diff options
Diffstat (limited to 'tests/src/com/android/inputmethod/latin')
60 files changed, 8818 insertions, 2336 deletions
diff --git a/tests/src/com/android/inputmethod/latin/AppWorkaroundsTests.java b/tests/src/com/android/inputmethod/latin/AppWorkaroundsTests.java new file mode 100644 index 000000000..c29257d34 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/AppWorkaroundsTests.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import com.android.inputmethod.latin.settings.Settings; + +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageManager; +import android.os.Build.VERSION_CODES; +import android.test.suitebuilder.annotation.LargeTest; +import android.view.inputmethod.EditorInfo; + +@LargeTest +public class AppWorkaroundsTests extends InputTestsBase { + String packageNameOfAppBeforeJellyBean; + String packageNameOfAppAfterJellyBean; + + @Override + protected void setUp() throws Exception { + // NOTE: this will fail if there is no app installed that targets an SDK + // before Jelly Bean. For the moment, it's fine. + final PackageManager pm = getContext().getPackageManager(); + for (ApplicationInfo ai : pm.getInstalledApplications(0 /* flags */)) { + if (ai.targetSdkVersion < VERSION_CODES.JELLY_BEAN) { + packageNameOfAppBeforeJellyBean = ai.packageName; + } else { + packageNameOfAppAfterJellyBean = ai.packageName; + } + } + super.setUp(); + } + + // We want to test if the app package info is correctly retrieved by LatinIME. Since it + // asks this information to the package manager from the package name, and that it takes + // the package name from the EditorInfo, all we have to do it put the correct package + // name in the editor info. + // To this end, our base class InputTestsBase offers a hook for us to touch the EditorInfo. + // We override this hook to write the package name that we need. + @Override + protected EditorInfo enrichEditorInfo(final EditorInfo ei) { + if ("testBeforeJellyBeanTrue".equals(getName())) { + ei.packageName = packageNameOfAppBeforeJellyBean; + } else if ("testBeforeJellyBeanFalse".equals(getName())) { + ei.packageName = packageNameOfAppAfterJellyBean; + } + return ei; + } + + public void testBeforeJellyBeanTrue() { + assertTrue("Couldn't successfully detect this app targets < Jelly Bean (package is " + + packageNameOfAppBeforeJellyBean + ")", + Settings.getInstance().getCurrent().isBeforeJellyBean()); + } + + public void testBeforeJellyBeanFalse() { + assertFalse("Couldn't successfully detect this app targets >= Jelly Bean (package is " + + packageNameOfAppAfterJellyBean + ")", + Settings.getInstance().getCurrent().isBeforeJellyBean()); + } +}
\ No newline at end of file diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index cd5384ea4..ae184268c 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -20,8 +20,18 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Pair; +import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; +import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.CodePointUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.DictionaryHeader; import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FusionDictionary; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; +import com.android.inputmethod.latin.utils.FileUtils; +import com.android.inputmethod.latin.utils.LocaleUtils; import java.io.File; import java.io.IOException; @@ -30,68 +40,183 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.concurrent.TimeUnit; @LargeTest public class BinaryDictionaryDecayingTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; - - // Note that these are corresponding definitions in native code in - // latinime::DynamicPatriciaTriePolicy. - private static final String SET_NEEDS_TO_DECAY_FOR_TESTING_KEY = - "SET_NEEDS_TO_DECAY_FOR_TESTING"; - private static final int DUMMY_PROBABILITY = 0; + private static final int[] DICT_FORMAT_VERSIONS = + new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; + + private int mCurrentTime = 0; @Override protected void setUp() throws Exception { super.setUp(); + mCurrentTime = 0; } @Override protected void tearDown() throws Exception { + stopTestModeInNativeCode(); super.tearDown(); } + private static boolean supportsBeginningOfSentence(final int formatVersion) { + return formatVersion > FormatSpec.VERSION401; + } + + private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, + final int probability) { + binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, + BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + false /* isBeginningOfSentence */, false /* isNotAWord */, + false /* isBlacklisted */, mCurrentTime /* timestamp */); + } + + private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, + final String word1, final int probability) { + binaryDictionary.addNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1, probability, + mCurrentTime /* timestamp */); + } + + private static boolean isValidBigram(final BinaryDictionary binaryDictionary, + final String word0, final String word1) { + return binaryDictionary.isValidNgram(new PrevWordsInfo(new WordInfo(word0)), word1); + } + private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { - // Entries having low probability would be suppressed once in 3 GCs. - final int count = 3; - for (int i = 0; i < count; i++) { - binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY); - binaryDictionary.flushWithGC(); - } + // 30 days. + final int timeToElapse = (int)TimeUnit.SECONDS.convert(30, TimeUnit.DAYS); + mCurrentTime += timeToElapse; + setCurrentTimeForTestMode(mCurrentTime); + binaryDictionary.flushWithGC(); } private void forcePassingLongTime(final BinaryDictionary binaryDictionary) { - // Currently, probabilities are decayed when GC is run. All entries that have never been - // typed in 128 GCs would be removed. - final int count = 128; - for (int i = 0; i < count; i++) { - binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY); - binaryDictionary.flushWithGC(); + // 365 days. + final int timeToElapse = (int)TimeUnit.SECONDS.convert(365, TimeUnit.DAYS); + mCurrentTime += timeToElapse; + setCurrentTimeForTestMode(mCurrentTime); + binaryDictionary.flushWithGC(); + } + + private File createEmptyDictionaryAndGetFile(final String dictId, + final int formatVersion) throws IOException { + if (formatVersion == FormatSpec.VERSION4 + || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING + || formatVersion == FormatSpec.VERSION4_DEV) { + return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion); + } else { + throw new IOException("Dictionary format version " + formatVersion + + " is not supported."); } } - private File createEmptyDictionaryAndGetFile(final String filename) throws IOException { - final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, + private File createEmptyVer4DictionaryAndGetFile(final String dictId, final int formatVersion) + throws IOException { + final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); - Map<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - 3 /* dictVersion */, attributeMap)) { + FileUtils.deleteRecursively(file); + Map<String, String> attributeMap = new HashMap<>(); + attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId); + attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, + String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); + attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion, + LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) { return file; } else { - throw new IOException("Empty dictionary cannot be created."); + throw new IOException("Empty dictionary " + file.getAbsolutePath() + + " cannot be created. Foramt version: " + formatVersion); + } + } + + private static int setCurrentTimeForTestMode(final int currentTime) { + return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime); + } + + private static int stopTestModeInNativeCode() { + return BinaryDictionaryUtils.setCurrentTimeForTest(-1); + } + + public void testReadDictInJavaSide() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testReadDictInJavaSide(formatVersion); } } + private void testReadDictInJavaSide(final int formatVersion) { + setCurrentTimeForTestMode(mCurrentTime); + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + final DictDecoder dictDecoder = + BinaryDictIOUtils.getDictDecoder(dictFile, 0, dictFile.length()); + try { + final FusionDictionary dict = + dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); + PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, "a"); + assertNotNull(ptNode); + assertTrue(ptNode.isTerminal()); + assertNotNull(ptNode.getBigram("aaa")); + ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, "ab"); + assertNotNull(ptNode); + assertTrue(ptNode.isTerminal()); + ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa"); + assertNotNull(ptNode); + assertTrue(ptNode.isTerminal()); + } catch (IOException e) { + fail("IOException while reading dictionary: " + e); + } catch (UnsupportedFormatException e) { + fail("Unsupported format: " + e); + } + dictFile.delete(); + } + + public void testControlCurrentTime() { + final int TEST_COUNT = 1000; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + final int startTime = stopTestModeInNativeCode(); + for (int i = 0; i < TEST_COUNT; i++) { + final int currentTime = random.nextInt(Integer.MAX_VALUE); + final int currentTimeInNativeCode = setCurrentTimeForTestMode(currentTime); + assertEquals(currentTime, currentTimeInNativeCode); + } + final int endTime = stopTestModeInNativeCode(); + final int MAX_ALLOWED_ELAPSED_TIME = 10; + assertTrue(startTime <= endTime && endTime <= startTime + MAX_ALLOWED_ELAPSED_TIME); + } + public void testAddValidAndInvalidWords() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddValidAndInvalidWords(formatVersion); + } + } + + private void testAddValidAndInvalidWords(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -99,46 +224,44 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); assertTrue(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidWord("b")); - final int unigramProbability = binaryDictionary.getFrequency("a"); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + assertFalse(isValidBigram(binaryDictionary, "a", "b")); + addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); - binaryDictionary.addUnigramWord("c", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "c", DUMMY_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "c")); + addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. - binaryDictionary.addBigramWords("x", "y", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("x", "y")); - binaryDictionary.addBigramWords("x", "y", DUMMY_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("x", "y")); + addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); + assertFalse(isValidBigram(binaryDictionary, "x", "y")); + addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); + assertFalse(isValidBigram(binaryDictionary, "x", "y")); binaryDictionary.close(); dictFile.delete(); } public void testDecayingProbability() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testDecayingProbability(formatVersion); + } + } + + private void testDecayingProbability(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -146,50 +269,53 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); - assertFalse(binaryDictionary.isValidBigram("a", "b")); - - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + assertFalse(isValidBigram(binaryDictionary, "a", "b")); + + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingLongTime(binaryDictionary); - assertFalse(binaryDictionary.isValidBigram("a", "b")); + assertFalse(isValidBigram(binaryDictionary, "a", "b")); binaryDictionary.close(); dictFile.delete(); } public void testAddManyUnigramsToDecayingDict() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddManyUnigramsToDecayingDict(formatVersion); + } + } + + private void testAddManyUnigramsToDecayingDict(final int formatVersion) { final int unigramCount = 30000; final int unigramTypedCount = 100000; final int codePointSetSize = 50; @@ -198,16 +324,17 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<String> words = new ArrayList<>(); for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); @@ -215,32 +342,102 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final int maxUnigramCount = Integer.parseInt( - binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); + binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); - binaryDictionary.addUnigramWord(word, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = - Integer.parseInt(binaryDictionary.getPropertyForTests( + Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.UNIGRAM_COUNT_QUERY)); while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDictionary.flushWithGC(); + forcePassingShortTime(binaryDictionary); } final int unigramCountAfterGC = - Integer.parseInt(binaryDictionary.getPropertyForTests( + Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.UNIGRAM_COUNT_QUERY)); assertTrue(unigramCountBeforeGC > unigramCountAfterGC); } } - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.UNIGRAM_COUNT_QUERY)) > 0); - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.UNIGRAM_COUNT_QUERY)) <= maxUnigramCount); + forcePassingLongTime(binaryDictionary); + assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest( + BinaryDictionary.UNIGRAM_COUNT_QUERY))); + } + + public void testOverflowUnigrams() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testOverflowUnigrams(formatVersion); + } + } + + private void testOverflowUnigrams(final int formatVersion) { + final int unigramCount = 20000; + final int eachUnigramTypedCount = 2; + final int strongUnigramTypedCount = 20; + final int weakUnigramTypedCount = 1; + final int codePointSetSize = 50; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + setCurrentTimeForTestMode(mCurrentTime); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + final String strong = "strong"; + final String weak = "weak"; + for (int j = 0; j < strongUnigramTypedCount; j++) { + addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); + } + for (int j = 0; j < weakUnigramTypedCount; j++) { + addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); + } + assertTrue(binaryDictionary.isValidWord(strong)); + assertTrue(binaryDictionary.isValidWord(weak)); + + for (int i = 0; i < unigramCount; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + for (int j = 0; j < eachUnigramTypedCount; j++) { + addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + } + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + final int unigramCountBeforeGC = + Integer.parseInt(binaryDictionary.getPropertyForTest( + BinaryDictionary.UNIGRAM_COUNT_QUERY)); + assertTrue(binaryDictionary.isValidWord(strong)); + assertTrue(binaryDictionary.isValidWord(weak)); + binaryDictionary.flushWithGC(); + final int unigramCountAfterGC = + Integer.parseInt(binaryDictionary.getPropertyForTest( + BinaryDictionary.UNIGRAM_COUNT_QUERY)); + assertTrue(unigramCountBeforeGC > unigramCountAfterGC); + assertFalse(binaryDictionary.isValidWord(weak)); + assertTrue(binaryDictionary.isValidWord(strong)); + break; + } + } } public void testAddManyBigramsToDecayingDict() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddManyBigramsToDecayingDict(formatVersion); + } + } + + private void testAddManyBigramsToDecayingDict(final int formatVersion) { final int unigramCount = 5000; final int bigramCount = 30000; final int bigramTypedCount = 100000; @@ -250,17 +447,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final ArrayList<String> words = new ArrayList<String>(); - final ArrayList<Pair<String, String>> bigrams = new ArrayList<Pair<String, String>>(); + final ArrayList<String> words = new ArrayList<>(); + final ArrayList<Pair<String, String>> bigrams = new ArrayList<>(); for (int i = 0; i < unigramCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); @@ -274,35 +472,221 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); - final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + final Pair<String, String> bigram = new Pair<>(word0, word1); bigrams.add(bigram); } final int maxBigramCount = Integer.parseInt( - binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); + binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); - binaryDictionary.addUnigramWord(bigram.first, DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord(bigram.second, DUMMY_PROBABILITY); - binaryDictionary.addBigramWords(bigram.first, bigram.second, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = - Integer.parseInt(binaryDictionary.getPropertyForTests( + Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.BIGRAM_COUNT_QUERY)); while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDictionary.flushWithGC(); + forcePassingShortTime(binaryDictionary); } final int bigramCountAfterGC = - Integer.parseInt(binaryDictionary.getPropertyForTests( + Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.BIGRAM_COUNT_QUERY)); assertTrue(bigramCountBeforeGC > bigramCountAfterGC); } } - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.BIGRAM_COUNT_QUERY)) > 0); - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTests( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.BIGRAM_COUNT_QUERY)) <= maxBigramCount); + forcePassingLongTime(binaryDictionary); + assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest( + BinaryDictionary.BIGRAM_COUNT_QUERY))); + } + + public void testOverflowBigrams() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testOverflowBigrams(formatVersion); + } + } + + private void testOverflowBigrams(final int formatVersion) { + final int bigramCount = 20000; + final int unigramCount = 1000; + final int unigramTypedCount = 20; + final int eachBigramTypedCount = 2; + final int strongBigramTypedCount = 20; + final int weakBigramTypedCount = 1; + final int codePointSetSize = 50; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + setCurrentTimeForTestMode(mCurrentTime); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + final ArrayList<String> words = new ArrayList<>(); + for (int i = 0; i < unigramCount; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + for (int j = 0; j < unigramTypedCount; j++) { + addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + } + } + final String strong = "strong"; + final String weak = "weak"; + final String target = "target"; + for (int j = 0; j < unigramTypedCount; j++) { + addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY); + } + binaryDictionary.flushWithGC(); + for (int j = 0; j < strongBigramTypedCount; j++) { + addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY); + } + for (int j = 0; j < weakBigramTypedCount; j++) { + addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); + } + assertTrue(isValidBigram(binaryDictionary, strong, target)); + assertTrue(isValidBigram(binaryDictionary, weak, target)); + + for (int i = 0; i < bigramCount; i++) { + final int word0Index = random.nextInt(words.size()); + final String word0 = words.get(word0Index); + final int index = random.nextInt(words.size() - 1); + final int word1Index = (index >= word0Index) ? index + 1 : index; + final String word1 = words.get(word1Index); + + for (int j = 0; j < eachBigramTypedCount; j++) { + addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY); + } + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + final int bigramCountBeforeGC = + Integer.parseInt(binaryDictionary.getPropertyForTest( + BinaryDictionary.BIGRAM_COUNT_QUERY)); + binaryDictionary.flushWithGC(); + final int bigramCountAfterGC = + Integer.parseInt(binaryDictionary.getPropertyForTest( + BinaryDictionary.BIGRAM_COUNT_QUERY)); + assertTrue(bigramCountBeforeGC > bigramCountAfterGC); + assertTrue(isValidBigram(binaryDictionary, strong, target)); + assertFalse(isValidBigram(binaryDictionary, weak, target)); + break; + } + } + } + + public void testDictMigration() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, formatVersion); + } + } + + private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) { + setCurrentTimeForTestMode(mCurrentTime); + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("aaa")); + addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidWord("bbb")); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); + addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); + + assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); + assertTrue(binaryDictionary.migrateTo(toFormatVersion)); + assertTrue(binaryDictionary.isValidDictionary()); + assertEquals(toFormatVersion, binaryDictionary.getFormatVersion()); + assertTrue(binaryDictionary.isValidWord("aaa")); + assertFalse(binaryDictionary.isValidWord("bbb")); + assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); + addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("bbb")); + assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); + assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); + addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); + binaryDictionary.close(); + dictFile.delete(); + } + + public void testBeginningOfSentence() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + if (supportsBeginningOfSentence(formatVersion)) { + testBeginningOfSentence(formatVersion); + } + } + } + + private void testBeginningOfSentence(final int formatVersion) { + setCurrentTimeForTestMode(mCurrentTime); + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */, + BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, + mCurrentTime); + final PrevWordsInfo prevWordsInfoStartOfSentence = PrevWordsInfo.BEGINNING_OF_SENTENCE; + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, + mCurrentTime); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, + mCurrentTime); + addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY, + mCurrentTime); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + + forcePassingLongTime(binaryDictionary); + assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, + mCurrentTime); + addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY, + mCurrentTime); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + binaryDictionary.close(); + dictFile.delete(); } } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 5b8f0e977..6ba18d665 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -21,8 +21,14 @@ import android.test.suitebuilder.annotation.LargeTest; import android.text.TextUtils; import android.util.Pair; +import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.WeightedString; +import com.android.inputmethod.latin.makedict.WordProperty; +import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; +import com.android.inputmethod.latin.utils.FileUtils; +import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; import java.io.IOException; @@ -33,39 +39,60 @@ import java.util.Locale; import java.util.Map; import java.util.Random; +// TODO Use the seed passed as an argument for makedict test. @LargeTest public class BinaryDictionaryTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; + private static final int[] DICT_FORMAT_VERSIONS = + new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; - @Override - protected void setUp() throws Exception { - super.setUp(); + private static boolean canCheckBigramProbability(final int formatVersion) { + return formatVersion > FormatSpec.VERSION401; } - @Override - protected void tearDown() throws Exception { - super.tearDown(); + private static boolean supportsBeginningOfSentence(final int formatVersion) { + return formatVersion > FormatSpec.VERSION401; } - private File createEmptyDictionaryAndGetFile(final String filename) throws IOException { - final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, + private File createEmptyDictionaryAndGetFile(final String dictId, + final int formatVersion) throws IOException { + if (formatVersion == FormatSpec.VERSION4 + || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING + || formatVersion == FormatSpec.VERSION4_DEV) { + return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion); + } else { + throw new IOException("Dictionary format version " + formatVersion + + " is not supported."); + } + } + + private File createEmptyVer4DictionaryAndGetFile(final String dictId, + final int formatVersion) throws IOException { + final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); - Map<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - 3 /* dictVersion */, attributeMap)) { + file.delete(); + file.mkdir(); + Map<String, String> attributeMap = new HashMap<>(); + if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion, + Locale.ENGLISH, attributeMap)) { return file; } else { - throw new IOException("Empty dictionary cannot be created."); + throw new IOException("Empty dictionary " + file.getAbsolutePath() + + " cannot be created. Format version: " + formatVersion); } } public void testIsValidDictionary() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testIsValidDictionary(formatVersion); + } + } + + private void testIsValidDictionary(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -77,7 +104,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.close(); assertFalse("binaryDictionary must be invalid after closing.", binaryDictionary.isValidDictionary()); - dictFile.delete(); + FileUtils.deleteRecursively(dictFile); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); @@ -86,10 +113,126 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.close(); } + public void testConstructingDictionaryOnMemory() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testConstructingDictionaryOnMemory(formatVersion); + } + } + + private void testConstructingDictionaryOnMemory(final int formatVersion) { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + FileUtils.deleteRecursively(dictFile); + assertFalse(dictFile.exists()); + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, formatVersion, + new HashMap<String, String>()); + assertTrue(binaryDictionary.isValidDictionary()); + assertEquals(formatVersion, binaryDictionary.getFormatVersion()); + final int probability = 100; + addUnigramWord(binaryDictionary, "word", probability); + assertEquals(probability, binaryDictionary.getFrequency("word")); + assertFalse(dictFile.exists()); + binaryDictionary.flush(); + assertTrue(dictFile.exists()); + assertTrue(binaryDictionary.isValidDictionary()); + assertEquals(formatVersion, binaryDictionary.getFormatVersion()); + assertEquals(probability, binaryDictionary.getFrequency("word")); + binaryDictionary.close(); + dictFile.delete(); + } + + public void testAddTooLongWord() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddTooLongWord(formatVersion); + } + } + + private void testAddTooLongWord(final int formatVersion) { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final StringBuffer stringBuilder = new StringBuffer(); + for (int i = 0; i < Constants.DICTIONARY_MAX_WORD_LENGTH; i++) { + stringBuilder.append('a'); + } + final String validLongWord = stringBuilder.toString(); + stringBuilder.append('a'); + final String invalidLongWord = stringBuilder.toString(); + final int probability = 100; + addUnigramWord(binaryDictionary, "aaa", probability); + addUnigramWord(binaryDictionary, validLongWord, probability); + addUnigramWord(binaryDictionary, invalidLongWord, probability); + // Too long short cut. + binaryDictionary.addUnigramEntry("a", probability, invalidLongWord, + 10 /* shortcutProbability */, false /* isBeginningOfSentence */, + false /* isNotAWord */, false /* isBlacklisted */, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); + addUnigramWord(binaryDictionary, "abc", probability); + final int updatedProbability = 200; + // Update. + addUnigramWord(binaryDictionary, validLongWord, updatedProbability); + addUnigramWord(binaryDictionary, invalidLongWord, updatedProbability); + addUnigramWord(binaryDictionary, "abc", updatedProbability); + + assertEquals(probability, binaryDictionary.getFrequency("aaa")); + assertEquals(updatedProbability, binaryDictionary.getFrequency(validLongWord)); + assertEquals(BinaryDictionary.NOT_A_PROBABILITY, + binaryDictionary.getFrequency(invalidLongWord)); + assertEquals(updatedProbability, binaryDictionary.getFrequency("abc")); + dictFile.delete(); + } + + private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, + final int probability) { + binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, + BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + false /* isBeginningOfSentence */, false /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + + private static void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, + final String word1, final int probability) { + binaryDictionary.addNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1, probability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + + private static boolean isValidBigram(final BinaryDictionary binaryDictionary, + final String word0, final String word1) { + return binaryDictionary.isValidNgram(new PrevWordsInfo(new WordInfo(word0)), word1); + } + + private static void removeBigramEntry(final BinaryDictionary binaryDictionary, + final String word0, final String word1) { + binaryDictionary.removeNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1); + } + + private static int getBigramProbability(final BinaryDictionary binaryDictionary, + final String word0, final String word1) { + return binaryDictionary.getNgramProbability(new PrevWordsInfo(new WordInfo(word0)), word1); + } + public void testAddUnigramWord() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddUnigramWord(formatVersion); + } + } + + private void testAddUnigramWord(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -98,21 +241,21 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = 100; - binaryDictionary.addUnigramWord("aaa", probability); + addUnigramWord(binaryDictionary, "aaa", probability); // Reallocate and create. - binaryDictionary.addUnigramWord("aab", probability); + addUnigramWord(binaryDictionary, "aab", probability); // Insert into children. - binaryDictionary.addUnigramWord("aac", probability); + addUnigramWord(binaryDictionary, "aac", probability); // Make terminal. - binaryDictionary.addUnigramWord("aa", probability); + addUnigramWord(binaryDictionary, "aa", probability); // Create children. - binaryDictionary.addUnigramWord("aaaa", probability); + addUnigramWord(binaryDictionary, "aaaa", probability); // Reallocate and make termianl. - binaryDictionary.addUnigramWord("a", probability); + addUnigramWord(binaryDictionary, "a", probability); final int updatedProbability = 200; // Update. - binaryDictionary.addUnigramWord("aaa", updatedProbability); + addUnigramWord(binaryDictionary, "aaa", updatedProbability); assertEquals(probability, binaryDictionary.getFrequency("aab")); assertEquals(probability, binaryDictionary.getFrequency("aac")); @@ -125,13 +268,19 @@ public class BinaryDictionaryTests extends AndroidTestCase { } public void testRandomlyAddUnigramWord() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testRandomlyAddUnigramWord(formatVersion); + } + } + + private void testRandomlyAddUnigramWord(final int formatVersion) { final int wordCount = 1000; final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -139,7 +288,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - final HashMap<String, Integer> probabilityMap = new HashMap<String, Integer>(); + final HashMap<String, Integer> probabilityMap = new HashMap<>(); // Test a word that isn't contained within the dictionary. final Random random = new Random(seed); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); @@ -148,7 +297,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { probabilityMap.put(word, random.nextInt(0xFF)); } for (String word : probabilityMap.keySet()) { - binaryDictionary.addUnigramWord(word, probabilityMap.get(word)); + addUnigramWord(binaryDictionary, word, probabilityMap.get(word)); } for (String word : probabilityMap.keySet()) { assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word)); @@ -157,9 +306,15 @@ public class BinaryDictionaryTests extends AndroidTestCase { } public void testAddBigramWords() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddBigramWords(formatVersion); + } + } + + private void testAddBigramWords(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -168,59 +323,73 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; - final int bigramProbability = 10; - final int updatedBigramProbability = 15; - binaryDictionary.addUnigramWord("aaa", unigramProbability); - binaryDictionary.addUnigramWord("abb", unigramProbability); - binaryDictionary.addUnigramWord("bcc", unigramProbability); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); - binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); - binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); - - final int probability = binaryDictionary.calculateProbability(unigramProbability, - bigramProbability); - assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); - assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); - assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); - assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); - assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); - assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); - assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); - assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); - - binaryDictionary.addBigramWords("aaa", "abb", updatedBigramProbability); - final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability, - updatedBigramProbability); - assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb")); - - assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); - assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); - assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + final int bigramProbability = 150; + final int updatedBigramProbability = 200; + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); + addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); + addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); + + assertTrue(isValidBigram(binaryDictionary, "aaa", "abb")); + assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc")); + assertTrue(isValidBigram(binaryDictionary, "abb", "aaa")); + assertTrue(isValidBigram(binaryDictionary, "abb", "bcc")); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc")); + } + + addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(updatedBigramProbability, + getBigramProbability(binaryDictionary, "aaa", "abb")); + } + + assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa")); + assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc")); + assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, - binaryDictionary.getBigramProbability("bcc", "aaa")); + getBigramProbability(binaryDictionary, "bcc", "aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, - binaryDictionary.getBigramProbability("bcc", "bbc")); + getBigramProbability(binaryDictionary, "bcc", "bbc")); assertEquals(Dictionary.NOT_A_PROBABILITY, - binaryDictionary.getBigramProbability("aaa", "aaa")); + getBigramProbability(binaryDictionary, "aaa", "aaa")); // Testing bigram link. - binaryDictionary.addUnigramWord("abcde", unigramProbability); - binaryDictionary.addUnigramWord("fghij", unigramProbability); - binaryDictionary.addBigramWords("abcde", "fghij", bigramProbability); - binaryDictionary.addUnigramWord("fgh", unigramProbability); - binaryDictionary.addUnigramWord("abc", unigramProbability); - binaryDictionary.addUnigramWord("f", unigramProbability); - assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij")); + addUnigramWord(binaryDictionary, "abcde", unigramProbability); + addUnigramWord(binaryDictionary, "fghij", unigramProbability); + addBigramWords(binaryDictionary, "abcde", "fghij", bigramProbability); + addUnigramWord(binaryDictionary, "fgh", unigramProbability); + addUnigramWord(binaryDictionary, "abc", unigramProbability); + addUnigramWord(binaryDictionary, "f", unigramProbability); + + if (canCheckBigramProbability(formatVersion)) { + assertEquals(bigramProbability, + getBigramProbability(binaryDictionary, "abcde", "fghij")); + } assertEquals(Dictionary.NOT_A_PROBABILITY, - binaryDictionary.getBigramProbability("abcde", "fgh")); - binaryDictionary.addBigramWords("abcde", "fghij", updatedBigramProbability); - assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij")); + getBigramProbability(binaryDictionary, "abcde", "fgh")); + addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(updatedBigramProbability, + getBigramProbability(binaryDictionary, "abcde", "fghij")); + } dictFile.delete(); } public void testRandomlyAddBigramWords() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testRandomlyAddBigramWords(formatVersion); + } + } + + private void testRandomlyAddBigramWords(final int formatVersion) { final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 50; @@ -229,7 +398,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -237,19 +406,18 @@ public class BinaryDictionaryTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - final ArrayList<String> words = new ArrayList<String>(); - final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); + final ArrayList<String> words = new ArrayList<>(); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); - final HashMap<Pair<String, String>, Integer> bigramProbabilities = - new HashMap<Pair<String, String>, Integer>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int i = 0; i < bigramCount; i++) { @@ -258,29 +426,38 @@ public class BinaryDictionaryTests extends AndroidTestCase { if (TextUtils.equals(word0, word1)) { continue; } - final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + final Pair<String, String> bigram = new Pair<>(word0, word1); bigramWords.add(bigram); - final int bigramProbability = random.nextInt(0xF); + final int unigramProbability = unigramProbabilities.get(word1); + final int bigramProbability = + unigramProbability + random.nextInt(0xFF - unigramProbability); bigramProbabilities.put(bigram, bigramProbability); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } for (final Pair<String, String> bigram : bigramWords) { - final int unigramProbability = unigramProbabilities.get(bigram.second); final int bigramProbability = bigramProbabilities.get(bigram); - final int probability = binaryDictionary.calculateProbability(unigramProbability, - bigramProbability); - assertEquals(probability, - binaryDictionary.getBigramProbability(bigram.first, bigram.second)); + assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, + isValidBigram(binaryDictionary, bigram.first, bigram.second)); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(bigramProbability, + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); + } } dictFile.delete(); } public void testRemoveBigramWords() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testRemoveBigramWords(formatVersion); + } + } + + private void testRemoveBigramWords(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -288,45 +465,51 @@ public class BinaryDictionaryTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; - final int bigramProbability = 10; - binaryDictionary.addUnigramWord("aaa", unigramProbability); - binaryDictionary.addUnigramWord("abb", unigramProbability); - binaryDictionary.addUnigramWord("bcc", unigramProbability); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); - binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); - binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); - - assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); - assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); - assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); - assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); - - binaryDictionary.removeBigramWords("aaa", "abb"); - assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb")); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); - - - binaryDictionary.removeBigramWords("aaa", "bcc"); - assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc")); - binaryDictionary.removeBigramWords("abb", "aaa"); - assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa")); - binaryDictionary.removeBigramWords("abb", "bcc"); - assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc")); - - binaryDictionary.removeBigramWords("aaa", "abb"); + final int bigramProbability = 150; + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); + addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); + addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); + + assertTrue(isValidBigram(binaryDictionary, "aaa", "abb")); + assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc")); + assertTrue(isValidBigram(binaryDictionary, "abb", "aaa")); + assertTrue(isValidBigram(binaryDictionary, "abb", "bcc")); + + removeBigramEntry(binaryDictionary, "aaa", "abb"); + assertFalse(isValidBigram(binaryDictionary, "aaa", "abb")); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + assertTrue(isValidBigram(binaryDictionary, "aaa", "abb")); + + + removeBigramEntry(binaryDictionary, "aaa", "bcc"); + assertFalse(isValidBigram(binaryDictionary, "aaa", "bcc")); + removeBigramEntry(binaryDictionary, "abb", "aaa"); + assertFalse(isValidBigram(binaryDictionary, "abb", "aaa")); + removeBigramEntry(binaryDictionary, "abb", "bcc"); + assertFalse(isValidBigram(binaryDictionary, "abb", "bcc")); + + removeBigramEntry(binaryDictionary, "aaa", "abb"); // Test remove non-existing bigram operation. - binaryDictionary.removeBigramWords("aaa", "abb"); - binaryDictionary.removeBigramWords("bcc", "aaa"); + removeBigramEntry(binaryDictionary, "aaa", "abb"); + removeBigramEntry(binaryDictionary, "bcc", "aaa"); dictFile.delete(); } public void testFlushDictionary() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testFlushDictionary(formatVersion); + } + } + + private void testFlushDictionary(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -335,8 +518,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = 100; - binaryDictionary.addUnigramWord("aaa", probability); - binaryDictionary.addUnigramWord("abcd", probability); + addUnigramWord(binaryDictionary, "aaa", probability); + addUnigramWord(binaryDictionary, "abcd", probability); // Close without flushing. binaryDictionary.close(); @@ -347,8 +530,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd")); - binaryDictionary.addUnigramWord("aaa", probability); - binaryDictionary.addUnigramWord("abcd", probability); + addUnigramWord(binaryDictionary, "aaa", probability); + addUnigramWord(binaryDictionary, "abcd", probability); binaryDictionary.flush(); binaryDictionary.close(); @@ -358,7 +541,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(probability, binaryDictionary.getFrequency("aaa")); assertEquals(probability, binaryDictionary.getFrequency("abcd")); - binaryDictionary.addUnigramWord("bcde", probability); + addUnigramWord(binaryDictionary, "bcde", probability); binaryDictionary.flush(); binaryDictionary.close(); @@ -372,9 +555,15 @@ public class BinaryDictionaryTests extends AndroidTestCase { } public void testFlushWithGCDictionary() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testFlushWithGCDictionary(formatVersion); + } + } + + private void testFlushWithGCDictionary(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -383,40 +572,46 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; - final int bigramProbability = 10; - binaryDictionary.addUnigramWord("aaa", unigramProbability); - binaryDictionary.addUnigramWord("abb", unigramProbability); - binaryDictionary.addUnigramWord("bcc", unigramProbability); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); - binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); - binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + final int bigramProbability = 150; + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); + addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); + addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); binaryDictionary.flushWithGC(); binaryDictionary.close(); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - final int probability = binaryDictionary.calculateProbability(unigramProbability, - bigramProbability); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); - assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); - assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); - assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); - assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); - assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); - assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); - assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc")); + } + assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa")); + assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc")); + assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa")); binaryDictionary.flushWithGC(); binaryDictionary.close(); dictFile.delete(); } - // TODO: Evaluate performance of GC public void testAddBigramWordsAndFlashWithGC() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddBigramWordsAndFlashWithGC(formatVersion); + } + } + + // TODO: Evaluate performance of GC + private void testAddBigramWordsAndFlashWithGC(final int formatVersion) { final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 30; @@ -425,7 +620,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -434,19 +629,18 @@ public class BinaryDictionaryTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - final ArrayList<String> words = new ArrayList<String>(); - final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); + final ArrayList<String> words = new ArrayList<>(); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); - final HashMap<Pair<String, String>, Integer> bigramProbabilities = - new HashMap<Pair<String, String>, Integer>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int i = 0; i < bigramCount; i++) { @@ -455,11 +649,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { if (TextUtils.equals(word0, word1)) { continue; } - final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + final Pair<String, String> bigram = new Pair<>(word0, word1); bigramWords.add(bigram); - final int bigramProbability = random.nextInt(0xF); + final int unigramProbability = unigramProbabilities.get(word1); + final int bigramProbability = + unigramProbability + random.nextInt(0xFF - unigramProbability); bigramProbabilities.put(bigram, bigramProbability); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } binaryDictionary.flushWithGC(); @@ -468,19 +664,27 @@ public class BinaryDictionaryTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + for (final Pair<String, String> bigram : bigramWords) { - final int unigramProbability = unigramProbabilities.get(bigram.second); final int bigramProbability = bigramProbabilities.get(bigram); - final int probability = binaryDictionary.calculateProbability(unigramProbability, - bigramProbability); - assertEquals(probability, - binaryDictionary.getBigramProbability(bigram.first, bigram.second)); + assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, + isValidBigram(binaryDictionary, bigram.first, bigram.second)); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(bigramProbability, + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); + } } dictFile.delete(); } - public void testRandomOperetionsAndFlashWithGC() { + public void testRandomOperationsAndFlashWithGC() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testRandomOperationsAndFlashWithGC(formatVersion); + } + } + + private void testRandomOperationsAndFlashWithGC(final int formatVersion) { final int flashWithGCIterationCount = 50; final int operationCountInEachIteration = 200; final int initialUnigramCount = 100; @@ -494,7 +698,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -502,18 +706,17 @@ public class BinaryDictionaryTests extends AndroidTestCase { BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - final ArrayList<String> words = new ArrayList<String>(); - final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); + final ArrayList<String> words = new ArrayList<>(); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); - final HashMap<Pair<String, String>, Integer> bigramProbabilities = - new HashMap<Pair<String, String>, Integer>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); for (int i = 0; i < initialUnigramCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -529,7 +732,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } // Add bigram. if (random.nextFloat() < addBigramProb && words.size() > 2) { @@ -543,11 +746,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { if (TextUtils.equals(word0, word1)) { continue; } - final int bigramProbability = random.nextInt(0xF); - final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + final int unigramProbability = unigramProbabilities.get(word1); + final int bigramProbability = + unigramProbability + random.nextInt(0xFF - unigramProbability); + final Pair<String, String> bigram = new Pair<>(word0, word1); bigramWords.add(bigram); bigramProbabilities.put(bigram, bigramProbability); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } // Remove bigram. if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) { @@ -555,7 +760,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final Pair<String, String> bigram = bigramWords.get(bigramIndex); bigramWords.remove(bigramIndex); bigramProbabilities.remove(bigram); - binaryDictionary.removeBigramWords(bigram.first, bigram.second); + removeBigramEntry(binaryDictionary, bigram.first, bigram.second); } } @@ -568,17 +773,20 @@ public class BinaryDictionaryTests extends AndroidTestCase { // Test whether the all bigram operations are collectlly handled. for (int i = 0; i < bigramWords.size(); i++) { final Pair<String, String> bigram = bigramWords.get(i); - final int unigramProbability = unigramProbabilities.get(bigram.second); final int probability; if (bigramProbabilities.containsKey(bigram)) { final int bigramProbability = bigramProbabilities.get(bigram); - probability = binaryDictionary.calculateProbability(unigramProbability, - bigramProbability); + probability = bigramProbability; } else { probability = Dictionary.NOT_A_PROBABILITY; } - assertEquals(probability, - binaryDictionary.getBigramProbability(bigram.first, bigram.second)); + + if (canCheckBigramProbability(formatVersion)) { + assertEquals(probability, + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); + } + assertEquals(probability != Dictionary.NOT_A_PROBABILITY, + isValidBigram(binaryDictionary, bigram.first, bigram.second)); } binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -588,6 +796,12 @@ public class BinaryDictionaryTests extends AndroidTestCase { } public void testAddManyUnigramsAndFlushWithGC() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddManyUnigramsAndFlushWithGC(formatVersion); + } + } + + private void testAddManyUnigramsAndFlushWithGC(final int formatVersion) { final int flashWithGCIterationCount = 3; final int codePointSetSize = 50; @@ -596,13 +810,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } - final ArrayList<String> words = new ArrayList<String>(); - final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final ArrayList<String> words = new ArrayList<>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); BinaryDictionary binaryDictionary; @@ -615,7 +829,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int j = 0; j < words.size(); j++) { @@ -632,6 +846,12 @@ public class BinaryDictionaryTests extends AndroidTestCase { } public void testUnigramAndBigramCount() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testUnigramAndBigramCount(formatVersion); + } + } + + private void testUnigramAndBigramCount(final int formatVersion) { final int flashWithGCIterationCount = 10; final int codePointSetSize = 50; final int unigramCountPerIteration = 1000; @@ -641,13 +861,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } - final ArrayList<String> words = new ArrayList<String>(); - final HashSet<Pair<String, String>> bigrams = new HashSet<Pair<String, String>>(); + final ArrayList<String> words = new ArrayList<>(); + final HashSet<Pair<String, String>> bigrams = new HashSet<>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); BinaryDictionary binaryDictionary; @@ -659,7 +879,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int j = 0; j < bigramCountPerIteration; j++) { final String word0 = words.get(random.nextInt(words.size())); @@ -667,22 +887,630 @@ public class BinaryDictionaryTests extends AndroidTestCase { if (TextUtils.equals(word0, word1)) { continue; } - bigrams.add(new Pair<String, String>(word0, word1)); + bigrams.add(new Pair<>(word0, word1)); final int bigramProbability = random.nextInt(0xF); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } - assertEquals(new HashSet<String>(words).size(), Integer.parseInt( - binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY))); - assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt( - binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY))); + assertEquals(new HashSet<>(words).size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + assertEquals(new HashSet<>(bigrams).size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.flushWithGC(); - assertEquals(new HashSet<String>(words).size(), Integer.parseInt( - binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY))); - assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt( - binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY))); + assertEquals(new HashSet<>(words).size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + assertEquals(new HashSet<>(bigrams).size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.close(); } dictFile.delete(); } + + public void testAddMultipleDictionaryEntries() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddMultipleDictionaryEntries(formatVersion); + } + } + + private void testAddMultipleDictionaryEntries(final int formatVersion) { + final int codePointSetSize = 20; + final int lmParamCount = 1000; + final double bigramContinueRate = 0.9; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); + + final LanguageModelParam[] languageModelParams = new LanguageModelParam[lmParamCount]; + String prevWord = null; + for (int i = 0; i < languageModelParams.length; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + final int probability = random.nextInt(0xFF); + final int bigramProbability = probability + random.nextInt(0xFF - probability); + unigramProbabilities.put(word, probability); + if (prevWord == null) { + languageModelParams[i] = new LanguageModelParam(word, probability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); + } else { + languageModelParams[i] = new LanguageModelParam(prevWord, word, probability, + bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + bigramProbabilities.put(new Pair<>(prevWord, word), + bigramProbability); + } + prevWord = (random.nextDouble() < bigramContinueRate) ? word : null; + } + + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + binaryDictionary.addMultipleDictionaryEntries(languageModelParams); + + for (Map.Entry<String, Integer> entry : unigramProbabilities.entrySet()) { + assertEquals((int)entry.getValue(), binaryDictionary.getFrequency(entry.getKey())); + } + + for (Map.Entry<Pair<String, String>, Integer> entry : bigramProbabilities.entrySet()) { + final String word0 = entry.getKey().first; + final String word1 = entry.getKey().second; + final int bigramProbability = entry.getValue(); + assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, + isValidBigram(binaryDictionary, word0, word1)); + if (canCheckBigramProbability(formatVersion)) { + assertEquals(bigramProbability, + getBigramProbability(binaryDictionary, word0, word1)); + } + } + } + + public void testGetWordProperties() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testGetWordProperties(formatVersion); + } + } + + private void testGetWordProperties(final int formatVersion) { + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + final int UNIGRAM_COUNT = 1000; + final int BIGRAM_COUNT = 1000; + final int codePointSetSize = 20; + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord", + false /* isBeginningOfSentence */); + assertFalse(invalidWordProperty.isValid()); + + final ArrayList<String> words = new ArrayList<>(); + final HashMap<String, Integer> wordProbabilities = new HashMap<>(); + final HashMap<String, HashSet<String>> bigrams = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); + + for (int i = 0; i < UNIGRAM_COUNT; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + final int unigramProbability = random.nextInt(0xFF); + final boolean isNotAWord = random.nextBoolean(); + final boolean isBlacklisted = random.nextBoolean(); + // TODO: Add tests for historical info. + binaryDictionary.addUnigramEntry(word, unigramProbability, + null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY, + false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); + if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + words.add(word); + wordProbabilities.put(word, unigramProbability); + final WordProperty wordProperty = binaryDictionary.getWordProperty(word, + false /* isBeginningOfSentence */); + assertEquals(word, wordProperty.mWord); + assertTrue(wordProperty.isValid()); + assertEquals(isNotAWord, wordProperty.mIsNotAWord); + assertEquals(isBlacklisted, wordProperty.mIsBlacklistEntry); + assertEquals(false, wordProperty.mHasBigrams); + assertEquals(false, wordProperty.mHasShortcuts); + assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability); + assertTrue(wordProperty.mShortcutTargets.isEmpty()); + } + + for (int i = 0; i < BIGRAM_COUNT; i++) { + final int word0Index = random.nextInt(wordProbabilities.size()); + final int word1Index = random.nextInt(wordProbabilities.size()); + if (word0Index == word1Index) { + continue; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int unigramProbability = wordProbabilities.get(word1); + final int bigramProbability = + unigramProbability + random.nextInt(0xFF - unigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); + if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + if (!bigrams.containsKey(word0)) { + final HashSet<String> bigramWord1s = new HashSet<>(); + bigrams.put(word0, bigramWord1s); + } + bigrams.get(word0).add(word1); + bigramProbabilities.put(new Pair<>(word0, word1), bigramProbability); + } + + for (int i = 0; i < words.size(); i++) { + final String word0 = words.get(i); + if (!bigrams.containsKey(word0)) { + continue; + } + final HashSet<String> bigramWord1s = bigrams.get(word0); + final WordProperty wordProperty = binaryDictionary.getWordProperty(word0, + false /* isBeginningOfSentence */); + assertEquals(bigramWord1s.size(), wordProperty.mBigrams.size()); + for (int j = 0; j < wordProperty.mBigrams.size(); j++) { + final String word1 = wordProperty.mBigrams.get(j).mWord; + assertTrue(bigramWord1s.contains(word1)); + if (canCheckBigramProbability(formatVersion)) { + final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1)); + assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability()); + } + } + } + } + + public void testIterateAllWords() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testIterateAllWords(formatVersion); + } + } + + private void testIterateAllWords(final int formatVersion) { + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + final int UNIGRAM_COUNT = 1000; + final int BIGRAM_COUNT = 1000; + final int codePointSetSize = 20; + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord", + false /* isBeginningOfSentence */); + assertFalse(invalidWordProperty.isValid()); + + final ArrayList<String> words = new ArrayList<>(); + final HashMap<String, Integer> wordProbabilitiesToCheckLater = new HashMap<>(); + final HashMap<String, HashSet<String>> bigrams = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilitiesToCheckLater = + new HashMap<>(); + + for (int i = 0; i < UNIGRAM_COUNT; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + final int unigramProbability = random.nextInt(0xFF); + addUnigramWord(binaryDictionary, word, unigramProbability); + if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + words.add(word); + wordProbabilitiesToCheckLater.put(word, unigramProbability); + } + + for (int i = 0; i < BIGRAM_COUNT; i++) { + final int word0Index = random.nextInt(wordProbabilitiesToCheckLater.size()); + final int word1Index = random.nextInt(wordProbabilitiesToCheckLater.size()); + if (word0Index == word1Index) { + continue; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int unigramProbability = wordProbabilitiesToCheckLater.get(word1); + final int bigramProbability = + unigramProbability + random.nextInt(0xFF - unigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); + if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + if (!bigrams.containsKey(word0)) { + final HashSet<String> bigramWord1s = new HashSet<>(); + bigrams.put(word0, bigramWord1s); + } + bigrams.get(word0).add(word1); + bigramProbabilitiesToCheckLater.put(new Pair<>(word0, word1), bigramProbability); + } + + final HashSet<String> wordSet = new HashSet<>(words); + final HashSet<Pair<String, String>> bigramSet = + new HashSet<>(bigramProbabilitiesToCheckLater.keySet()); + int token = 0; + do { + final BinaryDictionary.GetNextWordPropertyResult result = + binaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + final String word0 = wordProperty.mWord; + assertEquals((int)wordProbabilitiesToCheckLater.get(word0), + wordProperty.mProbabilityInfo.mProbability); + wordSet.remove(word0); + final HashSet<String> bigramWord1s = bigrams.get(word0); + for (int j = 0; j < wordProperty.mBigrams.size(); j++) { + final String word1 = wordProperty.mBigrams.get(j).mWord; + assertTrue(bigramWord1s.contains(word1)); + final Pair<String, String> bigram = new Pair<>(word0, word1); + if (canCheckBigramProbability(formatVersion)) { + final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram); + assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability()); + } + bigramSet.remove(bigram); + } + token = result.mNextToken; + } while (token != 0); + assertTrue(wordSet.isEmpty()); + assertTrue(bigramSet.isEmpty()); + } + + public void testAddShortcuts() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddShortcuts(formatVersion); + } + } + + private void testAddShortcuts(final int formatVersion) { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int shortcutProbability = 10; + binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", + shortcutProbability, false /* isBeginningOfSentence */, + false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); + WordProperty wordProperty = binaryDictionary.getWordProperty("aaa", + false /* isBeginningOfSentence */); + assertEquals(1, wordProperty.mShortcutTargets.size()); + assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); + assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability()); + final int updatedShortcutProbability = 2; + binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", + updatedShortcutProbability, false /* isBeginningOfSentence */, + false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); + wordProperty = binaryDictionary.getWordProperty("aaa", + false /* isBeginningOfSentence */); + assertEquals(1, wordProperty.mShortcutTargets.size()); + assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); + assertEquals(updatedShortcutProbability, + wordProperty.mShortcutTargets.get(0).getProbability()); + binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy", + shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, + false /* isBlacklisted */, 0 /* timestamp */); + final HashMap<String, Integer> shortcutTargets = new HashMap<>(); + shortcutTargets.put("zzz", updatedShortcutProbability); + shortcutTargets.put("yyy", shortcutProbability); + wordProperty = binaryDictionary.getWordProperty("aaa", + false /* isBeginningOfSentence */); + assertEquals(2, wordProperty.mShortcutTargets.size()); + for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); + assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), + shortcutTarget.getProbability()); + shortcutTargets.remove(shortcutTarget.mWord); + } + shortcutTargets.put("zzz", updatedShortcutProbability); + shortcutTargets.put("yyy", shortcutProbability); + binaryDictionary.flushWithGC(); + wordProperty = binaryDictionary.getWordProperty("aaa", + false /* isBeginningOfSentence */); + assertEquals(2, wordProperty.mShortcutTargets.size()); + for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); + assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), + shortcutTarget.getProbability()); + shortcutTargets.remove(shortcutTarget.mWord); + } + } + + public void testAddManyShortcuts() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddManyShortcuts(formatVersion); + } + } + + private void testAddManyShortcuts(final int formatVersion) { + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + final int UNIGRAM_COUNT = 1000; + final int SHORTCUT_COUNT = 10000; + final int codePointSetSize = 20; + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + final ArrayList<String> words = new ArrayList<>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); + final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>(); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + for (int i = 0; i < UNIGRAM_COUNT; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + final int unigramProbability = random.nextInt(0xFF); + addUnigramWord(binaryDictionary, word, unigramProbability); + words.add(word); + unigramProbabilities.put(word, unigramProbability); + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + } + for (int i = 0; i < SHORTCUT_COUNT; i++) { + final String shortcutTarget = CodePointUtils.generateWord(random, codePointSet); + final int shortcutProbability = random.nextInt(0xF); + final String word = words.get(random.nextInt(words.size())); + final int unigramProbability = unigramProbabilities.get(word); + binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget, + shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, + false /* isBlacklisted */, 0 /* timestamp */); + if (shortcutTargets.containsKey(word)) { + final HashMap<String, Integer> shortcutTargetsOfWord = shortcutTargets.get(word); + shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); + } else { + final HashMap<String, Integer> shortcutTargetsOfWord = new HashMap<>(); + shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); + shortcutTargets.put(word, shortcutTargetsOfWord); + } + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + } + + for (final String word : words) { + final WordProperty wordProperty = binaryDictionary.getWordProperty(word, + false /* isBeginningOfSentence */); + assertEquals((int)unigramProbabilities.get(word), + wordProperty.mProbabilityInfo.mProbability); + if (!shortcutTargets.containsKey(word)) { + // The word does not have shortcut targets. + continue; + } + assertEquals(shortcutTargets.get(word).size(), wordProperty.mShortcutTargets.size()); + for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + final String targetCodePonts = shortcutTarget.mWord; + assertEquals((int)shortcutTargets.get(word).get(targetCodePonts), + shortcutTarget.getProbability()); + } + } + } + + public void testDictMigration() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, formatVersion); + } + } + + private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final int unigramProbability = 100; + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "bbb", unigramProbability); + final int bigramProbability = 150; + addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability); + final int shortcutProbability = 10; + binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability, + false /* isBeginningOfSentence */, false /* isNotAWord */, + false /* isBlacklisted */, 0 /* timestamp */); + binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */, + Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */, + true /* isNotAWord */, true /* isBlacklisted */, 0 /* timestamp */); + binaryDictionary.addNgramEntry(PrevWordsInfo.BEGINNING_OF_SENTENCE, + "aaa", bigramProbability, 0 /* timestamp */); + assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); + assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); + assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); + assertTrue(binaryDictionary.migrateTo(toFormatVersion)); + assertTrue(binaryDictionary.isValidDictionary()); + assertEquals(toFormatVersion, binaryDictionary.getFormatVersion()); + assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); + if (canCheckBigramProbability(toFormatVersion)) { + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb")); + assertEquals(bigramProbability, binaryDictionary.getNgramProbability( + PrevWordsInfo.BEGINNING_OF_SENTENCE, "aaa")); + } + assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); + WordProperty wordProperty = binaryDictionary.getWordProperty("ccc", + false /* isBeginningOfSentence */); + assertEquals(1, wordProperty.mShortcutTargets.size()); + assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord); + wordProperty = binaryDictionary.getWordProperty("ddd", + false /* isBeginningOfSentence */); + assertTrue(wordProperty.mIsBlacklistEntry); + assertTrue(wordProperty.mIsNotAWord); + } + + public void testLargeDictMigration() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testLargeDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, formatVersion); + } + } + + private void testLargeDictMigration(final int fromFormatVersion, final int toFormatVersion) { + final int UNIGRAM_COUNT = 3000; + final int BIGRAM_COUNT = 3000; + final int codePointSetSize = 50; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final ArrayList<String> words = new ArrayList<>(); + final ArrayList<Pair<String, String>> bigrams = new ArrayList<>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); + + for (int i = 0; i < UNIGRAM_COUNT; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + final int unigramProbability = random.nextInt(0xFF); + addUnigramWord(binaryDictionary, word, unigramProbability); + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + words.add(word); + unigramProbabilities.put(word, unigramProbability); + } + + for (int i = 0; i < BIGRAM_COUNT; i++) { + final int word0Index = random.nextInt(words.size()); + final int word1Index = random.nextInt(words.size()); + if (word0Index == word1Index) { + continue; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int unigramProbability = unigramProbabilities.get(word1); + final int bigramProbability = + random.nextInt(0xFF - unigramProbability) + unigramProbability; + addBigramWords(binaryDictionary, word0, word1, bigramProbability); + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + final Pair<String, String> bigram = new Pair<>(word0, word1); + bigrams.add(bigram); + bigramProbabilities.put(bigram, bigramProbability); + } + assertTrue(binaryDictionary.migrateTo(toFormatVersion)); + + for (final String word : words) { + assertEquals((int)unigramProbabilities.get(word), binaryDictionary.getFrequency(word)); + } + assertEquals(unigramProbabilities.size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + + for (final Pair<String, String> bigram : bigrams) { + if (canCheckBigramProbability(toFormatVersion)) { + assertEquals((int)bigramProbabilities.get(bigram), + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); + } + assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second)); + } + assertEquals(bigramProbabilities.size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); + } + + public void testBeginningOfSentence() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + if (supportsBeginningOfSentence(formatVersion)) { + testBeginningOfSentence(formatVersion); + } + } + } + + private void testBeginningOfSentence(final int formatVersion) { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final int dummyProbability = 0; + final PrevWordsInfo prevWordsInfoBeginningOfSentence = PrevWordsInfo.BEGINNING_OF_SENTENCE; + final int bigramProbability = 200; + addUnigramWord(binaryDictionary, "aaa", dummyProbability); + binaryDictionary.addNgramEntry(prevWordsInfoBeginningOfSentence, "aaa", bigramProbability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + assertEquals(bigramProbability, + binaryDictionary.getNgramProbability(prevWordsInfoBeginningOfSentence, "aaa")); + binaryDictionary.addNgramEntry(prevWordsInfoBeginningOfSentence, "aaa", bigramProbability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + addUnigramWord(binaryDictionary, "bbb", dummyProbability); + binaryDictionary.addNgramEntry(prevWordsInfoBeginningOfSentence, "bbb", bigramProbability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + binaryDictionary.flushWithGC(); + assertEquals(bigramProbability, + binaryDictionary.getNgramProbability(prevWordsInfoBeginningOfSentence, "aaa")); + assertEquals(bigramProbability, + binaryDictionary.getNgramProbability(prevWordsInfoBeginningOfSentence, "bbb")); + } + + public void testGetMaxFrequencyOfExactMatches() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testGetMaxFrequencyOfExactMatches(formatVersion); + } + } + + private void testGetMaxFrequencyOfExactMatches(final int formatVersion) { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + addUnigramWord(binaryDictionary, "abc", 10); + addUnigramWord(binaryDictionary, "aBc", 15); + assertEquals(15, binaryDictionary.getMaxFrequencyOfExactMatches("abc")); + addUnigramWord(binaryDictionary, "ab'c", 20); + assertEquals(20, binaryDictionary.getMaxFrequencyOfExactMatches("abc")); + addUnigramWord(binaryDictionary, "a-b-c", 25); + assertEquals(25, binaryDictionary.getMaxFrequencyOfExactMatches("abc")); + addUnigramWord(binaryDictionary, "ab-'-'-'-c", 30); + assertEquals(30, binaryDictionary.getMaxFrequencyOfExactMatches("abc")); + addUnigramWord(binaryDictionary, "ab c", 255); + assertEquals(30, binaryDictionary.getMaxFrequencyOfExactMatches("abc")); + } } diff --git a/tests/src/com/android/inputmethod/latin/BlueUnderlineTests.java b/tests/src/com/android/inputmethod/latin/BlueUnderlineTests.java index c4fd5a0c4..6e894decf 100644 --- a/tests/src/com/android/inputmethod/latin/BlueUnderlineTests.java +++ b/tests/src/com/android/inputmethod/latin/BlueUnderlineTests.java @@ -50,8 +50,7 @@ public class BlueUnderlineTests extends InputTestsBase { final SpanGetter spanBefore = new SpanGetter(mEditText.getText(), SuggestionSpan.class); assertEquals("extend blue underline, span start", EXPECTED_SPAN_START, spanBefore.mStart); assertEquals("extend blue underline, span end", EXPECTED_SPAN_END, spanBefore.mEnd); - assertEquals("extend blue underline, span color", true, - spanBefore.isAutoCorrectionIndicator()); + assertTrue("extend blue underline, span color", spanBefore.isAutoCorrectionIndicator()); sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); // Now we have been able to re-evaluate the word, there shouldn't be an auto-correction span @@ -61,6 +60,7 @@ public class BlueUnderlineTests extends InputTestsBase { public void testBlueUnderlineOnBackspace() { final String STRING_TO_TYPE = "tgis"; + final int typedLength = STRING_TO_TYPE.length(); final int EXPECTED_SUGGESTION_SPAN_START = -1; final int EXPECTED_UNDERLINE_SPAN_START = 0; final int EXPECTED_UNDERLINE_SPAN_END = 4; @@ -68,6 +68,8 @@ public class BlueUnderlineTests extends InputTestsBase { sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); type(Constants.CODE_SPACE); + // typedLength + 1 because we also typed a space + mLatinIME.onUpdateSelection(0, 0, typedLength + 1, typedLength + 1, -1, -1); sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); type(Constants.CODE_DELETE); @@ -77,8 +79,8 @@ public class BlueUnderlineTests extends InputTestsBase { sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); final SpanGetter suggestionSpan = new SpanGetter(mEditText.getText(), SuggestionSpan.class); - assertEquals("show no blue underline after backspace, span start should be -1", - EXPECTED_SUGGESTION_SPAN_START, suggestionSpan.mStart); + assertFalse("show no blue underline after backspace, span should not be the auto-" + + "correction indicator", suggestionSpan.isAutoCorrectionIndicator()); final SpanGetter underlineSpan = new SpanGetter(mEditText.getText(), UnderlineSpan.class); assertEquals("should be composing, so should have an underline span", EXPECTED_UNDERLINE_SPAN_START, underlineSpan.mStart); @@ -104,7 +106,8 @@ public class BlueUnderlineTests extends InputTestsBase { sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); final SpanGetter span = new SpanGetter(mEditText.getText(), SuggestionSpan.class); - assertNull("blue underline removed when cursor is moved", span.mSpan); + assertFalse("blue underline removed when cursor is moved", + span.isAutoCorrectionIndicator()); } public void testComposingStopsOnSpace() { diff --git a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java new file mode 100644 index 000000000..70b8f530a --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import java.util.Locale; + +import android.test.suitebuilder.annotation.LargeTest; + +import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatches; + +/** + * Unit test for DistracterFilter + */ +@LargeTest +public class DistracterFilterTest extends InputTestsBase { + private DistracterFilterCheckingExactMatches mDistracterFilter; + + @Override + protected void setUp() throws Exception { + super.setUp(); + mDistracterFilter = new DistracterFilterCheckingExactMatches(getContext()); + mDistracterFilter.updateEnabledSubtypes(mLatinIME.getEnabledSubtypesForTest()); + } + + public void testIsDistractorToWordsInDictionaries() { + final PrevWordsInfo EMPTY_PREV_WORDS_INFO = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; + + final Locale localeEnUs = new Locale("en", "US"); + String typedWord; + + typedWord = "Bill"; + // For this test case, we consider "Bill" is a distracter to "bill". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "nOt"; + // For this test case, we consider "nOt" is a distracter to "not". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "youre"; + // For this test case, we consider "youre" is a distracter to "you're". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "Banana"; + // For this test case, we consider "Banana" is a distracter to "banana". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "orange"; + // For this test case, we consider "orange" is not a distracter to any word in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "Orange"; + // For this test case, we consider "Orange" is a distracter to "orange". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "café"; + // For this test case, we consider "café" is a distracter to "cafe". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "cafe"; + // For this test case, we consider "cafe" is not a distracter to any word in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "I'll"; + // For this test case, we consider "I'll" is not a distracter to any word in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "ill"; + // For this test case, we consider "ill" is a distracter to "I'll" + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "asdfd"; + // For this test case, we consider "asdfd" is not a distracter to any word in dictionaries. + assertFalse( + mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "thank"; + // For this test case, we consider "thank" is not a distracter to any other word + // in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + final Locale localeDeDe = new Locale("de", "DE"); + + typedWord = "fuer"; + // For this test case, we consider "fuer" is a distracter to "für". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); + + typedWord = "fUEr"; + // For this test case, we consider "fUEr" is a distracter to "für". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); + + typedWord = "fur"; + // For this test case, we consider "fur" is a distracter to "für". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); + + final Locale localeFrFr = new Locale("fr", "FR"); + + typedWord = "a"; + // For this test case, we consider "a" is a distracter to "à". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "à"; + // For this test case, we consider "à" is not a distracter to any word in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "etre"; + // For this test case, we consider "etre" is a distracter to "être". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "États-unis"; + // For this test case, we consider "États-unis" is a distracter to "États-Unis". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "ÉtatsUnis"; + // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + } +} diff --git a/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java b/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java deleted file mode 100644 index 6aae1044e..000000000 --- a/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin; - -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.SmallTest; - -/** - * Unit test for ExpandableDictionary - */ -@SmallTest -public class ExpandableDictionaryTests extends AndroidTestCase { - - private final static int UNIGRAM_FREQ = 50; - // See UserBinaryDictionary for more information about this variable. - // For tests, its actual value does not matter. - private final static int SHORTCUT_FREQ = 14; - - public void testAddWordAndGetWordFrequency() { - final ExpandableDictionary dict = new ExpandableDictionary(Dictionary.TYPE_USER); - - // Add words - dict.addWord("abcde", "abcde", UNIGRAM_FREQ, SHORTCUT_FREQ); - dict.addWord("abcef", null, UNIGRAM_FREQ + 1, 0); - - // Check words - assertFalse(dict.isValidWord("abcde")); - assertEquals(UNIGRAM_FREQ, dict.getWordFrequency("abcde")); - assertTrue(dict.isValidWord("abcef")); - assertEquals(UNIGRAM_FREQ+1, dict.getWordFrequency("abcef")); - - dict.addWord("abc", null, UNIGRAM_FREQ + 2, 0); - assertTrue(dict.isValidWord("abc")); - assertEquals(UNIGRAM_FREQ + 2, dict.getWordFrequency("abc")); - - // Add existing word with lower frequency - dict.addWord("abc", null, UNIGRAM_FREQ, 0); - assertEquals(UNIGRAM_FREQ + 2, dict.getWordFrequency("abc")); - - // Add existing word with higher frequency - dict.addWord("abc", null, UNIGRAM_FREQ + 3, 0); - assertEquals(UNIGRAM_FREQ + 3, dict.getWordFrequency("abc")); - } -} diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index cadd0f8f3..09309bcc0 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -19,7 +19,9 @@ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; +import com.android.inputmethod.latin.makedict.ProbabilityInfo; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import java.util.HashMap; @@ -31,18 +33,18 @@ import java.util.HashMap; public class FusionDictionaryTests extends AndroidTestCase { public void testFindWordInTree() { FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + new DictionaryOptions(new HashMap<String,String>())); - dict.add("abc", 10, null, false /* isNotAWord */); + dict.add("abc", new ProbabilityInfo(10), null, false /* isNotAWord */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc")); - dict.add("aa", 10, null, false /* isNotAWord */); + dict.add("aa", new ProbabilityInfo(10), null, false /* isNotAWord */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa")); - dict.add("babcd", 10, null, false /* isNotAWord */); - dict.add("bacde", 10, null, false /* isNotAWord */); + dict.add("babcd", new ProbabilityInfo(10), null, false /* isNotAWord */); + dict.add("bacde", new ProbabilityInfo(10), null, false /* isNotAWord */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "bacde")); diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java index 8ad8689d8..de9475af4 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java @@ -16,7 +16,10 @@ package com.android.inputmethod.latin; +import com.android.inputmethod.latin.settings.Settings; + import android.test.suitebuilder.annotation.LargeTest; +import android.text.TextUtils; import android.view.inputmethod.BaseInputConnection; @LargeTest @@ -32,7 +35,7 @@ public class InputLogicTests extends InputTestsBase { final String WORD_TO_TYPE = "this"; final String EXPECTED_RESULT = "thi"; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); mLatinIME.onUpdateSelection(0, 0, WORD_TO_TYPE.length(), WORD_TO_TYPE.length(), -1, -1); type(Constants.CODE_DELETE); assertEquals("press suggestion then backspace", EXPECTED_RESULT, @@ -44,9 +47,8 @@ public class InputLogicTests extends InputTestsBase { final String WORD_TO_PICK = "this"; final String EXPECTED_RESULT = "thi"; type(WORD_TO_TYPE); - // Choose the auto-correction, which is always in position 0. For "tgis", the - // auto-correction should be "this". - pickSuggestionManually(0, WORD_TO_PICK); + // Choose the auto-correction. For "tgis", the auto-correction should be "this". + pickSuggestionManually(WORD_TO_PICK); mLatinIME.onUpdateSelection(0, 0, WORD_TO_TYPE.length(), WORD_TO_TYPE.length(), -1, -1); assertEquals("pick typed word over auto-correction then backspace", WORD_TO_PICK, mEditText.getText().toString()); @@ -59,9 +61,8 @@ public class InputLogicTests extends InputTestsBase { final String WORD_TO_TYPE = "tgis"; final String EXPECTED_RESULT = "tgi"; type(WORD_TO_TYPE); - // Choose the typed word, which should be in position 1 (because position 0 should - // be occupied by the "this" auto-correction, as checked by testAutoCorrect()) - pickSuggestionManually(1, WORD_TO_TYPE); + // Choose the typed word. + pickSuggestionManually(WORD_TO_TYPE); mLatinIME.onUpdateSelection(0, 0, WORD_TO_TYPE.length(), WORD_TO_TYPE.length(), -1, -1); assertEquals("pick typed word over auto-correction then backspace", WORD_TO_TYPE, mEditText.getText().toString()); @@ -75,9 +76,8 @@ public class InputLogicTests extends InputTestsBase { final String WORD_TO_PICK = "thus"; final String EXPECTED_RESULT = "thu"; type(WORD_TO_TYPE); - // Choose the second suggestion, which should be in position 2 and should be "thus" - // when "tgis is typed. - pickSuggestionManually(2, WORD_TO_PICK); + // Choose the second suggestion, which should be "thus" when "tgis" is typed. + pickSuggestionManually(WORD_TO_PICK); mLatinIME.onUpdateSelection(0, 0, WORD_TO_TYPE.length(), WORD_TO_TYPE.length(), -1, -1); assertEquals("pick different suggestion then backspace", WORD_TO_PICK, mEditText.getText().toString()); @@ -179,6 +179,8 @@ public class InputLogicTests extends InputTestsBase { } public void testDoubleSpace() { + // Set default pref just in case + setBooleanPreference(Settings.PREF_KEY_USE_DOUBLE_SPACE_PERIOD, true, true); // U+1F607 is an emoji final String[] STRINGS_TO_TYPE = new String[] { "this ", "a+ ", "\u1F607 ", ".. ", ") ", "( ", "% " }; @@ -200,6 +202,76 @@ public class InputLogicTests extends InputTestsBase { assertEquals("double space make a period", EXPECTED_RESULT, mEditText.getText().toString()); } + private void testDoubleSpacePeriodWithSettings(final boolean expectsPeriod, + final Object... settingsKeysValues) { + final Object[] oldSettings = new Object[settingsKeysValues.length / 2]; + final String STRING_WITHOUT_PERIOD = "this "; + final String STRING_WITH_PERIOD = "this. "; + final String EXPECTED_RESULT = expectsPeriod ? STRING_WITH_PERIOD : STRING_WITHOUT_PERIOD; + try { + for (int i = 0; i < settingsKeysValues.length; i += 2) { + if (settingsKeysValues[i + 1] instanceof String) { + oldSettings[i / 2] = setStringPreference((String)settingsKeysValues[i], + (String)settingsKeysValues[i + 1], "0"); + } else { + oldSettings[i / 2] = setBooleanPreference((String)settingsKeysValues[i], + (Boolean)settingsKeysValues[i + 1], false); + } + } + mLatinIME.loadSettings(); + mEditText.setText(""); + type(STRING_WITHOUT_PERIOD); + assertEquals("double-space-to-period with specific settings " + + TextUtils.join(" ", settingsKeysValues), + EXPECTED_RESULT, mEditText.getText().toString()); + } finally { + // Restore old settings + for (int i = 0; i < settingsKeysValues.length; i += 2) { + if (null == oldSettings[i / 2]) { + break; + } if (oldSettings[i / 2] instanceof String) { + setStringPreference((String)settingsKeysValues[i], (String)oldSettings[i / 2], + ""); + } else { + setBooleanPreference((String)settingsKeysValues[i], (Boolean)oldSettings[i / 2], + false); + } + } + } + } + + public void testDoubleSpacePeriod() { + // Reset settings to default, else these tests will go flaky. + setStringPreference(Settings.PREF_SHOW_SUGGESTIONS_SETTING, "0", "0"); + setStringPreference(Settings.PREF_AUTO_CORRECTION_THRESHOLD, "1", "1"); + setBooleanPreference(Settings.PREF_KEY_USE_DOUBLE_SPACE_PERIOD, true, true); + testDoubleSpacePeriodWithSettings(true /* expectsPeriod */); + // "Suggestion visibility" to "always hide" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_SHOW_SUGGESTIONS_SETTING, "2"); + // "Suggestion visibility" to "portrait only" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_SHOW_SUGGESTIONS_SETTING, "1"); + // "Suggestion visibility" to "always show" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_SHOW_SUGGESTIONS_SETTING, "0"); + + // "Double-space period" to "off" + testDoubleSpacePeriodWithSettings(false, Settings.PREF_KEY_USE_DOUBLE_SPACE_PERIOD, false); + + // "Auto-correction" to "off" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_AUTO_CORRECTION_THRESHOLD, "0"); + // "Auto-correction" to "modest" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_AUTO_CORRECTION_THRESHOLD, "1"); + // "Auto-correction" to "very aggressive" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_AUTO_CORRECTION_THRESHOLD, "3"); + + // "Suggestion visibility" to "always hide" and "Auto-correction" to "off" + testDoubleSpacePeriodWithSettings(true, Settings.PREF_SHOW_SUGGESTIONS_SETTING, "0", + Settings.PREF_AUTO_CORRECTION_THRESHOLD, "0"); + // "Suggestion visibility" to "always hide" and "Auto-correction" to "off" + testDoubleSpacePeriodWithSettings(false, Settings.PREF_SHOW_SUGGESTIONS_SETTING, "0", + Settings.PREF_AUTO_CORRECTION_THRESHOLD, "0", + Settings.PREF_KEY_USE_DOUBLE_SPACE_PERIOD, false); + } + public void testBackspaceAtStartAfterAutocorrect() { final String STRING_TO_TYPE = "tgis "; final int typedLength = STRING_TO_TYPE.length(); @@ -234,7 +306,7 @@ public class InputLogicTests extends InputTestsBase { final String WORD_TO_TYPE = "this"; final String EXPECTED_RESULT = WORD_TO_TYPE; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); assertEquals("no space after manual pick", EXPECTED_RESULT, mEditText.getText().toString()); } @@ -244,7 +316,7 @@ public class InputLogicTests extends InputTestsBase { final String WORD2_TO_TYPE = "is"; final String EXPECTED_RESULT = "this is"; type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); + pickSuggestionManually(WORD1_TO_TYPE); type(WORD2_TO_TYPE); assertEquals("manual pick then type", EXPECTED_RESULT, mEditText.getText().toString()); } @@ -254,20 +326,32 @@ public class InputLogicTests extends InputTestsBase { final String WORD2_TO_TYPE = "!"; final String EXPECTED_RESULT = "this!"; type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); + pickSuggestionManually(WORD1_TO_TYPE); type(WORD2_TO_TYPE); assertEquals("manual pick then separator", EXPECTED_RESULT, mEditText.getText().toString()); } + // This test matches the one in InputLogicTestsNonEnglish. In some non-English languages, + // ! and ? are clustering punctuation signs. + public void testClusteringPunctuation() { + final String WORD1_TO_TYPE = "test"; + final String WORD2_TO_TYPE = "!!?!:!"; + final String EXPECTED_RESULT = "test!!?!:!"; + type(WORD1_TO_TYPE); + pickSuggestionManually(WORD1_TO_TYPE); + type(WORD2_TO_TYPE); + assertEquals("clustering punctuation", EXPECTED_RESULT, mEditText.getText().toString()); + } + public void testManualPickThenStripperThenPick() { final String WORD_TO_TYPE = "this"; final String STRIPPER = "\n"; final String EXPECTED_RESULT = "this\nthis"; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); type(STRIPPER); type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); assertEquals("manual pick then \\n then manual pick", EXPECTED_RESULT, mEditText.getText().toString()); } @@ -277,7 +361,7 @@ public class InputLogicTests extends InputTestsBase { final String WORD2_TO_TYPE = " is"; final String EXPECTED_RESULT = "this is"; type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); + pickSuggestionManually(WORD1_TO_TYPE); type(WORD2_TO_TYPE); assertEquals("manual pick then space then type", EXPECTED_RESULT, mEditText.getText().toString()); @@ -288,11 +372,9 @@ public class InputLogicTests extends InputTestsBase { final String WORD2_TO_PICK = "is"; final String EXPECTED_RESULT = "this is"; type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); - // Here we fake picking a word through bigram prediction. This test is taking - // advantage of the fact that Latin IME blindly trusts the caller of #pickSuggestionManually - // to actually pass the right string. - pickSuggestionManually(1, WORD2_TO_PICK); + pickSuggestionManually(WORD1_TO_TYPE); + // Here we fake picking a word through bigram prediction. + pickSuggestionManually(WORD2_TO_PICK); assertEquals("manual pick then manual pick", EXPECTED_RESULT, mEditText.getText().toString()); } @@ -307,12 +389,14 @@ public class InputLogicTests extends InputTestsBase { } public void testResumeSuggestionOnBackspace() { - final String WORD_TO_TYPE = "and this "; - type(WORD_TO_TYPE); + final String STRING_TO_TYPE = "and this "; + final int typedLength = STRING_TO_TYPE.length(); + type(STRING_TO_TYPE); assertEquals("resume suggestion on backspace", -1, BaseInputConnection.getComposingSpanStart(mEditText.getText())); assertEquals("resume suggestion on backspace", -1, BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + mLatinIME.onUpdateSelection(0, 0, typedLength, typedLength, -1, -1); type(Constants.CODE_DELETE); assertEquals("resume suggestion on backspace", 4, BaseInputConnection.getComposingSpanStart(mEditText.getText())); @@ -348,4 +432,211 @@ public class InputLogicTests extends InputTestsBase { helperTestComposing("a'", true); } // TODO: Add some tests for non-BMP characters + + public void testAutoCorrectByUserHistory() { + final String WORD_TO_BE_CORRECTED = "qpmx"; + final String NOT_CORRECTED_RESULT = "qpmx "; + final String DESIRED_WORD = "qpmz"; + final String CORRECTED_RESULT = "qpmz "; + final int typeCountNotToAutocorrect = 1; + final int typeCountToAutoCorrect = 16; + int startIndex = 0; + int endIndex = 0; + + for (int i = 0; i < typeCountNotToAutocorrect; i++) { + type(DESIRED_WORD); + type(Constants.CODE_SPACE); + } + startIndex = mEditText.getText().length(); + type(WORD_TO_BE_CORRECTED); + type(Constants.CODE_SPACE); + endIndex = mEditText.getText().length(); + assertEquals("not auto-corrected by user history", NOT_CORRECTED_RESULT, + mEditText.getText().subSequence(startIndex, endIndex).toString()); + for (int i = typeCountNotToAutocorrect; i < typeCountToAutoCorrect; i++) { + type(DESIRED_WORD); + type(Constants.CODE_SPACE); + } + startIndex = mEditText.getText().length(); + type(WORD_TO_BE_CORRECTED); + type(Constants.CODE_SPACE); + endIndex = mEditText.getText().length(); + assertEquals("auto-corrected by user history", + CORRECTED_RESULT, mEditText.getText().subSequence(startIndex, endIndex).toString()); + } + + public void testPredictionsAfterSpace() { + final String WORD_TO_TYPE = "Barack "; + type(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Test the first prediction is displayed + final SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("predictions after space", "Obama", + suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); + } + + public void testPredictionsWithDoubleSpaceToPeriod() { + mLatinIME.clearPersonalizedDictionariesForTest(); + final String WORD_TO_TYPE = "Barack "; + type(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // No need to test here, testPredictionsAfterSpace is testing it already + type(" "); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Test the predictions have been cleared + SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("predictions cleared after double-space-to-period", suggestedWords.size(), 0); + type(Constants.CODE_DELETE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Test the first prediction is displayed + suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("predictions after cancel double-space-to-period", "Obama", + suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); + } + + public void testPredictionsAfterManualPick() { + final String WORD_TO_TYPE = "Barack"; + type(WORD_TO_TYPE); + // Choose the auto-correction. For "Barack", the auto-correction should be "Barack". + pickSuggestionManually(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Test the first prediction is displayed + final SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("predictions after manual pick", "Obama", + suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); + } + + public void testPredictionsAfterPeriod() { + mLatinIME.clearPersonalizedDictionariesForTest(); + final String WORD_TO_TYPE = "Barack. "; + type(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("No prediction after period after inputting once.", 0, suggestedWords.size()); + + type(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("Beginning-of-Sentence prediction after inputting 2 times.", "Barack", + suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); + } + + public void testPredictionsAfterRecorrection() { + final String PREFIX = "A "; + final String WORD_TO_TYPE = "Barack"; + final String FIRST_NON_TYPED_SUGGESTION = "Barrack"; + final int endOfPrefix = PREFIX.length(); + final int endOfWord = endOfPrefix + WORD_TO_TYPE.length(); + final int endOfSuggestion = endOfPrefix + FIRST_NON_TYPED_SUGGESTION.length(); + final int indexForManualCursor = endOfPrefix + 3; // +3 because it's after "Bar" in "Barack" + type(PREFIX); + mLatinIME.onUpdateSelection(0, 0, endOfPrefix, endOfPrefix, -1, -1); + type(WORD_TO_TYPE); + pickSuggestionManually(FIRST_NON_TYPED_SUGGESTION); + mLatinIME.onUpdateSelection(endOfPrefix, endOfPrefix, endOfSuggestion, endOfSuggestion, + -1, -1); + runMessages(); + type(" "); + mLatinIME.onUpdateSelection(endOfSuggestion, endOfSuggestion, + endOfSuggestion + 1, endOfSuggestion + 1, -1, -1); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Simulate a manual cursor move + mInputConnection.setSelection(indexForManualCursor, indexForManualCursor); + mLatinIME.onUpdateSelection(endOfSuggestion + 1, endOfSuggestion + 1, + indexForManualCursor, indexForManualCursor, -1, -1); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + pickSuggestionManually(WORD_TO_TYPE); + mLatinIME.onUpdateSelection(indexForManualCursor, indexForManualCursor, + endOfWord, endOfWord, -1, -1); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Test the first prediction is displayed + final SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("predictions after recorrection", "Obama", + suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); + } + + public void testComposingMultipleBackspace() { + final String WORD_TO_TYPE = "radklro"; + final int TIMES_TO_TYPE = 3; + final int TIMES_TO_BACKSPACE = 8; + type(WORD_TO_TYPE); + type(Constants.CODE_DELETE); + type(Constants.CODE_DELETE); + type(Constants.CODE_DELETE); + type(WORD_TO_TYPE); + type(Constants.CODE_DELETE); + type(Constants.CODE_DELETE); + type(WORD_TO_TYPE); + type(Constants.CODE_DELETE); + type(Constants.CODE_DELETE); + type(Constants.CODE_DELETE); + assertEquals("composing with multiple backspace", + WORD_TO_TYPE.length() * TIMES_TO_TYPE - TIMES_TO_BACKSPACE, + mEditText.getText().length()); + } + + public void testManySingleQuotes() { + final String WORD_TO_AUTOCORRECT = "i"; + final String WORD_AUTOCORRECTED = "I"; + final String QUOTES = "''''''''''''''''''''"; + final String WORD_TO_TYPE = WORD_TO_AUTOCORRECT + QUOTES + " "; + final String EXPECTED_RESULT = WORD_AUTOCORRECTED + QUOTES + " "; + type(WORD_TO_TYPE); + assertEquals("auto-correct with many trailing single quotes", EXPECTED_RESULT, + mEditText.getText().toString()); + } + + public void testManySingleQuotesOneByOne() { + final String WORD_TO_AUTOCORRECT = "i"; + final String WORD_AUTOCORRECTED = "I"; + final String QUOTES = "''''''''''''''''''''"; + final String WORD_TO_TYPE = WORD_TO_AUTOCORRECT + QUOTES + " "; + final String EXPECTED_RESULT = WORD_AUTOCORRECTED + QUOTES + " "; + + for (int i = 0; i < WORD_TO_TYPE.length(); ++i) { + type(WORD_TO_TYPE.substring(i, i+1)); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + } + assertEquals("type many trailing single quotes one by one", EXPECTED_RESULT, + mEditText.getText().toString()); + } + + public void testTypingSingleQuotesOneByOne() { + final String WORD_TO_TYPE = "it's "; + final String EXPECTED_RESULT = WORD_TO_TYPE; + for (int i = 0; i < WORD_TO_TYPE.length(); ++i) { + type(WORD_TO_TYPE.substring(i, i+1)); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + } + assertEquals("type words letter by letter", EXPECTED_RESULT, + mEditText.getText().toString()); + } + + public void testSwitchLanguages() { + final String WORD_TO_TYPE_FIRST_PART = "com"; + final String WORD_TO_TYPE_SECOND_PART = "md"; + final String EXPECTED_RESULT = "comme"; + changeLanguage("en"); + type(WORD_TO_TYPE_FIRST_PART); + changeLanguage("fr"); + runMessages(); + type(WORD_TO_TYPE_SECOND_PART); + sleep(DELAY_TO_WAIT_FOR_UNDERLINE); + runMessages(); + final SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); + assertEquals("Suggestions updated after switching languages", + EXPECTED_RESULT, suggestedWords.size() > 0 ? suggestedWords.getWord(1) : null); + } } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java index 0f0ebafb9..2560407dc 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java @@ -19,8 +19,6 @@ package com.android.inputmethod.latin; import android.test.suitebuilder.annotation.LargeTest; import android.view.inputmethod.BaseInputConnection; -import com.android.inputmethod.latin.suggestions.SuggestionStripView; - @LargeTest public class InputLogicTestsLanguageWithoutSpaces extends InputTestsBase { public void testAutoCorrectForLanguageWithoutSpaces() { @@ -99,7 +97,8 @@ public class InputLogicTestsLanguageWithoutSpaces extends InputTestsBase { assertEquals("predictions in lang without spaces", "Barack", mEditText.getText().toString()); // Test the first prediction is displayed + final SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); assertEquals("predictions in lang without spaces", "Obama", - mLatinIME.getFirstSuggestedWord()); + suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); } } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTestsNonEnglish.java b/tests/src/com/android/inputmethod/latin/InputLogicTestsNonEnglish.java index 2d736e338..866f8894c 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTestsNonEnglish.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTestsNonEnglish.java @@ -18,8 +18,6 @@ package com.android.inputmethod.latin; import android.test.suitebuilder.annotation.LargeTest; -import com.android.inputmethod.latin.suggestions.SuggestionStripView; - @LargeTest public class InputLogicTestsNonEnglish extends InputTestsBase { final String NEXT_WORD_PREDICTION_OPTION = "next_word_prediction"; @@ -39,12 +37,25 @@ public class InputLogicTestsNonEnglish extends InputTestsBase { final String EXPECTED_RESULT = "test !"; changeLanguage("fr"); type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); + pickSuggestionManually(WORD1_TO_TYPE); type(WORD2_TO_TYPE); assertEquals("manual pick then separator for French", EXPECTED_RESULT, mEditText.getText().toString()); } + public void testClusteringPunctuationForFrench() { + final String WORD1_TO_TYPE = "test"; + final String WORD2_TO_TYPE = "!!?!:!"; + // In English, the expected result would be "test!!?!:!" + final String EXPECTED_RESULT = "test !!?! : !"; + changeLanguage("fr"); + type(WORD1_TO_TYPE); + pickSuggestionManually(WORD1_TO_TYPE); + type(WORD2_TO_TYPE); + assertEquals("clustering punctuation for French", EXPECTED_RESULT, + mEditText.getText().toString()); + } + public void testWordThenSpaceThenPunctuationFromStripTwiceForFrench() { final String WORD_TO_TYPE = "test "; final String PUNCTUATION_FROM_STRIP = "!"; @@ -60,9 +71,9 @@ public class InputLogicTestsNonEnglish extends InputTestsBase { sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); assertTrue("type word then type space should display punctuation strip", - mLatinIME.isShowingPunctuationList()); - pickSuggestionManually(0, PUNCTUATION_FROM_STRIP); - pickSuggestionManually(0, PUNCTUATION_FROM_STRIP); + mLatinIME.getSuggestedWordsForTest().isPunctuationSuggestions()); + pickSuggestionManually(PUNCTUATION_FROM_STRIP); + pickSuggestionManually(PUNCTUATION_FROM_STRIP); assertEquals("type word then type space then punctuation from strip twice for French", EXPECTED_RESULT, mEditText.getText().toString()); } finally { @@ -84,8 +95,9 @@ public class InputLogicTestsNonEnglish extends InputTestsBase { type(WORD_TO_TYPE); sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); + final SuggestedWords suggestedWords = mLatinIME.getSuggestedWordsForTest(); assertEquals("type word then type space yields predictions for French", - EXPECTED_RESULT, mLatinIME.getFirstSuggestedWord()); + EXPECTED_RESULT, suggestedWords.size() > 0 ? suggestedWords.getWord(0) : null); } finally { setBooleanPreference(NEXT_WORD_PREDICTION_OPTION, previousNextWordPredictionOption, defaultNextWordPredictionOption); diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTestsReorderingMyanmar.java b/tests/src/com/android/inputmethod/latin/InputLogicTestsReorderingMyanmar.java new file mode 100644 index 000000000..61eae4e8b --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/InputLogicTestsReorderingMyanmar.java @@ -0,0 +1,234 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.suitebuilder.annotation.LargeTest; +import android.util.Pair; + +/* + * Relevant characters for this test : + * Spurs the need to reorder : + * U+1031 MYANMAR VOWEL SIGN E : ေ + * U+1004 U+103A U+1039 Kinzi. It's a compound character. + * + * List of consonants : + * U+1000 MYANMAR LETTER KA က + * U+1001 MYANMAR LETTER KHA ခ + * U+1002 MYANMAR LETTER GA ဂ + * U+1003 MYANMAR LETTER GHA ဃ + * U+1004 MYANMAR LETTER NGA င + * U+1005 MYANMAR LETTER CA စ + * U+1006 MYANMAR LETTER CHA ဆ + * U+1007 MYANMAR LETTER JA ဇ + * U+1008 MYANMAR LETTER JHA ဈ + * U+1009 MYANMAR LETTER NYA ဉ + * U+100A MYANMAR LETTER NNYA ည + * U+100B MYANMAR LETTER TTA ဋ + * U+100C MYANMAR LETTER TTHA ဌ + * U+100D MYANMAR LETTER DDA ဍ + * U+100E MYANMAR LETTER DDHA ဎ + * U+100F MYANMAR LETTER NNA ဏ + * U+1010 MYANMAR LETTER TA တ + * U+1011 MYANMAR LETTER THA ထ + * U+1012 MYANMAR LETTER DA ဒ + * U+1013 MYANMAR LETTER DHA ဓ + * U+1014 MYANMAR LETTER NA န + * U+1015 MYANMAR LETTER PA ပ + * U+1016 MYANMAR LETTER PHA ဖ + * U+1017 MYANMAR LETTER BA ဗ + * U+1018 MYANMAR LETTER BHA ဘ + * U+1019 MYANMAR LETTER MA မ + * U+101A MYANMAR LETTER YA ယ + * U+101B MYANMAR LETTER RA ရ + * U+101C MYANMAR LETTER LA လ + * U+101D MYANMAR LETTER WA ဝ + * U+101E MYANMAR LETTER SA သ + * U+101F MYANMAR LETTER HA ဟ + * U+1020 MYANMAR LETTER LLA ဠ + * U+103F MYANMAR LETTER GREAT SA ဿ + * + * List of medials : + * U+103B MYANMAR CONSONANT SIGN MEDIAL YA ျ + * U+103C MYANMAR CONSONANT SIGN MEDIAL RA ြ + * U+103D MYANMAR CONSONANT SIGN MEDIAL WA ွ + * U+103E MYANMAR CONSONANT SIGN MEDIAL HA ှ + * U+105E MYANMAR CONSONANT SIGN MON MEDIAL NA ၞ + * U+105F MYANMAR CONSONANT SIGN MON MEDIAL MA ၟ + * U+1060 MYANMAR CONSONANT SIGN MON MEDIAL LA ၠ + * U+1082 MYANMAR CONSONANT SIGN SHAN MEDIAL WA ႂ + * + * Other relevant characters : + * U+200C ZERO WIDTH NON-JOINER + * U+200B ZERO WIDTH SPACE + */ + +@LargeTest +@SuppressWarnings("rawtypes") +public class InputLogicTestsReorderingMyanmar extends InputTestsBase { + // The tests are formatted as follows. + // Each test is an entry in the array of Pair arrays. + + // One test is an array of pairs. Each pair contains, in the `first' member, + // the code points that the next key press should contain. In the `second' + // member is stored the string that should be in the text view after this + // key press. + + private static final Pair[][] TESTS = { + + // Tests for U+1031 MYANMAR VOWEL SIGN E : ေ + new Pair[] { // Type : U+1031 U+1000 U+101F ေ က ဟ + Pair.create(new int[] { 0x1031 }, "\u1031"), // ေ + Pair.create(new int[] { 0x1000 }, "\u1000\u1031"), // ကေ + Pair.create(new int[] { 0x101F }, "\u1000\u1031\u101F") // ကေဟ + }, + + new Pair[] { // Type : U+1000 U+1031 U+101F က ေ ဟ + Pair.create(new int[] { 0x1000 }, "\u1000"), // က + Pair.create(new int[] { 0x1031 }, "\u1000\u200B\u1031"), // ကေ + Pair.create(new int[] { 0x101F }, "\u1000\u101F\u1031") // ကဟေ + }, + + new Pair[] { // Type : U+1031 U+101D U+103E U+1018 ေ ဝ ှ ဘ + Pair.create(new int[] { 0x1031 }, "\u1031"), // ေ + Pair.create(new int[] { 0x101D }, "\u101D\u1031"), // ဝေ + Pair.create(new int[] { 0x103E }, "\u101D\u103E\u1031"), // ဝှေ + Pair.create(new int[] { 0x1018 }, "\u101D\u103E\u1031\u1018") // ဝှေဘ + }, + + new Pair[] { // Type : U+1031 U+1014 U+1031 U+1000 U+102C U+1004 U+103A U+1038 U+101C + // U+102C U+1038 U+104B ေ န ေ က ာ င ် း လ ာ း ။ + Pair.create(new int[] { 0x1031 }, "\u1031"), // ေ + Pair.create(new int[] { 0x1014 }, "\u1014\u1031"), // နေ + Pair.create(new int[] { 0x1031 }, "\u1014\u1031\u1031"), // နေေ + Pair.create(new int[] { 0x1000 }, "\u1014\u1031\u1000\u1031"), // နေကေ + Pair.create(new int[] { 0x102C }, "\u1014\u1031\u1000\u1031\u102C"), // နေကော + Pair.create(new int[] { 0x1004 }, "\u1014\u1031\u1000\u1031\u102C\u1004"), // နေကောင + Pair.create(new int[] { 0x103A }, // နေကောင် + "\u1014\u1031\u1000\u1031\u102C\u1004\u103A"), + Pair.create(new int[] { 0x1038 }, // နေကောင်း + "\u1014\u1031\u1000\u1031\u102C\u1004\u103A\u1038"), + Pair.create(new int[] { 0x101C }, // နေကောင်းလ + "\u1014\u1031\u1000\u1031\u102C\u1004\u103A\u1038\u101C"), + Pair.create(new int[] { 0x102C }, // နေကောင်းလာ + "\u1014\u1031\u1000\u1031\u102C\u1004\u103A\u1038\u101C\u102C"), + Pair.create(new int[] { 0x1038 }, // နေကောင်းလား + "\u1014\u1031\u1000\u1031\u102C\u1004\u103A\u1038\u101C\u102C\u1038"), + Pair.create(new int[] { 0x104B }, // နေကောင်းလား။ + "\u1014\u1031\u1000\u1031\u102C\u1004\u103A\u1038\u101C\u102C\u1038\u104B") + }, + + new Pair[] { // Type : U+1031 U+1031 U+1031 U+1000 ေ ေ ေ က + Pair.create(new int[] { 0x1031 }, "\u1031"), // ေ + Pair.create(new int[] { 0x1031 }, "\u1031\u1031"), // ေေ + Pair.create(new int[] { 0x1031 }, "\u1031\u1031\u1031"), // U+1031ေေေ + Pair.create(new int[] { 0x1000 }, "\u1031\u1031\u1000\u1031") // ေေကေ + }, + + new Pair[] { // Type : U+1031 U+1001 U+103B U+103D U+1038 ေ ခ ျ ွ း + Pair.create(new int[] { 0x1031 }, "\u1031"), // ေ + Pair.create(new int[] { 0x1001 }, "\u1001\u1031"), // ခေ + Pair.create(new int[] { 0x103B }, "\u1001\u103B\u1031"), // ချေ + Pair.create(new int[] { 0x103D }, "\u1001\u103B\u103D\u1031"), // ချွေ + Pair.create(new int[] { 0x1038 }, "\u1001\u103B\u103D\u1031\u1038") // ချွေး + }, + + // Tests for Kinzi U+1004 U+103A U+1039 : + + /* Kinzi reordering is not implemented yet. Uncomment these tests when it is. + + new Pair[] { // Type : U+1021 U+1002 (U+1004 U+103A U+1039) + // U+101C U+1014 U+103A အ ဂ (င ် ္) လ န ် + Pair.create(new int[] { 0x1021 }, "\u1021"), // အ + Pair.create(new int[] { 0x1002 }, "\u1021\u1002"), // အဂ + Pair.create(new int[] { 0x1004, 0x103A, 0x1039 }, // အင်္ဂ + "\u1021\u1004\u103A\u1039\u1002"), + Pair.create(new int[] { 0x101C }, // အင်္ဂလ + "\u1021\u1004\u103A\u1039\u1002\u101C"), + Pair.create(new int[] { 0x1014 }, // အင်္ဂလန + "\u1021\u1004\u103A\u1039\u1002\u101C\u1014"), + Pair.create(new int[] { 0x103A }, // အင်္ဂလန် + "\u1021\u1004\u103A\u1039\u1002\u101C\u1014\u103A") + }, + + new Pair[] { //Type : kinzi after a whole syllable U+101E U+1001 U+103B U+102D U+102F + // (U+1004 U+103A U+1039) U+1004 U+103A U+1038 သ ခ ျ ိ ု င ် ္ င ် း + Pair.create(new int[] { 0x101E }, "\u101E"), // သခ + Pair.create(new int[] { 0x1001 }, "\u101E\u1001"), // သခ + Pair.create(new int[] { 0x103B }, "\u101E\u1001\u103B"), // သချ + Pair.create(new int[] { 0x102D }, "\u101E\u1001\u103B\u102D"), // သချိ + Pair.create(new int[] { 0x102F }, "\u101E\u1001\u103B\u102D\u102F"), // သချို + Pair.create(new int[] { 0x1004, 0x103A, 0x1039}, // သင်္ချို + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F"), + Pair.create(new int[] { 0x1004 }, // သင်္ချိုင + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F\u1004"), + Pair.create(new int[] { 0x103A }, // သင်္ချိုင် + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F\u1004\u103A"), + Pair.create(new int[] { 0x1038 }, // သင်္ချိုင်း + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F\u1004\u103A\u1038") + }, + + new Pair[] { // Type : kinzi after the consonant U+101E U+1001 (U+1004 U+103A U+1039) + // U+103B U+102D U+102F U+1004 U+103A U+1038 သ ခ င ် ္ ျ ိ ု င ် း + Pair.create(new int[] { 0x101E }, "\u101E"), // သခ + Pair.create(new int[] { 0x1001 }, "\u101E\u1001"), // သခ + Pair.create(new int[] { 0x1004, 0x103A, 0x1039 }, // သင်္ခ + "\u101E\u1004\u103A\u1039\u1001"), + Pair.create(new int[] { 0x103B }, // သင်္ချ + "\u101E\u1004\u103A\u1039\u1001\u103B"), + Pair.create(new int[] { 0x102D }, // သင်္ချိ + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D"), + Pair.create(new int[] { 0x102F }, // သင်္ချို + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F"), + Pair.create(new int[] { 0x1004 }, // သင်္ချိုင + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F\u1004"), + Pair.create(new int[] { 0x103A }, // သင်္ချိုင် + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F\u1004\u103A"), + Pair.create(new int[] { 0x1038 }, // သင်္ချိုင်း + "\u101E\u1004\u103A\u1039\u1001\u103B\u102D\u102F\u1004\u103A\u1038") + }, + */ + }; + + @SuppressWarnings("unchecked") + private void doMyanmarTest(final int testNumber, final Pair[] test) { + int stepNumber = 0; + for (final Pair<int[], String> step : test) { + ++stepNumber; + final int[] input = step.first; + final String expectedResult = step.second; + if (input.length > 1) { + mLatinIME.onTextInput(new String(input, 0, input.length)); + } else { + type(input[0]); + } + assertEquals("Myanmar reordering test " + testNumber + ", step " + stepNumber, + expectedResult, mEditText.getText().toString()); + } + } + + public void testMyanmarReordering() { + int testNumber = 0; + changeLanguage("my_MM", "CombiningRules=MyanmarReordering"); + for (final Pair[] test : TESTS) { + // Small trick to reset LatinIME : setText("") and send updateSelection with values + // LatinIME has never seen, and cursor pos 0,0. + mEditText.setText(""); + mLatinIME.onUpdateSelection(1, 1, 0, 0, -1, -1); + doMyanmarTest(++testNumber, test); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/InputPointersTests.java b/tests/src/com/android/inputmethod/latin/InputPointersTests.java index 5095f9606..1a47cddf4 100644 --- a/tests/src/com/android/inputmethod/latin/InputPointersTests.java +++ b/tests/src/com/android/inputmethod/latin/InputPointersTests.java @@ -55,14 +55,22 @@ public class InputPointersTests extends AndroidTestCase { final InputPointers src = new InputPointers(DEFAULT_CAPACITY); final int limit = src.getXCoordinates().length * 2 + 10; for (int i = 0; i < limit; i++) { - src.addPointer(i, i * 2, i * 3, i * 4); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + src.addPointer(x, y, pointerId, time); assertEquals("size after add " + i, i + 1, src.getPointerSize()); } for (int i = 0; i < limit; i++) { - assertEquals("xCoordinates at " + i, i, src.getXCoordinates()[i]); - assertEquals("yCoordinates at " + i, i * 2, src.getYCoordinates()[i]); - assertEquals("pointerIds at " + i, i * 3, src.getPointerIds()[i]); - assertEquals("times at " + i, i * 4, src.getTimes()[i]); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + assertEquals("xCoordinates at " + i, x, src.getXCoordinates()[i]); + assertEquals("yCoordinates at " + i, y, src.getYCoordinates()[i]); + assertEquals("pointerIds at " + i, pointerId, src.getPointerIds()[i]); + assertEquals("times at " + i, time, src.getTimes()[i]); } } @@ -70,14 +78,22 @@ public class InputPointersTests extends AndroidTestCase { final InputPointers src = new InputPointers(DEFAULT_CAPACITY); final int limit = 1000, step = 100; for (int i = 0; i < limit; i += step) { - src.addPointer(i, i, i * 2, i * 3, i * 4); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + src.addPointerAt(i, x, y, pointerId, time); assertEquals("size after add at " + i, i + 1, src.getPointerSize()); } for (int i = 0; i < limit; i += step) { - assertEquals("xCoordinates at " + i, i, src.getXCoordinates()[i]); - assertEquals("yCoordinates at " + i, i * 2, src.getYCoordinates()[i]); - assertEquals("pointerIds at " + i, i * 3, src.getPointerIds()[i]); - assertEquals("times at " + i, i * 4, src.getTimes()[i]); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + assertEquals("xCoordinates at " + i, x, src.getXCoordinates()[i]); + assertEquals("yCoordinates at " + i, y, src.getYCoordinates()[i]); + assertEquals("pointerIds at " + i, pointerId, src.getPointerIds()[i]); + assertEquals("times at " + i, time, src.getTimes()[i]); } } @@ -85,7 +101,11 @@ public class InputPointersTests extends AndroidTestCase { final InputPointers src = new InputPointers(DEFAULT_CAPACITY); final int limit = src.getXCoordinates().length * 2 + 10; for (int i = 0; i < limit; i++) { - src.addPointer(i, i * 2, i * 3, i * 4); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + src.addPointer(x, y, pointerId, time); } final InputPointers dst = new InputPointers(DEFAULT_CAPACITY); dst.set(src); @@ -100,7 +120,11 @@ public class InputPointersTests extends AndroidTestCase { final InputPointers src = new InputPointers(DEFAULT_CAPACITY); final int limit = 100; for (int i = 0; i < limit; i++) { - src.addPointer(i, i * 2, i * 3, i * 4); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + src.addPointer(x, y, pointerId, time); } final InputPointers dst = new InputPointers(DEFAULT_CAPACITY); dst.copy(src); @@ -121,106 +145,135 @@ public class InputPointersTests extends AndroidTestCase { } public void testAppend() { - final InputPointers src = new InputPointers(DEFAULT_CAPACITY); - final int srcLen = 100; - for (int i = 0; i < srcLen; i++) { - src.addPointer(i, i * 2, i * 3, i * 4); - } - final int dstLen = 50; + final int dstLength = 50; final InputPointers dst = new InputPointers(DEFAULT_CAPACITY); - for (int i = 0; i < dstLen; i++) { - final int value = -i - 1; - dst.addPointer(value * 4, value * 3, value * 2, value); + for (int i = 0; i < dstLength; i++) { + final int x = i * 4; + final int y = i * 3; + final int pointerId = i * 2; + final int time = i; + dst.addPointer(x, y, pointerId, time); } final InputPointers dstCopy = new InputPointers(DEFAULT_CAPACITY); dstCopy.copy(dst); - dst.append(src, 0, 0); - assertEquals("size after append zero", dstLen, dst.getPointerSize()); + final ResizableIntArray srcXCoords = new ResizableIntArray(DEFAULT_CAPACITY); + final ResizableIntArray srcYCoords = new ResizableIntArray(DEFAULT_CAPACITY); + final ResizableIntArray srcPointerIds = new ResizableIntArray(DEFAULT_CAPACITY); + final ResizableIntArray srcTimes = new ResizableIntArray(DEFAULT_CAPACITY); + final int srcLength = 100; + final int srcPointerId = 10; + for (int i = 0; i < srcLength; i++) { + final int x = i; + final int y = i * 2; + // The time value must be larger than <code>dst</code>. + final int time = i * 4 + dstLength; + srcXCoords.add(x); + srcYCoords.add(y); + srcPointerIds.add(srcPointerId); + srcTimes.add(time); + } + + final int startPos = 0; + dst.append(srcPointerId, srcTimes, srcXCoords, srcYCoords, + startPos, 0 /* length */); + assertEquals("size after append zero", dstLength, dst.getPointerSize()); assertIntArrayEquals("xCoordinates after append zero", - dstCopy.getXCoordinates(), 0, dst.getXCoordinates(), 0, dstLen); + dstCopy.getXCoordinates(), startPos, dst.getXCoordinates(), startPos, dstLength); assertIntArrayEquals("yCoordinates after append zero", - dstCopy.getYCoordinates(), 0, dst.getYCoordinates(), 0, dstLen); + dstCopy.getYCoordinates(), startPos, dst.getYCoordinates(), startPos, dstLength); assertIntArrayEquals("pointerIds after append zero", - dstCopy.getPointerIds(), 0, dst.getPointerIds(), 0, dstLen); + dstCopy.getPointerIds(), startPos, dst.getPointerIds(), startPos, dstLength); assertIntArrayEquals("times after append zero", - dstCopy.getTimes(), 0, dst.getTimes(), 0, dstLen); + dstCopy.getTimes(), startPos, dst.getTimes(), startPos, dstLength); - dst.append(src, 0, srcLen); - assertEquals("size after append", dstLen + srcLen, dst.getPointerSize()); + dst.append(srcPointerId, srcTimes, srcXCoords, srcYCoords, + startPos, srcLength); + assertEquals("size after append", dstLength + srcLength, dst.getPointerSize()); assertTrue("primitive length after append", - dst.getPointerIds().length >= dstLen + srcLen); + dst.getPointerIds().length >= dstLength + srcLength); assertIntArrayEquals("original xCoordinates values after append", - dstCopy.getXCoordinates(), 0, dst.getXCoordinates(), 0, dstLen); + dstCopy.getXCoordinates(), startPos, dst.getXCoordinates(), startPos, dstLength); assertIntArrayEquals("original yCoordinates values after append", - dstCopy.getYCoordinates(), 0, dst.getYCoordinates(), 0, dstLen); + dstCopy.getYCoordinates(), startPos, dst.getYCoordinates(), startPos, dstLength); assertIntArrayEquals("original pointerIds values after append", - dstCopy.getPointerIds(), 0, dst.getPointerIds(), 0, dstLen); + dstCopy.getPointerIds(), startPos, dst.getPointerIds(), startPos, dstLength); assertIntArrayEquals("original times values after append", - dstCopy.getTimes(), 0, dst.getTimes(), 0, dstLen); + dstCopy.getTimes(), startPos, dst.getTimes(), startPos, dstLength); assertIntArrayEquals("appended xCoordinates values after append", - src.getXCoordinates(), 0, dst.getXCoordinates(), dstLen, srcLen); + srcXCoords.getPrimitiveArray(), startPos, dst.getXCoordinates(), + dstLength, srcLength); assertIntArrayEquals("appended yCoordinates values after append", - src.getYCoordinates(), 0, dst.getYCoordinates(), dstLen, srcLen); + srcYCoords.getPrimitiveArray(), startPos, dst.getYCoordinates(), + dstLength, srcLength); assertIntArrayEquals("appended pointerIds values after append", - src.getPointerIds(), 0, dst.getPointerIds(), dstLen, srcLen); + srcPointerIds.getPrimitiveArray(), startPos, dst.getPointerIds(), + dstLength, srcLength); assertIntArrayEquals("appended times values after append", - src.getTimes(), 0, dst.getTimes(), dstLen, srcLen); + srcTimes.getPrimitiveArray(), startPos, dst.getTimes(), dstLength, srcLength); } public void testAppendResizableIntArray() { - final int srcLen = 100; + final int dstLength = 50; + final InputPointers dst = new InputPointers(DEFAULT_CAPACITY); + for (int i = 0; i < dstLength; i++) { + final int x = i * 4; + final int y = i * 3; + final int pointerId = i * 2; + final int time = i; + dst.addPointer(x, y, pointerId, time); + } + final InputPointers dstCopy = new InputPointers(DEFAULT_CAPACITY); + dstCopy.copy(dst); + + final int srcLength = 100; final int srcPointerId = 1; - final int[] srcPointerIds = new int[srcLen]; + final int[] srcPointerIds = new int[srcLength]; Arrays.fill(srcPointerIds, srcPointerId); final ResizableIntArray srcTimes = new ResizableIntArray(DEFAULT_CAPACITY); final ResizableIntArray srcXCoords = new ResizableIntArray(DEFAULT_CAPACITY); final ResizableIntArray srcYCoords= new ResizableIntArray(DEFAULT_CAPACITY); - for (int i = 0; i < srcLen; i++) { - srcTimes.add(i * 2); - srcXCoords.add(i * 3); - srcYCoords.add(i * 4); + for (int i = 0; i < srcLength; i++) { + // The time value must be larger than <code>dst</code>. + final int time = i * 2 + dstLength; + final int x = i * 3; + final int y = i * 4; + srcTimes.add(time); + srcXCoords.add(x); + srcYCoords.add(y); } - final int dstLen = 50; - final InputPointers dst = new InputPointers(DEFAULT_CAPACITY); - for (int i = 0; i < dstLen; i++) { - final int value = -i - 1; - dst.addPointer(value * 4, value * 3, value * 2, value); - } - final InputPointers dstCopy = new InputPointers(DEFAULT_CAPACITY); - dstCopy.copy(dst); dst.append(srcPointerId, srcTimes, srcXCoords, srcYCoords, 0, 0); - assertEquals("size after append zero", dstLen, dst.getPointerSize()); + assertEquals("size after append zero", dstLength, dst.getPointerSize()); assertIntArrayEquals("xCoordinates after append zero", - dstCopy.getXCoordinates(), 0, dst.getXCoordinates(), 0, dstLen); + dstCopy.getXCoordinates(), 0, dst.getXCoordinates(), 0, dstLength); assertIntArrayEquals("yCoordinates after append zero", - dstCopy.getYCoordinates(), 0, dst.getYCoordinates(), 0, dstLen); + dstCopy.getYCoordinates(), 0, dst.getYCoordinates(), 0, dstLength); assertIntArrayEquals("pointerIds after append zero", - dstCopy.getPointerIds(), 0, dst.getPointerIds(), 0, dstLen); + dstCopy.getPointerIds(), 0, dst.getPointerIds(), 0, dstLength); assertIntArrayEquals("times after append zero", - dstCopy.getTimes(), 0, dst.getTimes(), 0, dstLen); + dstCopy.getTimes(), 0, dst.getTimes(), 0, dstLength); - dst.append(srcPointerId, srcTimes, srcXCoords, srcYCoords, 0, srcLen); - assertEquals("size after append", dstLen + srcLen, dst.getPointerSize()); + dst.append(srcPointerId, srcTimes, srcXCoords, srcYCoords, 0, srcLength); + assertEquals("size after append", dstLength + srcLength, dst.getPointerSize()); assertTrue("primitive length after append", - dst.getPointerIds().length >= dstLen + srcLen); + dst.getPointerIds().length >= dstLength + srcLength); assertIntArrayEquals("original xCoordinates values after append", - dstCopy.getXCoordinates(), 0, dst.getXCoordinates(), 0, dstLen); + dstCopy.getXCoordinates(), 0, dst.getXCoordinates(), 0, dstLength); assertIntArrayEquals("original yCoordinates values after append", - dstCopy.getYCoordinates(), 0, dst.getYCoordinates(), 0, dstLen); + dstCopy.getYCoordinates(), 0, dst.getYCoordinates(), 0, dstLength); assertIntArrayEquals("original pointerIds values after append", - dstCopy.getPointerIds(), 0, dst.getPointerIds(), 0, dstLen); + dstCopy.getPointerIds(), 0, dst.getPointerIds(), 0, dstLength); assertIntArrayEquals("original times values after append", - dstCopy.getTimes(), 0, dst.getTimes(), 0, dstLen); + dstCopy.getTimes(), 0, dst.getTimes(), 0, dstLength); assertIntArrayEquals("appended xCoordinates values after append", - srcXCoords.getPrimitiveArray(), 0, dst.getXCoordinates(), dstLen, srcLen); + srcXCoords.getPrimitiveArray(), 0, dst.getXCoordinates(), dstLength, srcLength); assertIntArrayEquals("appended yCoordinates values after append", - srcYCoords.getPrimitiveArray(), 0, dst.getYCoordinates(), dstLen, srcLen); + srcYCoords.getPrimitiveArray(), 0, dst.getYCoordinates(), dstLength, srcLength); assertIntArrayEquals("appended pointerIds values after append", - srcPointerIds, 0, dst.getPointerIds(), dstLen, srcLen); + srcPointerIds, 0, dst.getPointerIds(), dstLength, srcLength); assertIntArrayEquals("appended times values after append", - srcTimes.getPrimitiveArray(), 0, dst.getTimes(), dstLen, srcLen); + srcTimes.getPrimitiveArray(), 0, dst.getTimes(), dstLength, srcLength); } // TODO: Consolidate this method with @@ -250,14 +303,24 @@ public class InputPointersTests extends AndroidTestCase { final int limit = 100; final int shiftAmount = 20; for (int i = 0; i < limit; i++) { - src.addPointer(i, i * 2, i * 3, i * 4); + final int x = i; + final int y = i * 2; + final int pointerId = i * 3; + final int time = i * 4; + src.addPointer(x, y, pointerId, time); } src.shift(shiftAmount); + assertEquals("length after shift", src.getPointerSize(), limit - shiftAmount); for (int i = 0; i < limit - shiftAmount; ++i) { - assertEquals("xCoordinates at " + i, i + shiftAmount, src.getXCoordinates()[i]); - assertEquals("yCoordinates at " + i, (i + shiftAmount) * 2, src.getYCoordinates()[i]); - assertEquals("pointerIds at " + i, (i + shiftAmount) * 3, src.getPointerIds()[i]); - assertEquals("times at " + i, (i + shiftAmount) * 4, src.getTimes()[i]); + final int oldIndex = i + shiftAmount; + final int x = oldIndex; + final int y = oldIndex * 2; + final int pointerId = oldIndex * 3; + final int time = oldIndex * 4; + assertEquals("xCoordinates at " + i, x, src.getXCoordinates()[i]); + assertEquals("yCoordinates at " + i, y, src.getYCoordinates()[i]); + assertEquals("pointerIds at " + i, pointerId, src.getPointerIds()[i]); + assertEquals("times at " + i, time, src.getTimes()[i]); } } } diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java index b9b52a6f3..986fb1097 100644 --- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java +++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java @@ -25,35 +25,48 @@ import android.text.InputType; import android.text.SpannableStringBuilder; import android.text.style.CharacterStyle; import android.text.style.SuggestionSpan; +import android.util.Log; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputConnection; +import android.view.inputmethod.InputMethodSubtype; import android.widget.EditText; import android.widget.FrameLayout; +import com.android.inputmethod.compat.InputMethodSubtypeCompatUtils; import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.settings.DebugSettings; +import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.LocaleUtils; +import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; import java.util.Locale; +import java.util.concurrent.TimeUnit; public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { + private static final String TAG = InputTestsBase.class.getSimpleName(); - private static final String PREF_DEBUG_MODE = "debug_mode"; + // Default value for auto-correction threshold. This is the string representation of the + // index in the resources array of auto-correction threshold settings. + private static final String DEFAULT_AUTO_CORRECTION_THRESHOLD = "1"; - // The message that sets the underline is posted with a 200 ms delay - protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 200; + // The message that sets the underline is posted with a 500 ms delay + protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 500; // The message that sets predictions is posted with a 200 ms delay protected static final int DELAY_TO_WAIT_FOR_PREDICTIONS = 200; + private final int TIMEOUT_TO_WAIT_FOR_LOADING_MAIN_DICTIONARY_IN_SECONDS = 60; protected LatinIME mLatinIME; protected Keyboard mKeyboard; protected MyEditText mEditText; protected View mInputView; protected InputConnection mInputConnection; + private boolean mPreviousBigramPredictionSettings; + private String mPreviousAutoCorrectSetting; // A helper class to ease span tests public static class SpanGetter { @@ -135,13 +148,30 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { final boolean previousSetting = prefs.getBoolean(key, defaultValue); final SharedPreferences.Editor editor = prefs.edit(); editor.putBoolean(key, value); - editor.commit(); + editor.apply(); return previousSetting; } - // returns the previous setting value - protected boolean setDebugMode(final boolean value) { - return setBooleanPreference(PREF_DEBUG_MODE, value, false); + protected String setStringPreference(final String key, final String value, + final String defaultValue) { + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(mLatinIME); + final String previousSetting = prefs.getString(key, defaultValue); + final SharedPreferences.Editor editor = prefs.edit(); + editor.putString(key, value); + editor.apply(); + return previousSetting; + } + + protected void setDebugMode(final boolean value) { + setBooleanPreference(DebugSettings.PREF_DEBUG_MODE, value, false); + setBooleanPreference(Settings.PREF_KEY_IS_INTERNAL, value, false); + } + + protected EditorInfo enrichEditorInfo(final EditorInfo ei) { + // Some tests that inherit from us need to add some data in the EditorInfo (see + // AppWorkaroundsTests#enrichEditorInfo() for a concrete example of this). Since we + // control the EditorInfo, we supply a hook here for children to override. + return ei; } @Override @@ -154,15 +184,19 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { mEditText.setEnabled(true); setupService(); mLatinIME = getService(); - final boolean previousDebugSetting = setDebugMode(true); + setDebugMode(true); + mPreviousBigramPredictionSettings = setBooleanPreference(Settings.PREF_BIGRAM_PREDICTIONS, + true, true /* defaultValue */); + mPreviousAutoCorrectSetting = setStringPreference(Settings.PREF_AUTO_CORRECTION_THRESHOLD, + DEFAULT_AUTO_CORRECTION_THRESHOLD, DEFAULT_AUTO_CORRECTION_THRESHOLD); mLatinIME.onCreate(); - setDebugMode(previousDebugSetting); - final EditorInfo ei = new EditorInfo(); + EditorInfo ei = new EditorInfo(); final InputConnection ic = mEditText.onCreateInputConnection(ei); final LayoutInflater inflater = (LayoutInflater)getContext().getSystemService(Context.LAYOUT_INFLATER_SERVICE); final ViewGroup vg = new FrameLayout(getContext()); mInputView = inflater.inflate(R.layout.input_view, vg); + ei = enrichEditorInfo(ei); mLatinIME.onCreateInputMethodInterface().startInput(ic, ei); mLatinIME.setInputView(mInputView); mLatinIME.onBindInput(); @@ -170,6 +204,27 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { mLatinIME.onStartInputView(ei, false); mInputConnection = ic; changeLanguage("en_US"); + // Run messages to avoid the messages enqueued by startInputView() and its friends + // to run on a later call and ruin things. We need to wait first because some of them + // can be posted with a delay (notably, MSG_RESUME_SUGGESTIONS) + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + } + + @Override + protected void tearDown() throws Exception { + mLatinIME.onFinishInputView(true); + mLatinIME.onFinishInput(); + runMessages(); + mLatinIME.mHandler.removeAllMessages(); + setBooleanPreference(Settings.PREF_BIGRAM_PREDICTIONS, mPreviousBigramPredictionSettings, + true /* defaultValue */); + setStringPreference(Settings.PREF_AUTO_CORRECTION_THRESHOLD, mPreviousAutoCorrectSetting, + DEFAULT_AUTO_CORRECTION_THRESHOLD); + setDebugMode(false); + mLatinIME.recycle(); + super.tearDown(); + mLatinIME = null; } // We need to run the messages added to the handler from LatinIME. The only way to do @@ -199,7 +254,7 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { } // type(int) and type(String): helper methods to send a code point resp. a string to LatinIME. - protected void type(final int codePoint) { + protected void typeInternal(final int codePoint, final boolean isKeyRepeat) { // onPressKey and onReleaseKey are explicitly deactivated here, but they do happen in the // code (although multitouch/slide input and other factors make the sequencing complicated). // They are supposed to be entirely deconnected from the input logic from LatinIME point of @@ -208,60 +263,90 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { // but keep them in mind if something breaks. Commenting them out as is should work. //mLatinIME.onPressKey(codePoint, 0 /* repeatCount */, true /* isSinglePointer */); final Key key = mKeyboard.getKey(codePoint); - if (key != null) { + if (key == null) { + mLatinIME.onCodeInput(codePoint, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, + isKeyRepeat); + } else { final int x = key.getX() + key.getWidth() / 2; final int y = key.getY() + key.getHeight() / 2; - mLatinIME.onCodeInput(codePoint, x, y); - return; + mLatinIME.onCodeInput(codePoint, x, y, isKeyRepeat); } - mLatinIME.onCodeInput(codePoint, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); + // Also see the comment at the top of this function about onReleaseKey //mLatinIME.onReleaseKey(codePoint, false /* withSliding */); } + protected void type(final int codePoint) { + typeInternal(codePoint, false /* isKeyRepeat */); + } + + protected void repeatKey(final int codePoint) { + typeInternal(codePoint, true /* isKeyRepeat */); + } + protected void type(final String stringToType) { for (int i = 0; i < stringToType.length(); i = stringToType.offsetByCodePoints(i, 1)) { type(stringToType.codePointAt(i)); } } - protected void waitForDictionaryToBeLoaded() { - int remainingAttempts = 300; - while (remainingAttempts > 0 && mLatinIME.isCurrentlyWaitingForMainDictionary()) { - try { - Thread.sleep(200); - } catch (InterruptedException e) { - // Don't do much - } finally { - --remainingAttempts; - } + protected void waitForDictionariesToBeLoaded() { + try { + mLatinIME.waitForLoadingDictionaries( + TIMEOUT_TO_WAIT_FOR_LOADING_MAIN_DICTIONARY_IN_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Log.e(TAG, "Interrupted during waiting for loading main dictionary.", e); } } protected void changeLanguage(final String locale) { - changeLanguageWithoutWait(locale); - waitForDictionaryToBeLoaded(); + changeLanguage(locale, null); + } + + protected void changeLanguage(final String locale, final String combiningSpec) { + changeLanguageWithoutWait(locale, combiningSpec); + waitForDictionariesToBeLoaded(); } - protected void changeLanguageWithoutWait(final String locale) { + protected void changeLanguageWithoutWait(final String locale, final String combiningSpec) { mEditText.mCurrentLocale = LocaleUtils.constructLocaleFromString(locale); - SubtypeSwitcher.getInstance().forceLocale(mEditText.mCurrentLocale); - mLatinIME.loadKeyboard(); + // TODO: this is forcing a QWERTY keyboard for all locales, which is wrong. + // It's still better than using whatever keyboard is the current one, but we + // should actually use the default keyboard for this locale. + // TODO: Use {@link InputMethodSubtype.InputMethodSubtypeBuilder} directly or indirectly so + // that {@link InputMethodSubtype#isAsciiCapable} can return the correct value. + final String EXTRA_VALUE_FOR_TEST = + "KeyboardLayoutSet=" + SubtypeLocaleUtils.QWERTY + + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE + + "," + Constants.Subtype.ExtraValue.ENABLED_WHEN_DEFAULT_IS_NOT_ASCII_CAPABLE + + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE + + null == combiningSpec ? "" : ("," + combiningSpec); + final InputMethodSubtype subtype = InputMethodSubtypeCompatUtils.newInputMethodSubtype( + R.string.subtype_no_language_qwerty, + R.drawable.ic_ime_switcher_dark, + locale, + Constants.Subtype.KEYBOARD_MODE, + EXTRA_VALUE_FOR_TEST, + false /* isAuxiliary */, + false /* overridesImplicitlyEnabledSubtype */, + 0 /* id */); + SubtypeSwitcher.getInstance().forceSubtype(subtype); + mLatinIME.onCurrentInputMethodSubtypeChanged(subtype); runMessages(); mKeyboard = mLatinIME.mKeyboardSwitcher.getKeyboard(); + mLatinIME.clearPersonalizedDictionariesForTest(); } protected void changeKeyboardLocaleAndDictLocale(final String keyboardLocale, final String dictLocale) { changeLanguage(keyboardLocale); if (!keyboardLocale.equals(dictLocale)) { - mLatinIME.replaceMainDictionaryForTest( - LocaleUtils.constructLocaleFromString(dictLocale)); + mLatinIME.replaceDictionariesForTest(LocaleUtils.constructLocaleFromString(dictLocale)); } - waitForDictionaryToBeLoaded(); + waitForDictionariesToBeLoaded(); } - protected void pickSuggestionManually(final int index, final String suggestion) { - mLatinIME.pickSuggestionManually(index, new SuggestedWordInfo(suggestion, 1, + protected void pickSuggestionManually(final String suggestion) { + mLatinIME.pickSuggestionManually(new SuggestedWordInfo(suggestion, 1, SuggestedWordInfo.KIND_CORRECTION, null /* sourceDict */, SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); diff --git a/tests/src/com/android/inputmethod/latin/LatinImeStressTests.java b/tests/src/com/android/inputmethod/latin/LatinImeStressTests.java index 5e98cdf8d..f5e993de8 100644 --- a/tests/src/com/android/inputmethod/latin/LatinImeStressTests.java +++ b/tests/src/com/android/inputmethod/latin/LatinImeStressTests.java @@ -30,10 +30,10 @@ public class LatinImeStressTests extends InputTestsBase { final int maxWordCountToTypeInEachIteration = 20; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - final int codePointSetSize = 30; final int[] codePointSet = CodePointUtils.LATIN_ALPHABETS_LOWER; for (int i = 0; i < switchCount; ++i) { - changeLanguageWithoutWait(locales[random.nextInt(locales.length)]); + changeLanguageWithoutWait(locales[random.nextInt(locales.length)], + null /* combiningSpec */); final int wordCount = random.nextInt(maxWordCountToTypeInEachIteration); for (int j = 0; j < wordCount; ++j) { final String word = CodePointUtils.generateWord(random, codePointSet); @@ -41,7 +41,7 @@ public class LatinImeStressTests extends InputTestsBase { } } } - public void testSwitchLanguagesAndInputRandamCodePoints() { + public void testSwitchLanguagesAndInputRandomCodePoints() { final String[] locales = {"en_US", "de", "el", "es", "fi", "it", "nl", "pt", "ru"}; final int switchCount = 50; final int maxWordCountToTypeInEachIteration = 20; @@ -50,7 +50,8 @@ public class LatinImeStressTests extends InputTestsBase { final int codePointSetSize = 30; final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); for (int i = 0; i < switchCount; ++i) { - changeLanguageWithoutWait(locales[random.nextInt(locales.length)]); + changeLanguageWithoutWait(locales[random.nextInt(locales.length)], + null /* combiningSpec */); final int wordCount = random.nextInt(maxWordCountToTypeInEachIteration); for (int j = 0; j < wordCount; ++j) { final String word = CodePointUtils.generateWord(random, codePointSet); diff --git a/tests/src/com/android/inputmethod/latin/PunctuationTests.java b/tests/src/com/android/inputmethod/latin/PunctuationTests.java index 84ff6b307..64750fbda 100644 --- a/tests/src/com/android/inputmethod/latin/PunctuationTests.java +++ b/tests/src/com/android/inputmethod/latin/PunctuationTests.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin; +import android.provider.Settings.Secure; import android.test.suitebuilder.annotation.LargeTest; import com.android.inputmethod.latin.R; @@ -40,9 +41,9 @@ public class PunctuationTests extends InputTestsBase { sleep(DELAY_TO_WAIT_FOR_UNDERLINE); runMessages(); assertTrue("type word then type space should display punctuation strip", - mLatinIME.isShowingPunctuationList()); - pickSuggestionManually(0, PUNCTUATION_FROM_STRIP); - pickSuggestionManually(0, PUNCTUATION_FROM_STRIP); + mLatinIME.getSuggestedWordsForTest().isPunctuationSuggestions()); + pickSuggestionManually(PUNCTUATION_FROM_STRIP); + pickSuggestionManually(PUNCTUATION_FROM_STRIP); assertEquals("type word then type space then punctuation from strip twice", EXPECTED_RESULT, mEditText.getText().toString()); } finally { @@ -65,9 +66,9 @@ public class PunctuationTests extends InputTestsBase { final String PUNCTUATION_FROM_STRIP = "!"; final String EXPECTED_RESULT = "this!! is"; type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); - pickSuggestionManually(0, PUNCTUATION_FROM_STRIP); - pickSuggestionManually(0, PUNCTUATION_FROM_STRIP); + pickSuggestionManually(WORD1_TO_TYPE); + pickSuggestionManually(PUNCTUATION_FROM_STRIP); + pickSuggestionManually(PUNCTUATION_FROM_STRIP); type(WORD2_TO_TYPE); assertEquals("pick word then pick punctuation twice then type", EXPECTED_RESULT, mEditText.getText().toString()); @@ -78,8 +79,8 @@ public class PunctuationTests extends InputTestsBase { final String WORD2_TO_PICK = "!is"; final String EXPECTED_RESULT = "this!is"; type(WORD1_TO_TYPE); - pickSuggestionManually(0, WORD1_TO_TYPE); - pickSuggestionManually(1, WORD2_TO_PICK); + pickSuggestionManually(WORD1_TO_TYPE); + pickSuggestionManually(WORD2_TO_PICK); assertEquals("manual pick then manual pick a word with punct at start", EXPECTED_RESULT, mEditText.getText().toString()); } @@ -89,7 +90,7 @@ public class PunctuationTests extends InputTestsBase { final String PUNCTUATION = ":"; final String EXPECTED_RESULT = "this:"; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); type(PUNCTUATION); assertEquals("manually pick word then colon", EXPECTED_RESULT, mEditText.getText().toString()); @@ -100,7 +101,7 @@ public class PunctuationTests extends InputTestsBase { final String PUNCTUATION = "("; final String EXPECTED_RESULT = "this ("; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); type(PUNCTUATION); assertEquals("manually pick word then open paren", EXPECTED_RESULT, mEditText.getText().toString()); @@ -111,7 +112,7 @@ public class PunctuationTests extends InputTestsBase { final String PUNCTUATION = ")"; final String EXPECTED_RESULT = "this)"; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); type(PUNCTUATION); assertEquals("manually pick word then close paren", EXPECTED_RESULT, mEditText.getText().toString()); @@ -122,7 +123,7 @@ public class PunctuationTests extends InputTestsBase { final String SPECIAL_KEY = ":-)"; final String EXPECTED_RESULT = "this :-)"; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); mLatinIME.onTextInput(SPECIAL_KEY); assertEquals("manually pick word then press the smiley key", EXPECTED_RESULT, mEditText.getText().toString()); @@ -133,7 +134,7 @@ public class PunctuationTests extends InputTestsBase { final String SPECIAL_KEY = ".com"; final String EXPECTED_RESULT = "this.com"; type(WORD_TO_TYPE); - pickSuggestionManually(0, WORD_TO_TYPE); + pickSuggestionManually(WORD_TO_TYPE); mLatinIME.onTextInput(SPECIAL_KEY); assertEquals("manually pick word then press the .com key", EXPECTED_RESULT, mEditText.getText().toString()); @@ -153,7 +154,9 @@ public class PunctuationTests extends InputTestsBase { final String WORD_TO_TYPE = "you'f "; final String EXPECTED_RESULT = "you'd "; type(WORD_TO_TYPE); - assertEquals("auto-correction with single quote inside", + assertEquals("auto-correction with single quote inside. ID = " + + Secure.getString(getContext().getContentResolver(), Secure.ANDROID_ID) + + " ; Suggestions = " + mLatinIME.getSuggestedWordsForTest(), EXPECTED_RESULT, mEditText.getText().toString()); } @@ -161,7 +164,37 @@ public class PunctuationTests extends InputTestsBase { final String WORD_TO_TYPE = "'tgis' "; final String EXPECTED_RESULT = "'this' "; type(WORD_TO_TYPE); - assertEquals("auto-correction with single quotes around", + assertEquals("auto-correction with single quotes around. ID = " + + Secure.getString(getContext().getContentResolver(), Secure.ANDROID_ID) + + " ; Suggestions = " + mLatinIME.getSuggestedWordsForTest(), + EXPECTED_RESULT, mEditText.getText().toString()); + } + + public void testAutoSpaceWithDoubleQuotes() { + final String STRING_TO_TYPE = "He said\"hello\"to me. I replied,\"hi\"." + + "Then, 5\"passed. He said\"bye\"and left."; + final String EXPECTED_RESULT = "He said \"hello\" to me. I replied, \"hi\". " + + "Then, 5\" passed. He said \"bye\" and left. \""; + // Split by double quote, so that we can type the double quotes individually. + for (final String partToType : STRING_TO_TYPE.split("\"")) { + // Split at word boundaries. This regexp means "anywhere that is preceded + // by a word character but not followed by a word character, OR that is not + // preceded by a word character but followed by a word character". + // We need to input word by word because auto-spaces are only active when + // manually picking or gesturing (which we can't simulate yet), but only words + // can be picked. + final String[] wordsToType = partToType.split("(?<=\\w)(?!\\w)|(?<!\\w)(?=\\w)"); + for (final String wordToType : wordsToType) { + type(wordToType); + if (wordToType.matches("^\\w+$")) { + // Only pick selection if that was a word, because if that was not a word, + // then we don't have a composition. + pickSuggestionManually(wordToType); + } + } + type("\""); + } + assertEquals("auto-space with double quotes", EXPECTED_RESULT, mEditText.getText().toString()); } } diff --git a/tests/src/com/android/inputmethod/latin/RichInputConnectionAndTextRangeTests.java b/tests/src/com/android/inputmethod/latin/RichInputConnectionAndTextRangeTests.java index c0dd9933c..199922491 100644 --- a/tests/src/com/android/inputmethod/latin/RichInputConnectionAndTextRangeTests.java +++ b/tests/src/com/android/inputmethod/latin/RichInputConnectionAndTextRangeTests.java @@ -16,15 +16,13 @@ package com.android.inputmethod.latin; -import com.android.inputmethod.latin.utils.TextRange; - +import android.content.res.Resources; import android.inputmethodservice.InputMethodService; import android.os.Parcel; import android.test.AndroidTestCase; import android.test.MoreAsserts; import android.test.suitebuilder.annotation.SmallTest; import android.text.SpannableString; -import android.text.Spanned; import android.text.TextUtils; import android.text.style.SuggestionSpan; import android.view.inputmethod.ExtractedText; @@ -32,6 +30,14 @@ import android.view.inputmethod.ExtractedTextRequest; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputConnectionWrapper; +import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; +import com.android.inputmethod.latin.utils.PrevWordsInfoUtils; +import com.android.inputmethod.latin.utils.RunInLocale; +import com.android.inputmethod.latin.utils.ScriptUtils; +import com.android.inputmethod.latin.utils.StringUtils; +import com.android.inputmethod.latin.utils.TextRange; + import java.util.Locale; @SmallTest @@ -39,11 +45,19 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { // The following is meant to be a reasonable default for // the "word_separators" resource. - private static final String sSeparators = ".,:;!?-"; + private SpacingAndPunctuations mSpacingAndPunctuations; @Override protected void setUp() throws Exception { super.setUp(); + final RunInLocale<SpacingAndPunctuations> job = new RunInLocale<SpacingAndPunctuations>() { + @Override + protected SpacingAndPunctuations job(final Resources res) { + return new SpacingAndPunctuations(res); + } + }; + final Resources res = getContext().getResources(); + mSpacingAndPunctuations = job.runInLocale(res, Locale.ENGLISH); } private class MockConnection extends InputConnectionWrapper { @@ -78,6 +92,10 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { mExtractedText = extractedText; } + public int cursorPos() { + return mTextBefore.length(); + } + /* (non-Javadoc) * @see android.view.inputmethod.InputConnectionWrapper#getTextBeforeCursor(int, int) */ @@ -120,13 +138,16 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { } private class MockInputMethodService extends InputMethodService { - InputConnection mInputConnection; - public void setInputConnection(final InputConnection inputConnection) { - mInputConnection = inputConnection; + private MockConnection mMockConnection; + public void setInputConnection(final MockConnection mockConnection) { + mMockConnection = mockConnection; + } + public int cursorPos() { + return mMockConnection.cursorPos(); } @Override public InputConnection getCurrentInputConnection() { - return mInputConnection; + return mMockConnection; } } @@ -137,9 +158,26 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { */ public void testGetPreviousWord() { // If one of the following cases breaks, the bigram suggestions won't work. - assertEquals(RichInputConnection.getNthPreviousWord("abc def", sSeparators, 2), "abc"); - assertNull(RichInputConnection.getNthPreviousWord("abc", sSeparators, 2)); - assertNull(RichInputConnection.getNthPreviousWord("abc. def", sSeparators, 2)); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def", mSpacingAndPunctuations, 2).mPrevWordsInfo[0].mWord, "abc"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc", mSpacingAndPunctuations, 2), PrevWordsInfo.BEGINNING_OF_SENTENCE); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc. def", mSpacingAndPunctuations, 2), PrevWordsInfo.BEGINNING_OF_SENTENCE); + + assertFalse(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def", mSpacingAndPunctuations, 2).mPrevWordsInfo[0].mIsBeginningOfSentence); + assertTrue(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc", mSpacingAndPunctuations, 2).mPrevWordsInfo[0].mIsBeginningOfSentence); + + // For n-gram + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def", mSpacingAndPunctuations, 1).mPrevWordsInfo[0].mWord, "def"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def", mSpacingAndPunctuations, 1).mPrevWordsInfo[1].mWord, "abc"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def", mSpacingAndPunctuations, 2).mPrevWordsInfo[1], + WordInfo.BEGINNING_OF_SENTENCE); // The following tests reflect the current behavior of the function // RichInputConnection#getNthPreviousWord. @@ -148,20 +186,46 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { // this function if needed - especially since it does not seem very // logical. These tests are just there to catch any unintentional // changes in the behavior of the RichInputConnection#getPreviousWord method. - assertEquals(RichInputConnection.getNthPreviousWord("abc def ", sSeparators, 2), "abc"); - assertEquals(RichInputConnection.getNthPreviousWord("abc def.", sSeparators, 2), "abc"); - assertEquals(RichInputConnection.getNthPreviousWord("abc def .", sSeparators, 2), "def"); - assertNull(RichInputConnection.getNthPreviousWord("abc ", sSeparators, 2)); - - assertEquals(RichInputConnection.getNthPreviousWord("abc def", sSeparators, 1), "def"); - assertEquals(RichInputConnection.getNthPreviousWord("abc def ", sSeparators, 1), "def"); - assertNull(RichInputConnection.getNthPreviousWord("abc def.", sSeparators, 1)); - assertNull(RichInputConnection.getNthPreviousWord("abc def .", sSeparators, 1)); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def ", mSpacingAndPunctuations, 2).mPrevWordsInfo[0].mWord, "abc"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def.", mSpacingAndPunctuations, 2).mPrevWordsInfo[0].mWord, "abc"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def .", mSpacingAndPunctuations, 2).mPrevWordsInfo[0].mWord, "def"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc ", mSpacingAndPunctuations, 2), PrevWordsInfo.BEGINNING_OF_SENTENCE); + + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def", mSpacingAndPunctuations, 1).mPrevWordsInfo[0].mWord, "def"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def ", mSpacingAndPunctuations, 1).mPrevWordsInfo[0].mWord, "def"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc 'def", mSpacingAndPunctuations, 1).mPrevWordsInfo[0].mWord, "'def"); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def.", mSpacingAndPunctuations, 1), PrevWordsInfo.BEGINNING_OF_SENTENCE); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc def .", mSpacingAndPunctuations, 1), PrevWordsInfo.BEGINNING_OF_SENTENCE); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc, def", mSpacingAndPunctuations, 2), PrevWordsInfo.EMPTY_PREV_WORDS_INFO); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc? def", mSpacingAndPunctuations, 2), PrevWordsInfo.EMPTY_PREV_WORDS_INFO); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc! def", mSpacingAndPunctuations, 2), PrevWordsInfo.EMPTY_PREV_WORDS_INFO); + assertEquals(PrevWordsInfoUtils.getPrevWordsInfoFromNthPreviousWord( + "abc 'def", mSpacingAndPunctuations, 2), PrevWordsInfo.EMPTY_PREV_WORDS_INFO); } /** * Test logic in getting the word range at the cursor. */ + private static final int[] SPACE = { Constants.CODE_SPACE }; + static final int[] TAB = { Constants.CODE_TAB }; + private static final int[] SPACE_TAB = StringUtils.toSortedCodePointArray(" \t"); + // A character that needs surrogate pair to represent its code point (U+2008A). + private static final String SUPPLEMENTARY_CHAR = "\uD840\uDC8A"; + private static final String HIRAGANA_WORD = "\u3042\u3044\u3046\u3048\u304A"; // あいうえお + private static final String GREEK_WORD = "\u03BA\u03B1\u03B9"; // και + public void testGetWordRangeAtCursor() { ExtractedText et = new ExtractedText(); final MockInputMethodService mockInputMethodService = new MockInputMethodService(); @@ -173,50 +237,42 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { ic.beginBatchEdit(); // basic case - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); assertTrue(TextUtils.equals("word", r.mWord)); - // more than one word - r = ic.getWordRangeAtCursor(" ", 1); - assertTrue(TextUtils.equals("word word", r.mWord)); - ic.endBatchEdit(); - // tab character instead of space mockInputMethodService.setInputConnection(new MockConnection("one\tword\two", "rd", et)); ic.beginBatchEdit(); - r = ic.getWordRangeAtCursor("\t", 1); + r = ic.getWordRangeAtCursor(TAB, ScriptUtils.SCRIPT_LATIN); ic.endBatchEdit(); - assertTrue(TextUtils.equals("word\tword", r.mWord)); - - // only one word doesn't go too far - mockInputMethodService.setInputConnection(new MockConnection("one\tword\two", "rd", et)); - ic.beginBatchEdit(); - r = ic.getWordRangeAtCursor("\t", 1); - ic.endBatchEdit(); - assertTrue(TextUtils.equals("word\tword", r.mWord)); + assertTrue(TextUtils.equals("word", r.mWord)); - // tab or space - mockInputMethodService.setInputConnection(new MockConnection("one word\two", "rd", et)); + // splitting on supplementary character + mockInputMethodService.setInputConnection( + new MockConnection("one word" + SUPPLEMENTARY_CHAR + "wo", "rd", et)); ic.beginBatchEdit(); - r = ic.getWordRangeAtCursor(" \t", 1); + r = ic.getWordRangeAtCursor(StringUtils.toSortedCodePointArray(SUPPLEMENTARY_CHAR), + ScriptUtils.SCRIPT_LATIN); ic.endBatchEdit(); - assertTrue(TextUtils.equals("word\tword", r.mWord)); + assertTrue(TextUtils.equals("word", r.mWord)); - // tab or space multiword - mockInputMethodService.setInputConnection(new MockConnection("one word\two", "rd", et)); + // split on chars outside the specified script + mockInputMethodService.setInputConnection( + new MockConnection(HIRAGANA_WORD + "wo", "rd" + GREEK_WORD, et)); ic.beginBatchEdit(); - r = ic.getWordRangeAtCursor(" \t", 2); + r = ic.getWordRangeAtCursor(StringUtils.toSortedCodePointArray(SUPPLEMENTARY_CHAR), + ScriptUtils.SCRIPT_LATIN); ic.endBatchEdit(); - assertTrue(TextUtils.equals("one word\tword", r.mWord)); + assertTrue(TextUtils.equals("word", r.mWord)); - // splitting on supplementary character - final String supplementaryChar = "\uD840\uDC8A"; + // likewise for greek mockInputMethodService.setInputConnection( - new MockConnection("one word" + supplementaryChar + "wo", "rd", et)); + new MockConnection("text" + GREEK_WORD, "text", et)); ic.beginBatchEdit(); - r = ic.getWordRangeAtCursor(supplementaryChar, 0); + r = ic.getWordRangeAtCursor(StringUtils.toSortedCodePointArray(SUPPLEMENTARY_CHAR), + ScriptUtils.SCRIPT_GREEK); ic.endBatchEdit(); - assertTrue(TextUtils.equals("word", r.mWord)); + assertTrue(TextUtils.equals(GREEK_WORD, r.mWord)); } /** @@ -244,7 +300,7 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { TextRange r; SuggestionSpan[] suggestions; - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); suggestions = r.getSuggestionSpansAtWord(); assertEquals(suggestions.length, 1); MoreAsserts.assertEquals(suggestions[0].getSuggestions(), SUGGESTIONS1); @@ -256,7 +312,7 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { text.setSpan(new SuggestionSpan(Locale.ENGLISH, SUGGESTIONS2, 0 /* flags */), 10 /* start */, 16 /* end */, 0 /* flags */); mockInputMethodService.setInputConnection(new MockConnection(text, cursorPos)); - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); suggestions = r.getSuggestionSpansAtWord(); assertEquals(suggestions.length, 2); MoreAsserts.assertEquals(suggestions[0].getSuggestions(), SUGGESTIONS1); @@ -269,7 +325,7 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { text.setSpan(new SuggestionSpan(Locale.ENGLISH, SUGGESTIONS2, 0 /* flags */), 5 /* start */, 16 /* end */, 0 /* flags */); mockInputMethodService.setInputConnection(new MockConnection(text, cursorPos)); - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); suggestions = r.getSuggestionSpansAtWord(); assertEquals(suggestions.length, 1); MoreAsserts.assertEquals(suggestions[0].getSuggestions(), SUGGESTIONS1); @@ -281,7 +337,7 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { text.setSpan(new SuggestionSpan(Locale.ENGLISH, SUGGESTIONS2, 0 /* flags */), 10 /* start */, 20 /* end */, 0 /* flags */); mockInputMethodService.setInputConnection(new MockConnection(text, cursorPos)); - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); suggestions = r.getSuggestionSpansAtWord(); assertEquals(suggestions.length, 1); MoreAsserts.assertEquals(suggestions[0].getSuggestions(), SUGGESTIONS1); @@ -293,7 +349,7 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { text.setSpan(new SuggestionSpan(Locale.ENGLISH, SUGGESTIONS2, 0 /* flags */), 5 /* start */, 20 /* end */, 0 /* flags */); mockInputMethodService.setInputConnection(new MockConnection(text, cursorPos)); - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); suggestions = r.getSuggestionSpansAtWord(); assertEquals(suggestions.length, 1); MoreAsserts.assertEquals(suggestions[0].getSuggestions(), SUGGESTIONS1); @@ -305,8 +361,86 @@ public class RichInputConnectionAndTextRangeTests extends AndroidTestCase { text.setSpan(new SuggestionSpan(Locale.ENGLISH, SUGGESTIONS2, 0 /* flags */), 5 /* start */, 20 /* end */, 0 /* flags */); mockInputMethodService.setInputConnection(new MockConnection(text, cursorPos)); - r = ic.getWordRangeAtCursor(" ", 0); + r = ic.getWordRangeAtCursor(SPACE, ScriptUtils.SCRIPT_LATIN); suggestions = r.getSuggestionSpansAtWord(); assertEquals(suggestions.length, 0); } + + public void testCursorTouchingWord() { + final MockInputMethodService ims = new MockInputMethodService(); + final RichInputConnection ic = new RichInputConnection(ims); + final SpacingAndPunctuations sap = mSpacingAndPunctuations; + + ims.setInputConnection(new MockConnection("users", 5)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("users'", 5)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("users'", 6)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("'users'", 6)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("'users'", 7)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("users '", 6)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("users '", 7)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("re-", 3)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("re--", 4)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("-", 1)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection("--", 2)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" -", 2)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" --", 3)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" users '", 1)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" users '", 3)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" users '", 7)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" users are", 7)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertTrue(ic.isCursorTouchingWord(sap)); + + ims.setInputConnection(new MockConnection(" users 'are", 7)); + ic.resetCachesUponCursorMoveAndReturnSuccess(ims.cursorPos(), ims.cursorPos(), true); + assertFalse(ic.isCursorTouchingWord(sap)); + } } diff --git a/tests/src/com/android/inputmethod/latin/ShiftModeTests.java b/tests/src/com/android/inputmethod/latin/ShiftModeTests.java new file mode 100644 index 000000000..db3c9baa9 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/ShiftModeTests.java @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.os.Build; +import android.test.suitebuilder.annotation.LargeTest; +import android.text.TextUtils; +import android.view.inputmethod.EditorInfo; + +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.WordComposer; + +@LargeTest +public class ShiftModeTests extends InputTestsBase { + + @Override + protected EditorInfo enrichEditorInfo(final EditorInfo ei) { + ei.inputType |= TextUtils.CAP_MODE_SENTENCES; + ei.initialCapsMode = TextUtils.CAP_MODE_SENTENCES; + return ei; + } + + private boolean isCapsModeAutoShifted() { + return mLatinIME.mKeyboardSwitcher.getKeyboardShiftMode() + == WordComposer.CAPS_MODE_AUTO_SHIFTED; + } + + public void testTypicalSentence() { + assertTrue("Initial auto caps state", isCapsModeAutoShifted()); + type("Test"); + assertFalse("Caps after letter", isCapsModeAutoShifted()); + type(" "); + assertFalse("Caps after space", isCapsModeAutoShifted()); + type("some,"); + assertFalse("Caps after comma", isCapsModeAutoShifted()); + type(" "); + assertFalse("Caps after comma space", isCapsModeAutoShifted()); + type("words."); + assertFalse("Caps directly after period", isCapsModeAutoShifted()); + type(" "); + assertTrue("Caps after period space", isCapsModeAutoShifted()); + } + + public void testBackspace() { + assertTrue("Initial auto caps state", isCapsModeAutoShifted()); + type("A"); + assertFalse("Caps state after one letter", isCapsModeAutoShifted()); + type(Constants.CODE_DELETE); + assertTrue("Auto caps state at start after delete", isCapsModeAutoShifted()); + } + + public void testRepeatingBackspace() { + final String SENTENCE_TO_TYPE = "Test sentence. Another."; + final int BACKSPACE_COUNT = + SENTENCE_TO_TYPE.length() - SENTENCE_TO_TYPE.lastIndexOf(' ') - 1; + + type(SENTENCE_TO_TYPE); + assertFalse("Caps after typing \"" + SENTENCE_TO_TYPE + "\"", isCapsModeAutoShifted()); + type(Constants.CODE_DELETE); + for (int i = 1; i < BACKSPACE_COUNT; ++i) { + repeatKey(Constants.CODE_DELETE); + } + assertFalse("Caps immediately after repeating Backspace a lot", isCapsModeAutoShifted()); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + assertTrue("Caps after a while after repeating Backspace a lot", isCapsModeAutoShifted()); + } + + public void testAutoCapsAfterDigitsPeriod() { + changeLanguage("en"); + type("On 22.11."); + assertFalse("(English) Auto caps after digits-period", isCapsModeAutoShifted()); + type(" "); + assertTrue("(English) Auto caps after digits-period-whitespace", isCapsModeAutoShifted()); + mEditText.setText(""); + changeLanguage("fr"); + type("Le 22."); + assertFalse("(French) Auto caps after digits-period", isCapsModeAutoShifted()); + type(" "); + assertTrue("(French) Auto caps after digits-period-whitespace", isCapsModeAutoShifted()); + mEditText.setText(""); + changeLanguage("de"); + type("Am 22."); + assertFalse("(German) Auto caps after digits-period", isCapsModeAutoShifted()); + type(" "); + // For German, no auto-caps in this case + assertFalse("(German) Auto caps after digits-period-whitespace", isCapsModeAutoShifted()); + } + + public void testAutoCapsAfterInvertedMarks() { + changeLanguage("es"); + assertTrue("(Spanish) Auto caps at start", isCapsModeAutoShifted()); + type("Hey. ¿"); + assertTrue("(Spanish) Auto caps after inverted what", isCapsModeAutoShifted()); + mEditText.setText(""); + type("¡"); + assertTrue("(Spanish) Auto caps after inverted bang", isCapsModeAutoShifted()); + } + + public void testOtherSentenceSeparators() { + changeLanguage("hy_AM"); + assertTrue("(Armenian) Auto caps at start", isCapsModeAutoShifted()); + type("Hey. "); + assertFalse("(Armenian) No auto-caps after latin period", isCapsModeAutoShifted()); + type("Hey\u0589"); + assertFalse("(Armenian) No auto-caps directly after armenian period", + isCapsModeAutoShifted()); + type(" "); + assertTrue("(Armenian) Auto-caps after armenian period-whitespace", + isCapsModeAutoShifted()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java index 375352067..66b4a9c71 100644 --- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java +++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java @@ -16,12 +16,10 @@ package com.android.inputmethod.latin; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; - import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; -import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.util.ArrayList; import java.util.Locale; @@ -33,7 +31,7 @@ public class SuggestedWordsTests extends AndroidTestCase { final String TYPED_WORD = "typed"; final int TYPED_WORD_FREQ = 5; final int NUMBER_OF_ADDED_SUGGESTIONS = 5; - final ArrayList<SuggestedWordInfo> list = CollectionUtils.newArrayList(); + final ArrayList<SuggestedWordInfo> list = new ArrayList<>(); list.add(new SuggestedWordInfo(TYPED_WORD, TYPED_WORD_FREQ, SuggestedWordInfo.KIND_TYPED, null /* sourceDict */, SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, @@ -46,24 +44,23 @@ public class SuggestedWordsTests extends AndroidTestCase { } final SuggestedWords words = new SuggestedWords( - list, + list, null /* rawSuggestions */, false /* typedWordValid */, false /* willAutoCorrect */, - false /* isPunctuationSuggestions */, false /* isObsoleteSuggestions */, false /* isPrediction*/); assertEquals(NUMBER_OF_ADDED_SUGGESTIONS + 1, words.size()); assertEquals("typed", words.getWord(0)); - assertEquals(SuggestedWordInfo.KIND_TYPED, words.getInfo(0).mKind); + assertTrue(words.getInfo(0).isKindOf(SuggestedWordInfo.KIND_TYPED)); assertEquals("0", words.getWord(1)); - assertEquals(SuggestedWordInfo.KIND_CORRECTION, words.getInfo(1).mKind); + assertTrue(words.getInfo(1).isKindOf(SuggestedWordInfo.KIND_CORRECTION)); assertEquals("4", words.getWord(5)); - assertEquals(SuggestedWordInfo.KIND_CORRECTION, words.getInfo(5).mKind); + assertTrue(words.getInfo(5).isKindOf(SuggestedWordInfo.KIND_CORRECTION)); final SuggestedWords wordsWithoutTyped = words.getSuggestedWordsExcludingTypedWord(); assertEquals(words.size() - 1, wordsWithoutTyped.size()); assertEquals("0", wordsWithoutTyped.getWord(0)); - assertEquals(SuggestedWordInfo.KIND_CORRECTION, wordsWithoutTyped.getInfo(0).mKind); + assertTrue(wordsWithoutTyped.getInfo(0).isKindOf(SuggestedWordInfo.KIND_CORRECTION)); } // Helper for testGetTransformedWordInfo diff --git a/tests/src/com/android/inputmethod/latin/WordComposerTests.java b/tests/src/com/android/inputmethod/latin/WordComposerTests.java index 1434c6b63..c44544f3d 100644 --- a/tests/src/com/android/inputmethod/latin/WordComposerTests.java +++ b/tests/src/com/android/inputmethod/latin/WordComposerTests.java @@ -19,6 +19,9 @@ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import com.android.inputmethod.latin.utils.CoordinateUtils; +import com.android.inputmethod.latin.utils.StringUtils; + /** * Unit tests for WordComposer. */ @@ -26,10 +29,19 @@ import android.test.suitebuilder.annotation.SmallTest; public class WordComposerTests extends AndroidTestCase { public void testMoveCursor() { final WordComposer wc = new WordComposer(); + // BMP is the Basic Multilingual Plane, as defined by Unicode. This includes + // most characters for most scripts, including all Roman alphabet languages, + // CJK, Arabic, Hebrew. Notable exceptions include some emoji and some + // very rare Chinese ideograms. BMP characters can be encoded on 2 bytes + // in UTF-16, whereas those outside the BMP need 4 bytes. + // http://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane final String STR_WITHIN_BMP = "abcdef"; - wc.setComposingWord(STR_WITHIN_BMP, null); - assertEquals(wc.size(), - STR_WITHIN_BMP.codePointCount(0, STR_WITHIN_BMP.length())); + final int[] CODEPOINTS_WITHIN_BMP = StringUtils.toCodePointArray(STR_WITHIN_BMP); + final int[] COORDINATES_WITHIN_BMP = + CoordinateUtils.newCoordinateArray(CODEPOINTS_WITHIN_BMP.length, + Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); + wc.setComposingWord(CODEPOINTS_WITHIN_BMP, COORDINATES_WITHIN_BMP); + assertEquals(wc.size(), STR_WITHIN_BMP.codePointCount(0, STR_WITHIN_BMP.length())); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); wc.setCursorPositionWithinWord(2); assertTrue(wc.isCursorFrontOrMiddleOfComposingWord()); @@ -46,12 +58,20 @@ public class WordComposerTests extends AndroidTestCase { // Move the cursor past the end of the word assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(15)); + // Do what LatinIME does when the cursor is moved outside of the word, + // and check the behavior is correct. + wc.reset(); // \uD861\uDED7 is 𨛗, a character outside the BMP final String STR_WITH_SUPPLEMENTARY_CHAR = "abcde\uD861\uDED7fgh"; - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); - assertEquals(wc.size(), STR_WITH_SUPPLEMENTARY_CHAR.codePointCount(0, - STR_WITH_SUPPLEMENTARY_CHAR.length())); + final int[] CODEPOINTS_WITH_SUPPLEMENTARY_CHAR = + StringUtils.toCodePointArray(STR_WITH_SUPPLEMENTARY_CHAR); + final int[] COORDINATES_WITH_SUPPLEMENTARY_CHAR = + CoordinateUtils.newCoordinateArray(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length, + Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); + assertEquals(wc.size(), CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); wc.setCursorPositionWithinWord(3); assertTrue(wc.isCursorFrontOrMiddleOfComposingWord()); @@ -60,33 +80,42 @@ public class WordComposerTests extends AndroidTestCase { assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-3)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-1)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); wc.setCursorPositionWithinWord(3); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-9)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-10)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-11)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0)); - wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null); + wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, + COORDINATES_WITH_SUPPLEMENTARY_CHAR); wc.setCursorPositionWithinWord(2); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0)); } diff --git a/tests/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java new file mode 100644 index 000000000..bc856f113 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.TreeMap; + +/** + * A base class of the binary dictionary decoder. + */ +public abstract class AbstractDictDecoder implements DictDecoder { + private static final int SUCCESS = 0; + private static final int ERROR_CANNOT_READ = 1; + private static final int ERROR_WRONG_FORMAT = 2; + + @Override @UsedForTesting + public int getTerminalPosition(final String word) + throws IOException, UnsupportedFormatException { + if (!isDictBufferOpen()) { + openDictBuffer(); + } + return BinaryDictIOUtils.getTerminalPosition(this, word); + } + + @Override @UsedForTesting + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException { + if (!isDictBufferOpen()) { + openDictBuffer(); + } + BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); + } + + /** + * Check whether the header contains the expected information. This is a no-error method, + * that will return an error code and never throw a checked exception. + * @return an error code, either ERROR_* or SUCCESS. + */ + private int checkHeader() { + try { + readHeader(); + } catch (IOException e) { + return ERROR_CANNOT_READ; + } catch (UnsupportedFormatException e) { + return ERROR_WRONG_FORMAT; + } + return SUCCESS; + } + + @Override + public boolean hasValidRawBinaryDictionary() { + return checkHeader() == SUCCESS; + } + + // Placeholder implementations below. These are actually unused. + @Override + public void openDictBuffer() throws FileNotFoundException, IOException, + UnsupportedFormatException { + } + + @Override + public boolean isDictBufferOpen() { + return false; + } + + @Override + public PtNodeInfo readPtNode(final int ptNodePos) { + return null; + } + + @Override + public void setPosition(int newPos) { + } + + @Override + public int getPosition() { + return 0; + } + + @Override + public int readPtNodeCount() { + return 0; + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 32c07e106..406046a74 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -17,30 +17,28 @@ package com.android.inputmethod.latin.makedict; import android.test.AndroidTestCase; -import android.test.MoreAsserts; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; +import android.util.Pair; import android.util.SparseArray; +import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; -import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map.Entry; import java.util.Random; import java.util.Set; @@ -52,39 +50,21 @@ import java.util.TreeMap; @LargeTest public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private static final String TAG = BinaryDictDecoderEncoderTests.class.getSimpleName(); - private static final int DEFAULT_MAX_UNIGRAMS = 100; + private static final int DEFAULT_MAX_UNIGRAMS = 300; private static final int DEFAULT_CODE_POINT_SET_SIZE = 50; + private static final int LARGE_CODE_POINT_SET_SIZE = 300; private static final int UNIGRAM_FREQ = 10; private static final int BIGRAM_FREQ = 50; private static final int TOLERANCE_OF_BIGRAM_FREQ = 5; private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50; private static final int NUM_OF_SHORTCUTS = 5; - private static final int USE_BYTE_ARRAY = 1; - private static final int USE_BYTE_BUFFER = 2; - - private static final ArrayList<String> sWords = CollectionUtils.newArrayList(); - private static final SparseArray<List<Integer>> sEmptyBigrams = - CollectionUtils.newSparseArray(); - private static final SparseArray<List<Integer>> sStarBigrams = CollectionUtils.newSparseArray(); - private static final SparseArray<List<Integer>> sChainBigrams = - CollectionUtils.newSparseArray(); - private static final HashMap<String, List<String>> sShortcuts = CollectionUtils.newHashMap(); - - private static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); - private static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = - new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */, - true /* hasTimestamp */); - - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final ArrayList<String> sWords = new ArrayList<>(); + private static final ArrayList<String> sWordsWithVariousCodePoints = new ArrayList<>(); + private static final SparseArray<List<Integer>> sEmptyBigrams = new SparseArray<>(); + private static final SparseArray<List<Integer>> sStarBigrams = new SparseArray<>(); + private static final SparseArray<List<Integer>> sChainBigrams = new SparseArray<>(); + private static final HashMap<String, List<String>> sShortcuts = new HashMap<>(); public BinaryDictDecoderEncoderTests() { this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); @@ -92,12 +72,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) { super(); + BinaryDictionaryUtils.setCurrentTimeForTest(0); Log.e(TAG, "Testing dictionary: seed is " + seed); final Random random = new Random(seed); sWords.clear(); - final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, - random); - generateWords(maxUnigrams, random, codePointSet); + sWordsWithVariousCodePoints.clear(); + generateWords(maxUnigrams, random); for (int i = 0; i < sWords.size(); ++i) { sChainBigrams.put(i, new ArrayList<Integer>()); @@ -124,23 +104,35 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } - private DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - return new Ver4DictEncoder(getContext().getCacheDir()); - } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { - return new Ver3DictEncoder(file); - } else { - throw new RuntimeException("The format option has a wrong version : " - + formatOptions.mVersion); - } + @Override + protected void setUp() throws Exception { + super.setUp(); + BinaryDictionaryUtils.setCurrentTimeForTest(0); } - private void generateWords(final int number, final Random random, final int[] codePointSet) { - final Set<String> wordSet = CollectionUtils.newHashSet(); + @Override + protected void tearDown() throws Exception { + // Quit test mode. + BinaryDictionaryUtils.setCurrentTimeForTest(-1); + super.tearDown(); + } + + private void generateWords(final int number, final Random random) { + final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, + random); + final Set<String> wordSet = new HashSet<>(); while (wordSet.size() < number) { wordSet.add(CodePointUtils.generateWord(random, codePointSet)); } sWords.addAll(wordSet); + + final int[] largeCodePointSet = CodePointUtils.generateCodePointSet( + LARGE_CODE_POINT_SET_SIZE, random); + wordSet.clear(); + while (wordSet.size() < number) { + wordSet.add(CodePointUtils.generateWord(random, largeCodePointSet)); + } + sWordsWithVariousCodePoints.addAll(wordSet); } /** @@ -150,14 +142,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final List<String> words, final HashMap<String, List<String>> shortcutMap) { for (int i = 0; i < number; ++i) { final String word = words.get(i); - final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList(); + final ArrayList<WeightedString> shortcuts = new ArrayList<>(); if (shortcutMap != null && shortcutMap.containsKey(word)) { for (final String shortcut : shortcutMap.get(word)) { shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ)); } } - dict.add(word, UNIGRAM_FREQ, (shortcutMap == null) ? null : shortcuts, - false /* isNotAWord */); + dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), + (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */); } } @@ -167,7 +159,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { for (int i = 0; i < bigrams.size(); ++i) { final int w1 = bigrams.keyAt(i); for (int w2 : bigrams.valueAt(i)) { - dict.setBigram(words.get(w1), words.get(w2), BIGRAM_FREQ); + dict.setBigram(words.get(w1), words.get(w2), new ProbabilityInfo(BIGRAM_FREQ)); } } } @@ -186,7 +178,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictEncoder dictEncoder = getDictEncoder(file, formatOptions); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions); now = System.currentTimeMillis(); // If you need to dump the dict to a textual file, uncomment the line below and the @@ -241,56 +233,23 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private String outputOptions(final int bufferType, final FormatSpec.FormatOptions formatOptions) { String result = " : buffer type = " - + ((bufferType == USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); - result += " : version = " + formatOptions.mVersion; - return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; + + ((bufferType == BinaryDictUtils.USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); + return result + " : version = " + formatOptions.mVersion; } - private DictionaryOptions getDictionaryOptions(final String id, final String version) { - final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), - false, false); - options.mAttributes.put("version", version); - options.mAttributes.put("dictionary", id); - return options; - } - - private File setUpDictionaryFile(final String name, final String version) { - File file = null; - try { - file = new File(getContext().getCacheDir(), name + "." + version - + TEST_DICT_FILE_EXTENSION); - file.createNewFile(); - } catch (IOException e) { - // do nothing - } - assertTrue("Failed to create the dictionary file.", file.exists()); - return file; - } - - private DictDecoder getDictDecoder(final File file, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - final FileHeader header = new FileHeader(0, dictOptions, formatOptions); - return FormatSpec.getDictDecoder(new File(getContext().getCacheDir(), - header.getId() + "." + header.getVersion()), bufferType); - } else { - return FormatSpec.getDictDecoder(file, bufferType); - } - } // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List<String> words, final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + final HashMap<String, List<String>> shortcutMap, final int bufferType) { long now, diff = -1; FusionDictionary dict = null; try { - final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, - dictOptions); + final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), + bufferType); now = System.currentTimeMillis(); - dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); + dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException while reading dictionary", e); @@ -310,17 +269,17 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final String dictName = "runReadAndWrite"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); checkDictionary(dict, words, bigrams, shortcuts); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType, - formatOptions, dict.mOptions); + final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -340,6 +299,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { "chain with shortcuts")); results.add(runReadAndWrite(sWords, sStarBigrams, sShortcuts, bufferType, formatOptions, "star with shortcuts")); + results.add(runReadAndWrite(sWordsWithVariousCodePoints, sEmptyBigrams, + null /* shortcuts */, bufferType, formatOptions, + "unigram with various code points")); } // Unit test for CharEncoding.readString and CharEncoding.writeString. @@ -349,8 +311,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final byte[] buffer = new byte[50 * 3]; final DictBuffer dictBuffer = new ByteArrayDictBuffer(buffer); for (final String word : sWords) { - Log.d("testReadAndWriteString", "write : " + word); - Arrays.fill(buffer, (byte)0); + Arrays.fill(buffer, (byte) 0); CharEncoding.writeString(buffer, 0, word); dictBuffer.position(0); final String str = CharEncoding.readString(dictBuffer); @@ -359,29 +320,28 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testReadAndWriteWithByteBuffer() { - final List<String> results = CollectionUtils.newArrayList(); - - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); - + final List<String> results = new ArrayList<>(); + + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2_OPTIONS); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); } } public void testReadAndWriteWithByteArray() { - final List<String> results = CollectionUtils.newArrayList(); + final List<String> results = new ArrayList<>(); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2_OPTIONS); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -394,62 +354,62 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final SparseArray<List<Integer>> expectedBigrams, final TreeMap<Integer, String> resultWords, final TreeMap<Integer, Integer> resultFrequencies, - final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams) { + final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams, + final boolean checkProbability) { // check unigrams - final Set<String> actualWordsSet = new HashSet<String>(resultWords.values()); - final Set<String> expectedWordsSet = new HashSet<String>(expectedWords); + final Set<String> actualWordsSet = new HashSet<>(resultWords.values()); + final Set<String> expectedWordsSet = new HashSet<>(expectedWords); assertEquals(actualWordsSet, expectedWordsSet); - - for (int freq : resultFrequencies.values()) { - assertEquals(freq, UNIGRAM_FREQ); + if (checkProbability) { + for (int freq : resultFrequencies.values()) { + assertEquals(freq, UNIGRAM_FREQ); + } } // check bigrams - final HashMap<String, List<String>> expBigrams = new HashMap<String, List<String>>(); + final HashMap<String, Set<String>> expBigrams = new HashMap<>(); for (int i = 0; i < expectedBigrams.size(); ++i) { final String word1 = expectedWords.get(expectedBigrams.keyAt(i)); for (int w2 : expectedBigrams.valueAt(i)) { if (expBigrams.get(word1) == null) { - expBigrams.put(word1, new ArrayList<String>()); + expBigrams.put(word1, new HashSet<String>()); } expBigrams.get(word1).add(expectedWords.get(w2)); } } - final HashMap<String, List<String>> actBigrams = new HashMap<String, List<String>>(); + final HashMap<String, Set<String>> actBigrams = new HashMap<>(); for (Entry<Integer, ArrayList<PendingAttribute>> entry : resultBigrams.entrySet()) { final String word1 = resultWords.get(entry.getKey()); final int unigramFreq = resultFrequencies.get(entry.getKey()); for (PendingAttribute attr : entry.getValue()) { final String word2 = resultWords.get(attr.mAddress); if (actBigrams.get(word1) == null) { - actBigrams.put(word1, new ArrayList<String>()); + actBigrams.put(word1, new HashSet<String>()); } actBigrams.get(word1).add(word2); - final int bigramFreq = BinaryDictIOUtils.reconstructBigramFrequency( - unigramFreq, attr.mFrequency); - assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); + if (checkProbability) { + final int bigramFreq = BinaryDictIOUtils.reconstructBigramFrequency( + unigramFreq, attr.mFrequency); + assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); + } } } - assertEquals(actBigrams, expBigrams); } private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words, final SparseArray<List<Integer>> bigrams, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { - FileInputStream inStream = null; - - final TreeMap<Integer, String> resultWords = CollectionUtils.newTreeMap(); - final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams = - CollectionUtils.newTreeMap(); - final TreeMap<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap(); + final boolean checkProbability) { + final TreeMap<Integer, String> resultWords = new TreeMap<>(); + final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams = new TreeMap<>(); + final TreeMap<Integer, Integer> resultFreqs = new TreeMap<>(); long now = -1, diff = -1; try { - final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, - dictOptions); + final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), + bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -457,17 +417,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { Log.e(TAG, "IOException", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "UnsupportedFormatException", e); - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } } - checkWordMap(words, bigrams, resultWords, resultFreqs, resultBigrams); + checkWordMap(words, bigrams, resultWords, resultFreqs, resultBigrams, checkProbability); return diff; } @@ -476,20 +428,24 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatSpec.FormatOptions formatOptions, final String message) { final String dictName = "runReadUnigrams"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); + // Caveat: Currently, the Java code to read a v4 dictionary doesn't calculate the + // probability when there's a timestamp for the entry. + // TODO: Abandon the Java code, and implement the v4 dictionary reading code in native. long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, - formatOptions, dict.mOptions); + !formatOptions.mHasTimestamp /* checkProbability */); long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, - bufferType, formatOptions, dict.mOptions); + bufferType); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap + " : " + message + " : " + outputOptions(bufferType, formatOptions); @@ -506,15 +462,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testReadUnigramsAndBigramsBinaryWithByteBuffer() { - final ArrayList<String> results = CollectionUtils.newArrayList(); + final ArrayList<String> results = new ArrayList<>(); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, - VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2_OPTIONS); for (final String result : results) { Log.d(TAG, result); @@ -522,15 +473,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testReadUnigramsAndBigramsBinaryWithByteArray() { - final ArrayList<String> results = CollectionUtils.newArrayList(); + final ArrayList<String> results = new ArrayList<>(); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, - VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2_OPTIONS); for (final String result : results) { Log.d(TAG, result); @@ -541,7 +487,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private String getWordFromBinary(final DictDecoder dictDecoder, final int address) { if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0); - FileHeader fileHeader = null; + DictionaryHeader fileHeader = null; try { fileHeader = dictDecoder.readHeader(); } catch (IOException e) { @@ -550,8 +496,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return null; } if (fileHeader == null) return null; - return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize, - address, fileHeader.mFormatOptions).mWord; + return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset, + address).mWord; } private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word, @@ -578,20 +524,22 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatOptions formatOptions, final String message) { final String dictName = "testGetTerminalPosition"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - final DictDecoder dictDecoder = getDictDecoder(file, DictDecoder.USE_BYTEARRAY, - formatOptions, dict.mOptions); + final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), + DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { - // ignore + Log.e(TAG, "IOException while opening the buffer", e); + } catch (UnsupportedFormatException e) { Log.e(TAG, "IOException while opening the buffer", e); } assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); @@ -636,67 +584,113 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testGetTerminalPosition() { - final ArrayList<String> results = CollectionUtils.newArrayList(); - - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION2); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); - - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION2); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + final ArrayList<String> results = new ArrayList<>(); + + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2_OPTIONS); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2_OPTIONS); for (final String result : results) { Log.d(TAG, result); } } - private void runTestDeleteWord(final FormatOptions formatOptions) { - final String dictName = "testDeleteWord"; + public void testVer2DictGetWordProperty() { + final FormatOptions formatOptions = BinaryDictUtils.VERSION2_OPTIONS; + final ArrayList<String> words = sWords; + final HashMap<String, List<String>> shortcuts = sShortcuts; + final String dictName = "testGetWordProperty"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); - addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); + BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); + addUnigrams(words.size(), dict, words, shortcuts); + addBigrams(dict, words, sEmptyBigrams); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); + file.delete(); timeWritingDictToFile(file, dict, formatOptions); - - final DictUpdater dictUpdater; - if (formatOptions.mVersion == 3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else if (formatOptions.mVersion == 4) { - dictUpdater = new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatOptions.mVersion - + " doesn't exist."); - } - - try { - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(0))); - dictUpdater.deleteWord(sWords.get(0)); - assertEquals(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(0))); - - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(5))); - dictUpdater.deleteWord(sWords.get(5)); - assertEquals(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(5))); - } catch (IOException e) { - } catch (UnsupportedFormatException e) { + final BinaryDictionary binaryDictionary = new BinaryDictionary(file.getAbsolutePath(), + 0 /* offset */, file.length(), true /* useFullEditDistance */, + Locale.ENGLISH, dictName, false /* isUpdatable */); + for (final String word : words) { + final WordProperty wordProperty = binaryDictionary.getWordProperty(word, + false /* isBeginningOfSentence */); + assertEquals(word, wordProperty.mWord); + assertEquals(UNIGRAM_FREQ, wordProperty.getProbability()); + if (shortcuts.containsKey(word)) { + assertEquals(shortcuts.get(word).size(), wordProperty.mShortcutTargets.size()); + final List<String> shortcutList = shortcuts.get(word); + assertTrue(wordProperty.mHasShortcuts); + for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + assertTrue(shortcutList.contains(shortcutTarget.mWord)); + assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability()); + shortcutList.remove(shortcutTarget.mWord); + } + assertTrue(shortcutList.isEmpty()); + } } } - public void testDeleteWord() { - runTestDeleteWord(VERSION3_WITH_DYNAMIC_UPDATE); - runTestDeleteWord(VERSION4_WITH_DYNAMIC_UPDATE); + public void testVer2DictIteration() { + final FormatOptions formatOptions = BinaryDictUtils.VERSION2_OPTIONS; + final ArrayList<String> words = sWords; + final HashMap<String, List<String>> shortcuts = sShortcuts; + final SparseArray<List<Integer>> bigrams = sEmptyBigrams; + final String dictName = "testGetWordProperty"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); + addUnigrams(words.size(), dict, words, shortcuts); + addBigrams(dict, words, bigrams); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); + timeWritingDictToFile(file, dict, formatOptions); + Log.d(TAG, file.getAbsolutePath()); + final BinaryDictionary binaryDictionary = new BinaryDictionary(file.getAbsolutePath(), + 0 /* offset */, file.length(), true /* useFullEditDistance */, + Locale.ENGLISH, dictName, false /* isUpdatable */); + + final HashSet<String> wordSet = new HashSet<>(words); + final HashSet<Pair<String, String>> bigramSet = new HashSet<>(); + + for (int i = 0; i < words.size(); i++) { + final List<Integer> bigramList = bigrams.get(i); + if (bigramList != null) { + for (final Integer word1Index : bigramList) { + final String word1 = words.get(word1Index); + bigramSet.add(new Pair<>(words.get(i), word1)); + } + } + } + int token = 0; + do { + final BinaryDictionary.GetNextWordPropertyResult result = + binaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + final String word0 = wordProperty.mWord; + assertEquals(UNIGRAM_FREQ, wordProperty.mProbabilityInfo.mProbability); + wordSet.remove(word0); + if (shortcuts.containsKey(word0)) { + assertEquals(shortcuts.get(word0).size(), wordProperty.mShortcutTargets.size()); + final List<String> shortcutList = shortcuts.get(word0); + assertNotNull(wordProperty.mShortcutTargets); + for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + assertTrue(shortcutList.contains(shortcutTarget.mWord)); + assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability()); + shortcutList.remove(shortcutTarget.mWord); + } + assertTrue(shortcutList.isEmpty()); + } + for (int j = 0; j < wordProperty.mBigrams.size(); j++) { + final String word1 = wordProperty.mBigrams.get(j).mWord; + final Pair<String, String> bigram = new Pair<>(word0, word1); + assertTrue(bigramSet.contains(bigram)); + bigramSet.remove(bigram); + } + token = result.mNextToken; + } while (token != 0); + assertTrue(wordSet.isEmpty()); + assertTrue(bigramSet.isEmpty()); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java new file mode 100644 index 000000000..96604a197 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -0,0 +1,361 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +/** + * Decodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + * + * TODO: Move this file to makedict/internal. + * TODO: Rename this class to DictDecoderUtils. + */ +public final class BinaryDictDecoderUtils { + private BinaryDictDecoderUtils() { + // This utility class is not publicly instantiable. + } + + @UsedForTesting + public interface DictBuffer { + public int readUnsignedByte(); + public int readUnsignedShort(); + public int readUnsignedInt24(); + public int readInt(); + public int position(); + public void position(int newPosition); + @UsedForTesting + public void put(final byte b); + public int limit(); + @UsedForTesting + public int capacity(); + } + + public static final class ByteBufferDictBuffer implements DictBuffer { + private ByteBuffer mBuffer; + + public ByteBufferDictBuffer(final ByteBuffer buffer) { + mBuffer = buffer; + } + + @Override + public int readUnsignedByte() { + return mBuffer.get() & 0xFF; + } + + @Override + public int readUnsignedShort() { + return mBuffer.getShort() & 0xFFFF; + } + + @Override + public int readUnsignedInt24() { + final int retval = readUnsignedByte(); + return (retval << 16) + readUnsignedShort(); + } + + @Override + public int readInt() { + return mBuffer.getInt(); + } + + @Override + public int position() { + return mBuffer.position(); + } + + @Override + public void position(int newPos) { + mBuffer.position(newPos); + } + + @Override + public void put(final byte b) { + mBuffer.put(b); + } + + @Override + public int limit() { + return mBuffer.limit(); + } + + @Override + public int capacity() { + return mBuffer.capacity(); + } + } + + /** + * A class grouping utility function for our specific character encoding. + */ + static final class CharEncoding { + private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; + private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; + + /** + * Helper method to find out whether this code fits on one byte + */ + private static boolean fitsOnOneByte(final int character) { + return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE + && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; + } + + /** + * Compute the size of a character given its character code. + * + * Char format is: + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + * + * @param character the character code. + * @return the size in binary encoded-form, either 1 or 3 bytes. + */ + static int getCharSize(final int character) { + // See char encoding in FusionDictionary.java + if (fitsOnOneByte(character)) return 1; + if (FormatSpec.INVALID_CHARACTER == character) return 1; + return 3; + } + + /** + * Compute the byte size of a character array. + */ + static int getCharArraySize(final int[] chars) { + int size = 0; + for (int character : chars) size += getCharSize(character); + return size; + } + + /** + * Writes a char array to a byte buffer. + * + * @param codePoints the code point array to write. + * @param buffer the byte buffer to write to. + * @param index the index in buffer to write the character array to. + * @return the index after the last character. + */ + static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { + for (int codePoint : codePoints) { + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + return index; + } + + /** + * Writes a string with our character format to a byte buffer. + * + * This will also write the terminator byte. + * + * @param buffer the byte buffer to write to. + * @param origin the offset to write from. + * @param word the string to write. + * @return the size written, in bytes. + */ + static int writeString(final byte[] buffer, final int origin, final String word) { + final int length = word.length(); + int index = origin; + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + return index - origin; + } + + /** + * Writes a string with our character format to an OutputStream. + * + * This will also write the terminator byte. + * + * @param stream the OutputStream to write to. + * @param word the string to write. + * @return the size written, in bytes. + */ + static int writeString(final OutputStream stream, final String word) throws IOException { + final int length = word.length(); + int written = 0; + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + final int charSize = getCharSize(codePoint); + if (1 == charSize) { + stream.write((byte) codePoint); + } else { + stream.write((byte) (0xFF & (codePoint >> 16))); + stream.write((byte) (0xFF & (codePoint >> 8))); + stream.write((byte) (0xFF & codePoint)); + } + written += charSize; + } + stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + written += FormatSpec.PTNODE_TERMINATOR_SIZE; + return written; + } + + /** + * Reads a string from a DictBuffer. This is the converse of the above method. + */ + static String readString(final DictBuffer dictBuffer) { + final StringBuilder s = new StringBuilder(); + int character = readChar(dictBuffer); + while (character != FormatSpec.INVALID_CHARACTER) { + s.appendCodePoint(character); + character = readChar(dictBuffer); + } + return s.toString(); + } + + /** + * Reads a character from the buffer. + * + * This follows the character format documented earlier in this source file. + * + * @param dictBuffer the buffer, positioned over an encoded character. + * @return the character code. + */ + static int readChar(final DictBuffer dictBuffer) { + int character = dictBuffer.readUnsignedByte(); + if (!fitsOnOneByte(character)) { + if (FormatSpec.PTNODE_CHARACTERS_TERMINATOR == character) { + return FormatSpec.INVALID_CHARACTER; + } + character <<= 16; + character += dictBuffer.readUnsignedShort(); + } + return character; + } + } + + /** + * Reads and returns the PtNode count out of a buffer and forwards the pointer. + */ + /* package */ static int readPtNodeCount(final DictBuffer dictBuffer) { + final int msb = dictBuffer.readUnsignedByte(); + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= msb) { + return msb; + } else { + return ((FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT & msb) << 8) + + dictBuffer.readUnsignedByte(); + } + } + + /** + * Finds, as a string, the word at the position passed as an argument. + * + * @param dictDecoder the dict decoder. + * @param headerSize the size of the header. + * @param pos the position to seek. + * @return the word with its frequency, as a weighted string. + */ + @UsedForTesting + /* package for tests */ static WeightedString getWordAtPosition(final DictDecoder dictDecoder, + final int headerSize, final int pos) { + final WeightedString result; + final int originalPos = dictDecoder.getPosition(); + dictDecoder.setPosition(pos); + result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos); + dictDecoder.setPosition(originalPos); + return result; + } + + private static WeightedString getWordAtPositionWithoutParentAddress( + final DictDecoder dictDecoder, final int headerSize, final int pos) { + dictDecoder.setPosition(headerSize); + final int count = dictDecoder.readPtNodeCount(); + int groupPos = dictDecoder.getPosition(); + final StringBuilder builder = new StringBuilder(); + WeightedString result = null; + + PtNodeInfo last = null; + for (int i = count - 1; i >= 0; --i) { + PtNodeInfo info = dictDecoder.readPtNode(groupPos); + groupPos = info.mEndAddress; + if (info.mOriginalAddress == pos) { + builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); + result = new WeightedString(builder.toString(), info.mProbabilityInfo); + break; // and return + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + if (info.mChildrenAddress > pos) { + if (null == last) continue; + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + dictDecoder.setPosition(last.mChildrenAddress); + i = dictDecoder.readPtNodeCount(); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); + last = null; + continue; + } + last = info; + } + if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + dictDecoder.setPosition(last.mChildrenAddress); + i = dictDecoder.readPtNodeCount(); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); + last = null; + continue; + } + } + return result; + } + + /** + * Helper method to pass a file name instead of a File object to isBinaryDictionary. + */ + public static boolean isBinaryDictionary(final String filename) { + final File file = new File(filename); + return isBinaryDictionary(file); + } + + /** + * Basic test to find out whether the file is a binary dictionary or not. + * + * @param file The file to test. + * @return true if it's a binary dictionary, false otherwise + */ + public static boolean isBinaryDictionary(final File file) { + final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length()); + if (dictDecoder == null) { + return false; + } + return dictDecoder.hasValidRawBinaryDictionary(); + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java new file mode 100644 index 000000000..084371944 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -0,0 +1,881 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; + +/** + * Encodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + * + * TODO: Rename this class to DictEncoderUtils. + */ +public class BinaryDictEncoderUtils { + + private static final boolean DBG = MakedictLog.DBG; + + private BinaryDictEncoderUtils() { + // This utility class is not publicly instantiable. + } + + // Arbitrary limit to how much passes we consider address size compression should + // terminate in. At the time of this writing, our largest dictionary completes + // compression in five passes. + // If the number of passes exceeds this number, makedict bails with an exception on + // suspicion that a bug might be causing an infinite loop. + private static final int MAX_PASSES = 24; + + /** + * Compute the binary size of the character array. + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param characters the character array + * @return the size of the char array, including the terminator if any + */ + static int getPtNodeCharactersSize(final int[] characters) { + int size = CharEncoding.getCharArraySize(characters); + if (characters.length > 1) size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the binary size of the character array in a PtNode + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param ptNode the PtNode + * @return the size of the char array, including the terminator if any + */ + private static int getPtNodeCharactersSize(final PtNode ptNode) { + return getPtNodeCharactersSize(ptNode.mChars); + } + + /** + * Compute the binary size of the PtNode count for a node array. + * @param nodeArray the nodeArray + * @return the size of the PtNode count, either 1 or 2 bytes. + */ + private static int getPtNodeCountSize(final PtNodeArray nodeArray) { + return BinaryDictIOUtils.getPtNodeCountSize(nodeArray.mData.size()); + } + + /** + * Compute the size of a shortcut in bytes. + */ + private static int getShortcutSize(final WeightedString shortcut) { + int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + final String word = shortcut.mWord; + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + size += CharEncoding.getCharSize(codePoint); + } + size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the size of a shortcut list in bytes. + * + * This is known in advance and does not change according to position in the file + * like address lists do. + */ + static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { + if (null == shortcutList || shortcutList.isEmpty()) return 0; + int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + for (final WeightedString shortcut : shortcutList) { + size += getShortcutSize(shortcut); + } + return size; + } + + /** + * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. + * + * @param ptNode the PtNode to compute the size of. + * @return the maximum size of the PtNode. + */ + private static int getPtNodeMaximumSize(final PtNode ptNode) { + int size = getNodeHeaderSize(ptNode); + if (ptNode.isTerminal()) { + // If terminal, one byte for the frequency. + size += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address + size += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE + + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE) + * ptNode.mBigrams.size(); + } + return size; + } + + /** + * Compute the maximum size of each PtNode of a PtNode array, assuming 3-byte addresses for + * everything, and caches it in the `mCachedSize' member of the nodes; deduce the size of + * the containing node array, and cache it it its 'mCachedSize' member. + * + * @param ptNodeArray the node array to compute the maximum size of. + */ + private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray) { + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode node : ptNodeArray.mData) { + final int nodeSize = getPtNodeMaximumSize(node); + node.mCachedSize = nodeSize; + size += nodeSize; + } + ptNodeArray.mCachedSize = size; + } + + /** + * Compute the size of the header (flag + [parent address] + characters size) of a PtNode. + * + * @param ptNode the PtNode of which to compute the size of the header + */ + private static int getNodeHeaderSize(final PtNode ptNode) { + return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode); + } + + /** + * Compute the size, in bytes, that an address will occupy. + * + * This can be used either for children addresses (which are always positive) or for + * attribute, which may be positive or negative but + * store their sign bit separately. + * + * @param address the address + * @return the byte size. + */ + static int getByteSize(final int address) { + assert(address <= FormatSpec.UINT24_MAX); + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + return 0; + } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) { + return 1; + } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) { + return 2; + } else { + return 3; + } + } + + static int writeUIntToBuffer(final byte[] buffer, int position, final int value, + final int size) { + switch(size) { + case 4: + buffer[position++] = (byte) ((value >> 24) & 0xFF); + /* fall through */ + case 3: + buffer[position++] = (byte) ((value >> 16) & 0xFF); + /* fall through */ + case 2: + buffer[position++] = (byte) ((value >> 8) & 0xFF); + /* fall through */ + case 1: + buffer[position++] = (byte) (value & 0xFF); + break; + default: + /* nop */ + } + return position; + } + + static void writeUIntToStream(final OutputStream stream, final int value, final int size) + throws IOException { + switch(size) { + case 4: + stream.write((value >> 24) & 0xFF); + /* fall through */ + case 3: + stream.write((value >> 16) & 0xFF); + /* fall through */ + case 2: + stream.write((value >> 8) & 0xFF); + /* fall through */ + case 1: + stream.write(value & 0xFF); + break; + default: + /* nop */ + } + } + + @UsedForTesting + static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value, + final int size) { + switch(size) { + case 4: + dictBuffer.put((byte) ((value >> 24) & 0xFF)); + /* fall through */ + case 3: + dictBuffer.put((byte) ((value >> 16) & 0xFF)); + /* fall through */ + case 2: + dictBuffer.put((byte) ((value >> 8) & 0xFF)); + /* fall through */ + case 1: + dictBuffer.put((byte) (value & 0xFF)); + break; + default: + /* nop */ + } + } + + // End utility methods + + // This method is responsible for finding a nice ordering of the nodes that favors run-time + // cache performance and dictionary size. + /* package for tests */ static ArrayList<PtNodeArray> flattenTree( + final PtNodeArray rootNodeArray) { + final int treeSize = FusionDictionary.countPtNodes(rootNodeArray); + MakedictLog.i("Counted nodes : " + treeSize); + final ArrayList<PtNodeArray> flatTree = new ArrayList<>(treeSize); + return flattenTreeInner(flatTree, rootNodeArray); + } + + private static ArrayList<PtNodeArray> flattenTreeInner(final ArrayList<PtNodeArray> list, + final PtNodeArray ptNodeArray) { + // Removing the node is necessary if the tails are merged, because we would then + // add the same node several times when we only want it once. A number of places in + // the code also depends on any node being only once in the list. + // Merging tails can only be done if there are no attributes. Searching for attributes + // in LatinIME code depends on a total breadth-first ordering, which merging tails + // breaks. If there are no attributes, it should be fine (and reduce the file size) + // to merge tails, and removing the node from the list would be necessary. However, + // we don't merge tails because breaking the breadth-first ordering would result in + // extreme overhead at bigram lookup time (it would make the search function O(n) instead + // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty + // high). + // If no nodes are ever merged, we can't have the same node twice in the list, hence + // searching for duplicates in unnecessary. It is also very performance consuming, + // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making + // this simple list.remove operation O(n*n) overall. On Android this overhead is very + // high. + // For future reference, the code to remove duplicate is a simple : list.remove(node); + list.add(ptNodeArray); + final ArrayList<PtNode> branches = ptNodeArray.mData; + for (PtNode ptNode : branches) { + if (null != ptNode.mChildren) flattenTreeInner(list, ptNode.mChildren); + } + return list; + } + + /** + * Get the offset from a position inside a current node array to a target node array, during + * update. + * + * If the current node array is before the target node array, the target node array has not + * been updated yet, so we should return the offset from the old position of the current node + * array to the old position of the target node array. If on the other hand the target is + * before the current node array, it already has been updated, so we should return the offset + * from the new position in the current node array to the new position in the target node + * array. + * + * @param currentNodeArray node array containing the PtNode where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetNodeArray the target node array to get the offset to + * @return the offset to the target node array + */ + private static int getOffsetToTargetNodeArrayDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNodeArray targetNodeArray) { + final boolean isTargetBeforeCurrent = (targetNodeArray.mCachedAddressBeforeUpdate + < currentNodeArray.mCachedAddressBeforeUpdate); + if (isTargetBeforeCurrent) { + return targetNodeArray.mCachedAddressAfterUpdate + - (currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray); + } else { + return targetNodeArray.mCachedAddressBeforeUpdate + - (currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray); + } + } + + /** + * Get the offset from a position inside a current node array to a target PtNode, during + * update. + * + * @param currentNodeArray node array containing the PtNode where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetPtNode the target PtNode to get the offset to + * @return the offset to the target PtNode + */ + // TODO: is there any way to factorize this method with the one above? + private static int getOffsetToTargetPtNodeDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNode targetPtNode) { + final int oldOffsetBasePoint = currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray; + final boolean isTargetBeforeCurrent = (targetPtNode.mCachedAddressBeforeUpdate + < oldOffsetBasePoint); + // If the target is before the current node array, then its address has already been + // updated. We can use the AfterUpdate member, and compare it to our own member after + // update. Otherwise, the AfterUpdate member is not updated yet, so we need to use the + // BeforeUpdate member, and of course we have to compare this to our own address before + // update. + if (isTargetBeforeCurrent) { + final int newOffsetBasePoint = currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray; + return targetPtNode.mCachedAddressAfterUpdate - newOffsetBasePoint; + } else { + return targetPtNode.mCachedAddressBeforeUpdate - oldOffsetBasePoint; + } + } + + /** + * Computes the actual node array size, based on the cached addresses of the children nodes. + * + * Each node array stores its tentative address. During dictionary address computing, these + * are not final, but they can be used to compute the node array size (the node array size + * depends on the address of the children because the number of bytes necessary to store an + * address depends on its numeric value. The return value indicates whether the node array + * contents (as in, any of the addresses stored in the cache fields) have changed with + * respect to their previous value. + * + * @param ptNodeArray the node array to compute the size of. + * @param dict the dictionary in which the word/attributes are to be found. + * @return false if none of the cached addresses inside the node array changed, true otherwise. + */ + private static boolean computeActualPtNodeArraySize(final PtNodeArray ptNodeArray, + final FusionDictionary dict) { + boolean changed = false; + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode ptNode : ptNodeArray.mData) { + ptNode.mCachedAddressAfterUpdate = ptNodeArray.mCachedAddressAfterUpdate + size; + if (ptNode.mCachedAddressAfterUpdate != ptNode.mCachedAddressBeforeUpdate) { + changed = true; + } + int nodeSize = getNodeHeaderSize(ptNode); + if (ptNode.isTerminal()) { + nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + if (null != ptNode.mChildren) { + nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, + nodeSize + size, ptNode.mChildren)); + } + nodeSize += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + for (WeightedString bigram : ptNode.mBigrams) { + final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, + nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE, + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); + nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + } + } + ptNode.mCachedSize = nodeSize; + size += nodeSize; + } + if (ptNodeArray.mCachedSize != size) { + ptNodeArray.mCachedSize = size; + changed = true; + } + return changed; + } + + /** + * Initializes the cached addresses of node arrays and their containing nodes from their size. + * + * @param flatNodes the list of node arrays. + * @return the byte size of the entire stack. + */ + private static int initializePtNodeArraysCachedAddresses( + final ArrayList<PtNodeArray> flatNodes) { + int nodeArrayOffset = 0; + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset; + int nodeCountSize = getPtNodeCountSize(nodeArray); + int nodeffset = 0; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate = + nodeCountSize + nodeArrayOffset + nodeffset; + nodeffset += ptNode.mCachedSize; + } + nodeArrayOffset += nodeArray.mCachedSize; + } + return nodeArrayOffset; + } + + /** + * Updates the cached addresses of node arrays after recomputing their new positions. + * + * @param flatNodes the list of node arrays. + */ + private static void updatePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArray.mCachedAddressAfterUpdate; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate; + } + } + } + + /** + * Compute the addresses and sizes of an ordered list of PtNode arrays. + * + * This method takes a list of PtNode arrays and will update their cached address and size + * values so that they can be written into a file. It determines the smallest size each of the + * PtNode arrays can be given the addresses of its children and attributes, and store that into + * each PtNode. + * The order of the PtNode is given by the order of the array. This method makes no effort + * to find a good order; it only mechanically computes the size this order results in. + * + * @param dict the dictionary + * @param flatNodes the ordered list of PtNode arrays + * @return the same array it was passed. The nodes have been updated for address and size. + */ + /* package */ static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, + final ArrayList<PtNodeArray> flatNodes) { + // First get the worst possible sizes and offsets + for (final PtNodeArray n : flatNodes) { + calculatePtNodeArrayMaximumSize(n); + } + final int offset = initializePtNodeArraysCachedAddresses(flatNodes); + + MakedictLog.i("Compressing the array addresses. Original size : " + offset); + MakedictLog.i("(Recursively seen size : " + offset + ")"); + + int passes = 0; + boolean changesDone = false; + do { + changesDone = false; + int ptNodeArrayStartOffset = 0; + for (final PtNodeArray ptNodeArray : flatNodes) { + ptNodeArray.mCachedAddressAfterUpdate = ptNodeArrayStartOffset; + final int oldNodeArraySize = ptNodeArray.mCachedSize; + final boolean changed = computeActualPtNodeArraySize(ptNodeArray, dict); + final int newNodeArraySize = ptNodeArray.mCachedSize; + if (oldNodeArraySize < newNodeArraySize) { + throw new RuntimeException("Increased size ?!"); + } + ptNodeArrayStartOffset += newNodeArraySize; + changesDone |= changed; + } + updatePtNodeArraysCachedAddresses(flatNodes); + ++passes; + if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); + } while (changesDone); + + final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); + MakedictLog.i("Compression complete in " + passes + " passes."); + MakedictLog.i("After address compression : " + + (lastPtNodeArray.mCachedAddressAfterUpdate + lastPtNodeArray.mCachedSize)); + + return flatNodes; + } + + /** + * Sanity-checking method. + * + * This method checks a list of PtNode arrays for juxtaposition, that is, it will do + * nothing if each node array's cached address is actually the previous node array's address + * plus the previous node's size. + * If this is not the case, it will throw an exception. + * + * @param arrays the list of node arrays to check + */ + /* package */ static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) { + int offset = 0; + int index = 0; + for (final PtNodeArray ptNodeArray : arrays) { + // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter + // which we use. + if (ptNodeArray.mCachedAddressAfterUpdate != offset) { + throw new RuntimeException("Wrong address for node " + index + + " : expected " + offset + ", got " + + ptNodeArray.mCachedAddressAfterUpdate); + } + ++index; + offset += ptNodeArray.mCachedSize; + } + } + + /** + * Helper method to write a children position to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param position the position to write. + * @return the size in bytes the address actually took. + */ + /* package */ static int writeChildrenPosition(final byte[] buffer, int index, + final int position) { + switch (getByteSize(position)) { + case 1: + buffer[index++] = (byte)position; + return 1; + case 2: + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); + return 2; + case 3: + buffer[index++] = (byte)(0xFF & (position >> 16)); + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); + return 3; + case 0: + return 0; + default: + throw new RuntimeException("Position " + position + " has a strange size"); + } + } + + /** + * Helper method to write a signed children position to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param position the position to write. + * @return the size in bytes the address actually took. + */ + /* package */ static int writeSignedChildrenPosition(final byte[] buffer, int index, + final int position) { + if (!BinaryDictIOUtils.hasChildrenAddress(position)) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absPosition = Math.abs(position); + buffer[index++] = + (byte)((position < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absPosition >> 16))); + buffer[index++] = (byte)(0xFF & (absPosition >> 8)); + buffer[index++] = (byte)(0xFF & absPosition); + } + return 3; + } + + /** + * Makes the flag value for a PtNode. + * + * @param hasMultipleChars whether the PtNode has multiple chars. + * @param isTerminal whether the PtNode is terminal. + * @param childrenAddressSize the size of a children address. + * @param hasShortcuts whether the PtNode has shortcuts. + * @param hasBigrams whether the PtNode has bigrams. + * @param isNotAWord whether the PtNode is not a word. + * @param isBlackListEntry whether the PtNode is a blacklist entry. + * @return the flags + */ + static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, + final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, + final boolean isNotAWord, final boolean isBlackListEntry) { + byte flags = 0; + if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; + if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; + switch (childrenAddressSize) { + case 1: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES; + break; + case 0: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS; + break; + default: + throw new RuntimeException("Node with a strange address"); + } + if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; + if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; + if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; + if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; + return flags; + } + + /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset) { + return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(), + getByteSize(childrenOffset), + node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), + node.mBigrams != null && !node.mBigrams.isEmpty(), + node.mIsNotAWord, node.mIsBlacklistEntry); + } + + /** + * Makes the flag value for a bigram. + * + * @param more whether there are more bigrams after this one. + * @param offset the offset of the bigram. + * @param bigramFrequency the frequency of the bigram, 0..255. + * @param unigramFrequency the unigram frequency of the same word, 0..255. + * @param word the second bigram, for debugging purposes + * @return the flags + */ + /* package */ static final int makeBigramFlags(final boolean more, final int offset, + int bigramFrequency, final int unigramFrequency, final String word) { + int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); + switch (getByteSize(offset)) { + case 1: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; + break; + default: + throw new RuntimeException("Strange offset size"); + } + if (unigramFrequency > bigramFrequency) { + MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word + + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " + + word + " is " + unigramFrequency); + bigramFrequency = unigramFrequency; + } + bigramFlags += getBigramFrequencyDiff(unigramFrequency, bigramFrequency) + & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; + return bigramFlags; + } + + public static int getBigramFrequencyDiff(final int unigramFrequency, + final int bigramFrequency) { + // We compute the difference between 255 (which means probability = 1) and the + // unigram score. We split this into a number of discrete steps. + // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 + // represents an increase of 16 steps: a value of 15 will be interpreted as the median + // value of the 16th step. In all justice, if the bigram frequency is low enough to be + // rounded below the first step (which means it is less than half a step higher than the + // unigram frequency) then the unigram frequency itself is the best approximation of the + // bigram freq that we could possibly supply, hence we should *not* include this bigram + // in the file at all. + // until this is done, we'll write 0 and slightly overestimate this case. + // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step + // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to + // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the + // step size. Then we compute the start of the first step (the one where value 0 starts) + // by adding half-a-step to the unigramFrequency. From there, we compute the integer + // number of steps to the bigramFrequency. One last thing: we want our steps to include + // their lower bound and exclude their higher bound so we need to have the first step + // start at exactly 1 unit higher than floor(unigramFreq + half a step). + // Note : to reconstruct the score, the dictionary reader will need to divide + // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, + // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best + // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the + // step pointed by the discretized frequency. + final float stepSize = + (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); + final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); + // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 + // here. The best approximation would be the unigram freq itself, so we should not + // include this bigram in the dictionary. For now, register as 0, and live with the + // small over-estimation that we get in this case. TODO: actually remove this bigram + // if discretizedFrequency < 0. + return discretizedFrequency > 0 ? discretizedFrequency : 0; + } + + /** + * Makes the flag value for a shortcut. + * + * @param more whether there are more attributes after this one. + * @param frequency the frequency of the attribute, 0..15 + * @return the flags + */ + static final int makeShortcutFlags(final boolean more, final int frequency) { + return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); + } + + /* package */ static final int getChildrenPosition(final PtNode ptNode) { + int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate + + getNodeHeaderSize(ptNode); + if (ptNode.isTerminal()) { + // A terminal node has the frequency. + // If positionOfChildrenPosField is incorrect, we may crash when jumping to the children + // position. + positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS + : ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField; + } + + /** + * Write a PtNodeArray. The PtNodeArray is expected to have its final position cached. + * + * @param dict the dictionary the node array is a part of (for relative offsets). + * @param dictEncoder the dictionary encoder. + * @param ptNodeArray the node array to write. + */ + @SuppressWarnings("unused") + /* package */ static void writePlacedPtNodeArray(final FusionDictionary dict, + final DictEncoder dictEncoder, final PtNodeArray ptNodeArray) { + // TODO: Make the code in common with BinaryDictIOUtils#writePtNode + dictEncoder.setPosition(ptNodeArray.mCachedAddressAfterUpdate); + + final int ptNodeCount = ptNodeArray.mData.size(); + dictEncoder.writePtNodeCount(ptNodeCount); + final int parentPosition = + (ptNodeArray.mCachedParentAddress == FormatSpec.NO_PARENT_ADDRESS) + ? FormatSpec.NO_PARENT_ADDRESS + : ptNodeArray.mCachedParentAddress + ptNodeArray.mCachedAddressAfterUpdate; + for (int i = 0; i < ptNodeCount; ++i) { + final PtNode ptNode = ptNodeArray.mData.get(i); + if (dictEncoder.getPosition() != ptNode.mCachedAddressAfterUpdate) { + throw new RuntimeException("Bug: write index is not the same as the cached address " + + "of the node : " + dictEncoder.getPosition() + " <> " + + ptNode.mCachedAddressAfterUpdate); + } + // Sanity checks. + if (DBG && ptNode.getProbability() > FormatSpec.MAX_TERMINAL_FREQUENCY) { + throw new RuntimeException("A node has a frequency > " + + FormatSpec.MAX_TERMINAL_FREQUENCY + + " : " + ptNode.mProbabilityInfo.toString()); + } + dictEncoder.writePtNode(ptNode, dict); + } + if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate + + ptNodeArray.mCachedSize) { + throw new RuntimeException("Not the same size : written " + + (dictEncoder.getPosition() - ptNodeArray.mCachedAddressAfterUpdate) + + " bytes from a node that should have " + ptNodeArray.mCachedSize + " bytes"); + } + } + + /** + * Dumps a collection of useful statistics about a list of PtNode arrays. + * + * This prints purely informative stuff, like the total estimated file size, the + * number of PtNode arrays, of PtNodes, the repartition of each address size, etc + * + * @param ptNodeArrays the list of PtNode arrays. + */ + /* package */ static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) { + int firstTerminalAddress = Integer.MAX_VALUE; + int lastTerminalAddress = Integer.MIN_VALUE; + int size = 0; + int ptNodes = 0; + int maxNodes = 0; + int maxRuns = 0; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + if (maxNodes < ptNodeArray.mData.size()) maxNodes = ptNodeArray.mData.size(); + for (final PtNode ptNode : ptNodeArray.mData) { + ++ptNodes; + if (ptNode.mChars.length > maxRuns) maxRuns = ptNode.mChars.length; + if (ptNode.isTerminal()) { + if (ptNodeArray.mCachedAddressAfterUpdate < firstTerminalAddress) + firstTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + if (ptNodeArray.mCachedAddressAfterUpdate > lastTerminalAddress) + lastTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + } + } + if (ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize > size) { + size = ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize; + } + } + final int[] ptNodeCounts = new int[maxNodes + 1]; + final int[] runCounts = new int[maxRuns + 1]; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + ++ptNodeCounts[ptNodeArray.mData.size()]; + for (final PtNode ptNode : ptNodeArray.mData) { + ++runCounts[ptNode.mChars.length]; + } + } + + MakedictLog.i("Statistics:\n" + + " total file size " + size + "\n" + + " " + ptNodeArrays.size() + " node arrays\n" + + " " + ptNodes + " PtNodes (" + ((float)ptNodes / ptNodeArrays.size()) + + " PtNodes per node)\n" + + " first terminal at " + firstTerminalAddress + "\n" + + " last terminal at " + lastTerminalAddress + "\n" + + " PtNode stats : max = " + maxNodes); + for (int i = 0; i < ptNodeCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + ptNodeCounts[i]); + } + MakedictLog.i(" Character run stats : max = " + maxRuns); + for (int i = 0; i < runCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + runCounts[i]); + } + } + + /** + * Writes a file header to an output stream. + * + * @param destination the stream to write the file header to. + * @param dict the dictionary to write. + * @param formatOptions file format options. + * @return the size of the header. + */ + /* package */ static int writeDictionaryHeader(final OutputStream destination, + final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + final int version = formatOptions.mVersion; + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("Requested file format version " + version + + ", but this implementation only supports versions " + + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + + ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); + + // The magic number in big-endian order. + // Magic number for all versions. + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8))); + headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER)); + // Dictionary version. + headerBuffer.write((byte) (0xFF & (version >> 8))); + headerBuffer.write((byte) (0xFF & version)); + + // Options flags + // TODO: Remove this field. + final int options = 0; + headerBuffer.write((byte) (0xFF & (options >> 8))); + headerBuffer.write((byte) (0xFF & options)); + final int headerSizeOffset = headerBuffer.size(); + // Placeholder to be written later with header size. + for (int i = 0; i < 4; ++i) { + headerBuffer.write(0); + } + // Write out the options. + for (final String key : dict.mOptions.mAttributes.keySet()) { + final String value = dict.mOptions.mAttributes.get(key); + CharEncoding.writeString(headerBuffer, key); + CharEncoding.writeString(headerBuffer, value); + } + final int size = headerBuffer.size(); + final byte[] bytes = headerBuffer.toByteArray(); + // Write out the header size. + bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); + bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); + bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); + bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); + destination.write(bytes); + + headerBuffer.close(); + return size; + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java new file mode 100644 index 000000000..9c3b37387 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -0,0 +1,306 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Map; +import java.util.Stack; + +public final class BinaryDictIOUtils { + private static final boolean DBG = false; + + private BinaryDictIOUtils() { + // This utility class is not publicly instantiable. + } + + /** + * Returns new dictionary decoder. + * + * @param dictFile the dictionary file. + * @param bufferType The type of buffer, as one of USE_* in DictDecoder. + * @return new dictionary decoder if the dictionary file exists, otherwise null. + */ + public static DictDecoder getDictDecoder(final File dictFile, final long offset, + final long length, final int bufferType) { + if (dictFile.isDirectory()) { + return new Ver4DictDecoder(dictFile, bufferType); + } else if (dictFile.isFile()) { + return new Ver2DictDecoder(dictFile, offset, length, bufferType); + } + return null; + } + + public static DictDecoder getDictDecoder(final File dictFile, final long offset, + final long length, final DictionaryBufferFactory factory) { + if (dictFile.isDirectory()) { + return new Ver4DictDecoder(dictFile, factory); + } else if (dictFile.isFile()) { + return new Ver2DictDecoder(dictFile, offset, length, factory); + } + return null; + } + + public static DictDecoder getDictDecoder(final File dictFile, final long offset, + final long length) { + return getDictDecoder(dictFile, offset, length, DictDecoder.USE_READONLY_BYTEBUFFER); + } + + private static final class Position { + public static final int NOT_READ_PTNODE_COUNT = -1; + + public int mAddress; + public int mNumOfPtNode; + public int mPosition; + public int mLength; + + public Position(int address, int length) { + mAddress = address; + mLength = length; + mNumOfPtNode = NOT_READ_PTNODE_COUNT; + } + } + + /** + * Retrieves all node arrays without recursive call. + */ + private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder, + final int bodyOffset, final Map<Integer, String> words, + final Map<Integer, Integer> frequencies, + final Map<Integer, ArrayList<PendingAttribute>> bigrams) { + int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; + + Stack<Position> stack = new Stack<>(); + int index = 0; + + Position initPos = new Position(bodyOffset, 0); + stack.push(initPos); + + while (!stack.empty()) { + Position p = stack.peek(); + + if (DBG) { + MakedictLog.d("read: address=" + p.mAddress + ", numOfPtNode=" + + p.mNumOfPtNode + ", position=" + p.mPosition + ", length=" + p.mLength); + } + + if (dictDecoder.getPosition() != p.mAddress) dictDecoder.setPosition(p.mAddress); + if (index != p.mLength) index = p.mLength; + + if (p.mNumOfPtNode == Position.NOT_READ_PTNODE_COUNT) { + p.mNumOfPtNode = dictDecoder.readPtNodeCount(); + p.mAddress = dictDecoder.getPosition(); + p.mPosition = 0; + } + if (p.mNumOfPtNode == 0) { + stack.pop(); + continue; + } + final PtNodeInfo ptNodeInfo = dictDecoder.readPtNode(p.mAddress); + for (int i = 0; i < ptNodeInfo.mCharacters.length; ++i) { + pushedChars[index++] = ptNodeInfo.mCharacters[i]; + } + p.mPosition++; + if (ptNodeInfo.isTerminal()) {// found word + words.put(ptNodeInfo.mOriginalAddress, new String(pushedChars, 0, index)); + frequencies.put( + ptNodeInfo.mOriginalAddress, ptNodeInfo.mProbabilityInfo.mProbability); + if (ptNodeInfo.mBigrams != null) { + bigrams.put(ptNodeInfo.mOriginalAddress, ptNodeInfo.mBigrams); + } + } + + if (p.mPosition == p.mNumOfPtNode) { + stack.pop(); + } else { + // The PtNode array has more PtNodes. + p.mAddress = dictDecoder.getPosition(); + } + + if (hasChildrenAddress(ptNodeInfo.mChildrenAddress)) { + final Position childrenPos = new Position(ptNodeInfo.mChildrenAddress, index); + stack.push(childrenPos); + } + } + } + + /** + * Reads unigrams and bigrams from the binary file. + * Doesn't store a full memory representation of the dictionary. + * + * @param dictDecoder the dict decoder. + * @param words the map to store the address as a key and the word as a value. + * @param frequencies the map to store the address as a key and the frequency as a value. + * @param bigrams the map to store the address as a key and the list of address as a value. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + /* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder, + final Map<Integer, String> words, final Map<Integer, Integer> frequencies, + final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, + UnsupportedFormatException { + // Read header + final DictionaryHeader header = dictDecoder.readHeader(); + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mBodyOffset, words, + frequencies, bigrams); + } + + /** + * Gets the address of the last PtNode of the exact matching word in the dictionary. + * If no match is found, returns NOT_VALID_WORD. + * + * @param dictDecoder the dict decoder. + * @param word the word we search for. + * @return the address of the terminal node. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + /* package */ static int getTerminalPosition(final DictDecoder dictDecoder, + final String word) throws IOException, UnsupportedFormatException { + if (word == null) return FormatSpec.NOT_VALID_WORD; + dictDecoder.setPosition(0); + dictDecoder.readHeader(); + int wordPos = 0; + final int wordLen = word.codePointCount(0, word.length()); + for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { + if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; + + do { + final int ptNodeCount = dictDecoder.readPtNodeCount(); + boolean foundNextPtNode = false; + for (int i = 0; i < ptNodeCount; ++i) { + final int ptNodePos = dictDecoder.getPosition(); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos); + boolean same = true; + for (int p = 0, j = word.offsetByCodePoints(0, wordPos); + p < currentInfo.mCharacters.length; + ++p, j = word.offsetByCodePoints(j, 1)) { + if (wordPos + p >= wordLen + || word.codePointAt(j) != currentInfo.mCharacters[p]) { + same = false; + break; + } + } + + if (same) { + // found the PtNode matches the word. + if (wordPos + currentInfo.mCharacters.length == wordLen) { + if (!currentInfo.isTerminal()) { + return FormatSpec.NOT_VALID_WORD; + } else { + return ptNodePos; + } + } + wordPos += currentInfo.mCharacters.length; + if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { + return FormatSpec.NOT_VALID_WORD; + } + foundNextPtNode = true; + dictDecoder.setPosition(currentInfo.mChildrenAddress); + break; + } + } + if (foundNextPtNode) break; + return FormatSpec.NOT_VALID_WORD; + } while(true); + } + return FormatSpec.NOT_VALID_WORD; + } + + /** + * Writes a PtNodeCount to the stream. + * + * @param destination the stream to write. + * @param ptNodeCount the count. + * @return the size written in bytes. + */ + @UsedForTesting + static int writePtNodeCount(final OutputStream destination, final int ptNodeCount) + throws IOException { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + // the count must fit on one byte or two bytes. + // Please see comments in FormatSpec. + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); + } + final int encodedPtNodeCount = (countSize == 2) ? + (ptNodeCount | FormatSpec.LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG) : ptNodeCount; + BinaryDictEncoderUtils.writeUIntToStream(destination, encodedPtNodeCount, countSize); + return countSize; + } + + /** + * Helper method to hide the actual value of the no children address. + */ + public static boolean hasChildrenAddress(final int address) { + return FormatSpec.NO_CHILDREN_ADDRESS != address; + } + + /** + * Compute the binary size of the node count + * @param count the node count + * @return the size of the node count, either 1 or 2 bytes. + */ + public static int getPtNodeCountSize(final int count) { + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= count) { + return 1; + } else if (FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY >= count) { + return 2; + } else { + throw new RuntimeException("Can't have more than " + + FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY + " PtNode in a PtNodeArray (found " + + count + ")"); + } + } + + static int getChildrenAddressSize(final int optionFlags) { + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return 1; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return 2; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return 3; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return 0; + } + } + + /** + * Calculate bigram frequency from compressed value + * + * @param unigramFrequency + * @param bigramFrequency compressed frequency + * @return approximate bigram frequency + */ + @UsedForTesting + public static int reconstructBigramFrequency(final int unigramFrequency, + final int bigramFrequency) { + final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); + return (int)resultFreqFloat; + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java deleted file mode 100644 index afe5adb73..000000000 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import android.test.AndroidTestCase; -import android.test.MoreAsserts; -import android.test.suitebuilder.annotation.LargeTest; -import android.util.Log; - -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Random; - -@LargeTest -public class BinaryDictIOUtilsTests extends AndroidTestCase { - private static final String TAG = BinaryDictIOUtilsTests.class.getSimpleName(); - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(3, true); - - private static final ArrayList<String> sWords = CollectionUtils.newArrayList(); - public static final int DEFAULT_MAX_UNIGRAMS = 1500; - private final int mMaxUnigrams; - - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - - private static final int VERSION3 = 3; - private static final int VERSION4 = 4; - - private static final String[] CHARACTERS = { - "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", - "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", - "\u00FC" /* ü */, "\u00E2" /* â */, "\u00F1" /* ñ */, // accented characters - "\u4E9C" /* 亜 */, "\u4F0A" /* 伊 */, "\u5B87" /* 宇 */, // kanji - "\uD841\uDE28" /* 𠘨 */, "\uD840\uDC0B" /* 𠀋 */, "\uD861\uDED7" /* 𨛗 */ // surrogate pair - }; - - public BinaryDictIOUtilsTests() { - // 1500 is the default max unigrams - this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); - } - - public BinaryDictIOUtilsTests(final long seed, final int maxUnigrams) { - super(); - Log.d(TAG, "Seed for test is " + seed + ", maxUnigrams is " + maxUnigrams); - mMaxUnigrams = maxUnigrams; - final Random random = new Random(seed); - sWords.clear(); - for (int i = 0; i < maxUnigrams; ++i) { - sWords.add(generateWord(random.nextInt())); - } - } - - // Utilities for test - private String generateWord(final int value) { - final int lengthOfChars = CHARACTERS.length; - StringBuilder builder = new StringBuilder(""); - long lvalue = Math.abs((long)value); - while (lvalue > 0) { - builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]); - lvalue /= lengthOfChars; - } - if (builder.toString().equals("")) return "a"; - return builder.toString(); - } - - private static void printPtNode(final PtNodeInfo info) { - Log.d(TAG, " PtNode at " + info.mOriginalAddress); - Log.d(TAG, " flags = " + info.mFlags); - Log.d(TAG, " parentAddress = " + info.mParentAddress); - Log.d(TAG, " characters = " + new String(info.mCharacters, 0, - info.mCharacters.length)); - if (info.mFrequency != -1) Log.d(TAG, " frequency = " + info.mFrequency); - if (info.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { - Log.d(TAG, " children address = no children address"); - } else { - Log.d(TAG, " children address = " + info.mChildrenAddress); - } - if (info.mShortcutTargets != null) { - for (final WeightedString ws : info.mShortcutTargets) { - Log.d(TAG, " shortcuts = " + ws.mWord); - } - } - if (info.mBigrams != null) { - for (final PendingAttribute attr : info.mBigrams) { - Log.d(TAG, " bigram = " + attr.mAddress); - } - } - Log.d(TAG, " end address = " + info.mEndAddress); - } - - private static void printNode(final Ver3DictDecoder dictDecoder, - final FormatSpec.FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - Log.d(TAG, "Node at " + dictBuffer.position()); - final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); - Log.d(TAG, " ptNodeCount = " + count); - for (int i = 0; i < count; ++i) { - final PtNodeInfo currentInfo = dictDecoder.readPtNode(dictBuffer.position(), - formatOptions); - printPtNode(currentInfo); - } - if (formatOptions.mSupportsDynamicUpdate) { - final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); - Log.d(TAG, " forwardLinkAddress = " + forwardLinkAddress); - } - } - - @SuppressWarnings("unused") - private static void printBinaryFile(final Ver3DictDecoder dictDecoder) - throws IOException, UnsupportedFormatException { - final FileHeader fileHeader = dictDecoder.readHeader(); - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - while (dictBuffer.position() < dictBuffer.limit()) { - printNode(dictDecoder, fileHeader.mFormatOptions); - } - } - - private int getWordPosition(final File file, final String word) { - int position = FormatSpec.NOT_VALID_WORD; - - try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, - DictDecoder.USE_READONLY_BYTEBUFFER); - position = dictDecoder.getTerminalPosition(word); - } catch (IOException e) { - } catch (UnsupportedFormatException e) { - } - return position; - } - - /** - * Find a word using the DictDecoder. - * - * @param dictDecoder the dict decoder - * @param word the word searched - * @return the found ptNodeInfo - * @throws IOException - * @throws UnsupportedFormatException - */ - private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder, - final String word) throws IOException, UnsupportedFormatException { - int position = dictDecoder.getTerminalPosition(word); - if (position != FormatSpec.NOT_VALID_WORD) { - dictDecoder.setPosition(0); - final FileHeader header = dictDecoder.readHeader(); - dictDecoder.setPosition(position); - return dictDecoder.readPtNode(position, header.mFormatOptions); - } - return null; - } - - private PtNodeInfo findWordFromFile(final File file, final String word) { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); - PtNodeInfo info = null; - try { - dictDecoder.openDictBuffer(); - info = findWordByBinaryDictReader(dictDecoder, word); - } catch (IOException e) { - } catch (UnsupportedFormatException e) { - } - return info; - } - - // return amount of time to insert a word - private long insertAndCheckWord(final File file, final String word, final int frequency, - final boolean exist, final ArrayList<WeightedString> bigrams, - final ArrayList<WeightedString> shortcuts, final int formatVersion) { - long amountOfTime = -1; - try { - final DictUpdater dictUpdater; - if (formatVersion == VERSION3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" - + " exist."); - } - - if (!exist) { - assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - } - final long now = System.nanoTime(); - dictUpdater.insertWord(word, frequency, bigrams, shortcuts, false, false); - amountOfTime = System.nanoTime() - now; - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - } catch (IOException e) { - Log.e(TAG, "Raised an IOException while inserting a word", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e); - } - return amountOfTime; - } - - private void deleteWord(final File file, final String word, final int formatVersion) { - try { - final DictUpdater dictUpdater; - if (formatVersion == VERSION3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" - + " exist."); - } - dictUpdater.deleteWord(word); - } catch (IOException e) { - } catch (UnsupportedFormatException e) { - } - } - - private void checkReverseLookup(final File file, final String word, final int position) { - - try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); - final FileHeader fileHeader = dictDecoder.readHeader(); - assertEquals(word, - BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize, - position, fileHeader.mFormatOptions).mWord); - } catch (IOException e) { - Log.e(TAG, "Raised an IOException while looking up a word", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Raised an UnsupportedFormatException error while looking up a word", e); - } - } - - private void runTestInsertWord(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - fail("IOException while creating temporary file: " + e); - } - - // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); - dict.add("abcd", 10, null, false); - - try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); - } - - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - insertAndCheckWord(file, "abcde", 10, false, null, null, formatVersion); - - insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatVersion); - checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - - insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatVersion); - checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); - - // update the existing word. - insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatVersion); - - // split 1 - insertAndCheckWord(file, "ab", 20, false, null, null, formatVersion); - - // split 2 - insertAndCheckWord(file, "ami", 30, false, null, null, formatVersion); - - deleteWord(file, "ami", formatVersion); - assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami")); - - insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatVersion); - - deleteWord(file, "abcd", formatVersion); - assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - } - - public void testInsertWord() { - runTestInsertWord(VERSION3); - } - - private void runTestInsertWordWithBigrams(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - fail("IOException while creating temporary file: " + e); - } - - // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); - dict.add("abcd", 10, null, false); - dict.add("efgh", 15, null, false); - - try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); - } - - final ArrayList<WeightedString> banana = new ArrayList<WeightedString>(); - banana.add(new WeightedString("banana", 10)); - - insertAndCheckWord(file, "banana", 0, false, null, null, formatVersion); - insertAndCheckWord(file, "recursive", 60, true, banana, null, formatVersion); - - final PtNodeInfo info = findWordFromFile(file, "recursive"); - int bananaPos = getWordPosition(file, "banana"); - assertNotNull(info.mBigrams); - assertEquals(info.mBigrams.size(), 1); - assertEquals(info.mBigrams.get(0).mAddress, bananaPos); - } - - public void testInsertWordWithBigrams() { - runTestInsertWordWithBigrams(VERSION3); - } - - private void runTestRandomWords(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - } - assertNotNull(file); - - // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, - false)); - dict.add("initial", 10, null, false); - - try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - assertTrue(false); - } catch (UnsupportedFormatException e) { - assertTrue(false); - } - - long maxTimeToInsert = 0, sum = 0; - long minTimeToInsert = 100000000; // 1000000000 is an upper bound for minTimeToInsert. - int cnt = 0; - for (final String word : sWords) { - final long diff = insertAndCheckWord(file, word, - cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatVersion); - maxTimeToInsert = Math.max(maxTimeToInsert, diff); - minTimeToInsert = Math.min(minTimeToInsert, diff); - sum += diff; - cnt++; - } - cnt = 0; - for (final String word : sWords) { - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - } - - Log.d(TAG, "Test version " + formatVersion); - Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms."); - Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms."); - Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms."); - } - - public void testRandomWords() { - runTestRandomWords(VERSION3); - } -} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java new file mode 100644 index 000000000..5a3eba801 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; + +import java.io.File; +import java.util.HashMap; + +public class BinaryDictUtils { + public static final int USE_BYTE_ARRAY = 1; + public static final int USE_BYTE_BUFFER = 2; + + public static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + + public static final FormatSpec.FormatOptions VERSION2_OPTIONS = + new FormatSpec.FormatOptions(FormatSpec.VERSION2); + public static final FormatSpec.FormatOptions VERSION4_OPTIONS_WITHOUT_TIMESTAMP = + new FormatSpec.FormatOptions(FormatSpec.VERSION4, false /* hasTimestamp */); + public static final FormatSpec.FormatOptions VERSION4_OPTIONS_WITH_TIMESTAMP = + new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* hasTimestamp */); + + public static DictionaryOptions makeDictionaryOptions(final String id, final String version, + final FormatSpec.FormatOptions formatOptions) { + final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>()); + options.mAttributes.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, "en_US"); + options.mAttributes.put(DictionaryHeader.DICTIONARY_ID_KEY, id); + options.mAttributes.put(DictionaryHeader.DICTIONARY_VERSION_KEY, version); + if (formatOptions.mHasTimestamp) { + options.mAttributes.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + options.mAttributes.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + } + return options; + } + + public static File getDictFile(final String name, final String version, + final FormatOptions formatOptions, final File directory) { + if (formatOptions.mVersion == FormatSpec.VERSION2) { + return new File(directory, name + "." + version + TEST_DICT_FILE_EXTENSION); + } else if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new File(directory, name + "." + version); + } else { + throw new RuntimeException("the format option has a wrong version : " + + formatOptions.mVersion); + } + } + + public static DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + if (!file.isDirectory()) { + file.mkdir(); + } + return new Ver4DictEncoder(file); + } else if (formatOptions.mVersion == FormatSpec.VERSION2) { + return new Ver2DictEncoder(file); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/DictDecoder.java new file mode 100644 index 000000000..a3b28a702 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -0,0 +1,222 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.TreeMap; + +/** + * An interface of binary dictionary decoders. + */ +// TODO: Straighten out responsibility for the buffer's file pointer. +public interface DictDecoder { + + /** + * Reads and returns the file header. + */ + public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException; + + /** + * Reads PtNode from ptNodePos. + * @param ptNodePos the position of PtNode. + * @return PtNodeInfo. + */ + public PtNodeInfo readPtNode(final int ptNodePos); + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. + * + * @param deleteDictIfBroken a flag indicating whether this method should remove the broken + * dictionary or not. + * @return the created dictionary. + */ + @UsedForTesting + public FusionDictionary readDictionaryBinary(final boolean deleteDictIfBroken) + throws FileNotFoundException, IOException, UnsupportedFormatException; + + /** + * Gets the address of the last PtNode of the exact matching word in the dictionary. + * If no match is found, returns NOT_VALID_WORD. + * + * @param word the word we search for. + * @return the address of the terminal node. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + public int getTerminalPosition(final String word) + throws IOException, UnsupportedFormatException; + + /** + * Reads unigrams and bigrams from the binary file. + * Doesn't store a full memory representation of the dictionary. + * + * @param words the map to store the address as a key and the word as a value. + * @param frequencies the map to store the address as a key and the frequency as a value. + * @param bigrams the map to store the address as a key and the list of address as a value. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException; + + /** + * Sets the position of the buffer to the given value. + * + * @param newPos the new position + */ + public void setPosition(final int newPos); + + /** + * Gets the position of the buffer. + * + * @return the position + */ + public int getPosition(); + + /** + * Reads and returns the PtNode count out of a buffer and forwards the pointer. + */ + public int readPtNodeCount(); + + /** + * Opens the dictionary file and makes DictBuffer. + */ + @UsedForTesting + public void openDictBuffer() throws FileNotFoundException, IOException, + UnsupportedFormatException; + @UsedForTesting + public boolean isDictBufferOpen(); + + // Constants for DictionaryBufferFactory. + public static final int USE_READONLY_BYTEBUFFER = 0x01000000; + public static final int USE_BYTEARRAY = 0x02000000; + public static final int USE_WRITABLE_BYTEBUFFER = 0x03000000; + public static final int MASK_DICTBUFFER = 0x0F000000; + + public interface DictionaryBufferFactory { + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException; + } + + /** + * Creates DictionaryBuffer using a ByteBuffer + * + * This class uses less memory than DictionaryBufferFromByteArrayFactory, + * but doesn't perform as fast. + * When operating on a big dictionary, this class is preferred. + */ + public static final class DictionaryBufferFromReadOnlyByteBufferFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + ByteBuffer buffer = null; + try { + inStream = new FileInputStream(file); + buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, + 0, file.length()); + } finally { + if (inStream != null) { + inStream.close(); + } + } + if (buffer != null) { + return new BinaryDictDecoderUtils.ByteBufferDictBuffer(buffer); + } + return null; + } + } + + /** + * Creates DictionaryBuffer using a byte array + * + * This class performs faster than other classes, but consumes more memory. + * When operating on a small dictionary, this class is preferred. + */ + public static final class DictionaryBufferFromByteArrayFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final byte[] array = new byte[(int) file.length()]; + inStream.read(array); + return new ByteArrayDictBuffer(array); + } finally { + if (inStream != null) { + inStream.close(); + } + } + } + } + + /** + * Creates DictionaryBuffer using a writable ByteBuffer and a RandomAccessFile. + * + * This class doesn't perform as fast as other classes, + * but this class is the only option available for destructive operations (insert or delete) + * on a dictionary. + */ + @UsedForTesting + public static final class DictionaryBufferFromWritableByteBufferFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + RandomAccessFile raFile = null; + ByteBuffer buffer = null; + try { + raFile = new RandomAccessFile(file, "rw"); + buffer = raFile.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, file.length()); + } finally { + if (raFile != null) { + raFile.close(); + } + } + if (buffer != null) { + return new BinaryDictDecoderUtils.ByteBufferDictBuffer(buffer); + } + return null; + } + } + + /** + * @return whether this decoder has a valid binary dictionary that it can decode. + */ + public boolean hasValidRawBinaryDictionary(); +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/DictEncoder.java new file mode 100644 index 000000000..678c5ca6b --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; + +import java.io.IOException; + +/** + * An interface of binary dictionary encoder. + */ +public interface DictEncoder { + @UsedForTesting + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException; + + public void setPosition(final int position); + public int getPosition(); + public void writePtNodeCount(final int ptNodeCount); + public void writeForwardLinkAddress(final int forwardLinkAddress); + public void writePtNode(final PtNode ptNode, final FusionDictionary dict); +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java new file mode 100644 index 000000000..4a8c178b5 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -0,0 +1,716 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; + +/** + * A dictionary that can fusion heads and tails of words for more compression. + */ +@UsedForTesting +public final class FusionDictionary implements Iterable<WordProperty> { + private static final boolean DBG = MakedictLog.DBG; + + private static int CHARACTER_NOT_FOUND_INDEX = -1; + + /** + * A node array of the dictionary, containing several PtNodes. + * + * A PtNodeArray is but an ordered array of PtNodes, which essentially contain all the + * real information. + * This class also contains fields to cache size and address, to help with binary + * generation. + */ + public static final class PtNodeArray { + ArrayList<PtNode> mData; + // To help with binary generation + int mCachedSize = Integer.MIN_VALUE; + // mCachedAddressBefore/AfterUpdate are helpers for binary dictionary generation. They + // always hold the same value except between dictionary address compression, during which + // the update process needs to know about both values at the same time. Updating will + // update the AfterUpdate value, and the code will move them to BeforeUpdate before + // the next update pass. + int mCachedAddressBeforeUpdate = Integer.MIN_VALUE; + int mCachedAddressAfterUpdate = Integer.MIN_VALUE; + int mCachedParentAddress = 0; + + public PtNodeArray() { + mData = new ArrayList<>(); + } + public PtNodeArray(ArrayList<PtNode> data) { + Collections.sort(data, PTNODE_COMPARATOR); + mData = data; + } + } + + /** + * PtNode is a group of characters, with probability information, shortcut targets, bigrams, + * and children (Pt means Patricia Trie). + * + * This is the central class of the in-memory representation. A PtNode is what can + * be seen as a traditional "trie node", except it can hold several characters at the + * same time. A PtNode essentially represents one or several characters in the middle + * of the trie tree; as such, it can be a terminal, and it can have children. + * In this in-memory representation, whether the PtNode is a terminal or not is represented + * by mProbabilityInfo. The PtNode is a terminal when the mProbabilityInfo is not null and the + * PtNode is not a terminal when the mProbabilityInfo is null. A terminal may have non-null + * shortcuts and/or bigrams, but a non-terminal may not. Moreover, children, if present, + * are non-null. + */ + public static final class PtNode { + private static final int NOT_A_TERMINAL = -1; + final int mChars[]; + ArrayList<WeightedString> mShortcutTargets; + ArrayList<WeightedString> mBigrams; + // null == mProbabilityInfo indicates this is not a terminal. + ProbabilityInfo mProbabilityInfo; + int mTerminalId; // NOT_A_TERMINAL == mTerminalId indicates this is not a terminal. + PtNodeArray mChildren; + boolean mIsNotAWord; // Only a shortcut + boolean mIsBlacklistEntry; + // mCachedSize and mCachedAddressBefore/AfterUpdate are helpers for binary dictionary + // generation. Before and After always hold the same value except during dictionary + // address compression, where the update process needs to know about both values at the + // same time. Updating will update the AfterUpdate value, and the code will move them + // to BeforeUpdate before the next update pass. + // The update process does not need two versions of mCachedSize. + int mCachedSize; // The size, in bytes, of this PtNode. + int mCachedAddressBeforeUpdate; // The address of this PtNode (before update) + int mCachedAddressAfterUpdate; // The address of this PtNode (after update) + + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo, + final boolean isNotAWord, final boolean isBlacklistEntry) { + mChars = chars; + mProbabilityInfo = probabilityInfo; + mTerminalId = probabilityInfo == null ? NOT_A_TERMINAL : probabilityInfo.mProbability; + mShortcutTargets = shortcutTargets; + mBigrams = bigrams; + mChildren = null; + mIsNotAWord = isNotAWord; + mIsBlacklistEntry = isBlacklistEntry; + } + + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo, + final boolean isNotAWord, final boolean isBlacklistEntry, + final PtNodeArray children) { + mChars = chars; + mProbabilityInfo = probabilityInfo; + mShortcutTargets = shortcutTargets; + mBigrams = bigrams; + mChildren = children; + mIsNotAWord = isNotAWord; + mIsBlacklistEntry = isBlacklistEntry; + } + + public void addChild(PtNode n) { + if (null == mChildren) { + mChildren = new PtNodeArray(); + } + mChildren.mData.add(n); + } + + public int getTerminalId() { + return mTerminalId; + } + + public boolean isTerminal() { + return mProbabilityInfo != null; + } + + public int getProbability() { + if (isTerminal()) { + return mProbabilityInfo.mProbability; + } else { + return NOT_A_TERMINAL; + } + } + + public boolean getIsNotAWord() { + return mIsNotAWord; + } + + public boolean getIsBlacklistEntry() { + return mIsBlacklistEntry; + } + + public ArrayList<WeightedString> getShortcutTargets() { + // We don't want write permission to escape outside the package, so we return a copy + if (null == mShortcutTargets) return null; + final ArrayList<WeightedString> copyOfShortcutTargets = + new ArrayList<>(mShortcutTargets); + return copyOfShortcutTargets; + } + + public ArrayList<WeightedString> getBigrams() { + // We don't want write permission to escape outside the package, so we return a copy + if (null == mBigrams) return null; + final ArrayList<WeightedString> copyOfBigrams = new ArrayList<>(mBigrams); + return copyOfBigrams; + } + + public boolean hasSeveralChars() { + assert(mChars.length > 0); + return 1 < mChars.length; + } + + /** + * Adds a word to the bigram list. Updates the probability information if the word already + * exists. + */ + public void addBigram(final String word, final ProbabilityInfo probabilityInfo) { + if (mBigrams == null) { + mBigrams = new ArrayList<>(); + } + WeightedString bigram = getBigram(word); + if (bigram != null) { + bigram.mProbabilityInfo = probabilityInfo; + } else { + bigram = new WeightedString(word, probabilityInfo); + mBigrams.add(bigram); + } + } + + /** + * Gets the shortcut target for the given word. Returns null if the word is not in the + * shortcut list. + */ + public WeightedString getShortcut(final String word) { + // TODO: Don't do a linear search + if (mShortcutTargets != null) { + final int size = mShortcutTargets.size(); + for (int i = 0; i < size; ++i) { + WeightedString shortcut = mShortcutTargets.get(i); + if (shortcut.mWord.equals(word)) { + return shortcut; + } + } + } + return null; + } + + /** + * Gets the bigram for the given word. + * Returns null if the word is not in the bigrams list. + */ + public WeightedString getBigram(final String word) { + // TODO: Don't do a linear search + if (mBigrams != null) { + final int size = mBigrams.size(); + for (int i = 0; i < size; ++i) { + WeightedString bigram = mBigrams.get(i); + if (bigram.mWord.equals(word)) { + return bigram; + } + } + } + return null; + } + + /** + * Updates the PtNode with the given properties. Adds the shortcut and bigram lists to + * the existing ones if any. Note: unigram, bigram, and shortcut frequencies are only + * updated if they are higher than the existing ones. + */ + private void update(final ProbabilityInfo probabilityInfo, + final ArrayList<WeightedString> shortcutTargets, + final ArrayList<WeightedString> bigrams, + final boolean isNotAWord, final boolean isBlacklistEntry) { + mProbabilityInfo = ProbabilityInfo.max(mProbabilityInfo, probabilityInfo); + if (shortcutTargets != null) { + if (mShortcutTargets == null) { + mShortcutTargets = shortcutTargets; + } else { + final int size = shortcutTargets.size(); + for (int i = 0; i < size; ++i) { + final WeightedString shortcut = shortcutTargets.get(i); + final WeightedString existingShortcut = getShortcut(shortcut.mWord); + if (existingShortcut == null) { + mShortcutTargets.add(shortcut); + } else { + existingShortcut.mProbabilityInfo = ProbabilityInfo.max( + existingShortcut.mProbabilityInfo, shortcut.mProbabilityInfo); + } + } + } + } + if (bigrams != null) { + if (mBigrams == null) { + mBigrams = bigrams; + } else { + final int size = bigrams.size(); + for (int i = 0; i < size; ++i) { + final WeightedString bigram = bigrams.get(i); + final WeightedString existingBigram = getBigram(bigram.mWord); + if (existingBigram == null) { + mBigrams.add(bigram); + } else { + existingBigram.mProbabilityInfo = ProbabilityInfo.max( + existingBigram.mProbabilityInfo, bigram.mProbabilityInfo); + } + } + } + } + mIsNotAWord = isNotAWord; + mIsBlacklistEntry = isBlacklistEntry; + } + } + + public final DictionaryOptions mOptions; + public final PtNodeArray mRootNodeArray; + + public FusionDictionary(final PtNodeArray rootNodeArray, final DictionaryOptions options) { + mRootNodeArray = rootNodeArray; + mOptions = options; + } + + public void addOptionAttribute(final String key, final String value) { + mOptions.mAttributes.put(key, value); + } + + /** + * Helper method to convert a String to an int array. + */ + static int[] getCodePoints(final String word) { + // TODO: this is a copy-paste of the old contents of StringUtils.toCodePointArray, + // which is not visible from the makedict package. Factor this code. + final int length = word.length(); + if (length <= 0) return new int[] {}; + final char[] characters = word.toCharArray(); + final int[] codePoints = new int[Character.codePointCount(characters, 0, length)]; + int codePoint = Character.codePointAt(characters, 0); + int dsti = 0; + for (int srci = Character.charCount(codePoint); + srci < length; srci += Character.charCount(codePoint), ++dsti) { + codePoints[dsti] = codePoint; + codePoint = Character.codePointAt(characters, srci); + } + codePoints[dsti] = codePoint; + return codePoints; + } + + /** + * Helper method to add a word as a string. + * + * This method adds a word to the dictionary with the given frequency. Optional + * lists of bigrams and shortcuts can be passed here. For each word inside, + * they will be added to the dictionary as necessary. + * + * @param word the word to add. + * @param probabilityInfo probability information of the word. + * @param shortcutTargets a list of shortcut targets for this word, or null. + * @param isNotAWord true if this should not be considered a word (e.g. shortcut only) + */ + public void add(final String word, final ProbabilityInfo probabilityInfo, + final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord) { + add(getCodePoints(word), probabilityInfo, shortcutTargets, isNotAWord, + false /* isBlacklistEntry */); + } + + /** + * Helper method to add a blacklist entry as a string. + * + * @param word the word to add as a blacklist entry. + * @param shortcutTargets a list of shortcut targets for this word, or null. + * @param isNotAWord true if this is not a word for spellcheking purposes (shortcut only or so) + */ + public void addBlacklistEntry(final String word, + final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord) { + add(getCodePoints(word), new ProbabilityInfo(0), shortcutTargets, isNotAWord, + true /* isBlacklistEntry */); + } + + /** + * Sanity check for a PtNode array. + * + * This method checks that all PtNodes in a node array are ordered as expected. + * If they are, nothing happens. If they aren't, an exception is thrown. + */ + private void checkStack(PtNodeArray ptNodeArray) { + ArrayList<PtNode> stack = ptNodeArray.mData; + int lastValue = -1; + for (int i = 0; i < stack.size(); ++i) { + int currentValue = stack.get(i).mChars[0]; + if (currentValue <= lastValue) + throw new RuntimeException("Invalid stack"); + else + lastValue = currentValue; + } + } + + /** + * Helper method to add a new bigram to the dictionary. + * + * @param word0 the previous word of the context + * @param word1 the next word of the context + * @param probabilityInfo the bigram probability info + */ + public void setBigram(final String word0, final String word1, + final ProbabilityInfo probabilityInfo) { + PtNode ptNode0 = findWordInTree(mRootNodeArray, word0); + if (ptNode0 != null) { + final PtNode ptNode1 = findWordInTree(mRootNodeArray, word1); + if (ptNode1 == null) { + add(getCodePoints(word1), new ProbabilityInfo(0), null, false /* isNotAWord */, + false /* isBlacklistEntry */); + // The PtNode for the first word may have moved by the above insertion, + // if word1 and word2 share a common stem that happens not to have been + // a cutting point until now. In this case, we need to refresh ptNode. + ptNode0 = findWordInTree(mRootNodeArray, word0); + } + ptNode0.addBigram(word1, probabilityInfo); + } else { + throw new RuntimeException("First word of bigram not found " + word0); + } + } + + /** + * Add a word to this dictionary. + * + * The shortcuts, if any, have to be in the dictionary already. If they aren't, + * an exception is thrown. + * + * @param word the word, as an int array. + * @param probabilityInfo the probability information of the word. + * @param shortcutTargets an optional list of shortcut targets for this word (null if none). + * @param isNotAWord true if this is not a word for spellcheking purposes (shortcut only or so) + * @param isBlacklistEntry true if this is a blacklisted word, false otherwise + */ + private void add(final int[] word, final ProbabilityInfo probabilityInfo, + final ArrayList<WeightedString> shortcutTargets, + final boolean isNotAWord, final boolean isBlacklistEntry) { + assert(probabilityInfo.mProbability <= FormatSpec.MAX_TERMINAL_FREQUENCY); + if (word.length >= Constants.DICTIONARY_MAX_WORD_LENGTH) { + MakedictLog.w("Ignoring a word that is too long: word.length = " + word.length); + return; + } + + PtNodeArray currentNodeArray = mRootNodeArray; + int charIndex = 0; + + PtNode currentPtNode = null; + int differentCharIndex = 0; // Set by the loop to the index of the char that differs + int nodeIndex = findIndexOfChar(mRootNodeArray, word[charIndex]); + while (CHARACTER_NOT_FOUND_INDEX != nodeIndex) { + currentPtNode = currentNodeArray.mData.get(nodeIndex); + differentCharIndex = compareCharArrays(currentPtNode.mChars, word, charIndex); + if (ARRAYS_ARE_EQUAL != differentCharIndex + && differentCharIndex < currentPtNode.mChars.length) break; + if (null == currentPtNode.mChildren) break; + charIndex += currentPtNode.mChars.length; + if (charIndex >= word.length) break; + currentNodeArray = currentPtNode.mChildren; + nodeIndex = findIndexOfChar(currentNodeArray, word[charIndex]); + } + + if (CHARACTER_NOT_FOUND_INDEX == nodeIndex) { + // No node at this point to accept the word. Create one. + final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); + final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length), + shortcutTargets, null /* bigrams */, probabilityInfo, isNotAWord, + isBlacklistEntry); + currentNodeArray.mData.add(insertionIndex, newPtNode); + if (DBG) checkStack(currentNodeArray); + } else { + // There is a word with a common prefix. + if (differentCharIndex == currentPtNode.mChars.length) { + if (charIndex + differentCharIndex >= word.length) { + // The new word is a prefix of an existing word, but the node on which it + // should end already exists as is. Since the old PtNode was not a terminal, + // make it one by filling in its frequency and other attributes + currentPtNode.update(probabilityInfo, shortcutTargets, null, isNotAWord, + isBlacklistEntry); + } else { + // The new word matches the full old word and extends past it. + // We only have to create a new node and add it to the end of this. + final PtNode newNode = new PtNode( + Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), + shortcutTargets, null /* bigrams */, probabilityInfo, + isNotAWord, isBlacklistEntry); + currentPtNode.mChildren = new PtNodeArray(); + currentPtNode.mChildren.mData.add(newNode); + } + } else { + if (0 == differentCharIndex) { + // Exact same word. Update the frequency if higher. This will also add the + // new shortcuts to the existing shortcut list if it already exists. + currentPtNode.update(probabilityInfo, shortcutTargets, null, + currentPtNode.mIsNotAWord && isNotAWord, + currentPtNode.mIsBlacklistEntry || isBlacklistEntry); + } else { + // Partial prefix match only. We have to replace the current node with a node + // containing the current prefix and create two new ones for the tails. + PtNodeArray newChildren = new PtNodeArray(); + final PtNode newOldWord = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex, + currentPtNode.mChars.length), currentPtNode.mShortcutTargets, + currentPtNode.mBigrams, currentPtNode.mProbabilityInfo, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry, + currentPtNode.mChildren); + newChildren.mData.add(newOldWord); + + final PtNode newParent; + if (charIndex + differentCharIndex >= word.length) { + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), + shortcutTargets, null /* bigrams */, probabilityInfo, + isNotAWord, isBlacklistEntry, newChildren); + } else { + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), + null /* shortcutTargets */, null /* bigrams */, + null /* probabilityInfo */, false /* isNotAWord */, + false /* isBlacklistEntry */, newChildren); + final PtNode newWord = new PtNode(Arrays.copyOfRange(word, + charIndex + differentCharIndex, word.length), + shortcutTargets, null /* bigrams */, probabilityInfo, + isNotAWord, isBlacklistEntry); + final int addIndex = word[charIndex + differentCharIndex] + > currentPtNode.mChars[differentCharIndex] ? 1 : 0; + newChildren.mData.add(addIndex, newWord); + } + currentNodeArray.mData.set(nodeIndex, newParent); + } + if (DBG) checkStack(currentNodeArray); + } + } + } + + private static int ARRAYS_ARE_EQUAL = 0; + + /** + * Custom comparison of two int arrays taken to contain character codes. + * + * This method compares the two arrays passed as an argument in a lexicographic way, + * with an offset in the dst string. + * This method does NOT test for the first character. It is taken to be equal. + * I repeat: this method starts the comparison at 1 <> dstOffset + 1. + * The index where the strings differ is returned. ARRAYS_ARE_EQUAL = 0 is returned if the + * strings are equal. This works BECAUSE we don't look at the first character. + * + * @param src the left-hand side string of the comparison. + * @param dst the right-hand side string of the comparison. + * @param dstOffset the offset in the right-hand side string. + * @return the index at which the strings differ, or ARRAYS_ARE_EQUAL = 0 if they don't. + */ + private static int compareCharArrays(final int[] src, final int[] dst, int dstOffset) { + // We do NOT test the first char, because we come from a method that already + // tested it. + for (int i = 1; i < src.length; ++i) { + if (dstOffset + i >= dst.length) return i; + if (src[i] != dst[dstOffset + i]) return i; + } + if (dst.length > src.length) return src.length; + return ARRAYS_ARE_EQUAL; + } + + /** + * Helper class that compares and sorts two PtNodes according to their + * first element only. I repeat: ONLY the first element is considered, the rest + * is ignored. + * This comparator imposes orderings that are inconsistent with equals. + */ + static private final class PtNodeComparator implements java.util.Comparator<PtNode> { + @Override + public int compare(PtNode p1, PtNode p2) { + if (p1.mChars[0] == p2.mChars[0]) return 0; + return p1.mChars[0] < p2.mChars[0] ? -1 : 1; + } + } + final static private PtNodeComparator PTNODE_COMPARATOR = new PtNodeComparator(); + + /** + * Finds the insertion index of a character within a node array. + */ + private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { + final ArrayList<PtNode> data = nodeArray.mData; + final PtNode reference = new PtNode(new int[] { character }, + null /* shortcutTargets */, null /* bigrams */, null /* probabilityInfo */, + false /* isNotAWord */, false /* isBlacklistEntry */); + int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR); + return result >= 0 ? result : -result - 1; + } + + /** + * Find the index of a char in a node array, if it exists. + * + * @param nodeArray the node array to search in. + * @param character the character to search for. + * @return the position of the character if it's there, or CHARACTER_NOT_FOUND_INDEX = -1 else. + */ + private static int findIndexOfChar(final PtNodeArray nodeArray, int character) { + final int insertionIndex = findInsertionIndex(nodeArray, character); + if (nodeArray.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; + return character == nodeArray.mData.get(insertionIndex).mChars[0] ? insertionIndex + : CHARACTER_NOT_FOUND_INDEX; + } + + /** + * Helper method to find a word in a given branch. + */ + public static PtNode findWordInTree(PtNodeArray nodeArray, final String string) { + int index = 0; + final StringBuilder checker = DBG ? new StringBuilder() : null; + final int[] codePoints = getCodePoints(string); + + PtNode currentPtNode; + do { + int indexOfGroup = findIndexOfChar(nodeArray, codePoints[index]); + if (CHARACTER_NOT_FOUND_INDEX == indexOfGroup) return null; + currentPtNode = nodeArray.mData.get(indexOfGroup); + + if (codePoints.length - index < currentPtNode.mChars.length) return null; + int newIndex = index; + while (newIndex < codePoints.length && newIndex - index < currentPtNode.mChars.length) { + if (currentPtNode.mChars[newIndex - index] != codePoints[newIndex]) return null; + newIndex++; + } + index = newIndex; + + if (DBG) { + checker.append(new String(currentPtNode.mChars, 0, currentPtNode.mChars.length)); + } + if (index < codePoints.length) { + nodeArray = currentPtNode.mChildren; + } + } while (null != nodeArray && index < codePoints.length); + + if (index < codePoints.length) return null; + if (!currentPtNode.isTerminal()) return null; + if (DBG && !string.equals(checker.toString())) return null; + return currentPtNode; + } + + /** + * Helper method to find out whether a word is in the dict or not. + */ + public boolean hasWord(final String s) { + if (null == s || "".equals(s)) { + throw new RuntimeException("Can't search for a null or empty string"); + } + return null != findWordInTree(mRootNodeArray, s); + } + + /** + * Recursively count the number of PtNodes in a given branch of the trie. + * + * @param nodeArray the parent node. + * @return the number of PtNodes in all the branch under this node. + */ + public static int countPtNodes(final PtNodeArray nodeArray) { + final int nodeSize = nodeArray.mData.size(); + int size = nodeSize; + for (int i = nodeSize - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countPtNodes(ptNode.mChildren); + } + return size; + } + + /** + * Iterator to walk through a dictionary. + * + * This is purely for convenience. + */ + public static final class DictionaryIterator implements Iterator<WordProperty> { + private static final class Position { + public Iterator<PtNode> pos; + public int length; + public Position(ArrayList<PtNode> ptNodes) { + pos = ptNodes.iterator(); + length = 0; + } + } + final StringBuilder mCurrentString; + final LinkedList<Position> mPositions; + + public DictionaryIterator(ArrayList<PtNode> ptRoot) { + mCurrentString = new StringBuilder(); + mPositions = new LinkedList<>(); + final Position rootPos = new Position(ptRoot); + mPositions.add(rootPos); + } + + @Override + public boolean hasNext() { + for (Position p : mPositions) { + if (p.pos.hasNext()) { + return true; + } + } + return false; + } + + @Override + public WordProperty next() { + Position currentPos = mPositions.getLast(); + mCurrentString.setLength(currentPos.length); + + do { + if (currentPos.pos.hasNext()) { + final PtNode currentPtNode = currentPos.pos.next(); + currentPos.length = mCurrentString.length(); + for (int i : currentPtNode.mChars) { + mCurrentString.append(Character.toChars(i)); + } + if (null != currentPtNode.mChildren) { + currentPos = new Position(currentPtNode.mChildren.mData); + currentPos.length = mCurrentString.length(); + mPositions.addLast(currentPos); + } + if (currentPtNode.isTerminal()) { + return new WordProperty(mCurrentString.toString(), + currentPtNode.mProbabilityInfo, + currentPtNode.mShortcutTargets, currentPtNode.mBigrams, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry); + } + } else { + mPositions.removeLast(); + currentPos = mPositions.getLast(); + mCurrentString.setLength(mPositions.getLast().length); + } + } while (true); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Unsupported yet"); + } + + } + + /** + * Method to return an iterator. + * + * This method enables Java's enhanced for loop. With this you can have a FusionDictionary x + * and say : for (Word w : x) {} + */ + @Override + public Iterator<WordProperty> iterator() { + return new DictionaryIterator(mRootNodeArray.mData); + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/MakedictLog.java b/tests/src/com/android/inputmethod/latin/makedict/MakedictLog.java new file mode 100644 index 000000000..7eccff2b4 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/MakedictLog.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.latin.makedict; + +/** + * Wrapper to redirect log events to the right output medium. + */ +public class MakedictLog { + public static final boolean DBG = true; + + private static void print(String message) { + System.out.println(message); + } + + public static void d(String message) { + print(message); + } + + public static void i(String message) { + print(message); + } + + public static void w(String message) { + print(message); + } + + public static void e(String message) { + print(message); + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/PendingAttribute.java b/tests/src/com/android/inputmethod/latin/makedict/PendingAttribute.java new file mode 100644 index 000000000..70e24cc98 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/PendingAttribute.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +/** + * A not-yet-resolved attribute. + * + * An attribute is either a bigram or a shortcut. + * All instances of this class are always immutable. + */ +public final class PendingAttribute { + public final int mFrequency; + public final int mAddress; + public PendingAttribute(final int frequency, final int address) { + mFrequency = frequency; + mAddress = address; + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java b/tests/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java new file mode 100644 index 000000000..862e8c101 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import java.util.ArrayList; + +/** + * Raw PtNode info straight out of a file. This will contain numbers for addresses. + */ +public final class PtNodeInfo { + public final int mOriginalAddress; + public final int mEndAddress; + public final int mFlags; + public final int[] mCharacters; + public final ProbabilityInfo mProbabilityInfo; + public final int mChildrenAddress; + public final ArrayList<WeightedString> mShortcutTargets; + public final ArrayList<PendingAttribute> mBigrams; + + public PtNodeInfo(final int originalAddress, final int endAddress, final int flags, + final int[] characters, final ProbabilityInfo probabilityInfo, + final int childrenAddress, final ArrayList<WeightedString> shortcutTargets, + final ArrayList<PendingAttribute> bigrams) { + mOriginalAddress = originalAddress; + mEndAddress = endAddress; + mFlags = flags; + mCharacters = characters; + mProbabilityInfo = probabilityInfo; + mChildrenAddress = childrenAddress; + mShortcutTargets = shortcutTargets; + mBigrams = bigrams; + } + + public boolean isTerminal() { + return mProbabilityInfo != null; + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java deleted file mode 100644 index aeb8552bd..000000000 --- a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.LargeTest; -import android.util.Log; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Random; - -/** - * Unit tests for SparseTable. - */ -@LargeTest -public class SparseTableTests extends AndroidTestCase { - private static final String TAG = SparseTableTests.class.getSimpleName(); - - private final Random mRandom; - private final ArrayList<Integer> mRandomIndex; - - private static final int DEFAULT_SIZE = 10000; - private static final int BLOCK_SIZE = 8; - - public SparseTableTests() { - this(System.currentTimeMillis(), DEFAULT_SIZE); - } - - public SparseTableTests(final long seed, final int tableSize) { - super(); - Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize); - mRandom = new Random(seed); - mRandomIndex = new ArrayList<Integer>(tableSize); - for (int i = 0; i < tableSize; ++i) { - mRandomIndex.add(SparseTable.NOT_EXIST); - } - } - - public void testSet() { - final SparseTable table = new SparseTable(16, BLOCK_SIZE, 1); - table.set(0, 3, 6); - table.set(0, 8, 16); - for (int i = 0; i < 16; ++i) { - if (i == 3 || i == 8) { - assertEquals(i * 2, table.get(0, i)); - } else { - assertEquals(SparseTable.NOT_EXIST, table.get(0, i)); - } - } - } - - private void generateRandomIndex(final int size, final int prop) { - for (int i = 0; i < size; ++i) { - if (mRandom.nextInt(100) < prop) { - mRandomIndex.set(i, mRandom.nextInt()); - } else { - mRandomIndex.set(i, SparseTable.NOT_EXIST); - } - } - } - - private void runTestRandomSet() { - final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, 1); - int elementCount = 0; - for (int i = 0; i < DEFAULT_SIZE; ++i) { - if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { - table.set(0, i, mRandomIndex.get(i)); - elementCount++; - } - } - - Log.d(TAG, "table size = " + table.getLookupTableSize() + " + " - + table.getContentTableSize()); - Log.d(TAG, "the table has " + elementCount + " elements"); - for (int i = 0; i < DEFAULT_SIZE; ++i) { - assertEquals(table.get(0, i), (int)mRandomIndex.get(i)); - } - - // flush and reload - OutputStream lookupOutStream = null; - OutputStream contentOutStream = null; - try { - final File lookupIndexFile = File.createTempFile("testRandomSet", ".small"); - final File contentFile = File.createTempFile("testRandomSet", ".big"); - lookupOutStream = new FileOutputStream(lookupIndexFile); - contentOutStream = new FileOutputStream(contentFile); - table.write(lookupOutStream, new OutputStream[] { contentOutStream }); - lookupOutStream.flush(); - contentOutStream.flush(); - final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, - new File[] { contentFile }, BLOCK_SIZE); - for (int i = 0; i < DEFAULT_SIZE; ++i) { - assertEquals(table.get(0, i), newTable.get(0, i)); - } - } catch (IOException e) { - Log.d(TAG, "IOException while flushing and realoding", e); - } finally { - if (lookupOutStream != null) { - try { - lookupOutStream.close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing the stream", e); - } - } - if (contentOutStream != null) { - try { - contentOutStream.close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing contentStream.", e); - } - } - } - } - - public void testRandomSet() { - for (int i = 0; i <= 100; i += 10) { - generateRandomIndex(DEFAULT_SIZE, i); - runTestRandomSet(); - } - } - - public void testMultipleContents() { - final int numOfContents = 5; - generateRandomIndex(DEFAULT_SIZE, 20); - final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, numOfContents); - for (int i = 0; i < mRandomIndex.size(); ++i) { - if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { - for (int j = 0; j < numOfContents; ++j) { - table.set(j, i, mRandomIndex.get(i)); - } - } - } - - OutputStream lookupOutStream = null; - OutputStream[] contentsOutStream = new OutputStream[numOfContents]; - try { - final File lookupIndexFile = File.createTempFile("testMultipleContents", "small"); - lookupOutStream = new FileOutputStream(lookupIndexFile); - final File[] contentFiles = new File[numOfContents]; - for (int i = 0; i < numOfContents; ++i) { - contentFiles[i] = File.createTempFile("testMultipleContents", "big" + i); - contentsOutStream[i] = new FileOutputStream(contentFiles[i]); - } - table.write(lookupOutStream, contentsOutStream); - lookupOutStream.flush(); - for (int i = 0; i < numOfContents; ++i) { - contentsOutStream[i].flush(); - } - final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, contentFiles, - BLOCK_SIZE); - for (int i = 0; i < numOfContents; ++i) { - for (int j = 0; j < DEFAULT_SIZE; ++j) { - assertEquals(table.get(i, j), newTable.get(i, j)); - } - } - } catch (IOException e) { - Log.d(TAG, "IOException while flushing and reloading", e); - } finally { - if (lookupOutStream != null) { - try { - lookupOutStream.close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing the stream", e); - } - } - for (int i = 0; i < numOfContents; ++i) { - if (contentsOutStream[i] != null) { - try { - contentsOutStream[i].close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing the stream.", e); - } - } - } - } - } -} diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java new file mode 100644 index 000000000..65b84d5f7 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * An implementation of DictDecoder for version 2 binary dictionary. + */ +// TODO: Separate logics that are used only for testing. +@UsedForTesting +public class Ver2DictDecoder extends AbstractDictDecoder { + /** + * A utility class for reading a PtNode. + */ + protected static class PtNodeReader { + private static ProbabilityInfo readProbabilityInfo(final DictBuffer dictBuffer) { + // Ver2 dicts don't contain historical information. + return new ProbabilityInfo(dictBuffer.readUnsignedByte()); + } + + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, + final int ptNodeFlags) { + switch (ptNodeFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigramAddresses(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + bigramAddress += sign * dictBuffer.readUnsignedInt24(); + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return readLength; + } + } + + protected final File mDictionaryBinaryFile; + protected final long mOffset; + protected final long mLength; + // TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now + // used only for testing. + private final DictionaryBufferFactory mBufferFactory; + protected DictBuffer mDictBuffer; + + @UsedForTesting + /* package */ Ver2DictDecoder(final File file, final long offset, final long length, + final int factoryFlag) { + mDictionaryBinaryFile = file; + mOffset = offset; + mLength = length; + mDictBuffer = null; + if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { + mBufferFactory = new DictionaryBufferFromByteArrayFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); + } else { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } + } + + /* package */ Ver2DictDecoder(final File file, final long offset, final long length, + final DictionaryBufferFactory factory) { + mDictionaryBinaryFile = file; + mOffset = offset; + mLength = length; + mBufferFactory = factory; + } + + @Override + public void openDictBuffer() throws FileNotFoundException, IOException { + mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); + } + + @Override + public boolean isDictBufferOpen() { + return mDictBuffer != null; + } + + /* package */ DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @UsedForTesting + /* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException { + openDictBuffer(); + return getDictBuffer(); + } + + @Override + public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { + // dictType is not being used in dicttool. Passing an empty string. + final BinaryDictionary binaryDictionary = new BinaryDictionary( + mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength, + true /* useFullEditDistance */, null /* locale */, "" /* dictType */, + false /* isUpdatable */); + final DictionaryHeader header = binaryDictionary.getHeader(); + binaryDictionary.close(); + if (header == null) { + throw new IOException("Cannot read the dictionary header."); + } + if (header.mFormatOptions.mVersion != FormatSpec.VERSION2) { + throw new UnsupportedFormatException("File header has a wrong version : " + + header.mFormatOptions.mVersion); + } + if (!isDictBufferOpen()) { + openDictBuffer(); + } + // Advance buffer reading position to the head of dictionary body. + setPosition(header.mBodyOffset); + return header; + } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public PtNodeInfo readPtNode(final int ptNodePos) { + int addressPointer = ptNodePos; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (FormatSpec.INVALID_CHARACTER != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final ProbabilityInfo probabilityInfo; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + probabilityInfo = PtNodeReader.readProbabilityInfo(mDictBuffer); + addressPointer += FormatSpec.PTNODE_FREQUENCY_SIZE; + } else { + probabilityInfo = null; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags); + final ArrayList<WeightedString> shortcutTargets; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + // readShortcut will add shortcuts to shortcutTargets. + shortcutTargets = new ArrayList<>(); + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } else { + shortcutTargets = null; + } + + final ArrayList<PendingAttribute> bigrams; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<>(); + addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, + addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() + + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); + } + } else { + bigrams = null; + } + return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo, + childrenAddress, shortcutTargets, bigrams); + } + + @Override + public FusionDictionary readDictionaryBinary(final boolean deleteDictIfBroken) + throws FileNotFoundException, IOException, UnsupportedFormatException { + // dictType is not being used in dicttool. Passing an empty string. + final BinaryDictionary binaryDictionary = new BinaryDictionary( + mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */, + mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */, + null /* locale */, "" /* dictType */, false /* isUpdatable */); + final DictionaryHeader header = readHeader(); + final FusionDictionary fusionDict = + new FusionDictionary(new FusionDictionary.PtNodeArray(), header.mDictionaryOptions); + int token = 0; + final ArrayList<WordProperty> wordProperties = new ArrayList<>(); + do { + final BinaryDictionary.GetNextWordPropertyResult result = + binaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + if (wordProperty == null) { + binaryDictionary.close(); + if (deleteDictIfBroken) { + mDictionaryBinaryFile.delete(); + } + return null; + } + wordProperties.add(wordProperty); + token = result.mNextToken; + } while (token != 0); + + // Insert unigrams into the fusion dictionary. + for (final WordProperty wordProperty : wordProperties) { + if (wordProperty.mIsBlacklistEntry) { + fusionDict.addBlacklistEntry(wordProperty.mWord, wordProperty.mShortcutTargets, + wordProperty.mIsNotAWord); + } else { + fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, + wordProperty.mShortcutTargets, wordProperty.mIsNotAWord); + } + } + // Insert bigrams into the fusion dictionary. + for (final WordProperty wordProperty : wordProperties) { + if (wordProperty.mBigrams == null) { + continue; + } + final String word0 = wordProperty.mWord; + for (final WeightedString bigram : wordProperty.mBigrams) { + fusionDict.setBigram(word0, bigram.mWord, bigram.mProbabilityInfo); + } + } + binaryDictionary.close(); + return fusionDict; + } + + @Override + public void setPosition(int newPos) { + mDictBuffer.position(newPos); + } + + @Override + public int getPosition() { + return mDictBuffer.position(); + } + + @Override + public int readPtNodeCount() { + return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java index 9611599b9..3882c2c55 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoderTests.java @@ -32,10 +32,10 @@ import java.io.FileOutputStream; import java.io.IOException; /** - * Unit tests for Ver3DictDecoder + * Unit tests for Ver2DictDecoder */ -public class Ver3DictDecoderTests extends AndroidTestCase { - private static final String TAG = Ver3DictDecoderTests.class.getSimpleName(); +public class Ver2DictDecoderTests extends AndroidTestCase { + private static final String TAG = Ver2DictDecoderTests.class.getSimpleName(); private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; @@ -58,7 +58,6 @@ public class Ver3DictDecoderTests extends AndroidTestCase { } } - @SuppressWarnings("null") public void runTestOpenBuffer(final String testName, final DictionaryBufferFactory factory) { File testFile = null; try { @@ -68,7 +67,8 @@ public class Ver3DictDecoderTests extends AndroidTestCase { } assertNotNull(testFile); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory); + final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(), + factory); try { dictDecoder.openDictBuffer(); } catch (Exception e) { @@ -101,7 +101,6 @@ public class Ver3DictDecoderTests extends AndroidTestCase { new DictionaryBufferFromWritableByteBufferFactory()); } - @SuppressWarnings("null") public void runTestGetBuffer(final String testName, final DictionaryBufferFactory factory) { File testFile = null; try { @@ -110,7 +109,8 @@ public class Ver3DictDecoderTests extends AndroidTestCase { Log.e(TAG, "IOException while the creating temporary file", e); } - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory); + final Ver2DictDecoder dictDecoder = new Ver2DictDecoder(testFile, 0, testFile.length(), + factory); // the default return value of getBuffer() must be null. assertNull("the default return value of getBuffer() is not null", diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java new file mode 100644 index 000000000..a286190cb --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * An implementation of DictEncoder for version 2 binary dictionary. + */ +@UsedForTesting +public class Ver2DictEncoder implements DictEncoder { + + private final File mDictFile; + private OutputStream mOutStream; + private byte[] mBuffer; + private int mPosition; + + @UsedForTesting + public Ver2DictEncoder(final File dictFile) { + mDictFile = dictFile; + mOutStream = null; + mBuffer = null; + } + + // This constructor is used only by BinaryDictOffdeviceUtilsTests. + // If you want to use this in the production code, you should consider keeping consistency of + // the interface of Ver3DictDecoder by using factory. + @UsedForTesting + public Ver2DictEncoder(final OutputStream outStream) { + mDictFile = null; + mOutStream = outStream; + } + + private void openStream() throws FileNotFoundException { + mOutStream = new FileOutputStream(mDictFile); + } + + private void close() throws IOException { + if (mOutStream != null) { + mOutStream.close(); + mOutStream = null; + } + } + + @Override + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion > FormatSpec.VERSION2) { + throw new UnsupportedFormatException( + "The given format options has wrong version number : " + + formatOptions.mVersion); + } + + if (mOutStream == null) { + openStream(); + } + BinaryDictEncoderUtils.writeDictionaryHeader(mOutStream, dict, formatOptions); + + // Addresses are limited to 3 bytes, but since addresses can be relative to each node + // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding + // the order of the PtNode arrays becomes a quite complicated problem, because though the + // dictionary itself does not have a size limit, each node array must still be within 16MB + // of all its children and parents. As long as this is ensured, the dictionary file may + // grow to any size. + + // Leave the choice of the optimal node order to the flattenTree function. + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes); + MakedictLog.i("Checking PtNode array..."); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + // Create a buffer that matches the final dictionary size. + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mBuffer = new byte[bufferSize]; + + MakedictLog.i("Writing file..."); + + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray); + } + if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes); + mOutStream.write(mBuffer, 0, mPosition); + + MakedictLog.i("Done"); + close(); + } + + @Override + public void setPosition(final int position) { + if (mBuffer == null || position < 0 || position >= mBuffer.length) return; + mPosition = position; + } + + @Override + public int getPosition() { + return mPosition; + } + + @Override + public void writePtNodeCount(final int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); + } + final int encodedPtNodeCount = (countSize == 2) ? + (ptNodeCount | FormatSpec.LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG) : ptNodeCount; + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, encodedPtNodeCount, + countSize); + } + + private void writePtNodeFlags(final PtNode ptNode) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos), + FormatSpec.PTNODE_FLAGS_SIZE); + } + + private void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) { + mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition); + if (hasSeveralChars) { + mBuffer[mPosition++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + private void writeFrequency(final int frequency) { + if (frequency >= 0) { + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, frequency, + FormatSpec.PTNODE_FREQUENCY_SIZE); + } + } + + private void writeChildrenPosition(final PtNode ptNode) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode); + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + childrenPos); + } + + /** + * Write a shortcut attributes list to mBuffer. + * + * @param shortcuts the shortcut attributes list. + */ + private void writeShortcuts(final ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mPosition; + mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.getProbability()); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord); + mPosition += shortcutShift; + } + final int shortcutByteSize = mPosition - indexOfShortcutByteSize; + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { + throw new RuntimeException("Shortcut list too large"); + } + BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, + FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + /** + * Write a bigram attributes list to mBuffer. + * + * @param bigrams the bigram attributes list. + * @param dict the dictionary the node array is a part of (for relative offsets). + */ + private void writeBigrams(final ArrayList<WeightedString> bigrams, + final FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.getProbability(); + final int offset = addressOfBigram + - (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.getProbability(), unigramFrequencyForThisWord, bigram.mWord); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(final int forwardLinkAddress) { + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, forwardLinkAddress, + FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final FusionDictionary dict) { + writePtNodeFlags(ptNode); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + writeFrequency(ptNode.getProbability()); + writeChildrenPosition(ptNode); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java new file mode 100644 index 000000000..5e8417ed6 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.utils.FileUtils; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; + +/** + * An implementation of binary dictionary decoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictDecoder extends AbstractDictDecoder { + final File mDictDirectory; + + @UsedForTesting + /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { + this(dictDirectory, null /* factory */); + } + + @UsedForTesting + /* package */ Ver4DictDecoder(final File dictDirectory, final DictionaryBufferFactory factory) { + mDictDirectory = dictDirectory; + + } + + @Override + public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { + // dictType is not being used in dicttool. Passing an empty string. + final BinaryDictionary binaryDictionary= new BinaryDictionary( + mDictDirectory.getAbsolutePath(), 0 /* offset */, 0 /* length */, + true /* useFullEditDistance */, null /* locale */, + "" /* dictType */, true /* isUpdatable */); + final DictionaryHeader header = binaryDictionary.getHeader(); + binaryDictionary.close(); + if (header == null) { + throw new IOException("Cannot read the dictionary header."); + } + return header; + } + + @Override + public FusionDictionary readDictionaryBinary(final boolean deleteDictIfBroken) + throws FileNotFoundException, IOException, UnsupportedFormatException { + // dictType is not being used in dicttool. Passing an empty string. + final BinaryDictionary binaryDictionary = new BinaryDictionary( + mDictDirectory.getAbsolutePath(), 0 /* offset */, 0 /* length */, + true /* useFullEditDistance */, null /* locale */, + "" /* dictType */, true /* isUpdatable */); + final DictionaryHeader header = readHeader(); + final FusionDictionary fusionDict = + new FusionDictionary(new FusionDictionary.PtNodeArray(), header.mDictionaryOptions); + int token = 0; + final ArrayList<WordProperty> wordProperties = new ArrayList<>(); + do { + final BinaryDictionary.GetNextWordPropertyResult result = + binaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + if (wordProperty == null) { + binaryDictionary.close(); + if (deleteDictIfBroken) { + FileUtils.deleteRecursively(mDictDirectory); + } + return null; + } + wordProperties.add(wordProperty); + token = result.mNextToken; + } while (token != 0); + + // Insert unigrams into the fusion dictionary. + for (final WordProperty wordProperty : wordProperties) { + if (wordProperty.mIsBlacklistEntry) { + fusionDict.addBlacklistEntry(wordProperty.mWord, wordProperty.mShortcutTargets, + wordProperty.mIsNotAWord); + } else { + fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, + wordProperty.mShortcutTargets, wordProperty.mIsNotAWord); + } + } + // Insert bigrams into the fusion dictionary. + for (final WordProperty wordProperty : wordProperties) { + if (wordProperty.mBigrams == null) { + continue; + } + final String word0 = wordProperty.mWord; + for (final WeightedString bigram : wordProperty.mBigrams) { + fusionDict.setBigram(word0, bigram.mWord, bigram.mProbabilityInfo); + } + } + binaryDictionary.close(); + return fusionDict; + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java new file mode 100644 index 000000000..76eaef431 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.PrevWordsInfo; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; +import com.android.inputmethod.latin.utils.LocaleUtils; + +import java.io.File; +import java.io.IOException; + +/** + * An implementation of DictEncoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictEncoder implements DictEncoder { + private final File mDictPlacedDir; + + @UsedForTesting + public Ver4DictEncoder(final File dictPlacedDir) { + mDictPlacedDir = dictPlacedDir; + } + + // TODO: This builds a FusionDictionary first and iterates it to add words to the binary + // dictionary. However, it is possible to just add words directly to the binary dictionary + // instead. + // In the long run, when we stop supporting version 2, FusionDictionary will become deprecated + // and we can remove it. Then we'll be able to just call BinaryDictionary directly. + @Override + public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion != FormatSpec.VERSION4) { + throw new UnsupportedFormatException("File header has a wrong version number : " + + formatOptions.mVersion); + } + if (!mDictPlacedDir.isDirectory()) { + throw new UnsupportedFormatException("Given path is not a directory."); + } + if (!BinaryDictionaryUtils.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(), + FormatSpec.VERSION4, LocaleUtils.constructLocaleFromString( + dict.mOptions.mAttributes.get(DictionaryHeader.DICTIONARY_LOCALE_KEY)), + dict.mOptions.mAttributes)) { + throw new IOException("Cannot create dictionary file : " + + mDictPlacedDir.getAbsolutePath()); + } + final BinaryDictionary binaryDict = new BinaryDictionary(mDictPlacedDir.getAbsolutePath(), + 0l, mDictPlacedDir.length(), true /* useFullEditDistance */, + LocaleUtils.constructLocaleFromString(dict.mOptions.mAttributes.get( + DictionaryHeader.DICTIONARY_LOCALE_KEY)), + Dictionary.TYPE_USER /* Dictionary type. Does not matter for us */, + true /* isUpdatable */); + if (!binaryDict.isValidDictionary()) { + // Somehow createEmptyDictFile returned true, but the file was not created correctly + throw new IOException("Cannot create dictionary file"); + } + for (final WordProperty wordProperty : dict) { + // TODO: switch to addMultipleDictionaryEntries when they support shortcuts + if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) { + if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(), + null /* shortcutTarget */, 0 /* shortcutProbability */, + wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, + wordProperty.mIsBlacklistEntry, 0 /* timestamp */)) { + MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord); + } + } else { + for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + if (!binaryDict.addUnigramEntry(wordProperty.mWord, + wordProperty.getProbability(), + shortcutTarget.mWord, shortcutTarget.getProbability(), + wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, + wordProperty.mIsBlacklistEntry, 0 /* timestamp */)) { + MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord + + ", shortcutTarget: " + shortcutTarget.mWord); + return; + } + } + } + if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { + if (!binaryDict.flushWithGC()) { + MakedictLog.e("Cannot flush dict with GC."); + return; + } + } + } + for (final WordProperty word0Property : dict) { + if (null == word0Property.mBigrams) continue; + for (final WeightedString word1 : word0Property.mBigrams) { + final PrevWordsInfo prevWordsInfo = + new PrevWordsInfo(new PrevWordsInfo.WordInfo(word0Property.mWord)); + if (!binaryDict.addNgramEntry(prevWordsInfo, word1.mWord, + word1.getProbability(), 0 /* timestamp */)) { + MakedictLog.e("Cannot add n-gram entry for " + + prevWordsInfo + " -> " + word1.mWord); + return; + } + if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { + if (!binaryDict.flushWithGC()) { + MakedictLog.e("Cannot flush dict with GC."); + return; + } + } + } + } + if (!binaryDict.flushWithGC()) { + MakedictLog.e("Cannot flush dict with GC."); + return; + } + binaryDict.close(); + } + + @Override + public void setPosition(int position) { + } + + @Override + public int getPosition() { + return 0; + } + + @Override + public void writePtNodeCount(int ptNodeCount) { + } + + @Override + public void writeForwardLinkAddress(int forwardLinkAddress) { + } + + @Override + public void writePtNode(PtNode ptNode, FusionDictionary dict) { + } +} diff --git a/tests/src/com/android/inputmethod/latin/personalization/ContextualDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/ContextualDictionaryTests.java new file mode 100644 index 000000000..565fadb2a --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/personalization/ContextualDictionaryTests.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.DictionaryFacilitator; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; + +/** + * Unit tests for contextual dictionary + */ +@LargeTest +public class ContextualDictionaryTests extends AndroidTestCase { + private static final String TAG = ContextualDictionaryTests.class.getSimpleName(); + + private static final Locale LOCALE_EN_US = new Locale("en", "US"); + + private DictionaryFacilitator getDictionaryFacilitator() { + final ArrayList<String> dictTypes = new ArrayList<>(); + dictTypes.add(Dictionary.TYPE_CONTEXTUAL); + final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(); + dictionaryFacilitator.resetDictionariesForTesting(getContext(), LOCALE_EN_US, dictTypes, + new HashMap<String, File>(), new HashMap<String, Map<String, String>>()); + return dictionaryFacilitator; + } + + public void testAddPhrase() { + final DictionaryFacilitator dictionaryFacilitator = getDictionaryFacilitator(); + final String[] phrase = new String[] {"a", "b", "c", "d"}; + final int probability = 100; + final int bigramProbabilityForWords = 150; + final int bigramProbabilityForPhrases = 200; + dictionaryFacilitator.addPhraseToContextualDictionary( + phrase, probability, bigramProbabilityForWords, bigramProbabilityForPhrases); + final ExpandableBinaryDictionary contextualDictionary = + dictionaryFacilitator.getSubDictForTesting(Dictionary.TYPE_CONTEXTUAL); + contextualDictionary.waitAllTasksForTests(); + // Word + assertTrue(contextualDictionary.isInDictionary("a")); + assertTrue(contextualDictionary.isInDictionary("b")); + assertTrue(contextualDictionary.isInDictionary("c")); + assertTrue(contextualDictionary.isInDictionary("d")); + // Phrase + assertTrue(contextualDictionary.isInDictionary("a b c d")); + assertTrue(contextualDictionary.isInDictionary("b c d")); + assertTrue(contextualDictionary.isInDictionary("c d")); + assertFalse(contextualDictionary.isInDictionary("a b c")); + assertFalse(contextualDictionary.isInDictionary("abcd")); + // TODO: Add tests for probability. + // TODO: Add tests for n-grams. + } +} diff --git a/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java new file mode 100644 index 000000000..0f2f9814b --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.DictionaryFacilitator; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback; +import com.android.inputmethod.latin.makedict.CodePointUtils; +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; +import android.util.Log; + +/** + * Unit tests for personalization dictionary + */ +@LargeTest +public class PersonalizationDictionaryTests extends AndroidTestCase { + private static final String TAG = PersonalizationDictionaryTests.class.getSimpleName(); + + private static final Locale LOCALE_EN_US = new Locale("en", "US"); + private static final String DUMMY_PACKAGE_NAME = "test.package.name"; + private static final long TIMEOUT_TO_WAIT_DICTIONARY_OPERATIONS_IN_SECONDS = 120; + + private DictionaryFacilitator getDictionaryFacilitator() { + final ArrayList<String> dictTypes = new ArrayList<>(); + dictTypes.add(Dictionary.TYPE_MAIN); + dictTypes.add(Dictionary.TYPE_PERSONALIZATION); + final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(); + dictionaryFacilitator.resetDictionariesForTesting(getContext(), LOCALE_EN_US, dictTypes, + new HashMap<String, File>(), new HashMap<String, Map<String, String>>()); + return dictionaryFacilitator; + } + + public void testAddManyTokens() { + final DictionaryFacilitator dictionaryFacilitator = getDictionaryFacilitator(); + dictionaryFacilitator.clearPersonalizationDictionary(); + final int dataChunkCount = 20; + final int wordCountInOneChunk = 2000; + final Random random = new Random(System.currentTimeMillis()); + final int[] codePointSet = CodePointUtils.LATIN_ALPHABETS_LOWER; + + final SpacingAndPunctuations spacingAndPunctuations = + new SpacingAndPunctuations(getContext().getResources()); + + final int timeStampInSeconds = (int)TimeUnit.MILLISECONDS.toSeconds( + System.currentTimeMillis()); + + for (int i = 0; i < dataChunkCount; i++) { + final ArrayList<String> tokens = new ArrayList<>(); + for (int j = 0; j < wordCountInOneChunk; j++) { + tokens.add(CodePointUtils.generateWord(random, codePointSet)); + } + final PersonalizationDataChunk personalizationDataChunk = new PersonalizationDataChunk( + true /* inputByUser */, tokens, timeStampInSeconds, DUMMY_PACKAGE_NAME); + final CountDownLatch countDownLatch = new CountDownLatch(1); + final AddMultipleDictionaryEntriesCallback callback = + new AddMultipleDictionaryEntriesCallback() { + @Override + public void onFinished() { + countDownLatch.countDown(); + } + }; + dictionaryFacilitator.addEntriesToPersonalizationDictionary(personalizationDataChunk, + spacingAndPunctuations, callback); + try { + countDownLatch.await(TIMEOUT_TO_WAIT_DICTIONARY_OPERATIONS_IN_SECONDS, + TimeUnit.SECONDS); + } catch (InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); + } + } + dictionaryFacilitator.flushPersonalizationDictionary(); + try { + dictionaryFacilitator.waitForLoadingDictionariesForTesting( + TIMEOUT_TO_WAIT_DICTIONARY_OPERATIONS_IN_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); + } + final String dictName = ExpandableBinaryDictionary.getDictName( + PersonalizationDictionary.NAME, LOCALE_EN_US, null /* dictFile */); + final File dictFile = ExpandableBinaryDictionary.getDictFile( + getContext(), dictName, null /* dictFile */); + + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, 0 /* size */, + true /* useFullEditDistance */, LOCALE_EN_US, Dictionary.TYPE_PERSONALIZATION, + true /* isUpdatable */); + assertTrue(binaryDictionary.isValidDictionary()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 7c1decb71..f87f3b494 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -16,20 +16,23 @@ package com.android.inputmethod.latin.personalization; -import android.content.SharedPreferences; -import android.preference.PreferenceManager; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; import com.android.inputmethod.latin.ExpandableBinaryDictionary; -import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.PrevWordsInfo; +import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; +import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; +import com.android.inputmethod.latin.utils.DistracterFilter; +import com.android.inputmethod.latin.utils.FileUtils; import java.io.File; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Random; -import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -38,25 +41,57 @@ import java.util.concurrent.TimeUnit; @LargeTest public class UserHistoryDictionaryTests extends AndroidTestCase { private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName(); - private SharedPreferences mPrefs; private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" }; - private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000; - private static final int WAIT_TERMINATING_IN_MILLISECONDS = 100; + private int mCurrentTime = 0; @Override - public void setUp() { - mPrefs = PreferenceManager.getDefaultSharedPreferences(getContext()); + protected void setUp() throws Exception { + super.setUp(); + resetCurrentTimeForTestMode(); + } + + @Override + protected void tearDown() throws Exception { + stopTestModeInNativeCode(); + super.tearDown(); + } + + private void resetCurrentTimeForTestMode() { + mCurrentTime = 0; + setCurrentTimeForTestMode(mCurrentTime); + } + + private void forcePassingShortTime() { + // 3 days. + final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(3); + mCurrentTime += timeToElapse; + setCurrentTimeForTestMode(mCurrentTime); + } + + private void forcePassingLongTime() { + // 365 days. + final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(365); + mCurrentTime += timeToElapse; + setCurrentTimeForTestMode(mCurrentTime); + } + + private static int setCurrentTimeForTestMode(final int currentTime) { + return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime); + } + + private static int stopTestModeInNativeCode() { + return BinaryDictionaryUtils.setCurrentTimeForTest(-1); } /** * Generates a random word. */ - private String generateWord(final int value) { + private static String generateWord(final int value) { final int lengthOfChars = CHARACTERS.length; StringBuilder builder = new StringBuilder(); long lvalue = Math.abs((long)value); @@ -67,19 +102,21 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { return builder.toString(); } - private List<String> generateWords(final int number, final Random random) { - final Set<String> wordSet = CollectionUtils.newHashSet(); + private static List<String> generateWords(final int number, final Random random) { + final HashSet<String> wordSet = new HashSet<>(); while (wordSet.size() < number) { wordSet.add(generateWord(random.nextInt())); } - return new ArrayList<String>(wordSet); + return new ArrayList<>(wordSet); } - private void addToDict(final UserHistoryDictionary dict, final List<String> words) { - String prevWord = null; + private static void addToDict(final UserHistoryDictionary dict, final List<String> words) { + PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; for (String word : words) { - dict.addToDictionary(prevWord, word, true); - prevWord = word; + UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, + (int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()), + DistracterFilter.EMPTY_DISTRACTER_FILTER); + prevWordsInfo = prevWordsInfo.getNextPrevWordsInfo(new WordInfo(word)); } } @@ -87,22 +124,18 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { * @param checkContents if true, checks whether written words are actually in the dictionary * or not. */ - private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords, + private void addAndWriteRandomWords(final Locale locale, final int numberOfWords, final Random random, final boolean checkContents) { final List<String> words = generateWords(numberOfWords, random); - final UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix /* locale */, mPrefs); + final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( + mContext, locale); // Add random words to the user history dictionary. addToDict(dict, words); if (checkContents) { - try { - Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); - } catch (InterruptedException e) { - } + dict.waitAllTasksForTests(); for (int i = 0; i < numberOfWords; ++i) { final String word = words.get(i); - assertTrue(dict.isInDictionaryForTests(word)); + assertTrue(dict.isInDictionary(word)); } } // write to file. @@ -111,57 +144,48 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { /** * Clear all entries in the user history dictionary. - * @param testFilenameSuffix file name suffix used for testing. + * @param locale dummy locale for testing. */ - private void clearHistory(final String testFilenameSuffix) { - final UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix /* locale */, mPrefs); - dict.clearAndFlushDictionary(); + private void clearHistory(final Locale locale) { + final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( + mContext, locale); + dict.waitAllTasksForTests(); + dict.clear(); dict.close(); + dict.waitAllTasksForTests(); } /** * Shut down executer and wait until all operations of user history are done. - * @param testFilenameSuffix file name suffix used for testing. + * @param locale dummy locale for testing. */ - private void waitForWriting(final String testFilenameSuffix) { - try { - final UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix, mPrefs); - dict.shutdownExecutorForTests(); - while (!dict.isTerminatedForTests()) { - Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); - } - } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: ", e); - } + private void waitForWriting(final Locale locale) { + final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( + mContext, locale); + dict.waitAllTasksForTests(); } public void testRandomWords() { Log.d(TAG, "This test can be used for profiling."); Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); - final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); - final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix - + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + final Locale dummyLocale = new Locale("test_random_words" + System.currentTimeMillis()); + final String dictName = ExpandableBinaryDictionary.getDictName( + UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */); + final File dictFile = ExpandableBinaryDictionary.getDictFile( + mContext, dictName, null /* dictFile */); final int numberOfWords = 1000; final Random random = new Random(123456); try { - clearHistory(testFilenameSuffix); - addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, + clearHistory(dummyLocale); + addAndWriteRandomWords(dummyLocale, numberOfWords, random, true /* checksContents */); } finally { Log.d(TAG, "waiting for writing ..."); - waitForWriting(testFilenameSuffix); - final File dictFile = new File(getContext().getFilesDir(), fileName); - if (dictFile != null) { - assertTrue(dictFile.exists()); - assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); - dictFile.delete(); - } + waitForWriting(dummyLocale); + assertTrue("check exisiting of " + dictFile, dictFile.exists()); + FileUtils.deleteRecursively(dictFile); } } @@ -171,17 +195,18 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final int numberOfWordsInsertedForEachLanguageSwitch = 100; final File dictFiles[] = new File[numberOfLanguages]; - final String testFilenameSuffixes[] = new String[numberOfLanguages]; + final Locale dummyLocales[] = new Locale[numberOfLanguages]; try { final Random random = new Random(123456); // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { - testFilenameSuffixes[i] = "testSwitchingLanguages" + i; - final String fileName = UserHistoryDictionary.NAME + "." + - testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; - dictFiles[i] = new File(getContext().getFilesDir(), fileName); - clearHistory(testFilenameSuffixes[i]); + dummyLocales[i] = new Locale("test_switching_languages" + i); + final String dictName = ExpandableBinaryDictionary.getDictName( + UserHistoryDictionary.NAME, dummyLocales[i], null /* dictFile */); + dictFiles[i] = ExpandableBinaryDictionary.getDictFile( + mContext, dictName, null /* dictFile */); + clearHistory(dummyLocales[i]); } final long start = System.currentTimeMillis(); @@ -189,7 +214,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { for (int i = 0; i < numberOfLanguageSwitching; i++) { final int index = i % numberOfLanguages; // Switch languages to testFilenameSuffixes[index]. - addAndWriteRandomWords(testFilenameSuffixes[index], + addAndWriteRandomWords(dummyLocales[index], numberOfWordsInsertedForEachLanguageSwitch, random, false /* checksContents */); } @@ -200,40 +225,63 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } finally { Log.d(TAG, "waiting for writing ..."); for (int i = 0; i < numberOfLanguages; i++) { - waitForWriting(testFilenameSuffixes[i]); + waitForWriting(dummyLocales[i]); } - for (final File file : dictFiles) { - if (file != null) { - assertTrue(file.exists()); - assertTrue(file.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); - file.delete(); - } + for (final File dictFile : dictFiles) { + assertTrue("check exisiting of " + dictFile, dictFile.exists()); + FileUtils.deleteRecursively(dictFile); } } } public void testAddManyWords() { - final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); - final int numberOfWords = - ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? - 10000 : 1000; + final Locale dummyLocale = new Locale("test_random_words" + System.currentTimeMillis()); + final String dictName = ExpandableBinaryDictionary.getDictName( + UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */); + final File dictFile = ExpandableBinaryDictionary.getDictFile( + mContext, dictName, null /* dictFile */); + final int numberOfWords = 10000; final Random random = new Random(123456); - clearHistory(testFilenameSuffix); + clearHistory(dummyLocale); try { - addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, - true /* checksContents */); + addAndWriteRandomWords(dummyLocale, numberOfWords, random, true /* checksContents */); } finally { Log.d(TAG, "waiting for writing ..."); - waitForWriting(testFilenameSuffix); - final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix - + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; - final File dictFile = new File(getContext().getFilesDir(), fileName); - if (dictFile != null) { - assertTrue(dictFile.exists()); - assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); - dictFile.delete(); - } + waitForWriting(dummyLocale); + assertTrue("check exisiting of " + dictFile, dictFile.exists()); + FileUtils.deleteRecursively(dictFile); } } + public void testDecaying() { + final Locale dummyLocale = new Locale("test_decaying" + System.currentTimeMillis()); + final int numberOfWords = 5000; + final Random random = new Random(123456); + resetCurrentTimeForTestMode(); + clearHistory(dummyLocale); + final List<String> words = generateWords(numberOfWords, random); + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), dummyLocale); + dict.waitAllTasksForTests(); + PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; + for (final String word : words) { + UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime, + DistracterFilter.EMPTY_DISTRACTER_FILTER); + prevWordsInfo = prevWordsInfo.getNextPrevWordsInfo(new WordInfo(word)); + dict.waitAllTasksForTests(); + assertTrue(dict.isInDictionary(word)); + } + forcePassingShortTime(); + dict.runGCIfRequired(); + dict.waitAllTasksForTests(); + for (final String word : words) { + assertTrue(dict.isInDictionary(word)); + } + forcePassingLongTime(); + dict.runGCIfRequired(); + dict.waitAllTasksForTests(); + for (final String word : words) { + assertFalse(dict.isInDictionary(word)); + } + } } diff --git a/tests/src/com/android/inputmethod/latin/settings/SpacingAndPunctuationsTests.java b/tests/src/com/android/inputmethod/latin/settings/SpacingAndPunctuationsTests.java new file mode 100644 index 000000000..2cc22fae4 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/settings/SpacingAndPunctuationsTests.java @@ -0,0 +1,477 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.settings; + +import android.content.res.Resources; +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; + +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.SuggestedWords; +import com.android.inputmethod.latin.utils.RunInLocale; + +import junit.framework.AssertionFailedError; + +import java.util.Locale; + +@SmallTest +public class SpacingAndPunctuationsTests extends AndroidTestCase { + private static final int ARMENIAN_FULL_STOP = '\u0589'; + private static final int ARMENIAN_COMMA = '\u055D'; + + private int mScreenMetrics; + + private boolean isPhone() { + return mScreenMetrics == Constants.SCREEN_METRICS_SMALL_PHONE + || mScreenMetrics == Constants.SCREEN_METRICS_LARGE_PHONE; + } + + private boolean isTablet() { + return mScreenMetrics == Constants.SCREEN_METRICS_SMALL_TABLET + || mScreenMetrics == Constants.SCREEN_METRICS_LARGE_TABLET; + } + + private SpacingAndPunctuations ENGLISH; + private SpacingAndPunctuations FRENCH; + private SpacingAndPunctuations GERMAN; + private SpacingAndPunctuations ARMENIAN; + private SpacingAndPunctuations THAI; + private SpacingAndPunctuations KHMER; + private SpacingAndPunctuations LAO; + private SpacingAndPunctuations ARABIC; + private SpacingAndPunctuations PERSIAN; + private SpacingAndPunctuations HEBREW; + + private SpacingAndPunctuations UNITED_STATES; + private SpacingAndPunctuations UNITED_KINGDOM; + private SpacingAndPunctuations CANADA_FRENCH; + private SpacingAndPunctuations SWISS_GERMAN; + private SpacingAndPunctuations INDIA_ENGLISH; + private SpacingAndPunctuations ARMENIA_ARMENIAN; + private SpacingAndPunctuations CAMBODIA_KHMER; + private SpacingAndPunctuations LAOS_LAO; + + @Override + protected void setUp() throws Exception { + super.setUp(); + + mScreenMetrics = mContext.getResources().getInteger(R.integer.config_screen_metrics); + + // Language only + ENGLISH = getSpacingAndPunctuations(Locale.ENGLISH); + FRENCH = getSpacingAndPunctuations(Locale.FRENCH); + GERMAN = getSpacingAndPunctuations(Locale.GERMAN); + THAI = getSpacingAndPunctuations(new Locale("th")); + ARMENIAN = getSpacingAndPunctuations(new Locale("hy")); + KHMER = getSpacingAndPunctuations(new Locale("km")); + LAO = getSpacingAndPunctuations(new Locale("lo")); + ARABIC = getSpacingAndPunctuations(new Locale("ar")); + PERSIAN = getSpacingAndPunctuations(new Locale("fa")); + HEBREW = getSpacingAndPunctuations(new Locale("iw")); + + // Language and Country + UNITED_STATES = getSpacingAndPunctuations(Locale.US); + UNITED_KINGDOM = getSpacingAndPunctuations(Locale.UK); + CANADA_FRENCH = getSpacingAndPunctuations(Locale.CANADA_FRENCH); + SWISS_GERMAN = getSpacingAndPunctuations(new Locale("de", "CH")); + INDIA_ENGLISH = getSpacingAndPunctuations(new Locale("en", "IN")); + ARMENIA_ARMENIAN = getSpacingAndPunctuations(new Locale("hy", "AM")); + CAMBODIA_KHMER = getSpacingAndPunctuations(new Locale("km", "KH")); + LAOS_LAO = getSpacingAndPunctuations(new Locale("lo", "LA")); + } + + private SpacingAndPunctuations getSpacingAndPunctuations(final Locale locale) { + final RunInLocale<SpacingAndPunctuations> job = new RunInLocale<SpacingAndPunctuations>() { + @Override + protected SpacingAndPunctuations job(Resources res) { + return new SpacingAndPunctuations(res); + } + }; + return job.runInLocale(getContext().getResources(), locale); + } + + private static void testingStandardWordSeparator(final SpacingAndPunctuations sp) { + assertTrue("Tab", sp.isWordSeparator('\t')); + assertTrue("Newline", sp.isWordSeparator('\n')); + assertTrue("Space", sp.isWordSeparator(' ')); + assertTrue("Exclamation", sp.isWordSeparator('!')); + assertTrue("Quotation", sp.isWordSeparator('"')); + assertFalse("Number", sp.isWordSeparator('#')); + assertFalse("Dollar", sp.isWordSeparator('$')); + assertFalse("Percent", sp.isWordSeparator('%')); + assertTrue("Ampersand", sp.isWordSeparator('&')); + assertFalse("Apostrophe", sp.isWordSeparator('\'')); + assertTrue("L Paren", sp.isWordSeparator('(')); + assertTrue("R Paren", sp.isWordSeparator(')')); + assertTrue("Asterisk", sp.isWordSeparator('*')); + assertTrue("Plus", sp.isWordSeparator('+')); + assertTrue("Comma", sp.isWordSeparator(',')); + assertFalse("Minus", sp.isWordSeparator('-')); + assertTrue("Period", sp.isWordSeparator('.')); + assertTrue("Slash", sp.isWordSeparator('/')); + assertTrue("Colon", sp.isWordSeparator(':')); + assertTrue("Semicolon", sp.isWordSeparator(';')); + assertTrue("L Angle", sp.isWordSeparator('<')); + assertTrue("Equal", sp.isWordSeparator('=')); + assertTrue("R Angle", sp.isWordSeparator('>')); + assertTrue("Question", sp.isWordSeparator('?')); + assertFalse("Atmark", sp.isWordSeparator('@')); + assertTrue("L S Bracket", sp.isWordSeparator('[')); + assertFalse("B Slash", sp.isWordSeparator('\\')); + assertTrue("R S Bracket", sp.isWordSeparator(']')); + assertFalse("Circumflex", sp.isWordSeparator('^')); + assertTrue("Underscore", sp.isWordSeparator('_')); + assertFalse("Grave", sp.isWordSeparator('`')); + assertTrue("L C Brace", sp.isWordSeparator('{')); + assertTrue("V Line", sp.isWordSeparator('|')); + assertTrue("R C Brace", sp.isWordSeparator('}')); + assertFalse("Tilde", sp.isWordSeparator('~')); + } + + public void testWordSeparator() { + testingStandardWordSeparator(ENGLISH); + testingStandardWordSeparator(FRENCH); + testingStandardWordSeparator(CANADA_FRENCH); + testingStandardWordSeparator(ARMENIA_ARMENIAN); + assertTrue(ARMENIA_ARMENIAN.isWordSeparator(ARMENIAN_FULL_STOP)); + assertTrue(ARMENIA_ARMENIAN.isWordSeparator(ARMENIAN_COMMA)); + // TODO: We should fix these. + testingStandardWordSeparator(ARMENIAN); + assertFalse(ARMENIAN.isWordSeparator(ARMENIAN_FULL_STOP)); + assertFalse(ARMENIAN.isWordSeparator(ARMENIAN_COMMA)); + } + + private static void testingStandardWordConnector(final SpacingAndPunctuations sp) { + assertFalse("Tab", sp.isWordConnector('\t')); + assertFalse("Newline", sp.isWordConnector('\n')); + assertFalse("Space", sp.isWordConnector(' ')); + assertFalse("Exclamation", sp.isWordConnector('!')); + assertFalse("Quotation", sp.isWordConnector('"')); + assertFalse("Number", sp.isWordConnector('#')); + assertFalse("Dollar", sp.isWordConnector('$')); + assertFalse("Percent", sp.isWordConnector('%')); + assertFalse("Ampersand", sp.isWordConnector('&')); + assertTrue("Apostrophe", sp.isWordConnector('\'')); + assertFalse("L Paren", sp.isWordConnector('(')); + assertFalse("R Paren", sp.isWordConnector(')')); + assertFalse("Asterisk", sp.isWordConnector('*')); + assertFalse("Plus", sp.isWordConnector('+')); + assertFalse("Comma", sp.isWordConnector(',')); + assertTrue("Minus", sp.isWordConnector('-')); + assertFalse("Period", sp.isWordConnector('.')); + assertFalse("Slash", sp.isWordConnector('/')); + assertFalse("Colon", sp.isWordConnector(':')); + assertFalse("Semicolon", sp.isWordConnector(';')); + assertFalse("L Angle", sp.isWordConnector('<')); + assertFalse("Equal", sp.isWordConnector('=')); + assertFalse("R Angle", sp.isWordConnector('>')); + assertFalse("Question", sp.isWordConnector('?')); + assertFalse("Atmark", sp.isWordConnector('@')); + assertFalse("L S Bracket", sp.isWordConnector('[')); + assertFalse("B Slash", sp.isWordConnector('\\')); + assertFalse("R S Bracket", sp.isWordConnector(']')); + assertFalse("Circumflex", sp.isWordConnector('^')); + assertFalse("Underscore", sp.isWordConnector('_')); + assertFalse("Grave", sp.isWordConnector('`')); + assertFalse("L C Brace", sp.isWordConnector('{')); + assertFalse("V Line", sp.isWordConnector('|')); + assertFalse("R C Brace", sp.isWordConnector('}')); + assertFalse("Tilde", sp.isWordConnector('~')); + + } + + public void testWordConnector() { + testingStandardWordConnector(ENGLISH); + testingStandardWordConnector(FRENCH); + testingStandardWordConnector(CANADA_FRENCH); + testingStandardWordConnector(ARMENIA_ARMENIAN); + } + + private static void testingCommonPrecededBySpace(final SpacingAndPunctuations sp) { + assertFalse("Tab", sp.isUsuallyPrecededBySpace('\t')); + assertFalse("Newline", sp.isUsuallyPrecededBySpace('\n')); + assertFalse("Space", sp.isUsuallyPrecededBySpace(' ')); + //assertFalse("Exclamation", sp.isUsuallyPrecededBySpace('!')); + assertFalse("Quotation", sp.isUsuallyPrecededBySpace('"')); + assertFalse("Number", sp.isUsuallyPrecededBySpace('#')); + assertFalse("Dollar", sp.isUsuallyPrecededBySpace('$')); + assertFalse("Percent", sp.isUsuallyPrecededBySpace('%')); + assertTrue("Ampersand", sp.isUsuallyPrecededBySpace('&')); + assertFalse("Apostrophe", sp.isUsuallyPrecededBySpace('\'')); + assertTrue("L Paren", sp.isUsuallyPrecededBySpace('(')); + assertFalse("R Paren", sp.isUsuallyPrecededBySpace(')')); + assertFalse("Asterisk", sp.isUsuallyPrecededBySpace('*')); + assertFalse("Plus", sp.isUsuallyPrecededBySpace('+')); + assertFalse("Comma", sp.isUsuallyPrecededBySpace(',')); + assertFalse("Minus", sp.isUsuallyPrecededBySpace('-')); + assertFalse("Period", sp.isUsuallyPrecededBySpace('.')); + assertFalse("Slash", sp.isUsuallyPrecededBySpace('/')); + //assertFalse("Colon", sp.isUsuallyPrecededBySpace(':')); + //assertFalse("Semicolon", sp.isUsuallyPrecededBySpace(';')); + assertFalse("L Angle", sp.isUsuallyPrecededBySpace('<')); + assertFalse("Equal", sp.isUsuallyPrecededBySpace('=')); + assertFalse("R Angle", sp.isUsuallyPrecededBySpace('>')); + //assertFalse("Question", sp.isUsuallyPrecededBySpace('?')); + assertFalse("Atmark", sp.isUsuallyPrecededBySpace('@')); + assertTrue("L S Bracket", sp.isUsuallyPrecededBySpace('[')); + assertFalse("B Slash", sp.isUsuallyPrecededBySpace('\\')); + assertFalse("R S Bracket", sp.isUsuallyPrecededBySpace(']')); + assertFalse("Circumflex", sp.isUsuallyPrecededBySpace('^')); + assertFalse("Underscore", sp.isUsuallyPrecededBySpace('_')); + assertFalse("Grave", sp.isUsuallyPrecededBySpace('`')); + assertTrue("L C Brace", sp.isUsuallyPrecededBySpace('{')); + assertFalse("V Line", sp.isUsuallyPrecededBySpace('|')); + assertFalse("R C Brace", sp.isUsuallyPrecededBySpace('}')); + assertFalse("Tilde", sp.isUsuallyPrecededBySpace('~')); + } + + private static void testingStandardPrecededBySpace(final SpacingAndPunctuations sp) { + testingCommonPrecededBySpace(sp); + assertFalse("Exclamation", sp.isUsuallyPrecededBySpace('!')); + assertFalse("Colon", sp.isUsuallyPrecededBySpace(':')); + assertFalse("Semicolon", sp.isUsuallyPrecededBySpace(';')); + assertFalse("Question", sp.isUsuallyPrecededBySpace('?')); + } + + public void testIsUsuallyPrecededBySpace() { + testingStandardPrecededBySpace(ENGLISH); + testingCommonPrecededBySpace(FRENCH); + assertTrue("Exclamation", FRENCH.isUsuallyPrecededBySpace('!')); + assertTrue("Colon", FRENCH.isUsuallyPrecededBySpace(':')); + assertTrue("Semicolon", FRENCH.isUsuallyPrecededBySpace(';')); + assertTrue("Question", FRENCH.isUsuallyPrecededBySpace('?')); + testingCommonPrecededBySpace(CANADA_FRENCH); + assertFalse("Exclamation", CANADA_FRENCH.isUsuallyPrecededBySpace('!')); + assertTrue("Colon", CANADA_FRENCH.isUsuallyPrecededBySpace(':')); + assertFalse("Semicolon", CANADA_FRENCH.isUsuallyPrecededBySpace(';')); + assertFalse("Question", CANADA_FRENCH.isUsuallyPrecededBySpace('?')); + testingStandardPrecededBySpace(ARMENIA_ARMENIAN); + } + + private static void testingStandardFollowedBySpace(final SpacingAndPunctuations sp) { + assertFalse("Tab", sp.isUsuallyFollowedBySpace('\t')); + assertFalse("Newline", sp.isUsuallyFollowedBySpace('\n')); + assertFalse("Space", sp.isUsuallyFollowedBySpace(' ')); + assertTrue("Exclamation", sp.isUsuallyFollowedBySpace('!')); + assertFalse("Quotation", sp.isUsuallyFollowedBySpace('"')); + assertFalse("Number", sp.isUsuallyFollowedBySpace('#')); + assertFalse("Dollar", sp.isUsuallyFollowedBySpace('$')); + assertFalse("Percent", sp.isUsuallyFollowedBySpace('%')); + assertTrue("Ampersand", sp.isUsuallyFollowedBySpace('&')); + assertFalse("Apostrophe", sp.isUsuallyFollowedBySpace('\'')); + assertFalse("L Paren", sp.isUsuallyFollowedBySpace('(')); + assertTrue("R Paren", sp.isUsuallyFollowedBySpace(')')); + assertFalse("Asterisk", sp.isUsuallyFollowedBySpace('*')); + assertFalse("Plus", sp.isUsuallyFollowedBySpace('+')); + assertTrue("Comma", sp.isUsuallyFollowedBySpace(',')); + assertFalse("Minus", sp.isUsuallyFollowedBySpace('-')); + assertTrue("Period", sp.isUsuallyFollowedBySpace('.')); + assertFalse("Slash", sp.isUsuallyFollowedBySpace('/')); + assertTrue("Colon", sp.isUsuallyFollowedBySpace(':')); + assertTrue("Semicolon", sp.isUsuallyFollowedBySpace(';')); + assertFalse("L Angle", sp.isUsuallyFollowedBySpace('<')); + assertFalse("Equal", sp.isUsuallyFollowedBySpace('=')); + assertFalse("R Angle", sp.isUsuallyFollowedBySpace('>')); + assertTrue("Question", sp.isUsuallyFollowedBySpace('?')); + assertFalse("Atmark", sp.isUsuallyFollowedBySpace('@')); + assertFalse("L S Bracket", sp.isUsuallyFollowedBySpace('[')); + assertFalse("B Slash", sp.isUsuallyFollowedBySpace('\\')); + assertTrue("R S Bracket", sp.isUsuallyFollowedBySpace(']')); + assertFalse("Circumflex", sp.isUsuallyFollowedBySpace('^')); + assertFalse("Underscore", sp.isUsuallyFollowedBySpace('_')); + assertFalse("Grave", sp.isUsuallyFollowedBySpace('`')); + assertFalse("L C Brace", sp.isUsuallyFollowedBySpace('{')); + assertFalse("V Line", sp.isUsuallyFollowedBySpace('|')); + assertTrue("R C Brace", sp.isUsuallyFollowedBySpace('}')); + assertFalse("Tilde", sp.isUsuallyFollowedBySpace('~')); + } + + public void testIsUsuallyFollowedBySpace() { + testingStandardFollowedBySpace(ENGLISH); + testingStandardFollowedBySpace(FRENCH); + testingStandardFollowedBySpace(CANADA_FRENCH); + testingStandardFollowedBySpace(ARMENIA_ARMENIAN); + assertTrue(ARMENIA_ARMENIAN.isUsuallyFollowedBySpace(ARMENIAN_FULL_STOP)); + assertTrue(ARMENIA_ARMENIAN.isUsuallyFollowedBySpace(ARMENIAN_COMMA)); + } + + private static void testingStandardSentenceSeparator(final SpacingAndPunctuations sp) { + assertFalse("Tab", sp.isUsuallyFollowedBySpace('\t')); + assertFalse("Newline", sp.isUsuallyFollowedBySpace('\n')); + assertFalse("Space", sp.isUsuallyFollowedBySpace(' ')); + assertFalse("Exclamation", sp.isUsuallyFollowedBySpace('!')); + assertFalse("Quotation", sp.isUsuallyFollowedBySpace('"')); + assertFalse("Number", sp.isUsuallyFollowedBySpace('#')); + assertFalse("Dollar", sp.isUsuallyFollowedBySpace('$')); + assertFalse("Percent", sp.isUsuallyFollowedBySpace('%')); + assertFalse("Ampersand", sp.isUsuallyFollowedBySpace('&')); + assertFalse("Apostrophe", sp.isUsuallyFollowedBySpace('\'')); + assertFalse("L Paren", sp.isUsuallyFollowedBySpace('(')); + assertFalse("R Paren", sp.isUsuallyFollowedBySpace(')')); + assertFalse("Asterisk", sp.isUsuallyFollowedBySpace('*')); + assertFalse("Plus", sp.isUsuallyFollowedBySpace('+')); + assertFalse("Comma", sp.isUsuallyFollowedBySpace(',')); + assertFalse("Minus", sp.isUsuallyFollowedBySpace('-')); + assertTrue("Period", sp.isUsuallyFollowedBySpace('.')); + assertFalse("Slash", sp.isUsuallyFollowedBySpace('/')); + assertFalse("Colon", sp.isUsuallyFollowedBySpace(':')); + assertFalse("Semicolon", sp.isUsuallyFollowedBySpace(';')); + assertFalse("L Angle", sp.isUsuallyFollowedBySpace('<')); + assertFalse("Equal", sp.isUsuallyFollowedBySpace('=')); + assertFalse("R Angle", sp.isUsuallyFollowedBySpace('>')); + assertFalse("Question", sp.isUsuallyFollowedBySpace('?')); + assertFalse("Atmark", sp.isUsuallyFollowedBySpace('@')); + assertFalse("L S Bracket", sp.isUsuallyFollowedBySpace('[')); + assertFalse("B Slash", sp.isUsuallyFollowedBySpace('\\')); + assertFalse("R S Bracket", sp.isUsuallyFollowedBySpace(']')); + assertFalse("Circumflex", sp.isUsuallyFollowedBySpace('^')); + assertFalse("Underscore", sp.isUsuallyFollowedBySpace('_')); + assertFalse("Grave", sp.isUsuallyFollowedBySpace('`')); + assertFalse("L C Brace", sp.isUsuallyFollowedBySpace('{')); + assertFalse("V Line", sp.isUsuallyFollowedBySpace('|')); + assertFalse("R C Brace", sp.isUsuallyFollowedBySpace('}')); + assertFalse("Tilde", sp.isUsuallyFollowedBySpace('~')); + } + + public void isSentenceSeparator() { + testingStandardSentenceSeparator(ENGLISH); + try { + testingStandardSentenceSeparator(ARMENIA_ARMENIAN); + fail("Armenian Sentence Separator"); + } catch (final AssertionFailedError e) { + assertEquals("Period", e.getMessage()); + } + assertTrue(ARMENIA_ARMENIAN.isSentenceSeparator(ARMENIAN_FULL_STOP)); + assertFalse(ARMENIA_ARMENIAN.isSentenceSeparator(ARMENIAN_COMMA)); + } + + public void testLanguageHasSpace() { + assertTrue(ENGLISH.mCurrentLanguageHasSpaces); + assertTrue(FRENCH.mCurrentLanguageHasSpaces); + assertTrue(GERMAN.mCurrentLanguageHasSpaces); + assertFalse(THAI.mCurrentLanguageHasSpaces); + assertFalse(CAMBODIA_KHMER.mCurrentLanguageHasSpaces); + assertFalse(LAOS_LAO.mCurrentLanguageHasSpaces); + // TODO: We should fix these. + assertTrue(KHMER.mCurrentLanguageHasSpaces); + assertTrue(LAO.mCurrentLanguageHasSpaces); + } + + public void testUsesAmericanTypography() { + assertTrue(ENGLISH.mUsesAmericanTypography); + assertTrue(UNITED_STATES.mUsesAmericanTypography); + assertTrue(UNITED_KINGDOM.mUsesAmericanTypography); + assertTrue(INDIA_ENGLISH.mUsesAmericanTypography); + assertFalse(FRENCH.mUsesAmericanTypography); + assertFalse(GERMAN.mUsesAmericanTypography); + assertFalse(SWISS_GERMAN.mUsesAmericanTypography); + } + + public void testUsesGermanRules() { + assertFalse(ENGLISH.mUsesGermanRules); + assertFalse(FRENCH.mUsesGermanRules); + assertTrue(GERMAN.mUsesGermanRules); + assertTrue(SWISS_GERMAN.mUsesGermanRules); + } + + // Punctuations for phone. + private static final String[] PUNCTUATION_LABELS_PHONE = { + "!", "?", ",", ":", ";", "\"", "(", ")", "'", "-", "/", "@", "_" + }; + private static final String[] PUNCTUATION_WORDS_PHONE_LTR = PUNCTUATION_LABELS_PHONE; + private static final String[] PUNCTUATION_WORDS_PHONE_HEBREW = { + "!", "?", ",", ":", ";", "\"", ")", "(", "'", "-", "/", "@", "_" + }; + // U+061F: "؟" ARABIC QUESTION MARK + // U+060C: "،" ARABIC COMMA + // U+061B: "؛" ARABIC SEMICOLON + private static final String[] PUNCTUATION_LABELS_PHONE_ARABIC_PERSIAN = { + "!", "\u061F", "\u060C", ":", "\u061B", "\"", "(", ")", "'", "-", "/", "@", "_" + }; + private static final String[] PUNCTUATION_WORDS_PHONE_ARABIC_PERSIAN = { + "!", "\u061F", "\u060C", ":", "\u061B", "\"", ")", "(", "'", "-", "/", "@", "_" + }; + + // Punctuations for tablet. + private static final String[] PUNCTUATION_LABELS_TABLET = { + ":", ";", "\"", "(", ")", "'", "-", "/", "@", "_" + }; + private static final String[] PUNCTUATION_WORDS_TABLET_LTR = PUNCTUATION_LABELS_TABLET; + private static final String[] PUNCTUATION_WORDS_TABLET_HEBREW = { + ":", ";", "\"", ")", "(", "'", "-", "/", "@", "_" + }; + private static final String[] PUNCTUATION_LABELS_TABLET_ARABIC_PERSIAN = { + "!", "\u061F", ":", "\u061B", "\"", "'", "(", ")", "-", "/", "@", "_" + }; + private static final String[] PUNCTUATION_WORDS_TABLET_ARABIC_PERSIAN = { + "!", "\u061F", ":", "\u061B", "\"", "'", ")", "(", "-", "/", "@", "_" + }; + + private static void testingStandardPunctuationSuggestions(final SpacingAndPunctuations sp, + final String[] punctuationLabels, final String[] punctuationWords) { + final SuggestedWords suggestedWords = sp.mSuggestPuncList; + assertFalse("typedWordValid", suggestedWords.mTypedWordValid); + assertFalse("willAutoCorrect", suggestedWords.mWillAutoCorrect); + assertTrue("isPunctuationSuggestions", suggestedWords.isPunctuationSuggestions()); + assertFalse("isObsoleteSuggestions", suggestedWords.mIsObsoleteSuggestions); + assertFalse("isPrediction", suggestedWords.mIsPrediction); + assertEquals("size", punctuationLabels.length, suggestedWords.size()); + for (int index = 0; index < suggestedWords.size(); index++) { + assertEquals("punctuation label at " + index, + punctuationLabels[index], suggestedWords.getLabel(index)); + assertEquals("punctuation word at " + index, + punctuationWords[index], suggestedWords.getWord(index)); + } + } + + public void testPhonePunctuationSuggestions() { + if (!isPhone()) { + return; + } + testingStandardPunctuationSuggestions(ENGLISH, + PUNCTUATION_LABELS_PHONE, PUNCTUATION_WORDS_PHONE_LTR); + testingStandardPunctuationSuggestions(FRENCH, + PUNCTUATION_LABELS_PHONE, PUNCTUATION_WORDS_PHONE_LTR); + testingStandardPunctuationSuggestions(GERMAN, + PUNCTUATION_LABELS_PHONE, PUNCTUATION_WORDS_PHONE_LTR); + testingStandardPunctuationSuggestions(ARABIC, + PUNCTUATION_LABELS_PHONE_ARABIC_PERSIAN, PUNCTUATION_WORDS_PHONE_ARABIC_PERSIAN); + testingStandardPunctuationSuggestions(PERSIAN, + PUNCTUATION_LABELS_PHONE_ARABIC_PERSIAN, PUNCTUATION_WORDS_PHONE_ARABIC_PERSIAN); + testingStandardPunctuationSuggestions(HEBREW, + PUNCTUATION_LABELS_PHONE, PUNCTUATION_WORDS_PHONE_HEBREW); + } + + public void testTabletPunctuationSuggestions() { + if (!isTablet()) { + return; + } + testingStandardPunctuationSuggestions(ENGLISH, + PUNCTUATION_LABELS_TABLET, PUNCTUATION_WORDS_TABLET_LTR); + testingStandardPunctuationSuggestions(FRENCH, + PUNCTUATION_LABELS_TABLET, PUNCTUATION_WORDS_TABLET_LTR); + testingStandardPunctuationSuggestions(GERMAN, + PUNCTUATION_LABELS_TABLET, PUNCTUATION_WORDS_TABLET_LTR); + testingStandardPunctuationSuggestions(ARABIC, + PUNCTUATION_LABELS_TABLET_ARABIC_PERSIAN, PUNCTUATION_WORDS_TABLET_ARABIC_PERSIAN); + testingStandardPunctuationSuggestions(PERSIAN, + PUNCTUATION_LABELS_TABLET_ARABIC_PERSIAN, PUNCTUATION_WORDS_TABLET_ARABIC_PERSIAN); + testingStandardPunctuationSuggestions(HEBREW, + PUNCTUATION_LABELS_TABLET, PUNCTUATION_WORDS_TABLET_HEBREW); + } +} diff --git a/tests/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerServiceTest.java b/tests/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerServiceTest.java index 995d7f07b..2272d6ba0 100644 --- a/tests/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerServiceTest.java +++ b/tests/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerServiceTest.java @@ -39,7 +39,7 @@ public class AndroidSpellCheckerServiceTest extends InputTestsBase { // it yields 5). assertTrue(suggestions.length >= 2); // We also assume the top suggestion should be "this". - assertEquals("", "this", suggestions[0]); + assertEquals("Test basic spell checking", "this", suggestions[0]); } public void testRussianSpellchecker() { @@ -62,4 +62,21 @@ public class AndroidSpellCheckerServiceTest extends InputTestsBase { // Russian dictionary. assertEquals("", "года", suggestions[0]); } + + public void testSpellcheckWithPeriods() { + changeLanguage("en_US"); + mEditText.setText("I'm.sure "); + mEditText.setSelection(mEditText.getText().length()); + mEditText.onAttachedToWindow(); + sleep(1000); + runMessages(); + sleep(1000); + + final SpanGetter span = new SpanGetter(mEditText.getText(), SuggestionSpan.class); + // If no span, the following will crash + final String[] suggestions = span.getSuggestions(); + // The first suggestion should be "I'm sure". + assertEquals("Test spell checking of mistyped period for space", "I'm sure", + suggestions[0]); + } } diff --git a/tests/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtilsTests.java new file mode 100644 index 000000000..91c9c3775 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtilsTests.java @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.content.Context; +import android.os.Build; +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; +import android.view.inputmethod.InputMethodSubtype; + +import com.android.inputmethod.compat.InputMethodSubtypeCompatUtils; + +import java.util.Locale; + +import static com.android.inputmethod.latin.Constants.Subtype.KEYBOARD_MODE; +import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.ASCII_CAPABLE; +import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.EMOJI_CAPABLE; +import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.IS_ADDITIONAL_SUBTYPE; +import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.KEYBOARD_LAYOUT_SET; +import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue + .UNTRANSLATABLE_STRING_IN_SUBTYPE_NAME; + +@SmallTest +public class AdditionalSubtypeUtilsTests extends AndroidTestCase { + + /** + * Predictable subtype ID for en_US dvorak layout. This is actually a hash code calculated as + * follows. + * <code> + * final boolean isAuxiliary = false; + * final boolean overrideImplicitlyEnabledSubtype = false; + * final int SUBTYPE_ID_EN_US_DVORAK = Arrays.hashCode(new Object[] { + * "en_US", + * "keyboard", + * "KeyboardLayoutSet=dvorak" + * + ",AsciiCapable" + * + ",UntranslatableReplacementStringInSubtypeName=Dvorak" + * + ",EmojiCapable" + * + ",isAdditionalSubtype", + * isAuxiliary, + * overrideImplicitlyEnabledSubtype }); + * </code> + */ + private static int SUBTYPE_ID_EN_US_DVORAK = 0xb3c0cc56; + private static String EXTRA_VALUE_EN_US_DVORAK_ICS = + "KeyboardLayoutSet=dvorak" + + ",AsciiCapable" + + ",isAdditionalSubtype"; + private static String EXTRA_VALUE_EN_US_DVORAK_JELLY_BEAN = + "KeyboardLayoutSet=dvorak" + + ",AsciiCapable" + + ",UntranslatableReplacementStringInSubtypeName=Dvorak" + + ",isAdditionalSubtype"; + private static String EXTRA_VALUE_EN_US_DVORAK_KITKAT = + "KeyboardLayoutSet=dvorak" + + ",AsciiCapable" + + ",UntranslatableReplacementStringInSubtypeName=Dvorak" + + ",EmojiCapable" + + ",isAdditionalSubtype"; + + /** + * Predictable subtype ID for azerty layout. This is actually a hash code calculated as follows. + * <code> + * final boolean isAuxiliary = false; + * final boolean overrideImplicitlyEnabledSubtype = false; + * final int SUBTYPE_ID_ZZ_AZERTY = Arrays.hashCode(new Object[] { + * "zz", + * "keyboard", + * "KeyboardLayoutSet=azerty" + * + ",AsciiCapable" + * + ",EmojiCapable" + * + ",isAdditionalSubtype", + * isAuxiliary, + * overrideImplicitlyEnabledSubtype }); + * </code> + */ + private static int SUBTYPE_ID_ZZ_AZERTY = 0x5b6be697; + private static String EXTRA_VALUE_ZZ_AZERTY_ICS = + "KeyboardLayoutSet=azerty" + + ",AsciiCapable" + + ",isAdditionalSubtype"; + private static String EXTRA_VALUE_ZZ_AZERTY_KITKAT = + "KeyboardLayoutSet=azerty" + + ",AsciiCapable" + + ",EmojiCapable" + + ",isAdditionalSubtype"; + + @Override + protected void setUp() throws Exception { + super.setUp(); + final Context context = getContext(); + SubtypeLocaleUtils.init(context); + } + + private static void assertEnUsDvorak(InputMethodSubtype subtype) { + assertEquals("en_US", subtype.getLocale()); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + assertEquals(EXTRA_VALUE_EN_US_DVORAK_KITKAT, subtype.getExtraValue()); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) { + assertEquals(EXTRA_VALUE_EN_US_DVORAK_JELLY_BEAN, subtype.getExtraValue()); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) { + assertEquals(EXTRA_VALUE_EN_US_DVORAK_ICS, subtype.getExtraValue()); + } + assertTrue(subtype.containsExtraValueKey(ASCII_CAPABLE)); + assertTrue(InputMethodSubtypeCompatUtils.isAsciiCapable(subtype)); + // TODO: Enable following test + // if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + // assertTrue(InputMethodSubtypeCompatUtils.isAsciiCapableWithAPI(subtype)); + // } + assertTrue(subtype.containsExtraValueKey(EMOJI_CAPABLE)); + assertTrue(subtype.containsExtraValueKey(IS_ADDITIONAL_SUBTYPE)); + assertEquals("dvorak", subtype.getExtraValueOf(KEYBOARD_LAYOUT_SET)); + assertEquals("Dvorak", subtype.getExtraValueOf(UNTRANSLATABLE_STRING_IN_SUBTYPE_NAME)); + assertEquals(KEYBOARD_MODE, subtype.getMode()); + assertEquals(SUBTYPE_ID_EN_US_DVORAK, subtype.hashCode()); + } + + private static void assertAzerty(InputMethodSubtype subtype) { + assertEquals("zz", subtype.getLocale()); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + assertEquals(EXTRA_VALUE_ZZ_AZERTY_KITKAT, subtype.getExtraValue()); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) { + assertEquals(EXTRA_VALUE_ZZ_AZERTY_ICS, subtype.getExtraValue()); + } + assertTrue(subtype.containsExtraValueKey(ASCII_CAPABLE)); + assertTrue(InputMethodSubtypeCompatUtils.isAsciiCapable(subtype)); + // TODO: Enable following test + // if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + // assertTrue(InputMethodSubtypeCompatUtils.isAsciiCapableWithAPI(subtype)); + // } + assertTrue(subtype.containsExtraValueKey(EMOJI_CAPABLE)); + assertTrue(subtype.containsExtraValueKey(IS_ADDITIONAL_SUBTYPE)); + assertEquals("azerty", subtype.getExtraValueOf(KEYBOARD_LAYOUT_SET)); + assertFalse(subtype.containsExtraValueKey(UNTRANSLATABLE_STRING_IN_SUBTYPE_NAME)); + assertEquals(KEYBOARD_MODE, subtype.getMode()); + assertEquals(SUBTYPE_ID_ZZ_AZERTY, subtype.hashCode()); + } + + public void testRestorable() { + final InputMethodSubtype EN_UK_DVORAK = + AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.US.toString(), "dvorak"); + final InputMethodSubtype ZZ_AZERTY = + AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + SubtypeLocaleUtils.NO_LANGUAGE, "azerty"); + assertEnUsDvorak(EN_UK_DVORAK); + assertAzerty(ZZ_AZERTY); + + // Make sure the subtype can be stored and restored in a deterministic manner. + final InputMethodSubtype[] subtypes = { EN_UK_DVORAK, ZZ_AZERTY }; + final String prefSubtype = AdditionalSubtypeUtils.createPrefSubtypes(subtypes); + final InputMethodSubtype[] restoredSubtypes = + AdditionalSubtypeUtils.createAdditionalSubtypesArray(prefSubtype); + assertEquals(2, restoredSubtypes.length); + final InputMethodSubtype restored_EN_UK_DVORAK = restoredSubtypes[0]; + final InputMethodSubtype restored_ZZ_AZERTY = restoredSubtypes[1]; + + assertEnUsDvorak(restored_EN_UK_DVORAK); + assertAzerty(restored_ZZ_AZERTY); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java b/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java index 7fd167977..1501e942a 100644 --- a/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java @@ -45,27 +45,27 @@ public class AsyncResultHolderTests extends AndroidTestCase { } public void testGetWithoutSet() { - final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<>(); final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); assertEquals(DEFAULT_VALUE, resultValue); } public void testGetBeforeSet() { - final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<>(); setAfterGivenTime(holder, SET_VALUE, TIMEOUT_IN_MILLISECONDS + MARGIN_IN_MILLISECONDS); final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); assertEquals(DEFAULT_VALUE, resultValue); } public void testGetAfterSet() { - final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<>(); holder.set(SET_VALUE); final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); assertEquals(SET_VALUE, resultValue); } public void testGetBeforeTimeout() { - final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<>(); setAfterGivenTime(holder, SET_VALUE, TIMEOUT_IN_MILLISECONDS - MARGIN_IN_MILLISECONDS); final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); assertEquals(SET_VALUE, resultValue); diff --git a/tests/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtilsTests.java new file mode 100644 index 000000000..a333ee9bc --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtilsTests.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; + +import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.makedict.DictionaryHeader; +import com.android.inputmethod.latin.makedict.FormatSpec; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@LargeTest +public class BinaryDictionaryUtilsTests extends AndroidTestCase { + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final String TEST_LOCALE = "test"; + + private File createEmptyDictionaryAndGetFile(final String dictId, + final int formatVersion) throws IOException { + if (formatVersion == FormatSpec.VERSION4) { + return createEmptyVer4DictionaryAndGetFile(dictId); + } else { + throw new IOException("Dictionary format version " + formatVersion + + " is not supported."); + } + } + + private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException { + final File file = getDictFile(dictId); + FileUtils.deleteRecursively(file); + Map<String, String> attributeMap = new HashMap<>(); + attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId); + attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, + String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); + attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, + LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) { + return file; + } else { + throw new IOException("Empty dictionary " + file.getAbsolutePath() + + " cannot be created."); + } + } + + private File getDictFile(final String dictId) { + return new File(getContext().getCacheDir(), dictId + TEST_DICT_FILE_EXTENSION); + } + + public void testRenameDictionary() { + final int formatVersion = FormatSpec.VERSION4; + File dictFile0 = null; + try { + dictFile0 = createEmptyDictionaryAndGetFile("MoveFromDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final File dictFile1 = getDictFile("MoveToDictionary"); + FileUtils.deleteRecursively(dictFile1); + assertTrue(BinaryDictionaryUtils.renameDict(dictFile0, dictFile1)); + assertFalse(dictFile0.exists()); + assertTrue(dictFile1.exists()); + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile1.getAbsolutePath(), + 0 /* offset */, dictFile1.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + assertTrue(binaryDictionary.isValidDictionary()); + assertTrue(binaryDictionary.getFormatVersion() == formatVersion); + binaryDictionary.close(); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java b/tests/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java new file mode 100644 index 000000000..2028298f2 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; + +/** + * This class provides an implementation for the FusionDictionary buffer interface that is backed + * by a simpled byte array. It allows to create a binary dictionary in memory. + */ +public final class ByteArrayDictBuffer implements DictBuffer { + private byte[] mBuffer; + private int mPosition; + + public ByteArrayDictBuffer(final byte[] buffer) { + mBuffer = buffer; + mPosition = 0; + } + + @Override + public int readUnsignedByte() { + return mBuffer[mPosition++] & 0xFF; + } + + @Override + public int readUnsignedShort() { + final int retval = readUnsignedByte(); + return (retval << 8) + readUnsignedByte(); + } + + @Override + public int readUnsignedInt24() { + final int retval = readUnsignedShort(); + return (retval << 8) + readUnsignedByte(); + } + + @Override + public int readInt() { + final int retval = readUnsignedShort(); + return (retval << 16) + readUnsignedShort(); + } + + @Override + public int position() { + return mPosition; + } + + @Override + public void position(int position) { + mPosition = position; + } + + @Override + public void put(final byte b) { + mBuffer[mPosition++] = b; + } + + @Override + public int limit() { + return mBuffer.length - 1; + } + + @Override + public int capacity() { + return mBuffer.length; + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/CapsModeUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/CapsModeUtilsTests.java index 1fd5c989a..c746c8345 100644 --- a/tests/src/com/android/inputmethod/latin/utils/CapsModeUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/CapsModeUtilsTests.java @@ -16,75 +16,113 @@ package com.android.inputmethod.latin.utils; -import com.android.inputmethod.latin.settings.SettingsValues; - +import android.content.res.Resources; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; import android.text.TextUtils; +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; +import com.android.inputmethod.latin.utils.LocaleUtils; + import java.util.Locale; @SmallTest public class CapsModeUtilsTests extends AndroidTestCase { private static void onePathForCaps(final CharSequence cs, final int expectedResult, - final int mask, final SettingsValues sv, final boolean hasSpaceBefore) { - int oneTimeResult = expectedResult & mask; + final int mask, final SpacingAndPunctuations sp, final boolean hasSpaceBefore) { + final int oneTimeResult = expectedResult & mask; assertEquals("After >" + cs + "<", oneTimeResult, - CapsModeUtils.getCapsMode(cs, mask, sv, hasSpaceBefore)); + CapsModeUtils.getCapsMode(cs, mask, sp, hasSpaceBefore)); } private static void allPathsForCaps(final CharSequence cs, final int expectedResult, - final SettingsValues sv, final boolean hasSpaceBefore) { + final SpacingAndPunctuations sp, final boolean hasSpaceBefore) { final int c = TextUtils.CAP_MODE_CHARACTERS; final int w = TextUtils.CAP_MODE_WORDS; final int s = TextUtils.CAP_MODE_SENTENCES; - onePathForCaps(cs, expectedResult, c | w | s, sv, hasSpaceBefore); - onePathForCaps(cs, expectedResult, w | s, sv, hasSpaceBefore); - onePathForCaps(cs, expectedResult, c | s, sv, hasSpaceBefore); - onePathForCaps(cs, expectedResult, c | w, sv, hasSpaceBefore); - onePathForCaps(cs, expectedResult, c, sv, hasSpaceBefore); - onePathForCaps(cs, expectedResult, w, sv, hasSpaceBefore); - onePathForCaps(cs, expectedResult, s, sv, hasSpaceBefore); + onePathForCaps(cs, expectedResult, c | w | s, sp, hasSpaceBefore); + onePathForCaps(cs, expectedResult, w | s, sp, hasSpaceBefore); + onePathForCaps(cs, expectedResult, c | s, sp, hasSpaceBefore); + onePathForCaps(cs, expectedResult, c | w, sp, hasSpaceBefore); + onePathForCaps(cs, expectedResult, c, sp, hasSpaceBefore); + onePathForCaps(cs, expectedResult, w, sp, hasSpaceBefore); + onePathForCaps(cs, expectedResult, s, sp, hasSpaceBefore); } public void testGetCapsMode() { final int c = TextUtils.CAP_MODE_CHARACTERS; final int w = TextUtils.CAP_MODE_WORDS; final int s = TextUtils.CAP_MODE_SENTENCES; - SettingsValues sv = SettingsValues.makeDummySettingsValuesForTest(Locale.ENGLISH); - allPathsForCaps("", c | w | s, sv, false); - allPathsForCaps("Word", c, sv, false); - allPathsForCaps("Word.", c, sv, false); - allPathsForCaps("Word ", c | w, sv, false); - allPathsForCaps("Word. ", c | w | s, sv, false); - allPathsForCaps("Word..", c, sv, false); - allPathsForCaps("Word.. ", c | w | s, sv, false); - allPathsForCaps("Word... ", c | w | s, sv, false); - allPathsForCaps("Word ... ", c | w | s, sv, false); - allPathsForCaps("Word . ", c | w, sv, false); - allPathsForCaps("In the U.S ", c | w, sv, false); - allPathsForCaps("In the U.S. ", c | w, sv, false); - allPathsForCaps("Some stuff (e.g. ", c | w, sv, false); - allPathsForCaps("In the U.S.. ", c | w | s, sv, false); - allPathsForCaps("\"Word.\" ", c | w | s, sv, false); - allPathsForCaps("\"Word\". ", c | w | s, sv, false); - allPathsForCaps("\"Word\" ", c | w, sv, false); + final RunInLocale<SpacingAndPunctuations> job = new RunInLocale<SpacingAndPunctuations>() { + @Override + protected SpacingAndPunctuations job(final Resources res) { + return new SpacingAndPunctuations(res); + } + }; + final Resources res = getContext().getResources(); + SpacingAndPunctuations sp = job.runInLocale(res, Locale.ENGLISH); + allPathsForCaps("", c | w | s, sp, false); + allPathsForCaps("Word", c, sp, false); + allPathsForCaps("Word.", c, sp, false); + allPathsForCaps("Word ", c | w, sp, false); + allPathsForCaps("Word. ", c | w | s, sp, false); + allPathsForCaps("Word..", c, sp, false); + allPathsForCaps("Word.. ", c | w | s, sp, false); + allPathsForCaps("Word... ", c | w | s, sp, false); + allPathsForCaps("Word ... ", c | w | s, sp, false); + allPathsForCaps("Word . ", c | w, sp, false); + allPathsForCaps("In the U.S ", c | w, sp, false); + allPathsForCaps("In the U.S. ", c | w, sp, false); + allPathsForCaps("Some stuff (e.g. ", c | w, sp, false); + allPathsForCaps("In the U.S.. ", c | w | s, sp, false); + allPathsForCaps("\"Word.\" ", c | w | s, sp, false); + allPathsForCaps("\"Word\". ", c | w | s, sp, false); + allPathsForCaps("\"Word\" ", c | w, sp, false); // Test for phantom space - allPathsForCaps("Word", c | w, sv, true); - allPathsForCaps("Word.", c | w | s, sv, true); + allPathsForCaps("Word", c | w, sp, true); + allPathsForCaps("Word.", c | w | s, sp, true); // Tests after some whitespace - allPathsForCaps("Word\n", c | w | s, sv, false); - allPathsForCaps("Word\n", c | w | s, sv, true); - allPathsForCaps("Word\n ", c | w | s, sv, true); - allPathsForCaps("Word.\n", c | w | s, sv, false); - allPathsForCaps("Word.\n", c | w | s, sv, true); - allPathsForCaps("Word.\n ", c | w | s, sv, true); + allPathsForCaps("Word\n", c | w | s, sp, false); + allPathsForCaps("Word\n", c | w | s, sp, true); + allPathsForCaps("Word\n ", c | w | s, sp, true); + allPathsForCaps("Word.\n", c | w | s, sp, false); + allPathsForCaps("Word.\n", c | w | s, sp, true); + allPathsForCaps("Word.\n ", c | w | s, sp, true); + + sp = job.runInLocale(res, Locale.FRENCH); + allPathsForCaps("\"Word.\" ", c | w, sp, false); + allPathsForCaps("\"Word\". ", c | w | s, sp, false); + allPathsForCaps("\"Word\" ", c | w, sp, false); + + // Test special case for German. German does not capitalize at the start of a + // line when the previous line starts with a comma. It does in other cases. + sp = job.runInLocale(res, Locale.GERMAN); + allPathsForCaps("Liebe Sara,\n", c | w, sp, false); + allPathsForCaps("Liebe Sara,\n", c | w, sp, true); + allPathsForCaps("Liebe Sara, \n ", c | w, sp, false); + allPathsForCaps("Liebe Sara \n ", c | w | s, sp, false); + allPathsForCaps("Liebe Sara.\n ", c | w | s, sp, false); + sp = job.runInLocale(res, Locale.ENGLISH); + allPathsForCaps("Liebe Sara,\n", c | w | s, sp, false); + allPathsForCaps("Liebe Sara,\n", c | w | s, sp, true); + allPathsForCaps("Liebe Sara, \n ", c | w | s, sp, false); + allPathsForCaps("Liebe Sara \n ", c | w | s, sp, false); + allPathsForCaps("Liebe Sara.\n ", c | w | s, sp, false); - sv = SettingsValues.makeDummySettingsValuesForTest(Locale.FRENCH); - allPathsForCaps("\"Word.\" ", c | w, sv, false); - allPathsForCaps("\"Word\". ", c | w | s, sv, false); - allPathsForCaps("\"Word\" ", c | w, sv, false); + // Test armenian period + sp = job.runInLocale(res, LocaleUtils.constructLocaleFromString("hy_AM")); + assertTrue("Period is not sentence separator in Armenian", + !sp.isSentenceSeparator('.')); + assertTrue("Sentence separator is Armenian period in Armenian", + sp.isSentenceSeparator(0x589)); + // No space : capitalize only if MODE_CHARACTERS + allPathsForCaps("Word", c, sp, false); + allPathsForCaps("Word.", c, sp, false); + // Space, but no armenian period : capitalize if MODE_WORDS but not SENTENCES + allPathsForCaps("Word. ", c | w, sp, false); + // Armenian period : capitalize if MODE_SENTENCES + allPathsForCaps("Word\u0589 ", c | w | s, sp, false); } } diff --git a/tests/src/com/android/inputmethod/latin/utils/DictionaryInfoUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/DictionaryInfoUtilsTests.java new file mode 100644 index 000000000..6e716074c --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/DictionaryInfoUtilsTests.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.content.res.Resources; +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; + +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; + +import java.util.Locale; + +@SmallTest +public class DictionaryInfoUtilsTests extends AndroidTestCase { + public void testLooksValidForDictionaryInsertion() { + final RunInLocale<SpacingAndPunctuations> job = new RunInLocale<SpacingAndPunctuations>() { + @Override + protected SpacingAndPunctuations job(final Resources res) { + return new SpacingAndPunctuations(res); + } + }; + final Resources res = getContext().getResources(); + final SpacingAndPunctuations sp = job.runInLocale(res, Locale.ENGLISH); + assertTrue(DictionaryInfoUtils.looksValidForDictionaryInsertion("aochaueo", sp)); + assertFalse(DictionaryInfoUtils.looksValidForDictionaryInsertion("", sp)); + assertTrue(DictionaryInfoUtils.looksValidForDictionaryInsertion("ao-ch'aueo", sp)); + assertFalse(DictionaryInfoUtils.looksValidForDictionaryInsertion("2908743256", sp)); + assertTrue(DictionaryInfoUtils.looksValidForDictionaryInsertion("31aochaueo", sp)); + assertFalse(DictionaryInfoUtils.looksValidForDictionaryInsertion("akeo raeoch oerch .", + sp)); + assertFalse(DictionaryInfoUtils.looksValidForDictionaryInsertion("!!!", sp)); + } +} diff --git a/tests/src/com/android/inputmethod/latin/EditDistanceTests.java b/tests/src/com/android/inputmethod/latin/utils/EditDistanceTests.java index 0b7fcbbe8..58312264b 100644 --- a/tests/src/com/android/inputmethod/latin/EditDistanceTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/EditDistanceTests.java @@ -14,23 +14,13 @@ * limitations under the License. */ -package com.android.inputmethod.latin; +package com.android.inputmethod.latin.utils; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; @SmallTest public class EditDistanceTests extends AndroidTestCase { - @Override - protected void setUp() throws Exception { - super.setUp(); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - } - /* * dist(kitten, sitting) == 3 * @@ -39,7 +29,7 @@ public class EditDistanceTests extends AndroidTestCase { * sitting */ public void testExample1() { - final int dist = BinaryDictionary.editDistance("kitten", "sitting"); + final int dist = BinaryDictionaryUtils.editDistance("kitten", "sitting"); assertEquals("edit distance between 'kitten' and 'sitting' is 3", 3, dist); } @@ -52,26 +42,26 @@ public class EditDistanceTests extends AndroidTestCase { * S--unday */ public void testExample2() { - final int dist = BinaryDictionary.editDistance("Saturday", "Sunday"); + final int dist = BinaryDictionaryUtils.editDistance("Saturday", "Sunday"); assertEquals("edit distance between 'Saturday' and 'Sunday' is 3", 3, dist); } public void testBothEmpty() { - final int dist = BinaryDictionary.editDistance("", ""); + final int dist = BinaryDictionaryUtils.editDistance("", ""); assertEquals("when both string are empty, no edits are needed", 0, dist); } public void testFirstArgIsEmpty() { - final int dist = BinaryDictionary.editDistance("", "aaaa"); + final int dist = BinaryDictionaryUtils.editDistance("", "aaaa"); assertEquals("when only one string of the arguments is empty," + " the edit distance is the length of the other.", 4, dist); } public void testSecoondArgIsEmpty() { - final int dist = BinaryDictionary.editDistance("aaaa", ""); + final int dist = BinaryDictionaryUtils.editDistance("aaaa", ""); assertEquals("when only one string of the arguments is empty," + " the edit distance is the length of the other.", 4, dist); @@ -80,27 +70,27 @@ public class EditDistanceTests extends AndroidTestCase { public void testSameStrings() { final String arg1 = "The quick brown fox jumps over the lazy dog."; final String arg2 = "The quick brown fox jumps over the lazy dog."; - final int dist = BinaryDictionary.editDistance(arg1, arg2); + final int dist = BinaryDictionaryUtils.editDistance(arg1, arg2); assertEquals("when same strings are passed, distance equals 0.", 0, dist); } public void testSameReference() { final String arg = "The quick brown fox jumps over the lazy dog."; - final int dist = BinaryDictionary.editDistance(arg, arg); + final int dist = BinaryDictionaryUtils.editDistance(arg, arg); assertEquals("when same string references are passed, the distance equals 0.", 0, dist); } public void testNullArg() { try { - BinaryDictionary.editDistance(null, "aaa"); + BinaryDictionaryUtils.editDistance(null, "aaa"); fail("IllegalArgumentException should be thrown."); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); } try { - BinaryDictionary.editDistance("aaa", null); + BinaryDictionaryUtils.editDistance("aaa", null); fail("IllegalArgumentException should be thrown."); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); diff --git a/tests/src/com/android/inputmethod/latin/utils/ExecutorUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/ExecutorUtilsTests.java new file mode 100644 index 000000000..ae2623d12 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/ExecutorUtilsTests.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.MediumTest; +import android.util.Log; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Unit tests for ExecutorUtils. + */ +@MediumTest +public class ExecutorUtilsTests extends AndroidTestCase { + private static final String TAG = ExecutorUtilsTests.class.getSimpleName(); + + private static final String TEST_EXECUTOR_ID = "test"; + private static final int NUM_OF_TASKS = 10; + private static final int DELAY_FOR_WAITING_TASKS_MILLISECONDS = 500; + + public void testExecute() { + final ExecutorService executor = ExecutorUtils.getExecutor(TEST_EXECUTOR_ID); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.execute(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + try { + executor.awaitTermination(DELAY_FOR_WAITING_TASKS_MILLISECONDS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(NUM_OF_TASKS, v.get()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/ForgettingCurveTests.java b/tests/src/com/android/inputmethod/latin/utils/ForgettingCurveTests.java deleted file mode 100644 index 823bd5d7d..000000000 --- a/tests/src/com/android/inputmethod/latin/utils/ForgettingCurveTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.SmallTest; - -@SmallTest -public class ForgettingCurveTests extends AndroidTestCase { - public void testFcToFreq() { - for (int i = 0; i < Byte.MAX_VALUE; ++i) { - final byte fc = (byte)i; - final int e = UserHistoryForgettingCurveUtils.fcToElapsedTime(fc); - final int c = UserHistoryForgettingCurveUtils.fcToCount(fc); - final int l = UserHistoryForgettingCurveUtils.fcToLevel(fc); - final byte fc2 = UserHistoryForgettingCurveUtils.calcFc(e, c, l); - assertEquals(fc, fc2); - } - byte fc = 0; - int l; - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < (UserHistoryForgettingCurveUtils.COUNT_MAX + 1); ++j) { - fc = UserHistoryForgettingCurveUtils.pushCount(fc, true); - } - l = UserHistoryForgettingCurveUtils.fcToLevel(fc); - assertEquals(l, Math.max(1, Math.min(i + 1, 3))); - } - fc = 0; - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < (UserHistoryForgettingCurveUtils.COUNT_MAX + 1); ++j) { - fc = UserHistoryForgettingCurveUtils.pushCount(fc, false); - } - l = UserHistoryForgettingCurveUtils.fcToLevel(fc); - assertEquals(l, Math.min(i + 1, 3)); - } - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < (UserHistoryForgettingCurveUtils.ELAPSED_TIME_MAX + 1); ++j) { - fc = UserHistoryForgettingCurveUtils.pushElapsedTime(fc); - } - l = UserHistoryForgettingCurveUtils.fcToLevel(fc); - assertEquals(l, Math.max(0, 2 - i)); - } - } -} diff --git a/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java b/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java deleted file mode 100644 index e0755483c..000000000 --- a/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.MediumTest; -import android.util.Log; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Unit tests for PrioritizedSerialExecutor. - * TODO: Add more detailed tests to make use of priorities, etc. - */ -@MediumTest -public class PrioritizedSerialExecutorTests extends AndroidTestCase { - private static final String TAG = PrioritizedSerialExecutorTests.class.getSimpleName(); - - private static final int NUM_OF_TASKS = 10; - private static final int DELAY_FOR_WAITING_TASKS_MILLISECONDS = 500; - - public void testExecute() { - final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); - final AtomicInteger v = new AtomicInteger(0); - for (int i = 0; i < NUM_OF_TASKS; ++i) { - executor.execute(new Runnable() { - @Override - public void run() { - v.incrementAndGet(); - } - }); - } - try { - Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); - } catch (InterruptedException e) { - Log.d(TAG, "Exception while sleeping.", e); - } - - assertEquals(NUM_OF_TASKS, v.get()); - } - - public void testExecutePrioritized() { - final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); - final AtomicInteger v = new AtomicInteger(0); - for (int i = 0; i < NUM_OF_TASKS; ++i) { - executor.executePrioritized(new Runnable() { - @Override - public void run() { - v.incrementAndGet(); - } - }); - } - try { - Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); - } catch (InterruptedException e) { - Log.d(TAG, "Exception while sleeping.", e); - } - - assertEquals(NUM_OF_TASKS, v.get()); - } - - public void testExecuteCombined() { - final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); - final AtomicInteger v = new AtomicInteger(0); - for (int i = 0; i < NUM_OF_TASKS; ++i) { - executor.execute(new Runnable() { - @Override - public void run() { - v.incrementAndGet(); - } - }); - } - - for (int i = 0; i < NUM_OF_TASKS; ++i) { - executor.executePrioritized(new Runnable() { - @Override - public void run() { - v.incrementAndGet(); - } - }); - } - - try { - Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); - } catch (InterruptedException e) { - Log.d(TAG, "Exception while sleeping.", e); - } - - assertEquals(2 * NUM_OF_TASKS, v.get()); - } -} diff --git a/tests/src/com/android/inputmethod/latin/utils/RecapitalizeStatusTests.java b/tests/src/com/android/inputmethod/latin/utils/RecapitalizeStatusTests.java index a52041264..a3f2ce586 100644 --- a/tests/src/com/android/inputmethod/latin/utils/RecapitalizeStatusTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/RecapitalizeStatusTests.java @@ -19,31 +19,35 @@ package com.android.inputmethod.latin.utils; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import com.android.inputmethod.latin.Constants; + import java.util.Locale; @SmallTest public class RecapitalizeStatusTests extends AndroidTestCase { + private static final int[] SPACE = { Constants.CODE_SPACE }; + public void testTrim() { final RecapitalizeStatus status = new RecapitalizeStatus(); - status.initialize(30, 40, "abcdefghij", Locale.ENGLISH, " "); + status.start(30, 40, "abcdefghij", Locale.ENGLISH, SPACE); status.trim(); assertEquals("abcdefghij", status.getRecapitalizedString()); assertEquals(30, status.getNewCursorStart()); assertEquals(40, status.getNewCursorEnd()); - status.initialize(30, 44, " abcdefghij", Locale.ENGLISH, " "); + status.start(30, 44, " abcdefghij", Locale.ENGLISH, SPACE); status.trim(); assertEquals("abcdefghij", status.getRecapitalizedString()); assertEquals(34, status.getNewCursorStart()); assertEquals(44, status.getNewCursorEnd()); - status.initialize(30, 40, "abcdefgh ", Locale.ENGLISH, " "); + status.start(30, 40, "abcdefgh ", Locale.ENGLISH, SPACE); status.trim(); assertEquals("abcdefgh", status.getRecapitalizedString()); assertEquals(30, status.getNewCursorStart()); assertEquals(38, status.getNewCursorEnd()); - status.initialize(30, 45, " abcdefghij ", Locale.ENGLISH, " "); + status.start(30, 45, " abcdefghij ", Locale.ENGLISH, SPACE); status.trim(); assertEquals("abcdefghij", status.getRecapitalizedString()); assertEquals(33, status.getNewCursorStart()); @@ -52,7 +56,7 @@ public class RecapitalizeStatusTests extends AndroidTestCase { public void testRotate() { final RecapitalizeStatus status = new RecapitalizeStatus(); - status.initialize(29, 40, "abcd efghij", Locale.ENGLISH, " "); + status.start(29, 40, "abcd efghij", Locale.ENGLISH, SPACE); status.rotate(); assertEquals("Abcd Efghij", status.getRecapitalizedString()); assertEquals(29, status.getNewCursorStart()); @@ -64,7 +68,7 @@ public class RecapitalizeStatusTests extends AndroidTestCase { status.rotate(); assertEquals("Abcd Efghij", status.getRecapitalizedString()); - status.initialize(29, 40, "Abcd Efghij", Locale.ENGLISH, " "); + status.start(29, 40, "Abcd Efghij", Locale.ENGLISH, SPACE); status.rotate(); assertEquals("ABCD EFGHIJ", status.getRecapitalizedString()); assertEquals(29, status.getNewCursorStart()); @@ -76,7 +80,7 @@ public class RecapitalizeStatusTests extends AndroidTestCase { status.rotate(); assertEquals("ABCD EFGHIJ", status.getRecapitalizedString()); - status.initialize(29, 40, "ABCD EFGHIJ", Locale.ENGLISH, " "); + status.start(29, 40, "ABCD EFGHIJ", Locale.ENGLISH, SPACE); status.rotate(); assertEquals("abcd efghij", status.getRecapitalizedString()); assertEquals(29, status.getNewCursorStart()); @@ -88,7 +92,7 @@ public class RecapitalizeStatusTests extends AndroidTestCase { status.rotate(); assertEquals("abcd efghij", status.getRecapitalizedString()); - status.initialize(29, 39, "AbCDefghij", Locale.ENGLISH, " "); + status.start(29, 39, "AbCDefghij", Locale.ENGLISH, SPACE); status.rotate(); assertEquals("abcdefghij", status.getRecapitalizedString()); assertEquals(29, status.getNewCursorStart()); @@ -102,7 +106,7 @@ public class RecapitalizeStatusTests extends AndroidTestCase { status.rotate(); assertEquals("abcdefghij", status.getRecapitalizedString()); - status.initialize(29, 40, "Abcd efghij", Locale.ENGLISH, " "); + status.start(29, 40, "Abcd efghij", Locale.ENGLISH, SPACE); status.rotate(); assertEquals("abcd efghij", status.getRecapitalizedString()); assertEquals(29, status.getNewCursorStart()); @@ -116,7 +120,8 @@ public class RecapitalizeStatusTests extends AndroidTestCase { status.rotate(); assertEquals("abcd efghij", status.getRecapitalizedString()); - status.initialize(30, 34, "grüß", Locale.GERMAN, " "); status.rotate(); + status.start(30, 34, "grüß", Locale.GERMAN, SPACE); + status.rotate(); assertEquals("Grüß", status.getRecapitalizedString()); assertEquals(30, status.getNewCursorStart()); assertEquals(34, status.getNewCursorEnd()); @@ -133,7 +138,8 @@ public class RecapitalizeStatusTests extends AndroidTestCase { assertEquals(30, status.getNewCursorStart()); assertEquals(34, status.getNewCursorEnd()); - status.initialize(30, 33, "œuf", Locale.FRENCH, " "); status.rotate(); + status.start(30, 33, "œuf", Locale.FRENCH, SPACE); + status.rotate(); assertEquals("Œuf", status.getRecapitalizedString()); assertEquals(30, status.getNewCursorStart()); assertEquals(33, status.getNewCursorEnd()); @@ -150,7 +156,8 @@ public class RecapitalizeStatusTests extends AndroidTestCase { assertEquals(30, status.getNewCursorStart()); assertEquals(33, status.getNewCursorEnd()); - status.initialize(30, 33, "œUf", Locale.FRENCH, " "); status.rotate(); + status.start(30, 33, "œUf", Locale.FRENCH, SPACE); + status.rotate(); assertEquals("œuf", status.getRecapitalizedString()); assertEquals(30, status.getNewCursorStart()); assertEquals(33, status.getNewCursorEnd()); @@ -171,7 +178,8 @@ public class RecapitalizeStatusTests extends AndroidTestCase { assertEquals(30, status.getNewCursorStart()); assertEquals(33, status.getNewCursorEnd()); - status.initialize(30, 35, "école", Locale.FRENCH, " "); status.rotate(); + status.start(30, 35, "école", Locale.FRENCH, SPACE); + status.rotate(); assertEquals("École", status.getRecapitalizedString()); assertEquals(30, status.getNewCursorStart()); assertEquals(35, status.getNewCursorEnd()); diff --git a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java index cad80d5ce..8f58e6873 100644 --- a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java @@ -39,7 +39,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { int[] array2 = null, array3 = null; final int limit = DEFAULT_CAPACITY * 2 + 10; for (int i = 0; i < limit; i++) { - src.add(i); + final int value = i; + src.add(value); assertEquals("length after add " + i, i + 1, src.getLength()); if (i == DEFAULT_CAPACITY) { array2 = src.getPrimitiveArray(); @@ -56,7 +57,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { } } for (int i = 0; i < limit; i++) { - assertEquals("value at " + i, i, src.get(i)); + final int value = i; + assertEquals("value at " + i, value, src.get(i)); } } @@ -64,11 +66,13 @@ public class ResizableIntArrayTests extends AndroidTestCase { final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); final int limit = DEFAULT_CAPACITY * 10, step = DEFAULT_CAPACITY * 2; for (int i = 0; i < limit; i += step) { - src.add(i, i); + final int value = i; + src.addAt(i, value); assertEquals("length after add at " + i, i + 1, src.getLength()); } for (int i = 0; i < limit; i += step) { - assertEquals("value at " + i, i, src.get(i)); + final int value = i; + assertEquals("value at " + i, value, src.get(i)); } } @@ -88,9 +92,10 @@ public class ResizableIntArrayTests extends AndroidTestCase { } final int index = DEFAULT_CAPACITY / 2; - src.add(index, 100); + final int valueAddAt = 100; + src.addAt(index, valueAddAt); assertEquals("legth after add at " + index, index + 1, src.getLength()); - assertEquals("value after add at " + index, 100, src.get(index)); + assertEquals("value after add at " + index, valueAddAt, src.get(index)); assertEquals("value after add at 0", 0, src.get(0)); try { final int value = src.get(src.getLength()); @@ -104,7 +109,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); final int[] array = src.getPrimitiveArray(); for (int i = 0; i < DEFAULT_CAPACITY; i++) { - src.add(i); + final int value = i; + src.add(value); assertEquals("length after add " + i, i + 1, src.getLength()); } @@ -116,7 +122,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { int[] array3 = null; for (int i = 0; i < DEFAULT_CAPACITY; i++) { - src.add(i); + final int value = i; + src.add(value); assertEquals("length after add " + i, i + 1, src.getLength()); if (i == smallerLength) { array3 = src.getPrimitiveArray(); @@ -133,7 +140,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); final int[] array = src.getPrimitiveArray(); for (int i = 0; i < DEFAULT_CAPACITY; i++) { - src.add(i); + final int value = i; + src.add(value); assertEquals("length after add " + i, i + 1, src.getLength()); } @@ -144,11 +152,11 @@ public class ResizableIntArrayTests extends AndroidTestCase { assertNotSame("array after larger setLength", array, array2); assertEquals("array length after larger setLength", largerLength, array2.length); for (int i = 0; i < largerLength; i++) { - final int v = src.get(i); + final int value = i; if (i < DEFAULT_CAPACITY) { - assertEquals("value at " + i, i, v); + assertEquals("value at " + i, value, src.get(i)); } else { - assertEquals("value at " + i, 0, v); + assertEquals("value at " + i, 0, src.get(i)); } } @@ -159,7 +167,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { assertSame("array after smaller setLength", array2, array3); assertEquals("array length after smaller setLength", largerLength, array3.length); for (int i = 0; i < smallerLength; i++) { - assertEquals("value at " + i, i, src.get(i)); + final int value = i; + assertEquals("value at " + i, value, src.get(i)); } } @@ -167,7 +176,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); final int limit = DEFAULT_CAPACITY * 2 + 10; for (int i = 0; i < limit; i++) { - src.add(i); + final int value = i; + src.add(value); } final ResizableIntArray dst = new ResizableIntArray(DEFAULT_CAPACITY); @@ -179,7 +189,8 @@ public class ResizableIntArrayTests extends AndroidTestCase { public void testCopy() { final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); for (int i = 0; i < DEFAULT_CAPACITY; i++) { - src.add(i); + final int value = i; + src.add(value); } final ResizableIntArray dst = new ResizableIntArray(DEFAULT_CAPACITY); @@ -204,119 +215,126 @@ public class ResizableIntArrayTests extends AndroidTestCase { } public void testAppend() { - final int srcLen = DEFAULT_CAPACITY; - final ResizableIntArray src = new ResizableIntArray(srcLen); - for (int i = 0; i < srcLen; i++) { - src.add(i); + final int srcLength = DEFAULT_CAPACITY; + final ResizableIntArray src = new ResizableIntArray(srcLength); + for (int i = 0; i < srcLength; i++) { + final int value = i; + src.add(value); } final ResizableIntArray dst = new ResizableIntArray(DEFAULT_CAPACITY * 2); final int[] array = dst.getPrimitiveArray(); - final int dstLen = DEFAULT_CAPACITY / 2; - for (int i = 0; i < dstLen; i++) { + final int dstLength = DEFAULT_CAPACITY / 2; + for (int i = 0; i < dstLength; i++) { final int value = -i - 1; dst.add(value); } final ResizableIntArray dstCopy = new ResizableIntArray(dst.getLength()); dstCopy.copy(dst); - dst.append(src, 0, 0); - assertEquals("length after append zero", dstLen, dst.getLength()); + final int startPos = 0; + dst.append(src, startPos, 0 /* length */); + assertEquals("length after append zero", dstLength, dst.getLength()); assertSame("array after append zero", array, dst.getPrimitiveArray()); - assertIntArrayEquals("values after append zero", - dstCopy.getPrimitiveArray(), 0, dst.getPrimitiveArray(), 0, dstLen); + assertIntArrayEquals("values after append zero", dstCopy.getPrimitiveArray(), startPos, + dst.getPrimitiveArray(), startPos, dstLength); - dst.append(src, 0, srcLen); - assertEquals("length after append", dstLen + srcLen, dst.getLength()); + dst.append(src, startPos, srcLength); + assertEquals("length after append", dstLength + srcLength, dst.getLength()); assertSame("array after append", array, dst.getPrimitiveArray()); assertTrue("primitive length after append", - dst.getPrimitiveArray().length >= dstLen + srcLen); - assertIntArrayEquals("original values after append", - dstCopy.getPrimitiveArray(), 0, dst.getPrimitiveArray(), 0, dstLen); - assertIntArrayEquals("appended values after append", - src.getPrimitiveArray(), 0, dst.getPrimitiveArray(), dstLen, srcLen); + dst.getPrimitiveArray().length >= dstLength + srcLength); + assertIntArrayEquals("original values after append", dstCopy.getPrimitiveArray(), startPos, + dst.getPrimitiveArray(), startPos, dstLength); + assertIntArrayEquals("appended values after append", src.getPrimitiveArray(), startPos, + dst.getPrimitiveArray(), dstLength, srcLength); - dst.append(src, 0, srcLen); - assertEquals("length after 2nd append", dstLen + srcLen * 2, dst.getLength()); + dst.append(src, startPos, srcLength); + assertEquals("length after 2nd append", dstLength + srcLength * 2, dst.getLength()); assertNotSame("array after 2nd append", array, dst.getPrimitiveArray()); assertTrue("primitive length after 2nd append", - dst.getPrimitiveArray().length >= dstLen + srcLen * 2); + dst.getPrimitiveArray().length >= dstLength + srcLength * 2); assertIntArrayEquals("original values after 2nd append", - dstCopy.getPrimitiveArray(), 0, dst.getPrimitiveArray(), 0, dstLen); + dstCopy.getPrimitiveArray(), startPos, dst.getPrimitiveArray(), startPos, + dstLength); assertIntArrayEquals("appended values after 2nd append", - src.getPrimitiveArray(), 0, dst.getPrimitiveArray(), dstLen, srcLen); + src.getPrimitiveArray(), startPos, dst.getPrimitiveArray(), dstLength, + srcLength); assertIntArrayEquals("appended values after 2nd append", - src.getPrimitiveArray(), 0, dst.getPrimitiveArray(), dstLen + srcLen, srcLen); + src.getPrimitiveArray(), startPos, dst.getPrimitiveArray(), dstLength + srcLength, + srcLength); } public void testFill() { - final int srcLen = DEFAULT_CAPACITY; - final ResizableIntArray src = new ResizableIntArray(srcLen); - for (int i = 0; i < srcLen; i++) { - src.add(i); + final int srcLength = DEFAULT_CAPACITY; + final ResizableIntArray src = new ResizableIntArray(srcLength); + for (int i = 0; i < srcLength; i++) { + final int value = i; + src.add(value); } final int[] array = src.getPrimitiveArray(); - final int startPos = srcLen / 3; - final int length = srcLen / 3; + final int startPos = srcLength / 3; + final int length = srcLength / 3; final int endPos = startPos + length; assertTrue(startPos >= 1); - final int value = 123; + final int fillValue = 123; try { - src.fill(value, -1, length); + src.fill(fillValue, -1 /* startPos */, length); fail("fill from -1 shouldn't succeed"); } catch (IllegalArgumentException e) { // success } try { - src.fill(value, startPos, -1); + src.fill(fillValue, startPos, -1 /* length */); fail("fill negative length shouldn't succeed"); } catch (IllegalArgumentException e) { // success } - src.fill(value, startPos, length); - assertEquals("length after fill", srcLen, src.getLength()); + src.fill(fillValue, startPos, length); + assertEquals("length after fill", srcLength, src.getLength()); assertSame("array after fill", array, src.getPrimitiveArray()); - for (int i = 0; i < srcLen; i++) { - final int v = src.get(i); + for (int i = 0; i < srcLength; i++) { + final int value = i; if (i >= startPos && i < endPos) { - assertEquals("new values after fill at " + i, value, v); + assertEquals("new values after fill at " + i, fillValue, src.get(i)); } else { - assertEquals("unmodified values after fill at " + i, i, v); + assertEquals("unmodified values after fill at " + i, value, src.get(i)); } } - final int length2 = srcLen * 2 - startPos; + final int length2 = srcLength * 2 - startPos; final int largeEnd = startPos + length2; - assertTrue(largeEnd > srcLen); - final int value2 = 456; - src.fill(value2, startPos, length2); + assertTrue(largeEnd > srcLength); + final int fillValue2 = 456; + src.fill(fillValue2, startPos, length2); assertEquals("length after large fill", largeEnd, src.getLength()); assertNotSame("array after large fill", array, src.getPrimitiveArray()); for (int i = 0; i < largeEnd; i++) { - final int v = src.get(i); + final int value = i; if (i >= startPos && i < largeEnd) { - assertEquals("new values after large fill at " + i, value2, v); + assertEquals("new values after large fill at " + i, fillValue2, src.get(i)); } else { - assertEquals("unmodified values after large fill at " + i, i, v); + assertEquals("unmodified values after large fill at " + i, value, src.get(i)); } } final int startPos2 = largeEnd + length2; final int endPos2 = startPos2 + length2; - final int value3 = 789; - src.fill(value3, startPos2, length2); + final int fillValue3 = 789; + src.fill(fillValue3, startPos2, length2); assertEquals("length after disjoint fill", endPos2, src.getLength()); for (int i = 0; i < endPos2; i++) { - final int v = src.get(i); + final int value = i; if (i >= startPos2 && i < endPos2) { - assertEquals("new values after disjoint fill at " + i, value3, v); + assertEquals("new values after disjoint fill at " + i, fillValue3, src.get(i)); } else if (i >= startPos && i < largeEnd) { - assertEquals("unmodified values after disjoint fill at " + i, value2, v); + assertEquals("unmodified values after disjoint fill at " + i, + fillValue2, src.get(i)); } else if (i < startPos) { - assertEquals("unmodified values after disjoint fill at " + i, i, v); + assertEquals("unmodified values after disjoint fill at " + i, value, src.get(i)); } else { - assertEquals("gap values after disjoint fill at " + i, 0, v); + assertEquals("gap values after disjoint fill at " + i, 0, src.get(i)); } } } @@ -346,12 +364,14 @@ public class ResizableIntArrayTests extends AndroidTestCase { final int limit = DEFAULT_CAPACITY * 10; final int shiftAmount = 20; for (int i = 0; i < limit; ++i) { - src.add(i, i); + final int value = i; + src.addAt(i, value); assertEquals("length after add at " + i, i + 1, src.getLength()); } src.shift(shiftAmount); for (int i = 0; i < limit - shiftAmount; ++i) { - assertEquals("value at " + i, i + shiftAmount, src.get(i)); + final int oldValue = i + shiftAmount; + assertEquals("value at " + i, oldValue, src.get(i)); } } } diff --git a/tests/src/com/android/inputmethod/latin/utils/ResourceUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/ResourceUtilsTests.java index 1ae22e307..8e764e40f 100644 --- a/tests/src/com/android/inputmethod/latin/utils/ResourceUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/ResourceUtilsTests.java @@ -19,48 +19,15 @@ package com.android.inputmethod.latin.utils; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; -import com.android.inputmethod.latin.utils.ResourceUtils.DeviceOverridePatternSyntaxError; - import java.util.HashMap; @SmallTest public class ResourceUtilsTests extends AndroidTestCase { - public void testFindDefaultConstant() { - final String[] nullArray = null; - final String[] emptyArray = {}; - final String[] array = { - "HARDWARE=grouper,0.3", - "HARDWARE=mako,0.4", - ",defaultValue1", - "HARDWARE=manta,0.2", - ",defaultValue2", - }; - - try { - assertNull(ResourceUtils.findDefaultConstant(nullArray)); - assertNull(ResourceUtils.findDefaultConstant(emptyArray)); - assertEquals(ResourceUtils.findDefaultConstant(array), "defaultValue1"); - } catch (final DeviceOverridePatternSyntaxError e) { - fail(e.getMessage()); - } - - final String[] errorArray = { - "HARDWARE=grouper,0.3", - "no_comma" - }; - try { - final String defaultValue = ResourceUtils.findDefaultConstant(errorArray); - fail("exception should be thrown: defaultValue=" + defaultValue); - } catch (final DeviceOverridePatternSyntaxError e) { - assertEquals("Array element has no comma: no_comma", e.getMessage()); - } - } - public void testFindConstantForKeyValuePairsSimple() { - final HashMap<String,String> anyKeyValue = CollectionUtils.newHashMap(); + final HashMap<String,String> anyKeyValue = new HashMap<>(); anyKeyValue.put("anyKey", "anyValue"); final HashMap<String,String> nullKeyValue = null; - final HashMap<String,String> emptyKeyValue = CollectionUtils.newHashMap(); + final HashMap<String,String> emptyKeyValue = new HashMap<>(); final String[] nullArray = null; assertNull(ResourceUtils.findConstantForKeyValuePairs(anyKeyValue, nullArray)); @@ -81,7 +48,7 @@ public class ResourceUtilsTests extends AndroidTestCase { "HARDWARE=mako,0.5", }; - final HashMap<String,String> keyValues = CollectionUtils.newHashMap(); + final HashMap<String,String> keyValues = new HashMap<>(); keyValues.put(HARDWARE_KEY, "grouper"); assertEquals("0.3", ResourceUtils.findConstantForKeyValuePairs(keyValues, array)); keyValues.put(HARDWARE_KEY, "mako"); @@ -121,7 +88,7 @@ public class ResourceUtilsTests extends AndroidTestCase { "HARDWARE=mantaray:MODEL=Nexus 10:MANUFACTURER=samsung,0.2" }; - final HashMap<String,String> keyValues = CollectionUtils.newHashMap(); + final HashMap<String,String> keyValues = new HashMap<>(); keyValues.put(HARDWARE_KEY, "grouper"); keyValues.put(MODEL_KEY, "Nexus 7"); keyValues.put(MANUFACTURER_KEY, "asus"); @@ -159,7 +126,7 @@ public class ResourceUtilsTests extends AndroidTestCase { "HARDWARE=manta.*:MODEL=Nexus 10:MANUFACTURER=samsung,0.2" }; - final HashMap<String,String> keyValues = CollectionUtils.newHashMap(); + final HashMap<String,String> keyValues = new HashMap<>(); keyValues.put(HARDWARE_KEY, "grouper"); keyValues.put(MODEL_KEY, "Nexus 7"); keyValues.put(MANUFACTURER_KEY, "asus"); diff --git a/tests/src/com/android/inputmethod/latin/utils/SpacebarLanguagetUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SpacebarLanguagetUtilsTests.java new file mode 100644 index 000000000..fdde34251 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/SpacebarLanguagetUtilsTests.java @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.content.Context; +import android.content.res.Resources; +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; +import android.view.inputmethod.InputMethodInfo; +import android.view.inputmethod.InputMethodSubtype; + +import com.android.inputmethod.latin.RichInputMethodManager; + +import java.util.ArrayList; +import java.util.Locale; + +@SmallTest +public class SpacebarLanguagetUtilsTests extends AndroidTestCase { + // All input method subtypes of LatinIME. + private final ArrayList<InputMethodSubtype> mSubtypesList = new ArrayList<>(); + + private RichInputMethodManager mRichImm; + private Resources mRes; + + InputMethodSubtype EN_US; + InputMethodSubtype EN_GB; + InputMethodSubtype ES_US; + InputMethodSubtype FR; + InputMethodSubtype FR_CA; + InputMethodSubtype FR_CH; + InputMethodSubtype DE; + InputMethodSubtype DE_CH; + InputMethodSubtype ZZ; + InputMethodSubtype DE_QWERTY; + InputMethodSubtype FR_QWERTZ; + InputMethodSubtype EN_US_AZERTY; + InputMethodSubtype EN_UK_DVORAK; + InputMethodSubtype ES_US_COLEMAK; + InputMethodSubtype ZZ_AZERTY; + InputMethodSubtype ZZ_PC; + + @Override + protected void setUp() throws Exception { + super.setUp(); + final Context context = getContext(); + RichInputMethodManager.init(context); + mRichImm = RichInputMethodManager.getInstance(); + mRes = context.getResources(); + SubtypeLocaleUtils.init(context); + + final InputMethodInfo imi = mRichImm.getInputMethodInfoOfThisIme(); + final int subtypeCount = imi.getSubtypeCount(); + for (int index = 0; index < subtypeCount; index++) { + final InputMethodSubtype subtype = imi.getSubtypeAt(index); + mSubtypesList.add(subtype); + } + + EN_US = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + Locale.US.toString(), "qwerty"); + EN_GB = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + Locale.UK.toString(), "qwerty"); + ES_US = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + "es_US", "spanish"); + FR = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + Locale.FRENCH.toString(), "azerty"); + FR_CA = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + Locale.CANADA_FRENCH.toString(), "qwerty"); + FR_CH = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + "fr_CH", "swiss"); + DE = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + Locale.GERMAN.toString(), "qwertz"); + DE_CH = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + "de_CH", "swiss"); + ZZ = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + SubtypeLocaleUtils.NO_LANGUAGE, "qwerty"); + DE_QWERTY = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.GERMAN.toString(), "qwerty"); + FR_QWERTZ = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.FRENCH.toString(), "qwertz"); + EN_US_AZERTY = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.US.toString(), "azerty"); + EN_UK_DVORAK = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.UK.toString(), "dvorak"); + ES_US_COLEMAK = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + "es_US", "colemak"); + ZZ_AZERTY = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + SubtypeLocaleUtils.NO_LANGUAGE, "azerty"); + ZZ_PC = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + SubtypeLocaleUtils.NO_LANGUAGE, "pcqwerty"); + } + + public void testAllFullDisplayNameForSpacebar() { + for (final InputMethodSubtype subtype : mSubtypesList) { + final String subtypeName = SubtypeLocaleUtils + .getSubtypeDisplayNameInSystemLocale(subtype); + final String spacebarText = SpacebarLanguageUtils.getFullDisplayName(subtype); + final String languageName = SubtypeLocaleUtils + .getSubtypeLocaleDisplayName(subtype.getLocale()); + if (SubtypeLocaleUtils.isNoLanguage(subtype)) { + assertFalse(subtypeName, spacebarText.contains(languageName)); + } else { + assertTrue(subtypeName, spacebarText.contains(languageName)); + } + } + } + + public void testAllMiddleDisplayNameForSpacebar() { + for (final InputMethodSubtype subtype : mSubtypesList) { + final String subtypeName = SubtypeLocaleUtils + .getSubtypeDisplayNameInSystemLocale(subtype); + final String spacebarText = SpacebarLanguageUtils.getMiddleDisplayName(subtype); + if (SubtypeLocaleUtils.isNoLanguage(subtype)) { + assertEquals(subtypeName, + SubtypeLocaleUtils.getKeyboardLayoutSetDisplayName(subtype), spacebarText); + } else { + final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); + assertEquals(subtypeName, + SubtypeLocaleUtils.getSubtypeLocaleDisplayName(locale.getLanguage()), + spacebarText); + } + } + } + + // InputMethodSubtype's display name for spacebar text in its locale. + // isAdditionalSubtype (T=true, F=false) + // locale layout | Middle Full + // ------ ------- - --------- ---------------------- + // en_US qwerty F English English (US) exception + // en_GB qwerty F English English (UK) exception + // es_US spanish F Español Español (EE.UU.) exception + // fr azerty F Français Français + // fr_CA qwerty F Français Français (Canada) + // fr_CH swiss F Français Français (Suisse) + // de qwertz F Deutsch Deutsch + // de_CH swiss F Deutsch Deutsch (Schweiz) + // zz qwerty F QWERTY QWERTY + // fr qwertz T Français Français + // de qwerty T Deutsch Deutsch + // en_US azerty T English English (US) + // zz azerty T AZERTY AZERTY + + private final RunInLocale<Void> testsPredefinedSubtypesForSpacebar = new RunInLocale<Void>() { + @Override + protected Void job(final Resources res) { + assertEquals("en_US", "English (US)", + SpacebarLanguageUtils.getFullDisplayName(EN_US)); + assertEquals("en_GB", "English (UK)", + SpacebarLanguageUtils.getFullDisplayName(EN_GB)); + assertEquals("es_US", "Español (EE.UU.)", + SpacebarLanguageUtils.getFullDisplayName(ES_US)); + assertEquals("fr", "Français", + SpacebarLanguageUtils.getFullDisplayName(FR)); + assertEquals("fr_CA", "Français (Canada)", + SpacebarLanguageUtils.getFullDisplayName(FR_CA)); + assertEquals("fr_CH", "Français (Suisse)", + SpacebarLanguageUtils.getFullDisplayName(FR_CH)); + assertEquals("de", "Deutsch", + SpacebarLanguageUtils.getFullDisplayName(DE)); + assertEquals("de_CH", "Deutsch (Schweiz)", + SpacebarLanguageUtils.getFullDisplayName(DE_CH)); + assertEquals("zz", "QWERTY", + SpacebarLanguageUtils.getFullDisplayName(ZZ)); + + assertEquals("en_US", "English", + SpacebarLanguageUtils.getMiddleDisplayName(EN_US)); + assertEquals("en_GB", "English", + SpacebarLanguageUtils.getMiddleDisplayName(EN_GB)); + assertEquals("es_US", "Español", + SpacebarLanguageUtils.getMiddleDisplayName(ES_US)); + assertEquals("fr", "Français", + SpacebarLanguageUtils.getMiddleDisplayName(FR)); + assertEquals("fr_CA", "Français", + SpacebarLanguageUtils.getMiddleDisplayName(FR_CA)); + assertEquals("fr_CH", "Français", + SpacebarLanguageUtils.getMiddleDisplayName(FR_CH)); + assertEquals("de", "Deutsch", + SpacebarLanguageUtils.getMiddleDisplayName(DE)); + assertEquals("de_CH", "Deutsch", + SpacebarLanguageUtils.getMiddleDisplayName(DE_CH)); + assertEquals("zz", "QWERTY", + SpacebarLanguageUtils.getMiddleDisplayName(ZZ)); + return null; + } + }; + + private final RunInLocale<Void> testsAdditionalSubtypesForSpacebar = new RunInLocale<Void>() { + @Override + protected Void job(final Resources res) { + assertEquals("fr qwertz", "Français", + SpacebarLanguageUtils.getFullDisplayName(FR_QWERTZ)); + assertEquals("de qwerty", "Deutsch", + SpacebarLanguageUtils.getFullDisplayName(DE_QWERTY)); + assertEquals("en_US azerty", "English (US)", + SpacebarLanguageUtils.getFullDisplayName(EN_US_AZERTY)); + assertEquals("en_UK dvorak", "English (UK)", + SpacebarLanguageUtils.getFullDisplayName(EN_UK_DVORAK)); + assertEquals("es_US colemak", "Español (EE.UU.)", + SpacebarLanguageUtils.getFullDisplayName(ES_US_COLEMAK)); + assertEquals("zz azerty", "AZERTY", + SpacebarLanguageUtils.getFullDisplayName(ZZ_AZERTY)); + assertEquals("zz pc", "PC", + SpacebarLanguageUtils.getFullDisplayName(ZZ_PC)); + + assertEquals("fr qwertz", "Français", + SpacebarLanguageUtils.getMiddleDisplayName(FR_QWERTZ)); + assertEquals("de qwerty", "Deutsch", + SpacebarLanguageUtils.getMiddleDisplayName(DE_QWERTY)); + assertEquals("en_US azerty", "English", + SpacebarLanguageUtils.getMiddleDisplayName(EN_US_AZERTY)); + assertEquals("en_UK dvorak", "English", + SpacebarLanguageUtils.getMiddleDisplayName(EN_UK_DVORAK)); + assertEquals("es_US colemak", "Español", + SpacebarLanguageUtils.getMiddleDisplayName(ES_US_COLEMAK)); + assertEquals("zz azerty", "AZERTY", + SpacebarLanguageUtils.getMiddleDisplayName(ZZ_AZERTY)); + assertEquals("zz pc", "PC", + SpacebarLanguageUtils.getMiddleDisplayName(ZZ_PC)); + return null; + } + }; + + public void testPredefinedSubtypesForSpacebarInEnglish() { + testsPredefinedSubtypesForSpacebar.runInLocale(mRes, Locale.ENGLISH); + } + + public void testAdditionalSubtypeForSpacebarInEnglish() { + testsAdditionalSubtypesForSpacebar.runInLocale(mRes, Locale.ENGLISH); + } + + public void testPredefinedSubtypesForSpacebarInFrench() { + testsPredefinedSubtypesForSpacebar.runInLocale(mRes, Locale.FRENCH); + } + + public void testAdditionalSubtypeForSpacebarInFrench() { + testsAdditionalSubtypesForSpacebar.runInLocale(mRes, Locale.FRENCH); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java index 4e396a1cf..4448a6baf 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java @@ -16,17 +16,17 @@ package com.android.inputmethod.latin.utils; -import com.android.inputmethod.latin.settings.SettingsValues; - import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import com.android.inputmethod.latin.Constants; + import java.util.Arrays; import java.util.List; import java.util.Locale; @SmallTest -public class StringUtilsTests extends AndroidTestCase { +public class StringAndJsonUtilsTests extends AndroidTestCase { public void testContainsInArray() { assertFalse("empty array", StringUtils.containsInArray("key", new String[0])); assertFalse("not in 1 element", StringUtils.containsInArray("key", new String[] { @@ -44,7 +44,7 @@ public class StringUtilsTests extends AndroidTestCase { })); } - public void testContainsInExtraValues() { + public void testContainsInCommaSplittableText() { assertFalse("null", StringUtils.containsInCommaSplittableText("key", null)); assertFalse("empty", StringUtils.containsInCommaSplittableText("key", "")); assertFalse("not in 1 element", @@ -56,28 +56,7 @@ public class StringUtilsTests extends AndroidTestCase { assertTrue("in 2 elements", StringUtils.containsInCommaSplittableText("key", "key1,key")); } - public void testAppendToExtraValuesIfNotExists() { - assertEquals("null", "key", - StringUtils.appendToCommaSplittableTextIfNotExists("key", null)); - assertEquals("empty", "key", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "")); - - assertEquals("not in 1 element", "key1,key", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "key1")); - assertEquals("not in 2 elements", "key1,key2,key", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "key1,key2")); - - assertEquals("in 1 element", "key", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "key")); - assertEquals("in 2 elements at position 1", "key,key2", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "key,key2")); - assertEquals("in 2 elements at position 2", "key1,key", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "key1,key")); - assertEquals("in 3 elements at position 2", "key1,key,key3", - StringUtils.appendToCommaSplittableTextIfNotExists("key", "key1,key,key3")); - } - - public void testRemoveFromExtraValuesIfExists() { + public void testRemoveFromCommaSplittableTextIfExists() { assertEquals("null", "", StringUtils.removeFromCommaSplittableTextIfExists("key", null)); assertEquals("empty", "", StringUtils.removeFromCommaSplittableTextIfExists("key", "")); @@ -187,54 +166,47 @@ public class StringUtilsTests extends AndroidTestCase { assertTrue(StringUtils.isIdenticalAfterDowncase("")); } - public void testLooksValidForDictionaryInsertion() { - final SettingsValues settings = - SettingsValues.makeDummySettingsValuesForTest(Locale.ENGLISH); - assertTrue(StringUtils.looksValidForDictionaryInsertion("aochaueo", settings)); - assertFalse(StringUtils.looksValidForDictionaryInsertion("", settings)); - assertTrue(StringUtils.looksValidForDictionaryInsertion("ao-ch'aueo", settings)); - assertFalse(StringUtils.looksValidForDictionaryInsertion("2908743256", settings)); - assertTrue(StringUtils.looksValidForDictionaryInsertion("31aochaueo", settings)); - assertFalse(StringUtils.looksValidForDictionaryInsertion("akeo raeoch oerch .", settings)); - assertFalse(StringUtils.looksValidForDictionaryInsertion("!!!", settings)); - } - - private static void checkCapitalize(final String src, final String dst, final String separators, - final Locale locale) { - assertEquals(dst, StringUtils.capitalizeEachWord(src, separators, locale)); + private static void checkCapitalize(final String src, final String dst, + final int[] sortedSeparators, final Locale locale) { + assertEquals(dst, StringUtils.capitalizeEachWord(src, sortedSeparators, locale)); assert(src.equals(dst) - == StringUtils.isIdenticalAfterCapitalizeEachWord(src, separators)); + == StringUtils.isIdenticalAfterCapitalizeEachWord(src, sortedSeparators)); } + private static final int[] SPACE = { Constants.CODE_SPACE }; + private static final int[] SPACE_PERIOD = StringUtils.toSortedCodePointArray(" ."); + private static final int[] SENTENCE_SEPARATORS = + StringUtils.toSortedCodePointArray(" \n.!?*()&"); + private static final int[] WORD_SEPARATORS = StringUtils.toSortedCodePointArray(" \n.!?*,();&"); + public void testCapitalizeEachWord() { - checkCapitalize("", "", " ", Locale.ENGLISH); - checkCapitalize("test", "Test", " ", Locale.ENGLISH); - checkCapitalize(" test", " Test", " ", Locale.ENGLISH); - checkCapitalize("Test", "Test", " ", Locale.ENGLISH); - checkCapitalize(" Test", " Test", " ", Locale.ENGLISH); - checkCapitalize(".Test", ".test", " ", Locale.ENGLISH); - checkCapitalize(".Test", ".Test", " .", Locale.ENGLISH); - checkCapitalize(".Test", ".Test", ". ", Locale.ENGLISH); - checkCapitalize("test and retest", "Test And Retest", " .", Locale.ENGLISH); - checkCapitalize("Test and retest", "Test And Retest", " .", Locale.ENGLISH); - checkCapitalize("Test And Retest", "Test And Retest", " .", Locale.ENGLISH); - checkCapitalize("Test And.Retest ", "Test And.Retest ", " .", Locale.ENGLISH); - checkCapitalize("Test And.retest ", "Test And.Retest ", " .", Locale.ENGLISH); - checkCapitalize("Test And.retest ", "Test And.retest ", " ", Locale.ENGLISH); - checkCapitalize("Test And.Retest ", "Test And.retest ", " ", Locale.ENGLISH); - checkCapitalize("test and ietest", "Test And İetest", " .", new Locale("tr")); - checkCapitalize("test and ietest", "Test And Ietest", " .", Locale.ENGLISH); - checkCapitalize("Test&Retest", "Test&Retest", " \n.!?*()&", Locale.ENGLISH); - checkCapitalize("Test&retest", "Test&Retest", " \n.!?*()&", Locale.ENGLISH); - checkCapitalize("test&Retest", "Test&Retest", " \n.!?*()&", Locale.ENGLISH); + checkCapitalize("", "", SPACE, Locale.ENGLISH); + checkCapitalize("test", "Test", SPACE, Locale.ENGLISH); + checkCapitalize(" test", " Test", SPACE, Locale.ENGLISH); + checkCapitalize("Test", "Test", SPACE, Locale.ENGLISH); + checkCapitalize(" Test", " Test", SPACE, Locale.ENGLISH); + checkCapitalize(".Test", ".test", SPACE, Locale.ENGLISH); + checkCapitalize(".Test", ".Test", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("test and retest", "Test And Retest", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("Test and retest", "Test And Retest", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("Test And Retest", "Test And Retest", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("Test And.Retest ", "Test And.Retest ", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("Test And.retest ", "Test And.Retest ", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("Test And.retest ", "Test And.retest ", SPACE, Locale.ENGLISH); + checkCapitalize("Test And.Retest ", "Test And.retest ", SPACE, Locale.ENGLISH); + checkCapitalize("test and ietest", "Test And İetest", SPACE_PERIOD, new Locale("tr")); + checkCapitalize("test and ietest", "Test And Ietest", SPACE_PERIOD, Locale.ENGLISH); + checkCapitalize("Test&Retest", "Test&Retest", SENTENCE_SEPARATORS, Locale.ENGLISH); + checkCapitalize("Test&retest", "Test&Retest", SENTENCE_SEPARATORS, Locale.ENGLISH); + checkCapitalize("test&Retest", "Test&Retest", SENTENCE_SEPARATORS, Locale.ENGLISH); checkCapitalize("rest\nrecreation! And in the end...", - "Rest\nRecreation! And In The End...", " \n.!?*,();&", Locale.ENGLISH); + "Rest\nRecreation! And In The End...", WORD_SEPARATORS, Locale.ENGLISH); checkCapitalize("lorem ipsum dolor sit amet", "Lorem Ipsum Dolor Sit Amet", - " \n.,!?*()&;", Locale.ENGLISH); + WORD_SEPARATORS, Locale.ENGLISH); checkCapitalize("Lorem!Ipsum (Dolor) Sit * Amet", "Lorem!Ipsum (Dolor) Sit * Amet", - " \n,.;!?*()&", Locale.ENGLISH); + WORD_SEPARATORS, Locale.ENGLISH); checkCapitalize("Lorem!Ipsum (dolor) Sit * Amet", "Lorem!Ipsum (Dolor) Sit * Amet", - " \n,.;!?*()&", Locale.ENGLISH); + WORD_SEPARATORS, Locale.ENGLISH); } public void testLooksLikeURL() { @@ -271,13 +243,87 @@ public class StringUtilsTests extends AndroidTestCase { assertTrue(bytesStr.equals(bytesStr2)); } - public void testJsonStringUtils() { + public void testJsonUtils() { final Object[] objs = new Object[] { 1, "aaa", "bbb", 3 }; final List<Object> objArray = Arrays.asList(objs); - final String str = StringUtils.listToJsonStr(objArray); - final List<Object> newObjArray = StringUtils.jsonStrToList(str); + final String str = JsonUtils.listToJsonStr(objArray); + final List<Object> newObjArray = JsonUtils.jsonStrToList(str); for (int i = 0; i < objs.length; ++i) { assertEquals(objs[i], newObjArray.get(i)); } } + + public void testToCodePointArray() { + final String STR_WITH_SUPPLEMENTARY_CHAR = "abcde\uD861\uDED7fgh\u0000\u2002\u2003\u3000xx"; + final int[] EXPECTED_RESULT = new int[] { 'a', 'b', 'c', 'd', 'e', 0x286D7, 'f', 'g', 'h', + 0, 0x2002, 0x2003, 0x3000, 'x', 'x'}; + final int[] codePointArray = StringUtils.toCodePointArray(STR_WITH_SUPPLEMENTARY_CHAR, 0, + STR_WITH_SUPPLEMENTARY_CHAR.length()); + assertEquals("toCodePointArray, size matches", codePointArray.length, + EXPECTED_RESULT.length); + for (int i = 0; i < EXPECTED_RESULT.length; ++i) { + assertEquals("toCodePointArray position " + i, codePointArray[i], EXPECTED_RESULT[i]); + } + } + + public void testCopyCodePointsAndReturnCodePointCount() { + final String STR_WITH_SUPPLEMENTARY_CHAR = "AbcDE\uD861\uDED7fGh\u0000\u2002\u3000あx"; + final int[] EXPECTED_RESULT = new int[] { 'A', 'b', 'c', 'D', 'E', 0x286D7, + 'f', 'G', 'h', 0, 0x2002, 0x3000, 'あ', 'x'}; + final int[] EXPECTED_RESULT_DOWNCASE = new int[] { 'a', 'b', 'c', 'd', 'e', 0x286D7, + 'f', 'g', 'h', 0, 0x2002, 0x3000, 'あ', 'x'}; + + int[] codePointArray = new int[50]; + int codePointCount = StringUtils.copyCodePointsAndReturnCodePointCount(codePointArray, + STR_WITH_SUPPLEMENTARY_CHAR, 0, + STR_WITH_SUPPLEMENTARY_CHAR.length(), false /* downCase */); + assertEquals("copyCodePointsAndReturnCodePointCount, size matches", codePointCount, + EXPECTED_RESULT.length); + for (int i = 0; i < codePointCount; ++i) { + assertEquals("copyCodePointsAndReturnCodePointCount position " + i, codePointArray[i], + EXPECTED_RESULT[i]); + } + + codePointCount = StringUtils.copyCodePointsAndReturnCodePointCount(codePointArray, + STR_WITH_SUPPLEMENTARY_CHAR, 0, + STR_WITH_SUPPLEMENTARY_CHAR.length(), true /* downCase */); + assertEquals("copyCodePointsAndReturnCodePointCount downcase, size matches", codePointCount, + EXPECTED_RESULT_DOWNCASE.length); + for (int i = 0; i < codePointCount; ++i) { + assertEquals("copyCodePointsAndReturnCodePointCount position " + i, codePointArray[i], + EXPECTED_RESULT_DOWNCASE[i]); + } + + final int JAVA_CHAR_COUNT = 8; + final int CODEPOINT_COUNT = 7; + codePointCount = StringUtils.copyCodePointsAndReturnCodePointCount(codePointArray, + STR_WITH_SUPPLEMENTARY_CHAR, 0, JAVA_CHAR_COUNT, false /* downCase */); + assertEquals("copyCodePointsAndReturnCodePointCount, size matches", codePointCount, + CODEPOINT_COUNT); + for (int i = 0; i < codePointCount; ++i) { + assertEquals("copyCodePointsAndReturnCodePointCount position " + i, codePointArray[i], + EXPECTED_RESULT[i]); + } + + boolean exceptionHappened = false; + codePointArray = new int[5]; + try { + codePointCount = StringUtils.copyCodePointsAndReturnCodePointCount(codePointArray, + STR_WITH_SUPPLEMENTARY_CHAR, 0, JAVA_CHAR_COUNT, false /* downCase */); + } catch (ArrayIndexOutOfBoundsException e) { + exceptionHappened = true; + } + assertTrue("copyCodePointsAndReturnCodePointCount throws when array is too small", + exceptionHappened); + } + + public void testGetTrailingSingleQuotesCount() { + assertEquals(0, StringUtils.getTrailingSingleQuotesCount("")); + assertEquals(1, StringUtils.getTrailingSingleQuotesCount("'")); + assertEquals(5, StringUtils.getTrailingSingleQuotesCount("'''''")); + assertEquals(0, StringUtils.getTrailingSingleQuotesCount("a")); + assertEquals(0, StringUtils.getTrailingSingleQuotesCount("'this")); + assertEquals(1, StringUtils.getTrailingSingleQuotesCount("'word'")); + assertEquals(0, StringUtils.getTrailingSingleQuotesCount("I'm")); + } } diff --git a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java index 856b2dbda..ce3df7dd6 100644 --- a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java @@ -20,9 +20,9 @@ import android.content.Context; import android.content.res.Resources; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import android.view.inputmethod.InputMethodInfo; import android.view.inputmethod.InputMethodSubtype; -import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.RichInputMethodManager; import java.util.ArrayList; @@ -30,8 +30,8 @@ import java.util.Locale; @SmallTest public class SubtypeLocaleUtilsTests extends AndroidTestCase { - // Locale to subtypes list. - private final ArrayList<InputMethodSubtype> mSubtypesList = CollectionUtils.newArrayList(); + // All input method subtypes of LatinIME. + private final ArrayList<InputMethodSubtype> mSubtypesList = new ArrayList<>(); private RichInputMethodManager mRichImm; private Resources mRes; @@ -41,7 +41,9 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { InputMethodSubtype ES_US; InputMethodSubtype FR; InputMethodSubtype FR_CA; + InputMethodSubtype FR_CH; InputMethodSubtype DE; + InputMethodSubtype DE_CH; InputMethodSubtype ZZ; InputMethodSubtype DE_QWERTY; InputMethodSubtype FR_QWERTZ; @@ -60,6 +62,13 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { mRes = context.getResources(); SubtypeLocaleUtils.init(context); + final InputMethodInfo imi = mRichImm.getInputMethodInfoOfThisIme(); + final int subtypeCount = imi.getSubtypeCount(); + for (int index = 0; index < subtypeCount; index++) { + final InputMethodSubtype subtype = imi.getSubtypeAt(index); + mSubtypesList.add(subtype); + } + EN_US = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( Locale.US.toString(), "qwerty"); EN_GB = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( @@ -70,37 +79,41 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { Locale.FRENCH.toString(), "azerty"); FR_CA = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( Locale.CANADA_FRENCH.toString(), "qwerty"); + FR_CH = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + "fr_CH", "swiss"); DE = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( Locale.GERMAN.toString(), "qwertz"); + DE_CH = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + "de_CH", "swiss"); ZZ = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( SubtypeLocaleUtils.NO_LANGUAGE, "qwerty"); - DE_QWERTY = AdditionalSubtypeUtils.createAdditionalSubtype( - Locale.GERMAN.toString(), "qwerty", null); - FR_QWERTZ = AdditionalSubtypeUtils.createAdditionalSubtype( - Locale.FRENCH.toString(), "qwertz", null); - EN_US_AZERTY = AdditionalSubtypeUtils.createAdditionalSubtype( - Locale.US.toString(), "azerty", null); - EN_UK_DVORAK = AdditionalSubtypeUtils.createAdditionalSubtype( - Locale.UK.toString(), "dvorak", null); - ES_US_COLEMAK = AdditionalSubtypeUtils.createAdditionalSubtype( - "es_US", "colemak", null); - ZZ_AZERTY = AdditionalSubtypeUtils.createAdditionalSubtype( - SubtypeLocaleUtils.NO_LANGUAGE, "azerty", null); - ZZ_PC = AdditionalSubtypeUtils.createAdditionalSubtype( - SubtypeLocaleUtils.NO_LANGUAGE, "pcqwerty", null); - + DE_QWERTY = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.GERMAN.toString(), "qwerty"); + FR_QWERTZ = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.FRENCH.toString(), "qwertz"); + EN_US_AZERTY = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.US.toString(), "azerty"); + EN_UK_DVORAK = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + Locale.UK.toString(), "dvorak"); + ES_US_COLEMAK = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + "es_US", "colemak"); + ZZ_AZERTY = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + SubtypeLocaleUtils.NO_LANGUAGE, "azerty"); + ZZ_PC = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( + SubtypeLocaleUtils.NO_LANGUAGE, "pcqwerty"); } public void testAllFullDisplayName() { for (final InputMethodSubtype subtype : mSubtypesList) { - final String subtypeName = - SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(subtype); + final String subtypeName = SubtypeLocaleUtils + .getSubtypeDisplayNameInSystemLocale(subtype); if (SubtypeLocaleUtils.isNoLanguage(subtype)) { - final String noLanguage = mRes.getString(R.string.subtype_no_language); - assertTrue(subtypeName, subtypeName.contains(noLanguage)); + final String layoutName = SubtypeLocaleUtils + .getKeyboardLayoutSetDisplayName(subtype); + assertTrue(subtypeName, subtypeName.contains(layoutName)); } else { - final String languageName = - SubtypeLocaleUtils.getSubtypeLocaleDisplayName(subtype.getLocale()); + final String languageName = SubtypeLocaleUtils + .getSubtypeLocaleDisplayNameInSystemLocale(subtype.getLocale()); assertTrue(subtypeName, subtypeName.contains(languageName)); } } @@ -110,10 +123,23 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { assertEquals("en_US", "qwerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(EN_US)); assertEquals("en_GB", "qwerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(EN_GB)); assertEquals("es_US", "spanish", SubtypeLocaleUtils.getKeyboardLayoutSetName(ES_US)); - assertEquals("fr ", "azerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(FR)); + assertEquals("fr", "azerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(FR)); assertEquals("fr_CA", "qwerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(FR_CA)); - assertEquals("de ", "qwertz", SubtypeLocaleUtils.getKeyboardLayoutSetName(DE)); - assertEquals("zz ", "qwerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(ZZ)); + assertEquals("fr_CH", "swiss", SubtypeLocaleUtils.getKeyboardLayoutSetName(FR_CH)); + assertEquals("de", "qwertz", SubtypeLocaleUtils.getKeyboardLayoutSetName(DE)); + assertEquals("de_CH", "swiss", SubtypeLocaleUtils.getKeyboardLayoutSetName(DE_CH)); + assertEquals("zz", "qwerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(ZZ)); + + assertEquals("de qwerty", "qwerty", SubtypeLocaleUtils.getKeyboardLayoutSetName(DE_QWERTY)); + assertEquals("fr qwertz", "qwertz", SubtypeLocaleUtils.getKeyboardLayoutSetName(FR_QWERTZ)); + assertEquals("en_US azerty", "azerty", + SubtypeLocaleUtils.getKeyboardLayoutSetName(EN_US_AZERTY)); + assertEquals("en_UK dvorak", "dvorak", + SubtypeLocaleUtils.getKeyboardLayoutSetName(EN_UK_DVORAK)); + assertEquals("es_US colemak", "colemak", + SubtypeLocaleUtils.getKeyboardLayoutSetName(ES_US_COLEMAK)); + assertEquals("zz azerty", "azerty", + SubtypeLocaleUtils.getKeyboardLayoutSetName(ZZ_AZERTY)); } // InputMethodSubtype's display name in system locale (en_US). @@ -125,7 +151,9 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { // es_US spanish F Spanish (US) exception // fr azerty F French // fr_CA qwerty F French (Canada) + // fr_CH swiss F French (Switzerland) // de qwertz F German + // de_CH swiss F German (Switzerland) // zz qwerty F Alphabet (QWERTY) // fr qwertz T French (QWERTZ) // de qwerty T German (QWERTY) @@ -144,13 +172,17 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_GB)); assertEquals("es_US", "Spanish (US)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US)); - assertEquals("fr ", "French", + assertEquals("fr", "French", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR)); assertEquals("fr_CA", "French (Canada)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CA)); - assertEquals("de ", "German", + assertEquals("fr_CH", "French (Switzerland)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CH)); + assertEquals("de", "German", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE)); - assertEquals("zz ", "Alphabet (QWERTY)", + assertEquals("de_CH", "German (Switzerland)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE_CH)); + assertEquals("zz", "Alphabet (QWERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ)); return null; } @@ -162,17 +194,19 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { final RunInLocale<Void> tests = new RunInLocale<Void>() { @Override protected Void job(final Resources res) { - assertEquals("fr qwertz", "French (QWERTZ)", + assertEquals("fr qwertz", "French (QWERTZ)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_QWERTZ)); - assertEquals("de qwerty", "German (QWERTY)", + assertEquals("de qwerty", "German (QWERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE_QWERTY)); assertEquals("en_US azerty", "English (US) (AZERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_US_AZERTY)); - assertEquals("en_UK dvorak", "English (UK) (Dvorak)", + assertEquals("en_UK dvorak","English (UK) (Dvorak)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_UK_DVORAK)); - assertEquals("es_US colemak","Spanish (US) (Colemak)", + assertEquals("es_US colemak", "Spanish (US) (Colemak)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US_COLEMAK)); - assertEquals("zz pc", "Alphabet (PC)", + assertEquals("zz azerty", "Alphabet (AZERTY)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_AZERTY)); + assertEquals("zz pc", "Alphabet (PC)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_PC)); return null; } @@ -189,14 +223,16 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { // es_US spanish F Espagnol (États-Unis) exception // fr azerty F Français // fr_CA qwerty F Français (Canada) + // fr_CH swiss F Français (Suisse) // de qwertz F Allemand - // zz qwerty F Aucune langue (QWERTY) + // de_CH swiss F Allemand (Suisse) + // zz qwerty F Alphabet latin (QWERTY) // fr qwertz T Français (QWERTZ) // de qwerty T Allemand (QWERTY) // en_US azerty T Anglais (États-Unis) (AZERTY) exception // en_UK dvorak T Anglais (Royaume-Uni) (Dvorak) exception // es_US colemak T Espagnol (États-Unis) (Colemak) exception - // zz pc T Alphabet (PC) + // zz pc T Alphabet latin (PC) public void testPredefinedSubtypesInFrenchSystemLocale() { final RunInLocale<Void> tests = new RunInLocale<Void>() { @@ -208,13 +244,17 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_GB)); assertEquals("es_US", "Espagnol (États-Unis)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US)); - assertEquals("fr ", "Français", + assertEquals("fr", "Français", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR)); assertEquals("fr_CA", "Français (Canada)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CA)); - assertEquals("de ", "Allemand", + assertEquals("fr_CH", "Français (Suisse)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CH)); + assertEquals("de", "Allemand", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE)); - assertEquals("zz ", "Alphabet latin (QWERTY)", + assertEquals("de_CH", "Allemand (Suisse)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE_CH)); + assertEquals("zz", "Alphabet latin (QWERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ)); return null; } @@ -226,17 +266,19 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { final RunInLocale<Void> tests = new RunInLocale<Void>() { @Override protected Void job(final Resources res) { - assertEquals("fr qwertz", "Français (QWERTZ)", + assertEquals("fr qwertz", "Français (QWERTZ)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_QWERTZ)); - assertEquals("de qwerty", "Allemand (QWERTY)", + assertEquals("de qwerty", "Allemand (QWERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE_QWERTY)); assertEquals("en_US azerty", "Anglais (États-Unis) (AZERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_US_AZERTY)); assertEquals("en_UK dvorak", "Anglais (Royaume-Uni) (Dvorak)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_UK_DVORAK)); - assertEquals("es_US colemak","Espagnol (États-Unis) (Colemak)", + assertEquals("es_US colemak", "Espagnol (États-Unis) (Colemak)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US_COLEMAK)); - assertEquals("zz pc", "Alphabet latin (PC)", + assertEquals("zz azerty", "Alphabet latin (AZERTY)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_AZERTY)); + assertEquals("zz pc", "Alphabet latin (PC)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_PC)); return null; } @@ -244,144 +286,26 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { tests.runInLocale(mRes, Locale.FRENCH); } - public void testAllFullDisplayNameForSpacebar() { - for (final InputMethodSubtype subtype : mSubtypesList) { - final String subtypeName = - SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(subtype); - final String spacebarText = SubtypeLocaleUtils.getFullDisplayName(subtype); - final String languageName = - SubtypeLocaleUtils.getSubtypeLocaleDisplayName(subtype.getLocale()); - if (SubtypeLocaleUtils.isNoLanguage(subtype)) { - assertFalse(subtypeName, spacebarText.contains(languageName)); - } else { - assertTrue(subtypeName, spacebarText.contains(languageName)); - } - } - } - - public void testAllMiddleDisplayNameForSpacebar() { - for (final InputMethodSubtype subtype : mSubtypesList) { - final String subtypeName = - SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(subtype); - final String spacebarText = SubtypeLocaleUtils.getMiddleDisplayName(subtype); - if (SubtypeLocaleUtils.isNoLanguage(subtype)) { - assertEquals(subtypeName, - SubtypeLocaleUtils.getKeyboardLayoutSetName(subtype), spacebarText); - } else { - assertEquals(subtypeName, - SubtypeLocaleUtils.getSubtypeLocaleDisplayName(subtype.getLocale()), - spacebarText); - } - } - } + public void testIsRtlLanguage() { + // Known Right-to-Left language subtypes. + final InputMethodSubtype ARABIC = mRichImm + .findSubtypeByLocaleAndKeyboardLayoutSet("ar", "arabic"); + assertNotNull("Arabic", ARABIC); + final InputMethodSubtype FARSI = mRichImm + .findSubtypeByLocaleAndKeyboardLayoutSet("fa", "farsi"); + assertNotNull("Farsi", FARSI); + final InputMethodSubtype HEBREW = mRichImm + .findSubtypeByLocaleAndKeyboardLayoutSet("iw", "hebrew"); + assertNotNull("Hebrew", HEBREW); - public void testAllShortDisplayNameForSpacebar() { for (final InputMethodSubtype subtype : mSubtypesList) { - final String subtypeName = - SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(subtype); - final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); - final String spacebarText = SubtypeLocaleUtils.getShortDisplayName(subtype); - final String languageCode = StringUtils.capitalizeFirstCodePoint( - locale.getLanguage(), locale); - if (SubtypeLocaleUtils.isNoLanguage(subtype)) { - assertEquals(subtypeName, "", spacebarText); + final String subtypeName = SubtypeLocaleUtils + .getSubtypeDisplayNameInSystemLocale(subtype); + if (subtype.equals(ARABIC) || subtype.equals(FARSI) || subtype.equals(HEBREW)) { + assertTrue(subtypeName, SubtypeLocaleUtils.isRtlLanguage(subtype)); } else { - assertEquals(subtypeName, languageCode, spacebarText); + assertFalse(subtypeName, SubtypeLocaleUtils.isRtlLanguage(subtype)); } } } - - // InputMethodSubtype's display name for spacebar text in its locale. - // isAdditionalSubtype (T=true, F=false) - // locale layout | Short Middle Full - // ------ ------- - ---- --------- ---------------------- - // en_US qwerty F En English English (US) exception - // en_GB qwerty F En English English (UK) exception - // es_US spanish F Es Español Español (EE.UU.) exception - // fr azerty F Fr Français Français - // fr_CA qwerty F Fr Français Français (Canada) - // de qwertz F De Deutsch Deutsch - // zz qwerty F QWERTY QWERTY - // fr qwertz T Fr Français Français - // de qwerty T De Deutsch Deutsch - // en_US azerty T En English English (US) - // zz azerty T AZERTY AZERTY - - private final RunInLocale<Void> testsPredefinedSubtypesForSpacebar = new RunInLocale<Void>() { - @Override - protected Void job(final Resources res) { - assertEquals("en_US", "English (US)", SubtypeLocaleUtils.getFullDisplayName(EN_US)); - assertEquals("en_GB", "English (UK)", SubtypeLocaleUtils.getFullDisplayName(EN_GB)); - assertEquals("es_US", "Español (EE.UU.)", - SubtypeLocaleUtils.getFullDisplayName(ES_US)); - assertEquals("fr ", "Français", SubtypeLocaleUtils.getFullDisplayName(FR)); - assertEquals("fr_CA", "Français (Canada)", - SubtypeLocaleUtils.getFullDisplayName(FR_CA)); - assertEquals("de ", "Deutsch", SubtypeLocaleUtils.getFullDisplayName(DE)); - assertEquals("zz ", "QWERTY", SubtypeLocaleUtils.getFullDisplayName(ZZ)); - - assertEquals("en_US", "English", SubtypeLocaleUtils.getMiddleDisplayName(EN_US)); - assertEquals("en_GB", "English", SubtypeLocaleUtils.getMiddleDisplayName(EN_GB)); - assertEquals("es_US", "Español", SubtypeLocaleUtils.getMiddleDisplayName(ES_US)); - assertEquals("fr ", "Français", SubtypeLocaleUtils.getMiddleDisplayName(FR)); - assertEquals("fr_CA", "Français", SubtypeLocaleUtils.getMiddleDisplayName(FR_CA)); - assertEquals("de ", "Deutsch", SubtypeLocaleUtils.getMiddleDisplayName(DE)); - assertEquals("zz ", "QWERTY", SubtypeLocaleUtils.getMiddleDisplayName(ZZ)); - - assertEquals("en_US", "En", SubtypeLocaleUtils.getShortDisplayName(EN_US)); - assertEquals("en_GB", "En", SubtypeLocaleUtils.getShortDisplayName(EN_GB)); - assertEquals("es_US", "Es", SubtypeLocaleUtils.getShortDisplayName(ES_US)); - assertEquals("fr ", "Fr", SubtypeLocaleUtils.getShortDisplayName(FR)); - assertEquals("fr_CA", "Fr", SubtypeLocaleUtils.getShortDisplayName(FR_CA)); - assertEquals("de ", "De", SubtypeLocaleUtils.getShortDisplayName(DE)); - assertEquals("zz ", "", SubtypeLocaleUtils.getShortDisplayName(ZZ)); - return null; - } - }; - - private final RunInLocale<Void> testsAdditionalSubtypesForSpacebar = new RunInLocale<Void>() { - @Override - protected Void job(final Resources res) { - assertEquals("fr qwertz", "Français", - SubtypeLocaleUtils.getFullDisplayName(FR_QWERTZ)); - assertEquals("de qwerty", "Deutsch", - SubtypeLocaleUtils.getFullDisplayName(DE_QWERTY)); - assertEquals("en_US azerty", "English (US)", - SubtypeLocaleUtils.getFullDisplayName(EN_US_AZERTY)); - assertEquals("zz azerty", "AZERTY", - SubtypeLocaleUtils.getFullDisplayName(ZZ_AZERTY)); - - assertEquals("fr qwertz", "Français", - SubtypeLocaleUtils.getMiddleDisplayName(FR_QWERTZ)); - assertEquals("de qwerty", "Deutsch", - SubtypeLocaleUtils.getMiddleDisplayName(DE_QWERTY)); - assertEquals("en_US azerty", "English", - SubtypeLocaleUtils.getMiddleDisplayName(EN_US_AZERTY)); - assertEquals("zz azerty", "AZERTY", - SubtypeLocaleUtils.getMiddleDisplayName(ZZ_AZERTY)); - - assertEquals("fr qwertz", "Fr", SubtypeLocaleUtils.getShortDisplayName(FR_QWERTZ)); - assertEquals("de qwerty", "De", SubtypeLocaleUtils.getShortDisplayName(DE_QWERTY)); - assertEquals("en_US azerty", "En", - SubtypeLocaleUtils.getShortDisplayName(EN_US_AZERTY)); - assertEquals("zz azerty", "", SubtypeLocaleUtils.getShortDisplayName(ZZ_AZERTY)); - return null; - } - }; - - public void testPredefinedSubtypesForSpacebarInEnglish() { - testsPredefinedSubtypesForSpacebar.runInLocale(mRes, Locale.ENGLISH); - } - - public void testAdditionalSubtypeForSpacebarInEnglish() { - testsAdditionalSubtypesForSpacebar.runInLocale(mRes, Locale.ENGLISH); - } - - public void testPredefinedSubtypesForSpacebarInFrench() { - testsPredefinedSubtypesForSpacebar.runInLocale(mRes, Locale.FRENCH); - } - - public void testAdditionalSubtypeForSpacebarInFrench() { - testsAdditionalSubtypesForSpacebar.runInLocale(mRes, Locale.FRENCH); - } } diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java deleted file mode 100644 index 1944fd332..000000000 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.content.Context; -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.LargeTest; -import android.util.Log; - -import com.android.inputmethod.latin.makedict.DictDecoder; -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; -import com.android.inputmethod.latin.makedict.Ver3DictEncoder; -import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; - -/** - * Unit tests for UserHistoryDictIOUtils - */ -@LargeTest -public class UserHistoryDictIOUtilsTests extends AndroidTestCase - implements BigramDictionaryInterface { - - private static final String TAG = UserHistoryDictIOUtilsTests.class.getSimpleName(); - private static final int UNIGRAM_FREQUENCY = 50; - private static final int BIGRAM_FREQUENCY = 100; - private static final ArrayList<String> NOT_HAVE_BIGRAM = new ArrayList<String>(); - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = new FormatSpec.FormatOptions(2); - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - - /** - * Return same frequency for all words and bigrams - */ - @Override - public int getFrequency(String word1, String word2) { - if (word1 == null) return UNIGRAM_FREQUENCY; - return BIGRAM_FREQUENCY; - } - - // Utilities for Testing - - private void addWord(final String word, - final HashMap<String, ArrayList<String> > addedWords) { - if (!addedWords.containsKey(word)) { - addedWords.put(word, new ArrayList<String>()); - } - } - - private void addBigram(final String word1, final String word2, - final HashMap<String, ArrayList<String> > addedWords) { - addWord(word1, addedWords); - addWord(word2, addedWords); - addedWords.get(word1).add(word2); - } - - private void addBigramToBigramList(final String word1, final String word2, - final HashMap<String, ArrayList<String> > addedWords, - final UserHistoryDictionaryBigramList bigramList) { - bigramList.addBigram(null, word1); - bigramList.addBigram(word1, word2); - - addBigram(word1, word2, addedWords); - } - - private void checkWordInFusionDict(final FusionDictionary dict, final String word, - final ArrayList<String> expectedBigrams) { - final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); - assertNotNull(ptNode); - assertTrue(ptNode.isTerminal()); - - for (final String bigram : expectedBigrams) { - assertNotNull(ptNode.getBigram(bigram)); - } - } - - private void checkWordsInFusionDict(final FusionDictionary dict, - final HashMap<String, ArrayList<String> > bigrams) { - for (final String word : bigrams.keySet()) { - if (bigrams.containsKey(word)) { - checkWordInFusionDict(dict, word, bigrams.get(word)); - } else { - checkWordInFusionDict(dict, word, NOT_HAVE_BIGRAM); - } - } - } - - private void checkWordInBigramList( - final UserHistoryDictionaryBigramList bigramList, final String word, - final ArrayList<String> expectedBigrams) { - // check unigram - final HashMap<String,Byte> unigramMap = bigramList.getBigrams(null); - assertTrue(unigramMap.containsKey(word)); - - // check bigrams - final ArrayList<String> actualBigrams = new ArrayList<String>( - bigramList.getBigrams(word).keySet()); - - Collections.sort(expectedBigrams); - Collections.sort(actualBigrams); - assertEquals(expectedBigrams, actualBigrams); - } - - private void checkWordsInBigramList(final UserHistoryDictionaryBigramList bigramList, - final HashMap<String, ArrayList<String> > addedWords) { - for (final String word : addedWords.keySet()) { - if (addedWords.containsKey(word)) { - checkWordInBigramList(bigramList, word, addedWords.get(word)); - } else { - checkWordInBigramList(bigramList, word, NOT_HAVE_BIGRAM); - } - } - } - - private void writeDictToFile(final File file, - final UserHistoryDictionaryBigramList bigramList) { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, bigramList, FORMAT_OPTIONS); - } - - private void readDictFromFile(final File file, final OnAddWordListener listener) { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); - try { - dictDecoder.openDictBuffer(); - } catch (FileNotFoundException e) { - Log.e(TAG, "file not found", e); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); - } - - public void testGenerateFusionDictionary() { - final UserHistoryDictionaryBigramList originalList = new UserHistoryDictionaryBigramList(); - - final HashMap<String, ArrayList<String> > addedWords = - new HashMap<String, ArrayList<String>>(); - addBigramToBigramList("this", "is", addedWords, originalList); - addBigramToBigramList("this", "was", addedWords, originalList); - addBigramToBigramList("hello", "world", addedWords, originalList); - - final FusionDictionary fusionDict = - UserHistoryDictIOUtils.constructFusionDictionary(this, originalList); - - checkWordsInFusionDict(fusionDict, addedWords); - } - - public void testReadAndWrite() { - final Context context = getContext(); - - File file = null; - try { - file = File.createTempFile("testReadAndWrite", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - Log.d(TAG, "IOException while creating a temporary file", e); - } - assertNotNull(file); - - // make original dictionary - final UserHistoryDictionaryBigramList originalList = new UserHistoryDictionaryBigramList(); - final HashMap<String, ArrayList<String>> addedWords = CollectionUtils.newHashMap(); - addBigramToBigramList("this" , "is" , addedWords, originalList); - addBigramToBigramList("this" , "was" , addedWords, originalList); - addBigramToBigramList("is" , "not" , addedWords, originalList); - addBigramToBigramList("hello", "world", addedWords, originalList); - - // write to file - writeDictToFile(file, originalList); - - // make result dict. - final UserHistoryDictionaryBigramList resultList = new UserHistoryDictionaryBigramList(); - final OnAddWordListener listener = new OnAddWordListener() { - @Override - public void setUnigram(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq) { - Log.d(TAG, "in: setUnigram: " + word + "," + frequency); - resultList.addBigram(null, word, (byte)frequency); - } - @Override - public void setBigram(final String word1, final String word2, final int frequency) { - Log.d(TAG, "in: setBigram: " + word1 + "," + word2 + "," + frequency); - resultList.addBigram(word1, word2, (byte)frequency); - } - }; - - // load from file - readDictFromFile(file, listener); - checkWordsInBigramList(resultList, addedWords); - - // add new bigram - addBigramToBigramList("hello", "java", addedWords, resultList); - - // rewrite - writeDictToFile(file, resultList); - final UserHistoryDictionaryBigramList resultList2 = new UserHistoryDictionaryBigramList(); - final OnAddWordListener listener2 = new OnAddWordListener() { - @Override - public void setUnigram(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq) { - Log.d(TAG, "in: setUnigram: " + word + "," + frequency); - resultList2.addBigram(null, word, (byte)frequency); - } - @Override - public void setBigram(final String word1, final String word2, final int frequency) { - Log.d(TAG, "in: setBigram: " + word1 + "," + word2 + "," + frequency); - resultList2.addBigram(word1, word2, (byte)frequency); - } - }; - - // load from file - readDictFromFile(file, listener2); - checkWordsInBigramList(resultList2, addedWords); - } -} |