diff options
Diffstat (limited to 'tests/src')
22 files changed, 1908 insertions, 406 deletions
diff --git a/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java b/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java index db3997651..6e3e37add 100644 --- a/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java +++ b/tests/src/com/android/inputmethod/keyboard/internal/MockKeyboardSwitcher.java @@ -120,6 +120,11 @@ public class MockKeyboardSwitcher implements KeyboardState.SwitchActions { } @Override + public void setEmojiKeyboard() { + // Just ignore. + } + + @Override public void requestUpdatingShiftState() { mState.onUpdateShiftState(mAutoCapsState, RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE); } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java new file mode 100644 index 000000000..7ed3ee180 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -0,0 +1,628 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; +import android.util.Pair; + +import com.android.inputmethod.latin.makedict.CodePointUtils; +import com.android.inputmethod.latin.makedict.FormatSpec; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Random; + +@LargeTest +public class BinaryDictionaryTests extends AndroidTestCase { + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final String TEST_LOCALE = "test"; + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + private File createEmptyDictionaryAndGetFile(final String filename) throws IOException { + final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); + Map<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + 3 /* dictVersion */, attributeMap)) { + return file; + } else { + throw new IOException("Empty dictionary cannot be created."); + } + } + + public void testIsValidDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + assertTrue("binaryDictionary must be valid for existing valid dictionary file.", + binaryDictionary.isValidDictionary()); + binaryDictionary.close(); + assertFalse("binaryDictionary must be invalid after closing.", + binaryDictionary.isValidDictionary()); + dictFile.delete(); + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, + dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), + TEST_LOCALE, true /* isUpdatable */); + assertFalse("binaryDictionary must be invalid for not existing dictionary file.", + binaryDictionary.isValidDictionary()); + binaryDictionary.close(); + } + + public void testAddUnigramWord() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int probability = 100; + binaryDictionary.addUnigramWord("aaa", probability); + // Reallocate and create. + binaryDictionary.addUnigramWord("aab", probability); + // Insert into children. + binaryDictionary.addUnigramWord("aac", probability); + // Make terminal. + binaryDictionary.addUnigramWord("aa", probability); + // Create children. + binaryDictionary.addUnigramWord("aaaa", probability); + // Reallocate and make termianl. + binaryDictionary.addUnigramWord("a", probability); + + final int updatedProbability = 200; + // Update. + binaryDictionary.addUnigramWord("aaa", updatedProbability); + + assertEquals(probability, binaryDictionary.getFrequency("aab")); + assertEquals(probability, binaryDictionary.getFrequency("aac")); + assertEquals(probability, binaryDictionary.getFrequency("aa")); + assertEquals(probability, binaryDictionary.getFrequency("aaaa")); + assertEquals(probability, binaryDictionary.getFrequency("a")); + assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa")); + + dictFile.delete(); + } + + public void testRandomlyAddUnigramWord() { + final int wordCount = 1000; + final int codePointSetSize = 50; + final int seed = 123456789; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final HashMap<String, Integer> probabilityMap = new HashMap<String, Integer>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + probabilityMap.put(word, random.nextInt(0xFF)); + } + for (String word : probabilityMap.keySet()) { + binaryDictionary.addUnigramWord(word, probabilityMap.get(word)); + } + for (String word : probabilityMap.keySet()) { + assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word)); + } + dictFile.delete(); + } + + public void testAddBigramWords() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int bigramProbability = 10; + final int updatedBigramProbability = 15; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); + + binaryDictionary.addBigramWords("aaa", "abb", updatedBigramProbability); + final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability, + updatedBigramProbability); + assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb")); + + assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("bcc", "aaa")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("bcc", "bbc")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("aaa", "aaa")); + + // Testing bigram link. + binaryDictionary.addUnigramWord("abcde", unigramProbability); + binaryDictionary.addUnigramWord("fghij", unigramProbability); + binaryDictionary.addBigramWords("abcde", "fghij", bigramProbability); + binaryDictionary.addUnigramWord("fgh", unigramProbability); + binaryDictionary.addUnigramWord("abc", unigramProbability); + binaryDictionary.addUnigramWord("f", unigramProbability); + assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("abcde", "fgh")); + binaryDictionary.addBigramWords("abcde", "fghij", updatedBigramProbability); + assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij")); + + dictFile.delete(); + } + + public void testRandomlyAddBigramWords() { + final int wordCount = 100; + final int bigramCount = 1000; + final int codePointSetSize = 50; + final int seed = 11111; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final int[] unigramProbabilities = new int[wordCount]; + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities[i] = unigramProbability; + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + final int[][] probabilities = new int[wordCount][wordCount]; + + for (int i = 0; i < wordCount; ++i) { + for (int j = 0; j < wordCount; ++j) { + probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; + } + } + + for (int i = 0; i < bigramCount; i++) { + final int word0Index = random.nextInt(wordCount); + final int word1Index = random.nextInt(wordCount); + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( + unigramProbabilities[word1Index], bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + + for (int i = 0; i < words.size(); i++) { + for (int j = 0; j < words.size(); j++) { + assertEquals(probabilities[i][j], + binaryDictionary.getBigramProbability(words.get(i), words.get(j))); + } + } + + dictFile.delete(); + } + + public void testRemoveBigramWords() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final int unigramProbability = 100; + final int bigramProbability = 10; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + + assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); + + binaryDictionary.removeBigramWords("aaa", "abb"); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb")); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); + + + binaryDictionary.removeBigramWords("aaa", "bcc"); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc")); + binaryDictionary.removeBigramWords("abb", "aaa"); + assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa")); + binaryDictionary.removeBigramWords("abb", "bcc"); + assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc")); + + binaryDictionary.removeBigramWords("aaa", "abb"); + // Test remove non-existing bigram operation. + binaryDictionary.removeBigramWords("aaa", "abb"); + binaryDictionary.removeBigramWords("bcc", "aaa"); + + dictFile.delete(); + } + + public void testFlushDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int probability = 100; + binaryDictionary.addUnigramWord("aaa", probability); + binaryDictionary.addUnigramWord("abcd", probability); + // Close without flushing. + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa")); + assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd")); + + binaryDictionary.addUnigramWord("aaa", probability); + binaryDictionary.addUnigramWord("abcd", probability); + binaryDictionary.flush(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + assertEquals(probability, binaryDictionary.getFrequency("aaa")); + assertEquals(probability, binaryDictionary.getFrequency("abcd")); + binaryDictionary.addUnigramWord("bcde", probability); + binaryDictionary.flush(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + assertEquals(probability, binaryDictionary.getFrequency("bcde")); + binaryDictionary.close(); + + dictFile.delete(); + } + + public void testFlushWithGCDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int bigramProbability = 10; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + dictFile.delete(); + } + + // TODO: Evaluate performance of GC + public void testAddBigramWordsAndFlashWithGC() { + final int wordCount = 100; + final int bigramCount = 1000; + final int codePointSetSize = 30; + // TODO: Use various seeds such as a current timestamp to make this test more random. + final int seed = 314159265; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final int[] unigramProbabilities = new int[wordCount]; + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities[i] = unigramProbability; + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + final int[][] probabilities = new int[wordCount][wordCount]; + + for (int i = 0; i < wordCount; ++i) { + for (int j = 0; j < wordCount; ++j) { + probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; + } + } + + for (int i = 0; i < bigramCount; i++) { + final int word0Index = random.nextInt(wordCount); + final int word1Index = random.nextInt(wordCount); + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( + unigramProbabilities[word1Index], bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + for (int i = 0; i < words.size(); i++) { + for (int j = 0; j < words.size(); j++) { + assertEquals(probabilities[i][j], + binaryDictionary.getBigramProbability(words.get(i), words.get(j))); + } + } + dictFile.delete(); + } + + public void testRandomOperetionsAndFlashWithGC() { + final int flashWithGCIterationCount = 50; + final int operationCountInEachIteration = 200; + final int initialUnigramCount = 100; + final float addUnigramProb = 0.5f; + final float addBigramProb = 0.8f; + final float removeBigramProb = 0.2f; + final int codePointSetSize = 30; + final int seed = 141421356; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = + new HashMap<Pair<String, String>, Integer>(); + for (int i = 0; i < initialUnigramCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) { + // Add unigram. + if (random.nextFloat() < addUnigramProb) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + // Add bigram. + if (random.nextFloat() < addBigramProb && words.size() > 2) { + final int word0Index = random.nextInt(words.size()); + int word1Index = random.nextInt(words.size() - 1); + if (word0Index <= word1Index) { + word1Index++; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + bigramWords.add(bigram); + bigramProbabilities.put(bigram, bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + // Remove bigram. + if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) { + final int bigramIndex = random.nextInt(bigramWords.size()); + final Pair<String, String> bigram = bigramWords.get(bigramIndex); + bigramWords.remove(bigramIndex); + bigramProbabilities.remove(bigram); + binaryDictionary.removeBigramWords(bigram.first, bigram.second); + } + } + + // Test whether the all unigram operations are collectlly handled. + for (int i = 0; i < words.size(); i++) { + final String word = words.get(i); + final int unigramProbability = unigramProbabilities.get(word); + assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); + } + // Test whether the all bigram operations are collectlly handled. + for (int i = 0; i < bigramWords.size(); i++) { + final Pair<String, String> bigram = bigramWords.get(i); + final int unigramProbability = unigramProbabilities.get(bigram.second); + final int probability; + if (bigramProbabilities.containsKey(bigram)) { + final int bigramProbability = bigramProbabilities.get(bigram); + probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + } else { + probability = Dictionary.NOT_A_PROBABILITY; + } + assertEquals(probability, + binaryDictionary.getBigramProbability(bigram.first, bigram.second)); + } + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + } + + dictFile.delete(); + } + + public void testAddManyUnigramsAndFlushWithGC() { + final int flashWithGCIterationCount = 3; + final int codePointSetSize = 50; + final int seed = 22360679; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + + final ArrayList<String> words = new ArrayList<String>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + BinaryDictionary binaryDictionary; + for (int i = 0; i < flashWithGCIterationCount; i++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + while(!binaryDictionary.needsToRunGC()) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + for (int j = 0; j < words.size(); j++) { + final String word = words.get(j); + final int unigramProbability = unigramProbabilities.get(word); + assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); + } + + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + } + + dictFile.delete(); + } +} diff --git a/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java b/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java new file mode 100644 index 000000000..ecf3af736 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; + +/** + * Unit test for ExpandableDictionary + */ +@SmallTest +public class ExpandableDictionaryTests extends AndroidTestCase { + + private final static int UNIGRAM_FREQ = 50; + + public void testAddWordAndGetWordFrequency() { + final ExpandableDictionary dict = new ExpandableDictionary(Dictionary.TYPE_USER); + + // Add words + dict.addWord("abcde", "abcde", UNIGRAM_FREQ); + dict.addWord("abcef", null, UNIGRAM_FREQ + 1); + + // Check words + assertFalse(dict.isValidWord("abcde")); + assertEquals(UNIGRAM_FREQ, dict.getWordFrequency("abcde")); + assertTrue(dict.isValidWord("abcef")); + assertEquals(UNIGRAM_FREQ+1, dict.getWordFrequency("abcef")); + + dict.addWord("abc", null, UNIGRAM_FREQ + 2); + assertTrue(dict.isValidWord("abc")); + assertEquals(UNIGRAM_FREQ + 2, dict.getWordFrequency("abc")); + + // Add existing word with lower frequency + dict.addWord("abc", null, UNIGRAM_FREQ); + assertEquals(UNIGRAM_FREQ + 2, dict.getWordFrequency("abc")); + + // Add existing word with higher frequency + dict.addWord("abc", null, UNIGRAM_FREQ + 3); + assertEquals(UNIGRAM_FREQ + 3, dict.getWordFrequency("abc")); + } +} diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index 65dfd2dde..cadd0f8f3 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -20,7 +20,7 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import java.util.HashMap; @@ -30,21 +30,21 @@ import java.util.HashMap; @SmallTest public class FusionDictionaryTests extends AndroidTestCase { public void testFindWordInTree() { - FusionDictionary dict = new FusionDictionary(new Node(), + FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); dict.add("abc", 10, null, false /* isNotAWord */); - assertNull(FusionDictionary.findWordInTree(dict.mRoot, "aaa")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "abc")); + assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc")); dict.add("aa", 10, null, false /* isNotAWord */); - assertNull(FusionDictionary.findWordInTree(dict.mRoot, "aaa")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "aa")); + assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa")); dict.add("babcd", 10, null, false /* isNotAWord */); dict.add("bacde", 10, null, false /* isNotAWord */); - assertNull(FusionDictionary.findWordInTree(dict.mRoot, "ba")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "babcd")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "bacde")); + assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "bacde")); } } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java index d27a7a903..cc2569f5e 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin; import android.test.suitebuilder.annotation.LargeTest; +import android.view.inputmethod.BaseInputConnection; @LargeTest public class InputLogicTests extends InputTestsBase { @@ -133,6 +134,13 @@ public class InputLogicTests extends InputTestsBase { assertEquals("simple auto-correct", EXPECTED_RESULT, mEditText.getText().toString()); } + public void testAutoCorrectWithQuote() { + final String STRING_TO_TYPE = "didn' "; + final String EXPECTED_RESULT = "didn't "; + type(STRING_TO_TYPE); + assertEquals("auto-correct with quote", EXPECTED_RESULT, mEditText.getText().toString()); + } + public void testAutoCorrectWithPeriod() { final String STRING_TO_TYPE = "tgis."; final String EXPECTED_RESULT = "this."; @@ -290,5 +298,47 @@ public class InputLogicTests extends InputTestsBase { } assertEquals("delete whole composing word", "", mEditText.getText().toString()); } + + public void testResumeSuggestionOnBackspace() { + final String WORD_TO_TYPE = "and this "; + type(WORD_TO_TYPE); + assertEquals("resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + type(Constants.CODE_DELETE); + assertEquals("resume suggestion on backspace", 4, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("resume suggestion on backspace", 8, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + private void helperTestComposing(final String wordToType, final boolean shouldBeComposing) { + mEditText.setText(""); + type(wordToType); + assertEquals("start composing inside text", shouldBeComposing ? 0 : -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("start composing inside text", shouldBeComposing ? wordToType.length() : -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testStartComposing() { + // Should start composing on a letter + helperTestComposing("a", true); + type(" "); // To reset the composing state + // Should not start composing on quote + helperTestComposing("'", false); + type(" "); + helperTestComposing("'-", false); + type(" "); + // Should not start composing on dash + helperTestComposing("-", false); + type(" "); + helperTestComposing("-'", false); + type(" "); + helperTestComposing("a-", true); + type(" "); + helperTestComposing("a'", true); + } // TODO: Add some tests for non-BMP characters } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java new file mode 100644 index 000000000..0f0ebafb9 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.suitebuilder.annotation.LargeTest; +import android.view.inputmethod.BaseInputConnection; + +import com.android.inputmethod.latin.suggestions.SuggestionStripView; + +@LargeTest +public class InputLogicTestsLanguageWithoutSpaces extends InputTestsBase { + public void testAutoCorrectForLanguageWithoutSpaces() { + final String STRING_TO_TYPE = "tgis is"; + final String EXPECTED_RESULT = "thisis"; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(STRING_TO_TYPE); + assertEquals("simple auto-correct for language without spaces", EXPECTED_RESULT, + mEditText.getText().toString()); + } + + public void testRevertAutoCorrectForLanguageWithoutSpaces() { + final String STRING_TO_TYPE = "tgis "; + final String EXPECTED_INTERMEDIATE_RESULT = "this"; + final String EXPECTED_FINAL_RESULT = "tgis"; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(STRING_TO_TYPE); + assertEquals("simple auto-correct for language without spaces", + EXPECTED_INTERMEDIATE_RESULT, mEditText.getText().toString()); + type(Constants.CODE_DELETE); + assertEquals("simple auto-correct for language without spaces", + EXPECTED_FINAL_RESULT, mEditText.getText().toString()); + // Check we are back to composing the word + assertEquals("don't resume suggestion on backspace", 0, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("don't resume suggestion on backspace", 4, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testDontResumeSuggestionOnBackspace() { + final String WORD_TO_TYPE = "and this "; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(WORD_TO_TYPE); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + type(" "); + type(Constants.CODE_DELETE); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testStartComposingInsideText() { + final String WORD_TO_TYPE = "abcdefgh "; + final int typedLength = WORD_TO_TYPE.length() - 1; // -1 because space gets eaten + final int CURSOR_POS = 4; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(WORD_TO_TYPE); + mLatinIME.onUpdateSelection(0, 0, typedLength, typedLength, -1, -1); + mInputConnection.setSelection(CURSOR_POS, CURSOR_POS); + mLatinIME.onUpdateSelection(typedLength, typedLength, + CURSOR_POS, CURSOR_POS, -1, -1); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + assertEquals("start composing inside text", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("start composing inside text", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + type("xxxx"); + assertEquals("start composing inside text", 4, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("start composing inside text", 8, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testPredictions() { + final String WORD_TO_TYPE = "Barack "; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Make sure there is no space + assertEquals("predictions in lang without spaces", "Barack", + mEditText.getText().toString()); + // Test the first prediction is displayed + assertEquals("predictions in lang without spaces", "Obama", + mLatinIME.getFirstSuggestedWord()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/InputPointersTests.java b/tests/src/com/android/inputmethod/latin/InputPointersTests.java index f0b6acc75..5095f9606 100644 --- a/tests/src/com/android/inputmethod/latin/InputPointersTests.java +++ b/tests/src/com/android/inputmethod/latin/InputPointersTests.java @@ -244,4 +244,20 @@ public class InputPointersTests extends AndroidTestCase { expecteds[i + expectedPos], actuals[i + actualPos]); } } + + public void testShift() { + final InputPointers src = new InputPointers(DEFAULT_CAPACITY); + final int limit = 100; + final int shiftAmount = 20; + for (int i = 0; i < limit; i++) { + src.addPointer(i, i * 2, i * 3, i * 4); + } + src.shift(shiftAmount); + for (int i = 0; i < limit - shiftAmount; ++i) { + assertEquals("xCoordinates at " + i, i + shiftAmount, src.getXCoordinates()[i]); + assertEquals("yCoordinates at " + i, (i + shiftAmount) * 2, src.getYCoordinates()[i]); + assertEquals("pointerIds at " + i, (i + shiftAmount) * 3, src.getPointerIds()[i]); + assertEquals("times at " + i, (i + shiftAmount) * 4, src.getTimes()[i]); + } + } } diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java index cc3e0d74d..234bb1b31 100644 --- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java +++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java @@ -44,8 +44,10 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { private static final String PREF_DEBUG_MODE = "debug_mode"; - // The message that sets the underline is posted with a 100 ms delay + // The message that sets the underline is posted with a 200 ms delay protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 200; + // The message that sets predictions is posted with a 200 ms delay + protected static final int DELAY_TO_WAIT_FOR_PREDICTIONS = 200; protected LatinIME mLatinIME; protected Keyboard mKeyboard; @@ -233,9 +235,6 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { --remainingAttempts; } } - if (!mLatinIME.hasMainDictionary()) { - throw new RuntimeException("Can't initialize the main dictionary"); - } } protected void changeLanguage(final String locale) { @@ -247,9 +246,21 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { waitForDictionaryToBeLoaded(); } + protected void changeKeyboardLocaleAndDictLocale(final String keyboardLocale, + final String dictLocale) { + changeLanguage(keyboardLocale); + if (!keyboardLocale.equals(dictLocale)) { + mLatinIME.replaceMainDictionaryForTest( + LocaleUtils.constructLocaleFromString(dictLocale)); + } + waitForDictionaryToBeLoaded(); + } + protected void pickSuggestionManually(final int index, final String suggestion) { mLatinIME.pickSuggestionManually(index, new SuggestedWordInfo(suggestion, 1, - SuggestedWordInfo.KIND_CORRECTION, "main")); + SuggestedWordInfo.KIND_CORRECTION, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } // Helper to avoid writing the try{}catch block each time diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java index 8d0fe014c..a594baf0b 100644 --- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java +++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java @@ -34,9 +34,14 @@ public class SuggestedWordsTests extends AndroidTestCase { final int NUMBER_OF_ADDED_SUGGESTIONS = 5; final ArrayList<SuggestedWordInfo> list = CollectionUtils.newArrayList(); list.add(new SuggestedWordInfo(TYPED_WORD, TYPED_WORD_FREQ, - SuggestedWordInfo.KIND_TYPED, "")); + SuggestedWordInfo.KIND_TYPED, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); for (int i = 0; i < NUMBER_OF_ADDED_SUGGESTIONS; ++i) { - list.add(new SuggestedWordInfo("" + i, 1, SuggestedWordInfo.KIND_CORRECTION, "")); + list.add(new SuggestedWordInfo("" + i, 1, SuggestedWordInfo.KIND_CORRECTION, + null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } final SuggestedWords words = new SuggestedWords( @@ -59,4 +64,37 @@ public class SuggestedWordsTests extends AndroidTestCase { assertEquals("0", wordsWithoutTyped.getWord(0)); assertEquals(SuggestedWordInfo.KIND_CORRECTION, wordsWithoutTyped.getInfo(0).mKind); } + + // Helper for testGetTransformedWordInfo + private SuggestedWordInfo createWordInfo(final String s) { + // Use 100 as the frequency because the numerical value does not matter as + // long as it's > 1 and < INT_MAX. + return new SuggestedWordInfo(s, 100, + SuggestedWordInfo.KIND_TYPED, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); + } + + // Helper for testGetTransformedWordInfo + private SuggestedWordInfo transformWordInfo(final String info, + final int trailingSingleQuotesCount) { + return Suggest.getTransformedSuggestedWordInfo(createWordInfo(info), + Locale.ENGLISH, false /* isAllUpperCase */, false /* isFirstCharCapitalized */, + trailingSingleQuotesCount); + } + + public void testGetTransformedSuggestedWordInfo() { + SuggestedWordInfo result = transformWordInfo("word", 0); + assertEquals(result.mWord, "word"); + result = transformWordInfo("word", 1); + assertEquals(result.mWord, "word'"); + result = transformWordInfo("word", 3); + assertEquals(result.mWord, "word'''"); + result = transformWordInfo("didn't", 0); + assertEquals(result.mWord, "didn't"); + result = transformWordInfo("didn't", 1); + assertEquals(result.mWord, "didn't"); + result = transformWordInfo("didn't", 3); + assertEquals(result.mWord, "didn't''"); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 9664779f0..a4d94262f 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -22,29 +22,32 @@ import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; import android.util.SparseArray; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.Random; import java.util.Set; +import java.util.TreeMap; /** - * Unit tests for BinaryDictDecoder and BinaryDictEncoder. + * Unit tests for BinaryDictDecoderUtils and BinaryDictEncoderUtils. */ @LargeTest public class BinaryDictDecoderEncoderTests extends AndroidTestCase { @@ -54,22 +57,31 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private static final int UNIGRAM_FREQ = 10; private static final int BIGRAM_FREQ = 50; private static final int TOLERANCE_OF_BIGRAM_FREQ = 5; + private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50; + private static final int NUM_OF_SHORTCUTS = 5; private static final int USE_BYTE_ARRAY = 1; private static final int USE_BYTE_BUFFER = 2; - private static final List<String> sWords = CollectionUtils.newArrayList(); + private static final ArrayList<String> sWords = CollectionUtils.newArrayList(); private static final SparseArray<List<Integer>> sEmptyBigrams = CollectionUtils.newSparseArray(); private static final SparseArray<List<Integer>> sStarBigrams = CollectionUtils.newSparseArray(); private static final SparseArray<List<Integer>> sChainBigrams = CollectionUtils.newSparseArray(); + private static final HashMap<String, List<String>> sShortcuts = CollectionUtils.newHashMap(); private static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); private static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); + private static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); + private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); + + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; public BinaryDictDecoderEncoderTests() { this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); @@ -80,7 +92,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { Log.e(TAG, "Testing dictionary: seed is " + seed); final Random random = new Random(seed); sWords.clear(); - final int[] codePointSet = generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, random); + final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, + random); generateWords(maxUnigrams, random, codePointSet); for (int i = 0; i < sWords.size(); ++i) { @@ -94,65 +107,33 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { for (int i = 1; i < sWords.size(); ++i) { sStarBigrams.get(0).add(i); } - } - - private int[] generateCodePointSet(final int codePointSetSize, final Random random) { - final int[] codePointSet = new int[codePointSetSize]; - for (int i = codePointSet.length - 1; i >= 0; ) { - final int r = Math.abs(random.nextInt()); - if (r < 0) continue; - // Don't insert 0~0x20, but insert any other code point. - // Code points are in the range 0~0x10FFFF. - final int candidateCodePoint = (int)(0x20 + r % (Character.MAX_CODE_POINT - 0x20)); - // Code points between MIN_ and MAX_SURROGATE are not valid on their own. - if (candidateCodePoint >= Character.MIN_SURROGATE - && candidateCodePoint <= Character.MAX_SURROGATE) continue; - codePointSet[i] = candidateCodePoint; - --i; - } - return codePointSet; - } - - // Utilities for test - /** - * Makes new buffer according to BUFFER_TYPE. - */ - private void getBuffer(final BinaryDictReader reader, final int bufferType) - throws FileNotFoundException, IOException { - if (bufferType == USE_BYTE_BUFFER) { - reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); - } else if (bufferType == USE_BYTE_ARRAY) { - reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + sShortcuts.clear(); + for (int i = 0; i < NUM_OF_NODES_HAVING_SHORTCUTS; ++i) { + final int from = Math.abs(random.nextInt()) % sWords.size(); + sShortcuts.put(sWords.get(from), new ArrayList<String>()); + for (int j = 0; j < NUM_OF_SHORTCUTS; ++j) { + final int to = Math.abs(random.nextInt()) % sWords.size(); + sShortcuts.get(sWords.get(from)).add(sWords.get(to)); + } } } - /** - * Generates a random word. - */ - private String generateWord(final Random random, final int[] codePointSet) { - StringBuilder builder = new StringBuilder(); - // 8 * 4 = 32 chars max, but we do it the following way so as to bias the random toward - // longer words. This should be closer to natural language, and more importantly, it will - // exercise the algorithms in dicttool much more. - final int count = 1 + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5); - while (builder.length() < count) { - builder.appendCodePoint(codePointSet[Math.abs(random.nextInt()) % codePointSet.length]); - } - return builder.toString(); + private DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new Ver4DictEncoder(getContext().getCacheDir()); + } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { + return new Ver3DictEncoder(file); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } } private void generateWords(final int number, final Random random, final int[] codePointSet) { final Set<String> wordSet = CollectionUtils.newHashSet(); while (wordSet.size() < number) { - wordSet.add(generateWord(random, codePointSet)); + wordSet.add(CodePointUtils.generateWord(random, codePointSet)); } sWords.addAll(wordSet); } @@ -161,7 +142,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { * Adds unigrams to the dictionary. */ private void addUnigrams(final int number, final FusionDictionary dict, - final List<String> words, final Map<String, List<String>> shortcutMap) { + final List<String> words, final HashMap<String, List<String>> shortcutMap) { for (int i = 0; i < number; ++i) { final String word = words.get(i); final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList(); @@ -200,17 +181,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final FileOutputStream out = new FileOutputStream(file); + final DictEncoder dictEncoder = getDictEncoder(file, formatOptions); now = System.currentTimeMillis(); // If you need to dump the dict to a textual file, uncomment the line below and the // function above // dumpToCombinedFileForDebug(file, "/tmp/foo"); - BinaryDictEncoder.writeDictionaryBinary(out, dict, formatOptions); + dictEncoder.writeDictionary(dict, formatOptions); diff = System.currentTimeMillis() - now; - - out.flush(); - out.close(); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); } catch (UnsupportedFormatException e) { @@ -221,31 +199,35 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private void checkDictionary(final FusionDictionary dict, final List<String> words, - final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap) { + final SparseArray<List<Integer>> bigrams, + final HashMap<String, List<String>> shortcutMap) { assertNotNull(dict); // check unigram for (final String word : words) { - final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, word); - assertNotNull(cg); + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); + assertNotNull(ptNode); } // check bigram for (int i = 0; i < bigrams.size(); ++i) { final int w1 = bigrams.keyAt(i); for (final int w2 : bigrams.valueAt(i)) { - final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, words.get(w1)); - assertNotNull(words.get(w1) + "," + words.get(w2), cg.getBigram(words.get(w2))); + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, + words.get(w1)); + assertNotNull(words.get(w1) + "," + words.get(w2), ptNode.getBigram(words.get(w2))); } } // check shortcut if (shortcutMap != null) { - for (final Map.Entry<String, List<String>> entry : shortcutMap.entrySet()) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, entry.getKey()); + for (final Entry<String, List<String>> entry : shortcutMap.entrySet()) { + assertTrue(words.contains(entry.getKey())); + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, + entry.getKey()); for (final String word : entry.getValue()) { assertNotNull("shortcut not found: " + entry.getKey() + ", " + word, - group.getShortcut(word)); + ptNode.getShortcut(word)); } } } @@ -259,20 +241,51 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; } + private DictionaryOptions getDictionaryOptions(final String id, final String version) { + final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), + false, false); + options.mAttributes.put("version", version); + options.mAttributes.put("dictionary", id); + return options; + } + + private File setUpDictionaryFile(final String name, final String version) { + File file = null; + try { + file = new File(getContext().getCacheDir(), name + "." + version + + TEST_DICT_FILE_EXTENSION); + file.createNewFile(); + } catch (IOException e) { + // do nothing + } + assertTrue("Failed to create the dictionary file.", file.exists()); + return file; + } + + private DictDecoder getDictDecoder(final File file, final int bufferType, + final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + final FileHeader header = new FileHeader(0, dictOptions, formatOptions); + return FormatSpec.getDictDecoder(new File(getContext().getCacheDir(), + header.getId() + "." + header.getVersion()), bufferType); + } else { + return FormatSpec.getDictDecoder(file, bufferType); + } + } // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List<String> words, - final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap, - final int bufferType) { + final SparseArray<List<Integer>> bigrams, + final HashMap<String, List<String>> shortcutMap, final int bufferType, + final FormatOptions formatOptions, final DictionaryOptions dictOptions) { long now, diff = -1; - final BinaryDictReader reader = new BinaryDictReader(file); FusionDictionary dict = null; try { - getBuffer(reader, bufferType); - assertNotNull(reader.getBuffer()); + final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, + dictOptions); now = System.currentTimeMillis(); - dict = BinaryDictDecoder.readDictionaryBinary(reader, null); + dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException while reading dictionary", e); @@ -286,25 +299,23 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Tests for readDictionaryBinary and writeDictionaryBinary private String runReadAndWrite(final List<String> words, - final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcuts, + final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcuts, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { - File file = null; - try { - file = File.createTempFile("runReadAndWrite", ".dict", getContext().getCacheDir()); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - assertNotNull(file); - final FusionDictionary dict = new FusionDictionary(new Node(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + final String dictName = "runReadAndWrite"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); + + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); checkDictionary(dict, words, bigrams, shortcuts); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); + final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType, + formatOptions, dict.mOptions); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -318,6 +329,28 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { formatOptions, "chain")); results.add(runReadAndWrite(sWords, sStarBigrams, null /* shortcuts */, bufferType, formatOptions, "star")); + results.add(runReadAndWrite(sWords, sEmptyBigrams, sShortcuts, bufferType, formatOptions, + "unigram with shortcuts")); + results.add(runReadAndWrite(sWords, sChainBigrams, sShortcuts, bufferType, formatOptions, + "chain with shortcuts")); + results.add(runReadAndWrite(sWords, sStarBigrams, sShortcuts, bufferType, formatOptions, + "star with shortcuts")); + } + + // Unit test for CharEncoding.readString and CharEncoding.writeString. + public void testCharEncoding() { + // the max length of a word in sWords is less than 50. + // See generateWords. + final byte[] buffer = new byte[50 * 3]; + final DictBuffer dictBuffer = new ByteArrayDictBuffer(buffer); + for (final String word : sWords) { + Log.d("testReadAndWriteString", "write : " + word); + Arrays.fill(buffer, (byte)0); + CharEncoding.writeString(buffer, 0, word); + dictBuffer.position(0); + final String str = CharEncoding.readString(dictBuffer); + assertEquals(word, str); + } } public void testReadAndWriteWithByteBuffer() { @@ -326,6 +359,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2); runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -338,6 +373,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2); runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -348,9 +385,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private void checkWordMap(final List<String> expectedWords, final SparseArray<List<Integer>> expectedBigrams, - final Map<Integer, String> resultWords, - final Map<Integer, Integer> resultFrequencies, - final Map<Integer, ArrayList<PendingAttribute>> resultBigrams) { + final TreeMap<Integer, String> resultWords, + final TreeMap<Integer, Integer> resultFrequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams) { // check unigrams final Set<String> actualWordsSet = new HashSet<String>(resultWords.values()); final Set<String> expectedWordsSet = new HashSet<String>(expectedWords); @@ -361,7 +398,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } // check bigrams - final Map<String, List<String>> expBigrams = new HashMap<String, List<String>>(); + final HashMap<String, List<String>> expBigrams = new HashMap<String, List<String>>(); for (int i = 0; i < expectedBigrams.size(); ++i) { final String word1 = expectedWords.get(expectedBigrams.keyAt(i)); for (int w2 : expectedBigrams.valueAt(i)) { @@ -372,7 +409,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } - final Map<String, List<String>> actBigrams = new HashMap<String, List<String>>(); + final HashMap<String, List<String>> actBigrams = new HashMap<String, List<String>>(); for (Entry<Integer, ArrayList<PendingAttribute>> entry : resultBigrams.entrySet()) { final String word1 = resultWords.get(entry.getKey()); final int unigramFreq = resultFrequencies.get(entry.getKey()); @@ -383,7 +420,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } actBigrams.get(word1).add(word2); - final int bigramFreq = BinaryDictDecoder.reconstructBigramFrequency( + final int bigramFreq = BinaryDictIOUtils.reconstructBigramFrequency( unigramFreq, attr.mFrequency); assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); } @@ -393,22 +430,21 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words, - final SparseArray<List<Integer>> bigrams, final int bufferType) { + final SparseArray<List<Integer>> bigrams, final int bufferType, + final FormatOptions formatOptions, final DictionaryOptions dictOptions) { FileInputStream inStream = null; - final Map<Integer, String> resultWords = CollectionUtils.newTreeMap(); - final Map<Integer, ArrayList<PendingAttribute>> resultBigrams = + final TreeMap<Integer, String> resultWords = CollectionUtils.newTreeMap(); + final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams = CollectionUtils.newTreeMap(); - final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap(); + final TreeMap<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap(); long now = -1, diff = -1; - final BinaryDictReader reader = new BinaryDictReader(file); try { - getBuffer(reader, bufferType); - assertNotNull("Can't get buffer.", reader.getBuffer()); + final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, + dictOptions); now = System.currentTimeMillis(); - BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, resultWords, resultFreqs, - resultBigrams); + dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException", e); @@ -428,50 +464,48 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return diff; } - private String runReadUnigramsAndBigramsBinary(final List<String> words, + private String runReadUnigramsAndBigramsBinary(final ArrayList<String> words, final SparseArray<List<Integer>> bigrams, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { - File file = null; - try { - file = File.createTempFile("runReadUnigrams", ".dict", getContext().getCacheDir()); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - assertNotNull(file); + final String dictName = "runReadUnigrams"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); // making the dictionary from lists of words. - final FusionDictionary dict = new FusionDictionary(new Node(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType); + long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, + formatOptions, dict.mOptions); long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, - bufferType); + bufferType, formatOptions, dict.mOptions); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap + " : " + message + " : " + outputOptions(bufferType, formatOptions); } - private void runReadUnigramsAndBigramsTests(final List<String> results, final int bufferType, - final FormatSpec.FormatOptions formatOptions) { + private void runReadUnigramsAndBigramsTests(final ArrayList<String> results, + final int bufferType, final FormatSpec.FormatOptions formatOptions) { results.add(runReadUnigramsAndBigramsBinary(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram")); results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, formatOptions, "chain")); - results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, + results.add(runReadUnigramsAndBigramsBinary(sWords, sStarBigrams, bufferType, formatOptions, "star")); } public void testReadUnigramsAndBigramsBinaryWithByteBuffer() { - final List<String> results = CollectionUtils.newArrayList(); + final ArrayList<String> results = CollectionUtils.newArrayList(); runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2); runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -479,11 +513,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testReadUnigramsAndBigramsBinaryWithByteArray() { - final List<String> results = CollectionUtils.newArrayList(); + final ArrayList<String> results = CollectionUtils.newArrayList(); runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2); runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -491,31 +527,30 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } // Tests for getTerminalPosition - private String getWordFromBinary(final FusionDictionaryBufferInterface buffer, - final int address) { - if (buffer.position() != 0) buffer.position(0); + private String getWordFromBinary(final DictDecoder dictDecoder, final int address) { + if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0); - FileHeader header = null; + FileHeader fileHeader = null; try { - header = BinaryDictDecoder.readHeader(buffer); + fileHeader = dictDecoder.readHeader(); } catch (IOException e) { return null; } catch (UnsupportedFormatException e) { return null; } - if (header == null) return null; - return BinaryDictDecoder.getWordAtAddress(buffer, header.mHeaderSize, - address - header.mHeaderSize, header.mFormatOptions).mWord; + if (fileHeader == null) return null; + return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize, + address, fileHeader.mFormatOptions).mWord; } - private long runGetTerminalPosition(final FusionDictionaryBufferInterface buffer, - final String word, int index, boolean contained) { + private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word, + int index, boolean contained) { final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; long diff = -1; int position = -1; try { final long now = System.nanoTime(); - position = BinaryDictIOUtils.getTerminalPosition(buffer, word); + position = dictDecoder.getTerminalPosition(word); diff = System.nanoTime() - now; } catch (IOException e) { Log.e(TAG, "IOException while getTerminalPosition", e); @@ -524,50 +559,43 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } assertEquals(FormatSpec.NOT_VALID_WORD != position, contained); - if (contained) assertEquals(getWordFromBinary(buffer, position), word); + if (contained) assertEquals(getWordFromBinary(dictDecoder, position), word); return diff; } - public void testGetTerminalPosition() { - File file = null; - try { - file = File.createTempFile("testGetTerminalPosition", ".dict", - getContext().getCacheDir()); - } catch (IOException e) { - // do nothing - } - assertNotNull(file); + private void runGetTerminalPosition(final ArrayList<String> words, + final SparseArray<List<Integer>> bigrams, final int bufferType, + final FormatOptions formatOptions, final String message) { + final String dictName = "testGetTerminalPosition"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); - final FusionDictionary dict = new FusionDictionary(new Node(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + getDictionaryOptions(dictName, dictVersion)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); - timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); + addBigrams(dict, words, bigrams); + timeWritingDictToFile(file, dict, formatOptions); - final BinaryDictReader reader = new BinaryDictReader(file); - FusionDictionaryBufferInterface buffer = null; + final DictDecoder dictDecoder = getDictDecoder(file, DictDecoder.USE_BYTEARRAY, + formatOptions, dict.mOptions); try { - buffer = reader.openAndGetBuffer( - new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + dictDecoder.openDictBuffer(); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", buffer); + assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); try { // too long word final String longWord = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; - assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, longWord)); + assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(longWord)); // null - assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, null)); + assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(null)); // empty string - assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, "")); + assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition("")); } catch (IOException e) { } catch (UnsupportedFormatException e) { } @@ -575,59 +603,80 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Test a word that is contained within the dictionary. long sum = 0; for (int i = 0; i < sWords.size(); ++i) { - final long time = runGetTerminalPosition(buffer, sWords.get(i), i, true); + final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), i, true); sum += time == -1 ? 0 : time; } - Log.d(TAG, "per a search : " + (((double)sum) / sWords.size() / 1000000)); + Log.d(TAG, "per search : " + (((double)sum) / sWords.size() / 1000000) + " : " + message + + " : " + outputOptions(bufferType, formatOptions)); // Test a word that isn't contained within the dictionary. final Random random = new Random((int)System.currentTimeMillis()); - final int[] codePointSet = generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, random); + final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, + random); for (int i = 0; i < 1000; ++i) { - final String word = generateWord(random, codePointSet); + final String word = CodePointUtils.generateWord(random, codePointSet); if (sWords.indexOf(word) != -1) continue; - runGetTerminalPosition(buffer, word, i, false); + checkGetTerminalPosition(dictDecoder, word, i, false); } } - public void testDeleteWord() { - File file = null; - try { - file = File.createTempFile("testDeleteWord", ".dict", getContext().getCacheDir()); - } catch (IOException e) { - // do nothing + private void runGetTerminalPositionTests(final ArrayList<String> results, final int bufferType, + final FormatOptions formatOptions) { + runGetTerminalPosition(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram"); + } + + public void testGetTerminalPosition() { + final ArrayList<String> results = CollectionUtils.newArrayList(); + + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION2); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); + + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION2); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); + + for (final String result : results) { + Log.d(TAG, result); } - assertNotNull(file); + } + + public void testDeleteWord() { + final String dictName = "testDeleteWord"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( new HashMap<String, String>(), false, false)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final BinaryDictReader reader = new BinaryDictReader(file); - FusionDictionaryBufferInterface buffer = null; + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); try { - buffer = reader.openAndGetBuffer( - new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + dictDecoder.openDictBuffer(); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", buffer); + assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0))); - DynamicBinaryDictIOUtils.deleteWord(buffer, sWords.get(0)); + dictDecoder.getTerminalPosition(sWords.get(0))); + DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(0)); assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0))); + dictDecoder.getTerminalPosition(sWords.get(0))); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5))); - DynamicBinaryDictIOUtils.deleteWord(buffer, sWords.get(5)); + dictDecoder.getTerminalPosition(sWords.get(5))); + DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(5)); assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5))); + dictDecoder.getTerminalPosition(sWords.get(5))); } catch (IOException e) { } catch (UnsupportedFormatException e) { } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index f2476b2e6..a83749499 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -21,20 +21,16 @@ import android.test.MoreAsserts; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder.ByteBufferWrapper; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.Random; @@ -49,6 +45,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { public static final int DEFAULT_MAX_UNIGRAMS = 1500; private final int mMaxUnigrams; + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", @@ -86,8 +84,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { return builder.toString(); } - private static void printCharGroup(final CharGroupInfo info) { - Log.d(TAG, " CharGroup at " + info.mOriginalAddress); + private static void printPtNode(final PtNodeInfo info) { + Log.d(TAG, " PtNode at " + info.mOriginalAddress); Log.d(TAG, " flags = " + info.mFlags); Log.d(TAG, " parentAddress = " + info.mParentAddress); Log.d(TAG, " characters = " + new String(info.mCharacters, 0, @@ -111,70 +109,75 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { Log.d(TAG, " end address = " + info.mEndAddress); } - private static void printNode(final FusionDictionaryBufferInterface buffer, + private static void printNode(final Ver3DictDecoder dictDecoder, final FormatSpec.FormatOptions formatOptions) { - Log.d(TAG, "Node at " + buffer.position()); - final int count = BinaryDictDecoder.readCharGroupCount(buffer); - Log.d(TAG, " charGroupCount = " + count); + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + Log.d(TAG, "Node at " + dictBuffer.position()); + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + Log.d(TAG, " ptNodeCount = " + count); for (int i = 0; i < count; ++i) { - final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer, - buffer.position(), formatOptions); - printCharGroup(currentInfo); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(dictBuffer.position(), + formatOptions); + printPtNode(currentInfo); } if (formatOptions.mSupportsDynamicUpdate) { - final int forwardLinkAddress = buffer.readUnsignedInt24(); + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); Log.d(TAG, " forwardLinkAddress = " + forwardLinkAddress); } } - private static void printBinaryFile(final FusionDictionaryBufferInterface buffer) + @SuppressWarnings("unused") + private static void printBinaryFile(final Ver3DictDecoder dictDecoder) throws IOException, UnsupportedFormatException { - FileHeader header = BinaryDictDecoder.readHeader(buffer); - while (buffer.position() < buffer.limit()) { - printNode(buffer, header.mFormatOptions); + final FileHeader fileHeader = dictDecoder.readHeader(); + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + while (dictBuffer.position() < dictBuffer.limit()) { + printNode(dictDecoder, fileHeader.mFormatOptions); } } private int getWordPosition(final File file, final String word) { int position = FormatSpec.NOT_VALID_WORD; - FileInputStream inStream = null; + try { - inStream = new FileInputStream(file); - final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( - inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); - position = BinaryDictIOUtils.getTerminalPosition(buffer, word); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + DictDecoder.USE_READONLY_BYTEBUFFER); + position = dictDecoder.getTerminalPosition(word); } catch (IOException e) { } catch (UnsupportedFormatException e) { - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } } return position; } - private CharGroupInfo findWordFromFile(final File file, final String word) { - FileInputStream inStream = null; - CharGroupInfo info = null; + /** + * Find a word using the DictDecoder. + * + * @param dictDecoder the dict decoder + * @param word the word searched + * @return the found ptNodeInfo + * @throws IOException + * @throws UnsupportedFormatException + */ + private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder, + final String word) throws IOException, UnsupportedFormatException { + int position = dictDecoder.getTerminalPosition(word); + if (position != FormatSpec.NOT_VALID_WORD) { + dictDecoder.setPosition(0); + final FileHeader header = dictDecoder.readHeader(); + dictDecoder.setPosition(position); + return dictDecoder.readPtNode(position, header.mFormatOptions); + } + return null; + } + + private PtNodeInfo findWordFromFile(final File file, final String word) { + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + PtNodeInfo info = null; try { - inStream = new FileInputStream(file); - final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( - inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); - info = BinaryDictIOUtils.findWordFromBuffer(buffer, word); + dictDecoder.openDictBuffer(); + info = findWordByBinaryDictReader(dictDecoder, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } } return info; } @@ -183,42 +186,34 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList<WeightedString> bigrams, final ArrayList<WeightedString> shortcuts) { - RandomAccessFile raFile = null; BufferedOutputStream outStream = null; - FusionDictionaryBufferInterface buffer = null; long amountOfTime = -1; try { - raFile = new RandomAccessFile(file, "rw"); - buffer = new ByteBufferWrapper(raFile.getChannel().map( - FileChannel.MapMode.READ_WRITE, 0, file.length())); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + DictDecoder.USE_WRITABLE_BYTEBUFFER); + dictDecoder.openDictBuffer(); outStream = new BufferedOutputStream(new FileOutputStream(file, true)); if (!exist) { assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } final long now = System.nanoTime(); - DynamicBinaryDictIOUtils.insertWord(buffer, outStream, word, frequency, bigrams, + DynamicBinaryDictIOUtils.insertWord(dictDecoder, outStream, word, frequency, bigrams, shortcuts, false, false); amountOfTime = System.nanoTime() - now; outStream.flush(); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); outStream.close(); - raFile.close(); } catch (IOException e) { + Log.e(TAG, "Raised an IOException while inserting a word", e); } catch (UnsupportedFormatException e) { + Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e); } finally { if (outStream != null) { try { outStream.close(); } catch (IOException e) { - // do nothing - } - } - if (raFile != null) { - try { - raFile.close(); - } catch (IOException e) { - // do nothing + Log.e(TAG, "Failed to close the output stream", e); } } } @@ -226,65 +221,48 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } private void deleteWord(final File file, final String word) { - RandomAccessFile raFile = null; - FusionDictionaryBufferInterface buffer = null; try { - raFile = new RandomAccessFile(file, "rw"); - buffer = new ByteBufferWrapper(raFile.getChannel().map( - FileChannel.MapMode.READ_WRITE, 0, file.length())); - DynamicBinaryDictIOUtils.deleteWord(buffer, word); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + DictDecoder.USE_WRITABLE_BYTEBUFFER); + dictDecoder.openDictBuffer(); + DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { - } finally { - if (raFile != null) { - try { - raFile.close(); - } catch (IOException e) { - // do nothing - } - } } } private void checkReverseLookup(final File file, final String word, final int position) { - FileInputStream inStream = null; + try { - inStream = new FileInputStream(file); - final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( - inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); - final FileHeader header = BinaryDictDecoder.readHeader(buffer); - assertEquals(word, BinaryDictDecoder.getWordAtAddress(buffer, header.mHeaderSize, - position - header.mHeaderSize, header.mFormatOptions).mWord); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + final FileHeader fileHeader = dictDecoder.readHeader(); + assertEquals(word, + BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize, + position, fileHeader.mFormatOptions).mWord); } catch (IOException e) { + Log.e(TAG, "Raised an IOException while looking up a word", e); } catch (UnsupportedFormatException e) { - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } + Log.e(TAG, "Raised an UnsupportedFormatException error while looking up a word", e); } } public void testInsertWord() { File file = null; try { - file = File.createTempFile("testInsertWord", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { fail("IOException while creating temporary file: " + e); } // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); dict.add("abcd", 10, null, false); try { - final FileOutputStream out = new FileOutputStream(file); - BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); - out.close(); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } catch (UnsupportedFormatException e) { @@ -321,22 +299,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { public void testInsertWordWithBigrams() { File file = null; try { - file = File.createTempFile("testInsertWordWithBigrams", ".dict", + file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); } catch (IOException e) { fail("IOException while creating temporary file: " + e); } // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); dict.add("abcd", 10, null, false); dict.add("efgh", 15, null, false); try { - final FileOutputStream out = new FileOutputStream(file); - BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); - out.close(); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } catch (UnsupportedFormatException e) { @@ -349,7 +326,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { insertAndCheckWord(file, "banana", 0, false, null, null); insertAndCheckWord(file, "recursive", 60, true, banana, null); - final CharGroupInfo info = findWordFromFile(file, "recursive"); + final PtNodeInfo info = findWordFromFile(file, "recursive"); int bananaPos = getWordPosition(file, "banana"); assertNotNull(info.mBigrams); assertEquals(info.mBigrams.size(), 1); @@ -359,21 +336,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { public void testRandomWords() { File file = null; try { - file = File.createTempFile("testRandomWord", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { } assertNotNull(file); // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, false)); dict.add("initial", 10, null, false); try { - final FileOutputStream out = new FileOutputStream(file); - BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); - out.close(); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); } catch (IOException e) { assertTrue(false); } catch (UnsupportedFormatException e) { diff --git a/tests/src/com/android/inputmethod/latin/makedict/CodePointUtils.java b/tests/src/com/android/inputmethod/latin/makedict/CodePointUtils.java new file mode 100644 index 000000000..36b958af8 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/CodePointUtils.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import java.util.Random; + +// Utility methods related with code points used for tests. +public class CodePointUtils { + private CodePointUtils() { + // This utility class is not publicly instantiable. + } + + public static int[] generateCodePointSet(final int codePointSetSize, final Random random) { + final int[] codePointSet = new int[codePointSetSize]; + for (int i = codePointSet.length - 1; i >= 0; ) { + final int r = Math.abs(random.nextInt()); + if (r < 0) continue; + // Don't insert 0~0x20, but insert any other code point. + // Code points are in the range 0~0x10FFFF. + final int candidateCodePoint = 0x20 + r % (Character.MAX_CODE_POINT - 0x20); + // Code points between MIN_ and MAX_SURROGATE are not valid on their own. + if (candidateCodePoint >= Character.MIN_SURROGATE + && candidateCodePoint <= Character.MAX_SURROGATE) continue; + codePointSet[i] = candidateCodePoint; + --i; + } + return codePointSet; + } + + /** + * Generates a random word. + */ + public static String generateWord(final Random random, final int[] codePointSet) { + StringBuilder builder = new StringBuilder(); + // 8 * 4 = 32 chars max, but we do it the following way so as to bias the random toward + // longer words. This should be closer to natural language, and more importantly, it will + // exercise the algorithms in dicttool much more. + final int count = 1 + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5); + while (builder.length() < count) { + builder.appendCodePoint(codePointSet[Math.abs(random.nextInt()) % codePointSet.length]); + } + return builder.toString(); + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java new file mode 100644 index 000000000..132483d5e --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; +import android.util.Log; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Random; + +/** + * Unit tests for SparseTable. + */ +@LargeTest +public class SparseTableTests extends AndroidTestCase { + private static final String TAG = SparseTableTests.class.getSimpleName(); + + private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 }; + private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7}; + + private final Random mRandom; + private final ArrayList<Integer> mRandomIndex; + + private static final int DEFAULT_SIZE = 10000; + private static final int BLOCK_SIZE = 8; + + public SparseTableTests() { + this(System.currentTimeMillis(), DEFAULT_SIZE); + } + + public SparseTableTests(final long seed, final int tableSize) { + super(); + Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize); + mRandom = new Random(seed); + mRandomIndex = new ArrayList<Integer>(tableSize); + for (int i = 0; i < tableSize; ++i) { + mRandomIndex.add(SparseTable.NOT_EXIST); + } + } + + public void testInitializeWithArray() { + final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE); + for (int i = 0; i < 8; ++i) { + assertEquals(SparseTable.NOT_EXIST, table.get(i)); + } + assertEquals(SparseTable.NOT_EXIST, table.get(8)); + for (int i = 9; i < 16; ++i) { + assertEquals(i - 8, table.get(i)); + } + } + + public void testSet() { + final SparseTable table = new SparseTable(16, BLOCK_SIZE); + table.set(3, 6); + table.set(8, 16); + for (int i = 0; i < 16; ++i) { + if (i == 3 || i == 8) { + assertEquals(i * 2, table.get(i)); + } else { + assertEquals(SparseTable.NOT_EXIST, table.get(i)); + } + } + } + + private void generateRandomIndex(final int size, final int prop) { + for (int i = 0; i < DEFAULT_SIZE; ++i) { + if (mRandom.nextInt(100) < prop) { + mRandomIndex.set(i, mRandom.nextInt()); + } else { + mRandomIndex.set(i, SparseTable.NOT_EXIST); + } + } + } + + private void runTestRandomSet() { + final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE); + int elementCount = 0; + for (int i = 0; i < DEFAULT_SIZE; ++i) { + if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { + table.set(i, mRandomIndex.get(i)); + elementCount++; + } + } + + Log.d(TAG, "table size = " + table.getLookupTableSize() + " + " + + table.getContentTableSize()); + Log.d(TAG, "the table has " + elementCount + " elements"); + for (int i = 0; i < DEFAULT_SIZE; ++i) { + assertEquals(table.get(i), (int)mRandomIndex.get(i)); + } + + // flush and reload + OutputStream lookupOutStream = null; + OutputStream contentOutStream = null; + InputStream lookupInStream = null; + InputStream contentInStream = null; + try { + final File lookupIndexFile = File.createTempFile("testRandomSet", ".small"); + final File contentFile = File.createTempFile("testRandomSet", ".big"); + lookupOutStream = new FileOutputStream(lookupIndexFile); + contentOutStream = new FileOutputStream(contentFile); + table.write(lookupOutStream, contentOutStream); + lookupInStream = new FileInputStream(lookupIndexFile); + contentInStream = new FileInputStream(contentFile); + final byte[] lookupArray = new byte[(int) lookupIndexFile.length()]; + final byte[] contentArray = new byte[(int) contentFile.length()]; + lookupInStream.read(lookupArray); + contentInStream.read(contentArray); + final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE); + for (int i = 0; i < DEFAULT_SIZE; ++i) { + assertEquals(table.get(i), newTable.get(i)); + } + } catch (IOException e) { + Log.d(TAG, "IOException while flushing and realoding", e); + } finally { + if (lookupOutStream != null) { + try { + lookupOutStream.close(); + } catch (IOException e) { + Log.d(TAG, "IOException while closing the stream", e); + } + } + if (contentOutStream != null) { + try { + contentOutStream.close(); + } catch (IOException e) { + Log.d(TAG, "IOException while closing contentStream.", e); + } + } + } + } + + public void testRandomSet() { + for (int i = 0; i <= 100; i += 10) { + generateRandomIndex(DEFAULT_SIZE, i); + runTestRandomSet(); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java index 1c6de50f0..9611599b9 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java @@ -16,14 +16,13 @@ package com.android.inputmethod.latin.makedict; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; -import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory; -import com.android.inputmethod.latin.makedict.BinaryDictReader. - FusionDictionaryBufferFromByteArrayFactory; -import com.android.inputmethod.latin.makedict.BinaryDictReader. - FusionDictionaryBufferFromByteBufferFactory; -import com.android.inputmethod.latin.makedict.BinaryDictReader. - FusionDictionaryBufferFromWritableByteBufferFactory; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFromByteArrayFactory; +import com.android.inputmethod.latin.makedict.DictDecoder. + DictionaryBufferFromReadOnlyByteBufferFactory; +import com.android.inputmethod.latin.makedict.DictDecoder. + DictionaryBufferFromWritableByteBufferFactory; import android.test.AndroidTestCase; import android.util.Log; @@ -33,10 +32,10 @@ import java.io.FileOutputStream; import java.io.IOException; /** - * Unit tests for BinaryDictReader + * Unit tests for Ver3DictDecoder */ -public class BinaryDictReaderTests extends AndroidTestCase { - private static final String TAG = BinaryDictReaderTests.class.getSimpleName(); +public class Ver3DictDecoderTests extends AndroidTestCase { + private static final String TAG = Ver3DictDecoderTests.class.getSimpleName(); private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; @@ -60,8 +59,7 @@ public class BinaryDictReaderTests extends AndroidTestCase { } @SuppressWarnings("null") - public void runTestOpenBuffer(final String testName, - final FusionDictionaryBufferFactory factory) { + public void runTestOpenBuffer(final String testName, final DictionaryBufferFactory factory) { File testFile = null; try { testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir()); @@ -70,9 +68,9 @@ public class BinaryDictReaderTests extends AndroidTestCase { } assertNotNull(testFile); - final BinaryDictReader reader = new BinaryDictReader(testFile); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory); try { - reader.openBuffer(factory); + dictDecoder.openDictBuffer(); } catch (Exception e) { Log.e(TAG, "Failed to open the buffer", e); } @@ -80,32 +78,31 @@ public class BinaryDictReaderTests extends AndroidTestCase { writeDataToFile(testFile); try { - reader.openBuffer(factory); + dictDecoder.openDictBuffer(); } catch (Exception e) { Log.e(TAG, "Raised the exception while opening buffer", e); } - assertEquals(testFile.length(), reader.getBuffer().capacity()); + assertEquals(testFile.length(), dictDecoder.getDictBuffer().capacity()); } public void testOpenBufferWithByteBuffer() { runTestOpenBuffer("testOpenBufferWithByteBuffer", - new FusionDictionaryBufferFromByteBufferFactory()); + new DictionaryBufferFromReadOnlyByteBufferFactory()); } public void testOpenBufferWithByteArray() { runTestOpenBuffer("testOpenBufferWithByteArray", - new FusionDictionaryBufferFromByteArrayFactory()); + new DictionaryBufferFromByteArrayFactory()); } public void testOpenBufferWithWritableByteBuffer() { runTestOpenBuffer("testOpenBufferWithWritableByteBuffer", - new FusionDictionaryBufferFromWritableByteBufferFactory()); + new DictionaryBufferFromWritableByteBufferFactory()); } @SuppressWarnings("null") - public void runTestGetBuffer(final String testName, - final FusionDictionaryBufferFactory factory) { + public void runTestGetBuffer(final String testName, final DictionaryBufferFactory factory) { File testFile = null; try { testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir()); @@ -113,40 +110,41 @@ public class BinaryDictReaderTests extends AndroidTestCase { Log.e(TAG, "IOException while the creating temporary file", e); } - final BinaryDictReader reader = new BinaryDictReader(testFile); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory); // the default return value of getBuffer() must be null. - assertNull("the default return value of getBuffer() is not null", reader.getBuffer()); + assertNull("the default return value of getBuffer() is not null", + dictDecoder.getDictBuffer()); writeDataToFile(testFile); assertTrue(testFile.exists()); Log.d(TAG, "file length = " + testFile.length()); - FusionDictionaryBufferInterface buffer = null; + DictBuffer dictBuffer = null; try { - buffer = reader.openAndGetBuffer(factory); + dictBuffer = dictDecoder.openAndGetDictBuffer(); } catch (IOException e) { Log.e(TAG, "Failed to open and get the buffer", e); } - assertNotNull("the buffer must not be null", buffer); + assertNotNull("the buffer must not be null", dictBuffer); for (int i = 0; i < data.length; ++i) { - assertEquals(data[i], buffer.readUnsignedByte()); + assertEquals(data[i], dictBuffer.readUnsignedByte()); } } public void testGetBufferWithByteBuffer() { runTestGetBuffer("testGetBufferWithByteBuffer", - new FusionDictionaryBufferFromByteBufferFactory()); + new DictionaryBufferFromReadOnlyByteBufferFactory()); } public void testGetBufferWithByteArray() { runTestGetBuffer("testGetBufferWithByteArray", - new FusionDictionaryBufferFromByteArrayFactory()); + new DictionaryBufferFromByteArrayFactory()); } public void testGetBufferWithWritableByteBuffer() { runTestGetBuffer("testGetBufferWithWritableByteBuffer", - new FusionDictionaryBufferFromWritableByteBufferFactory()); + new DictionaryBufferFromWritableByteBufferFactory()); } } diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index b3e2ee0ff..d605cdb84 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -22,6 +22,7 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; @@ -29,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.Set; +import java.util.concurrent.TimeUnit; /** * Unit tests for UserHistoryDictionary @@ -43,6 +45,9 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" }; + private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000; + private static final int WAIT_TERMINATING_IN_MILLISECONDS = 100; + @Override public void setUp() { mPrefs = PreferenceManager.getDefaultSharedPreferences(getContext()); @@ -73,48 +78,74 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private void addToDict(final UserHistoryPredictionDictionary dict, final List<String> words) { String prevWord = null; for (String word : words) { - dict.forceAddWordForTest(prevWord, word, true); + dict.addToPersonalizationPredictionDictionary(prevWord, word, true); prevWord = word; } } - public void testRandomWords() { - File dictFile = null; - try { - Log.d(TAG, "This test can be used for profiling."); - Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); - final int numberOfWords = 1000; - final Random random = new Random(123456); - List<String> words = generateWords(numberOfWords, random); - - final String locale = "testRandomWords"; - final String fileName = "UserHistoryDictionary." + locale + ".dict"; - dictFile = new File(getContext().getFilesDir(), fileName); - final UserHistoryPredictionDictionary dict = - PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary( - getContext(), locale, mPrefs); - dict.mIsTest = true; - - addToDict(dict, words); - + /** + * @param checksContents if true, checks whether written words are actually in the dictionary + * or not. + */ + private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords, + final Random random, final boolean checksContents) { + final List<String> words = generateWords(numberOfWords, random); + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix /* locale */, mPrefs); + // Add random words to the user history dictionary. + addToDict(dict, words); + if (checksContents) { try { - Log.d(TAG, "waiting for adding the word ..."); - Thread.sleep(2000); + Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: " + e); } + // Limit word count to check when using a Java on memory dictionary. + final int wordCountToCheck = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + numberOfWords : 10; + for (int i = 0; i < wordCountToCheck; ++i) { + final String word = words.get(i); + // This may fail as long as we use tryLock on inserting the bigram words + assertTrue(dict.isInDictionaryForTests(word)); + } + } + // write to file. + dict.close(); + } - // write to file - dict.close(); + public void testRandomWords() { + File dictFile = null; + Log.d(TAG, "This test can be used for profiling."); + Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); + final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final int numberOfWords = 1000; + final Random random = new Random(123456); + try { + addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, + true /* checksContents */); + } finally { try { + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); Log.d(TAG, "waiting for writing ..."); - Thread.sleep(5000); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } - } finally { + + final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + dictFile = new File(getContext().getFilesDir(), fileName); + if (dictFile != null) { + assertTrue(dictFile.exists()); + assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); dictFile.delete(); } } @@ -122,52 +153,94 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { public void testStressTestForSwitchingLanguagesAndAddingWords() { final int numberOfLanguages = 2; - final int numberOfLanguageSwitching = 100; - final int numberOfWordsIntertedForEachLanguageSwitch = 100; + final int numberOfLanguageSwitching = 80; + final int numberOfWordsInsertedForEachLanguageSwitch = 100; final File dictFiles[] = new File[numberOfLanguages]; + final String testFilenameSuffixes[] = new String[numberOfLanguages]; try { final Random random = new Random(123456); - // Create locales for this test. - String locales[] = new String[numberOfLanguages]; + // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { - locales[i] = "testSwitchingLanguages" + i; - final String fileName = "UserHistoryDictionary." + locales[i] + ".dict"; + testFilenameSuffixes[i] = "testSwitchingLanguages" + i; + final String fileName = UserHistoryPredictionDictionary.NAME + "." + + testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFiles[i] = new File(getContext().getFilesDir(), fileName); } - final long now = System.currentTimeMillis(); + final long start = System.currentTimeMillis(); for (int i = 0; i < numberOfLanguageSwitching; i++) { final int index = i % numberOfLanguages; - // Switch languages to locales[index]. - final UserHistoryPredictionDictionary dict = - PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary( - getContext(), locales[index], mPrefs); - final List<String> words = generateWords( - numberOfWordsIntertedForEachLanguageSwitch, random); - // Add random words to the user history dictionary. - addToDict(dict, words); - // write to file - dict.close(); + // Switch languages to testFilenameSuffixes[index]. + addAndWriteRandomWords(testFilenameSuffixes[index], + numberOfWordsInsertedForEachLanguageSwitch, random, + false /* checksContents */); } final long end = System.currentTimeMillis(); Log.d(TAG, "testStressTestForSwitchingLanguageAndAddingWords took " - + (end - now) + " ms"); + + (end - start) + " ms"); + } finally { try { Log.d(TAG, "waiting for writing ..."); - Thread.sleep(5000); + for (int i = 0; i < numberOfLanguages; i++) { + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffixes[i], mPrefs); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } - } finally { for (final File file : dictFiles) { if (file != null) { + assertTrue(file.exists()); + assertTrue(file.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); file.delete(); } } } } + + public void testAddManyWords() { + File dictFile = null; + final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final int numberOfWords = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + 10000 : 1000; + final Random random = new Random(123456); + + UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); + try { + addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, + true /* checksContents */); + dict.close(); + } finally { + try { + Log.d(TAG, "waiting for writing ..."); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } catch (InterruptedException e) { + Log.d(TAG, "InterruptedException: ", e); + } + final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + dictFile = new File(getContext().getFilesDir(), fileName); + if (dictFile != null) { + assertTrue(dictFile.exists()); + assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); + dictFile.delete(); + } + } + } + } diff --git a/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java b/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java new file mode 100644 index 000000000..7fd167977 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.MediumTest; +import android.util.Log; + +@MediumTest +public class AsyncResultHolderTests extends AndroidTestCase { + private static final String TAG = AsyncResultHolderTests.class.getSimpleName(); + + private static final int TIMEOUT_IN_MILLISECONDS = 500; + private static final int MARGIN_IN_MILLISECONDS = 250; + private static final int DEFAULT_VALUE = 2; + private static final int SET_VALUE = 1; + + private <T> void setAfterGivenTime(final AsyncResultHolder<T> holder, final T value, + final long time) { + new Thread(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(time); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping", e); + } + holder.set(value); + } + }).start(); + } + + public void testGetWithoutSet() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(DEFAULT_VALUE, resultValue); + } + + public void testGetBeforeSet() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + setAfterGivenTime(holder, SET_VALUE, TIMEOUT_IN_MILLISECONDS + MARGIN_IN_MILLISECONDS); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(DEFAULT_VALUE, resultValue); + } + + public void testGetAfterSet() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + holder.set(SET_VALUE); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(SET_VALUE, resultValue); + } + + public void testGetBeforeTimeout() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + setAfterGivenTime(holder, SET_VALUE, TIMEOUT_IN_MILLISECONDS - MARGIN_IN_MILLISECONDS); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(SET_VALUE, resultValue); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java b/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java new file mode 100644 index 000000000..e0755483c --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.MediumTest; +import android.util.Log; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Unit tests for PrioritizedSerialExecutor. + * TODO: Add more detailed tests to make use of priorities, etc. + */ +@MediumTest +public class PrioritizedSerialExecutorTests extends AndroidTestCase { + private static final String TAG = PrioritizedSerialExecutorTests.class.getSimpleName(); + + private static final int NUM_OF_TASKS = 10; + private static final int DELAY_FOR_WAITING_TASKS_MILLISECONDS = 500; + + public void testExecute() { + final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.execute(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + try { + Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(NUM_OF_TASKS, v.get()); + } + + public void testExecutePrioritized() { + final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.executePrioritized(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + try { + Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(NUM_OF_TASKS, v.get()); + } + + public void testExecuteCombined() { + final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.execute(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.executePrioritized(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + + try { + Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(2 * NUM_OF_TASKS, v.get()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java index cfff61ef8..cad80d5ce 100644 --- a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java @@ -340,4 +340,18 @@ public class ResizableIntArrayTests extends AndroidTestCase { expecteds[i + expectedPos], actuals[i + actualPos]); } } + + public void testShift() { + final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); + final int limit = DEFAULT_CAPACITY * 10; + final int shiftAmount = 20; + for (int i = 0; i < limit; ++i) { + src.add(i, i); + assertEquals("length after add at " + i, i + 1, src.getLength()); + } + src.shift(shiftAmount); + for (int i = 0; i < limit - shiftAmount; ++i) { + assertEquals("value at " + i, i + shiftAmount, src.get(i)); + } + } } diff --git a/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java new file mode 100644 index 000000000..fa6ad16c1 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; +import android.text.style.SuggestionSpan; +import android.text.style.URLSpan; +import android.text.SpannableStringBuilder; +import android.text.Spannable; +import android.text.Spanned; + +@SmallTest +public class SpannableStringUtilsTests extends AndroidTestCase { + public void testConcatWithSuggestionSpansOnly() { + SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n" + + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n" + + "test string\ntest string\n"); + final int N = 10; + for (int i = 0; i < N; ++i) { + // Put a PARAGRAPH-flagged span that should not be found in the result. + s.setSpan(new SuggestionSpan(getContext(), + new String[] {"" + i}, Spannable.SPAN_PARAGRAPH), + i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH); + // Put a normal suggestion span that should be found in the result. + s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0); + // Put a URL span than should not be found in the result. + s.setSpan(new URLSpan("http://a"), i, i * 2, 0); + } + + final CharSequence a = s.subSequence(0, 15); + final CharSequence b = s.subSequence(15, s.length()); + final Spanned result = + (Spanned)SpannableStringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b); + + Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class); + for (int i = 0; i < spans.length; i++) { + final int flags = result.getSpanFlags(spans[i]); + assertEquals("Should not find a span with PARAGRAPH flag", + flags & Spannable.SPAN_PARAGRAPH, 0); + assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java index 175e511b0..4e396a1cf 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java @@ -21,6 +21,8 @@ import com.android.inputmethod.latin.settings.SettingsValues; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import java.util.Arrays; +import java.util.List; import java.util.Locale; @SmallTest @@ -256,4 +258,26 @@ public class StringUtilsTests extends AndroidTestCase { // code for now True is acceptable. assertTrue(StringUtils.lastPartLooksLikeURL(".abc/def")); } + + public void testHexStringUtils() { + final byte[] bytes = new byte[] { (byte)0x01, (byte)0x11, (byte)0x22, (byte)0x33, + (byte)0x55, (byte)0x88, (byte)0xEE }; + final String bytesStr = StringUtils.byteArrayToHexString(bytes); + final byte[] bytes2 = StringUtils.hexStringToByteArray(bytesStr); + for (int i = 0; i < bytes.length; ++i) { + assertTrue(bytes[i] == bytes2[i]); + } + final String bytesStr2 = StringUtils.byteArrayToHexString(bytes2); + assertTrue(bytesStr.equals(bytesStr2)); + } + + public void testJsonStringUtils() { + final Object[] objs = new Object[] { 1, "aaa", "bbb", 3 }; + final List<Object> objArray = Arrays.asList(objs); + final String str = StringUtils.listToJsonStr(objArray); + final List<Object> newObjArray = StringUtils.jsonStrToList(str); + for (int i = 0; i < objs.length; ++i) { + assertEquals(objs[i], newObjArray.get(i)); + } + } } diff --git a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java index baebda2ed..856b2dbda 100644 --- a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java @@ -214,7 +214,7 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CA)); assertEquals("de ", "Allemand", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE)); - assertEquals("zz ", "Alphabet (QWERTY)", + assertEquals("zz ", "Alphabet latin (QWERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ)); return null; } @@ -236,7 +236,7 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_UK_DVORAK)); assertEquals("es_US colemak","Espagnol (États-Unis) (Colemak)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US_COLEMAK)); - assertEquals("zz pc", "Alphabet (PC)", + assertEquals("zz pc", "Alphabet latin (PC)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_PC)); return null; } diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java index ce62bf21a..3eabe2b3c 100644 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java @@ -21,17 +21,19 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictReader; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -49,6 +51,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase private static final int BIGRAM_FREQUENCY = 100; private static final ArrayList<String> NOT_HAVE_BIGRAM = new ArrayList<String>(); private static final FormatSpec.FormatOptions FORMAT_OPTIONS = new FormatSpec.FormatOptions(2); + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; /** * Return same frequency for all words and bigrams @@ -86,12 +89,12 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase private void checkWordInFusionDict(final FusionDictionary dict, final String word, final ArrayList<String> expectedBigrams) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word); - assertNotNull(group); - assertTrue(group.isTerminal()); + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); + assertNotNull(ptNode); + assertTrue(ptNode.isTerminal()); for (final String bigram : expectedBigrams) { - assertNotNull(group.getBigram(bigram)); + assertNotNull(ptNode.getBigram(bigram)); } } @@ -135,26 +138,20 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase private void writeDictToFile(final File file, final UserHistoryDictionaryBigramList bigramList) { - try { - final FileOutputStream out = new FileOutputStream(file); - UserHistoryDictIOUtils.writeDictionaryBinary(out, this, bigramList, FORMAT_OPTIONS); - out.flush(); - out.close(); - } catch (IOException e) { - Log.e(TAG, "IO exception while writing file", e); - } + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, bigramList, FORMAT_OPTIONS); } private void readDictFromFile(final File file, final OnAddWordListener listener) { - final BinaryDictReader reader = new BinaryDictReader(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { - reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + dictDecoder.openDictBuffer(); } catch (FileNotFoundException e) { Log.e(TAG, "file not found", e); } catch (IOException e) { Log.e(TAG, "IOException", e); } - UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); + UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); } public void testGenerateFusionDictionary() { @@ -177,7 +174,8 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase File file = null; try { - file = File.createTempFile("testReadAndWrite", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testReadAndWrite", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { Log.d(TAG, "IOException while creating a temporary file", e); } |