diff options
14 files changed, 362 insertions, 162 deletions
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java new file mode 100644 index 000000000..ebbcedc96 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.content.Context; +import android.util.Log; + +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +// TODO: Quit extending Dictionary after implementing dynamic binary dictionary. +abstract public class AbstractDictionaryWriter extends Dictionary { + /** Used for Log actions from this class */ + private static final String TAG = AbstractDictionaryWriter.class.getSimpleName(); + + private final Context mContext; + + public AbstractDictionaryWriter(final Context context, final String dictType) { + super(dictType); + mContext = context; + } + + abstract public void clear(); + + abstract public void addUnigramWord(final String word, final String shortcutTarget, + final int frequency, final boolean isNotAWord); + + abstract public void addBigramWords(final String word0, final String word1, + final int frequency, final boolean isValid); + + abstract public void removeBigramWords(final String word0, final String word1); + + abstract protected void writeBinaryDictionary(final FileOutputStream out) + throws IOException, UnsupportedFormatException; + + public void write(final String fileName) { + final String tempFileName = fileName + ".temp"; + final File file = new File(mContext.getFilesDir(), fileName); + final File tempFile = new File(mContext.getFilesDir(), tempFileName); + FileOutputStream out = null; + try { + out = new FileOutputStream(tempFile); + writeBinaryDictionary(out); + out.flush(); + out.close(); + tempFile.renameTo(file); + } catch (IOException e) { + Log.e(TAG, "IO exception while writing file", e); + } catch (UnsupportedFormatException e) { + Log.e(TAG, "Unsupported format", e); + } finally { + if (out != null) { + try { + out.close(); + } catch (IOException e) { + // ignore + } + } + } + } +} diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index 110be9db3..c99d0e2ea 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -109,7 +109,6 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { @Override public void loadDictionaryAsync() { - clearFusionDictionary(); loadDeviceAccountsEmailAddresses(); loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI); // TODO: Switch this URL to the newer ContactsContract too @@ -236,6 +235,11 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } @Override + protected boolean needsToReloadBeforeWriting() { + return true; + } + + @Override protected boolean hasContentChanged() { final long startTime = SystemClock.uptimeMillis(); final int contactCount = getContactCount(); diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java new file mode 100644 index 000000000..8be04c1c0 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.content.Context; + +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FusionDictionary; +import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.utils.CollectionUtils; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; + +/** + * An in memory dictionary for memorizing entries and writing a binary dictionary. + */ +public class DictionaryWriter extends AbstractDictionaryWriter { + // TODO: Regenerate version 3 binary dictionary. + private static final int BINARY_DICT_VERSION = 2; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(BINARY_DICT_VERSION); + + private FusionDictionary mFusionDictionary; + + public DictionaryWriter(final Context context, final String dictType) { + super(context, dictType); + clear(); + } + + @Override + public void clear() { + final HashMap<String, String> attributes = CollectionUtils.newHashMap(); + mFusionDictionary = new FusionDictionary(new Node(), + new FusionDictionary.DictionaryOptions(attributes, false, false)); + } + + /** + * Adds a word unigram to the fusion dictionary. + */ + // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries, + // considering performance regression. + @Override + public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, + final boolean isNotAWord) { + if (shortcutTarget == null) { + mFusionDictionary.add(word, frequency, null, isNotAWord); + } else { + // TODO: Do this in the subclass, with this class taking an arraylist. + final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList(); + shortcutTargets.add(new WeightedString(shortcutTarget, frequency)); + mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord); + } + } + + @Override + public void addBigramWords(final String word0, final String word1, final int frequency, + final boolean isValid) { + mFusionDictionary.setBigram(word0, word1, frequency); + } + + @Override + public void removeBigramWords(final String word0, final String word1) { + // This class don't support removing bigram words. + } + + @Override + protected void writeBinaryDictionary(final FileOutputStream out) + throws IOException, UnsupportedFormatException { + BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + boolean blockOffensiveWords) { + // This class doesn't support suggestion. + return null; + } + + @Override + public boolean isValidWord(String word) { + // This class doesn't support dictionary retrieval. + return false; + } +} diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index f4c4c6533..a563c7686 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -22,17 +22,9 @@ import android.util.Log; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -76,8 +68,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private BinaryDictionary mBinaryDictionary; - /** The expandable fusion dictionary used to generate the binary dictionary. */ - private FusionDictionary mFusionDictionary; + /** The in-memory dictionary used to generate the binary dictionary. */ + private AbstractDictionaryWriter mDictionaryWriter; /** * The name of this dictionary, used as the filename for storing the binary dictionary. Multiple @@ -92,11 +84,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Controls access to the local binary dictionary for this instance. */ private final DictionaryController mLocalDictionaryController = new DictionaryController(); - // TODO: Regenerate version 3 binary dictionary. - private static final int BINARY_DICT_VERSION = 2; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(BINARY_DICT_VERSION); - /** * Abstract method for loading the unigrams and bigrams of a given dictionary in a background * thread. @@ -138,7 +125,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mContext = context; mBinaryDictionary = null; mSharedDictionaryController = getSharedDictionaryController(filename); - clearFusionDictionary(); + mDictionaryWriter = new DictionaryWriter(context, dictType); } protected static String getFilenameWithLocale(final String name, final String localeStr) { @@ -157,47 +144,51 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary.close(); mBinaryDictionary = null; } + mDictionaryWriter.close(); } finally { mLocalDictionaryController.writeLock().unlock(); } } /** - * Clears the fusion dictionary on the Java side. Note: Does not modify the binary dictionary on - * the native side. + * Adds a word unigram to the dictionary. Used for loading a dictionary. */ - public void clearFusionDictionary() { - final HashMap<String, String> attributes = CollectionUtils.newHashMap(); - mFusionDictionary = new FusionDictionary(new Node(), - new FusionDictionary.DictionaryOptions(attributes, false, false)); + protected void addWord(final String word, final String shortcutTarget, + final int frequency, final boolean isNotAWord) { + mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); } /** - * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes - * are done to update the binary dictionary. + * Sets a word bigram in the dictionary. Used for loading a dictionary. */ - // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries, - // considering performance regression. - protected void addWord(final String word, final String shortcutTarget, final int frequency, - final boolean isNotAWord) { - if (shortcutTarget == null) { - mFusionDictionary.add(word, frequency, null, isNotAWord); - } else { - // TODO: Do this in the subclass, with this class taking an arraylist. - final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList(); - shortcutTargets.add(new WeightedString(shortcutTarget, frequency)); - mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord); + protected void setBigram(final String prevWord, final String word, final int frequency) { + mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */); + } + + /** + * Dynamically adds a word unigram to the dictionary. + */ + protected void addWordDynamically(final String word, final String shortcutTarget, + final int frequency, final boolean isNotAWord) { + mLocalDictionaryController.writeLock().lock(); + try { + mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); + } finally { + mLocalDictionaryController.writeLock().unlock(); } } /** - * Sets a word bigram in the fusion dictionary. Call updateBinaryDictionary when all changes are - * done to update the binary dictionary. + * Dynamically sets a word bigram in the dictionary. */ - // TODO: Create "cache dictionary" to cache fresh bigrams for frequently updated dictionaries, - // considering performance regression. - protected void setBigram(final String prevWord, final String word, final int frequency) { - mFusionDictionary.setBigram(prevWord, word, frequency); + protected void setBigramDynamically(final String prevWord, final String word, + final int frequency) { + mLocalDictionaryController.writeLock().lock(); + try { + mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } } @Override @@ -207,9 +198,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { asyncReloadDictionaryIfRequired(); if (mLocalDictionaryController.readLock().tryLock()) { try { + final ArrayList<SuggestedWordInfo> inMemDictSuggestion = + mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo, + blockOffensiveWords); if (mBinaryDictionary != null) { - return mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords); + final ArrayList<SuggestedWordInfo> binarySuggestion = + mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, + blockOffensiveWords); + if (inMemDictSuggestion == null) { + return binarySuggestion; + } else if (binarySuggestion == null) { + return inMemDictSuggestion; + } else { + binarySuggestion.addAll(binarySuggestion); + return binarySuggestion; + } + } else { + return inMemDictSuggestion; } } finally { mLocalDictionaryController.readLock().unlock(); @@ -240,22 +245,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return mBinaryDictionary.isValidWord(word); } - protected boolean isValidBigram(final String word1, final String word2) { - if (mBinaryDictionary == null) return false; - return mBinaryDictionary.isValidBigram(word1, word2); - } - - protected boolean isValidBigramInner(final String word1, final String word2) { - if (mLocalDictionaryController.readLock().tryLock()) { - try { - return isValidBigramLocked(word1, word2); - } finally { - mLocalDictionaryController.readLock().unlock(); - } - } - return false; - } - protected boolean isValidBigramLocked(final String word1, final String word2) { if (mBinaryDictionary == null) return false; return mBinaryDictionary.isValidBigram(word1, word2); @@ -274,7 +263,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Loads the current binary dictionary from internal storage. Assumes the dictionary file * exists. */ - protected void loadBinaryDictionary() { + private void loadBinaryDictionary() { if (DEBUG) { Log.d(TAG, "Loading binary dictionary: " + mFilename + " request=" + mSharedDictionaryController.mLastUpdateRequestTime + " update=" @@ -306,6 +295,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** + * Abstract method for checking if it is required to reload the dictionary before writing + * a binary dictionary. + */ + abstract protected boolean needsToReloadBeforeWriting(); + + /** * Generates and writes a new binary dictionary based on the contents of the fusion dictionary. */ private void generateBinaryDictionary() { @@ -314,33 +309,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { + mSharedDictionaryController.mLastUpdateRequestTime + " update=" + mSharedDictionaryController.mLastUpdateTime); } - - loadDictionaryAsync(); - - final String tempFileName = mFilename + ".temp"; - final File file = new File(mContext.getFilesDir(), mFilename); - final File tempFile = new File(mContext.getFilesDir(), tempFileName); - FileOutputStream out = null; - try { - out = new FileOutputStream(tempFile); - BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); - out.flush(); - out.close(); - tempFile.renameTo(file); - clearFusionDictionary(); - } catch (IOException e) { - Log.e(TAG, "IO exception while writing file", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Unsupported format", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // ignore - } - } + if (needsToReloadBeforeWriting()) { + mDictionaryWriter.clear(); + loadDictionaryAsync(); } + mDictionaryWriter.write(mFilename); } /** diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index ba84c1ad3..ab8f34893 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -240,7 +240,6 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { private void addWords(final Cursor cursor) { final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; - clearFusionDictionary(); if (cursor == null) return; if (cursor.moveToFirst()) { final int indexWord = cursor.getColumnIndex(Words.WORD); @@ -267,4 +266,9 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { protected boolean hasContentChanged() { return true; } + + @Override + protected boolean needsToReloadBeforeWriting() { + return true; + } } diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 2b8dbbcaf..6e1b80ee0 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -201,7 +201,7 @@ static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, j static jint latinime_BinaryDictionary_getProbability(JNIEnv *env, jclass clazz, jlong dict, jintArray word) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); - if (!dictionary) return 0; + if (!dictionary) return NOT_A_PROBABILITY; const jsize wordLength = env->GetArrayLength(word); int codePoints[wordLength]; env->GetIntArrayRegion(word, 0, wordLength, codePoints); diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp index f48386bba..5d14a0554 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp @@ -31,9 +31,9 @@ const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4; // The versions of Latin IME that only handle format version 1 only test for the magic // number, so we had to change it so that version 2 files would be rejected by older // implementations. On this occasion, we made the magic number 32 bits long. -const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; +const uint32_t BinaryDictionaryFormatUtils::HEADER_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; // Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12 -const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12; +const int BinaryDictionaryFormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12; /* static */ BinaryDictionaryFormatUtils::FORMAT_VERSION BinaryDictionaryFormatUtils::detectFormatVersion(const uint8_t *const dict, @@ -46,25 +46,28 @@ const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12; } const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0); switch (magicNumber) { - case FORMAT_VERSION_2_MAGIC_NUMBER: - // Version 2 dictionaries are at least 12 bytes long. - // If this dictionary has the version 2 magic number but is less than 12 bytes long, - // then it's an unknown format and we need to avoid confidently reading the next bytes. - if (dictSize < FORMAT_VERSION_2_MINIMUM_SIZE) { + case HEADER_VERSION_2_MAGIC_NUMBER: + // Version 2 header are at least 12 bytes long. + // If this header has the version 2 magic number but is less than 12 bytes long, + // then it's an unknown format and we need to avoid confidently reading the next bytes. + if (dictSize < HEADER_VERSION_2_MINIMUM_SIZE) { + return UNKNOWN_VERSION; + } + // Version 2 header is as follows: + // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE + // Version number (2 bytes) + // Options (2 bytes) + // Header size (4 bytes) : integer, big endian + if (ByteArrayUtils::readUint16(dict, 4) == 2) { + return VERSION_2; + } else if (ByteArrayUtils::readUint16(dict, 4) == 3) { + // TODO: Support version 3 dictionary. + return UNKNOWN_VERSION; + } else { + return UNKNOWN_VERSION; + } + default: return UNKNOWN_VERSION; - } - // Format 2 header is as follows: - // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE - // Version number (2 bytes) 0x00 0x02 - // Options (2 bytes) - // Header size (4 bytes) : integer, big endian - if (ByteArrayUtils::readUint16(dict, 4) == 2) { - return VERSION_2; - } else { - return UNKNOWN_VERSION; - } - default: - return UNKNOWN_VERSION; } } diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h index 80067b255..830684c70 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h @@ -33,9 +33,9 @@ namespace latinime { */ class BinaryDictionaryFormatUtils { public: - // TODO: Support version 3 format. enum FORMAT_VERSION { - VERSION_2 = 1, + VERSION_2, + VERSION_3, UNKNOWN_VERSION }; @@ -45,8 +45,8 @@ class BinaryDictionaryFormatUtils { DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils); static const int DICTIONARY_MINIMUM_SIZE; - static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER; - static const int FORMAT_VERSION_2_MINIMUM_SIZE; + static const uint32_t HEADER_VERSION_2_MAGIC_NUMBER; + static const int HEADER_VERSION_2_MINIMUM_SIZE; }; } // namespace latinime #endif /* LATINIME_BINARY_DICTIONARY_FORMAT_UTILS_H */ diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp index c4c4bedde..a57b0f859 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp @@ -26,10 +26,10 @@ namespace latinime { const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256; -const int BinaryDictionaryHeaderReadingUtils::VERSION_2_MAGIC_NUMBER_SIZE = 4; -const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_VERSION_SIZE = 2; -const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_FLAG_SIZE = 2; -const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_HEADER_SIZE_SIZE = 4; +const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_MAGIC_NUMBER_SIZE = 4; +const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_DICTIONARY_VERSION_SIZE = 2; +const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_FLAG_SIZE = 2; +const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_SIZE_FIELD_SIZE = 4; const BinaryDictionaryHeaderReadingUtils::DictionaryFlags BinaryDictionaryHeaderReadingUtils::NO_FLAGS = 0; @@ -45,13 +45,13 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags /* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize( const BinaryDictionaryInfo *const binaryDictionaryInfo) { - switch (binaryDictionaryInfo->getFormat()) { - case BinaryDictionaryFormatUtils::VERSION_2: + switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) { + case HEADER_VERSION_2: // See the format of the header in the comment in // BinaryDictionaryFormatUtils::detectFormatVersion() return ByteArrayUtils::readUint32(binaryDictionaryInfo->getDictBuf(), - VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE - + VERSION_2_DICTIONARY_FLAG_SIZE); + VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE + + VERSION_2_HEADER_FLAG_SIZE); default: return S_INT_MAX; } @@ -60,10 +60,10 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags /* static */ BinaryDictionaryHeaderReadingUtils::DictionaryFlags BinaryDictionaryHeaderReadingUtils::getFlags( const BinaryDictionaryInfo *const binaryDictionaryInfo) { - switch (binaryDictionaryInfo->getFormat()) { - case BinaryDictionaryFormatUtils::VERSION_2: + switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) { + case HEADER_VERSION_2: return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(), - VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE); + VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE); default: return NO_FLAGS; } @@ -73,11 +73,15 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags /* static */ bool BinaryDictionaryHeaderReadingUtils::readHeaderValue( const BinaryDictionaryInfo *const binaryDictionaryInfo, const char *const key, int *outValue, const int outValueSize) { - if (outValueSize <= 0 || !hasHeaderAttributes(binaryDictionaryInfo->getFormat())) { + if (outValueSize <= 0) { return false; } const int headerSize = getHeaderSize(binaryDictionaryInfo); int pos = getHeaderOptionsPosition(binaryDictionaryInfo->getFormat()); + if (pos == NOT_A_DICT_POS) { + // The header doesn't have header options. + return false; + } while (pos < headerSize) { if(ByteArrayUtils::compareStringInBufferWithCharArray( binaryDictionaryInfo->getDictBuf(), key, headerSize - pos, &pos) == 0) { diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h index 94b9e124d..61748227e 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h @@ -48,27 +48,15 @@ class BinaryDictionaryHeaderReadingUtils { return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0; } - static AK_FORCE_INLINE bool hasHeaderAttributes( - const BinaryDictionaryFormatUtils::FORMAT_VERSION format) { - // Only format 2 and above have header attributes as {key,value} string pairs. - switch (format) { - case BinaryDictionaryFormatUtils::VERSION_2: - return true; - break; - default: - return false; - } - } - static AK_FORCE_INLINE int getHeaderOptionsPosition( - const BinaryDictionaryFormatUtils::FORMAT_VERSION format) { - switch (format) { - case BinaryDictionaryFormatUtils::VERSION_2: - return VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE - + VERSION_2_DICTIONARY_FLAG_SIZE + VERSION_2_DICTIONARY_HEADER_SIZE_SIZE; + const BinaryDictionaryFormatUtils::FORMAT_VERSION dictionaryFormat) { + switch (getHeaderVersion(dictionaryFormat)) { + case HEADER_VERSION_2: + return VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE + + VERSION_2_HEADER_FLAG_SIZE + VERSION_2_HEADER_SIZE_FIELD_SIZE; break; default: - return 0; + return NOT_A_DICT_POS; } } @@ -82,10 +70,15 @@ class BinaryDictionaryHeaderReadingUtils { private: DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils); - static const int VERSION_2_MAGIC_NUMBER_SIZE; - static const int VERSION_2_DICTIONARY_VERSION_SIZE; - static const int VERSION_2_DICTIONARY_FLAG_SIZE; - static const int VERSION_2_DICTIONARY_HEADER_SIZE_SIZE; + enum HEADER_VERSION { + HEADER_VERSION_2, + UNKNOWN_HEADER_VERSION + }; + + static const int VERSION_2_HEADER_MAGIC_NUMBER_SIZE; + static const int VERSION_2_HEADER_DICTIONARY_VERSION_SIZE; + static const int VERSION_2_HEADER_FLAG_SIZE; + static const int VERSION_2_HEADER_SIZE_FIELD_SIZE; static const DictionaryFlags NO_FLAGS; // Flags for special processing @@ -95,6 +88,18 @@ class BinaryDictionaryHeaderReadingUtils { static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG; static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG; static const DictionaryFlags CONTAINS_BIGRAMS_FLAG; + + static HEADER_VERSION getHeaderVersion( + const BinaryDictionaryFormatUtils::FORMAT_VERSION formatVersion) { + switch(formatVersion) { + case BinaryDictionaryFormatUtils::VERSION_2: + // Fall through + case BinaryDictionaryFormatUtils::VERSION_3: + return HEADER_VERSION_2; + default: + return UNKNOWN_HEADER_VERSION; + } + } }; } #endif /* LATINIME_DICTIONARY_HEADER_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h index c0e24fa4e..70dad67e8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h @@ -32,6 +32,9 @@ class DictionaryStructurePolicyFactory { switch (dictionaryFormat) { case BinaryDictionaryFormatUtils::VERSION_2: return PatriciaTriePolicy::getInstance(); + case BinaryDictionaryFormatUtils::VERSION_3: + // TODO: support version 3 dictionaries. + return 0; default: ASSERT(false); return 0; diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java index cca81a0e0..87acafee6 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java @@ -51,7 +51,7 @@ import java.util.Set; @LargeTest public class BinaryDictIOTests extends AndroidTestCase { private static final String TAG = BinaryDictIOTests.class.getSimpleName(); - private static final int MAX_UNIGRAMS = 100; + private static final int DEFAULT_MAX_UNIGRAMS = 100; private static final int UNIGRAM_FREQ = 10; private static final int BIGRAM_FREQ = 50; private static final int TOLERANCE_OF_BIGRAM_FREQ = 5; @@ -73,13 +73,15 @@ public class BinaryDictIOTests extends AndroidTestCase { new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); public BinaryDictIOTests() { - super(); + this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); + } - final long time = System.currentTimeMillis(); - Log.e(TAG, "Testing dictionary: seed is " + time); - final Random random = new Random(time); + public BinaryDictIOTests(final long seed, final int maxUnigrams) { + super(); + Log.e(TAG, "Testing dictionary: seed is " + seed); + final Random random = new Random(seed); sWords.clear(); - generateWords(MAX_UNIGRAMS, random); + generateWords(maxUnigrams, random); for (int i = 0; i < sWords.size(); ++i) { sChainBigrams.put(i, new ArrayList<Integer>()); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java index 7b311c3ec..cacee5268 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java @@ -72,15 +72,21 @@ public class Dicttool { return command; } - private void execute(final String[] arguments) { + /** + * Executes the specified command with the specified arguments. + * @param arguments the arguments passed to dicttool. + * @return 0 for success, an error code otherwise (always 1 at the moment) + */ + private int execute(final String[] arguments) { final Command command = getCommand(arguments); try { command.run(); + return 0; } catch (Exception e) { System.out.println("Exception while processing command " + command.getClass().getSimpleName() + " : " + e); e.printStackTrace(); - return; + return 1; } } @@ -89,6 +95,7 @@ public class Dicttool { help(); return; } - new Dicttool().execute(arguments); + // Exit with the success/error code from #execute() as status. + System.exit(new Dicttool().execute(arguments)); } } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java index df5ea35bb..972b6e7e6 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java @@ -16,10 +16,12 @@ package com.android.inputmethod.latin.dicttool; +import com.android.inputmethod.latin.makedict.BinaryDictIOTests; import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests; import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest; import com.android.inputmethod.latin.makedict.FusionDictionaryTest; +import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -36,7 +38,8 @@ public class Test extends Dicttool.Command { BinaryDictOffdeviceUtilsTests.class, FusionDictionaryTest.class, BinaryDictInputOutputTest.class, - BinaryDictIOUtilsTests.class + BinaryDictIOUtilsTests.class, + BinaryDictIOTests.class }; private ArrayList<Method> mAllTestMethods = new ArrayList<Method>(); private ArrayList<String> mUsedTestMethods = new ArrayList<String>(); @@ -86,12 +89,19 @@ public class Test extends Dicttool.Command { for (final Method m : mAllTestMethods) { final Class<?> declaringClass = m.getDeclaringClass(); if (!mUsedTestMethods.isEmpty() && !mUsedTestMethods.contains(m.getName())) continue; - final Object instance; - if (BinaryDictIOUtilsTests.class == declaringClass) { - instance = new BinaryDictIOUtilsTests(mSeed, mMaxUnigrams); - } else { - instance = declaringClass.newInstance(); + // Some of the test classes expose a two-argument constructor, taking a long as a + // seed for Random, and an int for a vocabulary size to test the dictionary with. They + // correspond respectively to the -s and -m numerical arguments to this command, which + // are stored in mSeed and mMaxUnigrams. If the two-arguments constructor is present, + // then invoke it; otherwise, invoke the default constructor. + Constructor<?> twoArgsConstructor = null; + try { + twoArgsConstructor = declaringClass.getDeclaredConstructor(Long.TYPE, Integer.TYPE); + } catch (NoSuchMethodException e) { + // No constructor with two args } + final Object instance = null == twoArgsConstructor ? declaringClass.newInstance() + : twoArgsConstructor.newInstance(mSeed, mMaxUnigrams); m.invoke(instance); } } |