diff options
Diffstat (limited to 'java/src/org/kelar/inputmethod/latin/makedict')
7 files changed, 819 insertions, 0 deletions
diff --git a/java/src/org/kelar/inputmethod/latin/makedict/DictionaryHeader.java b/java/src/org/kelar/inputmethod/latin/makedict/DictionaryHeader.java new file mode 100644 index 000000000..6d771af61 --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/DictionaryHeader.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +import org.kelar.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; +import org.kelar.inputmethod.latin.makedict.FormatSpec.FormatOptions; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Class representing dictionary header. + */ +public final class DictionaryHeader { + public final int mBodyOffset; + @Nonnull + public final DictionaryOptions mDictionaryOptions; + @Nonnull + public final FormatOptions mFormatOptions; + @Nonnull + public final String mLocaleString; + @Nonnull + public final String mVersionString; + @Nonnull + public final String mIdString; + + // Note that these are corresponding definitions in native code in latinime::HeaderPolicy + // and latinime::HeaderReadWriteUtils. + // TODO: Standardize the key names and bump up the format version, taking care not to + // break format version 2 dictionaries. + public static final String DICTIONARY_VERSION_KEY = "version"; + public static final String DICTIONARY_LOCALE_KEY = "locale"; + public static final String DICTIONARY_ID_KEY = "dictionary"; + public static final String DICTIONARY_DESCRIPTION_KEY = "description"; + public static final String DICTIONARY_DATE_KEY = "date"; + public static final String HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO"; + public static final String USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE"; + public static final String FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY = + "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID"; + public static final String MAX_UNIGRAM_COUNT_KEY = "MAX_UNIGRAM_ENTRY_COUNT"; + public static final String MAX_BIGRAM_COUNT_KEY = "MAX_BIGRAM_ENTRY_COUNT"; + public static final String MAX_TRIGRAM_COUNT_KEY = "MAX_TRIGRAM_ENTRY_COUNT"; + public static final String ATTRIBUTE_VALUE_TRUE = "1"; + public static final String CODE_POINT_TABLE_KEY = "codePointTable"; + + public DictionaryHeader(final int headerSize, + @Nonnull final DictionaryOptions dictionaryOptions, + @Nonnull final FormatOptions formatOptions) throws UnsupportedFormatException { + mDictionaryOptions = dictionaryOptions; + mFormatOptions = formatOptions; + mBodyOffset = formatOptions.mVersion < FormatSpec.VERSION4 ? headerSize : 0; + final String localeString = dictionaryOptions.mAttributes.get(DICTIONARY_LOCALE_KEY); + if (null == localeString) { + throw new UnsupportedFormatException("Cannot create a FileHeader without a locale"); + } + final String versionString = dictionaryOptions.mAttributes.get(DICTIONARY_VERSION_KEY); + if (null == versionString) { + throw new UnsupportedFormatException( + "Cannot create a FileHeader without a version"); + } + final String idString = dictionaryOptions.mAttributes.get(DICTIONARY_ID_KEY); + if (null == idString) { + throw new UnsupportedFormatException("Cannot create a FileHeader without an ID"); + } + mLocaleString = localeString; + mVersionString = versionString; + mIdString = idString; + } + + // Helper method to get the description + @Nullable + public String getDescription() { + // TODO: Right now each dictionary file comes with a description in its own language. + // It will display as is no matter the device's locale. It should be internationalized. + return mDictionaryOptions.mAttributes.get(DICTIONARY_DESCRIPTION_KEY); + } +}
\ No newline at end of file diff --git a/java/src/org/kelar/inputmethod/latin/makedict/FormatSpec.java b/java/src/org/kelar/inputmethod/latin/makedict/FormatSpec.java new file mode 100644 index 000000000..35ed0c7ec --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/FormatSpec.java @@ -0,0 +1,310 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +import org.kelar.inputmethod.annotations.UsedForTesting; +import org.kelar.inputmethod.latin.define.DecoderSpecificConstants; + +import java.util.Date; +import java.util.HashMap; + +/** + * Dictionary File Format Specification. + */ +public final class FormatSpec { + + /* + * File header layout is as follows: + * + * v | + * e | MAGIC_NUMBER + version of the file format, 2 bytes. + * r | + * sion + * + * o | + * p | not used, 2 bytes. + * o | + * nflags + * + * h | + * e | size of the file header, 4bytes + * a | including the size of the magic number, the option flags and the header size + * d | + * ersize + * + * attributes list + * + * attributes list is: + * <key> = | string of characters at the char format described below, with the terminator used + * | to signal the end of the string. + * <value> = | string of characters at the char format described below, with the terminator used + * | to signal the end of the string. + * if the size of already read < headersize, goto key. + * + */ + + /* + * Node array (FusionDictionary.PtNodeArray) layout is as follows: + * + * n | + * o | the number of PtNodes, 1 or 2 bytes. + * d | 1 byte = bbbbbbbb match + * e | case 1xxxxxxx => xxxxxxx << 8 + next byte + * c | otherwise => bbbbbbbb + * o | + * unt + * + * n | + * o | sequence of PtNodes, + * d | the layout of each PtNode is described below. + * e | + * s + * + * f | + * o | forward link address, 3byte + * r | 1 byte = bbbbbbbb match + * w | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) + * a | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte + * r | + * dlinkaddress + */ + + /* Node (FusionDictionary.PtNode) layout is as follows: + * | CHILDREN_ADDRESS_TYPE 2 bits, 11 : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 10 : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES + * f | 01 : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE + * l | 00 : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS + * a | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS + * g | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL + * s | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS + * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS + * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD + * | is possibly offensive ? 1 bit, 1 = yes, 0 = no : FLAG_IS_POSSIBLY_OFFENSIVE + * + * c | IF FLAG_HAS_MULTIPLE_CHARS + * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers + * a | end 1 byte, = 0 + * r | ELSE + * s | char 1 or 3 bytes + * | END + * + * f | + * r | IF FLAG_IS_TERMINAL + * e | frequency 1 byte + * q | + * + * c | + * h | children address, CHILDREN_ADDRESS_TYPE bytes + * i | This address is relative to the position of this field. + * l | + * drenaddress + * + * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS + * | shortcut string list + * | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS + * | bigrams address list + * + * Char format is: + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + * + * bigram address list is: + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT + * | addressSign = 1 bit, : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE + * | 1 = must take -address, 0 = must take +address + * | xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE + * | addressFormat = 2 bits, 00 = unused : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 01 = 1 byte : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES + * | 11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY + * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat) + * | read 1 byte, add top 4 bits + * | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat) + * | read 2 bytes, add top 4 bits + * | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat + * | read 3 bytes, add top 4 bits + * | END + * | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is + * + * shortcut string list is: + * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT + * | reserved = 3 bits, must be 0 + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY + * <shortcut> = | string of characters at the char format described above, with the terminator + * | used to signal the end of the string. + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags + */ + + public static final int MAGIC_NUMBER = 0x9BC13AFE; + static final int NOT_A_VERSION_NUMBER = -1; + + // These MUST have the same values as the relevant constants in format_utils.h. + // From version 2.01 on, we use version * 100 + revision as a version number. That allows + // us to change the format during development while having testing devices remove + // older files with each upgrade, while still having a readable versioning scheme. + // When we bump up the dictionary format version, we should update + // ExpandableDictionary.needsToMigrateDictionary() and + // ExpandableDictionary.matchesExpectedBinaryDictFormatVersionForThisType(). + public static final int VERSION2 = 2; + public static final int VERSION201 = 201; + public static final int VERSION202 = 202; + // format version for Fava Dictionaries. + public static final int VERSION_DELIGHT3 = 86736212; + public static final int MINIMUM_SUPPORTED_VERSION_OF_CODE_POINT_TABLE = VERSION201; + // Dictionary version used for testing. + public static final int VERSION4_ONLY_FOR_TESTING = 399; + public static final int VERSION402 = 402; + public static final int VERSION403 = 403; + public static final int VERSION4 = VERSION403; + public static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202; + public static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION_DELIGHT3; + static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4; + static final int MAXIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION403; + + // TODO: Make this value adaptative to content data, store it in the header, and + // use it in the reading code. + static final int MAX_WORD_LENGTH = DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH; + + // These flags are used only in the static dictionary. + static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0; + static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00; + static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40; + static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80; + static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0; + + static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; + + static final int FLAG_IS_TERMINAL = 0x10; + static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08; + static final int FLAG_HAS_BIGRAMS = 0x04; + static final int FLAG_IS_NOT_A_WORD = 0x02; + static final int FLAG_IS_POSSIBLY_OFFENSIVE = 0x01; + + static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80; + static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40; + static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F; + + static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F; + + static final int PTNODE_TERMINATOR_SIZE = 1; + static final int PTNODE_FLAGS_SIZE = 1; + static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; + static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; + + static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; + static final int INVALID_CHARACTER = -1; + + static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 + // Large PtNode array size field size is 2 bytes. + static final int LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; + static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 + static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; + static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF; + + static final int MAX_TERMINAL_FREQUENCY = 255; + static final int MAX_BIGRAM_FREQUENCY = 15; + + public static final int SHORTCUT_WHITELIST_FREQUENCY = 15; + + // This option needs to be the same numeric value as the one in binary_format.h. + static final int NOT_VALID_WORD = -99; + + static final int UINT8_MAX = 0xFF; + static final int UINT16_MAX = 0xFFFF; + static final int UINT24_MAX = 0xFFFFFF; + static final int MSB8 = 0x80; + static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; + static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; + + /** + * Options about file format. + */ + public static final class FormatOptions { + public final int mVersion; + public final boolean mHasTimestamp; + + @UsedForTesting + public FormatOptions(final int version) { + this(version, false /* hasTimestamp */); + } + + public FormatOptions(final int version, final boolean hasTimestamp) { + mVersion = version; + mHasTimestamp = hasTimestamp; + } + } + + /** + * Options global to the dictionary. + */ + public static final class DictionaryOptions { + public final HashMap<String, String> mAttributes; + public DictionaryOptions(final HashMap<String, String> attributes) { + mAttributes = attributes; + } + @Override + public String toString() { // Convenience method + return toString(0, false); + } + public String toString(final int indentCount, final boolean plumbing) { + final StringBuilder indent = new StringBuilder(); + if (plumbing) { + indent.append("H:"); + } else { + for (int i = 0; i < indentCount; ++i) { + indent.append(" "); + } + } + final StringBuilder s = new StringBuilder(); + for (final String optionKey : mAttributes.keySet()) { + s.append(indent); + s.append(optionKey); + s.append(" = "); + if ("date".equals(optionKey) && !plumbing) { + // Date needs a number of milliseconds, but the dictionary contains seconds + s.append(new Date( + 1000 * Long.parseLong(mAttributes.get(optionKey))).toString()); + } else { + s.append(mAttributes.get(optionKey)); + } + s.append("\n"); + } + return s.toString(); + } + } + + private FormatSpec() { + // This utility class is not publicly instantiable. + } +} diff --git a/java/src/org/kelar/inputmethod/latin/makedict/NgramProperty.java b/java/src/org/kelar/inputmethod/latin/makedict/NgramProperty.java new file mode 100644 index 000000000..a9a762553 --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/NgramProperty.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +import org.kelar.inputmethod.latin.NgramContext; + +public class NgramProperty { + public final WeightedString mTargetWord; + public final NgramContext mNgramContext; + + public NgramProperty(final WeightedString targetWord, final NgramContext ngramContext) { + mTargetWord = targetWord; + mNgramContext = ngramContext; + } + + @Override + public int hashCode() { + return mTargetWord.hashCode() ^ mNgramContext.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof NgramProperty)) return false; + final NgramProperty n = (NgramProperty)o; + return mTargetWord.equals(n.mTargetWord) && mNgramContext.equals(n.mNgramContext); + } +} diff --git a/java/src/org/kelar/inputmethod/latin/makedict/ProbabilityInfo.java b/java/src/org/kelar/inputmethod/latin/makedict/ProbabilityInfo.java new file mode 100644 index 000000000..bd397191a --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/ProbabilityInfo.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +import org.kelar.inputmethod.annotations.UsedForTesting; +import org.kelar.inputmethod.latin.BinaryDictionary; +import org.kelar.inputmethod.latin.utils.CombinedFormatUtils; + +import java.util.Arrays; + +public final class ProbabilityInfo { + public final int mProbability; + // mTimestamp, mLevel and mCount are historical info. These values are depend on the + // implementation in native code; thus, we must not use them and have any assumptions about + // them except for tests. + public final int mTimestamp; + public final int mLevel; + public final int mCount; + + @UsedForTesting + public static ProbabilityInfo max(final ProbabilityInfo probabilityInfo1, + final ProbabilityInfo probabilityInfo2) { + if (probabilityInfo1 == null) { + return probabilityInfo2; + } + if (probabilityInfo2 == null) { + return probabilityInfo1; + } + return (probabilityInfo1.mProbability > probabilityInfo2.mProbability) ? probabilityInfo1 + : probabilityInfo2; + } + + public ProbabilityInfo(final int probability) { + this(probability, BinaryDictionary.NOT_A_VALID_TIMESTAMP, 0, 0); + } + + public ProbabilityInfo(final int probability, final int timestamp, final int level, + final int count) { + mProbability = probability; + mTimestamp = timestamp; + mLevel = level; + mCount = count; + } + + public boolean hasHistoricalInfo() { + return mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP; + } + + @Override + public int hashCode() { + if (hasHistoricalInfo()) { + return Arrays.hashCode(new Object[] { mProbability, mTimestamp, mLevel, mCount }); + } + return Arrays.hashCode(new Object[] { mProbability }); + } + + @Override + public String toString() { + return CombinedFormatUtils.formatProbabilityInfo(this); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof ProbabilityInfo)) return false; + final ProbabilityInfo p = (ProbabilityInfo)o; + if (!hasHistoricalInfo() && !p.hasHistoricalInfo()) { + return mProbability == p.mProbability; + } + return mProbability == p.mProbability && mTimestamp == p.mTimestamp && mLevel == p.mLevel + && mCount == p.mCount; + } +}
\ No newline at end of file diff --git a/java/src/org/kelar/inputmethod/latin/makedict/UnsupportedFormatException.java b/java/src/org/kelar/inputmethod/latin/makedict/UnsupportedFormatException.java new file mode 100644 index 000000000..a8d60e5fb --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/UnsupportedFormatException.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +/** + * Simple exception thrown when a file format is not recognized. + */ +public final class UnsupportedFormatException extends Exception { + public UnsupportedFormatException(String description) { + super(description); + } +} diff --git a/java/src/org/kelar/inputmethod/latin/makedict/WeightedString.java b/java/src/org/kelar/inputmethod/latin/makedict/WeightedString.java new file mode 100644 index 000000000..e2b910b29 --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/WeightedString.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +import org.kelar.inputmethod.annotations.UsedForTesting; + +import java.util.Arrays; + +/** + * A string with a probability. + * + * This represents an "attribute", that is either a bigram or a shortcut. + */ +public final class WeightedString { + public final String mWord; + public ProbabilityInfo mProbabilityInfo; + + public WeightedString(final String word, final int probability) { + this(word, new ProbabilityInfo(probability)); + } + + public WeightedString(final String word, final ProbabilityInfo probabilityInfo) { + mWord = word; + mProbabilityInfo = probabilityInfo; + } + + @UsedForTesting + public int getProbability() { + return mProbabilityInfo.mProbability; + } + + public void setProbability(final int probability) { + mProbabilityInfo = new ProbabilityInfo(probability); + } + + @Override + public int hashCode() { + return Arrays.hashCode(new Object[] { mWord, mProbabilityInfo}); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof WeightedString)) return false; + final WeightedString w = (WeightedString)o; + return mWord.equals(w.mWord) && mProbabilityInfo.equals(w.mProbabilityInfo); + } +}
\ No newline at end of file diff --git a/java/src/org/kelar/inputmethod/latin/makedict/WordProperty.java b/java/src/org/kelar/inputmethod/latin/makedict/WordProperty.java new file mode 100644 index 000000000..e28615c40 --- /dev/null +++ b/java/src/org/kelar/inputmethod/latin/makedict/WordProperty.java @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kelar.inputmethod.latin.makedict; + +import org.kelar.inputmethod.annotations.UsedForTesting; +import org.kelar.inputmethod.latin.BinaryDictionary; +import org.kelar.inputmethod.latin.Dictionary; +import org.kelar.inputmethod.latin.NgramContext; +import org.kelar.inputmethod.latin.NgramContext.WordInfo; +import org.kelar.inputmethod.latin.common.StringUtils; +import org.kelar.inputmethod.latin.utils.CombinedFormatUtils; + +import java.util.ArrayList; +import java.util.Arrays; + +import javax.annotation.Nullable; + +/** + * Utility class for a word with a probability. + * + * This is chiefly used to iterate a dictionary. + */ +public final class WordProperty implements Comparable<WordProperty> { + public final String mWord; + public final ProbabilityInfo mProbabilityInfo; + public final ArrayList<NgramProperty> mNgrams; + // TODO: Support mIsBeginningOfSentence. + public final boolean mIsBeginningOfSentence; + public final boolean mIsNotAWord; + public final boolean mIsPossiblyOffensive; + public final boolean mHasNgrams; + + private int mHashCode = 0; + + // TODO: Support n-gram. + @UsedForTesting + public WordProperty(final String word, final ProbabilityInfo probabilityInfo, + @Nullable final ArrayList<WeightedString> bigrams, + final boolean isNotAWord, final boolean isPossiblyOffensive) { + mWord = word; + mProbabilityInfo = probabilityInfo; + if (null == bigrams) { + mNgrams = null; + } else { + mNgrams = new ArrayList<>(); + final NgramContext ngramContext = new NgramContext(new WordInfo(mWord)); + for (final WeightedString bigramTarget : bigrams) { + mNgrams.add(new NgramProperty(bigramTarget, ngramContext)); + } + } + mIsBeginningOfSentence = false; + mIsNotAWord = isNotAWord; + mIsPossiblyOffensive = isPossiblyOffensive; + mHasNgrams = bigrams != null && !bigrams.isEmpty(); + } + + private static ProbabilityInfo createProbabilityInfoFromArray(final int[] probabilityInfo) { + return new ProbabilityInfo( + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_PROBABILITY_INDEX], + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX], + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_LEVEL_INDEX], + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_COUNT_INDEX]); + } + + // Construct word property using information from native code. + // This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY. + public WordProperty(final int[] codePoints, final boolean isNotAWord, + final boolean isPossiblyOffensive, final boolean hasBigram, + final boolean isBeginningOfSentence, final int[] probabilityInfo, + final ArrayList<int[][]> ngramPrevWordsArray, + final ArrayList<boolean[]> ngramPrevWordIsBeginningOfSentenceArray, + final ArrayList<int[]> ngramTargets, final ArrayList<int[]> ngramProbabilityInfo) { + mWord = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); + mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo); + final ArrayList<NgramProperty> ngrams = new ArrayList<>(); + mIsBeginningOfSentence = isBeginningOfSentence; + mIsNotAWord = isNotAWord; + mIsPossiblyOffensive = isPossiblyOffensive; + mHasNgrams = hasBigram; + + final int relatedNgramCount = ngramTargets.size(); + for (int i = 0; i < relatedNgramCount; i++) { + final String ngramTargetString = + StringUtils.getStringFromNullTerminatedCodePointArray(ngramTargets.get(i)); + final WeightedString ngramTarget = new WeightedString(ngramTargetString, + createProbabilityInfoFromArray(ngramProbabilityInfo.get(i))); + final int[][] prevWords = ngramPrevWordsArray.get(i); + final boolean[] isBeginningOfSentenceArray = + ngramPrevWordIsBeginningOfSentenceArray.get(i); + final WordInfo[] wordInfoArray = new WordInfo[prevWords.length]; + for (int j = 0; j < prevWords.length; j++) { + wordInfoArray[j] = isBeginningOfSentenceArray[j] + ? WordInfo.BEGINNING_OF_SENTENCE_WORD_INFO + : new WordInfo(StringUtils.getStringFromNullTerminatedCodePointArray( + prevWords[j])); + } + final NgramContext ngramContext = new NgramContext(wordInfoArray); + ngrams.add(new NgramProperty(ngramTarget, ngramContext)); + } + mNgrams = ngrams.isEmpty() ? null : ngrams; + } + + // TODO: Remove + @UsedForTesting + public ArrayList<WeightedString> getBigrams() { + if (null == mNgrams) { + return null; + } + final ArrayList<WeightedString> bigrams = new ArrayList<>(); + for (final NgramProperty ngram : mNgrams) { + if (ngram.mNgramContext.getPrevWordCount() == 1) { + bigrams.add(ngram.mTargetWord); + } + } + return bigrams; + } + + public int getProbability() { + return mProbabilityInfo.mProbability; + } + + private static int computeHashCode(WordProperty word) { + return Arrays.hashCode(new Object[] { + word.mWord, + word.mProbabilityInfo, + word.mNgrams, + word.mIsNotAWord, + word.mIsPossiblyOffensive + }); + } + + /** + * Three-way comparison. + * + * A Word x is greater than a word y if x has a higher frequency. If they have the same + * frequency, they are sorted in lexicographic order. + */ + @Override + public int compareTo(final WordProperty w) { + if (getProbability() < w.getProbability()) return 1; + if (getProbability() > w.getProbability()) return -1; + return mWord.compareTo(w.mWord); + } + + /** + * Equality test. + * + * Words are equal if they have the same frequency, the same spellings, and the same + * attributes. + */ + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof WordProperty)) return false; + WordProperty w = (WordProperty)o; + return mProbabilityInfo.equals(w.mProbabilityInfo) + && mWord.equals(w.mWord) && equals(mNgrams, w.mNgrams) + && mIsNotAWord == w.mIsNotAWord && mIsPossiblyOffensive == w.mIsPossiblyOffensive + && mHasNgrams == w.mHasNgrams; + } + + // TDOO: Have a utility method like java.util.Objects.equals. + private static <T> boolean equals(final ArrayList<T> a, final ArrayList<T> b) { + if (null == a) { + return null == b; + } + return a.equals(b); + } + + @Override + public int hashCode() { + if (mHashCode == 0) { + mHashCode = computeHashCode(this); + } + return mHashCode; + } + + @UsedForTesting + public boolean isValid() { + return getProbability() != Dictionary.NOT_A_PROBABILITY; + } + + @Override + public String toString() { + return CombinedFormatUtils.formatWordProperty(this); + } +} |