diff options
75 files changed, 590 insertions, 836 deletions
diff --git a/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java index f0feb2513..e2fd39017 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java +++ b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java @@ -25,7 +25,7 @@ import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.StringUtils; +import com.android.inputmethod.latin.utils.JsonUtils; import java.util.ArrayDeque; import java.util.ArrayList; @@ -139,7 +139,7 @@ public class DynamicGridKeyboard extends Keyboard { keys.add(key.getCode()); } } - final String jsonStr = StringUtils.listToJsonStr(keys); + final String jsonStr = JsonUtils.listToJsonStr(keys); Settings.writeEmojiRecentKeys(mPrefs, jsonStr); } @@ -167,7 +167,7 @@ public class DynamicGridKeyboard extends Keyboard { public void loadRecentKeys(final Collection<DynamicGridKeyboard> keyboards) { final String str = Settings.readEmojiRecentKeys(mPrefs); - final List<Object> keys = StringUtils.jsonStrToList(str); + final List<Object> keys = JsonUtils.jsonStrToList(str); for (final Object o : keys) { final Key key; if (o instanceof Integer) { diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java index 59d556956..e6fb9807e 100644 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -60,10 +60,9 @@ abstract public class AbstractDictionaryWriter { abstract protected void writeDictionary(final DictEncoder dictEncoder, final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException; - public void write(final String fileName, final Map<String, String> attributeMap) { - final String tempFileName = fileName + ".temp"; - final File file = new File(mContext.getFilesDir(), fileName); - final File tempFile = new File(mContext.getFilesDir(), tempFileName); + public void write(final File file, final Map<String, String> attributeMap) { + final String tempFilePath = file.getAbsolutePath() + ".temp"; + final File tempFile = new File(tempFilePath); try { final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile); writeDictionary(dictEncoder, attributeMap); diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index f960c5343..89ef96d7f 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -18,8 +18,6 @@ package com.android.inputmethod.latin; import android.content.Context; -import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; @@ -52,7 +50,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { public void clear() { final HashMap<String, String> attributes = CollectionUtils.newHashMap(); mFusionDictionary = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(attributes, false, false)); + new FusionDictionary.DictionaryOptions(attributes)); } /** diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index b52045e3c..ed80a9629 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -144,20 +145,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return mBinaryDictionary.isValidDictionary(); } - protected String getFileNameToCreateDict(final String dictName) { - return dictName + DICT_FILE_EXTENSION; - } - - protected String getFileNameToOpenDict(final String dictName) { - return getFileNameToCreateDict(dictName); - } - - private File getFileToCreateDict() { - return new File(mContext.getFilesDir(), getFileNameToCreateDict(mDictName)); - } - - private File getFileToOpenDict() { - return new File(mContext.getFilesDir(), getFileNameToOpenDict(mDictName)); + private File getDictFile() { + return new File(mContext.getFilesDir(), mDictName + DICT_FILE_EXTENSION); } /** @@ -270,17 +259,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (mDictionaryWriter == null) { mBinaryDictionary.close(); - final File file = getFileToCreateDict(); - file.delete(); + final File file = getDictFile(); + if (file.exists() && !FileUtils.deleteRecursively(file)) { + Log.e(TAG, "Can't remove a file: " + file.getName()); + } BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); - // We have 'fileToOpen' in addition to 'file' for the v4 dictionary format - // where 'file' is a directory, and 'fileToOpen' is a normal file. - final File fileToOpen = getFileToOpenDict(); - // TODO: Make BinaryDictionary's constructor be able to accept filename - // without extension. mBinaryDictionary = new BinaryDictionary( - fileToOpen.getAbsolutePath(), 0 /* offset */, fileToOpen.length(), + file.getAbsolutePath(), 0 /* offset */, file.length(), true /* useFullEditDistance */, null, mDictType, mIsUpdatable); } else { mDictionaryWriter.clear(); @@ -531,7 +517,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - final File file = getFileToOpenDict(); + final File file = getDictFile(); final String filename = file.getAbsolutePath(); final long length = file.length(); @@ -572,14 +558,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); loadDictionaryAsync(); - mDictionaryWriter.write(getFileNameToCreateDict(mDictName), getHeaderAttributeMap()); + mDictionaryWriter.write(getDictFile(), getHeaderAttributeMap()); } else { if (mBinaryDictionary == null || !isValidDictionary() // TODO: remove the check below || !matchesExpectedBinaryDictFormatVersionForThisType( mBinaryDictionary.getFormatVersion())) { - final File file = getFileToCreateDict(); - if (!FileUtils.deleteRecursively(file)) { + final File file = getDictFile(); + if (file.exists() && !FileUtils.deleteRecursively(file)) { Log.e(TAG, "Can't remove a file: " + file.getName()); } BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), @@ -706,8 +692,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // TODO: cache the file's existence so that we avoid doing a disk access each time. private boolean dictionaryFileExists() { - final File file = getFileToOpenDict(); - return file.exists(); + return getDictFile().exists(); } /** @@ -754,13 +739,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } @UsedForTesting - public void shutdownExecutorForTests() { - getExecutor(mDictName).shutdown(); - } - - @UsedForTesting - public boolean isTerminatedForTests() { - return getExecutor(mDictName).isTerminated(); + public void waitAllTasksForTests() { + final CountDownLatch countDownLatch = new CountDownLatch(1); + getExecutor(mDictName).execute(new Runnable() { + @Override + public void run() { + countDownLatch.countDown(); + } + }); + try { + countDownLatch.await(); + } catch (InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); + } } @UsedForTesting diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index bbbb8e461..f8fa68f45 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -58,12 +58,9 @@ public abstract class AbstractDictDecoder implements DictDecoder { headerSize); final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE), - 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); + new FusionDictionary.DictionaryOptions(attributes), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); return header; } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 83ee7d685..7f0aa777f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -330,7 +330,7 @@ public final class BinaryDictDecoderUtils { static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { - if (options.mSupportsDynamicUpdate) { + if (options.supportsDynamicUpdate()) { final int address = dictBuffer.readUnsignedInt24(); if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; if ((address & FormatSpec.MSB24) != 0) { @@ -540,11 +540,11 @@ public final class BinaryDictDecoderUtils { } // reach the end of the array. - if (options.mSupportsDynamicUpdate) { + if (options.supportsDynamicUpdate()) { final boolean hasValidForwardLink = dictDecoder.readAndFollowForwardLink(); if (!hasValidForwardLink) break; } - } while (options.mSupportsDynamicUpdate && dictDecoder.hasNextPtNodeArray()); + } while (options.supportsDynamicUpdate() && dictDecoder.hasNextPtNodeArray()); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index c0dad3db2..8ba0797de 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -20,7 +20,6 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -161,7 +160,7 @@ public class BinaryDictEncoderUtils { node.mCachedSize = nodeSize; size += nodeSize; } - if (options.mSupportsDynamicUpdate) { + if (options.supportsDynamicUpdate()) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } ptNodeArray.mCachedSize = size; @@ -398,7 +397,7 @@ public class BinaryDictEncoderUtils { nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; } } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; } else if (null != ptNode.mChildren) { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, @@ -418,7 +417,7 @@ public class BinaryDictEncoderUtils { ptNode.mCachedSize = nodeSize; size += nodeSize; } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } if (ptNodeArray.mCachedSize != size) { @@ -534,7 +533,7 @@ public class BinaryDictEncoderUtils { if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); } while (changesDone); - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { computeParentAddresses(flatNodes); } final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); @@ -643,7 +642,7 @@ public class BinaryDictEncoderUtils { byte flags = 0; if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { flags |= FormatSpec.FLAG_IS_NOT_MOVED; } else if (true) { switch (childrenAddressSize) { @@ -755,16 +754,11 @@ public class BinaryDictEncoderUtils { } /** - * Makes the 2-byte value for options flags. + * Makes the 2-byte value for options flags. Unused at the moment, and always 0. */ - private static final int makeOptionsValue(final FusionDictionary dictionary, - final FormatOptions formatOptions) { - final DictionaryOptions options = dictionary.mOptions; - final boolean hasBigrams = dictionary.hasBigrams(); - return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) - + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) - + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) - + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); + private static final int makeOptionsValue(final FormatOptions formatOptions) { + // TODO: why doesn't this handle CONTAINS_TIMESTAMP_FLAG? + return 0; } /** @@ -852,7 +846,7 @@ public class BinaryDictEncoderUtils { } dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); } if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate @@ -953,7 +947,7 @@ public class BinaryDictEncoderUtils { headerBuffer.write((byte) (0xFF & version)); // Options flags - final int options = makeOptionsValue(dict, formatOptions); + final int options = makeOptionsValue(formatOptions); headerBuffer.write((byte) (0xFF & (options >> 8))); headerBuffer.write((byte) (0xFF & options)); final int headerSizeOffset = headerBuffer.size(); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 86ebf5844..640d778bb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -112,7 +112,7 @@ public final class BinaryDictIOUtils { } if (p.mPosition == p.mNumOfPtNode) { - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { final boolean hasValidForwardLinkAddress = dictDecoder.readAndFollowForwardLink(); if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) { @@ -228,7 +228,7 @@ public final class BinaryDictIOUtils { // a forward link address that we need to consult and possibly resume // search on the next node array in the linked list. if (foundNextPtNode) break; - if (!header.mFormatOptions.mSupportsDynamicUpdate) { + if (!header.mFormatOptions.supportsDynamicUpdate()) { return FormatSpec.NOT_VALID_WORD; } @@ -507,7 +507,7 @@ public final class BinaryDictIOUtils { * Helper method to check whether the node is moved. */ public static boolean isMovedPtNode(final int flags, final FormatOptions options) { - return options.mSupportsDynamicUpdate + return options.supportsDynamicUpdate() && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); } @@ -516,14 +516,14 @@ public final class BinaryDictIOUtils { */ public static boolean supportsDynamicUpdate(final FormatOptions options) { return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.mSupportsDynamicUpdate; + && options.supportsDynamicUpdate(); } /** * Helper method to check whether the node is deleted. */ public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { - return formatOptions.mSupportsDynamicUpdate + return formatOptions.supportsDynamicUpdate() && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); } @@ -546,7 +546,7 @@ public final class BinaryDictIOUtils { static int getChildrenAddressSize(final int optionFlags, final FormatOptions formatOptions) { - if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + if (formatOptions.supportsDynamicUpdate()) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return 1; diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 971b4ff9f..ff03190a3 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -61,7 +61,7 @@ public final class DynamicBinaryDictIOUtils { final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); final int originalPosition = dictBuffer.position(); dictBuffer.position(ptNodeOriginAddress); - if (!formatOptions.mSupportsDynamicUpdate) { + if (!formatOptions.supportsDynamicUpdate()) { throw new RuntimeException("this file format does not support parent addresses"); } final int flags = dictBuffer.readUnsignedByte(); @@ -102,7 +102,7 @@ public final class DynamicBinaryDictIOUtils { } if (!dictUpdater.readAndFollowForwardLink()) break; if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break; - } while (formatOptions.mSupportsDynamicUpdate); + } while (formatOptions.supportsDynamicUpdate()); dictUpdater.setPosition(originalPosition); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 846aacf11..20ddba836 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -40,12 +40,8 @@ public final class FormatSpec { * p | not used 3 bits * t | each unigram and bigram entry has a time stamp? * i | 1 bit, 1 = yes, 0 = no : CONTAINS_TIMESTAMP_FLAG - * o | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG - * n | FRENCH_LIGATURE_PROCESSING_FLAG - * f | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE - * l | GERMAN_UMLAUT_PROCESSING_FLAG - * a | - * gs + * o | + * nflags * * h | * e | size of the file header, 4bytes @@ -82,45 +78,36 @@ public final class FormatSpec { * s * * f | - * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) - * r | forward link address, 3byte - * w | 1 byte = bbbbbbbb match - * a | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) - * r | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte - * d | - * linkaddress + * o | forward link address, 3byte + * r | 1 byte = bbbbbbbb match + * w | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) + * a | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte + * r | + * dlinkaddress */ /* Node (FusionDictionary.PtNode) layout is as follows: - * | IF !SUPPORTS_DYNAMIC_UPDATE - * | addressType xx : mask with MASK_CHILDREN_ADDRESS_TYPE - * | 2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS - * f | 01 = 1 byte : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE - * l | 10 = 2 bytes : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES - * a | 11 = 3 bytes : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES - * g | ELSE - * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED - * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES - * | 01 = yes : FLAG_IS_MOVED - * | the new address is stored in the same place as the parent address - * | is deleted? 10 = yes : FLAG_IS_DELETED - * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS - * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL - * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS + * | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED + * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 01 = yes : FLAG_IS_MOVED + * f | the new address is stored in the same place as the parent address + * l | is deleted? 10 = yes : FLAG_IS_DELETED + * a | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS + * g | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL + * s | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD * | is blacklisted ? 1 bit, 1 = yes, 0 = no : FLAG_IS_BLACKLISTED * * p | - * a | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) - * r | parent address, 3byte - * e | 1 byte = bbbbbbbb match - * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) - * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte - * a | This address is relative to the head of the PtNode. - * d | If the node doesn't have a parent, this field is set to 0. + * a | parent address, 3byte + * r | 1 byte = bbbbbbbb match + * e | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * n | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte + * t | This address is relative to the head of the PtNode. + * a | If the node doesn't have a parent, this field is set to 0. * d | - * ress + * dress * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers @@ -134,23 +121,16 @@ public final class FormatSpec { * e | frequency 1 byte * q | * - * c | IF SUPPORTS_DYNAMIC_UPDATE - * h | children address, 3 bytes - * i | 1 byte = bbbbbbbb match - * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) - * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte - * r | if this node doesn't have children, this field is set to 0. - * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) - * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType - * a | // nothing - * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType - * d | children address, 1 byte - * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType - * e | children address, 2 bytes - * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType - * s | children address, 3 bytes - * | END - * | This address is relative to the position of this field. + * c | + * h | children address, 3 bytes + * i | 1 byte = bbbbbbbb match + * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte + * r | if this node doesn't have children, this field is set to 0. + * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) + * n | This address is relative to the position of this field. + * a | + * ddress * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list @@ -214,11 +194,7 @@ public final class FormatSpec { static final int MAXIMUM_SUPPORTED_VERSION = VERSION4; // These options need to be the same numeric values as the one in the native reading code. - static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; // TODO: Make the native reading code read this variable. - static final int SUPPORTS_DYNAMIC_UPDATE = 0x2; - static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; - static final int CONTAINS_BIGRAMS_FLAG = 0x8; static final int CONTAINS_TIMESTAMP_FLAG = 0x10; // TODO: Make this value adaptative to content data, store it in the header, and @@ -339,30 +315,23 @@ public final class FormatSpec { */ public static final class FormatOptions { public final int mVersion; - public final boolean mSupportsDynamicUpdate; public final boolean mHasTerminalId; public final boolean mHasTimestamp; - @UsedForTesting - public FormatOptions(final int version) { - this(version, false); - } @UsedForTesting - public FormatOptions(final int version, final boolean supportsDynamicUpdate) { - this(version, supportsDynamicUpdate, false /* hasTimestamp */); + public FormatOptions(final int version) { + this(version, false /* hasTimestamp */); } - public FormatOptions(final int version, final boolean supportsDynamicUpdate, - final boolean hasTimestamp) { + public FormatOptions(final int version, final boolean hasTimestamp) { mVersion = version; - if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) { - throw new RuntimeException("Dynamic updates are only supported with versions " - + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); - } - mSupportsDynamicUpdate = supportsDynamicUpdate; mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); mHasTimestamp = hasTimestamp; } + + public boolean supportsDynamicUpdate() { + return mVersion >= FIRST_VERSION_WITH_DYNAMIC_UPDATE; + } } /** @@ -374,7 +343,6 @@ public final class FormatSpec { public final FormatOptions mFormatOptions; // Note that these are corresponding definitions in native code in latinime::HeaderPolicy // and latinime::HeaderReadWriteUtils. - public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; public static final String HAS_HISTORICAL_INFO_ATTRIBUTE = "HAS_HISTORICAL_INFO"; public static final String ATTRIBUTE_VALUE_TRUE = "1"; diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 3bb218bea..fdf2ae7b5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -303,14 +303,9 @@ public final class FusionDictionary implements Iterable<Word> { * Options global to the dictionary. */ public static final class DictionaryOptions { - public final boolean mGermanUmlautProcessing; - public final boolean mFrenchLigatureProcessing; public final HashMap<String, String> mAttributes; - public DictionaryOptions(final HashMap<String, String> attributes, - final boolean germanUmlautProcessing, final boolean frenchLigatureProcessing) { + public DictionaryOptions(final HashMap<String, String> attributes) { mAttributes = attributes; - mGermanUmlautProcessing = germanUmlautProcessing; - mFrenchLigatureProcessing = frenchLigatureProcessing; } @Override public String toString() { // Convenience method @@ -339,14 +334,6 @@ public final class FusionDictionary implements Iterable<Word> { } s.append("\n"); } - if (mGermanUmlautProcessing) { - s.append(indent); - s.append("Needs German umlaut processing\n"); - } - if (mFrenchLigatureProcessing) { - s.append(indent); - s.append("Needs French ligature processing\n"); - } return s.toString(); } } @@ -701,138 +688,6 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Recursively count the number of nodes in a given branch of the trie. - * - * @param nodeArray the node array to count. - * @return the number of nodes in this branch. - */ - public static int countNodeArrays(final PtNodeArray nodeArray) { - int size = 1; - for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { - PtNode ptNode = nodeArray.mData.get(i); - if (null != ptNode.mChildren) - size += countNodeArrays(ptNode.mChildren); - } - return size; - } - - // Recursively find out whether there are any bigrams. - // This can be pretty expensive especially if there aren't any (we return as soon - // as we find one, so it's much cheaper if there are bigrams) - private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { - if (null == nodeArray) return false; - for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { - PtNode ptNode = nodeArray.mData.get(i); - if (null != ptNode.mBigrams) return true; - if (hasBigramsInternal(ptNode.mChildren)) return true; - } - return false; - } - - /** - * Finds out whether there are any bigrams in this dictionary. - * - * @return true if there is any bigram, false otherwise. - */ - // TODO: this is expensive especially for large dictionaries without any bigram. - // The up side is, this is always accurate and correct and uses no memory. We should - // find a more efficient way of doing this, without compromising too much on memory - // and ease of use. - public boolean hasBigrams() { - return hasBigramsInternal(mRootNodeArray); - } - - // Historically, the tails of the words were going to be merged to save space. - // However, that would prevent the code to search for a specific address in log(n) - // time so this was abandoned. - // The code is still of interest as it does add some compression to any dictionary - // that has no need for attributes. Implementations that does not read attributes should be - // able to read a dictionary with merged tails. - // Also, the following code does support frequencies, as in, it will only merges - // tails that share the same frequency. Though it would result in the above loss of - // performance while searching by address, it is still technically possible to merge - // tails that contain attributes, but this code does not take that into account - it does - // not compare attributes and will merge terminals with different attributes regardless. - public void mergeTails() { - MakedictLog.i("Do not merge tails"); - return; - -// MakedictLog.i("Merging PtNodes. Number of PtNodes : " + countPtNodes(root)); -// MakedictLog.i("Number of PtNodes : " + countPtNodes(root)); -// -// final HashMap<String, ArrayList<PtNodeArray>> repository = -// new HashMap<String, ArrayList<PtNodeArray>>(); -// mergeTailsInner(repository, root); -// -// MakedictLog.i("Number of different pseudohashes : " + repository.size()); -// int size = 0; -// for (ArrayList<PtNodeArray> a : repository.values()) { -// size += a.size(); -// } -// MakedictLog.i("Number of nodes after merge : " + (1 + size)); -// MakedictLog.i("Recursively seen nodes : " + countNodes(root)); - } - - // The following methods are used by the deactivated mergeTails() -// private static boolean isEqual(PtNodeArray a, PtNodeArray b) { -// if (null == a && null == b) return true; -// if (null == a || null == b) return false; -// if (a.data.size() != b.data.size()) return false; -// final int size = a.data.size(); -// for (int i = size - 1; i >= 0; --i) { -// PtNode aPtNode = a.data.get(i); -// PtNode bPtNode = b.data.get(i); -// if (aPtNode.frequency != bPtNode.frequency) return false; -// if (aPtNode.alternates == null && bPtNode.alternates != null) return false; -// if (aPtNode.alternates != null && !aPtNode.equals(bPtNode.alternates)) return false; -// if (!Arrays.equals(aPtNode.chars, bPtNode.chars)) return false; -// if (!isEqual(aPtNode.children, bPtNode.children)) return false; -// } -// return true; -// } - -// static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner( -// final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) { -// final ArrayList<PtNode> branches = nodeArray.data; -// final int nodeSize = branches.size(); -// for (int i = 0; i < nodeSize; ++i) { -// PtNode ptNode = branches.get(i); -// if (null != ptNode.children) { -// String pseudoHash = getPseudoHash(ptNode.children); -// ArrayList<PtNodeArray> similarList = map.get(pseudoHash); -// if (null == similarList) { -// similarList = new ArrayList<PtNodeArray>(); -// map.put(pseudoHash, similarList); -// } -// boolean merged = false; -// for (PtNodeArray similar : similarList) { -// if (isEqual(ptNode.children, similar)) { -// ptNode.children = similar; -// merged = true; -// break; -// } -// } -// if (!merged) { -// similarList.add(ptNode.children); -// } -// mergeTailsInner(map, ptNode.children); -// } -// } -// return map; -// } - -// private static String getPseudoHash(final PtNodeArray nodeArray) { -// StringBuilder s = new StringBuilder(); -// for (PtNode ptNode : nodeArray.data) { -// s.append(ptNode.frequency); -// for (int ch : ptNode.chars) { -// s.append(Character.toChars(ch)); -// } -// } -// return s.toString(); -// } - - /** * Iterator to walk through a dictionary. * * This is purely for convenience. diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 5da34534e..92eb861d6 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -169,7 +169,7 @@ public class Ver3DictEncoder implements DictEncoder { private void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition, childrenPos); } else { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 07522b54b..3be62f066 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -183,14 +183,11 @@ public class Ver4DictDecoder extends AbstractDictDecoder { * An auxiliary class for reading bigrams. */ protected static class BigramContentReader extends SparseTableContentReader { - private final boolean mHasTimestamp; - public BigramContentReader(final String name, final File baseDir, final DictionaryBufferFactory factory, final boolean hasTimestamp) { super(name + FormatSpec.BIGRAM_FILE_EXTENSION, FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, getContentFilenames(name, hasTimestamp), getContentIds(hasTimestamp), factory); - mHasTimestamp = hasTimestamp; } // TODO: Consolidate this method and BigramContentWriter.getContentFilenames. diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 1a245b6db..8b80ebe63 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -365,7 +365,7 @@ public class Ver4DictEncoder implements DictEncoder { private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, mTriePos, childrenPos); } else { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java index 91d9cf345..c46bc36bb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -54,8 +54,6 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { } private static class BigramContentUpdater extends SparseTableContentUpdater { - private final boolean mHasTimestamp; - public BigramContentUpdater(final String name, final File baseDir, final boolean hasTimestamp) { super(name + FormatSpec.BIGRAM_FILE_EXTENSION, @@ -63,7 +61,6 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { BigramContentReader.getContentFilenames(name, hasTimestamp), BigramContentReader.getContentIds(hasTimestamp), new DictionaryBufferFromWritableByteBufferFactory()); - mHasTimestamp = hasTimestamp; } public void insertBigramEntries(final int terminalId, final int frequency, diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 8321df94b..2dc9ecd7b 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -82,8 +82,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB @Override protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE, @@ -112,16 +110,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION; } - @Override - protected String getFileNameToCreateDict(final String dictName) { - return dictName; - } - - @Override - protected String getFileNameToOpenDict(final String dictName) { - return dictName + "/" + dictName + FormatSpec.HEADER_FILE_EXTENSION; - } - public void addMultipleDictionaryEntriesToDictionary( final ArrayList<LanguageModelParam> languageModelParams, final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { diff --git a/java/src/com/android/inputmethod/latin/utils/JsonUtils.java b/java/src/com/android/inputmethod/latin/utils/JsonUtils.java new file mode 100644 index 000000000..764ef72ce --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/JsonUtils.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.util.JsonReader; +import android.util.JsonWriter; +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class JsonUtils { + private static final String TAG = JsonUtils.class.getSimpleName(); + + private static final String INTEGER_CLASS_NAME = Integer.class.getSimpleName(); + private static final String STRING_CLASS_NAME = String.class.getSimpleName(); + + private static final String EMPTY_STRING = ""; + + public static List<Object> jsonStrToList(final String s) { + final ArrayList<Object> list = CollectionUtils.newArrayList(); + final JsonReader reader = new JsonReader(new StringReader(s)); + try { + reader.beginArray(); + while (reader.hasNext()) { + reader.beginObject(); + while (reader.hasNext()) { + final String name = reader.nextName(); + if (name.equals(INTEGER_CLASS_NAME)) { + list.add(reader.nextInt()); + } else if (name.equals(STRING_CLASS_NAME)) { + list.add(reader.nextString()); + } else { + Log.w(TAG, "Invalid name: " + name); + reader.skipValue(); + } + } + reader.endObject(); + } + reader.endArray(); + return list; + } catch (final IOException e) { + } finally { + close(reader); + } + return Collections.<Object>emptyList(); + } + + public static String listToJsonStr(final List<Object> list) { + if (list == null || list.isEmpty()) { + return EMPTY_STRING; + } + final StringWriter sw = new StringWriter(); + final JsonWriter writer = new JsonWriter(sw); + try { + writer.beginArray(); + for (final Object o : list) { + writer.beginObject(); + if (o instanceof Integer) { + writer.name(INTEGER_CLASS_NAME).value((Integer)o); + } else if (o instanceof String) { + writer.name(STRING_CLASS_NAME).value((String)o); + } + writer.endObject(); + } + writer.endArray(); + return sw.toString(); + } catch (final IOException e) { + } finally { + close(writer); + } + return EMPTY_STRING; + } + + private static void close(final Closeable closeable) { + try { + if (closeable != null) { + closeable.close(); + } + } catch (final IOException e) { + // Ignore + } + } +} diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index 928000ec9..df420417d 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -17,20 +17,14 @@ package com.android.inputmethod.latin.utils; import android.text.TextUtils; -import android.util.JsonReader; -import android.util.JsonWriter; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.settings.SettingsValues; -import java.io.Closeable; import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; @@ -426,72 +420,4 @@ public final class StringUtils { } return bytes; } - - private static final String INTEGER_CLASS_NAME = Integer.class.getSimpleName(); - private static final String STRING_CLASS_NAME = String.class.getSimpleName(); - - public static List<Object> jsonStrToList(final String s) { - final ArrayList<Object> list = CollectionUtils.newArrayList(); - final JsonReader reader = new JsonReader(new StringReader(s)); - try { - reader.beginArray(); - while (reader.hasNext()) { - reader.beginObject(); - while (reader.hasNext()) { - final String name = reader.nextName(); - if (name.equals(INTEGER_CLASS_NAME)) { - list.add(reader.nextInt()); - } else if (name.equals(STRING_CLASS_NAME)) { - list.add(reader.nextString()); - } else { - Log.w(TAG, "Invalid name: " + name); - reader.skipValue(); - } - } - reader.endObject(); - } - reader.endArray(); - return list; - } catch (final IOException e) { - } finally { - close(reader); - } - return Collections.<Object>emptyList(); - } - - public static String listToJsonStr(final List<Object> list) { - if (list == null || list.isEmpty()) { - return EMPTY_STRING; - } - final StringWriter sw = new StringWriter(); - final JsonWriter writer = new JsonWriter(sw); - try { - writer.beginArray(); - for (final Object o : list) { - writer.beginObject(); - if (o instanceof Integer) { - writer.name(INTEGER_CLASS_NAME).value((Integer)o); - } else if (o instanceof String) { - writer.name(STRING_CLASS_NAME).value((String)o); - } - writer.endObject(); - } - writer.endArray(); - return sw.toString(); - } catch (final IOException e) { - } finally { - close(writer); - } - return EMPTY_STRING; - } - - private static void close(final Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (final IOException e) { - // Ignore - } - } } diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index a2c3ed44d..db628fe18 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -95,8 +95,7 @@ public final class UserHistoryDictIOUtils { static FusionDictionary constructFusionDictionary(final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams, final HashMap<String, String> options) { final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(options, false, - false)); + new FusionDictionary.DictionaryOptions(options)); int profTotal = 0; for (final String word1 : bigrams.keySet()) { final HashMap<String, Byte> word1Bigrams = bigrams.getBigrams(word1); diff --git a/native/jni/Android.mk b/native/jni/Android.mk index f2c6d3bec..e11e706f3 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -76,15 +76,15 @@ LATIN_IME_CORE_SRC_FILES := \ $(addprefix suggest/policyimpl/dictionary/bigram/, \ bigram_list_read_write_utils.cpp \ ver4_bigram_list_policy.cpp) \ + $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \ + dynamic_pt_gc_event_listeners.cpp \ + dynamic_pt_reading_helper.cpp \ + dynamic_pt_reading_utils.cpp \ + dynamic_pt_updating_helper.cpp \ + dynamic_pt_writing_utils.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v2/, \ patricia_trie_policy.cpp \ patricia_trie_reading_utils.cpp) \ - $(addprefix suggest/policyimpl/dictionary/structure/v3/, \ - dynamic_patricia_trie_gc_event_listeners.cpp \ - dynamic_patricia_trie_reading_helper.cpp \ - dynamic_patricia_trie_reading_utils.cpp \ - dynamic_patricia_trie_updating_helper.cpp \ - dynamic_patricia_trie_writing_utils.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v4/, \ ver4_dict_buffers.cpp \ ver4_dict_constants.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp index 0ef6ccf37..7d0d09631 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp @@ -16,7 +16,6 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -78,11 +77,6 @@ const BigramListReadWriteUtils::BigramFlags offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos); break; } - if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) { - return NOT_A_DICT_POS; - } else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) { - return origin; - } if (isOffsetNegative(flags)) { return origin - offset; } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 5e8b5f11e..1208d2c2a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -87,8 +87,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { switch (mDictFormatVersion) { case FormatUtils::VERSION_2: return FormatUtils::VERSION_2; - case FormatUtils::VERSION_3: - return FormatUtils::VERSION_3; case FormatUtils::VERSION_4: return FormatUtils::VERSION_4; default: diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index 2d4547066..6b4598642 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -89,9 +89,6 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0; case FormatUtils::VERSION_2: // Version 2 dictionary writing is not supported. return false; - case FormatUtils::VERSION_3: - return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_3 /* data */, - HEADER_DICTIONARY_VERSION_SIZE, writingPos); case FormatUtils::VERSION_4: return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */, HEADER_DICTIONARY_VERSION_SIZE, writingPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index dfb110cdd..c81c61d23 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -16,11 +16,13 @@ #include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h" +#include <climits> #include <stdint.h> #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" @@ -32,9 +34,28 @@ namespace latinime { DictionaryStructureWithBufferPolicyFactory ::newDictionaryStructureWithBufferPolicy(const char *const path, const int bufOffset, const int size, const bool isUpdatable) { - // Allocated buffer in MmapedBuffer::newBuffer() will be freed in the destructor of - // MmappedBufferWrapper if the instance has the responsibility. - MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size, + if (FileUtils::existsDir(path)) { + // Given path represents a directory. + return newPolicyforDirectoryDict(path, isUpdatable); + } else { + if (isUpdatable) { + AKLOGE("One file dictionaries don't support updating. path: %s", path); + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); + } + return newPolicyforFileDict(path, bufOffset, size); + } +} + +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyforDirectoryDict( + const char *const path, const bool isUpdatable) { + const int headerFilePathBufSize = PATH_MAX + 1 /* terminator */; + char headerFilePath[headerFilePathBufSize]; + getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath); + // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of + // MmappedBufferPtr if the instance has the responsibility. + MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(headerFilePath, isUpdatable); if (!mmappedBuffer.get()) { return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); @@ -42,20 +63,22 @@ namespace latinime { switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(), mmappedBuffer.get()->getBufferSize())) { case FormatUtils::VERSION_2: - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( - new PatriciaTriePolicy(mmappedBuffer)); + AKLOGE("Given path is a directory but the format is version 2. path: %s", path); + break; case FormatUtils::VERSION_4: { - const int dictDirPathBufSize = strlen(path) + 1 /* terminator */; - char dictDirPath[dictDirPathBufSize]; - if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::HEADER_FILE_EXTENSION, - dictDirPathBufSize, dictDirPath)) { - // Dictionary file name is not valid as a version 4 dictionary. + const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */; + char dictPath[dictDirPathBufSize]; + if (!FileUtils::getFilePathWithoutSuffix(headerFilePath, + Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) { + AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path); + ASSERT(false); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); } const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = - Ver4DictBuffers::openVer4DictBuffers(dictDirPath, mmappedBuffer); + Ver4DictBuffers::openVer4DictBuffers(dictPath, mmappedBuffer); if (!dictBuffers.get()->isValid()) { - AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements."); + AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s", + path); ASSERT(false); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); } @@ -63,10 +86,47 @@ namespace latinime { new Ver4PatriciaTriePolicy(dictBuffers)); } default: - AKLOGE("DICT: dictionary format is unknown, bad magic number"); - ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); + AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path); + break; } + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); +} + +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyforFileDict( + const char *const path, const int bufOffset, const int size) { + // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of + // MmappedBufferPtr if the instance has the responsibility. + MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, + size, false /* isUpdatable */); + if (!mmappedBuffer.get()) { + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); + } + switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(), + mmappedBuffer.get()->getBufferSize())) { + case FormatUtils::VERSION_2: + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( + new PatriciaTriePolicy(mmappedBuffer)); + case FormatUtils::VERSION_4: + AKLOGE("Given path is a file but the format is version 4. path: %s", path); + break; + default: + AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path); + break; + } + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); +} + +/* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir( + const char *const dictDirPath, const int outHeaderFileBufSize, + char *const outHeaderFilePath) { + const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */; + char dictName[dictNameBufSize]; + FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName); + snprintf(outHeaderFilePath, outHeaderFileBufSize, "%s/%s%s", dictDirPath, + dictName, Ver4DictConstants::HEADER_FILE_EXTENSION); } } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h index 45237e4aa..45ab52931 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h @@ -33,6 +33,15 @@ class DictionaryStructureWithBufferPolicyFactory { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory); + + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr + newPolicyforDirectoryDict(const char *const path, const bool isUpdatable); + + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr + newPolicyforFileDict(const char *const path, const int bufOffset, const int size); + + static void getHeaderFilePathInDictDir(const char *const dirPath, + const int outHeaderFileBufSize, char *const outHeaderFilePath); }; } // namespace latinime #endif // LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp index bcba67035..8f42df6d2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp @@ -14,16 +14,16 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" #include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" namespace latinime { -bool DynamicPatriciaTrieGcEventListeners +bool DynamicPtGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless @@ -63,7 +63,7 @@ bool DynamicPatriciaTrieGcEventListeners return true; } -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability +bool DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { if (!ptNodeParams->isDeleted() && ptNodeParams->hasBigrams()) { int bigramEntryCount = 0; @@ -77,7 +77,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit } // Writes dummy PtNode array size when the head of PtNode array is read. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onDescend(const int ptNodeArrayPos) { mValidPtNodeCount = 0; int writingPos = mBufferToWrite->getTailPosition(); @@ -86,21 +86,21 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. mPtNodeArraySizeFieldPos = writingPos; - return DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( + return DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( mBufferToWrite, 0 /* arraySize */, &writingPos); } // Write PtNode array terminal and actual PtNode array size. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onReadingPtNodeArrayTail() { int writingPos = mBufferToWrite->getTailPosition(); // Write PtNode array terminal. - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition( mBufferToWrite, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } // Write actual PtNode array size. - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( mBufferToWrite, mValidPtNodeCount, &mPtNodeArraySizeFieldPos)) { return false; } @@ -108,7 +108,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo } // Write valid PtNode to buffer and memorize mapping from the old position to the new position. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { if (ptNodeParams->isDeleted()) { // Current PtNode is not written in new buffer because it has been deleted. @@ -126,7 +126,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo return mPtNodeWriter->writePtNodeAndAdvancePosition(ptNodeParams, &writingPos); } -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields +bool DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // Updates parent position. int bigramCount = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h index 562eb69bd..d8867754d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h @@ -14,14 +14,14 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H +#ifndef LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H +#define LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H #include <vector> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "utils/hash_map_compat.h" @@ -29,14 +29,13 @@ namespace latinime { class PtNodeParams; -// TODO: Move to pt_common. -class DynamicPatriciaTrieGcEventListeners { +class DynamicPtGcEventListeners { public: // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC. // TODO: Concatenate non-terminal PtNodes. class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( PtNodeWriter *const ptNodeWriter) @@ -81,7 +80,7 @@ class DynamicPatriciaTrieGcEventListeners { // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram // entries. class TraversePolicyToUpdateBigramProbability - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter) : mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {} @@ -106,7 +105,7 @@ class DynamicPatriciaTrieGcEventListeners { }; class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite, @@ -134,7 +133,7 @@ class DynamicPatriciaTrieGcEventListeners { }; class TraversePolicyToUpdateAllPositionFields - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter, const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap) @@ -168,7 +167,7 @@ class DynamicPatriciaTrieGcEventListeners { }; private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtGcEventListeners); }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H */ +#endif /* LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp index b3fdbeb78..b918e0765 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp @@ -14,25 +14,25 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "utils/char_utils.h" namespace latinime { // To avoid infinite loop caused by invalid or malicious forward links. -const int DynamicPatriciaTrieReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; -const int DynamicPatriciaTrieReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; -const size_t DynamicPatriciaTrieReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; +const int DynamicPtReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; +const int DynamicPtReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; +const size_t DynamicPtReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; // Visits all PtNodes in post-order depth first manner. // For example, visits c -> b -> y -> x -> a for the following dictionary: // a _ b _ c // \ x _ y -bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( +bool DynamicPtReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( TraversingEventListener *const listener) { bool alreadyVisitedChildren = false; // Descend from the root to the root PtNode array. @@ -92,7 +92,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstMa // For example, visits a -> b -> x -> c -> y for the following dictionary: // a _ b _ c // \ x _ y -bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( +bool DynamicPtReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( TraversingEventListener *const listener) { bool alreadyVisitedAllPtNodesInArray = false; bool alreadyVisitedChildren = false; @@ -169,7 +169,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreor return !isError(); } -int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( +int DynamicPtReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) { // This method traverses parent nodes from the terminal by following parent pointers; thus, // node code points are stored in the buffer in the reverse order. @@ -211,7 +211,7 @@ int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePo return totalCodePointCount; } -int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, +int DynamicPtReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) { int searchCodePoints[length]; for (int i = 0; i < length; ++i) { @@ -257,7 +257,7 @@ int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int // Read node array size and process empty node arrays. Nodes and arrays are counted up in this // method to avoid an infinite loop. -void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { +void DynamicPtReadingHelper::nextPtNodeArray() { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { // Reading invalid position because of a bug or a broken dictionary. AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d", @@ -308,7 +308,7 @@ void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { } // Follow the forward link and read the next node array if exists. -void DynamicPatriciaTrieReadingHelper::followForwardLink() { +void DynamicPtReadingHelper::followForwardLink() { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { // Reading invalid position because of bug or broken dictionary. AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d", @@ -324,12 +324,12 @@ void DynamicPatriciaTrieReadingHelper::followForwardLink() { mReadingState.mPos -= mBuffer->getOriginalBufferSize(); } const int forwardLinkPosition = - DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); + DynamicPtReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); if (usesAdditionalBuffer) { mReadingState.mPos += mBuffer->getOriginalBufferSize(); } mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos; - if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { + if (DynamicPtReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { // Follow the forward link. mReadingState.mPos += forwardLinkPosition; nextPtNodeArray(); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h index 197027313..a69490943 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H +#ifndef LATINIME_DYNAMIC_PT_READING_HELPER_H +#define LATINIME_DYNAMIC_PT_READING_HELPER_H #include <cstddef> #include <vector> @@ -34,8 +34,7 @@ class DictionaryShortcutsStructurePolicy; * This class is used for traversing dynamic patricia trie. This class supports iterating nodes and * dealing with additional buffer. This class counts nodes and node arrays to avoid infinite loop. */ -// TODO: Move to pt_common. -class DynamicPatriciaTrieReadingHelper { +class DynamicPtReadingHelper { public: class TraversingEventListener { public: @@ -60,12 +59,12 @@ class DynamicPatriciaTrieReadingHelper { DISALLOW_COPY_AND_ASSIGN(TraversingEventListener); }; - DynamicPatriciaTrieReadingHelper(const BufferWithExtendableBuffer *const buffer, + DynamicPtReadingHelper(const BufferWithExtendableBuffer *const buffer, const PtNodeReader *const ptNodeReader) : mIsError(false), mReadingState(), mBuffer(buffer), mPtNodeReader(ptNodeReader), mReadingStateStack() {} - ~DynamicPatriciaTrieReadingHelper() {} + ~DynamicPtReadingHelper() {} AK_FORCE_INLINE bool isError() const { return mIsError; @@ -205,7 +204,7 @@ class DynamicPatriciaTrieReadingHelper { const bool forceLowerCaseSearch); private: - DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieReadingHelper); + DISALLOW_COPY_AND_ASSIGN(DynamicPtReadingHelper); // This class encapsulates the reading state of a position in the dictionary. It points at a // specific PtNode in the dictionary. @@ -267,4 +266,4 @@ class DynamicPatriciaTrieReadingHelper { } }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H */ +#endif /* LATINIME_DYNAMIC_PT_READING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp index 6cd127487..3586b50ab 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp @@ -14,39 +14,38 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { -typedef DynamicPatriciaTrieReadingUtils DptReadingUtils; - -const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::MASK_MOVED = 0xC0; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_MOVED = 0x40; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_DELETED = 0x80; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; // TODO: Make DICT_OFFSET_ZERO_OFFSET = 0. // Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum // value of offsets, which is 0x7FFFFF is used to represent 0 offset. -const int DptReadingUtils::DICT_OFFSET_INVALID = 0; -const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; +const int DynamicPtReadingUtils::DICT_OFFSET_INVALID = 0; +const int DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; -/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, +/* static */ int DynamicPtReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } -/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( +/* static */ int DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } -/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) { +/* static */ int DynamicPtReadingUtils::getParentPtNodePos(const int parentOffset, + const int ptNodePos) { if (parentOffset == DICT_OFFSET_INVALID) { return NOT_A_DICT_POS; } else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) { @@ -56,7 +55,7 @@ const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; } } -/* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( +/* static */ int DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const int base = *pos; const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h index 25b0333a5..89ae12c0b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H +#ifndef LATINIME_DYNAMIC_PT_READING_UTILS_H +#define LATINIME_DYNAMIC_PT_READING_UTILS_H #include <stdint.h> @@ -23,7 +23,7 @@ namespace latinime { -class DynamicPatriciaTrieReadingUtils { +class DynamicPtReadingUtils { public: typedef uint8_t NodeFlags; @@ -71,7 +71,7 @@ class DynamicPatriciaTrieReadingUtils { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieReadingUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtReadingUtils); static const NodeFlags MASK_MOVED; static const NodeFlags FLAG_IS_NOT_MOVED; @@ -80,4 +80,4 @@ class DynamicPatriciaTrieReadingUtils { static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL; }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */ +#endif /* LATINIME_DYNAMIC_PT_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp index f12a8995e..2457b49c8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp @@ -14,21 +14,21 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { -const int DynamicPatriciaTrieUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; +const int DynamicPtUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; -bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( - DynamicPatriciaTrieReadingHelper *const readingHelper, +bool DynamicPtUpdatingHelper::addUnigramWord( + DynamicPtReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability, const bool isNotAWord, const bool isBlacklisted, const int timestamp, bool *const outAddedNewUnigram) { @@ -86,7 +86,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( isNotAWord, isBlacklisted, probability, timestamp, &pos); } -bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, +bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, const int probability, const int timestamp, bool *const outAddedNewBigram) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); @@ -97,7 +97,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const } // Remove a bigram relation from word0Pos to word1Pos. -bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { +bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); const PtNodeParams targetPtNodeParams( @@ -105,7 +105,7 @@ bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, co return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams); } -bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, +bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos, const int *const targetCodePoints, const int targetCodePointCount, const int shortcutProbability) { const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos)); @@ -113,12 +113,12 @@ bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, shortcutProbability); } -bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, +bool DynamicPtUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, int *const forwardLinkFieldPos) { const int newPtNodeArrayPos = mBuffer->getTailPosition(); - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, newPtNodeArrayPos, forwardLinkFieldPos)) { return false; } @@ -126,7 +126,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const isNotAWord, isBlacklisted, probability, timestamp); } -bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( +bool DynamicPtUpdatingHelper::setPtNodeProbability( const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, bool *const outAddedNewUnigram) { @@ -154,7 +154,7 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( return true; } -bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( +bool DynamicPtUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( const PtNodeParams *const parentPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, const int *const codePoints, const int codePointCount) { @@ -166,12 +166,12 @@ bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode codePointCount, isNotAWord, isBlacklisted, probability, timestamp); } -bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( +bool DynamicPtUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp) { int writingPos = mBuffer->getTailPosition(); - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, 1 /* arraySize */, &writingPos)) { return false; } @@ -182,7 +182,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( &writingPos)) { return false; } - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } @@ -190,7 +190,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( } // Returns whether the dictionary updating was succeeded or not. -bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( +bool DynamicPtUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probabilityOfNewPtNode, const int timestamp, const int *const newNodeCodePoints, const int newNodeCodePointCount) { @@ -227,7 +227,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( const int actualChildrenPos = writingPos; // Create new children PtNode array. const size_t newPtNodeCount = addsExtraChild ? 2 : 1; - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, newPtNodeCount, &writingPos)) { return false; } @@ -252,7 +252,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( return false; } } - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } @@ -268,7 +268,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos); } -const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( +const PtNodeParams DynamicPtUpdatingHelper::getUpdatedPtNodeParams( const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const int parentPos, const int codePointCount, const int *const codePoints, const int probability) const { @@ -280,7 +280,7 @@ const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( probability); } -const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getPtNodeParamsForNewPtNode( +const PtNodeParams DynamicPtUpdatingHelper::getPtNodeParamsForNewPtNode( const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const int parentPos, const int codePointCount, const int *const codePoints, const int probability) const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h index f02635fe2..71f473096 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H +#ifndef LATINIME_DYNAMIC_PT_UPDATING_HELPER_H +#define LATINIME_DYNAMIC_PT_UPDATING_HELPER_H #include <stdint.h> @@ -26,21 +26,20 @@ namespace latinime { class BufferWithExtendableBuffer; -class DynamicPatriciaTrieReadingHelper; +class DynamicPtReadingHelper; class PtNodeReader; class PtNodeWriter; -// TODO: Move to pt_common. -class DynamicPatriciaTrieUpdatingHelper { +class DynamicPtUpdatingHelper { public: - DynamicPatriciaTrieUpdatingHelper(BufferWithExtendableBuffer *const buffer, + DynamicPtUpdatingHelper(BufferWithExtendableBuffer *const buffer, const PtNodeReader *const ptNodeReader, PtNodeWriter *const ptNodeWriter) : mBuffer(buffer), mPtNodeReader(ptNodeReader), mPtNodeWriter(ptNodeWriter) {} - ~DynamicPatriciaTrieUpdatingHelper() {} + ~DynamicPtUpdatingHelper() {} // Add a word to the dictionary. If the word already exists, update the probability. - bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper, + bool addUnigramWord(DynamicPtReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability, const bool isNotAWord, const bool isBlacklisted, const int timestamp, bool *const outAddedNewUnigram); @@ -57,7 +56,7 @@ class DynamicPatriciaTrieUpdatingHelper { const int targetCodePointCount, const int shortcutProbability); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieUpdatingHelper); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtUpdatingHelper); static const int CHILDREN_POSITION_FIELD_SIZE; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp index 45eeefd4c..ebbdc2ea2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include <cstddef> #include <cstdlib> @@ -24,18 +24,18 @@ namespace latinime { -const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; -const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; -const int DynamicPatriciaTrieWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; -const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; -const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; -const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; -const int DynamicPatriciaTrieWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; -const int DynamicPatriciaTrieWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; -const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; -const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; +const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; +const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; +const int DynamicPtWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; +const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; +const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; +const int DynamicPtWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; +const int DynamicPtWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; +const int DynamicPtWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; +const int DynamicPtWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; +const int DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE = 1; -/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( +/* static */ bool DynamicPtWritingUtils::writeEmptyDictionary( BufferWithExtendableBuffer *const buffer, const int rootPos) { int writingPos = rootPos; if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) { @@ -45,13 +45,13 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; &writingPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos) { return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const size_t arraySize, int *const arraySizeFieldPos) { // Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to @@ -73,20 +73,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; } } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeFlagsAndAdvancePosition( BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { + const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos); } // Note that parentOffset is offset from node's head position. -/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos, int *const parentPosFieldPos) { return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeCodePointsAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int *const codePoints, const int codePointCount, int *const codePointFieldPos) { if (codePointCount <= 0) { @@ -100,21 +100,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; hasMultipleCodePoints, codePointFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int childrenPosition, int *const childrenPositionFieldPos) { return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos), childrenPositionFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset( - BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos, - int *const offsetFieldPos) { +/* static */ bool DynamicPtWritingUtils::writeDictOffset(BufferWithExtendableBuffer *const buffer, + const int targetPos, const int basePos, int *const offsetFieldPos) { int offset = targetPos - basePos; if (targetPos == NOT_A_DICT_POS) { - offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; + offset = DynamicPtReadingUtils::DICT_OFFSET_INVALID; } else if (offset == 0) { - offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; + offset = DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET; } if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) { AKLOGI("offset cannot be written because the offset is too large or too small: %d", diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h index 43e8ea649..362fbd1cc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h @@ -14,19 +14,19 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H +#ifndef LATINIME_DYNAMIC_PT_WRITING_UTILS_H +#define LATINIME_DYNAMIC_PT_WRITING_UTILS_H #include <cstddef> #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" namespace latinime { class BufferWithExtendableBuffer; -class DynamicPatriciaTrieWritingUtils { +class DynamicPtWritingUtils { public: static const int NODE_FLAG_FIELD_SIZE; @@ -40,14 +40,14 @@ class DynamicPatriciaTrieWritingUtils { const size_t arraySize, int *const arraySizeFieldPos); static bool writeFlags(BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, + const DynamicPtReadingUtils::NodeFlags nodeFlags, const int nodeFlagsFieldPos) { int writingPos = nodeFlagsFieldPos; return writeFlagsAndAdvancePosition(buffer, nodeFlags, &writingPos); } static bool writeFlagsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, + const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos); static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, @@ -60,7 +60,7 @@ class DynamicPatriciaTrieWritingUtils { const int childrenPosition, int *const childrenPositionFieldPos); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtWritingUtils); static const size_t MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD; static const size_t MAX_PTNODE_ARRAY_SIZE; @@ -76,4 +76,4 @@ class DynamicPatriciaTrieWritingUtils { const int basePos, int *const offsetFieldPos); }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */ +#endif /* LATINIME_DYNAMIC_PT_WRITING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h index 2a43e39bf..84731eb17 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h @@ -20,8 +20,8 @@ #include <cstring> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" namespace latinime { @@ -111,11 +111,11 @@ class PtNodeParams { // Flags AK_FORCE_INLINE bool isDeleted() const { - return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags); + return DynamicPtReadingUtils::isDeleted(mFlags); } AK_FORCE_INLINE bool willBecomeNonTerminal() const { - return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags); + return DynamicPtReadingUtils::willBecomeNonTerminal(mFlags); } AK_FORCE_INLINE bool hasChildren() const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h index 8420ee95a..b28f58336 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h @@ -23,6 +23,7 @@ namespace latinime { +// TODO: Move to pt_common class PatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h index 95ee74f99..ba2a05209 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h @@ -27,9 +27,9 @@ namespace latinime { class BigramDictContent : public SparseTableDictContent { public: - BigramDictContent(const char *const dictDirPath, const bool hasHistoricalInfo, + BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo, const bool isUpdatable) - : SparseTableDictContent(dictDirPath, + : SparseTableDictContent(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable, @@ -73,8 +73,8 @@ class BigramDictContent : public SparseTableDictContent { bool copyBigramList(const int bigramListPos, const int toPos); - bool flushToFile(const char *const dictBasePath) const { - return flush(dictBasePath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, + bool flushToFile(const char *const dictPath) const { + return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_FILE_EXTENSION); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp index 749e3fe8c..3b7c70efd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp @@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId, return writeEntry(probabilityEntry, entryPos); } -bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const { +bool ProbabilityDictContent::flushToFile(const char *const dictPath) const { if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo); for (int i = 0; i < mSize; ++i) { @@ -81,10 +81,10 @@ bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const { return false; } } - return probabilityDictContentToWrite.flush(dictBasePath, + return probabilityDictContentToWrite.flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION); } else { - return flush(dictBasePath, Ver4DictConstants::FREQ_FILE_EXTENSION); + return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h index db96f9082..b065bc954 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h @@ -29,9 +29,9 @@ class ProbabilityEntry; class ProbabilityDictContent : public SingleDictContent { public: - ProbabilityDictContent(const char *const dictDirPath, const bool hasHistoricalInfo, + ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo, const bool isUpdatable) - : SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable), + : SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable), mHasHistoricalInfo(hasHistoricalInfo), mSize(getBuffer()->getTailPosition() / getEntrySize()) {} @@ -42,7 +42,7 @@ class ProbabilityDictContent : public SingleDictContent { bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry); - bool flushToFile(const char *const dictDirPath) const; + bool flushToFile(const char *const dictPath) const; bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ProbabilityDictContent *const originalProbabilityDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp index 555217837..29972a4e8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp @@ -46,8 +46,8 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const { return addressLookupTable->get(terminalId); } -bool ShortcutDictContent::flushToFile(const char *const dictBasePath) const { - return flush(dictBasePath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, +bool ShortcutDictContent::flushToFile(const char *const dictPath) const { + return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_FILE_EXTENSION); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h index a52214ca2..eaafc27bc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h @@ -26,8 +26,8 @@ namespace latinime { class ShortcutDictContent : public SparseTableDictContent { public: - ShortcutDictContent(const char *const dictDirPath, const bool isUpdatable) - : SparseTableDictContent(dictDirPath, + ShortcutDictContent(const char *const dictPath, const bool isUpdatable) + : SparseTableDictContent(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable, @@ -53,7 +53,7 @@ class ShortcutDictContent : public SparseTableDictContent { // Returns head position of shortcut list for a PtNode specified by terminalId. int getShortcutListHeadPos(const int terminalId) const; - bool flushToFile(const char *const dictBasePath) const; + bool flushToFile(const char *const dictPath) const; bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ShortcutDictContent *const originalShortcutDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index d8eedf36e..9064b7e72 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -28,9 +28,9 @@ namespace latinime { class SingleDictContent : public DictContent { public: - SingleDictContent(const char *const dictDirPath, const char *const contentFileName, + SingleDictContent(const char *const dictPath, const char *const contentFileName, const bool isUpdatable) - : mMmappedBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)), + : mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), mExpandableContentBuffer(mMmappedBuffer.get() ? mMmappedBuffer.get()->getBuffer() : 0, mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), @@ -59,8 +59,8 @@ class SingleDictContent : public DictContent { return &mExpandableContentBuffer; } - bool flush(const char *const dictBasePath, const char *const contentFileNameSuffix) const { - return DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, + bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const { + return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix, &mExpandableContentBuffer); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp index abb7d5fd2..63c6ea3a4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp @@ -18,18 +18,18 @@ namespace latinime { -bool SparseTableDictContent::flush(const char *const dictBasePath, +bool SparseTableDictContent::flush(const char *const dictPath, const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix, const char *const contentFileNameSuffix) const { - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, lookupTableFileNameSuffix, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix, &mExpandableLookupTableBuffer)){ return false; } - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, addressTableFileNameSuffix, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix, &mExpandableAddressTableBuffer)) { return false; } - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, contentFileNameSuffix, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix, &mExpandableContentBuffer)) { return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h index 9a4f1e1c0..a82e3f50a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h @@ -30,15 +30,15 @@ namespace latinime { // TODO: Support multiple contents. class SparseTableDictContent : public DictContent { public: - AK_FORCE_INLINE SparseTableDictContent(const char *const dictDirPath, + AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath, const char *const lookupTableFileName, const char *const addressTableFileName, const char *const contentFileName, const bool isUpdatable, const int sparseTableBlockSize, const int sparseTableDataSize) : mLookupTableBuffer( - MmappedBuffer::openBuffer(dictDirPath, lookupTableFileName, isUpdatable)), + MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)), mAddressTableBuffer( - MmappedBuffer::openBuffer(dictDirPath, addressTableFileName, isUpdatable)), - mContentBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)), + MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)), + mContentBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), mExpandableLookupTableBuffer( mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBuffer() : 0, mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBufferSize() : 0, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp index c889cf5d1..0b17a009d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp @@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition( Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); } -bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) const { +bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const { // If the used buffer size is smaller than the actual buffer size, regenerate the lookup // table and write the new table to the file. if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { @@ -63,12 +63,12 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) co return false; } } - return lookupTableToWrite.flush(dictBasePath, + return lookupTableToWrite.flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } else { // We can simply use this lookup table because the buffer size has not been // changed. - return flush(dictBasePath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h index 5a28f52fd..f73e22754 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h @@ -28,8 +28,8 @@ class TerminalPositionLookupTable : public SingleDictContent { public: typedef hash_map_compat<int, int> TerminalIdMap; - TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable) - : SingleDictContent(dictDirPath, + TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable) + : SingleDictContent(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), mSize(getBuffer()->getTailPosition() / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {} @@ -44,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent { return mSize; } - bool flushToFile(const char *const dictBasePath) const; + bool flushToFile(const char *const dictPath) const; bool runGCTerminalIds(TerminalIdMap *const terminalIdMap); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index e2355407a..918c02ba2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -27,10 +27,10 @@ namespace latinime { /* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( - const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) { + const char *const dictPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) { const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false; // TODO: take only dictDirPath, and open both header and trie files in the constructor below - return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, headerBuffer, isUpdatable)); + return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, headerBuffer, isUpdatable)); } bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, @@ -57,38 +57,38 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */; char dictName[dictNameBufSize]; FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName); - const int dictBasePathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName); - char dictBasePath[dictBasePathBufSize]; - FileUtils::getFilePath(tmpDirPath, dictName, dictBasePathBufSize, dictBasePath); + const int dictPathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName); + char dictPath[dictPathBufSize]; + FileUtils::getFilePath(tmpDirPath, dictName, dictPathBufSize, dictPath); // Write header file. - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) { AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath, Ver4DictConstants::HEADER_FILE_EXTENSION); return false; } // Write trie file. - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) { AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath, Ver4DictConstants::TRIE_FILE_EXTENSION); return false; } // Write dictionary contents. - if (!mTerminalPositionLookupTable.flushToFile(dictBasePath)) { + if (!mTerminalPositionLookupTable.flushToFile(dictPath)) { AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); return false; } - if (!mProbabilityDictContent.flushToFile(dictBasePath)) { + if (!mProbabilityDictContent.flushToFile(dictPath)) { AKLOGE("Probability dict content cannot be written. %s", tmpDirPath); return false; } - if (!mBigramDictContent.flushToFile(dictBasePath)) { + if (!mBigramDictContent.flushToFile(dictPath)) { AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath); return false; } - if (!mShortcutDictContent.flushToFile(dictBasePath)) { + if (!mShortcutDictContent.flushToFile(dictPath)) { AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath); return false; } @@ -107,10 +107,10 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, return true; } -Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath, +Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable) : mHeaderBuffer(headerBuffer), - mDictBuffer(MmappedBuffer::openBuffer(dictDirPath, + mDictBuffer(MmappedBuffer::openBuffer(dictPath, Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4), mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), @@ -118,12 +118,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath, mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(), mDictBuffer.get()->getBufferSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mTerminalPositionLookupTable(dictDirPath, isUpdatable), - mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), + mTerminalPositionLookupTable(dictPath, isUpdatable), + mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), - mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), + mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), - mShortcutDictContent(dictDirPath, isUpdatable), + mShortcutDictContent(dictPath, isUpdatable), mIsUpdatable(isUpdatable) {} Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy) diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index c16e0bdcb..d6d22c5c1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -21,6 +21,7 @@ namespace latinime { +// TODO: Create PtConstants under the pt_common and move some constant values there. // Note that there are corresponding definitions in FormatSpec.java. class Ver4DictConstants { public: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 8ea029076..17fc9483b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -16,8 +16,8 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" @@ -45,10 +45,10 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce const PatriciaTrieReadingUtils::NodeFlags flags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const int parentPosOffset = - DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( + DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( dictBuf, &pos); const int parentPos = - DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, headPos); + DynamicPtReadingUtils::getParentPtNodePos(parentPosOffset, headPos); int codePoints[MAX_WORD_LENGTH]; const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos); @@ -74,7 +74,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce if (usesAdditionalBuffer) { childrenPosFieldPos += mBuffer->getOriginalBufferSize(); } - int childrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( + int childrenPos = DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, &pos); if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) { childrenPos += mBuffer->getOriginalBufferSize(); @@ -85,7 +85,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce // Sibling position is the tail position of original PtNode. int newSiblingNodePos = (siblingNodePos == NOT_A_DICT_POS) ? pos : siblingNodePos; // Read destination node if the read node is a moved node. - if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) { + if (DynamicPtReadingUtils::isMoved(flags)) { // The destination position is stored at the same place as the parent position. return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos); } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 10f90523a..32576cf0a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -19,11 +19,11 @@ #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" @@ -44,11 +44,11 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, true /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos)) { return false; } @@ -74,16 +74,16 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, false /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos)) { return false; } // Update moved position, which is stored in the parent offset field. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { return false; } @@ -93,8 +93,8 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( while (!mReadingHelper.isEnd()) { const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams()); int parentOffsetFieldPos = childPtNodeParams.getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(), &parentOffsetFieldPos)) { // Parent offset cannot be written because of a bug or a broken dictionary; thus, @@ -119,7 +119,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, false /* isDeleted */, true /* willBecomeNonTerminal */); if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) { @@ -129,7 +129,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( } // Update flags. int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); - return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + return DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos); } @@ -186,7 +186,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA bool Ver4PatriciaTrieNodeWriter::updateChildrenPosition( const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) { int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos(); - return DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, + return DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, newChildrenPosition, &childrenPosFieldPos); } @@ -264,9 +264,9 @@ bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields( } } int writingPos = toBeUpdatedPtNodeParams->getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; // Write updated parent offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { return false; } @@ -328,17 +328,17 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( const int nodePos = *ptNodeWritingPos; // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // PtNode writing. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, 0 /* nodeFlags */, ptNodeWritingPos)) { return false; } // Calculate a parent offset and write the offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) { return false; } // Write code points - if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) { return false; } @@ -369,7 +369,7 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( } } // Write children position - if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, ptNodeParams->getChildrenPos(), ptNodeWritingPos)) { return false; } @@ -401,7 +401,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos, PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal, hasShortcutTargets, hasBigrams, hasMultipleChars, CHILDREN_POSITION_FIELD_SIZE); - if (!DynamicPatriciaTrieWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { + if (!DynamicPtWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos); return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index 1df853a30..69576d8e5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -20,9 +20,9 @@ #include <stdint.h> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" @@ -115,7 +115,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { BufferWithExtendableBuffer *const mTrieBuffer; Ver4DictBuffers *const mBuffers; const Ver4PatriciaTrieNodeReader *const mPtNodeReader; - DynamicPatriciaTrieReadingHelper mReadingHelper; + DynamicPtReadingHelper mReadingHelper; Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 7b5461d74..96bb8128e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -21,7 +21,7 @@ #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_vector.h" #include "suggest/core/dictionary/unigram_property.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" @@ -48,7 +48,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d if (!dicNode->hasChildren()) { return; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos()); while (!readingHelper.isEnd()) { const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams(); @@ -75,7 +75,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) const { - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodePos(ptNodePos); return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount( maxCodePointCount, outCodePoints, outUnigramProbability); @@ -83,7 +83,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) const { - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); } @@ -154,7 +154,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len mDictBuffer->getTailPosition()); return false; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); bool addedNewUnigram = false; if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, isNotAWord, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 2d46c0ae2..8187b7a39 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -22,7 +22,7 @@ #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" @@ -131,7 +131,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { Ver4ShortcutListPolicy mShortcutPolicy; Ver4PatriciaTrieNodeReader mNodeReader; Ver4PatriciaTrieNodeWriter mNodeWriter; - DynamicPatriciaTrieUpdatingHelper mUpdatingHelper; + DynamicPtUpdatingHelper mUpdatingHelper; Ver4PatriciaTrieWritingHelper mWritingHelper; int mUnigramCount; int mBigramCount; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 2ee9483f7..43227635c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -32,12 +32,9 @@ namespace latinime { -void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePath, +void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath, const int unigramCount, const int bigramCount) const { const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy(); - const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */; - char dirPath[dirPathBufSize]; - FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath); BufferWithExtendableBuffer headerBuffer( BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE); const int extendedRegionSize = headerPolicy->getExtendedRegionSize() @@ -50,11 +47,11 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa extendedRegionSize); return; } - mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer); + mBuffers->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer); } void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, - const char *const trieFilePath) { + const char *const dictDirPath) { const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy(); Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( Ver4DictBuffers::createVer4DictBuffers(headerPolicy)); @@ -70,10 +67,7 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeAr 0 /* extendedRegionSize */)) { return; } - const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */; - char dirPath[dirPathBufSize]; - FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath); - dictBuffers.get()->flushHeaderAndDictBuffers(dirPath, &headerBuffer); + dictBuffers.get()->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer); } bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, @@ -88,9 +82,9 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy); - DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); + DynamicPtReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners + DynamicPtGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( &ptNodeWriter); @@ -111,7 +105,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, } readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability + DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability traversePolicyToUpdateBigramProbability(&ptNodeWriter); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( &traversePolicyToUpdateBigramProbability)) { @@ -132,7 +126,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(), buffersToWrite, &ptNodeReader, &bigramPolicy, &shortcutPolicy); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer + DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers, buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap); if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( @@ -170,10 +164,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, mBuffers->getShortcutDictContent())) { return false; } - DynamicPatriciaTrieReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), + DynamicPtReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), &newPtNodeReader); newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields + DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields traversePolicyToUpdateAllPositionFields(&newPtNodeWriter, &dictPositionRelocationMap); if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( &traversePolicyToUpdateAllPositionFields)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h index 73e63ef45..c3a155e0e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h @@ -18,7 +18,7 @@ #define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" namespace latinime { @@ -33,17 +33,16 @@ class Ver4PatriciaTrieWritingHelper { Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers) : mBuffers(buffers) {} - void writeToDictFile(const char *const trieFilePath, const int unigramCount, + void writeToDictFile(const char *const dictDirPath, const int unigramCount, const int bigramCount) const; - void writeToDictFileWithGC(const int rootPtNodeArrayPos, - const char *const trieFilePath); + void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const dictDirPath); private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); class TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds( Ver4PatriciaTrieNodeWriter *const ptNodeWriter, diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index b7db21365..442373b29 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -19,7 +19,7 @@ #include <cstdio> #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" @@ -51,7 +51,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); - if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( + if (!DynamicPtWritingUtils::writeEmptyDictionary( dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) { AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp index 49ae7f156..1f25cfa1e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp @@ -147,7 +147,7 @@ namespace latinime { const char *const baseName = basename(filePathBuf); const int baseNameLength = strlen(baseName); if (baseNameLength >= outNameBufSize) { - AKLOGE("outNameBufSize is too small. dirPath: %s, outNameBufSize: %d", + AKLOGE("outNameBufSize is too small. filePath: %s, outNameBufSize: %d", filePath, outNameBufSize); return; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp index e3d783835..cd3c403fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp @@ -46,8 +46,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12; // same so we use them for both here. if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) { return VERSION_2; - } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_3) { - return VERSION_3; } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) { return VERSION_4; } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 34727b8d8..eb2227d60 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -31,7 +31,6 @@ class FormatUtils { enum FORMAT_VERSION { // These MUST have the same values as the relevant constants in FormatSpec.java. VERSION_2 = 2, - VERSION_3 = 3, VERSION_4 = 400, UNKNOWN_VERSION = -1 }; diff --git a/native/jni/src/utils/exclusive_ownership_pointer.h b/native/jni/src/utils/exclusive_ownership_pointer.h index 617b34968..081802e8b 100644 --- a/native/jni/src/utils/exclusive_ownership_pointer.h +++ b/native/jni/src/utils/exclusive_ownership_pointer.h @@ -39,25 +39,15 @@ class ExclusiveOwnershipPointer { deletePointersIfHavingOwnership(); } - // Move the ownership. - AK_FORCE_INLINE ExclusiveOwnershipPointer<T> &operator=( - const ExclusiveOwnershipPointer<T> &pointer) { - // Delete pointers when this is an owner of another pointer. - deletePointersIfHavingOwnership(); - mPointer = pointer.mPointer; - mSharedOwnerPtr = pointer.mSharedOwnerPtr; - transferOwnership(pointer); - return *this; - } - AK_FORCE_INLINE T *get() const { return mPointer; } private: - // This class allows to copy and assign and ensures only one instance has the ownership of the + // This class allows to copy and ensures only one instance has the ownership of the // managed pointer. DISALLOW_DEFAULT_CONSTRUCTOR(ExclusiveOwnershipPointer); + DISALLOW_ASSIGNMENT_OPERATOR(ExclusiveOwnershipPointer); void transferOwnership(const ExclusiveOwnershipPointer<T> *const src) { if (*mSharedOwnerPtr != src) { diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index 931ba7d3c..c7bc55d96 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -114,10 +114,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); FileUtils.deleteRecursively(file); - file.mkdir(); Map<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE, @@ -125,7 +122,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION; if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, attributeMap)) { - return new File(file, headerFileName); + return file; } else { throw new IOException("Empty dictionary " + file.getAbsolutePath() + " " + headerFileName + " cannot be created."); diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index d9d4a5584..a26c25886 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -25,6 +25,7 @@ import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.UnigramProperty; import java.io.File; @@ -68,15 +69,12 @@ public class BinaryDictionaryTests extends AndroidTestCase { file.delete(); file.mkdir(); Map<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION; if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, attributeMap)) { - return new File(file, headerFileName); + return file; } else { - throw new IOException("Empty dictionary " + file.getAbsolutePath() + " " - + headerFileName + " cannot be created."); + throw new IOException("Empty dictionary " + file.getAbsolutePath() + + " cannot be created."); } } @@ -99,7 +97,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.close(); assertFalse("binaryDictionary must be invalid after closing.", binaryDictionary.isValidDictionary()); - dictFile.delete(); + FileUtils.deleteRecursively(dictFile); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index cadd0f8f3..e211d94b8 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -31,7 +31,7 @@ import java.util.HashMap; public class FusionDictionaryTests extends AndroidTestCase { public void testFindWordInTree() { FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + new FusionDictionary.DictionaryOptions(new HashMap<String,String>())); dict.add("abc", 10, null, false /* isNotAWord */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index e3ec2eca9..aff045c2b 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -226,8 +226,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatSpec.FormatOptions formatOptions) { String result = " : buffer type = " + ((bufferType == BinaryDictUtils.USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); - result += " : version = " + formatOptions.mVersion; - return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; + return result + " : version = " + formatOptions.mVersion; } // Tests for readDictionaryBinary and writeDictionaryBinary @@ -315,17 +314,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final List<String> results = CollectionUtils.newArrayList(); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION2); + BinaryDictUtils.VERSION2_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -336,17 +329,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final List<String> results = CollectionUtils.newArrayList(); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION2); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -472,17 +459,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final ArrayList<String> results = CollectionUtils.newArrayList(); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION2); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -493,17 +474,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final ArrayList<String> results = CollectionUtils.newArrayList(); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION2); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -612,29 +587,19 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testGetTerminalPosition() { final ArrayList<String> results = CollectionUtils.newArrayList(); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -668,7 +633,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testDeleteWord() throws IOException, UnsupportedFormatException { - runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(BinaryDictUtils.VERSION3_OPTIONS); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index 308919499..cdbcc832f 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -114,7 +114,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { formatOptions); printPtNode(currentInfo); } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); Log.d(TAG, " forwardLinkAddress = " + forwardLinkAddress); } @@ -289,8 +289,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testInsertWord() { - runTestInsertWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestInsertWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestInsertWord(BinaryDictUtils.VERSION3_OPTIONS); + runTestInsertWord(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runTestInsertWord(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); } private void runTestInsertWordWithBigrams(final FormatOptions formatOptions) { @@ -329,8 +330,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testInsertWordWithBigrams() { - runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_OPTIONS); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); } private void runTestRandomWords(final FormatOptions formatOptions) { @@ -377,7 +379,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testRandomWords() { - runTestRandomWords(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestRandomWords(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestRandomWords(BinaryDictUtils.VERSION3_OPTIONS); + runTestRandomWords(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runTestRandomWords(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java index ad17a7118..34226dcd3 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java @@ -29,23 +29,17 @@ public class BinaryDictUtils { public static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - public static final FormatSpec.FormatOptions VERSION2 = + public static final FormatSpec.FormatOptions VERSION2_OPTIONS = new FormatSpec.FormatOptions(FormatSpec.VERSION2); - public static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION3, false /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION3, true /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION4, false /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = - new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* supportsDynamicUpdate */, - true /* hasTimestamp */); + public static final FormatSpec.FormatOptions VERSION3_OPTIONS = + new FormatSpec.FormatOptions(FormatSpec.VERSION3); + public static final FormatSpec.FormatOptions VERSION4_OPTIONS_WITHOUT_TIMESTAMP = + new FormatSpec.FormatOptions(FormatSpec.VERSION4, false /* hasTimestamp */); + public static final FormatSpec.FormatOptions VERSION4_OPTIONS_WITH_TIMESTAMP = + new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* hasTimestamp */); public static DictionaryOptions makeDictionaryOptions(final String id, final String version) { - final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */); + final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>()); options.mAttributes.put(FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, "en_US"); options.mAttributes.put(FileHeader.DICTIONARY_ID_ATTRIBUTE, id); options.mAttributes.put(FileHeader.DICTIONARY_VERSION_ATTRIBUTE, version); diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index f0fe3f2e1..17423a7ba 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -20,6 +20,7 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; @@ -42,9 +43,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" }; - private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000; - private static final int WAIT_TERMINATING_IN_MILLISECONDS = 100; - /** * Generates a random word. */ @@ -70,8 +68,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private void addToDict(final UserHistoryDictionary dict, final List<String> words) { String prevWord = null; for (String word : words) { - // TODO: Use timestamp properly. - dict.addToDictionary(prevWord, word, true, 0 /* timestamp */); + dict.addToDictionary(prevWord, word, true, + (int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())); prevWord = word; } } @@ -89,10 +87,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { // Add random words to the user history dictionary. addToDict(dict, words); if (checkContents) { - try { - Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); - } catch (InterruptedException e) { - } + dict.waitAllTasksForTests(); for (int i = 0; i < numberOfWords; ++i) { final String word = words.get(i); assertTrue(dict.isInDictionaryForTests(word)); @@ -119,24 +114,18 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { * @param testFilenameSuffix file name suffix used for testing. */ private void waitForWriting(final String testFilenameSuffix) { - try { - final UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - new Locale(testFilenameSuffix)); - dict.shutdownExecutorForTests(); - while (!dict.isTerminatedForTests()) { - Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); - } - } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: ", e); - } + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), + new Locale(testFilenameSuffix)); + dict.waitAllTasksForTests(); } public void testRandomWords() { Log.d(TAG, "This test can be used for profiling."); Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); final String testFilenameSuffix = "test_random_words" + System.currentTimeMillis(); - final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix; + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; final int numberOfWords = 1000; final Random random = new Random(123456); @@ -151,7 +140,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final File dictFile = new File(getContext().getFilesDir(), fileName); if (dictFile != null) { assertTrue(dictFile.exists()); - assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); dictFile.delete(); } } @@ -170,7 +158,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { testFilenameSuffixes[i] = "test_switching_languages" + i; - final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i]; + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i] + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFiles[i] = new File(getContext().getFilesDir(), fileName); clearHistory(testFilenameSuffixes[i]); } @@ -196,7 +185,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { for (final File file : dictFiles) { if (file != null) { assertTrue(file.exists()); - assertTrue(file.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); file.delete(); } } @@ -214,11 +202,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } finally { Log.d(TAG, "waiting for writing ..."); waitForWriting(testFilenameSuffix); - final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix; + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; final File dictFile = new File(getContext().getFilesDir(), fileName); if (dictFile != null) { assertTrue(dictFile.exists()); - assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); dictFile.delete(); } } diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java index 21fcf1117..2123e84e8 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.Locale; @SmallTest -public class StringUtilsTests extends AndroidTestCase { +public class StringAndJsonUtilsTests extends AndroidTestCase { public void testContainsInArray() { assertFalse("empty array", StringUtils.containsInArray("key", new String[0])); assertFalse("not in 1 element", StringUtils.containsInArray("key", new String[] { @@ -292,11 +292,11 @@ public class StringUtilsTests extends AndroidTestCase { assertTrue(bytesStr.equals(bytesStr2)); } - public void testJsonStringUtils() { + public void testJsonUtils() { final Object[] objs = new Object[] { 1, "aaa", "bbb", 3 }; final List<Object> objArray = Arrays.asList(objs); - final String str = StringUtils.listToJsonStr(objArray); - final List<Object> newObjArray = StringUtils.jsonStrToList(str); + final String str = JsonUtils.listToJsonStr(objArray); + final List<Object> newObjArray = JsonUtils.jsonStrToList(str); for (int i = 0; i < objs.length; ++i) { assertEquals(objs[i], newObjArray.get(i)); } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java index 4b6716936..16f82dafd 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java @@ -50,8 +50,6 @@ public class CombinedInputOutput { private static final String NOT_A_WORD_TAG = "not_a_word"; private static final String WHITELIST_TAG = "whitelist"; private static final String OPTIONS_TAG = "options"; - private static final String GERMAN_UMLAUT_PROCESSING_OPTION = "german_umlaut_processing"; - private static final String FRENCH_LIGATURE_PROCESSING_OPTION = "french_ligature_processing"; private static final String COMMENT_LINE_STARTER = "#"; /** @@ -112,13 +110,9 @@ public class CombinedInputOutput { attributes.put(keyValue[0], keyValue[1]); } - final boolean processUmlauts = - GERMAN_UMLAUT_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); - final boolean processLigatures = - FRENCH_LIGATURE_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); attributes.remove(OPTIONS_TAG); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions( - attributes, processUmlauts, processLigatures)); + final FusionDictionary dict = + new FusionDictionary(new PtNodeArray(), new DictionaryOptions(attributes)); String line; String word = null; @@ -216,11 +210,6 @@ public class CombinedInputOutput { destination.write(options.get(DICTIONARY_TAG)); options.remove(DICTIONARY_TAG); } - if (dict.mOptions.mGermanUmlautProcessing) { - destination.write("," + OPTIONS_TAG + "=" + GERMAN_UMLAUT_PROCESSING_OPTION); - } else if (dict.mOptions.mFrenchLigatureProcessing) { - destination.write("," + OPTIONS_TAG + "=" + FRENCH_LIGATURE_PROCESSING_OPTION); - } for (final String key : dict.mOptions.mAttributes.keySet()) { final String value = dict.mOptions.mAttributes.get(key); destination.write("," + key + "=" + value); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java index 66fd084cd..7ac3c67a1 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java @@ -85,18 +85,6 @@ public class Diff extends Dicttool.Command { private static void diffHeaders(final FusionDictionary dict0, final FusionDictionary dict1) { boolean hasDifferences = false; - if (dict0.mOptions.mFrenchLigatureProcessing != dict1.mOptions.mFrenchLigatureProcessing) { - System.out.println(" French ligature processing : " - + dict0.mOptions.mFrenchLigatureProcessing + " <=> " - + dict1.mOptions.mFrenchLigatureProcessing); - hasDifferences = true; - } - else if (dict0.mOptions.mGermanUmlautProcessing != dict1.mOptions.mGermanUmlautProcessing) { - System.out.println(" German umlaut processing : " - + dict0.mOptions.mGermanUmlautProcessing + " <=> " - + dict1.mOptions.mGermanUmlautProcessing); - hasDifferences = true; - } final HashMap<String, String> options1 = new HashMap<String, String>(dict1.mOptions.mAttributes); for (final String optionKey : dict0.mOptions.mAttributes.keySet()) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java index 4e99bf979..d226251c1 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java @@ -57,8 +57,6 @@ public class XmlDictInputOutput { private static final String NOT_A_WORD_ATTR = "not_a_word"; private static final String OPTIONS_KEY = "options"; - private static final String GERMAN_UMLAUT_PROCESSING_OPTION = "german_umlaut_processing"; - private static final String FRENCH_LIGATURE_PROCESSING_OPTION = "french_ligature_processing"; /** * SAX handler for a unigram XML file. @@ -120,12 +118,8 @@ public class XmlDictInputOutput { attributes.put(attrName, attrs.getValue(attrIndex)); } final String optionsString = attributes.get(OPTIONS_KEY); - final boolean processUmlauts = - GERMAN_UMLAUT_PROCESSING_OPTION.equals(optionsString); - final boolean processLigatures = - FRENCH_LIGATURE_PROCESSING_OPTION.equals(optionsString); mDictionary = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(attributes, processUmlauts, processLigatures)); + new DictionaryOptions(attributes)); } else { mState = UNKNOWN; } @@ -361,11 +355,6 @@ public class XmlDictInputOutput { // TODO: use an XMLSerializer if this gets big destination.write("<wordlist format=\"2\""); final HashMap<String, String> options = dict.mOptions.mAttributes; - if (dict.mOptions.mGermanUmlautProcessing) { - destination.write(" " + OPTIONS_KEY + "=\"" + GERMAN_UMLAUT_PROCESSING_OPTION + "\""); - } else if (dict.mOptions.mFrenchLigatureProcessing) { - destination.write(" " + OPTIONS_KEY + "=\"" + FRENCH_LIGATURE_PROCESSING_OPTION + "\""); - } for (final String key : dict.mOptions.mAttributes.keySet()) { final String value = dict.mOptions.mAttributes.get(key); destination.write(" " + key + "=\"" + value + "\""); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 1baeb7a47..2b0a50464 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -44,8 +44,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { public void testGetRawDictWorks() throws IOException, UnsupportedFormatException { // Create a thrice-compressed dictionary file. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(new HashMap<String, String>(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); + new DictionaryOptions(new HashMap<String, String>())); dict.add("foo", TEST_FREQ, null, false /* isNotAWord */); dict.add("fta", 1, null, false /* isNotAWord */); dict.add("ftb", 1, null, false /* isNotAWord */); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java index 55058238c..c6e497615 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java @@ -32,8 +32,7 @@ public class BinaryDictEncoderFlattenTreeTests extends TestCase { // that it does not contain any duplicates. public void testFlattenNodes() { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(new HashMap<String, String>(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); + new DictionaryOptions(new HashMap<String, String>())); dict.add("foo", 1, null, false /* isNotAWord */); dict.add("fta", 1, null, false /* isNotAWord */); dict.add("ftb", 1, null, false /* isNotAWord */); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java index 659650a05..76dadc25c 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java @@ -96,8 +96,7 @@ public class FusionDictionaryTest extends TestCase { // that it does not contain any duplicates. public void testFusion() { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(new HashMap<String, String>(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); + new DictionaryOptions(new HashMap<String, String>())); final long time = System.currentTimeMillis(); prepare(time); for (int i = 0; i < sWords.size(); ++i) { |