diff options
25 files changed, 500 insertions, 276 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 877fff267..2e6c4b2f8 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -87,7 +87,7 @@ public final class BinaryDictIOUtils { if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) { p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer); - p.mAddress += BinaryDictInputOutput.getGroupCountSize(p.mNumOfCharGroup); + p.mAddress += getGroupCountSize(p.mNumOfCharGroup); p.mPosition = 0; } if (p.mNumOfCharGroup == 0) { @@ -101,9 +101,9 @@ public final class BinaryDictIOUtils { } p.mPosition++; - final boolean isMovedGroup = BinaryDictInputOutput.isMovedGroup(info.mFlags, + final boolean isMovedGroup = isMovedGroup(info.mFlags, formatOptions); - final boolean isDeletedGroup = BinaryDictInputOutput.isDeletedGroup(info.mFlags, + final boolean isDeletedGroup = isDeletedGroup(info.mFlags, formatOptions); if (!isMovedGroup && !isDeletedGroup && info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) {// found word @@ -130,7 +130,7 @@ public final class BinaryDictIOUtils { p.mAddress = buffer.position(); } - if (!isMovedGroup && BinaryDictInputOutput.hasChildrenAddress(info.mChildrenAddress)) { + if (!isMovedGroup && hasChildrenAddress(info.mChildrenAddress)) { Position childrenPos = new Position(info.mChildrenAddress + headerSize, index); stack.push(childrenPos); } @@ -187,12 +187,10 @@ public final class BinaryDictIOUtils { final int charGroupPos = buffer.position(); final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, buffer.position(), header.mFormatOptions); - final boolean isMovedGroup = - BinaryDictInputOutput.isMovedGroup(currentInfo.mFlags, - header.mFormatOptions); - final boolean isDeletedGroup = - BinaryDictInputOutput.isDeletedGroup(currentInfo.mFlags, - header.mFormatOptions); + final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, + header.mFormatOptions); + final boolean isDeletedGroup = isDeletedGroup(currentInfo.mFlags, + header.mFormatOptions); if (isMovedGroup) continue; boolean same = true; for (int p = 0, j = word.offsetByCodePoints(0, wordPos); @@ -490,8 +488,8 @@ public final class BinaryDictIOUtils { */ static int writeNode(final OutputStream destination, final CharGroupInfo[] infos) throws IOException { - int size = BinaryDictInputOutput.getGroupCountSize(infos.length); - switch (BinaryDictInputOutput.getGroupCountSize(infos.length)) { + int size = getGroupCountSize(infos.length); + switch (getGroupCountSize(infos.length)) { case 1: destination.write((byte)infos.length); break; @@ -566,4 +564,52 @@ public final class BinaryDictIOUtils { return null; } } + + /** + * Helper method to hide the actual value of the no children address. + */ + public static boolean hasChildrenAddress(final int address) { + return FormatSpec.NO_CHILDREN_ADDRESS != address; + } + + /** + * Helper method to check whether the group is moved. + */ + public static boolean isMovedGroup(final int flags, final FormatOptions options) { + return options.mSupportsDynamicUpdate + && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); + } + + /** + * Helper method to check whether the dictionary can be updated dynamically. + */ + public static boolean supportsDynamicUpdate(final FormatOptions options) { + return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE + && options.mSupportsDynamicUpdate; + } + + /** + * Helper method to check whether the group is deleted. + */ + public static boolean isDeletedGroup(final int flags, final FormatOptions formatOptions) { + return formatOptions.mSupportsDynamicUpdate + && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); + } + + /** + * Compute the binary size of the group count + * @param count the group count + * @return the size of the group count, either 1 or 2 bytes. + */ + public static int getGroupCountSize(final int count) { + if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { + return 1; + } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) { + return 2; + } else { + throw new RuntimeException("Can't have more than " + + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count + + ")"); + } + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java index 5710e14a4..a33cddb49 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java @@ -23,6 +23,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.Node; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.JniUtils; import java.io.ByteArrayOutputStream; import java.io.File; @@ -48,6 +49,13 @@ public final class BinaryDictInputOutput { private static final boolean DBG = MakedictLog.DBG; + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + // Arbitrary limit to how much passes we consider address size compression should // terminate in. At the time of this writing, our largest dictionary completes // compression in five passes. @@ -307,29 +315,12 @@ public final class BinaryDictInputOutput { } /** - * Compute the binary size of the group count - * @param count the group count - * @return the size of the group count, either 1 or 2 bytes. - */ - public static int getGroupCountSize(final int count) { - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { - return 1; - } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) { - return 2; - } else { - throw new RuntimeException("Can't have more than " - + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count - + ")"); - } - } - - /** * Compute the binary size of the group count for a node * @param node the node * @return the size of the group count, either 1 or 2 bytes. */ private static int getGroupCountSize(final Node node) { - return getGroupCountSize(node.mData.size()); + return BinaryDictIOUtils.getGroupCountSize(node.mData.size()); } /** @@ -404,44 +395,13 @@ public final class BinaryDictInputOutput { } /** - * Helper method to hide the actual value of the no children address. - */ - public static boolean hasChildrenAddress(final int address) { - return FormatSpec.NO_CHILDREN_ADDRESS != address; - } - - /** - * Helper method to check whether the group is moved. - */ - public static boolean isMovedGroup(final int flags, final FormatOptions options) { - return options.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); - } - - /** - * Helper method to check whether the group is deleted. - */ - public static boolean isDeletedGroup(final int flags, final FormatOptions formatOptions) { - return formatOptions.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); - } - - /** - * Helper method to check whether the dictionary can be updated dynamically. - */ - public static boolean supportsDynamicUpdate(final FormatOptions options) { - return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.mSupportsDynamicUpdate; - } - - /** * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. * * @param group the group of which to compute the size of the header * @param options file format options. */ private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { - if (supportsDynamicUpdate(options)) { + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + getGroupCharactersSize(group); } else { @@ -461,7 +421,7 @@ public final class BinaryDictInputOutput { */ static int getByteSize(final int address) { assert(address <= FormatSpec.UINT24_MAX); - if (!hasChildrenAddress(address)) { + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { return 0; } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) { return 1; @@ -802,7 +762,7 @@ public final class BinaryDictInputOutput { */ private static int writeVariableSignedAddress(final byte[] buffer, int index, final int address) { - if (!hasChildrenAddress(address)) { + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; } else { final int absAddress = Math.abs(address); @@ -966,7 +926,7 @@ public final class BinaryDictInputOutput { private static final int writeParentAddress(final byte[] buffer, final int index, final int address, final FormatOptions formatOptions) { - if (supportsDynamicUpdate(formatOptions)) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { if (address == FormatSpec.NO_PARENT_ADDRESS) { buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; } else { @@ -1315,7 +1275,7 @@ public final class BinaryDictInputOutput { static int readParentAddress(final FusionDictionaryBufferInterface buffer, final FormatOptions formatOptions) { - if (supportsDynamicUpdate(formatOptions)) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { final int parentAddress = buffer.readUnsignedInt24(); final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; return sign * (parentAddress & FormatSpec.SINT24_MAX); @@ -1332,7 +1292,7 @@ public final class BinaryDictInputOutput { ++addressPointer; final int parentAddress = readParentAddress(buffer, options); - if (supportsDynamicUpdate(options)) { + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { addressPointer += 3; } @@ -1459,7 +1419,7 @@ public final class BinaryDictInputOutput { final int originalPointer = buffer.position(); buffer.position(address); - if (supportsDynamicUpdate(formatOptions)) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions); } else { result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address, @@ -1488,13 +1448,13 @@ public final class BinaryDictInputOutput { do { buffer.position(currentAddress + headerSize); currentInfo = readCharGroup(buffer, currentAddress, options); - if (isMovedGroup(currentInfo.mFlags, options)) { + if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } if (DBG && loopCounter++ > MAX_JUMPS) { MakedictLog.d("Too many jumps - probably a bug"); } - } while (isMovedGroup(currentInfo.mFlags, options)); + } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)); if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; for (int i = 0; i < currentInfo.mCharacters.length; ++i) { sGetWordBuffer[index--] = @@ -1514,7 +1474,7 @@ public final class BinaryDictInputOutput { final FormatOptions options) { buffer.position(headerSize); final int count = readCharGroupCount(buffer); - int groupOffset = getGroupCountSize(count); + int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); final StringBuilder builder = new StringBuilder(); WeightedString result = null; @@ -1527,23 +1487,23 @@ public final class BinaryDictInputOutput { result = new WeightedString(builder.toString(), info.mFrequency); break; // and return } - if (hasChildrenAddress(info.mChildrenAddress)) { + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { if (info.mChildrenAddress > address) { if (null == last) continue; builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); buffer.position(last.mChildrenAddress + headerSize); i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + getGroupCountSize(i); + groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); last = null; continue; } last = info; } - if (0 == i && hasChildrenAddress(last.mChildrenAddress)) { + if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); buffer.position(last.mChildrenAddress + headerSize); i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + getGroupCountSize(i); + groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); last = null; continue; } @@ -1576,10 +1536,10 @@ public final class BinaryDictInputOutput { do { // Scan the linked-list node. final int nodeHeadPosition = buffer.position() - headerSize; final int count = readCharGroupCount(buffer); - int groupOffset = nodeHeadPosition + getGroupCountSize(count); + int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); for (int i = count; i > 0; --i) { // Scan the array of CharGroup. CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - if (isMovedGroup(info.mFlags, options)) continue; + if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; ArrayList<WeightedString> bigrams = null; if (null != info.mBigrams) { @@ -1592,7 +1552,7 @@ public final class BinaryDictInputOutput { bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); } } - if (hasChildrenAddress(info.mChildrenAddress)) { + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { Node children = reverseNodeMap.get(info.mChildrenAddress); if (null == children) { final int currentPosition = buffer.position(); diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index fd728f1d7..5b10912ea 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -85,7 +85,7 @@ public final class DynamicBinaryDictIOUtils { throw new RuntimeException("this file format does not support parent addresses"); } final int flags = buffer.readUnsignedByte(); - if (BinaryDictInputOutput.isMovedGroup(flags, formatOptions)) { + if (BinaryDictIOUtils.isMovedGroup(flags, formatOptions)) { // if the group is moved, the parent address is stored in the destination group. // We are guaranteed to process the destination group later, so there is no need to // update anything here. @@ -286,7 +286,7 @@ public final class DynamicBinaryDictIOUtils { address = buffer.position(); final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, buffer.position(), header.mFormatOptions); - final boolean isMovedGroup = BinaryDictInputOutput.isMovedGroup(currentInfo.mFlags, + final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, header.mFormatOptions); if (isMovedGroup) continue; nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java index f21db25a6..9f289e9ff 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java @@ -53,7 +53,7 @@ public final class UserHistoryDictionaryBigramList { * Called when loaded from the SQL DB. */ public void addBigram(String word1, String word2, byte fcValue) { - if (UserHistoryPredictionDictionary.DBG_SAVE_RESTORE) { + if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; @@ -73,7 +73,7 @@ public final class UserHistoryDictionaryBigramList { * Called when inserted to the SQL DB. */ public void updateBigram(String word1, String word2, byte fcValue) { - if (UserHistoryPredictionDictionary.DBG_SAVE_RESTORE) { + if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 6e1d765b3..8d51a2f26 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -43,6 +43,7 @@ LATIN_IME_JNI_SRC_FILES := \ com_android_inputmethod_keyboard_ProximityInfo.cpp \ com_android_inputmethod_latin_BinaryDictionary.cpp \ com_android_inputmethod_latin_DicTraverseSession.cpp \ + com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \ jni_common.cpp LATIN_IME_CORE_SRC_FILES := \ diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index a25cef555..86c2394d1 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -18,37 +18,20 @@ #include "com_android_inputmethod_latin_BinaryDictionary.h" -#include <cerrno> #include <cstring> // for memset() -#include <fcntl.h> -#include <sys/mman.h> -#include <unistd.h> #include "defines.h" #include "jni.h" #include "jni_common.h" -#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/dictionary.h" #include "suggest/core/suggest_options.h" -#include "suggest/policyimpl/dictionary/utils/format_utils.h" +#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h" #include "utils/autocorrection_threshold_utils.h" namespace latinime { class ProximityInfo; -// Helper method -static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) { - int ret = munmap(const_cast<void *>(dictBuf), length); - if (ret != 0) { - AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); - } - ret = close(fd); - if (ret != 0) { - AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); - } -} - static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir, jlong dictOffset, jlong dictSize, jboolean isUpdatable) { PROF_OPEN; @@ -61,41 +44,16 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s char sourceDirChars[sourceDirUtf8Length + 1]; env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars); sourceDirChars[sourceDirUtf8Length] = '\0'; - int fd = 0; - void *dictBuf = 0; - int offset = 0; - const bool updatableMmap = (isUpdatable == JNI_TRUE); - const int openMode = updatableMmap ? O_RDWR : O_RDONLY; - fd = open(sourceDirChars, openMode); - if (fd < 0) { - AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno); - return 0; - } - int pagesize = getpagesize(); - offset = static_cast<int>(dictOffset) % pagesize; - int adjDictOffset = static_cast<int>(dictOffset) - offset; - int adjDictSize = static_cast<int>(dictSize) + offset; - const int protMode = updatableMmap ? PROT_READ | PROT_WRITE : PROT_READ; - dictBuf = mmap(0, adjDictSize, protMode, MAP_PRIVATE, fd, adjDictOffset); - if (dictBuf == MAP_FAILED) { - AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno); + DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy = + DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy( + sourceDirChars, static_cast<int>(sourceDirUtf8Length), + static_cast<int>(dictOffset), static_cast<int>(dictSize), + isUpdatable == JNI_TRUE); + if (!dictionaryStructureWithBufferPolicy) { return 0; } - dictBuf = static_cast<char *>(dictBuf) + offset; - if (!dictBuf) { - AKLOGE("DICT: dictBuf is null"); - return 0; - } - Dictionary *dictionary = 0; - if (FormatUtils::UNKNOWN_VERSION - == FormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf), - static_cast<int>(dictSize))) { - AKLOGE("DICT: dictionary format is unknown, bad magic number"); - releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd); - } else { - dictionary = new Dictionary(env, dictBuf, static_cast<int>(dictSize), fd, offset, - updatableMmap); - } + + Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy); PROF_END(66); PROF_CLOSE; return reinterpret_cast<jlong>(dictionary); @@ -104,13 +62,6 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return; - const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo(); - const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset(); - const void *dictBuf = binaryDictionaryInfo->getDictBuf(); - if (!dictBuf) return; - releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset, - binaryDictionaryInfo->getDictSize() + dictBufOffset, - binaryDictionaryInfo->getMmapFd()); delete dictionary; } diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp new file mode 100644 index 000000000..f78883c2d --- /dev/null +++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "LatinIME: jni: BinaryDictInputOutput" + +#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h" + +#include "defines.h" +#include "jni.h" +#include "jni_common.h" + +namespace latinime { +static int latinime_BinaryDictInputOutput_doNothing(JNIEnv *env, jclass clazz) { + // This is a phony method for test - it does nothing. It just returns some value + // unlikely to be in memory by chance for testing purposes. + // TODO: remove this method. + return 2097; +} + +static const JNINativeMethod sMethods[] = { + { + // TODO: remove this entry when we have one useful method in here + const_cast<char *>("doNothing"), + const_cast<char *>("()I"), + reinterpret_cast<void *>(latinime_BinaryDictInputOutput_doNothing) + }, +}; + +int register_BinaryDictInputOutput(JNIEnv *env) { + const char *const kClassPathName = + "com/android/inputmethod/latin/makedict/BinaryDictInputOutput"; + return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods)); +} +} // namespace latinime diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h new file mode 100644 index 000000000..e622ed4ba --- /dev/null +++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H +#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H + +#include "jni.h" + +namespace latinime { +int register_BinaryDictInputOutput(JNIEnv *env); +} // namespace latinime +#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp index f2867d7c3..733e15f73 100644 --- a/native/jni/jni_common.cpp +++ b/native/jni/jni_common.cpp @@ -18,9 +18,12 @@ #include "jni_common.h" +#ifndef HOST_TOOL #include "com_android_inputmethod_keyboard_ProximityInfo.h" #include "com_android_inputmethod_latin_BinaryDictionary.h" #include "com_android_inputmethod_latin_DicTraverseSession.h" +#endif +#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h" #include "defines.h" /* @@ -38,6 +41,7 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { AKLOGE("ERROR: JNIEnv is invalid"); return -1; } +#ifndef HOST_TOOL if (!latinime::register_BinaryDictionary(env)) { AKLOGE("ERROR: BinaryDictionary native registration failed"); return -1; @@ -50,6 +54,11 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { AKLOGE("ERROR: ProximityInfo native registration failed"); return -1; } +#endif + if (!latinime::register_BinaryDictInputOutput(env)) { + AKLOGE("ERROR: BinaryDictInputOutput native registration failed"); + return -1; + } /* success -- return valid version number */ return JNI_VERSION_1_6; } diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h deleted file mode 100644 index e50baae0b..000000000 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2013, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_BINARY_DICTIONARY_INFO_H -#define LATINIME_BINARY_DICTIONARY_INFO_H - -#include <stdint.h> - -#include "defines.h" - -namespace latinime { - -class BinaryDictionaryInfo { - public: - AK_FORCE_INLINE BinaryDictionaryInfo(const uint8_t *const dictBuf, - const int dictSize, const int mmapFd, const int dictBufOffset, const bool isUpdatable) - : mDictBuf(dictBuf), mDictSize(dictSize), mMmapFd(mmapFd), - mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable) {} - - ~BinaryDictionaryInfo() {} - - AK_FORCE_INLINE const uint8_t *getDictBuf() const { - return mDictBuf; - } - - AK_FORCE_INLINE int getDictSize() const { - return mDictSize; - } - - AK_FORCE_INLINE int getMmapFd() const { - return mMmapFd; - } - - AK_FORCE_INLINE int getDictBufOffset() const { - return mDictBufOffset; - } - - AK_FORCE_INLINE bool isDynamicallyUpdatable() const { - // TODO: Support dynamic dictionary formats. - const bool isUpdatableDictionaryFormat = false; - return mIsUpdatable && isUpdatableDictionaryFormat; - } - - private: - DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo); - - const uint8_t *const mDictBuf; - const int mDictSize; - const int mMmapFd; - const int mDictBufOffset; - const bool mIsUpdatable; -}; -} -#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */ diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index af00e9927..8418a608a 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -23,23 +23,19 @@ #include "defines.h" #include "suggest/core/dictionary/bigram_dictionary.h" #include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" #include "suggest/core/session/dic_traverse_session.h" #include "suggest/core/suggest.h" #include "suggest/core/suggest_options.h" -#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h" #include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h" #include "suggest/policyimpl/typing/typing_suggest_policy_factory.h" #include "utils/log_utils.h" namespace latinime { -Dictionary::Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd, - int dictBufOffset, bool isUpdatable) - : mBinaryDictionaryInfo(static_cast<const uint8_t *>(dict), dictSize, mmapFd, - dictBufOffset, isUpdatable), - mDictionaryStructureWithBufferPolicy(DictionaryStructureWithBufferPolicyFactory - ::newDictionaryStructureWithBufferPolicy( - static_cast<const uint8_t *>(dict), dictSize)), +Dictionary::Dictionary(JNIEnv *env, + DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy) + : mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy), mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)), mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())), mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) { @@ -102,32 +98,18 @@ bool Dictionary::isValidBigram(const int *word0, int length0, const int *word1, } void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) { - if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: Dictionary::addUnigramWord() is called for non-updatable dictionary."); - return; - } - // TODO: Support dynamic update + mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability); } void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1, const int length1, const int probability) { - if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: Dictionary::addBigramWords() is called for non-updatable dictionary."); - return; - } - // TODO: Support dynamic update + mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1, + probability); } void Dictionary::removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1) { - if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: Dictionary::removeBigramWords() is called for non-updatable dictionary."); - return; - } - // TODO: Support dynamic update + mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1); } void Dictionary::logDictionaryInfo(JNIEnv *const env) const { @@ -154,9 +136,8 @@ void Dictionary::logDictionaryInfo(JNIEnv *const env) const { dateStringCharBuffer, BUFFER_SIZE); LogUtils::logToJava(env, - "Dictionary info: dictionary = %s ; version = %s ; date = %s ; filesize = %i", - dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer, - mBinaryDictionaryInfo.getDictSize()); + "Dictionary info: dictionary = %s ; version = %s ; date = %s", + dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer); } } // namespace latinime diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 17ce47974..0afe5a54b 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -21,7 +21,6 @@ #include "defines.h" #include "jni.h" -#include "suggest/core/dictionary/binary_dictionary_info.h" namespace latinime { @@ -54,8 +53,8 @@ class Dictionary { static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000; static const int KIND_FLAG_EXACT_MATCH = 0x40000000; - Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd, int dictBufOffset, - bool isUpdatable); + Dictionary(JNIEnv *env, + DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPoilcy); int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession, int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, @@ -78,11 +77,6 @@ class Dictionary { void removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1); - // TODO: Remove. - const BinaryDictionaryInfo *getBinaryDictionaryInfo() const { - return &mBinaryDictionaryInfo; - } - const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const { return mDictionaryStructureWithBufferPolicy; } @@ -92,11 +86,10 @@ class Dictionary { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); - const BinaryDictionaryInfo mBinaryDictionaryInfo; DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy; - const BigramDictionary *mBigramDictionary; - SuggestInterface *mGestureSuggest; - SuggestInterface *mTypingSuggest; + const BigramDictionary *const mBigramDictionary; + const SuggestInterface *const mGestureSuggest; + const SuggestInterface *const mTypingSuggest; void logDictionaryInfo(JNIEnv *const env) const; }; diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index 37daef98a..532411509 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -59,6 +59,18 @@ class DictionaryStructureWithBufferPolicy { virtual const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const = 0; + // Returns whether the update was success or not. + virtual bool addUnigramWord(const int *const word, const int length, + const int probability) = 0; + + // Returns whether the update was success or not. + virtual bool addBigramWords(const int *const word0, const int length0, const int *const word1, + const int length1, const int probability) = 0; + + // Returns whether the update was success or not. + virtual bool removeBigramWords(const int *const word0, const int length0, + const int *const word1, const int length1) = 0; + protected: DictionaryStructureWithBufferPolicy() {} diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp index 34f092f49..cc5252c43 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp @@ -22,18 +22,29 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" +#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h" namespace latinime { /* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory - ::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf, - const int dictSize) { - switch (FormatUtils::detectFormatVersion(dictBuf, dictSize)) { + ::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength, + const int bufOffset, const int size, const bool isUpdatable) { + // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of + // impl classes of DictionaryStructureWithBufferPolicy. + const MmapedBuffer *const mmapedBuffer = MmapedBuffer::openBuffer(path, pathLength, bufOffset, + size, isUpdatable); + if (!mmapedBuffer) { + return 0; + } + switch (FormatUtils::detectFormatVersion(mmapedBuffer->getBuffer(), + mmapedBuffer->getBufferSize())) { case FormatUtils::VERSION_2: - return new PatriciaTriePolicy(dictBuf); + return new PatriciaTriePolicy(mmapedBuffer); case FormatUtils::VERSION_3: - return new DynamicPatriciaTriePolicy(dictBuf); + return new DynamicPatriciaTriePolicy(mmapedBuffer); default: + AKLOGE("DICT: dictionary format is unknown, bad magic number"); + delete mmapedBuffer; ASSERT(false); return 0; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h index 53eb8f927..1cb7a89c4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h @@ -27,7 +27,8 @@ namespace latinime { class DictionaryStructureWithBufferPolicyFactory { public: static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy( - const uint8_t *const dictBuf, const int dictSize); + const char *const path, const int pathLength, const int bufOffset, const int size, + const bool isUpdatable); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index cca801098..3000860a3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -231,4 +231,34 @@ int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const return nodeReader.getBigramsPos(); } +bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length, + const int probability) { + if (!mBuffer->isUpdatable()) { + AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); + return false; + } + // TODO: Implement. + return false; +} + +bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0, + const int *const word1, const int length1, const int probability) { + if (!mBuffer->isUpdatable()) { + AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); + return false; + } + // TODO: Implement. + return false; +} + +bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0, + const int *const word1, const int length1) { + if (!mBuffer->isUpdatable()) { + AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary."); + return false; + } + // TODO: Implement. + return false; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index ad8911c80..708967d7b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -24,6 +24,7 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h" namespace latinime { @@ -32,11 +33,14 @@ class DicNodeVector; class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: - DynamicPatriciaTriePolicy(const uint8_t *const dictBuf) - : mHeaderPolicy(dictBuf), mDictRoot(dictBuf + mHeaderPolicy.getSize()), + DynamicPatriciaTriePolicy(const MmapedBuffer *const buffer) + : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), + mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()), mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {} - ~DynamicPatriciaTriePolicy() {} + ~DynamicPatriciaTriePolicy() { + delete mBuffer; + } AK_FORCE_INLINE int getRootPosition() const { return 0; @@ -70,10 +74,19 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { return &mShortcutListPolicy; } + bool addUnigramWord(const int *const word, const int length, const int probability); + + bool addBigramWords(const int *const word0, const int length0, const int *const word1, + const int length1, const int probability); + + bool removeBigramWords(const int *const word0, const int length0, const int *const word1, + const int length1); + private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP; + const MmapedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; // TODO: Consolidate mDictRoot. const uint8_t *const mDictRoot; diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp index 23b88ecdf..f323876c4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp @@ -34,7 +34,7 @@ const int HeaderReadingUtils::HEADER_SIZE_FIELD_SIZE = 4; const HeaderReadingUtils::DictionaryFlags HeaderReadingUtils::NO_FLAGS = 0; // Flags for special processing -// Those *must* match the flags in makedict (BinaryDictInputOutput#*_PROCESSING_FLAG) or +// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or // something very bad (like, the apocalypse) will happen. Please update both at the same time. const HeaderReadingUtils::DictionaryFlags HeaderReadingUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index 2e34480aa..0d85050f3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -24,6 +24,7 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h" namespace latinime { @@ -32,11 +33,14 @@ class DicNodeVector; class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: - PatriciaTriePolicy(const uint8_t *const dictBuf) - : mHeaderPolicy(dictBuf), mDictRoot(dictBuf + mHeaderPolicy.getSize()), + PatriciaTriePolicy(const MmapedBuffer *const buffer) + : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), + mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()), mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {} - ~PatriciaTriePolicy() {} + ~PatriciaTriePolicy() { + delete mBuffer; + } AK_FORCE_INLINE int getRootPosition() const { return 0; @@ -70,9 +74,30 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { return &mShortcutListPolicy; } + bool addUnigramWord(const int *const word, const int length, const int probability) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); + return false; + } + + bool addBigramWords(const int *const word0, const int length0, const int *const word1, + const int length1, const int probability) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); + return false; + } + + bool removeBigramWords(const int *const word0, const int length0, const int *const word1, + const int length1) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary."); + return false; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy); + const MmapedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; const uint8_t *const mDictRoot; const BigramListPolicy mBigramListPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h new file mode 100644 index 000000000..e3aabb084 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_MMAPED_BUFFER_H +#define LATINIME_MMAPED_BUFFER_H + +#include <cerrno> +#include <fcntl.h> +#include <stdint.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "defines.h" + +namespace latinime { + +class MmapedBuffer { + public: + static MmapedBuffer* openBuffer(const char *const path, const int pathLength, + const int bufOffset, const int size, const bool isUpdatable) { + const int openMode = isUpdatable ? O_RDWR : O_RDONLY; + const int fd = open(path, openMode); + if (fd < 0) { + AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno); + return 0; + } + const int pagesize = getpagesize(); + const int offset = bufOffset % pagesize; + int adjOffset = bufOffset - offset; + int adjSize = size + offset; + const int protMode = isUpdatable ? PROT_READ | PROT_WRITE : PROT_READ; + void *const mmapedBuffer = mmap(0, adjSize, protMode, MAP_PRIVATE, fd, adjOffset); + if (mmapedBuffer == MAP_FAILED) { + AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno); + close(fd); + return 0; + } + uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + bufOffset; + if (!buffer) { + AKLOGE("DICT: buffer is null"); + close(fd); + return 0; + } + return new MmapedBuffer(buffer, adjSize, fd, adjOffset, isUpdatable); + } + + ~MmapedBuffer() { + int ret = munmap(static_cast<void *>(mBuffer - mBufferOffset), + mBufferSize + mBufferOffset); + if (ret != 0) { + AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); + } + ret = close(mMmapFd); + if (ret != 0) { + AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); + } + } + + AK_FORCE_INLINE uint8_t *getBuffer() const { + return mBuffer; + } + + AK_FORCE_INLINE int getBufferSize() const { + return mBufferSize; + } + + AK_FORCE_INLINE bool isUpdatable() const { + return mIsUpdatable; + } + + private: + AK_FORCE_INLINE MmapedBuffer(uint8_t *const buffer, const int bufferSize, const int mmapFd, + const int bufferOffset, const bool isUpdatable) + : mBuffer(buffer), mBufferSize(bufferSize), mMmapFd(mmapFd), + mBufferOffset(bufferOffset), mIsUpdatable(isUpdatable) {} + + DISALLOW_IMPLICIT_CONSTRUCTORS(MmapedBuffer); + + uint8_t *const mBuffer; + const int mBufferSize; + const int mMmapFd; + const int mBufferOffset; + const bool mIsUpdatable; +}; +} +#endif /* LATINIME_MMAPED_BUFFER_H */ diff --git a/tools/dicttool/Android.mk b/tools/dicttool/Android.mk index d06be58a7..f076ef277 100644 --- a/tools/dicttool/Android.mk +++ b/tools/dicttool/Android.mk @@ -13,27 +13,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -LOCAL_PATH := $(call my-dir) +LATINIME_DICTTOOL_AOSP_LOCAL_PATH := $(call my-dir) +LOCAL_PATH := $(LATINIME_DICTTOOL_AOSP_LOCAL_PATH) +LATINIME_HOST_NATIVE_LIBNAME := liblatinime-aosp-dicttool-host +include $(LOCAL_PATH)/NativeLib.mk + +###################################### +LOCAL_PATH := $(LATINIME_DICTTOOL_AOSP_LOCAL_PATH) include $(CLEAR_VARS) -BUILD_TOP := ../../../../.. -LATINIME_DIR := $(BUILD_TOP)/packages/inputmethods/LatinIME -LATINIME_BASE_SOURCE_DIRECTORY := $(LATINIME_DIR)/java/src/com/android/inputmethod -LATINIME_CORE_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/latin +LATINIME_LOCAL_DIR := ../.. +LATINIME_BASE_SOURCE_DIRECTORY := $(LATINIME_LOCAL_DIR)/java/src/com/android/inputmethod LATINIME_ANNOTATIONS_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/annotations +LATINIME_CORE_SOURCE_DIRECTORY := $(LATINIME_BASE_SOURCE_DIRECTORY)/latin MAKEDICT_CORE_SOURCE_DIRECTORY := $(LATINIME_CORE_SOURCE_DIRECTORY)/makedict -DICTTOOL_COMPAT_TESTS_DIRECTORY := compat -DICTTOOL_ONDEVICE_TESTS_DIRECTORY := \ - $(LATINIME_DIR)/tests/src/com/android/inputmethod/latin/makedict/ - USED_TARGETTED_UTILS := \ $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ByteArrayWrapper.java \ - $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java + $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java \ + $(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java + +DICTTOOL_ONDEVICE_TESTS_DIRECTORY := \ + $(LATINIME_LOCAL_DIR)/tests/src/com/android/inputmethod/latin/makedict/ +DICTTOOL_COMPAT_TESTS_DIRECTORY := compat LOCAL_MAIN_SRC_FILES := $(call all-java-files-under, $(MAKEDICT_CORE_SOURCE_DIRECTORY)) LOCAL_TOOL_SRC_FILES := $(call all-java-files-under, src) LOCAL_ANNOTATIONS_SRC_FILES := \ $(call all-java-files-under, $(LATINIME_ANNOTATIONS_SOURCE_DIRECTORY)) + LOCAL_SRC_FILES := $(LOCAL_TOOL_SRC_FILES) \ $(filter-out $(addprefix %/, $(notdir $(LOCAL_TOOL_SRC_FILES))), $(LOCAL_MAIN_SRC_FILES)) \ $(LOCAL_ANNOTATIONS_SRC_FILES) \ @@ -44,9 +51,13 @@ LOCAL_SRC_FILES := $(LOCAL_TOOL_SRC_FILES) \ $(USED_TARGETTED_UTILS) LOCAL_JAVA_LIBRARIES := junit - +LOCAL_ADDITIONAL_DEPENDENCIES := $(LATINIME_HOST_NATIVE_LIBNAME) LOCAL_JAR_MANIFEST := etc/manifest.txt LOCAL_MODULE := dicttool_aosp include $(BUILD_HOST_JAVA_LIBRARY) include $(LOCAL_PATH)/etc/Android.mk + +# Clear our private variables +LATINIME_DICTTOOL_AOSP_LOCAL_PATH := +LATINIME_LOCAL_DIR := diff --git a/tools/dicttool/NativeLib.mk b/tools/dicttool/NativeLib.mk new file mode 100644 index 000000000..eeae003fb --- /dev/null +++ b/tools/dicttool/NativeLib.mk @@ -0,0 +1,51 @@ +# +# Copyright (C) 2013 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCAL_PATH := $(call my-dir) +include $(CLEAR_VARS) + +# Need to define the name of the library in the caller in LATINIME_HOST_NATIVE_LIBNAME + +LATINIME_DIR_RELATIVE_TO_DICTTOOL := ../.. + +ifneq ($(strip $(HOST_JDK_IS_64BIT_VERSION)),) +LOCAL_CFLAGS += -m64 +LOCAL_LDFLAGS += -m64 +endif #HOST_JDK_IS_64BIT_VERSION + +LOCAL_CFLAGS += -DHOST_TOOL -fPIC +LOCAL_NO_DEFAULT_COMPILER_FLAGS := true + +LATINIME_NATIVE_JNI_DIR := $(LATINIME_DIR_RELATIVE_TO_DICTTOOL)/native/jni +LATINIME_NATIVE_SRC_DIR := $(LATINIME_DIR_RELATIVE_TO_DICTTOOL)/native/jni/src +LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(LATINIME_NATIVE_SRC_DIR) +# Used in jni_common.cpp to avoid registering useless methods. + +LATIN_IME_JNI_SRC_FILES := \ + com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \ + jni_common.cpp + +LATIN_IME_CORE_SRC_FILES := + +LOCAL_SRC_FILES := \ + $(addprefix $(LATINIME_NATIVE_JNI_DIR)/, $(LATIN_IME_JNI_SRC_FILES)) \ + $(addprefix $(LATINIME_NATIVE_SRC_DIR)/, $(LATIN_IME_CORE_SRC_FILES)) + +LOCAL_MODULE := $(LATINIME_HOST_NATIVE_LIBNAME) + +include $(BUILD_HOST_SHARED_LIBRARY) + +# Clear our private variables +LATINIME_DIR_RELATIVE_TO_DICTTOOL := ../.. diff --git a/tools/dicttool/compat/android/util/Log.java b/tools/dicttool/compat/android/util/Log.java index d9df3a4ae..b3b6dd847 100644 --- a/tools/dicttool/compat/android/util/Log.java +++ b/tools/dicttool/compat/android/util/Log.java @@ -25,13 +25,13 @@ public class Log { public static void d(final String tag, final String message) { System.out.println(tag + " : " + message); } - public static void d(final String tag, final String message, final Exception e) { + public static void d(final String tag, final String message, final Throwable e) { System.out.println(tag + " : " + message + " : " + e); } public static void e(final String tag, final String message) { d(tag, message); } - public static void e(final String tag, final String message, final Exception e) { - e(tag, message, e); + public static void e(final String tag, final String message, final Throwable e) { + d(tag, message, e); } } diff --git a/tools/dicttool/compat/com/android/inputmethod/latin/define/JniLibName.java b/tools/dicttool/compat/com/android/inputmethod/latin/define/JniLibName.java new file mode 100644 index 000000000..c68bdaabf --- /dev/null +++ b/tools/dicttool/compat/com/android/inputmethod/latin/define/JniLibName.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.define; + +public final class JniLibName { + private JniLibName() { + // This class is not publicly instantiable. + } + + public static final String JNI_LIB_NAME = "latinime-dicttool-host"; +} diff --git a/tools/dicttool/etc/dicttool_aosp b/tools/dicttool/etc/dicttool_aosp index cc7111a2c..65a1c3a1c 100755 --- a/tools/dicttool/etc/dicttool_aosp +++ b/tools/dicttool/etc/dicttool_aosp @@ -69,4 +69,4 @@ else fi # might need more memory, e.g. -Xmx128M -exec java -ea -classpath "$libpath":"$jarpath" "$classname" "$@" +exec java -ea -classpath "$libpath":"$jarpath" -Djava.library.path="$libdir" "$classname" "$@" |