diff options
6 files changed, 185 insertions, 6 deletions
diff --git a/native/jni/Android.mk b/native/jni/Android.mk index c83ae065a..55a5c06d7 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -85,7 +85,9 @@ LATIN_IME_CORE_SRC_FILES := \ dynamic_patricia_trie_reading_utils.cpp \ dynamic_patricia_trie_writing_helper.cpp \ dynamic_patricia_trie_writing_utils.cpp) \ - suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp \ + $(addprefix suggest/policyimpl/dictionary/structure/v4/, \ + ver4_dict_constants.cpp \ + ver4_patricia_trie_policy.cpp) \ $(addprefix suggest/policyimpl/dictionary/utils/, \ buffer_with_extendable_buffer.cpp \ byte_array_utils.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index 81783b53e..3ab6a8e21 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -17,10 +17,13 @@ #include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h" #include <stdint.h> +#include <string> #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" @@ -45,9 +48,26 @@ namespace latinime { case FormatUtils::VERSION_3: return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr( new DynamicPatriciaTriePolicy(mmappedBuffer)); - case FormatUtils::VERSION_4: - // TODO: Support version 4 dictionary format. - // Fall through. + case FormatUtils::VERSION_4: { + std::string dictDirPath(path); + const std::string::size_type pos = + dictDirPath.rfind(Ver4DictConstants::TRIE_FILE_EXTENSION); + if (pos == std::string::npos) { + // Dictionary file name is not valid as a version 4 dictionary. + return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(0); + } + // Removing extension to get the base path. + dictDirPath.erase(pos); + const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( + Ver4DictBuffers::openVer4DictBuffers(dictDirPath.c_str(), mmappedBuffer)); + if (!dictBuffers.get()->isValid()) { + AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements."); + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(0); + } + return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr( + new Ver4PatriciaTriePolicy(dictBuffers)); + } default: AKLOGE("DICT: dictionary format is unknown, bad magic number"); ASSERT(false); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h new file mode 100644 index 000000000..1164c408a --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_VER4_DICT_BUFFER_H +#define LATINIME_VER4_DICT_BUFFER_H + +#include "defines.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" +#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" + +namespace latinime { + +class Ver4DictBuffers { + public: + typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr; + + static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, + const MmappedBuffer::MmappedBufferPtr &dictBuffer) { + const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false; + return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable)); + } + + AK_FORCE_INLINE bool isValid() const { + return mDictBuffer.get() != 0 && mProbabilityDictContent.isValid() + && mTerminalAddressTable.isValid() && mBigramDictContent.isValid() + && mShortcutDictContent.isValid(); + } + + AK_FORCE_INLINE const uint8_t *getRawDictBuffer() const { + return mDictBuffer.get()->getBuffer(); + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictBuffers); + + AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath, + const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable) + : mDictBuffer(dictBuffer), + mTerminalAddressTable(dictDirPath, + Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), + mProbabilityDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, + isUpdatable), + mBigramDictContent(dictDirPath, + Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, + Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, + Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable), + mShortcutDictContent(dictDirPath, + Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, + Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, + Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable) {} + + const MmappedBuffer::MmappedBufferPtr mDictBuffer; + SingleDictContent mTerminalAddressTable; + SingleDictContent mProbabilityDictContent; + SparseTableDictContent mBigramDictContent; + SparseTableDictContent mShortcutDictContent; +}; +} // namespace latinime +#endif /* LATINIME_VER4_DICT_BUFFER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp new file mode 100644 index 000000000..aff11f39f --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" + +namespace latinime { + +const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie"; +const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq"; +// tat = Terminal Address Table +const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; +const char *const Ver4DictConstants::BIGRAM_FILE_EXTENSION = ".bigram_freq"; +const char *const Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup"; +const char *const Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION = ".bigram_index_freq"; +const char *const Ver4DictConstants::SHORTCUT_FILE_EXTENSION = ".shortcut_shortcut"; +const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".shortcut_lookup"; +const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION = + ".shortcut_index_shortcut"; + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h new file mode 100644 index 000000000..a65f11abb --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_VER4_DICT_CONSTANTS_H +#define LATINIME_VER4_DICT_CONSTANTS_H + +#include "defines.h" + +namespace latinime { + +// Note that there are corresponding definitions in FormatSpec.java. +class Ver4DictConstants { + public: + static const char *const TRIE_FILE_EXTENSION; + static const char *const FREQ_FILE_EXTENSION; + static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION; + static const char *const BIGRAM_FILE_EXTENSION; + static const char *const BIGRAM_LOOKUP_TABLE_FILE_EXTENSION; + static const char *const BIGRAM_CONTENT_TABLE_FILE_EXTENSION; + static const char *const SHORTCUT_FILE_EXTENSION; + static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION; + static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION; + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants); +}; +} // namespace latinime +#endif /* LATINIME_VER4_DICT_CONSTANTS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 86a4f8bce..f7bfb3b0d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -19,6 +19,9 @@ #include "defines.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" +#include "suggest/policyimpl/dictionary/header/header_policy.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { @@ -28,7 +31,9 @@ class DicNodeVector; // TODO: Implement. class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: - ~Ver4PatriciaTriePolicy() {} + Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers) + : mBuffers(buffers), + mHeaderPolicy(mBuffers.get()->getRawDictBuffer(), FormatUtils::VERSION_4) {}; AK_FORCE_INLINE int getRootPosition() const { return 0; @@ -53,7 +58,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { int getBigramsPositionOfPtNode(const int ptNodePos) const; const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const { - return 0; + return &mHeaderPolicy; } const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const { @@ -83,6 +88,9 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTriePolicy); + + const Ver4DictBuffers::Ver4DictBuffersPtr mBuffers; + const HeaderPolicy mHeaderPolicy; }; } // namespace latinime #endif // LATINIME_VER4_PATRICIA_TRIE_POLICY_H |