diff options
6 files changed, 56 insertions, 3 deletions
diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk index 09777497b..1e3775b0a 100644 --- a/native/jni/NativeFileList.mk +++ b/native/jni/NativeFileList.mk @@ -72,6 +72,7 @@ LATIN_IME_CORE_SRC_FILES := \ ver4_pt_node_array_reader.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \ bigram_dict_content.cpp \ + language_model_dict_content.cpp \ probability_dict_content.cpp \ shortcut_dict_content.cpp \ sparse_table_dict_content.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp new file mode 100644 index 000000000..b165bf4b7 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2014, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h" + +namespace latinime { + +bool LanguageModelDictContent::save(FILE *const file) const { + return mTrieMap.save(file); +} + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h index 0cc0df2be..263911380 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h @@ -17,16 +17,28 @@ #ifndef LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H #define LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H +#include <cstdio> + #include "defines.h" +#include "suggest/policyimpl/dictionary/utils/trie_map.h" +#include "utils/byte_array_view.h" namespace latinime { class LanguageModelDictContent { public: - explicit LanguageModelDictContent(const bool hasHistoricalInfo) {} + LanguageModelDictContent(const ReadWriteByteArrayView trieMapBuffer, + const bool hasHistoricalInfo) + : mTrieMap(trieMapBuffer) {} + + explicit LanguageModelDictContent(const bool hasHistoricalInfo) : mTrieMap() {} + + bool save(FILE *const file) const; private: DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent); + + TrieMap mTrieMap; }; } // namespace latinime #endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index a2920e4b7..125ae1711 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -162,6 +162,11 @@ bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const { AKLOGE("Probability dict content cannot be written."); return false; } + // Write language model content. + if (!mLanguageModelDictContent.save(file)) { + AKLOGE("Language model dict content cannot be written."); + return false; + } // Write bigram dict content. if (!mBigramDictContent.flushToFile(file)) { AKLOGE("Bigram dict content cannot be written."); @@ -195,7 +200,11 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], mHeaderPolicy.hasHistoricalInfoOfWords()), - mLanguageModelDictContent(mHeaderPolicy.hasHistoricalInfoOfWords()), + mLanguageModelDictContent( + ReadWriteByteArrayView( + contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX], + contentBufferSizes[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX]), + mHeaderPolicy.hasHistoricalInfoOfWords()), mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX], &contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX], mHeaderPolicy.hasHistoricalInfoOfWords()), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index d45dfe377..e7e31e96f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -31,14 +31,17 @@ const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024; // NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut. const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE = NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3 + + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2; const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0; const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX = TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX = TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; -const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX = +const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX = PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; +const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX = + LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT; const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX = BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT; @@ -73,5 +76,6 @@ const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80; const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1; const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3; +const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT = 1; } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index e8f6739ba..e75db9f75 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -36,6 +36,7 @@ class Ver4DictConstants { static const int TRIE_BUFFER_INDEX; static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX; static const int PROBABILITY_BUFFER_INDEX; + static const int LANGUAGE_MODEL_BUFFER_INDEX; static const int BIGRAM_BUFFERS_INDEX; static const int SHORTCUT_BUFFERS_INDEX; @@ -71,6 +72,7 @@ class Ver4DictConstants { static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT; + static const size_t NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT; }; } // namespace latinime #endif /* LATINIME_VER4_DICT_CONSTANTS_H */ |