diff options
Diffstat (limited to 'native')
11 files changed, 180 insertions, 5 deletions
diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 55a5c06d7..f4c8f7a40 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -87,7 +87,8 @@ LATIN_IME_CORE_SRC_FILES := \ dynamic_patricia_trie_writing_utils.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v4/, \ ver4_dict_constants.cpp \ - ver4_patricia_trie_policy.cpp) \ + ver4_patricia_trie_policy.cpp \ + ver4_patricia_trie_reading_utils.cpp ) \ $(addprefix suggest/policyimpl/dictionary/utils/, \ buffer_with_extendable_buffer.cpp \ byte_array_utils.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h new file mode 100644 index 000000000..daaf08f61 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_PROBABILITY_DICT_CONTENT_H +#define LATINIME_PROBABILITY_DICT_CONTENT_H + +#include "defines.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" + +namespace latinime { + +class ProbabilityDictContent : public SingleDictContent { + public: + ProbabilityDictContent(const char *const dictDirPath, const bool isUpdatable) + : SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, + isUpdatable) {} + + int getProbability(const int terminalId) const { + if (terminalId < 0 || terminalId >= getSize()) { + return NOT_A_PROBABILITY; + } + return Ver4PatriciaTrieReadingUtils::getProbability(getBuffer(), terminalId); + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityDictContent); + + int getSize() const { + return getBuffer()->getTailPosition() / (Ver4DictConstants::PROBABILITY_SIZE + + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE); + } +}; +} // namespace latinime +#endif /* LATINIME_PROBABILITY_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index e415881a9..4cb96da6a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -39,6 +39,15 @@ class SingleDictContent : public DictContent { return mMmappedBuffer.get() != 0; } + protected: + BufferWithExtendableBuffer *getWritableBuffer() { + return &mExpandableContentBuffer; + } + + const BufferWithExtendableBuffer *getBuffer() const { + return &mExpandableContentBuffer; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(SingleDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index 1164c408a..333a7c209 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -18,6 +18,7 @@ #define LATINIME_VER4_DICT_BUFFER_H #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" @@ -42,10 +43,18 @@ class Ver4DictBuffers { && mShortcutDictContent.isValid(); } - AK_FORCE_INLINE const uint8_t *getRawDictBuffer() const { + AK_FORCE_INLINE uint8_t *getRawDictBuffer() const { return mDictBuffer.get()->getBuffer(); } + AK_FORCE_INLINE int getRawDictBufferSize() const { + return mDictBuffer.get()->getBufferSize(); + } + + AK_FORCE_INLINE const ProbabilityDictContent *getProbabilityDictContent() const { + return &mProbabilityDictContent; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictBuffers); @@ -54,8 +63,7 @@ class Ver4DictBuffers { : mDictBuffer(dictBuffer), mTerminalAddressTable(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), - mProbabilityDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, - isUpdatable), + mProbabilityDictContent(dictDirPath, isUpdatable), mBigramDictContent(dictDirPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, @@ -67,7 +75,7 @@ class Ver4DictBuffers { const MmappedBuffer::MmappedBufferPtr mDictBuffer; SingleDictContent mTerminalAddressTable; - SingleDictContent mProbabilityDictContent; + ProbabilityDictContent mProbabilityDictContent; SparseTableDictContent mBigramDictContent; SparseTableDictContent mShortcutDictContent; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index 0bfd07b04..160259afc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -31,5 +31,7 @@ const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION = ".shortcut_index_shortcut"; const int Ver4DictConstants::NOT_A_TERMINAL = -1; +const int Ver4DictConstants::PROBABILITY_SIZE = 1; +const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1; } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index 6498ce428..dcc36d1dc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -35,6 +35,8 @@ class Ver4DictConstants { static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION; static const int NOT_A_TERMINAL; + static const int PROBABILITY_SIZE; + static const int FLAGS_IN_PROBABILITY_FILE_SIZE; private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp new file mode 100644 index 000000000..3304f1521 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" + +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" + +namespace latinime { + +/* static */ int Ver4PatriciaTrieReadingUtils::getProbability( + const BufferWithExtendableBuffer *const probabilityBuffer, const int terminalId) { + int pos = terminalId * (Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE + + Ver4DictConstants::PROBABILITY_SIZE) + + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE; + return probabilityBuffer->readUintAndAdvancePosition(Ver4DictConstants::PROBABILITY_SIZE, &pos); +} + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h new file mode 100644 index 000000000..ee369d1e1 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_VER4_PATRICIA_TRIE_READING_UTILS_H +#define LATINIME_VER4_PATRICIA_TRIE_READING_UTILS_H + +#include <stdint.h> + +#include "defines.h" + +namespace latinime { + +class BufferWithExtendableBuffer; + +class Ver4PatriciaTrieReadingUtils { + public: + static int getProbability(const BufferWithExtendableBuffer *const probabilityBuffer, + const int terminalId); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils); +}; +} // namespace latinime +#endif /* LATINIME_VER4_PATRICIA_TRIE_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp index 5032131ab..e028de526 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp @@ -23,6 +23,20 @@ const int BufferWithExtendableBuffer::NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE = 9 // TODO: Needs to allocate larger memory corresponding to the current vector size. const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 128 * 1024; +uint32_t BufferWithExtendableBuffer::readUintAndAdvancePosition(const int size, + int *const pos) const { + const bool readingPosIsInAdditionalBuffer = isInAdditionalBuffer(*pos); + if (readingPosIsInAdditionalBuffer) { + *pos -= mOriginalBufferSize; + } + const int value = ByteArrayUtils::readUintAndAdvancePosition( + getBuffer(readingPosIsInAdditionalBuffer), size, pos); + if (readingPosIsInAdditionalBuffer) { + *pos += mOriginalBufferSize; + } + return value; +} + bool BufferWithExtendableBuffer::writeUintAndAdvancePosition(const uint32_t data, const int size, int *const pos) { if (!(size >= 1 && size <= 4)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h index 1e27a1bec..2d89f71b1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h @@ -71,6 +71,8 @@ class BufferWithExtendableBuffer { } } + uint32_t readUintAndAdvancePosition(const int size, int *const pos) const; + AK_FORCE_INLINE int getOriginalBufferSize() const { return mOriginalBufferSize; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h index 0c1576818..1ca01b868 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h @@ -114,6 +114,24 @@ class ByteArrayUtils { return buffer[(*pos)++]; } + static AK_FORCE_INLINE int readUintAndAdvancePosition(const uint8_t *const buffer, + const int size, int *const pos) { + // size must be in 1 to 4. + ASSERT(size >= 1 && size <= 4); + switch (size) { + case 1: + return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); + case 2: + return ByteArrayUtils::readUint16AndAdvancePosition(buffer, pos); + case 3: + return ByteArrayUtils::readUint24AndAdvancePosition(buffer, pos); + case 4: + return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos); + default: + return 0; + } + } + /** * Code Point Reading * |