diff options
7 files changed, 40 insertions, 82 deletions
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp index 0e8d72f2e..15f2aa630 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp @@ -16,6 +16,8 @@ #include "suggest/core/dictionary/binary_dictionary_format_utils.h" +#include "suggest/core/dictionary/byte_array_utils.h" + namespace latinime { /** @@ -27,7 +29,6 @@ const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4; /** * Format versions */ - // The versions of Latin IME that only handle format version 1 only test for the magic // number, so we had to change it so that version 2 files would be rejected by older // implementations. On this occasion, we made the magic number 32 bits long. diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h index 830684c70..62c7376e1 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp index 91c643a5f..16f534c21 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp @@ -17,7 +17,6 @@ #include "suggest/core/dictionary/binary_dictionary_header.h" #include "defines.h" -#include "suggest/core/dictionary/binary_dictionary_info.h" namespace latinime { @@ -26,16 +25,15 @@ const char *const BinaryDictionaryHeader::MULTIPLE_WORDS_DEMOTION_RATE_KEY = const float BinaryDictionaryHeader::DEFAULT_MULTI_WORD_COST_MULTIPLIER = 1.0f; const float BinaryDictionaryHeader::MULTI_WORD_COST_MULTIPLIER_SCALE = 100.0f; -BinaryDictionaryHeader::BinaryDictionaryHeader( - const BinaryDictionaryInfo *const binaryDictionaryInfo) - : mBinaryDictionaryInfo(binaryDictionaryInfo), - mDictionaryFlags(BinaryDictionaryHeaderReadingUtils::getFlags(binaryDictionaryInfo)), - mSize(BinaryDictionaryHeaderReadingUtils::getHeaderSize(binaryDictionaryInfo)), +BinaryDictionaryHeader::BinaryDictionaryHeader(const uint8_t *const dictBuf) + : mDictBuf(dictBuf), + mDictionaryFlags(BinaryDictionaryHeaderReadingUtils::getFlags(mDictBuf)), + mSize(BinaryDictionaryHeaderReadingUtils::getHeaderSize(mDictBuf)), mMultiWordCostMultiplier(readMultiWordCostMultiplier()) {} float BinaryDictionaryHeader::readMultiWordCostMultiplier() const { const int headerValue = BinaryDictionaryHeaderReadingUtils::readHeaderValueInt( - mBinaryDictionaryInfo, MULTIPLE_WORDS_DEMOTION_RATE_KEY); + mDictBuf, MULTIPLE_WORDS_DEMOTION_RATE_KEY); if (headerValue == S_INT_MIN) { // not found return DEFAULT_MULTI_WORD_COST_MULTIPLIER; diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_header.h index 240512bce..4d9295229 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header.h @@ -17,6 +17,8 @@ #ifndef LATINIME_BINARY_DICTIONARY_HEADER_H #define LATINIME_BINARY_DICTIONARY_HEADER_H +#include <stdint.h> + #include "defines.h" #include "suggest/core/dictionary/binary_dictionary_header_reading_utils.h" @@ -28,9 +30,10 @@ class BinaryDictionaryInfo; * This class abstracts dictionary header structures and provide interface to access dictionary * header information. */ +// TODO:: Move header classes to policyimpl. class BinaryDictionaryHeader { public: - explicit BinaryDictionaryHeader(const BinaryDictionaryInfo *const binaryDictionaryInfo); + explicit BinaryDictionaryHeader(const uint8_t *const dictBuf); AK_FORCE_INLINE int getSize() const { return mSize; @@ -60,7 +63,7 @@ class BinaryDictionaryHeader { outValue[0] = '\0'; return; } - if (!BinaryDictionaryHeaderReadingUtils::readHeaderValue(mBinaryDictionaryInfo, + if (!BinaryDictionaryHeaderReadingUtils::readHeaderValue(mDictBuf, key, outValue, outValueSize)) { outValue[0] = '?'; outValue[1] = '\0'; @@ -74,7 +77,7 @@ class BinaryDictionaryHeader { static const float DEFAULT_MULTI_WORD_COST_MULTIPLIER; static const float MULTI_WORD_COST_MULTIPLIER_SCALE; - const BinaryDictionaryInfo *const mBinaryDictionaryInfo; + const uint8_t *const mDictBuf; const BinaryDictionaryHeaderReadingUtils::DictionaryFlags mDictionaryFlags; const int mSize; const float mMultiWordCostMultiplier; diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp index a57b0f859..bcf0e612c 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp @@ -20,12 +20,11 @@ #include <cstdlib> #include "defines.h" -#include "suggest/core/dictionary/binary_dictionary_info.h" +#include "suggest/core/dictionary/byte_array_utils.h" namespace latinime { const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256; - const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_MAGIC_NUMBER_SIZE = 4; const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_DICTIONARY_VERSION_SIZE = 2; const int BinaryDictionaryHeaderReadingUtils::VERSION_2_HEADER_FLAG_SIZE = 2; @@ -43,68 +42,54 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags const BinaryDictionaryHeaderReadingUtils::DictionaryFlags BinaryDictionaryHeaderReadingUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; -/* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize( - const BinaryDictionaryInfo *const binaryDictionaryInfo) { - switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) { - case HEADER_VERSION_2: - // See the format of the header in the comment in - // BinaryDictionaryFormatUtils::detectFormatVersion() - return ByteArrayUtils::readUint32(binaryDictionaryInfo->getDictBuf(), - VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE - + VERSION_2_HEADER_FLAG_SIZE); - default: - return S_INT_MAX; - } +/* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(const uint8_t *const dictBuf) { + // See the format of the header in the comment in + // BinaryDictionaryFormatUtils::detectFormatVersion() + return ByteArrayUtils::readUint32(dictBuf, + VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE + + VERSION_2_HEADER_FLAG_SIZE); } /* static */ BinaryDictionaryHeaderReadingUtils::DictionaryFlags - BinaryDictionaryHeaderReadingUtils::getFlags( - const BinaryDictionaryInfo *const binaryDictionaryInfo) { - switch (getHeaderVersion(binaryDictionaryInfo->getFormat())) { - case HEADER_VERSION_2: - return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(), - VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE); - default: - return NO_FLAGS; - } + BinaryDictionaryHeaderReadingUtils::getFlags(const uint8_t *const dictBuf) { + return ByteArrayUtils::readUint16(dictBuf, VERSION_2_HEADER_MAGIC_NUMBER_SIZE + + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE); } // Returns if the key is found or not and reads the found value into outValue. -/* static */ bool BinaryDictionaryHeaderReadingUtils::readHeaderValue( - const BinaryDictionaryInfo *const binaryDictionaryInfo, +/* static */ bool BinaryDictionaryHeaderReadingUtils::readHeaderValue(const uint8_t *const dictBuf, const char *const key, int *outValue, const int outValueSize) { if (outValueSize <= 0) { return false; } - const int headerSize = getHeaderSize(binaryDictionaryInfo); - int pos = getHeaderOptionsPosition(binaryDictionaryInfo->getFormat()); + const int headerSize = getHeaderSize(dictBuf); + int pos = getHeaderOptionsPosition(); if (pos == NOT_A_DICT_POS) { // The header doesn't have header options. return false; } while (pos < headerSize) { if(ByteArrayUtils::compareStringInBufferWithCharArray( - binaryDictionaryInfo->getDictBuf(), key, headerSize - pos, &pos) == 0) { + dictBuf, key, headerSize - pos, &pos) == 0) { // The key was found. const int length = ByteArrayUtils::readStringAndAdvancePosition( - binaryDictionaryInfo->getDictBuf(), outValueSize, outValue, &pos); + dictBuf, outValueSize, outValue, &pos); // Add a 0 terminator to the string. outValue[length < outValueSize ? length : outValueSize - 1] = '\0'; return true; } - ByteArrayUtils::advancePositionToBehindString( - binaryDictionaryInfo->getDictBuf(), headerSize - pos, &pos); + ByteArrayUtils::advancePositionToBehindString(dictBuf, headerSize - pos, &pos); } // The key was not found. return false; } /* static */ int BinaryDictionaryHeaderReadingUtils::readHeaderValueInt( - const BinaryDictionaryInfo *const binaryDictionaryInfo, const char *const key) { + const uint8_t *const dictBuf, const char *const key) { const int bufferSize = LARGEST_INT_DIGIT_COUNT; int intBuffer[bufferSize]; char charBuffer[bufferSize]; - if (!readHeaderValue(binaryDictionaryInfo, key, intBuffer, bufferSize)) { + if (!readHeaderValue(dictBuf, key, intBuffer, bufferSize)) { return S_INT_MIN; } for (int i = 0; i < bufferSize; ++i) { diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h index 61748227e..deae9be27 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h @@ -20,21 +20,19 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/binary_dictionary_format_utils.h" namespace latinime { -class BinaryDictionaryInfo; - +// TODO:: Move header classes to policyimpl. class BinaryDictionaryHeaderReadingUtils { public: typedef uint16_t DictionaryFlags; static const int MAX_OPTION_KEY_LENGTH; - static int getHeaderSize(const BinaryDictionaryInfo *const binaryDictionaryInfo); + static int getHeaderSize(const uint8_t *const dictBuf); - static DictionaryFlags getFlags(const BinaryDictionaryInfo *const binaryDictionaryInfo); + static DictionaryFlags getFlags(const uint8_t *const dictBuf); static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) { return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0; @@ -48,33 +46,19 @@ class BinaryDictionaryHeaderReadingUtils { return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0; } - static AK_FORCE_INLINE int getHeaderOptionsPosition( - const BinaryDictionaryFormatUtils::FORMAT_VERSION dictionaryFormat) { - switch (getHeaderVersion(dictionaryFormat)) { - case HEADER_VERSION_2: - return VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE - + VERSION_2_HEADER_FLAG_SIZE + VERSION_2_HEADER_SIZE_FIELD_SIZE; - break; - default: - return NOT_A_DICT_POS; - } + static AK_FORCE_INLINE int getHeaderOptionsPosition() { + return VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE + + VERSION_2_HEADER_FLAG_SIZE + VERSION_2_HEADER_SIZE_FIELD_SIZE; } - static bool readHeaderValue( - const BinaryDictionaryInfo *const binaryDictionaryInfo, + static bool readHeaderValue(const uint8_t *const dictBuf, const char *const key, int *outValue, const int outValueSize); - static int readHeaderValueInt( - const BinaryDictionaryInfo *const binaryDictionaryInfo, const char *const key); + static int readHeaderValueInt(const uint8_t *const dictBuf, const char *const key); private: DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils); - enum HEADER_VERSION { - HEADER_VERSION_2, - UNKNOWN_HEADER_VERSION - }; - static const int VERSION_2_HEADER_MAGIC_NUMBER_SIZE; static const int VERSION_2_HEADER_DICTIONARY_VERSION_SIZE; static const int VERSION_2_HEADER_FLAG_SIZE; @@ -88,18 +72,6 @@ class BinaryDictionaryHeaderReadingUtils { static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG; static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG; static const DictionaryFlags CONTAINS_BIGRAMS_FLAG; - - static HEADER_VERSION getHeaderVersion( - const BinaryDictionaryFormatUtils::FORMAT_VERSION formatVersion) { - switch(formatVersion) { - case BinaryDictionaryFormatUtils::VERSION_2: - // Fall through - case BinaryDictionaryFormatUtils::VERSION_3: - return HEADER_VERSION_2; - default: - return UNKNOWN_HEADER_VERSION; - } - } }; } #endif /* LATINIME_DICTIONARY_HEADER_READING_UTILS_H */ diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h index c694c6a3a..818b2af56 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h @@ -36,7 +36,7 @@ class BinaryDictionaryInfo { mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable), mDictionaryFormat(BinaryDictionaryFormatUtils::detectFormatVersion( mDictBuf, mDictSize)), - mDictionaryHeader(this), mDictRoot(mDictBuf + mDictionaryHeader.getSize()), + mDictionaryHeader(dictBuf), mDictRoot(mDictBuf + mDictionaryHeader.getSize()), // TODO: Remove. mStructurePolicy(DictionaryStructureWithBufferPolicyFactory ::newDictionaryStructurePolicy(this)) { |