diff options
Diffstat (limited to 'native/src')
-rw-r--r-- | native/src/binary_format.h | 4 | ||||
-rw-r--r-- | native/src/dictionary.h | 75 | ||||
-rw-r--r-- | native/src/unigram_dictionary.cpp | 7 |
3 files changed, 7 insertions, 79 deletions
diff --git a/native/src/binary_format.h b/native/src/binary_format.h index 9944fa2bd..1d74998f6 100644 --- a/native/src/binary_format.h +++ b/native/src/binary_format.h @@ -61,7 +61,9 @@ inline int BinaryFormat::detectFormat(const uint8_t* const dict) { } inline int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t* const dict, int* pos) { - return dict[(*pos)++]; + const int msb = dict[(*pos)++]; + if (msb < 0x80) return msb; + return ((msb & 0x7F) << 8) | dict[(*pos)++]; } inline uint8_t BinaryFormat::getFlagsAndForwardPointer(const uint8_t* const dict, int* pos) { diff --git a/native/src/dictionary.h b/native/src/dictionary.h index 79d377a4f..90d7148d5 100644 --- a/native/src/dictionary.h +++ b/native/src/dictionary.h @@ -56,16 +56,7 @@ class Dictionary { // public static utility methods // static inline methods should be defined in the header file - static unsigned short getChar(const unsigned char *dict, int *pos); - static int getCount(const unsigned char *dict, int *pos); - static bool getTerminal(const unsigned char *dict, int *pos); - static int getAddress(const unsigned char *dict, int *pos); - static int getFreq(const unsigned char *dict, const bool isLatestDictVersion, int *pos); static int wideStrLen(unsigned short *str); - // returns next sibling's position - static int setDictionaryValues(const unsigned char *dict, const bool isLatestDictVersion, - const int pos, unsigned short *c, int *childrenPosition, - bool *terminal, int *freq); private: bool hasBigram(); @@ -87,56 +78,6 @@ class Dictionary { // public static utility methods // static inline methods should be defined in the header file -inline unsigned short Dictionary::getChar(const unsigned char *dict, int *pos) { - unsigned short ch = (unsigned short) (dict[(*pos)++] & 0xFF); - // If the code is 255, then actual 16 bit code follows (in big endian) - if (ch == 0xFF) { - ch = ((dict[*pos] & 0xFF) << 8) | (dict[*pos + 1] & 0xFF); - (*pos) += 2; - } - return ch; -} - -inline int Dictionary::getCount(const unsigned char *dict, int *pos) { - return dict[(*pos)++] & 0xFF; -} - -inline bool Dictionary::getTerminal(const unsigned char *dict, int *pos) { - return (dict[*pos] & FLAG_TERMINAL_MASK) > 0; -} - -inline int Dictionary::getAddress(const unsigned char *dict, int *pos) { - int address = 0; - if ((dict[*pos] & FLAG_ADDRESS_MASK) == 0) { - *pos += 1; - } else { - address += (dict[*pos] & (ADDRESS_MASK >> 16)) << 16; - address += (dict[*pos + 1] & 0xFF) << 8; - address += (dict[*pos + 2] & 0xFF); - *pos += 3; - } - return address; -} - -inline int Dictionary::getFreq(const unsigned char *dict, - const bool isLatestDictVersion, int *pos) { - int freq = dict[(*pos)++] & 0xFF; - if (isLatestDictVersion) { - // skipping bigram - int bigramExist = (dict[*pos] & FLAG_BIGRAM_READ); - if (bigramExist > 0) { - int nextBigramExist = 1; - while (nextBigramExist > 0) { - (*pos) += 3; - nextBigramExist = (dict[(*pos)++] & FLAG_BIGRAM_CONTINUED); - } - } else { - (*pos)++; - } - } - return freq; -} - inline int Dictionary::wideStrLen(unsigned short *str) { if (!str) return 0; unsigned short *end = str; @@ -144,22 +85,6 @@ inline int Dictionary::wideStrLen(unsigned short *str) { end++; return end - str; } - -inline int Dictionary::setDictionaryValues(const unsigned char *dict, - const bool isLatestDictVersion, const int pos, unsigned short *c,int *childrenPosition, - bool *terminal, int *freq) { - int position = pos; - // -- at char - *c = Dictionary::getChar(dict, &position); - // -- at flag/add - *terminal = Dictionary::getTerminal(dict, &position); - *childrenPosition = Dictionary::getAddress(dict, &position); - // -- after address or flag - *freq = (*terminal) ? Dictionary::getFreq(dict, isLatestDictVersion, &position) : 1; - // returns next sibling's position - return position; -} - } // namespace latinime #endif // LATINIME_DICTIONARY_H diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp index 1cff4d838..85e7e9bf2 100644 --- a/native/src/unigram_dictionary.cpp +++ b/native/src/unigram_dictionary.cpp @@ -277,7 +277,7 @@ void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, doAutoCompletion, maxErrors); int rootPosition = ROOT_POS; // Get the number of children of root, then increment the position - int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition); + int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition); int outputIndex = 0; correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0)); @@ -523,9 +523,10 @@ int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWor int maxFreq = -1; const uint8_t* const root = DICT_ROOT; - mStackChildCount[0] = root[0]; + int startPos = 0; + mStackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos); mStackInputIndex[0] = 0; - mStackSiblingPos[0] = 1; + mStackSiblingPos[0] = startPos; while (depth >= 0) { const int charGroupCount = mStackChildCount[depth]; int pos = mStackSiblingPos[depth]; |