diff options
Diffstat (limited to 'native/src/unigram_dictionary.cpp')
| -rw-r--r-- | native/src/unigram_dictionary.cpp | 97 |
1 files changed, 60 insertions, 37 deletions
diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp index 6c2c89231..d21413d8b 100644 --- a/native/src/unigram_dictionary.cpp +++ b/native/src/unigram_dictionary.cpp @@ -30,21 +30,25 @@ namespace latinime { const UnigramDictionary::digraph_t UnigramDictionary::GERMAN_UMLAUT_DIGRAPHS[] = - { { 'a', 'e' }, - { 'o', 'e' }, - { 'u', 'e' } }; + { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS + { 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS + { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS + +const UnigramDictionary::digraph_t UnigramDictionary::FRENCH_LIGATURES_DIGRAPHS[] = + { { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE + { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE // TODO: check the header UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier, int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars, const bool isLatestDictVersion) : DICT_ROOT(streamStart), MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords), - MAX_PROXIMITY_CHARS(maxProximityChars), IS_LATEST_DICT_VERSION(isLatestDictVersion), + IS_LATEST_DICT_VERSION(isLatestDictVersion), TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier), // TODO : remove this variable. ROOT_POS(0), - BYTES_IN_ONE_CHAR(MAX_PROXIMITY_CHARS * sizeof(int)), - MAX_UMLAUT_SEARCH_DEPTH(DEFAULT_MAX_UMLAUT_SEARCH_DEPTH) { + BYTES_IN_ONE_CHAR(sizeof(int)), + MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH) { if (DEBUG_DICT) { AKLOGI("UnigramDictionary - constructor"); } @@ -53,9 +57,8 @@ UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typed UnigramDictionary::~UnigramDictionary() { } -static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize, - const int MAX_PROXIMITY_CHARS) { - return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize; +static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize) { + return sizeof(*codes) * codesSize; } // TODO: This needs to take a const unsigned short* and not tinker with its contents @@ -64,23 +67,28 @@ static inline void addWord( queue->push(frequency, word, length); } -bool UnigramDictionary::isDigraph(const int *codes, const int i, const int codesSize) const { +// Return the replacement code point for a digraph, or 0 if none. +int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int codesSize, + const digraph_t* const digraphs, const unsigned int digraphsSize) const { // There can't be a digraph if we don't have at least 2 characters to examine if (i + 2 > codesSize) return false; // Search for the first char of some digraph int lastDigraphIndex = -1; - const int thisChar = codes[i * MAX_PROXIMITY_CHARS]; - for (lastDigraphIndex = sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0]) - 1; - lastDigraphIndex >= 0; --lastDigraphIndex) { - if (thisChar == GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].first) break; + const int thisChar = codes[i]; + for (lastDigraphIndex = digraphsSize - 1; lastDigraphIndex >= 0; --lastDigraphIndex) { + if (thisChar == digraphs[lastDigraphIndex].first) break; } // No match: return early - if (lastDigraphIndex < 0) return false; + if (lastDigraphIndex < 0) return 0; // It's an interesting digraph if the second char matches too. - return GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].second == codes[(i + 1) * MAX_PROXIMITY_CHARS]; + if (digraphs[lastDigraphIndex].second == codes[i + 1]) { + return digraphs[lastDigraphIndex].replacement; + } else { + return 0; + } } // Mostly the same arguments as the non-recursive version, except: @@ -94,38 +102,46 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit int *xCoordinatesBuffer, int *yCoordinatesBuffer, const int codesBufferSize, const int flags, const int *codesSrc, const int codesRemain, const int currentDepth, int *codesDest, Correction *correction, - WordsPriorityQueuePool *queuePool) { + WordsPriorityQueuePool *queuePool, + const digraph_t* const digraphs, const unsigned int digraphsSize) { - const int startIndex = (codesDest - codesBuffer) / MAX_PROXIMITY_CHARS; - if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) { + const int startIndex = codesDest - codesBuffer; + if (currentDepth < MAX_DIGRAPH_SEARCH_DEPTH) { for (int i = 0; i < codesRemain; ++i) { xCoordinatesBuffer[startIndex + i] = xcoordinates[codesBufferSize - codesRemain + i]; yCoordinatesBuffer[startIndex + i] = ycoordinates[codesBufferSize - codesRemain + i]; - if (isDigraph(codesSrc, i, codesRemain)) { + const int replacementCodePoint = + getDigraphReplacement(codesSrc, i, codesRemain, digraphs, digraphsSize); + if (0 != replacementCodePoint) { // Found a digraph. We will try both spellings. eg. the word is "pruefen" - // Copy the word up to the first char of the digraph, then continue processing - // on the remaining part of the word, skipping the second char of the digraph. - // In our example, copy "pru" and continue running on "fen" + // Copy the word up to the first char of the digraph, including proximity chars, + // and overwrite the primary code with the replacement code point. Then, continue + // processing on the remaining part of the word, skipping the second char of the + // digraph. + // In our example, copy "pru", replace "u" with the version with the diaeresis and + // continue running on "fen". // Make i the index of the second char of the digraph for simplicity. Forgetting // to do that results in an infinite recursion so take care! ++i; memcpy(codesDest, codesSrc, i * BYTES_IN_ONE_CHAR); + codesDest[(i - 1) * (BYTES_IN_ONE_CHAR / sizeof(codesDest[0]))] = + replacementCodePoint; getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize, flags, - codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1, - currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, correction, - queuePool); + codesSrc + i + 1, codesRemain - i - 1, + currentDepth + 1, codesDest + i, correction, + queuePool, digraphs, digraphsSize); // Copy the second char of the digraph in place, then continue processing on // the remaining part of the word. // In our example, after "pru" in the buffer copy the "e", and continue on "fen" - memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS, - BYTES_IN_ONE_CHAR); + memcpy(codesDest + i, codesSrc + i, BYTES_IN_ONE_CHAR); getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize, flags, - codesSrc + i * MAX_PROXIMITY_CHARS, codesRemain - i, currentDepth + 1, - codesDest + i * MAX_PROXIMITY_CHARS, correction, queuePool); + codesSrc + i, codesRemain - i, currentDepth + 1, + codesDest + i, correction, queuePool, + digraphs, digraphsSize); return; } } @@ -159,12 +175,23 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, Correction* masterCorrection = correction; if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags) { // Incrementally tune the word and try all possibilities - int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)]; + int codesBuffer[getCodesBufferSize(codes, codesSize)]; int xCoordinatesBuffer[codesSize]; int yCoordinatesBuffer[codesSize]; getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, - codesSize, flags, codes, codesSize, 0, codesBuffer, masterCorrection, queuePool); + codesSize, flags, codes, codesSize, 0, codesBuffer, masterCorrection, queuePool, + GERMAN_UMLAUT_DIGRAPHS, + sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0])); + } else if (REQUIRES_FRENCH_LIGATURES_PROCESSING & flags) { + int codesBuffer[getCodesBufferSize(codes, codesSize)]; + int xCoordinatesBuffer[codesSize]; + int yCoordinatesBuffer[codesSize]; + getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, + xCoordinatesBuffer, yCoordinatesBuffer, + codesSize, flags, codes, codesSize, 0, codesBuffer, masterCorrection, queuePool, + FRENCH_LIGATURES_DIGRAPHS, + sizeof(FRENCH_LIGATURES_DIGRAPHS) / sizeof(FRENCH_LIGATURES_DIGRAPHS[0])); } else { // Normal processing getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, flags, masterCorrection, queuePool); @@ -398,7 +425,7 @@ bool UnigramDictionary::getSubStringSuggestion( if (inputWordStartPos > 0) { const int offset = inputWordStartPos; initSuggestions(proximityInfo, &xcoordinates[offset], &ycoordinates[offset], - codes + offset * MAX_PROXIMITY_CHARS, inputWordLength, correction); + codes + offset, inputWordLength, correction); queuePool->clearSubQueue(currentWordIndex); getSuggestionCandidates(useFullEditDistance, inputWordLength, correction, queuePool, false, MAX_ERRORS_FOR_TWO_WORDS, currentWordIndex); @@ -557,10 +584,6 @@ void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximit const bool hasAutoCorrectionCandidate) { if (inputLength >= MAX_WORD_LENGTH) return; if (DEBUG_DICT) { - // MAX_PROXIMITY_CHARS_SIZE in ProximityInfo.java should be 16 - assert(MAX_PROXIMITY_CHARS == 16); - } - if (DEBUG_DICT) { AKLOGI("--- Suggest multiple words"); } |