diff options
13 files changed, 201 insertions, 125 deletions
diff --git a/java/res/values/donottranslate.xml b/java/res/values/donottranslate.xml index 82c5ce456..42e692d2f 100644 --- a/java/res/values/donottranslate.xml +++ b/java/res/values/donottranslate.xml @@ -158,47 +158,47 @@ <!-- Compatibility map from subtypeLocale:subtypeExtraValue to keyboardLayoutSet --> <string-array name="locale_and_extra_value_to_keyboard_layout_set_map"> - <item>en_US:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection</item> + <item>en_US:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> - <item>en_GB:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection</item> + <item>en_GB:TrySuppressingImeSwitcher,AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> - <item>ar:SupportTouchPositionCorrection</item> + <item>ar:SupportTouchPositionCorrection,EmojiCapable</item> <item>arabic</item> - <item>cs:AsciiCapable,SupportTouchPositionCorrection</item> + <item>cs:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwertz</item> - <item>da:AsciiCapable,SupportTouchPositionCorrection</item> + <item>da:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>nordic</item> - <item>de:AsciiCapable,SupportTouchPositionCorrection</item> + <item>de:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwertz</item> - <item>es:AsciiCapable,SupportTouchPositionCorrection</item> + <item>es:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>spanish</item> - <item>fi:AsciiCapable,SupportTouchPositionCorrection</item> + <item>fi:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>nordic</item> - <item>fr:AsciiCapable,SupportTouchPositionCorrection</item> + <item>fr:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>azerty</item> - <item>fr_CA:AsciiCapable,SupportTouchPositionCorrection</item> + <item>fr_CA:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> - <item>hr:AsciiCapable,SupportTouchPositionCorrection</item> + <item>hr:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwertz</item> - <item>hu:AsciiCapable,SupportTouchPositionCorrection</item> + <item>hu:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwertz</item> - <item>it:AsciiCapable,SupportTouchPositionCorrection</item> + <item>it:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> - <item>iw:SupportTouchPositionCorrection</item> + <item>iw:SupportTouchPositionCorrection,EmojiCapable</item> <item>hebrew</item> - <item>nb:AsciiCapable,SupportTouchPositionCorrection</item> + <item>nb:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>nordic</item> - <item>nl:AsciiCapable,SupportTouchPositionCorrection</item> + <item>nl:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> - <item>pl:AsciiCapable,SupportTouchPositionCorrection</item> + <item>pl:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> - <item>ru:SupportTouchPositionCorrection</item> + <item>ru:SupportTouchPositionCorrection,EmojiCapable</item> <item>east_slavic</item> - <item>sr:SupportTouchPositionCorrection</item> + <item>sr:SupportTouchPositionCorrection,EmojiCapable</item> <item>south_slavic</item> - <item>sv:AsciiCapable,SupportTouchPositionCorrection</item> + <item>sv:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>nordic</item> - <item>tr:AsciiCapable,SupportTouchPositionCorrection</item> + <item>tr:AsciiCapable,SupportTouchPositionCorrection,EmojiCapable</item> <item>qwerty</item> </string-array> diff --git a/java/res/xml-sw600dp/row_pcqwerty5.xml b/java/res/xml-sw600dp/row_pcqwerty5.xml index a79d2a87f..b854f1051 100644 --- a/java/res/xml-sw600dp/row_pcqwerty5.xml +++ b/java/res/xml-sw600dp/row_pcqwerty5.xml @@ -53,7 +53,7 @@ latin:keyXPos="-9.0%p" latin:keyWidth="9.0%p" latin:backgroundType="functional" - latin:keyboardLayout="@xml/key_symbols_period" /> + latin:keyboardLayout="@xml/key_f2" /> </default> </switch> </Row> diff --git a/java/res/xml/method.xml b/java/res/xml/method.xml index d7424c0c7..d92dc29da 100644 --- a/java/res/xml/method.xml +++ b/java/res/xml/method.xml @@ -62,8 +62,8 @@ mn: Mongolian/mongolian ms: Malay/qwerty nb: Norwegian Bokmål/nordic - ne: Nepali Romanized/nepali_romanized - ne: Nepali Traditional/nepali_traditional + ne: Nepali Romanized/nepali_romanized # disabled temporarily + ne: Nepali Traditional/nepali_traditional # disabled temporarily nl: Dutch/qwerty nl_BE: Dutch Belgium/azerty pl: Polish/qwerty @@ -380,6 +380,7 @@ android:imeSubtypeMode="keyboard" android:imeSubtypeExtraValue="AsciiCapable,SupportTouchPositionCorrection,EmojiCapable" /> + <!-- <subtype android:icon="@drawable/ic_subtype_keyboard" android:label="@string/subtype_generic" android:subtypeId="0xd80a4cee" @@ -394,6 +395,7 @@ android:imeSubtypeMode="keyboard" android:imeSubtypeExtraValue="KeyboardLayoutSet=nepali_traditional,EmojiCapable" /> + --> <subtype android:icon="@drawable/ic_subtype_keyboard" android:label="@string/subtype_generic" android:subtypeId="0x3f9fd91e" diff --git a/java/res/xml/row_pcqwerty5.xml b/java/res/xml/row_pcqwerty5.xml index 0e618059e..4ec908ba1 100644 --- a/java/res/xml/row_pcqwerty5.xml +++ b/java/res/xml/row_pcqwerty5.xml @@ -51,13 +51,13 @@ latin:keyWidth="11.538%p" /> <Key latin:keyStyle="spaceKeyStyle" - latin:keyWidth="42.310%p" /> + latin:keyWidth="38.464%p" /> </case> <!-- languageSwitchKeyEnabled="false" --> <default> <Key latin:keyStyle="spaceKeyStyle" - latin:keyWidth="53.848%p" /> + latin:keyWidth="50.002%p" /> </default> </switch> <Key @@ -71,9 +71,9 @@ </case> <!-- keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" --> <default> - <include - latin:keyboardLayout="@xml/key_symbols_period" - latin:backgroundType="functional" /> + <Key + latin:keyStyle="emojiKeyStyle" + latin:keyWidth="fillRight" /> </default> </switch> </Row> diff --git a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java index cd127c760..a0316696c 100644 --- a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java +++ b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java @@ -245,8 +245,8 @@ public class ProximityInfo { final int threshold = (int) (defaultWidth * SEARCH_DISTANCE); final int thresholdSquared = threshold * threshold; // Round-up so we don't have any pixels outside the grid - final int fullGridWidth = mGridWidth * mCellWidth; - final int fullGridHeight = mGridHeight * mCellHeight; + final int lastPixelXCoordinate = mGridWidth * mCellWidth - 1; + final int lastPixelYCoordinate = mGridHeight * mCellHeight - 1; // For large layouts, 'neighborsFlatBuffer' is about 80k of memory: gridSize is usually 512, // keycount is about 40 and a pointer to a Key is 4 bytes. This contains, for each cell, @@ -329,22 +329,20 @@ y |---+---+---+---+-v-+-|-+---+---+---+---+---| | thresholdBase and get final int yMiddleOfTopCell = topPixelWithinThreshold - yDeltaToGrid + halfCellHeight; final int yStart = Math.max(halfCellHeight, yMiddleOfTopCell + (yDeltaToGrid <= halfCellHeight ? 0 : mCellHeight)); - final int yEnd = Math.min(fullGridHeight, keyY + key.getHeight() + threshold); + final int yEnd = Math.min(lastPixelYCoordinate, keyY + key.getHeight() + threshold); final int leftPixelWithinThreshold = keyX - threshold; final int xDeltaToGrid = leftPixelWithinThreshold % mCellWidth; final int xMiddleOfLeftCell = leftPixelWithinThreshold - xDeltaToGrid + halfCellWidth; final int xStart = Math.max(halfCellWidth, xMiddleOfLeftCell + (xDeltaToGrid <= halfCellWidth ? 0 : mCellWidth)); - final int xEnd = Math.min(fullGridWidth, keyX + key.getWidth() + threshold); + final int xEnd = Math.min(lastPixelXCoordinate, keyX + key.getWidth() + threshold); int baseIndexOfCurrentRow = (yStart / mCellHeight) * mGridWidth + (xStart / mCellWidth); for (int centerY = yStart; centerY <= yEnd; centerY += mCellHeight) { int index = baseIndexOfCurrentRow; for (int centerX = xStart; centerX <= xEnd; centerX += mCellWidth) { - // TODO: Remove "index < neighborCountPerCell.length" below. - if (index < neighborCountPerCell.length - && key.squaredDistanceToEdge(centerX, centerY) < thresholdSquared) { + if (key.squaredDistanceToEdge(centerX, centerY) < thresholdSquared) { neighborsFlatBuffer[index * keyCount + neighborCountPerCell[index]] = key; ++neighborCountPerCell[index]; } diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 2a9076436..fcd7ede1a 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -311,9 +311,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } @Override - public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, + final int sessionId) { reloadDictionaryIfRequired(); final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = @@ -321,14 +322,16 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - final ArrayList<SuggestedWordInfo> inMemDictSuggestion = - mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords, additionalFeaturesOptions); + final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ? + null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + sessionId); // TODO: Remove checking mIsUpdatable and use native suggestion. if (mBinaryDictionary != null && !mIsUpdatable) { final ArrayList<SuggestedWordInfo> binarySuggestion = - mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords, additionalFeaturesOptions); + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + sessionId); if (inMemDictSuggestion == null) { holder.set(binarySuggestion); } else if (binarySuggestion == null) { @@ -342,11 +345,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } }); - return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, 0 /* sessionId */); + } + + @Override public boolean isValidWord(final String word) { reloadDictionaryIfRequired(); return isValidWordInner(word); diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 1684d47b5..6c18c948f 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -361,12 +361,6 @@ public final class Suggest { // At second character typed, search the unigrams (scores being affected by bigrams) for (final String key : mDictionaries.keySet()) { - // Skip User history dictionary for lookup - // TODO: The user history dictionary should just override getSuggestionsWithSessionId - // to make sure it doesn't return anything and we should remove this test - if (key.equals(Dictionary.TYPE_USER_HISTORY)) { - continue; - } final Dictionary dictionary = mDictionaries.get(key); suggestionsSet.addAll(dictionary.getSuggestionsWithSessionId(wordComposer, prevWordForBigram, proximityInfo, blockOffensiveWords, diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h index 884bcd7a9..9092a8019 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h @@ -98,6 +98,13 @@ public: flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; *outOffsetFieldSize = 1; } + + // Currently, all newly written bigram position fields are 3 bytes to simplify dictionary + // writing. + // TODO: Remove following 2 lines and optimize memory space. + flags = (flags & (~MASK_ATTRIBUTE_ADDRESS_TYPE)) | FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + *outOffsetFieldSize = 3; + *outBigramFlags = flags; *outOffset = absOffest; return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index 4c44d22fd..bd58b99e7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -54,8 +54,8 @@ void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const { } } -bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos, - int *outBigramsCount) { +bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, + int *const fromPos, int *const toPos, int *const outBigramsCount) const { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); @@ -86,10 +86,10 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo continue; } // Write bigram entry. Target buffer is always the additional buffer. - if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) { + if (!bufferToWrite->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) { return false; } - if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, + if (!bufferToWrite->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, toPos)) { return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index dafb62d80..5d02d3274 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -44,10 +44,11 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { void skipAllBigrams(int *const pos) const; - // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these - // positions after bigram lists. This method skips invalid bigram entries and write the valid - // bigram entry count to outBigramsCount. - bool copyAllBigrams(int *const fromPos, int *const toPos, int *outBigramsCount); + // Copy bigrams from the bigram list that starts at fromPos in mBuffer to toPos in + // bufferToWrite and advance these positions after bigram lists. This method skips invalid + // bigram entries and write the valid bigram entry count to outBigramsCount. + bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos, + int *const toPos, int *const outBigramsCount) const; bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index 31178fb5c..f4c98d848 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -97,8 +97,8 @@ bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const return false; } int writingPos = newNodePos; - // Write a new PtNode using original PtNode's info to the tail of the dictionary. - if (!writePtNodeToBufferByCopyingPtNodeInfo(&nodeReader, nodeReader.getParentPos(), + // Write a new PtNode using original PtNode's info to the tail of the dictionary in mBuffer. + if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, &nodeReader, nodeReader.getParentPos(), mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(), &writingPos)) { return false; @@ -143,38 +143,20 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) { return; } - const int tmpFileNameBufSize = strlen(fileName) - + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1; - char tmpFileName[tmpFileNameBufSize]; - snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, - TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); - FILE *const file = fopen(tmpFileName, "wb"); - if (!file) { - return; - } - // Write header. - if (fwrite(headerBuffer.getBuffer(true /* usesAdditionalBuffer */), - headerBuffer.getTailPosition(), 1, file) < 1) { - fclose(file); - remove(tmpFileName); - return; - } - // Write data in original buffer. - if (fwrite(mBuffer->getBuffer(false /* usesAdditionalBuffer */), - mBuffer->getOriginalBufferSize(), 1, file) < 1) { - fclose(file); - remove(tmpFileName); + flushAllToFile(fileName, &headerBuffer, mBuffer); +} + +void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, + const char *const fileName, const HeaderPolicy *const headerPolicy) { + BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); + if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) { return; } - // Write data in additional buffer. - if (fwrite(mBuffer->getBuffer(true /* usesAdditionalBuffer */), - mBuffer->getTailPosition() - mBuffer->getOriginalBufferSize(), 1, file) < 1) { - fclose(file); - remove(tmpFileName); + BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); + if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) { return; } - fclose(file); - rename(tmpFileName, fileName); + flushAllToFile(fileName, &headerBuffer, &newDictBuffer); } bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( @@ -232,7 +214,8 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( } // Write new PtNode at writingPos. -bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, +bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer( + BufferWithExtendableBuffer *const bufferToWrite, const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, @@ -240,38 +223,39 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo const int nodePos = *writingPos; // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // PtNode writing. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */, - writingPos)) { + if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, + 0 /* nodeFlags */, writingPos)) { return false; } // Calculate a parent offset and write the offset. const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; - if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBuffer, + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(bufferToWrite, parentOffset, writingPos)) { return false; } // Write code points - if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints, - codePointCount, writingPos)) { + if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(bufferToWrite, + codePoints, codePointCount, writingPos)) { return false; } // Write probability when the probability is a valid probability, which means this node is // terminal. if (probability != NOT_A_PROBABILITY) { - if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer, + if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(bufferToWrite, probability, writingPos)) { return false; } } // Write children position - if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(bufferToWrite, childrenPos, writingPos)) { return false; } // Copy shortcut list when the originalShortcutListPos is valid dictionary position. if (originalShortcutListPos != NOT_A_DICT_POS) { int fromPos = originalShortcutListPos; - if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(&fromPos, writingPos)) { + if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(bufferToWrite, &fromPos, + writingPos)) { return false; } } @@ -279,7 +263,7 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo int bigramCount = 0; if (originalBigramListPos != NOT_A_DICT_POS) { int fromPos = originalBigramListPos; - if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) { + if (!mBigramPolicy->copyAllBigrams(bufferToWrite, &fromPos, writingPos, &bigramCount)) { return false; } } @@ -291,27 +275,29 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */, CHILDREN_POSITION_FIELD_SIZE); int flagsFieldPos = nodePos; - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags, + if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, nodeFlags, &flagsFieldPos)) { return false; } return true; } -bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(const int parentPos, +bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer( + BufferWithExtendableBuffer *const bufferToWrite, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos) { - return writePtNodeWithFullInfoToBuffer(false /* isBlacklisted */, false /* isNotAWord */, - parentPos, codePoints, codePointCount, probability, + return writePtNodeWithFullInfoToBuffer(bufferToWrite, false /* isBlacklisted */, + false /* isNotAWord */, parentPos, codePoints, codePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, writingPos); } bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( + BufferWithExtendableBuffer *const bufferToWrite, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos) { - return writePtNodeWithFullInfoToBuffer(originalNode->isBlacklisted(), + return writePtNodeWithFullInfoToBuffer(bufferToWrite, originalNode->isBlacklisted(), originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, originalNode->getChildrenPos(), originalNode->getBigramsPos(), originalNode->getShortcutPos(), writingPos); @@ -345,8 +331,9 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { return false; } - if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), - codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { + if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, originalPtNode, + originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(), + probability, &movedPos)) { return false; } } @@ -374,8 +361,8 @@ bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( 1 /* arraySize */, &writingPos)) { return false; } - if (!writePtNodeToBuffer(parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, - &writingPos)) { + if (!writePtNodeToBuffer(mBuffer, parentPtNodePos, nodeCodePoints, nodeCodePointCount, + probability, &writingPos)) { return false; } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, @@ -404,8 +391,9 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( // Write the 1st part of the reallocating node. The children position will be updated later // with actual children position. const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; - if (!writePtNodeToBuffer(reallocatingPtNode->getParentPos(), reallocatingPtNodeCodePoints, - overlappingCodePointCount, newProbability, &writingPos)) { + if (!writePtNodeToBuffer(mBuffer, reallocatingPtNode->getParentPos(), + reallocatingPtNodeCodePoints, overlappingCodePointCount, newProbability, + &writingPos)) { return false; } const int actualChildrenPos = writingPos; @@ -417,14 +405,15 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( } // Write the 2nd part of the reallocating node. const int secondPartOfReallocatedPtNodePos = writingPos; - if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, firstPartOfReallocatedPtNodePos, + if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, reallocatingPtNode, + firstPartOfReallocatedPtNodePos, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getProbability(), &writingPos)) { return false; } if (addsExtraChild) { - if (!writePtNodeToBuffer(firstPartOfReallocatedPtNodePos, + if (!writePtNodeToBuffer(mBuffer, firstPartOfReallocatedPtNodePos, newNodeCodePoints + overlappingCodePointCount, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, &writingPos)) { @@ -452,4 +441,64 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( return true; } +// TODO: Create a struct which contains header, body and etc... and use here as an argument. +void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName, + BufferWithExtendableBuffer *const dictHeader, + BufferWithExtendableBuffer *const dictBody) const { + const int tmpFileNameBufSize = strlen(fileName) + + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */; + // Name of a temporary file used for writing that is a connected string of original name and + // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE. + char tmpFileName[tmpFileNameBufSize]; + snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, + TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); + FILE *const file = fopen(tmpFileName, "wb"); + if (!file) { + AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName); + ASSERT(false); + return; + } + // Write the dictionary header. + if (!writeBufferToFilePointer(file, dictHeader)) { + remove(tmpFileName); + AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition()); + ASSERT(false); + return; + } + // Write the dictionary body. + if (!writeBufferToFilePointer(file, dictBody)) { + remove(tmpFileName); + AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition()); + ASSERT(false); + return; + } + fclose(file); + rename(tmpFileName, fileName); +} + +// This closes file pointer when an error is caused and returns whether the writing was succeeded +// or not. +bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file, + const BufferWithExtendableBuffer *const buffer) const { + const int originalBufSize = buffer->getOriginalBufferSize(); + if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */), + originalBufSize, 1, file) < 1) { + fclose(file); + return false; + } + const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize(); + if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */), + additionalBufSize, 1, file) < 1) { + fclose(file); + return false; + } + return true; +} + +bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, + BufferWithExtendableBuffer *const bufferToWrite) { + // TODO: Implement. + return false; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index 219ea9857..8f78d84d0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -17,6 +17,7 @@ #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H +#include <cstdio> #include <stdint.h> #include "defines.h" @@ -51,7 +52,8 @@ class DynamicPatriciaTrieWritingHelper { void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy); - void writeToDictFileWithGC(const char *const fileName, const HeaderPolicy *const headerPolicy); + void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName, + const HeaderPolicy *const headerPolicy); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); @@ -66,15 +68,17 @@ class DynamicPatriciaTrieWritingHelper { bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos, const int bigramLinkedNodePos); - bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, + bool writePtNodeWithFullInfoToBuffer(BufferWithExtendableBuffer *const bufferToWrite, + const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos); - bool writePtNodeToBuffer(const int parentPos, const int *const codePoints, - const int codePointCount, const int probability, int *const writingPos); + bool writePtNodeToBuffer(BufferWithExtendableBuffer *const bufferToWrite, + const int parentPos, const int *const codePoints, const int codePointCount, + const int probability, int *const writingPos); - bool writePtNodeToBufferByCopyingPtNodeInfo( + bool writePtNodeToBufferByCopyingPtNodeInfo(BufferWithExtendableBuffer *const bufferToWrite, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos); @@ -97,6 +101,15 @@ class DynamicPatriciaTrieWritingHelper { const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int newNodeCodePointCount); + + void flushAllToFile(const char *const fileName, + BufferWithExtendableBuffer *const dictHeader, + BufferWithExtendableBuffer *const dictBody) const; + + bool writeBufferToFilePointer(FILE *const file, + const BufferWithExtendableBuffer *const buffer) const; + + bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite); }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h index 1803c09cb..bd3211f6a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h @@ -31,7 +31,7 @@ namespace latinime { */ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { public: - explicit DynamicShortcutListPolicy(BufferWithExtendableBuffer *const buffer) + explicit DynamicShortcutListPolicy(const BufferWithExtendableBuffer *const buffer) : mBuffer(buffer) {} ~DynamicShortcutListPolicy() {} @@ -82,18 +82,20 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { } } - // Copy shortcuts from the shortcut list that starts at fromPos to toPos and advance these - // positions after the shortcut lists. This returns whether the copy was succeeded or not. - bool copyAllShortcutsAndReturnIfSucceededOrNot(int *const fromPos, int *const toPos) { + // Copy shortcuts from the shortcut list that starts at fromPos in mBuffer to toPos in + // bufferToWrite and advance these positions after the shortcut lists. This returns whether + // the copy was succeeded or not. + bool copyAllShortcutsAndReturnIfSucceededOrNot(BufferWithExtendableBuffer *const bufferToWrite, + int *const fromPos, int *const toPos) const { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); } const int shortcutListSize = ShortcutListReadingUtils - ::getShortcutListSizeAndForwardPointer(buffer, fromPos); + ::getShortcutListSizeAndForwardPointer(mBuffer->getBuffer(usesAdditionalBuffer), + fromPos); // Copy shortcut list size. - if (!mBuffer->writeUintAndAdvancePosition( + if (!bufferToWrite->writeUintAndAdvancePosition( shortcutListSize + ShortcutListReadingUtils::getShortcutListSizeFieldSize(), ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos)) { return false; @@ -102,7 +104,7 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { for (int i = 0; i < shortcutListSize; ++i) { const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition( mBuffer->getBuffer(usesAdditionalBuffer), fromPos); - if (!mBuffer->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) { + if (!bufferToWrite->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) { return false; } } @@ -115,7 +117,7 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicShortcutListPolicy); - BufferWithExtendableBuffer *const mBuffer; + const BufferWithExtendableBuffer *const mBuffer; }; } // namespace latinime #endif // LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H |