diff options
18 files changed, 214 insertions, 47 deletions
diff --git a/java/src/com/android/inputmethod/dictionarypack/DictionaryService.java b/java/src/com/android/inputmethod/dictionarypack/DictionaryService.java index 41916b614..568c80abd 100644 --- a/java/src/com/android/inputmethod/dictionarypack/DictionaryService.java +++ b/java/src/com/android/inputmethod/dictionarypack/DictionaryService.java @@ -76,19 +76,29 @@ public final class DictionaryService extends Service { * How often, in milliseconds, we want to update the metadata. This is a * floor value; actually, it may happen several hours later, or even more. */ - private static final long UPDATE_FREQUENCY = TimeUnit.DAYS.toMillis(4); + private static final long UPDATE_FREQUENCY_MILLIS = TimeUnit.DAYS.toMillis(4); /** * We are waked around midnight, local time. We want to wake between midnight and 6 am, * roughly. So use a random time between 0 and this delay. */ - private static final int MAX_ALARM_DELAY = (int)TimeUnit.HOURS.toMillis(6); + private static final int MAX_ALARM_DELAY_MILLIS = (int)TimeUnit.HOURS.toMillis(6); /** * How long we consider a "very long time". If no update took place in this time, * the content provider will trigger an update in the background. */ - private static final long VERY_LONG_TIME = TimeUnit.DAYS.toMillis(14); + private static final long VERY_LONG_TIME_MILLIS = TimeUnit.DAYS.toMillis(14); + + /** + * After starting a download, how long we wait before considering it may be stuck. After this + * period is elapsed, if the keyboard tries to download again, then we cancel and re-register + * the request; if it's within this time, we just leave it be. + * It's important to note that we do not re-submit the request merely because the time is up. + * This is only to decide whether to cancel the old one and re-requesting when the keyboard + * fires a new request for the same data. + */ + public static final long NO_CANCEL_DOWNLOAD_PERIOD_MILLIS = TimeUnit.SECONDS.toMillis(30); /** * An executor that serializes tasks given to it. @@ -188,16 +198,16 @@ public final class DictionaryService extends Service { */ private static void checkTimeAndMaybeSetupUpdateAlarm(final Context context) { // Of all clients, if the one that hasn't been updated for the longest - // is still more recent than UPDATE_FREQUENCY, do nothing. - if (!isLastUpdateAtLeastThisOld(context, UPDATE_FREQUENCY)) return; + // is still more recent than UPDATE_FREQUENCY_MILLIS, do nothing. + if (!isLastUpdateAtLeastThisOld(context, UPDATE_FREQUENCY_MILLIS)) return; PrivateLog.log("Date changed - registering alarm"); AlarmManager alarmManager = (AlarmManager)context.getSystemService(Context.ALARM_SERVICE); - // Best effort to wake between midnight and MAX_ALARM_DELAY in the morning. + // Best effort to wake between midnight and MAX_ALARM_DELAY_MILLIS in the morning. // It doesn't matter too much if this is very inexact. final long now = System.currentTimeMillis(); - final long alarmTime = now + new Random().nextInt(MAX_ALARM_DELAY); + final long alarmTime = now + new Random().nextInt(MAX_ALARM_DELAY_MILLIS); final Intent updateIntent = new Intent(DictionaryPackConstants.UPDATE_NOW_INTENT_ACTION); final PendingIntent pendingIntent = PendingIntent.getBroadcast(context, 0, updateIntent, PendingIntent.FLAG_CANCEL_CURRENT); @@ -223,11 +233,11 @@ public final class DictionaryService extends Service { /** * Refreshes data if it hasn't been refreshed in a very long time. * - * This will check the last update time, and if it's been more than VERY_LONG_TIME, + * This will check the last update time, and if it's been more than VERY_LONG_TIME_MILLIS, * update metadata now - and possibly take subsequent update actions. */ public static void updateNowIfNotUpdatedInAVeryLongTime(final Context context) { - if (!isLastUpdateAtLeastThisOld(context, VERY_LONG_TIME)) return; + if (!isLastUpdateAtLeastThisOld(context, VERY_LONG_TIME_MILLIS)) return; UpdateHandler.tryUpdate(context, false); } diff --git a/java/src/com/android/inputmethod/dictionarypack/DownloadIdAndStartDate.java b/java/src/com/android/inputmethod/dictionarypack/DownloadIdAndStartDate.java new file mode 100644 index 000000000..6247a15e2 --- /dev/null +++ b/java/src/com/android/inputmethod/dictionarypack/DownloadIdAndStartDate.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.dictionarypack; + +/** + * A simple container of download ID and download start date. + */ +public class DownloadIdAndStartDate { + public final long mId; + public final long mStartDate; + public DownloadIdAndStartDate(final long id, final long startDate) { + mId = id; + mStartDate = startDate; + } +} diff --git a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java index e9dde4245..c9e8f9118 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java +++ b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java @@ -433,18 +433,18 @@ public class MetadataDbHelper extends SQLiteOpenHelper { * * @param context a context instance to open the database on * @param uri the URI to retrieve the metadata download ID of - * @return the metadata download ID, or NOT_AN_ID if no download is in progress + * @return the download id and start date, or null if the URL is not known */ - public static long getMetadataDownloadIdForURI(final Context context, - final String uri) { + public static DownloadIdAndStartDate getMetadataDownloadIdAndStartDateForURI( + final Context context, final String uri) { SQLiteDatabase defaultDb = getDb(context, null); final Cursor cursor = defaultDb.query(CLIENT_TABLE_NAME, - new String[] { CLIENT_PENDINGID_COLUMN }, + new String[] { CLIENT_PENDINGID_COLUMN, CLIENT_LAST_UPDATE_DATE_COLUMN }, CLIENT_METADATA_URI_COLUMN + " = ?", new String[] { uri }, null, null, null, null); try { - if (!cursor.moveToFirst()) return UpdateHandler.NOT_AN_ID; - return cursor.getInt(0); // Only one column, return it + if (!cursor.moveToFirst()) return null; + return new DownloadIdAndStartDate(cursor.getInt(0), cursor.getLong(1)); } finally { cursor.close(); } @@ -922,6 +922,7 @@ public class MetadataDbHelper extends SQLiteOpenHelper { final long downloadId) { final ContentValues values = new ContentValues(); values.put(CLIENT_PENDINGID_COLUMN, downloadId); + values.put(CLIENT_LAST_UPDATE_DATE_COLUMN, System.currentTimeMillis()); final SQLiteDatabase defaultDb = getDb(context, ""); final Cursor cursor = MetadataDbHelper.queryClientIds(context); if (null == cursor) return; diff --git a/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java b/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java index d8c5f3165..90a750493 100644 --- a/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java +++ b/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java @@ -251,12 +251,16 @@ public final class UpdateHandler { res.getBoolean(R.bool.metadata_downloads_visible_in_download_UI)); final DownloadManagerWrapper manager = new DownloadManagerWrapper(context); - cancelUpdateWithDownloadManager(context, metadataUri, manager); + if (maybeCancelUpdateAndReturnIfStillRunning(context, metadataUri, manager, + DictionaryService.NO_CANCEL_DOWNLOAD_PERIOD_MILLIS)) { + // We already have a recent download in progress. Don't register a new download. + return; + } final long downloadId; synchronized (sSharedIdProtector) { downloadId = manager.enqueue(metadataRequest); DebugLogUtils.l("Metadata download requested with id", downloadId); - // If there is already a download in progress, it's been there for a while and + // If there is still a download in progress, it's been there for a while and // there is probably something wrong with download manager. It's best to just // overwrite the id and request it again. If the old one happens to finish // anyway, we don't know about its ID any more, so the downloadFinished @@ -267,21 +271,29 @@ public final class UpdateHandler { } /** - * Cancels downloading a file, if there is one for this URI. + * Cancels downloading a file if there is one for this URI and it's too long. * * If we are not currently downloading the file at this URI, this is a no-op. * * @param context the context to open the database on * @param metadataUri the URI to cancel * @param manager an wrapped instance of DownloadManager + * @param graceTime if there was a download started less than this many milliseconds, don't + * cancel and return true + * @return whether the download is still active */ - private static void cancelUpdateWithDownloadManager(final Context context, - final String metadataUri, final DownloadManagerWrapper manager) { + private static boolean maybeCancelUpdateAndReturnIfStillRunning(final Context context, + final String metadataUri, final DownloadManagerWrapper manager, final long graceTime) { synchronized (sSharedIdProtector) { - final long metadataDownloadId = - MetadataDbHelper.getMetadataDownloadIdForURI(context, metadataUri); - if (NOT_AN_ID == metadataDownloadId) return; - manager.remove(metadataDownloadId); + final DownloadIdAndStartDate metadataDownloadIdAndStartDate = + MetadataDbHelper.getMetadataDownloadIdAndStartDateForURI(context, metadataUri); + if (null == metadataDownloadIdAndStartDate) return false; + if (NOT_AN_ID == metadataDownloadIdAndStartDate.mId) return false; + if (metadataDownloadIdAndStartDate.mStartDate + graceTime + > System.currentTimeMillis()) { + return true; + } + manager.remove(metadataDownloadIdAndStartDate.mId); writeMetadataDownloadId(context, metadataUri, NOT_AN_ID); } // Consider a cancellation as a failure. As such, inform listeners that the download @@ -289,6 +301,7 @@ public final class UpdateHandler { for (UpdateEventListener listener : linkedCopyOfList(sUpdateEventListeners)) { listener.downloadedMetadata(false); } + return false; } /** @@ -303,7 +316,7 @@ public final class UpdateHandler { public static void cancelUpdate(final Context context, final String clientId) { final DownloadManagerWrapper manager = new DownloadManagerWrapper(context); final String metadataUri = MetadataDbHelper.getMetadataUriAsString(context, clientId); - cancelUpdateWithDownloadManager(context, metadataUri, manager); + maybeCancelUpdateAndReturnIfStillRunning(context, metadataUri, manager, 0 /* graceTime */); } /** @@ -387,7 +400,7 @@ public final class UpdateHandler { // If any of these is metadata, we should update the DB boolean hasMetadata = false; for (DownloadRecord record : downloadRecords) { - if (null == record.mAttributes) { + if (record.isMetadata()) { hasMetadata = true; break; } diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java index b421a7eb5..e40fd8800 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java @@ -400,6 +400,7 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick if (mStripVisibilityGroup.isShowingImportantNoticeStrip()) { return false; } + // Detecting sliding up finger to show {@link MoreSuggestionsView}. if (!mMoreSuggestionsView.isShowingInParent()) { mLastX = (int)me.getX(); mLastY = (int)me.getY(); @@ -439,6 +440,11 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick @Override public boolean onTouchEvent(final MotionEvent me) { + if (!mMoreSuggestionsView.isShowingInParent()) { + // Ignore any touch event while more suggestions panel hasn't been shown. + // Detecting sliding up is done at {@link #onInterceptTouchEvent}. + return true; + } // In the sliding input mode. {@link MotionEvent} should be forwarded to // {@link MoreSuggestionsView}. final int index = me.getActionIndex(); diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 1e6baa5ec..365217a60 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -374,7 +374,7 @@ static bool latinime_BinaryDictionary_addUnigramEntry(JNIEnv *env, jclass clazz, // Use 1 for count to indicate the word has inputted. const UnigramProperty unigramProperty(isBeginningOfSentence, isNotAWord, isBlacklisted, probability, HistoricalInfo(timestamp, 0 /* level */, 1 /* count */), - &shortcuts); + std::move(shortcuts)); return dictionary->addUnigramEntry(CodePointArrayView(codePoints, codePointCount), &unigramProperty); } @@ -434,10 +434,16 @@ static bool latinime_BinaryDictionary_updateCounter(JNIEnv *env, jclass clazz, j if (!dictionary) { return false; } - jsize wordLength = env->GetArrayLength(word); - int wordCodePoints[wordLength]; - env->GetIntArrayRegion(word, 0, wordLength, wordCodePoints); - return false; + const PrevWordsInfo prevWordsInfo = JniDataUtils::constructPrevWordsInfo(env, + prevWordCodePointArrays, isBeginningOfSentenceArray, + env->GetArrayLength(prevWordCodePointArrays)); + jsize codePointCount = env->GetArrayLength(word); + int wordCodePoints[codePointCount]; + env->GetIntArrayRegion(word, 0, codePointCount, wordCodePoints); + const HistoricalInfo historicalInfo(timestamp, 0 /* level */, count); + return dictionary->updateCounter(&prevWordsInfo, + CodePointArrayView(wordCodePoints, codePointCount), isValidWord == JNI_TRUE, + historicalInfo); } // Returns how many language model params are processed. @@ -509,7 +515,7 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j // Use 1 for count to indicate the word has inputted. const UnigramProperty unigramProperty(false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, unigramProbability, - HistoricalInfo(timestamp, 0 /* level */, 1 /* count */), &shortcuts); + HistoricalInfo(timestamp, 0 /* level */, 1 /* count */), std::move(shortcuts)); dictionary->addUnigramEntry(CodePointArrayView(word1CodePoints, word1Length), &unigramProperty); if (word0) { diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index c3f422916..8d3f8a9f8 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -155,6 +155,14 @@ bool Dictionary::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, return mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo, codePoints); } +bool Dictionary::updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView codePoints, const bool isValidWord, + const HistoricalInfo historicalInfo) { + TimeKeeper::setCurrentTime(); + return mDictionaryStructureWithBufferPolicy->updateCounter(prevWordsInfo, codePoints, + isValidWord, historicalInfo); +} + bool Dictionary::flush(const char *const filePath) { TimeKeeper::setCurrentTime(); return mDictionaryStructureWithBufferPolicy->flush(filePath); diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 09f8eaceb..a58dbfbd7 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -22,6 +22,7 @@ #include "defines.h" #include "jni.h" #include "suggest/core/dictionary/ngram_listener.h" +#include "suggest/core/dictionary/property/historical_info.h" #include "suggest/core/dictionary/property/word_property.h" #include "suggest/core/policy/dictionary_header_structure_policy.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" @@ -90,6 +91,10 @@ class Dictionary { bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const CodePointArrayView codePoints); + bool updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView codePoints, const bool isValidWord, + const HistoricalInfo historicalInfo); + bool flush(const char *const filePath); bool flushWithGC(const char *const filePath); diff --git a/native/jni/src/suggest/core/dictionary/property/ngram_property.h b/native/jni/src/suggest/core/dictionary/property/ngram_property.h index 49f683bdc..dce460099 100644 --- a/native/jni/src/suggest/core/dictionary/property/ngram_property.h +++ b/native/jni/src/suggest/core/dictionary/property/ngram_property.h @@ -27,7 +27,7 @@ namespace latinime { class NgramProperty { public: NgramProperty(const std::vector<int> &&targetCodePoints, const int probability, - const HistoricalInfo &historicalInfo) + const HistoricalInfo historicalInfo) : mTargetCodePoints(std::move(targetCodePoints)), mProbability(probability), mHistoricalInfo(historicalInfo) {} diff --git a/native/jni/src/suggest/core/dictionary/property/unigram_property.h b/native/jni/src/suggest/core/dictionary/property/unigram_property.h index 4c61f96e6..d1f0ab4ca 100644 --- a/native/jni/src/suggest/core/dictionary/property/unigram_property.h +++ b/native/jni/src/suggest/core/dictionary/property/unigram_property.h @@ -54,11 +54,18 @@ class UnigramProperty { mProbability(NOT_A_PROBABILITY), mHistoricalInfo(), mShortcuts() {} UnigramProperty(const bool representsBeginningOfSentence, const bool isNotAWord, - const bool isBlacklisted, const int probability, const HistoricalInfo &historicalInfo, - const std::vector<ShortcutProperty> *const shortcuts) + const bool isBlacklisted, const int probability, const HistoricalInfo historicalInfo, + const std::vector<ShortcutProperty> &&shortcuts) : mRepresentsBeginningOfSentence(representsBeginningOfSentence), mIsNotAWord(isNotAWord), mIsBlacklisted(isBlacklisted), mProbability(probability), - mHistoricalInfo(historicalInfo), mShortcuts(*shortcuts) {} + mHistoricalInfo(historicalInfo), mShortcuts(std::move(shortcuts)) {} + + // Without shortcuts. + UnigramProperty(const bool representsBeginningOfSentence, const bool isNotAWord, + const bool isBlacklisted, const int probability, const HistoricalInfo historicalInfo) + : mRepresentsBeginningOfSentence(representsBeginningOfSentence), + mIsNotAWord(isNotAWord), mIsBlacklisted(isBlacklisted), mProbability(probability), + mHistoricalInfo(historicalInfo), mShortcuts() {} bool representsBeginningOfSentence() const { return mRepresentsBeginningOfSentence; diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index f4b97989f..6624b7921 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -21,6 +21,7 @@ #include "defines.h" #include "suggest/core/dictionary/binary_dictionary_shortcut_iterator.h" +#include "suggest/core/dictionary/property/historical_info.h" #include "suggest/core/dictionary/property/word_property.h" #include "suggest/core/dictionary/word_attributes.h" #include "utils/int_array_view.h" @@ -87,6 +88,11 @@ class DictionaryStructureWithBufferPolicy { virtual bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const CodePointArrayView wordCodePoints) = 0; + // Returns whether the update was success or not. + virtual bool updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView wordCodePoints, const bool isValidWord, + const HistoricalInfo historicalInfo) = 0; + // Returns whether the flush was success or not. virtual bool flush(const char *const filePath) = 0; diff --git a/native/jni/src/suggest/core/session/prev_words_info.h b/native/jni/src/suggest/core/session/prev_words_info.h index 02e82a8e0..553d5ad07 100644 --- a/native/jni/src/suggest/core/session/prev_words_info.h +++ b/native/jni/src/suggest/core/session/prev_words_info.h @@ -33,7 +33,7 @@ class PrevWordsInfo { clear(); } - PrevWordsInfo(PrevWordsInfo &&prevWordsInfo) + PrevWordsInfo(const PrevWordsInfo &prevWordsInfo) : mPrevWordCount(prevWordsInfo.mPrevWordCount) { for (size_t i = 0; i < mPrevWordCount; ++i) { mPrevWordCodePointCount[i] = prevWordsInfo.mPrevWordCodePointCount[i]; @@ -73,6 +73,16 @@ class PrevWordsInfo { mIsBeginningOfSentence[0] = isBeginningOfSentence; } + size_t getPrevWordCount() const { + return mPrevWordCount; + } + + // TODO: Remove. + const PrevWordsInfo getTrimmedPrevWordsInfo(const size_t maxPrevWordCount) const { + return PrevWordsInfo(mPrevWordCodePoints, mPrevWordCodePointCount, mIsBeginningOfSentence, + std::min(mPrevWordCount, maxPrevWordCount)); + } + bool isValid() const { if (mPrevWordCodePointCount[0] > 0) { return true; @@ -112,7 +122,7 @@ class PrevWordsInfo { } private: - DISALLOW_COPY_AND_ASSIGN(PrevWordsInfo); + DISALLOW_ASSIGNMENT_OPERATOR(PrevWordsInfo); static int getWordId(const DictionaryStructureWithBufferPolicy *const dictStructurePolicy, const int *const wordCodePoints, const int wordCodePointCount, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp index 40e393c6c..11f7b305f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp @@ -52,6 +52,7 @@ const char *const Ver4PatriciaTriePolicy::MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_C const int Ver4PatriciaTriePolicy::MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS = 1024; const int Ver4PatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS = Ver4DictConstants::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS; +const int Ver4PatriciaTriePolicy::DUMMY_PROBABILITY_FOR_VALID_WORDS = 1; void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const dicNode, DicNodeVector *const childDicNodes) const { @@ -339,11 +340,9 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI } if (prevWordIds[0] == NOT_A_WORD_ID) { if (prevWordsInfo->isNthPrevWordBeginningOfSentence(1 /* n */)) { - const std::vector<UnigramProperty::ShortcutProperty> shortcuts; const UnigramProperty beginningOfSentenceUnigramProperty( true /* representsBeginningOfSentence */, true /* isNotAWord */, - false /* isBlacklisted */, MAX_PROBABILITY /* probability */, - HistoricalInfo(), &shortcuts); + false /* isBlacklisted */, MAX_PROBABILITY /* probability */, HistoricalInfo()); if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */), &beginningOfSentenceUnigramProperty)) { AKLOGE("Cannot add unigram entry for the beginning-of-sentence."); @@ -414,6 +413,29 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor } } + +bool Ver4PatriciaTriePolicy::updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView wordCodePoints, const bool isValidWord, + const HistoricalInfo historicalInfo) { + if (!mBuffers->isUpdatable()) { + AKLOGI("Warning: updateCounter() is called for non-updatable dictionary."); + return false; + } + const int probability = isValidWord ? DUMMY_PROBABILITY_FOR_VALID_WORDS : NOT_A_PROBABILITY; + const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */, + false /* isNotAWord */, false /*isBlacklisted*/, probability, historicalInfo); + if (!addUnigramEntry(wordCodePoints, &unigramProperty)) { + AKLOGE("Cannot update unigarm entry in updateCounter()."); + return false; + } + const NgramProperty ngramProperty(wordCodePoints.toVector(), probability, historicalInfo); + if (!addNgramEntry(prevWordsInfo, &ngramProperty)) { + AKLOGE("Cannot update unigarm entry in updateCounter()."); + return false; + } + return true; +} + bool Ver4PatriciaTriePolicy::flush(const char *const filePath) { if (!mBuffers->isUpdatable()) { AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath); @@ -551,7 +573,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty( } const UnigramProperty unigramProperty(ptNodeParams.representsBeginningOfSentence(), ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.getProbability(), - *historicalInfo, &shortcuts); + *historicalInfo, std::move(shortcuts)); return WordProperty(wordCodePoints.toVector(), &unigramProperty, &ngrams); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h index 324a53e62..995d7764f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h @@ -118,6 +118,10 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const CodePointArrayView wordCodePoints); + bool updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView wordCodePoints, const bool isValidWord, + const HistoricalInfo historicalInfo); + bool flush(const char *const filePath); bool flushWithGC(const char *const filePath); @@ -147,6 +151,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { // prevent the dictionary from overflowing. static const int MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS; static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS; + static const int DUMMY_PROBABILITY_FOR_VALID_WORDS; const Ver4DictBuffers::Ver4DictBuffersPtr mBuffers; const HeaderPolicy *const mHeaderPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp index 7800758c9..d3d684bfa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp @@ -477,7 +477,7 @@ const WordProperty PatriciaTriePolicy::getWordProperty( } const UnigramProperty unigramProperty(ptNodeParams.representsBeginningOfSentence(), ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.getProbability(), - HistoricalInfo(), &shortcuts); + HistoricalInfo(), std::move(shortcuts)); return WordProperty(wordCodePoints.toVector(), &unigramProperty, &ngrams); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h index a912d03be..32a95bb6c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h @@ -107,6 +107,14 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { return false; } + bool updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView wordCodePoints, const bool isValidWord, + const HistoricalInfo historicalInfo) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: updateCounter() is called for non-updatable dictionary."); + return false; + } + bool flush(const char *const filePath) { // This method should not be called for non-updatable dictionary. AKLOGI("Warning: flush() is called for non-updatable dictionary."); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 0badabf03..41b109f95 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -43,6 +43,7 @@ const char *const Ver4PatriciaTriePolicy::MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_C const int Ver4PatriciaTriePolicy::MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS = 1024; const int Ver4PatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS = Ver4DictConstants::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS; +const int Ver4PatriciaTriePolicy::DUMMY_PROBABILITY_FOR_VALID_WORDS = 1; void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const dicNode, DicNodeVector *const childDicNodes) const { @@ -298,11 +299,9 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI if (!prevWordsInfo->isNthPrevWordBeginningOfSentence(i + 1 /* n */)) { return false; } - const std::vector<UnigramProperty::ShortcutProperty> shortcuts; const UnigramProperty beginningOfSentenceUnigramProperty( true /* representsBeginningOfSentence */, true /* isNotAWord */, - false /* isBlacklisted */, MAX_PROBABILITY /* probability */, - HistoricalInfo(), &shortcuts); + false /* isBlacklisted */, MAX_PROBABILITY /* probability */, HistoricalInfo()); if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */), &beginningOfSentenceUnigramProperty)) { AKLOGE("Cannot add unigram entry for the beginning-of-sentence."); @@ -364,6 +363,32 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor } } +bool Ver4PatriciaTriePolicy::updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView wordCodePoints, const bool isValidWord, + const HistoricalInfo historicalInfo) { + if (!mBuffers->isUpdatable()) { + AKLOGI("Warning: updateCounter() is called for non-updatable dictionary."); + return false; + } + // TODO: Have count up method in language model dict content. + const int probability = isValidWord ? DUMMY_PROBABILITY_FOR_VALID_WORDS : NOT_A_PROBABILITY; + const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */, + false /* isNotAWord */, false /*isBlacklisted*/, probability, historicalInfo); + if (!addUnigramEntry(wordCodePoints, &unigramProperty)) { + AKLOGE("Cannot update unigarm entry in updateCounter()."); + return false; + } + const NgramProperty ngramProperty(wordCodePoints.toVector(), probability, historicalInfo); + for (size_t i = 1; i <= prevWordsInfo->getPrevWordCount(); ++i) { + const PrevWordsInfo trimmedPrevWordsInfo(prevWordsInfo->getTrimmedPrevWordsInfo(i)); + if (!addNgramEntry(&trimmedPrevWordsInfo, &ngramProperty)) { + AKLOGE("Cannot update ngram entry in updateCounter()."); + return false; + } + } + return true; +} + bool Ver4PatriciaTriePolicy::flush(const char *const filePath) { if (!mBuffers->isUpdatable()) { AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath); @@ -486,7 +511,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty( } const UnigramProperty unigramProperty(probabilityEntry.representsBeginningOfSentence(), probabilityEntry.isNotAWord(), probabilityEntry.isBlacklisted(), - probabilityEntry.getProbability(), *historicalInfo, &shortcuts); + probabilityEntry.getProbability(), *historicalInfo, std::move(shortcuts)); return WordProperty(wordCodePoints.toVector(), &unigramProperty, &ngrams); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 598122bf2..662bb8d4b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -98,6 +98,10 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const CodePointArrayView wordCodePoints); + bool updateCounter(const PrevWordsInfo *const prevWordsInfo, + const CodePointArrayView wordCodePoints, const bool isValidWord, + const HistoricalInfo historicalInfo); + bool flush(const char *const filePath); bool flushWithGC(const char *const filePath); @@ -127,6 +131,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { // prevent the dictionary from overflowing. static const int MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS; static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS; + // TODO: Remove + static const int DUMMY_PROBABILITY_FOR_VALID_WORDS; const Ver4DictBuffers::Ver4DictBuffersPtr mBuffers; const HeaderPolicy *const mHeaderPolicy; |