diff options
Diffstat (limited to 'native/jni/src')
9 files changed, 43 insertions, 3 deletions
diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index f88388c75..0bcde2294 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -99,6 +99,11 @@ bool Dictionary::addUnigramEntry(const int *const word, const int length, return mDictionaryStructureWithBufferPolicy->addUnigramEntry(word, length, unigramProperty); } +bool Dictionary::removeUnigramEntry(const int *const codePoints, const int codePointCount) { + TimeKeeper::setCurrentTime(); + return mDictionaryStructureWithBufferPolicy->removeUnigramEntry(codePoints, codePointCount); +} + bool Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty) { TimeKeeper::setCurrentTime(); diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 10010b21c..542ba7291 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -81,6 +81,8 @@ class Dictionary { bool addUnigramEntry(const int *const codePoints, const int codePointCount, const UnigramProperty *const unigramProperty); + bool removeUnigramEntry(const int *const codePoints, const int codePointCount); + bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty); diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index cda89406c..e2771f97c 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -74,6 +74,9 @@ class DictionaryStructureWithBufferPolicy { const UnigramProperty *const unigramProperty) = 0; // Returns whether the update was success or not. + virtual bool removeUnigramEntry(const int *const word, const int length) = 0; + + // Returns whether the update was success or not. virtual bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty) = 0; diff --git a/native/jni/src/suggest/core/result/suggestions_output_utils.cpp b/native/jni/src/suggest/core/result/suggestions_output_utils.cpp index 23908255b..7b0e7e1b4 100644 --- a/native/jni/src/suggest/core/result/suggestions_output_utils.cpp +++ b/native/jni/src/suggest/core/result/suggestions_output_utils.cpp @@ -26,6 +26,7 @@ #include "suggest/core/policy/scoring.h" #include "suggest/core/result/suggestion_results.h" #include "suggest/core/session/dic_traverse_session.h" +#include "suggest/core/suggest_options.h" namespace latinime { @@ -105,6 +106,11 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16; // Entries that are blacklisted or do not represent a word should not be output. const bool isValidWord = !terminalDicNode->isBlacklistedOrNotAWord(); + // When we have to block offensive words, non-exact matched offensive words should not be + // output. + const bool blockOffensiveWords = traverseSession->getSuggestOptions()->blockOffensiveWords(); + const bool isBlockedOffensiveWord = blockOffensiveWords && isPossiblyOffensiveWord + && !isSafeExactMatch; // Increase output score of top typing suggestion to ensure autocorrection. // TODO: Better integration with java side autocorrection logic. @@ -115,8 +121,9 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16; || (isValidWord && scoringPolicy->doesAutoCorrectValidWord()), boostExactMatches); - // Don't output invalid words. However, we still need to submit their shortcuts if any. - if (isValidWord) { + // Don't output invalid or blocked offensive words. However, we still need to submit their + // shortcuts if any. + if (isValidWord && !isBlockedOffensiveWord) { int codePoints[MAX_WORD_LENGTH]; terminalDicNode->outputResult(codePoints); const int indexToPartialCommit = outputSecondWordFirstLetterInputIndex ? diff --git a/native/jni/src/suggest/core/suggest_options.h b/native/jni/src/suggest/core/suggest_options.h index 1b21aafcf..2e22a7ac3 100644 --- a/native/jni/src/suggest/core/suggest_options.h +++ b/native/jni/src/suggest/core/suggest_options.h @@ -34,6 +34,10 @@ class SuggestOptions{ return getBoolOption(USE_FULL_EDIT_DISTANCE); } + AK_FORCE_INLINE bool blockOffensiveWords() const { + return getBoolOption(BLOCK_OFFENSIVE_WORDS); + } + AK_FORCE_INLINE bool getAdditionalFeaturesBoolOption(const int key) const { return getBoolOption(key + ADDITIONAL_FEATURES_OPTIONS); } @@ -45,9 +49,10 @@ class SuggestOptions{ // reorder options. static const int IS_GESTURE = 0; static const int USE_FULL_EDIT_DISTANCE = 1; + static const int BLOCK_OFFENSIVE_WORDS = 2; // Additional features options are stored after the other options and used as setting values of // experimental features. - static const int ADDITIONAL_FEATURES_OPTIONS = 2; + static const int ADDITIONAL_FEATURES_OPTIONS = 3; const int *const mOptions; const int mLength; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h index e323652d4..2e948ac4a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h @@ -111,6 +111,11 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool addUnigramEntry(const int *const word, const int length, const UnigramProperty *const unigramProperty); + bool removeUnigramEntry(const int *const word, const int length) { + // Removing unigram entry is not supported. + return false; + } + bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h index 88bbfd966..dce94363a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h @@ -88,6 +88,12 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { return false; } + bool removeUnigramEntry(const int *const word, const int length) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary."); + return false; + } + bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty) { // This method should not be called for non-updatable dictionary. diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 1e10f24c5..aec3b8ea3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -221,6 +221,11 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le } } +bool Ver4PatriciaTriePolicy::removeUnigramEntry(const int *const word, const int length) { + // TODO: Implement. + return false; +} + bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty) { if (!mBuffers->isUpdatable()) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index d198c97fd..0a20965f3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -93,6 +93,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool addUnigramEntry(const int *const word, const int length, const UnigramProperty *const unigramProperty); + bool removeUnigramEntry(const int *const word, const int length); + bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty); |