diff options
5 files changed, 37 insertions, 1 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp index f3bc4a0cb..f6721c063 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp @@ -53,6 +53,16 @@ bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView pr return mTrieMap.put(terminalId, probabilityEntry->encode(mHasHistoricalInfo), bitmapEntryIndex); } +bool LanguageModelDictContent::removeNgramProbabilityEntry(const WordIdArrayView prevWordIds, + const int wordId) { + const int bitmapEntryIndex = getBitmapEntryIndex(prevWordIds); + if (bitmapEntryIndex == TrieMap::INVALID_INDEX) { + // Cannot find bitmap entry for the probability entry. The entry doesn't exist. + return false; + } + return mTrieMap.remove(wordId, bitmapEntryIndex); +} + bool LanguageModelDictContent::runGCInner( const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const TrieMap::TrieMapRange trieMapRange, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h index 104ee2520..bd07f2f62 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h @@ -61,12 +61,18 @@ class LanguageModelDictContent { return setNgramProbabilityEntry(WordIdArrayView(), wordId, probabilityEntry); } + bool removeProbabilityEntry(const int wordId) { + return removeNgramProbabilityEntry(WordIdArrayView(), wordId); + } + ProbabilityEntry getNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId) const; bool setNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId, const ProbabilityEntry *const probabilityEntry); + bool removeNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId); + private: DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp index e630aba9a..944a59c52 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp @@ -420,6 +420,10 @@ bool TrieMap::addNewEntryByExpandingTable(const uint32_t key, const uint64_t val bool TrieMap::removeInner(const Entry &bitmapEntry) { const int tableSize = popCount(bitmapEntry.getBitmap()); + if (tableSize <= 0) { + // The table is empty. No need to remove any entries. + return true; + } for (int i = 0; i < tableSize; ++i) { const int entryIndex = bitmapEntry.getTableIndex() + i; const Entry entry = readEntry(entryIndex); @@ -444,7 +448,7 @@ bool TrieMap::removeInner(const Entry &bitmapEntry) { } } } - return freeTable(bitmapEntry.getTableIndex(), tableSize); + return true; } } // namespace latinime diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp index 6eef2040b..a1a3b1a75 100644 --- a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp +++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp @@ -35,6 +35,13 @@ TEST(LanguageModelDictContentTest, TestUnigramProbability) { LanguageModelDictContent.getProbabilityEntry(wordId); EXPECT_EQ(flag, entry.getFlags()); EXPECT_EQ(probability, entry.getProbability()); + + // Remove + EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId)); + EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid()); + EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId)); + EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry)); + EXPECT_TRUE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid()); } TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) { @@ -53,6 +60,13 @@ TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) { EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimeStamp()); EXPECT_EQ(level, entry.getHistoricalInfo()->getLevel()); EXPECT_EQ(count, entry.getHistoricalInfo()->getCount()); + + // Remove + EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId)); + EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid()); + EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId)); + EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry)); + EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId)); } } // namespace diff --git a/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp index 8c8e8838a..9904a3aa8 100644 --- a/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp +++ b/native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp @@ -70,6 +70,8 @@ TEST(TrieMapTest, TestRemove) { EXPECT_FALSE(result.mIsValid); EXPECT_EQ(TrieMap::INVALID_INDEX, result.mNextLevelBitmapEntryIndex); EXPECT_EQ(11ull, trieMap.getRoot(12).mValue); + EXPECT_TRUE(trieMap.putRoot(S_INT_MAX, 0xFFFFFFFFFull)); + EXPECT_TRUE(trieMap.remove(S_INT_MAX, trieMap.getRootBitmapEntryIndex())); } TEST(TrieMapTest, TestSetAndGetLarge) { |