aboutsummaryrefslogtreecommitdiffstats
path: root/native/jni/src
diff options
context:
space:
mode:
Diffstat (limited to 'native/jni/src')
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h9
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h36
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp22
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h18
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp11
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h7
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp3
7 files changed, 59 insertions, 47 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index 9726fb856..4c4e43b95 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -68,6 +68,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
+ // Temporary dummy header.
+ HeaderPolicy()
+ : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
+ mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false),
+ mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
+ mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
+
~HeaderPolicy() {}
virtual int getFormatVersionNumber() const {
@@ -139,7 +146,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int unigramCount, const int bigramCount, const int extendedRegionSize) const;
private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
+ DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
static const char *const IS_DECAYING_DICT_KEY;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
index 07b8f181d..40f6469fa 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
@@ -18,7 +18,7 @@
#define LATINIME_VER4_DICT_BUFFER_H
#include "defines.h"
-#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
+#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h"
@@ -39,8 +39,9 @@ class Ver4DictBuffers {
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
}
- static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() {
- return Ver4DictBuffersPtr(new Ver4DictBuffers());
+ static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
+ const HeaderPolicy *const headerPolicy) {
+ return Ver4DictBuffersPtr(new Ver4DictBuffers(headerPolicy));
}
AK_FORCE_INLINE bool isValid() const {
@@ -57,6 +58,10 @@ class Ver4DictBuffers {
|| mShortcutDictContent.isNearSizeLimit();
}
+ AK_FORCE_INLINE const HeaderPolicy *getHeaderPolicy() const {
+ return &mHeaderPolicy;
+ }
+
AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() {
return &mExpandableHeaderBuffer;
}
@@ -118,30 +123,31 @@ class Ver4DictBuffers {
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
: mDictBuffer(dictBuffer),
- mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())),
- mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize,
+ mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
+ mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
- mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize,
- dictBuffer.get()->getBufferSize() - mHeaderSize,
+ mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(),
+ dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
// TODO: Quit using header size.
- mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize),
- mProbabilityDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable),
- mBigramDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable),
+ mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()),
+ mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(),
+ isUpdatable),
+ mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(), isUpdatable),
mShortcutDictContent(dictDirPath, isUpdatable),
mIsUpdatable(isUpdatable) {}
- AK_FORCE_INLINE Ver4DictBuffers()
- : mDictBuffer(0), mHeaderSize(0),
+ AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
+ : mDictBuffer(0), mHeaderPolicy(),
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mTerminalPositionLookupTable(),
- mProbabilityDictContent(false /* hasHistoricalInfo */),
- mBigramDictContent(false /* hasHistoricalInfo */), mShortcutDictContent(),
+ mProbabilityDictContent(headerPolicy->hasHistricalInfoOfWords()),
+ mBigramDictContent(headerPolicy->hasHistricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {}
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
- const int mHeaderSize;
+ const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mExpandableHeaderBuffer;
BufferWithExtendableBuffer mExpandableTrieBuffer;
TerminalPositionLookupTable mTerminalPositionLookupTable;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index 719f35807..f9baeee31 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -50,7 +50,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
break;
}
bool isTerminal = ptNodeParams.isTerminal() && !ptNodeParams.isDeleted();
- if (isTerminal && mHeaderPolicy.isDecayingDict()) {
+ if (isTerminal && mHeaderPolicy->isDecayingDict()) {
// A DecayingDict may have a terminal PtNode that has a terminal DicNode whose
// probability is NOT_A_PROBABILITY. In such case, we don't want to treat it as a
// valid terminal DicNode.
@@ -85,7 +85,7 @@ int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inW
int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
const int bigramProbability) const {
- if (mHeaderPolicy.isDecayingDict()) {
+ if (mHeaderPolicy->isDecayingDict()) {
// Both probabilities are encoded. Decode them and get probability.
return ForgettingCurveUtils::getProbability(unigramProbability, bigramProbability);
} else {
@@ -229,7 +229,7 @@ void Ver4PatriciaTriePolicy::flush(const char *const filePath) {
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
return;
}
- mWritingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount);
+ mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount);
}
void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
@@ -237,10 +237,10 @@ void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
return;
}
- const bool needsToDecay = mHeaderPolicy.isDecayingDict()
+ const bool needsToDecay = mHeaderPolicy->isDecayingDict()
&& (mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay(
- false /* mindsBlockByDecay */, mUnigramCount, mBigramCount, &mHeaderPolicy));
- mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy, needsToDecay);
+ false /* mindsBlockByDecay */, mUnigramCount, mBigramCount, mHeaderPolicy));
+ mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath, needsToDecay);
mNeedsToDecayForTesting = false;
}
@@ -252,7 +252,7 @@ bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
if (mBuffers.get()->isNearSizeLimit()) {
// Additional buffer size is near the limit.
return true;
- } else if (mHeaderPolicy.getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
+ } else if (mHeaderPolicy->getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
> Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE) {
// Total extended region size of the trie exceeds the limit.
return true;
@@ -260,9 +260,9 @@ bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
&& mDictBuffer->getUsedAdditionalBufferSize() > 0) {
// Needs to reduce dictionary size.
return true;
- } else if (mHeaderPolicy.isDecayingDict()) {
+ } else if (mHeaderPolicy->isDecayingDict()) {
return mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay(
- mindsBlockByGC, mUnigramCount, mBigramCount, &mHeaderPolicy);
+ mindsBlockByGC, mUnigramCount, mBigramCount, mHeaderPolicy);
}
return false;
}
@@ -276,11 +276,11 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer
snprintf(outResult, maxResultLength, "%d", mBigramCount);
} else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d",
- mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
+ mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT :
static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
} else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, compareLength) == 0) {
snprintf(outResult, maxResultLength, "%d",
- mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
+ mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT :
static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE));
} else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, compareLength) == 0) {
mNeedsToDecayForTesting = true;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
index 360b6b82e..e38510708 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
@@ -38,22 +38,20 @@ class DicNodeVector;
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers)
- : mBuffers(buffers),
- mHeaderPolicy(mBuffers.get()->getWritableHeaderBuffer()->getBuffer(
- false /* usesAdditionalBuffer*/), FormatUtils::VERSION_4),
+ : mBuffers(buffers), mHeaderPolicy(mBuffers.get()->getHeaderPolicy()),
mDictBuffer(mBuffers.get()->getWritableTrieBuffer()),
mBigramPolicy(mBuffers.get()->getUpdatableBigramDictContent(),
- mBuffers.get()->getTerminalPositionLookupTable(), &mHeaderPolicy,
- mHeaderPolicy.isDecayingDict()),
+ mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy,
+ mHeaderPolicy->isDecayingDict()),
mShortcutPolicy(mBuffers.get()->getShortcutDictContent(),
mBuffers.get()->getTerminalPositionLookupTable()),
mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()),
mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy,
- &mShortcutPolicy, mHeaderPolicy.isDecayingDict()),
+ &mShortcutPolicy, mHeaderPolicy->isDecayingDict()),
mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter),
mWritingHelper(mBuffers.get()),
- mUnigramCount(mHeaderPolicy.getUnigramCount()),
- mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {};
+ mUnigramCount(mHeaderPolicy->getUnigramCount()),
+ mBigramCount(mHeaderPolicy->getBigramCount()), mNeedsToDecayForTesting(false) {};
AK_FORCE_INLINE int getRootPosition() const {
return 0;
@@ -78,7 +76,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
int getBigramsPositionOfPtNode(const int ptNodePos) const;
const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const {
- return &mHeaderPolicy;
+ return mHeaderPolicy;
}
const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const {
@@ -121,7 +119,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
- const HeaderPolicy mHeaderPolicy;
+ const HeaderPolicy *const mHeaderPolicy;
BufferWithExtendableBuffer *const mDictBuffer;
Ver4BigramListPolicy mBigramPolicy;
Ver4ShortcutListPolicy mShortcutPolicy;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
index e8368af27..dd62b9c32 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
@@ -32,8 +32,8 @@
namespace latinime {
void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePath,
- const HeaderPolicy *const headerPolicy, const int unigramCount,
- const int bigramCount) const {
+ const int unigramCount, const int bigramCount) const {
+ const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */;
char dirPath[dirPathBufSize];
FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath);
@@ -53,9 +53,10 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa
}
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
- const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
- const bool needsToDecay) {
- Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers());
+ const char *const trieFilePath, const bool needsToDecay) {
+ const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
+ Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
+ Ver4DictBuffers::createVer4DictBuffers(headerPolicy));
int unigramCount = 0;
int bigramCount = 0;
if (needsToDecay) {
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
index ea2fd532b..9344bde39 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
@@ -32,12 +32,11 @@ class Ver4PatriciaTrieWritingHelper {
Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers)
: mBuffers(buffers) {}
- void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
- const int unigramCount, const int bigramCount) const;
+ void writeToDictFile(const char *const trieFilePath, const int unigramCount,
+ const int bigramCount) const;
void writeToDictFileWithGC(const int rootPtNodeArrayPos,
- const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
- const bool needsToDecay);
+ const char *const trieFilePath, const bool needsToDecay);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
index ff762d75d..d959b26a8 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
@@ -43,8 +43,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
- Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers();
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
+ Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
+ Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-07-20 05:34:36 +0000