1 files changed, 69 insertions, 0 deletions
diff --git a/native/jni/src/dictionary/utils/probability_utils.h b/native/jni/src/dictionary/utils/probability_utils.h
new file mode 100644
index 000000000..2050af1e9
--- /dev/null
+++ b/native/jni/src/dictionary/utils/probability_utils.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_PROBABILITY_UTILS_H
+#define LATINIME_PROBABILITY_UTILS_H
+
+#include <algorithm>
+#include <cmath>
+
+#include "defines.h"
+
+namespace latinime {
+
+// TODO: Quit using bigram probability to indicate the delta.
+class ProbabilityUtils {
+ public:
+    static AK_FORCE_INLINE int backoff(const int unigramProbability) {
+        return unigramProbability;
+        // For some reason, applying the backoff weight gives bad results in tests. To apply the
+        // backoff weight, we divide the probability by 2, which in our storing format means
+        // decreasing the score by 8.
+        // TODO: figure out what's wrong with this.
+        // return unigramProbability > 8 ?
+        //         unigramProbability - 8 : (0 == unigramProbability ? 0 : 8);
+    }
+
+    static AK_FORCE_INLINE int computeProbabilityForBigram(
+            const int unigramProbability, const int bigramProbability) {
+        // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want
+        // the unigram probability to be the median value of the 17th step from the top. A value of
+        // 0 for the bigram probability represents the middle of the 16th step from the top,
+        // while a value of 15 represents the middle of the top step.
+        // See makedict.BinaryDictEncoder#makeBigramFlags for details.
+        const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
+                / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
+        return unigramProbability
+                + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
+    }
+
+    // Encode probability using the same way as we are doing for main dictionaries.
+    static AK_FORCE_INLINE int encodeRawProbability(const float rawProbability) {
+        const float probability = static_cast<float>(MAX_PROBABILITY)
+                + log2f(rawProbability) * PROBABILITY_ENCODING_SCALER;
+        if (probability < 0.0f) {
+            return 0;
+        }
+        return std::min(static_cast<int>(probability + 0.5f), MAX_PROBABILITY);
+    }
+
+ private:
+    DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils);
+
+    static const float PROBABILITY_ENCODING_SCALER;
+};
+}
+#endif /* LATINIME_PROBABILITY_UTILS_H */