Make PtNode have ProbabilityInfo instead of raw value.

Bug: 11281877
Bug: 12810574
Change-Id: Id1cda0afc74c4e30633c735729143491b2274a7b

7 files changed, 48 insertions, 29 deletions

diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
index 8d2f5fbbf..b6795ea6d 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
@@ -21,6 +21,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
 import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.makedict.ProbabilityInfo;
 import com.android.inputmethod.latin.makedict.WordProperty;
 import com.android.inputmethod.latin.utils.CombinedFormatUtils;
 
@@ -112,7 +113,7 @@ public class CombinedInputOutput {
 
         String line;
         String word = null;
-        int freq = 0;
+        ProbabilityInfo probabilityInfo = new ProbabilityInfo(0);
         boolean isNotAWord = false;
         ArrayList<WeightedString> bigrams = new ArrayList<WeightedString>();
         ArrayList<WeightedString> shortcuts = new ArrayList<WeightedString>();
@@ -121,9 +122,10 @@ public class CombinedInputOutput {
             final String args[] = line.trim().split(",");
             if (args[0].matches(CombinedFormatUtils.WORD_TAG + "=.*")) {
                 if (null != word) {
-                    dict.add(word, freq, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
+                    dict.add(word, probabilityInfo, shortcuts.isEmpty() ? null : shortcuts,
+                            isNotAWord);
                     for (WeightedString s : bigrams) {
-                        dict.setBigram(word, s.mWord, s.getProbability());
+                        dict.setBigram(word, s.mWord, s.mProbabilityInfo);
                     }
                 }
                 if (!shortcuts.isEmpty()) shortcuts = new ArrayList<WeightedString>();
@@ -135,14 +137,19 @@ public class CombinedInputOutput {
                     if (CombinedFormatUtils.WORD_TAG.equals(params[0])) {
                         word = params[1];
                     } else if (CombinedFormatUtils.PROBABILITY_TAG.equals(params[0])) {
-                        freq = Integer.parseInt(params[1]);
+                        probabilityInfo = new ProbabilityInfo(Integer.parseInt(params[1]),
+                                probabilityInfo.mTimestamp, probabilityInfo.mLevel,
+                                probabilityInfo.mCount);
                     } else if (CombinedFormatUtils.HISTORICAL_INFO_TAG.equals(params[0])) {
                         final String[] historicalInfoParams =
                                 params[1].split(CombinedFormatUtils.HISTORICAL_INFO_SEPARATOR);
                         if (historicalInfoParams.length != HISTORICAL_INFO_ELEMENT_COUNT) {
                             throw new RuntimeException("Wrong format (historical info) : " + line);
                         }
-                        // TODO: Use parsed historical info.
+                        probabilityInfo = new ProbabilityInfo(probabilityInfo.mProbability,
+                                Integer.parseInt(historicalInfoParams[0]),
+                                Integer.parseInt(historicalInfoParams[1]),
+                                Integer.parseInt(historicalInfoParams[2]));
                     } else if (CombinedFormatUtils.NOT_A_WORD_TAG.equals(params[0])) {
                         isNotAWord = "true".equals(params[1]);
                     }
@@ -168,34 +175,40 @@ public class CombinedInputOutput {
                 }
             } else if (args[0].matches(CombinedFormatUtils.BIGRAM_TAG + "=.*")) {
                 String secondWordOfBigram = null;
-                int bigramFreq = 0;
+                ProbabilityInfo bigramProbabilityInfo = new ProbabilityInfo(0);
                 for (String param : args) {
                     final String params[] = param.split("=", 2);
                     if (2 != params.length) throw new RuntimeException("Wrong format : " + line);
                     if (CombinedFormatUtils.BIGRAM_TAG.equals(params[0])) {
                         secondWordOfBigram = params[1];
                     } else if (CombinedFormatUtils.PROBABILITY_TAG.equals(params[0])) {
-                        bigramFreq = Integer.parseInt(params[1]);
+                        bigramProbabilityInfo = new ProbabilityInfo(Integer.parseInt(params[1]),
+                                bigramProbabilityInfo.mTimestamp, bigramProbabilityInfo.mLevel,
+                                bigramProbabilityInfo.mCount);
                     }  else if (CombinedFormatUtils.HISTORICAL_INFO_TAG.equals(params[0])) {
                         final String[] historicalInfoParams =
                                 params[1].split(CombinedFormatUtils.HISTORICAL_INFO_SEPARATOR);
                         if (historicalInfoParams.length != HISTORICAL_INFO_ELEMENT_COUNT) {
                             throw new RuntimeException("Wrong format (historical info) : " + line);
                         }
-                        // TODO: Use parsed historical info.
+                        bigramProbabilityInfo = new ProbabilityInfo(
+                                bigramProbabilityInfo.mProbability,
+                                Integer.parseInt(historicalInfoParams[0]),
+                                Integer.parseInt(historicalInfoParams[1]),
+                                Integer.parseInt(historicalInfoParams[2]));
                     }
                 }
                 if (null != secondWordOfBigram) {
-                    bigrams.add(new WeightedString(secondWordOfBigram, bigramFreq));
+                    bigrams.add(new WeightedString(secondWordOfBigram, bigramProbabilityInfo));
                 } else {
                     throw new RuntimeException("Wrong format : " + line);
                 }
             }
         }
         if (null != word) {
-            dict.add(word, freq, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
+            dict.add(word, probabilityInfo, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
             for (WeightedString s : bigrams) {
-                dict.setBigram(word, s.mWord, s.getProbability());
+                dict.setBigram(word, s.mWord, s.mProbabilityInfo);
             }
         }
 
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
index 9947608ea..ce9b9f306 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
@@ -118,9 +118,10 @@ public class Diff extends Dicttool.Command {
                 hasDifferences = true;
             } else {
                 // We found the word. Compare frequencies, shortcuts, bigrams
-                if (word0Property.getProbability() != word1PtNode.getFrequency()) {
+                if (word0Property.getProbability() != word1PtNode.getProbability()) {
                     System.out.println("Probability changed: " + word0Property.mWord + " "
-                            + word0Property.getProbability() + " -> " + word1PtNode.getFrequency());
+                            + word0Property.getProbability() + " -> "
+                            + word1PtNode.getProbability());
                     hasDifferences = true;
                 }
                 if (word0Property.mIsNotAWord != word1PtNode.getIsNotAWord()) {
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
index c1eb0f8e7..178df5cec 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
@@ -72,7 +72,7 @@ public class Info extends Dicttool.Command {
             return;
         }
         System.out.println("Word: " + word);
-        System.out.println("  Freq: " + ptNode.getFrequency());
+        System.out.println("  Freq: " + ptNode.getProbability());
         if (ptNode.getIsNotAWord()) {
             System.out.println("  Is not a word");
         }
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
index c6c60b8e2..f45615cbe 100644
--- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
+++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
@@ -20,6 +20,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
 import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.makedict.ProbabilityInfo;
 import com.android.inputmethod.latin.makedict.WordProperty;
 
 import java.io.BufferedReader;
@@ -66,6 +67,7 @@ public class XmlDictInputOutput {
         private static final int START = 1;
         private static final int WORD = 2;
         private static final int UNKNOWN = 3;
+        private static final int SHORTCUT_ONLY_WORD_PROBABILITY = 1;
 
         FusionDictionary mDictionary;
         int mState; // the state of the parser
@@ -90,7 +92,8 @@ public class XmlDictInputOutput {
             final FusionDictionary dict = mDictionary;
             for (final String shortcutOnly : mShortcutsMap.keySet()) {
                 if (dict.hasWord(shortcutOnly)) continue;
-                dict.add(shortcutOnly, 1, mShortcutsMap.get(shortcutOnly), true /* isNotAWord */);
+                dict.add(shortcutOnly, new ProbabilityInfo(SHORTCUT_ONLY_WORD_PROBABILITY),
+                        mShortcutsMap.get(shortcutOnly), true /* isNotAWord */);
             }
             mDictionary = null;
             mShortcutsMap.clear();
@@ -138,7 +141,8 @@ public class XmlDictInputOutput {
         @Override
         public void endElement(String uri, String localName, String qName) {
             if (WORD == mState) {
-                mDictionary.add(mWord, mFreq, mShortcutsMap.get(mWord), false /* isNotAWord */);
+                mDictionary.add(mWord, new ProbabilityInfo(mFreq), mShortcutsMap.get(mWord),
+                        false /* isNotAWord */);
                 mState = START;
             }
         }
@@ -319,7 +323,7 @@ public class XmlDictInputOutput {
             final ArrayList<WeightedString> bigramList = bigramMap.get(firstWord);
             for (final WeightedString bigram : bigramList) {
                 if (!dict.hasWord(bigram.mWord)) continue;
-                dict.setBigram(firstWord, bigram.mWord, bigram.getProbability());
+                dict.setBigram(firstWord, bigram.mWord, bigram.mProbabilityInfo);
             }
         }
         return dict;
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
index 9ed4dd5a2..69b49f015 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
@@ -24,6 +24,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.ProbabilityInfo;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
 import com.android.inputmethod.latin.makedict.Ver2DictEncoder;
 
@@ -53,11 +54,11 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
         testOptions.mAttributes.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, LOCALE);
         testOptions.mAttributes.put(DictionaryHeader.DICTIONARY_ID_KEY, ID);
         final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), testOptions);
-        dict.add("foo", TEST_FREQ, null, false /* isNotAWord */);
-        dict.add("fta", 1, null, false /* isNotAWord */);
-        dict.add("ftb", 1, null, false /* isNotAWord */);
-        dict.add("bar", 1, null, false /* isNotAWord */);
-        dict.add("fool", 1, null, false /* isNotAWord */);
+        dict.add("foo", new ProbabilityInfo(TEST_FREQ), null, false /* isNotAWord */);
+        dict.add("fta", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("ftb", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("bar", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("fool", new ProbabilityInfo(1), null, false /* isNotAWord */);
 
         final File dst = File.createTempFile("testGetRawDict", ".tmp");
         dst.deleteOnExit();
@@ -87,7 +88,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
         assertEquals("Wrong id attribute", ID, resultDict.mOptions.mAttributes.get(
                 DictionaryHeader.DICTIONARY_ID_KEY));
         assertEquals("Dictionary can't be read back correctly",
-                FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(),
+                FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getProbability(),
                 TEST_FREQ);
     }
 
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
index c6e497615..283abcdd1 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java
@@ -33,11 +33,11 @@ public class BinaryDictEncoderFlattenTreeTests extends TestCase {
     public void testFlattenNodes() {
         final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
                 new DictionaryOptions(new HashMap<String, String>()));
-        dict.add("foo", 1, null, false /* isNotAWord */);
-        dict.add("fta", 1, null, false /* isNotAWord */);
-        dict.add("ftb", 1, null, false /* isNotAWord */);
-        dict.add("bar", 1, null, false /* isNotAWord */);
-        dict.add("fool", 1, null, false /* isNotAWord */);
+        dict.add("foo", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("fta", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("ftb", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("bar", new ProbabilityInfo(1), null, false /* isNotAWord */);
+        dict.add("fool", new ProbabilityInfo(1), null, false /* isNotAWord */);
         final ArrayList<PtNodeArray> result =
                 BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
         assertEquals(4, result.size());
diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
index 191546433..d0d47da96 100644
--- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
+++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
@@ -101,7 +101,7 @@ public class FusionDictionaryTest extends TestCase {
         prepare(time);
         for (int i = 0; i < sWords.size(); ++i) {
             System.out.println("Adding in pos " + i + " : " + dumpWord(sWords.get(i)));
-            dict.add(sWords.get(i), 180, null, false);
+            dict.add(sWords.get(i), new ProbabilityInfo(180), null, false);
             dumpDict(dict);
             checkDictionary(dict, sWords, i);
         }