From 607a9244861ee22c25aaea6ffdfa19fccf497b0b Mon Sep 17 00:00:00 2001 From: satok Date: Thu, 17 May 2012 19:53:32 +0900 Subject: Add utils for forgetting curve Bug: 4192129 Change-Id: Iebb7ac355e1c1891da31c33754315bd76a61889b --- .../latin/UserHistoryForgettingCurveUtils.java | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java new file mode 100644 index 000000000..eb3881726 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +public class UserHistoryForgettingCurveUtils { + private static final int FC_FREQ_MAX = 127; + /* package */ static final int COUNT_MAX = 3; + private static final int FC_LEVEL_MAX = 3; + /* package */ static final int ELAPSED_TIME_MAX = 15; + private static final int ELAPSED_TIME_INTERVAL_HOURS = 6; + private static final int HALF_LIFE_HOURS = 48; + + /* package */ static int fcToElapsedTime(byte fc) { + return fc & 0x0F; + } + + /* package */ static int fcToCount(byte fc) { + return (fc >> 4) & 0x03; + } + + /* package */ static int fcToLevel(byte fc) { + return (fc >> 6) & 0x03; + } + + private static int calcFreq(int elapsedTime, int count, int level) { + if (level <= 0) { + // Reserved words, just return 0 + return 0; + } + if (count == COUNT_MAX) { + // Temporary promote because it's frequently typed recently + ++level; + } + final int et = Math.min(FC_FREQ_MAX, Math.max(0, elapsedTime)); + final int l = Math.min(FC_LEVEL_MAX, Math.max(0, level)); + return MathUtils.SCORE_TABLE[l - 1][et]; + } + + /* pakcage */ static byte calcFc(int elapsedTime, int count, int level) { + final int et = Math.min(FC_FREQ_MAX, Math.max(0, elapsedTime)); + final int c = Math.min(COUNT_MAX, Math.max(0, count)); + final int l = Math.min(FC_LEVEL_MAX, Math.max(0, level)); + return (byte)(et | (c << 4) | (l << 6)); + } + + public static int fcToFreq(byte fc) { + final int elapsedTime = fcToElapsedTime(fc); + final int count = fcToCount(fc); + final int level = fcToLevel(fc); + return calcFreq(elapsedTime, count, level); + } + + public static byte pushElapsedTime(byte fc) { + int elapsedTime = fcToElapsedTime(fc); + int count = fcToCount(fc); + int level = fcToLevel(fc); + if (elapsedTime >= ELAPSED_TIME_MAX) { + // Downgrade level + elapsedTime = 0; + count = COUNT_MAX; + --level; + } else { + ++elapsedTime; + } + return calcFc(elapsedTime, count, level); + } + + public static byte pushCount(byte fc, boolean isValid) { + final int elapsedTime = fcToElapsedTime(fc); + int count = fcToCount(fc); + int level = fcToLevel(fc); + if ((elapsedTime == 0 && count >= COUNT_MAX) || (isValid && level == 0)) { + // Upgrade level + ++level; + count = 0; + } else { + ++count; + } + return calcFc(0, count, level); + } + + private static class MathUtils { + public static final int[][] SCORE_TABLE = new int[FC_LEVEL_MAX][ELAPSED_TIME_MAX + 1]; + static { + for (int i = 0; i < FC_LEVEL_MAX; ++i) { + final double initialFreq; + if (i >= 2) { + initialFreq = (double)FC_FREQ_MAX; + } else if (i == 1) { + initialFreq = (double)FC_FREQ_MAX / 2; + } else if (i == 0) { + initialFreq = (double)FC_FREQ_MAX / 4; + } else { + continue; + } + for (int j = 0; j < ELAPSED_TIME_MAX; ++j) { + final double elapsedHour = j * ELAPSED_TIME_INTERVAL_HOURS; + final double freq = + initialFreq * Math.pow(initialFreq, elapsedHour / HALF_LIFE_HOURS); + final int intFreq = Math.min(FC_FREQ_MAX, Math.max(0, (int)freq)); + SCORE_TABLE[i][j] = intFreq; + } + } + } + } +} -- cgit v1.2.3-83-g751a From fd53b8cc2b78acd7e33f4dc39cfc2faaea92f0f8 Mon Sep 17 00:00:00 2001 From: satok Date: Fri, 18 May 2012 21:28:09 +0900 Subject: Forget user history Bug: 4192129 Change-Id: Ic98398e5383093a1c24373849eadc48fc4d3626f --- .../inputmethod/latin/ContactsDictionary.java | 2 +- .../inputmethod/latin/ExpandableDictionary.java | 136 ++++++++++++++----- .../com/android/inputmethod/latin/LatinIME.java | 3 +- .../com/android/inputmethod/latin/Settings.java | 3 +- .../android/inputmethod/latin/SettingsValues.java | 21 +++ .../inputmethod/latin/UserHistoryDictionary.java | 148 +++++++++++++-------- .../latin/UserHistoryForgettingCurveUtils.java | 82 +++++++++++- java/src/com/android/inputmethod/latin/Utils.java | 38 ++++++ 8 files changed, 333 insertions(+), 100 deletions(-) (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/ContactsDictionary.java b/java/src/com/android/inputmethod/latin/ContactsDictionary.java index c9b8d6eb1..cbfbd0ec8 100644 --- a/java/src/com/android/inputmethod/latin/ContactsDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsDictionary.java @@ -159,7 +159,7 @@ public class ContactsDictionary extends ExpandableDictionary { super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS); if (!TextUtils.isEmpty(prevWord)) { - super.setBigram(prevWord, word, + super.setBigramAndGetFrequency(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM); } prevWord = word; diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index 6c457afd2..358cd4d4d 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -21,6 +21,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.KeyDetector; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import java.util.ArrayList; import java.util.LinkedList; @@ -80,31 +81,73 @@ public class ExpandableDictionary extends Dictionary { } } - private static class NextWord { - public final Node mWord; - private int mFrequency; + protected interface NextWord { + public Node getWordNode(); + public int getFrequency(); + /** FcValue is a bit set */ + public int getFcValue(); + public int notifyTypedAgainAndGetFrequency(); + } - public NextWord(Node word, int frequency) { + private static class NextStaticWord implements NextWord { + public final Node mWord; + private final int mFrequency; + public NextStaticWord(Node word, int frequency) { mWord = word; mFrequency = frequency; } + @Override + public Node getWordNode() { + return mWord; + } + + @Override public int getFrequency() { return mFrequency; } - public int setFrequency(int freq) { - mFrequency = freq; + @Override + public int getFcValue() { return mFrequency; } - public int addFrequency(int add) { - mFrequency += add; - if (mFrequency > BIGRAM_MAX_FREQUENCY) mFrequency = BIGRAM_MAX_FREQUENCY; + @Override + public int notifyTypedAgainAndGetFrequency() { return mFrequency; } } + private static class NextHistoryWord implements NextWord { + public final Node mWord; + public final ForgettingCurveParams mFcp; + + public NextHistoryWord(Node word, ForgettingCurveParams fcp) { + mWord = word; + mFcp = fcp; + } + + @Override + public Node getWordNode() { + return mWord; + } + + @Override + public int getFrequency() { + return mFcp.getFrequency(); + } + + @Override + public int getFcValue() { + return mFcp.getFc(); + } + + @Override + public int notifyTypedAgainAndGetFrequency() { + return mFcp.notifyTypedAgainAndGetFrequency(); + } + } + private NodeArray mRoots; private int[][] mCodes; @@ -183,7 +226,7 @@ public class ExpandableDictionary extends Dictionary { childNode.mShortcutOnly = isShortcutOnly; children.add(childNode); } - if (wordLength == depth + 1) { + if (wordLength == depth + 1 && shortcutTarget != null) { // Terminate this word childNode.mTerminal = true; if (isShortcutOnly) { @@ -221,7 +264,7 @@ public class ExpandableDictionary extends Dictionary { protected final void getWordsInner(final WordComposer codes, final CharSequence prevWordForBigrams, final WordCallback callback, - @SuppressWarnings("unused") final ProximityInfo proximityInfo) { + final ProximityInfo proximityInfo) { mInputLength = codes.size(); if (mCodes.length < mInputLength) mCodes = new int[mInputLength][]; final int[] xCoordinates = codes.getXCoordinates(); @@ -265,13 +308,13 @@ public class ExpandableDictionary extends Dictionary { // Refer to addOrSetBigram() about word1.toLowerCase() final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); final Node secondWord = searchWord(mRoots, word2, 0, null); - LinkedList bigram = firstWord.mNGrams; + LinkedList bigrams = firstWord.mNGrams; NextWord bigramNode = null; - if (bigram == null || bigram.size() == 0) { + if (bigrams == null || bigrams.size() == 0) { return false; } else { - for (NextWord nw : bigram) { - if (nw.mWord == secondWord) { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { bigramNode = nw; break; } @@ -280,7 +323,7 @@ public class ExpandableDictionary extends Dictionary { if (bigramNode == null) { return false; } - return bigram.remove(bigramNode); + return bigrams.remove(bigramNode); } /** @@ -292,6 +335,23 @@ public class ExpandableDictionary extends Dictionary { return (node == null) ? -1 : node.mFrequency; } + protected NextWord getBigramWord(String word1, String word2) { + // Refer to addOrSetBigram() about word1.toLowerCase() + final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); + final Node secondWord = searchWord(mRoots, word2, 0, null); + LinkedList bigrams = firstWord.mNGrams; + if (bigrams == null || bigrams.size() == 0) { + return null; + } else { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + return nw; + } + } + } + return null; + } + private static int computeSkippedWordFinalFreq(int freq, int snr, int inputLength) { // The computation itself makes sense for >= 2, but the == 2 case returns 0 // anyway so we may as well test against 3 instead and return the constant @@ -445,43 +505,45 @@ public class ExpandableDictionary extends Dictionary { } } - protected int setBigram(String word1, String word2, int frequency) { - return addOrSetBigram(word1, word2, frequency, false); + public int setBigramAndGetFrequency(String word1, String word2, int frequency) { + return setBigramAndGetFrequency(word1, word2, frequency, null /* unused */); } - protected int addBigram(String word1, String word2, int frequency) { - return addOrSetBigram(word1, word2, frequency, true); + public int setBigramAndGetFrequency(String word1, String word2, ForgettingCurveParams fcp) { + return setBigramAndGetFrequency(word1, word2, 0 /* unused */, fcp); } /** * Adds bigrams to the in-memory trie structure that is being used to retrieve any word * @param frequency frequency for this bigram * @param addFrequency if true, it adds to current frequency, else it overwrites the old value - * @return returns the final frequency + * @return returns the final bigram frequency */ - private int addOrSetBigram(String word1, String word2, int frequency, boolean addFrequency) { + private int setBigramAndGetFrequency( + String word1, String word2, int frequency, ForgettingCurveParams fcp) { // We don't want results to be different according to case of the looked up left hand side // word. We do want however to return the correct case for the right hand side. // So we want to squash the case of the left hand side, and preserve that of the right // hand side word. Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); Node secondWord = searchWord(mRoots, word2, 0, null); - LinkedList bigram = firstWord.mNGrams; - if (bigram == null || bigram.size() == 0) { + LinkedList bigrams = firstWord.mNGrams; + if (bigrams == null || bigrams.size() == 0) { firstWord.mNGrams = new LinkedList(); - bigram = firstWord.mNGrams; + bigrams = firstWord.mNGrams; } else { - for (NextWord nw : bigram) { - if (nw.mWord == secondWord) { - if (addFrequency) { - return nw.addFrequency(frequency); - } else { - return nw.setFrequency(frequency); - } + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + return nw.notifyTypedAgainAndGetFrequency(); } } } - firstWord.mNGrams.add(new NextWord(secondWord, frequency)); + if (fcp != null) { + // history + firstWord.mNGrams.add(new NextHistoryWord(secondWord, fcp)); + } else { + firstWord.mNGrams.add(new NextStaticWord(secondWord, frequency)); + } return frequency; } @@ -580,7 +642,7 @@ public class ExpandableDictionary extends Dictionary { Node node; int freq; for (NextWord nextWord : terminalNodes) { - node = nextWord.mWord; + node = nextWord.getWordNode(); freq = nextWord.getFrequency(); int index = BinaryDictionary.MAX_WORD_LENGTH; do { @@ -589,8 +651,10 @@ public class ExpandableDictionary extends Dictionary { node = node.mParent; } while (node != null); - callback.addWord(mLookedUpString, index, BinaryDictionary.MAX_WORD_LENGTH - index, - freq, mDicTypeId, Dictionary.BIGRAM); + if (freq >= 0) { + callback.addWord(mLookedUpString, index, BinaryDictionary.MAX_WORD_LENGTH - index, + freq, mDicTypeId, Dictionary.BIGRAM); + } } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index c58549497..e9117e29a 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -494,7 +494,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen resetContactsDictionary(oldContactsDictionary); mUserHistoryDictionary = new UserHistoryDictionary( - this, localeStr, Suggest.DIC_USER_HISTORY); + this, localeStr, Suggest.DIC_USER_HISTORY, mPrefs); mSuggest.setUserHistoryDictionary(mUserHistoryDictionary); } @@ -745,7 +745,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen KeyboardView inputView = mKeyboardSwitcher.getKeyboardView(); if (inputView != null) inputView.closing(); - if (mUserHistoryDictionary != null) mUserHistoryDictionary.flushPendingWrites(); } private void onFinishInputViewInternal(boolean finishingInput) { diff --git a/java/src/com/android/inputmethod/latin/Settings.java b/java/src/com/android/inputmethod/latin/Settings.java index 74c4aea0c..08f3e8456 100644 --- a/java/src/com/android/inputmethod/latin/Settings.java +++ b/java/src/com/android/inputmethod/latin/Settings.java @@ -58,6 +58,8 @@ public class Settings extends InputMethodSettingsFragment public static final String PREF_SHOW_SUGGESTIONS_SETTING = "show_suggestions_setting"; public static final String PREF_MISC_SETTINGS = "misc_settings"; public static final String PREF_USABILITY_STUDY_MODE = "usability_study_mode"; + public static final String PREF_LAST_USER_DICTIONARY_WRITE_TIME = + "last_user_dictionary_write_time"; public static final String PREF_ADVANCED_SETTINGS = "pref_advanced_settings"; public static final String PREF_SUPPRESS_LANGUAGE_SWITCH_KEY = "pref_suppress_language_switch_key"; @@ -244,7 +246,6 @@ public class Settings extends InputMethodSettingsFragment refreshEnablingsOfKeypressSoundAndVibrationSettings(prefs, res); } - @SuppressWarnings("unused") @Override public void onResume() { super.onResume(); diff --git a/java/src/com/android/inputmethod/latin/SettingsValues.java b/java/src/com/android/inputmethod/latin/SettingsValues.java index 932920a60..4aae6a85e 100644 --- a/java/src/com/android/inputmethod/latin/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/SettingsValues.java @@ -28,6 +28,8 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; /** * When you call the constructor of this class, you may want to change the current system locale by @@ -351,4 +353,23 @@ public class SettingsValues { // TODO: use mUsabilityStudyMode instead of reading it again here return prefs.getBoolean(Settings.PREF_USABILITY_STUDY_MODE, true); } + + public static long getLastUserHistoryWriteTime( + final SharedPreferences prefs, final String locale) { + final String str = prefs.getString(Settings.PREF_LAST_USER_DICTIONARY_WRITE_TIME, ""); + final HashMap map = Utils.localeAndTimeStrToHashMap(str); + if (map.containsKey(locale)) { + return map.get(locale); + } + return 0; + } + + public static void setLastUserHistoryWriteTime( + final SharedPreferences prefs, final String locale) { + final String oldStr = prefs.getString(Settings.PREF_LAST_USER_DICTIONARY_WRITE_TIME, ""); + final HashMap map = Utils.localeAndTimeStrToHashMap(oldStr); + map.put(locale, System.currentTimeMillis()); + final String newStr = Utils.localeAndTimeHashMapToStr(map); + prefs.edit().putString(Settings.PREF_LAST_USER_DICTIONARY_WRITE_TIME, newStr).apply(); + } } diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java index efafacc8a..fa3d1be11 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.content.ContentValues; import android.content.Context; +import android.content.SharedPreferences; import android.database.Cursor; import android.database.sqlite.SQLiteDatabase; import android.database.sqlite.SQLiteOpenHelper; @@ -26,6 +27,8 @@ import android.os.AsyncTask; import android.provider.BaseColumns; import android.util.Log; +import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -40,9 +43,6 @@ public class UserHistoryDictionary extends ExpandableDictionary { /** Any pair being typed or picked */ private static final int FREQUENCY_FOR_TYPED = 2; - /** Maximum frequency for all pairs */ - private static final int FREQUENCY_MAX = 127; - /** Maximum number of pairs. Pruning will start when databases goes above this number. */ private static int sMaxHistoryBigrams = 10000; @@ -81,6 +81,7 @@ public class UserHistoryDictionary extends ExpandableDictionary { private HashSet mPendingWrites = new HashSet(); private final Object mPendingWritesLock = new Object(); private static volatile boolean sUpdatingDB = false; + private final SharedPreferences mPrefs; private final static HashMap sDictProjectionMap; @@ -101,12 +102,10 @@ public class UserHistoryDictionary extends ExpandableDictionary { private static class Bigram { public final String mWord1; public final String mWord2; - public final int mFrequency; - Bigram(String word1, String word2, int frequency) { + Bigram(String word1, String word2) { this.mWord1 = word1; this.mWord2 = word2; - this.mFrequency = frequency; } @Override @@ -137,7 +136,8 @@ public class UserHistoryDictionary extends ExpandableDictionary { sDeleteHistoryBigrams = deleteHistoryBigram; } - public UserHistoryDictionary(final Context context, final String locale, final int dicTypeId) { + public UserHistoryDictionary(final Context context, final String locale, final int dicTypeId, + SharedPreferences sp) { super(context, dicTypeId); mLocale = locale; if (sOpenHelper == null) { @@ -146,11 +146,13 @@ public class UserHistoryDictionary extends ExpandableDictionary { if (mLocale != null && mLocale.length() > 1) { loadDictionary(); } + mPrefs = sp; } @Override public void close() { flushPendingWrites(); + SettingsValues.setLastUserHistoryWriteTime(mPrefs, mLocale); // Don't close the database as locale changes will require it to be reopened anyway // Also, the database is written to somewhat frequently, so it needs to be kept alive // throughout the life of the process. @@ -176,25 +178,20 @@ public class UserHistoryDictionary extends ExpandableDictionary { * The second word may not be null (a NullPointerException would be thrown). */ public int addToUserHistory(final String word1, String word2) { - super.addWord(word2, null /* shortcut */, FREQUENCY_FOR_TYPED); + super.addWord(word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED); // Do not insert a word as a bigram of itself if (word2.equals(word1)) { return 0; } - - int freq; + final int freq; if (null == word1) { freq = FREQUENCY_FOR_TYPED; } else { - freq = super.addBigram(word1, word2, FREQUENCY_FOR_TYPED); + freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams()); } - if (freq > FREQUENCY_MAX) freq = FREQUENCY_MAX; synchronized (mPendingWritesLock) { - if (freq == FREQUENCY_FOR_TYPED || mPendingWrites.isEmpty()) { - mPendingWrites.add(new Bigram(word1, word2, freq)); - } else { - Bigram bi = new Bigram(word1, word2, freq); - mPendingWrites.remove(bi); + final Bigram bi = new Bigram(word1, word2); + if (!mPendingWrites.contains(bi)) { mPendingWrites.add(bi); } } @@ -203,7 +200,7 @@ public class UserHistoryDictionary extends ExpandableDictionary { } public boolean cancelAddingUserHistory(String word1, String word2) { - final Bigram bi = new Bigram(word1, word2, 0); + final Bigram bi = new Bigram(word1, word2); if (mPendingWrites.contains(bi)) { mPendingWrites.remove(bi); return super.removeBigram(word1, word2); @@ -214,12 +211,12 @@ public class UserHistoryDictionary extends ExpandableDictionary { /** * Schedules a background thread to write any pending words to the database. */ - public void flushPendingWrites() { + private void flushPendingWrites() { synchronized (mPendingWritesLock) { // Nothing pending? Return if (mPendingWrites.isEmpty()) return; // Create a background thread to write the pending entries - new UpdateDbTask(sOpenHelper, mPendingWrites, mLocale).execute(); + new UpdateDbTask(sOpenHelper, mPendingWrites, mLocale, this).execute(); // Create a new map for writing new entries into while the old one is written to db mPendingWrites = new HashSet(); } @@ -240,25 +237,30 @@ public class UserHistoryDictionary extends ExpandableDictionary { @Override public void loadDictionaryAsync() { + final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale); + final long now = System.currentTimeMillis(); // Load the words that correspond to the current input locale final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale }); if (null == cursor) return; try { if (cursor.moveToFirst()) { - int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1); - int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2); - int frequencyIndex = cursor.getColumnIndex(FREQ_COLUMN_FREQUENCY); + final int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1); + final int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2); + final int frequencyIndex = cursor.getColumnIndex(FREQ_COLUMN_FREQUENCY); while (!cursor.isAfterLast()) { - String word1 = cursor.getString(word1Index); - String word2 = cursor.getString(word2Index); - int frequency = cursor.getInt(frequencyIndex); + final String word1 = cursor.getString(word1Index); + final String word2 = cursor.getString(word2Index); + final int frequency = cursor.getInt(frequencyIndex); // Safeguard against adding really long words. Stack may overflow due // to recursive lookup if (null == word1) { super.addWord(word2, null /* shortcut */, frequency); } else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH && word2.length() < BinaryDictionary.MAX_WORD_LENGTH) { - super.setBigram(word1, word2, frequency); + super.setBigramAndGetFrequency( + word1, word2, new ForgettingCurveParams(frequency, now, last)); + // TODO: optimize + mPendingWrites.add(new Bigram(word1, word2)); } cursor.moveToNext(); } @@ -340,12 +342,14 @@ public class UserHistoryDictionary extends ExpandableDictionary { private final HashSet mMap; private final DatabaseHelper mDbHelper; private final String mLocale; + private final UserHistoryDictionary mUserHistoryDictionary; public UpdateDbTask(DatabaseHelper openHelper, HashSet pendingWrites, - String locale) { + String locale, UserHistoryDictionary dict) { mMap = pendingWrites; mLocale = locale; mDbHelper = openHelper; + mUserHistoryDictionary = dict; } /** Prune any old data if the database is getting too big. */ @@ -357,7 +361,8 @@ public class UserHistoryDictionary extends ExpandableDictionary { int totalRowCount = c.getCount(); // prune out old data if we have too much data if (totalRowCount > sMaxHistoryBigrams) { - int numDeleteRows = (totalRowCount - sMaxHistoryBigrams) + sDeleteHistoryBigrams; + int numDeleteRows = (totalRowCount - sMaxHistoryBigrams) + + sDeleteHistoryBigrams; int pairIdColumnId = c.getColumnIndex(FREQ_COLUMN_PAIR_ID); c.moveToFirst(); int count = 0; @@ -397,43 +402,70 @@ public class UserHistoryDictionary extends ExpandableDictionary { } db.execSQL("PRAGMA foreign_keys = ON;"); // Write all the entries to the db - Iterator iterator = mMap.iterator(); + final Iterator iterator = mMap.iterator(); while (iterator.hasNext()) { // TODO: this process of making a text search for each pair each time // is terribly inefficient. Optimize this. - Bigram bi = iterator.next(); + final Bigram bi = iterator.next(); // find pair id - final Cursor c; - if (null != bi.mWord1) { - c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, - MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " - + MAIN_COLUMN_LOCALE + "=?", - new String[] { bi.mWord1, bi.mWord2, mLocale }, null, null, null); - } else { - c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, - MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + "=? AND " - + MAIN_COLUMN_LOCALE + "=?", - new String[] { bi.mWord2, mLocale }, null, null, null); - } + Cursor c = null; + try { + if (null != bi.mWord1) { + c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, + MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " + + MAIN_COLUMN_LOCALE + "=?", + new String[] { bi.mWord1, bi.mWord2, mLocale }, null, null, + null); + } else { + c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, + MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + "=? AND " + + MAIN_COLUMN_LOCALE + "=?", + new String[] { bi.mWord2, mLocale }, null, null, null); + } - int pairId; - if (c.moveToFirst()) { - // existing pair - pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); - db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", - new String[] { Integer.toString(pairId) }); - } else { - // new pair - Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, - getContentValues(bi.mWord1, bi.mWord2, mLocale)); - pairId = pairIdLong.intValue(); + final int pairId; + if (c.moveToFirst()) { + // existing pair + pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); + db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", + new String[] { Integer.toString(pairId) }); + } else { + // new pair + Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, + getContentValues(bi.mWord1, bi.mWord2, mLocale)); + pairId = pairIdLong.intValue(); + } + // insert new frequency + final int freq; + if (bi.mWord1 == null) { + freq = FREQUENCY_FOR_TYPED; + } else { + final NextWord nw = mUserHistoryDictionary.getBigramWord( + bi.mWord1, bi.mWord2); + if (nw != null) { + final int tempFreq = nw.getFcValue(); + // TODO: Check whether the word is valid or not + if (UserHistoryForgettingCurveUtils.needsToSave( + (byte)tempFreq, false)) { + freq = tempFreq; + } else { + freq = -1; + } + } else { + freq = -1; + } + } + if (freq > 0) { + db.insert(FREQ_TABLE_NAME, null, getFrequencyContentValues(pairId, freq)); + } + } finally { + if (c != null) { + c.close(); + } } - c.close(); - - // insert new frequency - db.insert(FREQ_TABLE_NAME, null, getFrequencyContentValues(pairId, bi.mFrequency)); } + checkPruneData(db); sUpdatingDB = false; diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index eb3881726..f30fee23e 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -16,14 +16,78 @@ package com.android.inputmethod.latin; +import android.text.format.DateUtils; +import android.util.Log; + public class UserHistoryForgettingCurveUtils { + private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); + private static final boolean DEBUG = false; private static final int FC_FREQ_MAX = 127; /* package */ static final int COUNT_MAX = 3; private static final int FC_LEVEL_MAX = 3; /* package */ static final int ELAPSED_TIME_MAX = 15; private static final int ELAPSED_TIME_INTERVAL_HOURS = 6; + private static final long ELAPSED_TIME_INTERVAL_MILLIS = ELAPSED_TIME_INTERVAL_HOURS + * DateUtils.HOUR_IN_MILLIS; private static final int HALF_LIFE_HOURS = 48; + private UserHistoryForgettingCurveUtils() { + // This utility class is not publicly instantiable. + } + + public static class ForgettingCurveParams { + private byte mFc; + long mLastTouchedTime = 0; + + private void updateLastTouchedTime() { + mLastTouchedTime = System.currentTimeMillis(); + } + + public ForgettingCurveParams() { + // TODO: Check whether this word is valid or not + this(System.currentTimeMillis()); + } + + private ForgettingCurveParams(long now) { + this((int)pushCount((byte)0, false), now, now); + } + + public ForgettingCurveParams(int fc, long now, long last) { + mFc = (byte)fc; + mLastTouchedTime = last; + updateElapsedTime(now); + } + + public byte getFc() { + updateElapsedTime(System.currentTimeMillis()); + return mFc; + } + + public int getFrequency() { + updateElapsedTime(System.currentTimeMillis()); + return UserHistoryForgettingCurveUtils.fcToFreq(mFc); + } + + public int notifyTypedAgainAndGetFrequency() { + updateLastTouchedTime(); + // TODO: Check whether this word is valid or not + mFc = pushCount(mFc, false); + return UserHistoryForgettingCurveUtils.fcToFreq(mFc); + } + + private void updateElapsedTime(long now) { + final int elapsedTimeCount = + (int)((now - mLastTouchedTime) / ELAPSED_TIME_INTERVAL_MILLIS); + if (elapsedTimeCount <= 0) { + return; + } + for (int i = 0; i < elapsedTimeCount; ++i) { + mLastTouchedTime += ELAPSED_TIME_INTERVAL_MILLIS; + mFc = pushElapsedTime(mFc); + } + } + } + /* package */ static int fcToElapsedTime(byte fc) { return fc & 0x0F; } @@ -38,8 +102,8 @@ public class UserHistoryForgettingCurveUtils { private static int calcFreq(int elapsedTime, int count, int level) { if (level <= 0) { - // Reserved words, just return 0 - return 0; + // Reserved words, just return -1 + return -1; } if (count == COUNT_MAX) { // Temporary promote because it's frequently typed recently @@ -87,12 +151,26 @@ public class UserHistoryForgettingCurveUtils { // Upgrade level ++level; count = 0; + if (DEBUG) { + Log.d(TAG, "Upgrade level."); + } } else { ++count; } return calcFc(0, count, level); } + // TODO: isValid should be false for a word whose frequency is 0, + // or that is not in the dictionary. + public static boolean needsToSave(byte fc, boolean isValid) { + int level = fcToLevel(fc); + if (isValid && level == 0) { + return false; + } + final int elapsedTime = fcToElapsedTime(fc); + return (elapsedTime < ELAPSED_TIME_MAX - 1 || level > 0); + } + private static class MathUtils { public static final int[][] SCORE_TABLE = new int[FC_LEVEL_MAX][ELAPSED_TIME_MAX + 1]; static { diff --git a/java/src/com/android/inputmethod/latin/Utils.java b/java/src/com/android/inputmethod/latin/Utils.java index 036ff74b8..91996674b 100644 --- a/java/src/com/android/inputmethod/latin/Utils.java +++ b/java/src/com/android/inputmethod/latin/Utils.java @@ -44,8 +44,10 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.Map; public class Utils { private Utils() { @@ -484,4 +486,40 @@ public class Utils { } return sDeviceOverrideValueMap.get(key); } + + private static final HashMap EMPTY_LT_HASH_MAP = new HashMap(); + private static final String LOCALE_AND_TIME_STR_SEPARATER = ","; + public static HashMap localeAndTimeStrToHashMap(String str) { + if (TextUtils.isEmpty(str)) { + return EMPTY_LT_HASH_MAP; + } + final String[] ss = str.split(LOCALE_AND_TIME_STR_SEPARATER); + final int N = ss.length; + if (N < 2 || N % 2 != 0) { + return EMPTY_LT_HASH_MAP; + } + final HashMap retval = new HashMap(); + for (int i = 0; i < N / 2; ++i) { + final String localeStr = ss[i]; + final long time = Long.valueOf(ss[i + 1]); + retval.put(localeStr, time); + } + return retval; + } + + public static String localeAndTimeHashMapToStr(HashMap map) { + if (map == null || map.isEmpty()) { + return ""; + } + final StringBuilder builder = new StringBuilder(); + for (String localeStr : map.keySet()) { + if (builder.length() > 0) { + builder.append(LOCALE_AND_TIME_STR_SEPARATER); + } + final Long time = map.get(localeStr); + builder.append(localeStr).append(LOCALE_AND_TIME_STR_SEPARATER); + builder.append(String.valueOf(time)); + } + return builder.toString(); + } } -- cgit v1.2.3-83-g751a From bc5688506229bd5cd5e6f4dcdc73c21dc6b80ecb Mon Sep 17 00:00:00 2001 From: Satoshi Kataoka Date: Mon, 28 May 2012 19:08:34 +0900 Subject: Optimize the store of bigram list Bug: 4192129 Change-Id: Idcc62e4f9696b56b1d7013891b2da37b1784423e --- .../inputmethod/latin/UserHistoryDictionary.java | 183 +++++++++------------ .../latin/UserHistoryDictionaryBigramList.java | 91 ++++++++++ .../latin/UserHistoryForgettingCurveUtils.java | 11 +- 3 files changed, 179 insertions(+), 106 deletions(-) create mode 100644 java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java index fa3d1be11..d5163f2a1 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java @@ -30,8 +30,6 @@ import android.util.Log; import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; /** * Locally gathers stats about the words user types and various other signals like auto-correction @@ -39,6 +37,7 @@ import java.util.Iterator; */ public class UserHistoryDictionary extends ExpandableDictionary { private static final String TAG = "UserHistoryDictionary"; + public static final boolean DBG_SAVE_RESTORE = false; /** Any pair being typed or picked */ private static final int FREQUENCY_FOR_TYPED = 2; @@ -78,7 +77,8 @@ public class UserHistoryDictionary extends ExpandableDictionary { /** Locale for which this auto dictionary is storing words */ private String mLocale; - private HashSet mPendingWrites = new HashSet(); + private UserHistoryDictionaryBigramList mBigramList = + new UserHistoryDictionaryBigramList(); private final Object mPendingWritesLock = new Object(); private static volatile boolean sUpdatingDB = false; private final SharedPreferences mPrefs; @@ -99,35 +99,6 @@ public class UserHistoryDictionary extends ExpandableDictionary { private static DatabaseHelper sOpenHelper = null; - private static class Bigram { - public final String mWord1; - public final String mWord2; - - Bigram(String word1, String word2) { - this.mWord1 = word1; - this.mWord2 = word2; - } - - @Override - public boolean equals(Object bigram) { - if (!(bigram instanceof Bigram)) { - return false; - } - final Bigram bigram2 = (Bigram) bigram; - final boolean eq1 = - mWord1 == null ? bigram2.mWord1 == null : mWord1.equals(bigram2.mWord1); - if (!eq1) { - return false; - } - return mWord2 == null ? bigram2.mWord2 == null : mWord2.equals(bigram2.mWord2); - } - - @Override - public int hashCode() { - return (mWord1 + " " + mWord2).hashCode(); - } - } - public void setDatabaseMax(int maxHistoryBigram) { sMaxHistoryBigrams = maxHistoryBigram; } @@ -190,20 +161,17 @@ public class UserHistoryDictionary extends ExpandableDictionary { freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams()); } synchronized (mPendingWritesLock) { - final Bigram bi = new Bigram(word1, word2); - if (!mPendingWrites.contains(bi)) { - mPendingWrites.add(bi); - } + mBigramList.addBigram(word1, word2); } return freq; } public boolean cancelAddingUserHistory(String word1, String word2) { - final Bigram bi = new Bigram(word1, word2); - if (mPendingWrites.contains(bi)) { - mPendingWrites.remove(bi); - return super.removeBigram(word1, word2); + synchronized (mPendingWritesLock) { + if (mBigramList.removeBigram(word1, word2)) { + return super.removeBigram(word1, word2); + } } return false; } @@ -214,11 +182,11 @@ public class UserHistoryDictionary extends ExpandableDictionary { private void flushPendingWrites() { synchronized (mPendingWritesLock) { // Nothing pending? Return - if (mPendingWrites.isEmpty()) return; + if (mBigramList.isEmpty()) return; // Create a background thread to write the pending entries - new UpdateDbTask(sOpenHelper, mPendingWrites, mLocale, this).execute(); + new UpdateDbTask(sOpenHelper, mBigramList, mLocale, this).execute(); // Create a new map for writing new entries into while the old one is written to db - mPendingWrites = new HashSet(); + mBigramList = new UserHistoryDictionaryBigramList(); } } @@ -251,6 +219,9 @@ public class UserHistoryDictionary extends ExpandableDictionary { final String word1 = cursor.getString(word1Index); final String word2 = cursor.getString(word2Index); final int frequency = cursor.getInt(frequencyIndex); + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "--- Load user history: " + word1 + ", " + word2); + } // Safeguard against adding really long words. Stack may overflow due // to recursive lookup if (null == word1) { @@ -259,8 +230,9 @@ public class UserHistoryDictionary extends ExpandableDictionary { && word2.length() < BinaryDictionary.MAX_WORD_LENGTH) { super.setBigramAndGetFrequency( word1, word2, new ForgettingCurveParams(frequency, now, last)); - // TODO: optimize - mPendingWrites.add(new Bigram(word1, word2)); + } + synchronized(mPendingWritesLock) { + mBigramList.addBigram(word1, word2); } cursor.moveToNext(); } @@ -339,14 +311,15 @@ public class UserHistoryDictionary extends ExpandableDictionary { * the in-memory trie. */ private static class UpdateDbTask extends AsyncTask { - private final HashSet mMap; + private final UserHistoryDictionaryBigramList mBigramList; private final DatabaseHelper mDbHelper; private final String mLocale; private final UserHistoryDictionary mUserHistoryDictionary; - public UpdateDbTask(DatabaseHelper openHelper, HashSet pendingWrites, + public UpdateDbTask( + DatabaseHelper openHelper, UserHistoryDictionaryBigramList pendingWrites, String locale, UserHistoryDictionary dict) { - mMap = pendingWrites; + mBigramList = pendingWrites; mLocale = locale; mDbHelper = openHelper; mUserHistoryDictionary = dict; @@ -401,67 +374,71 @@ public class UserHistoryDictionary extends ExpandableDictionary { return null; } db.execSQL("PRAGMA foreign_keys = ON;"); + final boolean addLevel0Bigram = mBigramList.size() <= sMaxHistoryBigrams; + // Write all the entries to the db - final Iterator iterator = mMap.iterator(); - while (iterator.hasNext()) { - // TODO: this process of making a text search for each pair each time - // is terribly inefficient. Optimize this. - final Bigram bi = iterator.next(); - - // find pair id - Cursor c = null; - try { - if (null != bi.mWord1) { - c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, - MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " - + MAIN_COLUMN_LOCALE + "=?", - new String[] { bi.mWord1, bi.mWord2, mLocale }, null, null, - null); - } else { - c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, - MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + "=? AND " - + MAIN_COLUMN_LOCALE + "=?", - new String[] { bi.mWord2, mLocale }, null, null, null); - } + for (String word1 : mBigramList.keySet()) { + for (String word2 : mBigramList.getBigrams(word1)) { + // TODO: this process of making a text search for each pair each time + // is terribly inefficient. Optimize this. + // find pair id + Cursor c = null; + try { + if (null != word1) { + c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, + MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " + + MAIN_COLUMN_LOCALE + "=?", + new String[] { word1, word2, mLocale }, null, null, + null); + } else { + c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, + MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + + "=? AND " + MAIN_COLUMN_LOCALE + "=?", + new String[] { word2, mLocale }, null, null, null); + } - final int pairId; - if (c.moveToFirst()) { - // existing pair - pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); - db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", - new String[] { Integer.toString(pairId) }); - } else { - // new pair - Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, - getContentValues(bi.mWord1, bi.mWord2, mLocale)); - pairId = pairIdLong.intValue(); - } - // insert new frequency - final int freq; - if (bi.mWord1 == null) { - freq = FREQUENCY_FOR_TYPED; - } else { - final NextWord nw = mUserHistoryDictionary.getBigramWord( - bi.mWord1, bi.mWord2); - if (nw != null) { - final int tempFreq = nw.getFcValue(); - // TODO: Check whether the word is valid or not - if (UserHistoryForgettingCurveUtils.needsToSave( - (byte)tempFreq, false)) { - freq = tempFreq; + final int pairId; + if (c.moveToFirst()) { + // existing pair + pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); + db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", + new String[] { Integer.toString(pairId) }); + } else { + // new pair + Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, + getContentValues(word1, word2, mLocale)); + pairId = pairIdLong.intValue(); + } + // insert new frequency + final int freq; + if (word1 == null) { + freq = FREQUENCY_FOR_TYPED; + } else { + final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2); + if (nw != null) { + final int tempFreq = nw.getFcValue(); + // TODO: Check whether the word is valid or not + if (UserHistoryForgettingCurveUtils.needsToSave( + (byte)tempFreq, false, addLevel0Bigram)) { + freq = tempFreq; + } else { + freq = -1; + } } else { freq = -1; } - } else { - freq = -1; } - } - if (freq > 0) { - db.insert(FREQ_TABLE_NAME, null, getFrequencyContentValues(pairId, freq)); - } - } finally { - if (c != null) { - c.close(); + if (freq > 0) { + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "--- Save user history: " + word1 + ", " + word2); + } + db.insert(FREQ_TABLE_NAME, null, + getFrequencyContentValues(pairId, freq)); + } + } finally { + if (c != null) { + c.close(); + } } } } diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java new file mode 100644 index 000000000..409f921ff --- /dev/null +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.util.Log; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; + +/** + * A store of bigrams which will be updated when the user history dictionary is closed + * All bigrams including stale ones in SQL DB should be stored in this class to avoid adding stale + * bigrams when we write to the SQL DB. + */ +public class UserHistoryDictionaryBigramList { + private static final String TAG = UserHistoryDictionaryBigramList.class.getSimpleName(); + private static final HashSet EMPTY_STRING_SET = new HashSet(); + private final HashMap> mBigramMap = + new HashMap>(); + private int mSize = 0; + + public void evictAll() { + mSize = 0; + mBigramMap.clear(); + } + + public void addBigram(String word1, String word2) { + if (UserHistoryDictionary.DBG_SAVE_RESTORE) { + Log.d(TAG, "--- add bigram: " + word1 + ", " + word2); + } + final HashSet set; + if (mBigramMap.containsKey(word1)) { + set = mBigramMap.get(word1); + } else { + set = new HashSet(); + mBigramMap.put(word1, set); + } + if (!set.contains(word2)) { + ++mSize; + set.add(word2); + } + } + + public int size() { + return mSize; + } + + public boolean isEmpty() { + return mBigramMap.isEmpty(); + } + + public Set keySet() { + return mBigramMap.keySet(); + } + + public HashSet getBigrams(String word1) { + if (!mBigramMap.containsKey(word1)) { + return EMPTY_STRING_SET; + } else { + return mBigramMap.get(word1); + } + } + + public boolean removeBigram(String word1, String word2) { + final HashSet set = getBigrams(word1); + if (set.isEmpty()) { + return false; + } + if (set.contains(word2)) { + set.remove(word2); + --mSize; + return true; + } + return false; + } +} diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index f30fee23e..feb1d0029 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -162,10 +162,15 @@ public class UserHistoryForgettingCurveUtils { // TODO: isValid should be false for a word whose frequency is 0, // or that is not in the dictionary. - public static boolean needsToSave(byte fc, boolean isValid) { + /** + * Check wheather we should save the bigram to the SQL DB or not + */ + public static boolean needsToSave(byte fc, boolean isValid, boolean addLevel0Bigram) { int level = fcToLevel(fc); - if (isValid && level == 0) { - return false; + if (level == 0) { + if (isValid || !addLevel0Bigram) { + return false; + } } final int elapsedTime = fcToElapsedTime(fc); return (elapsedTime < ELAPSED_TIME_MAX - 1 || level > 0); -- cgit v1.2.3-83-g751a From c88f61215c5b9ca6e0cc3f776e3b7da19eec9cae Mon Sep 17 00:00:00 2001 From: Satoshi Kataoka Date: Tue, 29 May 2012 19:07:22 +0900 Subject: Set level 1 as the initial value of the valid words Bug: 4192129 Change-Id: I867e78ce79c78977d08e8b66881a25b6fe5bf41f --- .../android/inputmethod/latin/AutoCorrection.java | 20 ++++++++++++++++++- .../inputmethod/latin/BinaryDictionary.java | 10 +++++++--- .../com/android/inputmethod/latin/Dictionary.java | 5 +++++ .../inputmethod/latin/DictionaryCollection.java | 12 +++++++++++ .../inputmethod/latin/ExpandableDictionary.java | 11 +++++------ .../com/android/inputmethod/latin/LatinIME.java | 6 +++++- .../inputmethod/latin/UserHistoryDictionary.java | 11 ++++++----- .../latin/UserHistoryForgettingCurveUtils.java | 23 +++++++++++++++++----- 8 files changed, 77 insertions(+), 21 deletions(-) (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/AutoCorrection.java b/java/src/com/android/inputmethod/latin/AutoCorrection.java index 32b213e67..e0452483c 100644 --- a/java/src/com/android/inputmethod/latin/AutoCorrection.java +++ b/java/src/com/android/inputmethod/latin/AutoCorrection.java @@ -50,7 +50,7 @@ public class AutoCorrection { } public static boolean isValidWord(final ConcurrentHashMap dictionaries, - CharSequence word, boolean ignoreCase) { + CharSequence word, boolean ignoreCase) { if (TextUtils.isEmpty(word)) { return false; } @@ -74,6 +74,24 @@ public class AutoCorrection { return false; } + public static int getMaxFrequency(final ConcurrentHashMap dictionaries, + CharSequence word) { + if (TextUtils.isEmpty(word)) { + return Dictionary.NOT_A_PROBABILITY; + } + int maxFreq = -1; + for (final String key : dictionaries.keySet()) { + if (key.equals(Suggest.DICT_KEY_WHITELIST)) continue; + final Dictionary dictionary = dictionaries.get(key); + if (null == dictionary) continue; + final int tempFreq = dictionary.getFrequency(word); + if (tempFreq >= maxFreq) { + maxFreq = tempFreq; + } + } + return maxFreq; + } + public static boolean allowsToBeAutoCorrected( final ConcurrentHashMap dictionaries, final CharSequence word, final boolean ignoreCase) { diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index cb1069cfb..d0613bd72 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -201,10 +201,14 @@ public class BinaryDictionary extends Dictionary { @Override public boolean isValidWord(CharSequence word) { - if (word == null) return false; + return getFrequency(word) >= 0; + } + + @Override + public int getFrequency(CharSequence word) { + if (word == null) return -1; int[] chars = StringUtils.toCodePointArray(word.toString()); - final int freq = getFrequencyNative(mNativeDict, chars, chars.length); - return freq >= 0; + return getFrequencyNative(mNativeDict, chars, chars.length); } // TODO: Add a batch process version (isValidBigramMultiple?) to avoid excessive numbers of jni diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 231e9ab81..7cd9bc2a8 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -31,6 +31,7 @@ public abstract class Dictionary { public static final int UNIGRAM = 0; public static final int BIGRAM = 1; + public static final int NOT_A_PROBABILITY = -1; /** * Interface to be implemented by classes requesting words to be fetched from the dictionary. * @see #getWords(WordComposer, CharSequence, WordCallback, ProximityInfo) @@ -84,6 +85,10 @@ public abstract class Dictionary { */ abstract public boolean isValidWord(CharSequence word); + public int getFrequency(CharSequence word) { + return NOT_A_PROBABILITY; + } + /** * Compares the contents of the character array with the typed word and returns true if they * are the same. diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java index f3aa27a22..1a05fcd86 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java +++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java @@ -70,6 +70,18 @@ public class DictionaryCollection extends Dictionary { return false; } + @Override + public int getFrequency(CharSequence word) { + int maxFreq = -1; + for (int i = mDictionaries.size() - 1; i >= 0; --i) { + final int tempFreq = mDictionaries.get(i).getFrequency(word); + if (tempFreq >= maxFreq) { + maxFreq = tempFreq; + } + } + return maxFreq; + } + public boolean isEmpty() { return mDictionaries.isEmpty(); } diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index 358cd4d4d..34a92fd30 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -84,8 +84,7 @@ public class ExpandableDictionary extends Dictionary { protected interface NextWord { public Node getWordNode(); public int getFrequency(); - /** FcValue is a bit set */ - public int getFcValue(); + public ForgettingCurveParams getFcParams(); public int notifyTypedAgainAndGetFrequency(); } @@ -108,8 +107,8 @@ public class ExpandableDictionary extends Dictionary { } @Override - public int getFcValue() { - return mFrequency; + public ForgettingCurveParams getFcParams() { + return null; } @Override @@ -138,8 +137,8 @@ public class ExpandableDictionary extends Dictionary { } @Override - public int getFcValue() { - return mFcp.getFc(); + public ForgettingCurveParams getFcParams() { + return mFcp; } @Override diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 52088812d..38549436b 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -2093,8 +2093,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { secondWord = suggestion.toString(); } + // We demote unrecognized word and words with 0-frequency (assuming they would be + // profanity etc.) by specifying them as "invalid". + final int maxFreq = AutoCorrection.getMaxFrequency( + mSuggest.getUnigramDictionaries(), suggestion); mUserHistoryDictionary.addToUserHistory(null == prevWord ? null : prevWord.toString(), - secondWord); + secondWord, maxFreq > 0); return prevWord; } return null; diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java index d5163f2a1..c8ad40b12 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java @@ -148,7 +148,7 @@ public class UserHistoryDictionary extends ExpandableDictionary { * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public int addToUserHistory(final String word1, String word2) { + public int addToUserHistory(final String word1, String word2, boolean isValid) { super.addWord(word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED); // Do not insert a word as a bigram of itself if (word2.equals(word1)) { @@ -158,7 +158,7 @@ public class UserHistoryDictionary extends ExpandableDictionary { if (null == word1) { freq = FREQUENCY_FOR_TYPED; } else { - freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams()); + freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams(isValid)); } synchronized (mPendingWritesLock) { mBigramList.addBigram(word1, word2); @@ -416,10 +416,11 @@ public class UserHistoryDictionary extends ExpandableDictionary { } else { final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2); if (nw != null) { - final int tempFreq = nw.getFcValue(); - // TODO: Check whether the word is valid or not + final ForgettingCurveParams fcp = nw.getFcParams(); + final int tempFreq = fcp.getFc(); + final boolean isValid = fcp.isValid(); if (UserHistoryForgettingCurveUtils.needsToSave( - (byte)tempFreq, false, addLevel0Bigram)) { + (byte)tempFreq, isValid, addLevel0Bigram)) { freq = tempFreq; } else { freq = -1; diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index feb1d0029..9cd8c6778 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -38,26 +38,39 @@ public class UserHistoryForgettingCurveUtils { public static class ForgettingCurveParams { private byte mFc; long mLastTouchedTime = 0; + private final boolean mIsValid; private void updateLastTouchedTime() { mLastTouchedTime = System.currentTimeMillis(); } - public ForgettingCurveParams() { - // TODO: Check whether this word is valid or not - this(System.currentTimeMillis()); + public ForgettingCurveParams(boolean isValid) { + this(System.currentTimeMillis(), isValid); } - private ForgettingCurveParams(long now) { - this((int)pushCount((byte)0, false), now, now); + private ForgettingCurveParams(long now, boolean isValid) { + this((int)pushCount((byte)0, isValid), now, now, isValid); } + /** This constructor is called when the user history bigram dictionary is being restored. */ public ForgettingCurveParams(int fc, long now, long last) { + // All words with level >= 1 had been saved. + // Invalid words with level == 0 had been saved. + // Valid words words with level == 0 had *not* been saved. + this(fc, now, last, fcToLevel((byte)fc) > 0); + } + + private ForgettingCurveParams(int fc, long now, long last, boolean isValid) { + mIsValid = isValid; mFc = (byte)fc; mLastTouchedTime = last; updateElapsedTime(now); } + public boolean isValid() { + return mIsValid; + } + public byte getFc() { updateElapsedTime(System.currentTimeMillis()); return mFc; -- cgit v1.2.3-83-g751a From ec2981a487b91a682caade486700d8b2377a5c52 Mon Sep 17 00:00:00 2001 From: satok Date: Fri, 8 Jun 2012 02:23:39 +0900 Subject: Fix a bug on upgrading user history scheme Bug: 6626700 Change-Id: I3190d7404e45e704be79ee2031e79b18475feb8c --- java/src/com/android/inputmethod/latin/UserHistoryDictionary.java | 4 +++- .../android/inputmethod/latin/UserHistoryForgettingCurveUtils.java | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java index a73e71bb2..61b012f2d 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java @@ -229,6 +229,7 @@ public class UserHistoryDictionary extends ExpandableDictionary { public void loadDictionaryAsync() { synchronized(mBigramList) { final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale); + final boolean initializing = last == 0; final long now = System.currentTimeMillis(); // Load the words that correspond to the current input locale final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale }); @@ -253,7 +254,8 @@ public class UserHistoryDictionary extends ExpandableDictionary { } else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH && word2.length() < BinaryDictionary.MAX_WORD_LENGTH) { super.setBigramAndGetFrequency( - word1, word2, new ForgettingCurveParams(fc, now, last)); + word1, word2, initializing ? new ForgettingCurveParams(true) + : new ForgettingCurveParams(fc, now, last)); } mBigramList.addBigram(word1, word2, (byte)fc); cursor.moveToNext(); diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index 9cd8c6778..e5516dc62 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -30,6 +30,7 @@ public class UserHistoryForgettingCurveUtils { private static final long ELAPSED_TIME_INTERVAL_MILLIS = ELAPSED_TIME_INTERVAL_HOURS * DateUtils.HOUR_IN_MILLIS; private static final int HALF_LIFE_HOURS = 48; + private static final int MAX_PUSH_ELAPSED = (FC_LEVEL_MAX + 1) * (ELAPSED_TIME_MAX + 1); private UserHistoryForgettingCurveUtils() { // This utility class is not publicly instantiable. @@ -94,6 +95,11 @@ public class UserHistoryForgettingCurveUtils { if (elapsedTimeCount <= 0) { return; } + if (elapsedTimeCount >= MAX_PUSH_ELAPSED) { + mLastTouchedTime = now; + mFc = 0; + return; + } for (int i = 0; i < elapsedTimeCount; ++i) { mLastTouchedTime += ELAPSED_TIME_INTERVAL_MILLIS; mFc = pushElapsedTime(mFc); -- cgit v1.2.3-83-g751a From 7214617622fce8f3fea6620e782c16336260a2a3 Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Fri, 8 Jun 2012 16:00:02 +0900 Subject: Remove a slew of Eclipse warnings. Change-Id: I03236386aea13fbd4fb8eaeee18e0008aa136502 --- java/src/com/android/inputmethod/keyboard/KeyboardView.java | 1 + java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java | 2 -- java/src/com/android/inputmethod/latin/ExpandableDictionary.java | 4 +++- java/src/com/android/inputmethod/latin/LatinIME.java | 1 + java/src/com/android/inputmethod/latin/ResearchLogger.java | 4 ++-- java/src/com/android/inputmethod/latin/SettingsValues.java | 1 - java/src/com/android/inputmethod/latin/UserHistoryDictionary.java | 2 +- .../android/inputmethod/latin/UserHistoryForgettingCurveUtils.java | 4 ++-- java/src/com/android/inputmethod/latin/Utils.java | 2 -- .../com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java | 2 +- 10 files changed, 11 insertions(+), 12 deletions(-) (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java index 51a0f537f..18e01fb49 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java @@ -873,6 +873,7 @@ public class KeyboardView extends View implements PointerTracker.DrawingProxy { keyPreview, ViewLayoutUtils.newLayoutParam(mPreviewPlacer, 0, 0)); } + @SuppressWarnings("deprecation") // setBackgroundDrawable is replaced by setBackground in API16 @Override public void showKeyPreview(PointerTracker tracker) { if (!mShowKeyPreviewPopup) return; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java index 5aa9a0887..4ab6832c3 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java @@ -21,8 +21,6 @@ import android.util.Log; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.ResearchLogger; -import com.android.inputmethod.latin.define.ProductionFlag; /** * Keyboard state machine. diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index 34a92fd30..4a5471c85 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -514,8 +514,10 @@ public class ExpandableDictionary extends Dictionary { /** * Adds bigrams to the in-memory trie structure that is being used to retrieve any word + * @param word1 the first word of this bigram + * @param word2 the second word of this bigram * @param frequency frequency for this bigram - * @param addFrequency if true, it adds to current frequency, else it overwrites the old value + * @param fcp an instance of ForgettingCurveParams to use for decay policy * @return returns the final bigram frequency */ private int setBigramAndGetFrequency( diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index ae9e197a1..f5025e54a 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -747,6 +747,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (TRACE) Debug.startMethodTracing("/data/trace/latinime"); } + @Override public void onTargetApplicationKnown(final ApplicationInfo info) { mTargetApplicationInfo = info; } diff --git a/java/src/com/android/inputmethod/latin/ResearchLogger.java b/java/src/com/android/inputmethod/latin/ResearchLogger.java index bb003f766..a7e7738d8 100644 --- a/java/src/com/android/inputmethod/latin/ResearchLogger.java +++ b/java/src/com/android/inputmethod/latin/ResearchLogger.java @@ -101,13 +101,13 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang private static class NullOutputStream extends OutputStream { /** {@inheritDoc} */ @Override - public void write(byte[] buffer, int offset, int count) throws IOException { + public void write(byte[] buffer, int offset, int count) { // nop } /** {@inheritDoc} */ @Override - public void write(byte[] buffer) throws IOException { + public void write(byte[] buffer) { // nop } diff --git a/java/src/com/android/inputmethod/latin/SettingsValues.java b/java/src/com/android/inputmethod/latin/SettingsValues.java index 4aae6a85e..dfe207cf2 100644 --- a/java/src/com/android/inputmethod/latin/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/SettingsValues.java @@ -29,7 +29,6 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.Map; /** * When you call the constructor of this class, you may want to change the current system locale by diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java index 9c54e0b81..10f92d29e 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java @@ -440,7 +440,7 @@ public class UserHistoryDictionary extends ExpandableDictionary { if (nw != null) { final ForgettingCurveParams fcp = nw.getFcParams(); final byte prevFc = word1Bigrams.get(word2); - final byte fc = (byte)fcp.getFc(); + final byte fc = fcp.getFc(); final boolean isValid = fcp.isValid(); if (prevFc > 0 && prevFc == fc) { // No need to update since we found no changes for this entry. diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index e5516dc62..3ae1bd336 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -50,7 +50,7 @@ public class UserHistoryForgettingCurveUtils { } private ForgettingCurveParams(long now, boolean isValid) { - this((int)pushCount((byte)0, isValid), now, now, isValid); + this(pushCount((byte)0, isValid), now, now, isValid); } /** This constructor is called when the user history bigram dictionary is being restored. */ @@ -201,7 +201,7 @@ public class UserHistoryForgettingCurveUtils { for (int i = 0; i < FC_LEVEL_MAX; ++i) { final double initialFreq; if (i >= 2) { - initialFreq = (double)FC_FREQ_MAX; + initialFreq = FC_FREQ_MAX; } else if (i == 1) { initialFreq = (double)FC_FREQ_MAX / 2; } else if (i == 0) { diff --git a/java/src/com/android/inputmethod/latin/Utils.java b/java/src/com/android/inputmethod/latin/Utils.java index 4178955bc..903b5a357 100644 --- a/java/src/com/android/inputmethod/latin/Utils.java +++ b/java/src/com/android/inputmethod/latin/Utils.java @@ -44,10 +44,8 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; -import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.Map; public class Utils { private Utils() { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java index 89c59f809..0c5d41a5c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java @@ -788,7 +788,7 @@ public class BinaryDictInputOutput { // which is the best approximation. This is how we get the most precise result with // only four bits. final double stepSize = - (double)(MAX_TERMINAL_FREQUENCY - unigramFrequency) / (1.5 + MAX_BIGRAM_FREQUENCY); + (MAX_TERMINAL_FREQUENCY - unigramFrequency) / (1.5 + MAX_BIGRAM_FREQUENCY); final double firstStepStart = 1 + unigramFrequency + (stepSize / 2.0); final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 -- cgit v1.2.3-83-g751a From d10c473347c7e21c383c56786c9eb96fd6513a5c Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Fri, 8 Jun 2012 17:05:28 +0900 Subject: Small performance tweak Change-Id: Icd540742073d49d12e70b2d8bd99aaf7ccb5802d --- .../inputmethod/latin/UserHistoryForgettingCurveUtils.java | 10 +++++----- .../inputmethod/latin/makedict/BinaryDictInputOutput.java | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index 3ae1bd336..6e71885cc 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -199,20 +199,20 @@ public class UserHistoryForgettingCurveUtils { public static final int[][] SCORE_TABLE = new int[FC_LEVEL_MAX][ELAPSED_TIME_MAX + 1]; static { for (int i = 0; i < FC_LEVEL_MAX; ++i) { - final double initialFreq; + final float initialFreq; if (i >= 2) { initialFreq = FC_FREQ_MAX; } else if (i == 1) { - initialFreq = (double)FC_FREQ_MAX / 2; + initialFreq = FC_FREQ_MAX / 2; } else if (i == 0) { - initialFreq = (double)FC_FREQ_MAX / 4; + initialFreq = FC_FREQ_MAX / 4; } else { continue; } for (int j = 0; j < ELAPSED_TIME_MAX; ++j) { - final double elapsedHour = j * ELAPSED_TIME_INTERVAL_HOURS; + final float elapsedHours = j * ELAPSED_TIME_INTERVAL_HOURS; final double freq = - initialFreq * Math.pow(initialFreq, elapsedHour / HALF_LIFE_HOURS); + initialFreq * Math.pow(initialFreq, elapsedHours / HALF_LIFE_HOURS); final int intFreq = Math.min(FC_FREQ_MAX, Math.max(0, (int)freq)); SCORE_TABLE[i][j] = intFreq; } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java index 0c5d41a5c..2c3eee74c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java @@ -787,9 +787,9 @@ public class BinaryDictInputOutput { // (discretizedFrequency + 0.5) times this value to get the median value of the step, // which is the best approximation. This is how we get the most precise result with // only four bits. - final double stepSize = - (MAX_TERMINAL_FREQUENCY - unigramFrequency) / (1.5 + MAX_BIGRAM_FREQUENCY); - final double firstStepStart = 1 + unigramFrequency + (stepSize / 2.0); + final float stepSize = + (MAX_TERMINAL_FREQUENCY - unigramFrequency) / (1.5f + MAX_BIGRAM_FREQUENCY); + final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 // here. The best approximation would be the unigram freq itself, so we should not -- cgit v1.2.3-83-g751a From e7b34b9f867b64eabc3606e5ef21e26eda8de0f6 Mon Sep 17 00:00:00 2001 From: Ken Wakasa Date: Fri, 8 Jun 2012 19:05:47 +0900 Subject: Add NativeUtils.powf() Change-Id: I0339f39d49bc6390d098017f32d92c776974e03e --- .../com/android/inputmethod/latin/NativeUtils.java | 32 +++++++++++++++++ .../latin/UserHistoryForgettingCurveUtils.java | 4 +-- native/jni/Android.mk | 1 + .../com_android_inputmethod_latin_NativeUtils.cpp | 40 ++++++++++++++++++++++ .../com_android_inputmethod_latin_NativeUtils.h | 29 ++++++++++++++++ native/jni/jni_common.cpp | 6 ++++ 6 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 java/src/com/android/inputmethod/latin/NativeUtils.java create mode 100644 native/jni/com_android_inputmethod_latin_NativeUtils.cpp create mode 100644 native/jni/com_android_inputmethod_latin_NativeUtils.h (limited to 'java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java') diff --git a/java/src/com/android/inputmethod/latin/NativeUtils.java b/java/src/com/android/inputmethod/latin/NativeUtils.java new file mode 100644 index 000000000..9cc2bc02e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/NativeUtils.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +public class NativeUtils { + static { + JniUtils.loadNativeLibrary(); + } + + private NativeUtils() { + // This utility class is not publicly instantiable. + } + + /** + * This method just calls up libm's powf() directly. + */ + public static native float powf(float x, float y); +} diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index 6e71885cc..1de95d7b8 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -211,8 +211,8 @@ public class UserHistoryForgettingCurveUtils { } for (int j = 0; j < ELAPSED_TIME_MAX; ++j) { final float elapsedHours = j * ELAPSED_TIME_INTERVAL_HOURS; - final double freq = - initialFreq * Math.pow(initialFreq, elapsedHours / HALF_LIFE_HOURS); + final float freq = initialFreq + * NativeUtils.powf(initialFreq, elapsedHours / HALF_LIFE_HOURS); final int intFreq = Math.min(FC_FREQ_MAX, Math.max(0, (int)freq)); SCORE_TABLE[i][j] = intFreq; } diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 0f7aef269..edcc0677d 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -35,6 +35,7 @@ LOCAL_CFLAGS += -Wno-unused-parameter -Wno-unused-function LATIN_IME_JNI_SRC_FILES := \ com_android_inputmethod_keyboard_ProximityInfo.cpp \ com_android_inputmethod_latin_BinaryDictionary.cpp \ + com_android_inputmethod_latin_NativeUtils.cpp \ jni_common.cpp LATIN_IME_CORE_SRC_FILES := \ diff --git a/native/jni/com_android_inputmethod_latin_NativeUtils.cpp b/native/jni/com_android_inputmethod_latin_NativeUtils.cpp new file mode 100644 index 000000000..c1e586a4b --- /dev/null +++ b/native/jni/com_android_inputmethod_latin_NativeUtils.cpp @@ -0,0 +1,40 @@ +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#include "com_android_inputmethod_latin_NativeUtils.h" +#include "jni.h" +#include "jni_common.h" + +#include + +namespace latinime { + +static float latinime_NativeUtils_powf(float x, float y) { + return powf(x, y); +} + +static JNINativeMethod sMethods[] = { + {"powf", "(FF)F", (void*)latinime_NativeUtils_powf} +}; + +int register_NativeUtils(JNIEnv *env) { + const char* const kClassPathName = "com/android/inputmethod/latin/NativeUtils"; + return registerNativeMethods(env, kClassPathName, sMethods, + sizeof(sMethods) / sizeof(sMethods[0])); +} + +} // namespace latinime diff --git a/native/jni/com_android_inputmethod_latin_NativeUtils.h b/native/jni/com_android_inputmethod_latin_NativeUtils.h new file mode 100644 index 000000000..13a348a5c --- /dev/null +++ b/native/jni/com_android_inputmethod_latin_NativeUtils.h @@ -0,0 +1,29 @@ +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_NATIVEUTILS_H +#define _COM_ANDROID_INPUTMETHOD_LATIN_NATIVEUTILS_H + +#include "jni.h" + +namespace latinime { + +int register_NativeUtils(JNIEnv *env); + +} + +#endif // _COM_ANDROID_INPUTMETHOD_LATIN_NATIVEUTILS_H diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp index b9e2c3255..1314bab27 100644 --- a/native/jni/jni_common.cpp +++ b/native/jni/jni_common.cpp @@ -19,6 +19,7 @@ #include "com_android_inputmethod_keyboard_ProximityInfo.h" #include "com_android_inputmethod_latin_BinaryDictionary.h" +#include "com_android_inputmethod_latin_NativeUtils.h" #include "defines.h" #include "jni.h" #include "proximity_info.h" @@ -52,6 +53,11 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { goto bail; } + if (!register_NativeUtils(env)) { + AKLOGE("ERROR: NativeUtils native registration failed"); + goto bail; + } + /* success -- return valid version number */ result = JNI_VERSION_1_4; -- cgit v1.2.3-83-g751a