aboutsummaryrefslogtreecommitdiffstats
path: root/java/src
diff options
context:
space:
mode:
Diffstat (limited to 'java/src')
-rw-r--r--java/src/com/android/inputmethod/latin/BinaryDictionary.java17
-rw-r--r--java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java46
-rw-r--r--java/src/com/android/inputmethod/latin/makedict/FormatSpec.java6
-rw-r--r--java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java (renamed from java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java)24
-rw-r--r--java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java17
-rw-r--r--java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java2
-rw-r--r--java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java4
-rw-r--r--java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java2
8 files changed, 71 insertions, 47 deletions
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index fdde98da1..a463651d5 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -31,6 +31,7 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;
+import java.util.Map;
/**
* Implements a static, compacted, binary dictionary of standard words.
@@ -104,6 +105,8 @@ public final class BinaryDictionary extends Dictionary {
JniUtils.loadNativeLibrary();
}
+ private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
+ String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void flushNative(long dict, String filePath);
@@ -127,6 +130,20 @@ public final class BinaryDictionary extends Dictionary {
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
+ @UsedForTesting
+ public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
+ final Map<String, String> attributeMap) {
+ final String[] keyArray = new String[attributeMap.size()];
+ final String[] valueArray = new String[attributeMap.size()];
+ int index = 0;
+ for (final String key : attributeMap.keySet()) {
+ keyArray[index] = key;
+ valueArray[index] = attributeMap.get(key);
+ index++;
+ }
+ return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
+ }
+
// TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset,
final long length, final boolean isUpdatable) {
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 0774ce203..99859decf 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,12 +22,7 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
-import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
@@ -35,9 +30,9 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
@@ -68,8 +63,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
- private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
- new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
+ private static final int DICTIONARY_FORMAT_VERSION = 3;
+
+ private static final String SUPPORTS_DYNAMIC_UPDATE =
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE;
/**
* A static map of time recorders, each of which records the time of accesses to a single binary
@@ -233,6 +230,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
});
}
+ protected Map<String, String> getHeaderAttributeMap() {
+ HashMap<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ SUPPORTS_DYNAMIC_UPDATE);
+ return attributeMap;
+ }
+
protected void clear() {
getExecutor(mFilename).execute(new Runnable() {
@Override
@@ -240,17 +244,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename);
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
- false, false));
- final DictEncoder dictEncoder = new Ver3DictEncoder(file);
- try {
- dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
- } catch (IOException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- }
+ BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else {
mDictionaryWriter.clear();
}
@@ -498,17 +493,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
final File file = new File(mContext.getFilesDir(), mFilename);
- final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
- new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
- false, false));
- final DictEncoder dictEncoder = new Ver3DictEncoder(file);
- try {
- dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
- } catch (IOException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- } catch (UnsupportedFormatException e) {
- Log.e(TAG, "Exception in creating new dictionary file.", e);
- }
+ BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+ DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else {
if (mBinaryDictionary.needsToRunGC()) {
mBinaryDictionary.flushWithGC();
diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
index aa5129ccb..849bff050 100644
--- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
+++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
@@ -325,6 +325,12 @@ public final class FormatSpec {
public final int mHeaderSize;
public final DictionaryOptions mDictionaryOptions;
public final FormatOptions mFormatOptions;
+ // Note that these are corresponding definitions in native code in latinime::HeaderPolicy
+ // and latinime::HeaderReadWriteUtils.
+ public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";
+ public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
+ public static final String ATTRIBUTE_VALUE_TRUE = "1";
+
private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index 075d7e3c3..66517a800 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -34,12 +34,15 @@ import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListe
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
/**
- * This class is a base class of a dictionary for the personalized prediction language model.
+ * This class is a base class of a dictionary that supports decaying for the personalized language
+ * model.
*/
-public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {
- private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();
+public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
+ private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
public static final boolean DBG_SAVE_RESTORE = false;
private static final boolean DBG_STRESS_TEST = false;
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
@@ -60,8 +63,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Should always be false except when we use this class for test
@UsedForTesting boolean mIsTest = false;
- /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,
- final SharedPreferences sp, final String dictionaryType, final String fileName) {
+ /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
+ final String locale, final SharedPreferences sp, final String dictionaryType,
+ final String fileName) {
super(context, fileName, dictionaryType, true);
mLocale = locale;
mFileName = fileName;
@@ -84,6 +88,16 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
}
@Override
+ protected Map<String, String> getHeaderAttributeMap() {
+ HashMap<String, String> attributeMap = new HashMap<String, String>();
+ attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
+ FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+ return attributeMap;
+ }
+
+ @Override
protected boolean hasContentChanged() {
return false;
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index ab3de801c..c616a296c 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -46,7 +46,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Use a dynamic binary dictionary instead
public WeakReference<PersonalizationDictionary> mDictionary;
- public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary;
+ public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;
public final String mSystemLocale;
public PersonalizationDictionaryUpdateSession(String locale) {
mSystemLocale = locale;
@@ -60,15 +60,16 @@ public abstract class PersonalizationDictionaryUpdateSession {
mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
}
- public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) {
- mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary);
+ public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {
+ mPredictionDictionary =
+ new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);
}
protected PersonalizationDictionary getDictionary() {
return mDictionary == null ? null : mDictionary.get();
}
- protected DynamicPredictionDictionaryBase getPredictionDictionary() {
+ protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {
return mPredictionDictionary == null ? null : mPredictionDictionary.get();
}
@@ -81,7 +82,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
}
private void unsetPredictionDictionary() {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
@@ -89,7 +90,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
}
public void clearAndFlushPredictionDictionary(Context context) {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
@@ -105,7 +106,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Support multi locale to add bigram
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
int frequency) {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
@@ -116,7 +117,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Support multi locale to add bigram
public void addBigramsToPersonalizationDictionary(
final ArrayList<PersonalizationLanguageModelParam> lmParams) {
- final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
+ final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
index e80953c05..432954453 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
@@ -22,7 +22,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import android.content.Context;
import android.content.SharedPreferences;
-public class PersonalizationPredictionDictionary extends DynamicPredictionDictionaryBase {
+public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
index 4c1803bdf..55a90ee51 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
@@ -54,7 +54,7 @@ public final class UserHistoryDictionaryBigramList {
* Called when loaded from the SQL DB.
*/
public void addBigram(String word1, String word2, byte fcValue) {
- if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) {
+ if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue);
}
final HashMap<String, Byte> map;
@@ -74,7 +74,7 @@ public final class UserHistoryDictionaryBigramList {
* Called when inserted to the SQL DB.
*/
public void updateBigram(String word1, String word2, byte fcValue) {
- if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) {
+ if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue);
}
final HashMap<String, Byte> map;
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
index b140c919b..38e308a4e 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
@@ -26,7 +26,7 @@ import android.content.SharedPreferences;
* Locally gathers stats about the words user types and various other signals like auto-correction
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
*/
-public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase {
+public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
/* package for tests */ static final String NAME =
UserHistoryPredictionDictionary.class.getSimpleName();
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-05-05 03:19:31 +0000