aboutsummaryrefslogtreecommitdiffstats
path: root/java/src/com/android/inputmethod/latin/utils
diff options
context:
space:
mode:
Diffstat (limited to 'java/src/com/android/inputmethod/latin/utils')
-rw-r--r--java/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java (renamed from java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java)6
-rw-r--r--java/src/com/android/inputmethod/latin/utils/CollectionUtils.java5
-rw-r--r--java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java10
-rw-r--r--java/src/com/android/inputmethod/latin/utils/StringUtils.java82
-rw-r--r--java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java1
-rw-r--r--java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java40
-rw-r--r--java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java12
7 files changed, 119 insertions, 37 deletions
diff --git a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java b/java/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java
index 1bb27aa2b..2028298f2 100644
--- a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
+++ b/java/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java
@@ -16,17 +16,17 @@
package com.android.inputmethod.latin.utils;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
/**
* This class provides an implementation for the FusionDictionary buffer interface that is backed
* by a simpled byte array. It allows to create a binary dictionary in memory.
*/
-public final class ByteArrayWrapper implements FusionDictionaryBufferInterface {
+public final class ByteArrayDictBuffer implements DictBuffer {
private byte[] mBuffer;
private int mPosition;
- public ByteArrayWrapper(final byte[] buffer) {
+ public ByteArrayDictBuffer(final byte[] buffer) {
mBuffer = buffer;
mPosition = 0;
}
diff --git a/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java b/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java
index 98f0d8b68..cc25102ce 100644
--- a/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java
@@ -18,6 +18,7 @@ package com.android.inputmethod.latin.utils;
import android.util.SparseArray;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -94,6 +95,10 @@ public final class CollectionUtils {
return new CopyOnWriteArrayList<E>(array);
}
+ public static <E> ArrayDeque<E> newArrayDeque() {
+ return new ArrayDeque<E>();
+ }
+
public static <E> SparseArray<E> newSparseArray() {
return new SparseArray<E>();
}
diff --git a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java
index 34eccd65b..021bf0825 100644
--- a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java
@@ -27,10 +27,8 @@ import com.android.inputmethod.latin.BinaryDictionaryGetter;
import com.android.inputmethod.latin.R;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Locale;
@@ -281,13 +279,7 @@ public class DictionaryInfoUtils {
}
public static FileHeader getDictionaryFileHeaderOrNull(final File file) {
- try {
- return BinaryDictIOUtils.getDictionaryFileHeader(file, 0, file.length());
- } catch (UnsupportedFormatException e) {
- return null;
- } catch (IOException e) {
- return null;
- }
+ return BinaryDictIOUtils.getDictionaryFileHeaderOrNull(file, 0, file.length());
}
private static DictionaryInfo createDictionaryInfoFromFileAddress(
diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
index 7406d855a..be4184093 100644
--- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java
@@ -18,7 +18,9 @@ package com.android.inputmethod.latin.utils;
import android.text.TextUtils;
+import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
+import com.android.inputmethod.latin.settings.SettingsValues;
import java.util.ArrayList;
import java.util.Locale;
@@ -193,27 +195,56 @@ public final class StringUtils {
}
public static boolean isIdenticalAfterUpcase(final String text) {
- final int len = text.length();
- for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) {
+ final int length = text.length();
+ int i = 0;
+ while (i < length) {
final int codePoint = text.codePointAt(i);
if (Character.isLetter(codePoint) && !Character.isUpperCase(codePoint)) {
return false;
}
+ i += Character.charCount(codePoint);
}
return true;
}
public static boolean isIdenticalAfterDowncase(final String text) {
- final int len = text.length();
- for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) {
+ final int length = text.length();
+ int i = 0;
+ while (i < length) {
final int codePoint = text.codePointAt(i);
if (Character.isLetter(codePoint) && !Character.isLowerCase(codePoint)) {
return false;
}
+ i += Character.charCount(codePoint);
}
return true;
}
+ @UsedForTesting
+ public static boolean looksValidForDictionaryInsertion(final CharSequence text,
+ final SettingsValues settings) {
+ if (TextUtils.isEmpty(text)) return false;
+ final int length = text.length();
+ int i = 0;
+ int digitCount = 0;
+ while (i < length) {
+ final int codePoint = Character.codePointAt(text, i);
+ final int charCount = Character.charCount(codePoint);
+ i += charCount;
+ if (Character.isDigit(codePoint)) {
+ // Count digits: see below
+ digitCount += charCount;
+ continue;
+ }
+ if (!settings.isWordCodePoint(codePoint)) return false;
+ }
+ // We reject strings entirely comprised of digits to avoid using PIN codes or credit
+ // card numbers. It would come in handy for word prediction though; a good example is
+ // when writing one's address where the street number is usually quite discriminative,
+ // as well as the postal code.
+ return digitCount < length;
+ }
+
public static boolean isIdenticalAfterCapitalizeEachWord(final String text,
final String separators) {
boolean needCapsNext = true;
@@ -316,4 +347,47 @@ public final class StringUtils {
// Otherwise, it doesn't look like an URL.
return false;
}
+
+ public static boolean isEmptyStringOrWhiteSpaces(String s) {
+ final int N = codePointCount(s);
+ for (int i = 0; i < N; ++i) {
+ if (!Character.isWhitespace(s.codePointAt(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @UsedForTesting
+ public static String byteArrayToHexString(byte[] bytes) {
+ if (bytes == null || bytes.length == 0) {
+ return "";
+ }
+ final StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Convert hex string to byte array. The string length must be an even number.
+ */
+ @UsedForTesting
+ public static byte[] hexStringToByteArray(String hexString) {
+ if (TextUtils.isEmpty(hexString)) {
+ return null;
+ }
+ final int N = hexString.length();
+ if (N % 2 != 0) {
+ throw new NumberFormatException("Input hex string length must be an even number."
+ + " Length = " + N);
+ }
+ final byte[] bytes = new byte[N / 2];
+ for (int i = 0; i < N; i += 2) {
+ bytes[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4)
+ + Character.digit(hexString.charAt(i + 1), 16));
+ }
+ return bytes;
+ }
}
diff --git a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java
index 16728092d..102a41b4e 100644
--- a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java
@@ -40,6 +40,7 @@ public final class SubtypeLocaleUtils {
// Special language code to represent "no language".
public static final String NO_LANGUAGE = "zz";
public static final String QWERTY = "qwerty";
+ public static final String EMOJI = "emoji";
public static final int UNKNOWN_KEYBOARD_LAYOUT = R.string.subtype_generic;
private static boolean sInitialized = false;
diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
index d02f7187e..99788f6f2 100644
--- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
@@ -20,20 +20,20 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
-import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
+import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
-import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.PendingAttribute;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
import java.io.IOException;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
/**
* Reads and writes Binary files for a UserHistoryDictionary.
@@ -57,12 +57,12 @@ public final class UserHistoryDictIOUtils {
/**
* Writes dictionary to file.
*/
- public static void writeDictionaryBinary(final OutputStream destination,
+ public static void writeDictionary(final DictEncoder dictEncoder,
final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams,
final FormatOptions formatOptions) {
final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams);
try {
- BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions);
+ dictEncoder.writeDictionary(fusionDict, formatOptions);
Log.d(TAG, "end writing");
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
@@ -77,7 +77,7 @@ public final class UserHistoryDictIOUtils {
@UsedForTesting
static FusionDictionary constructFusionDictionary(
final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams) {
- final FusionDictionary fusionDict = new FusionDictionary(new Node(),
+ final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false,
false));
int profTotal = 0;
@@ -101,7 +101,7 @@ public final class UserHistoryDictIOUtils {
if (word1 == null) { // unigram
fusionDict.add(word2, freq, null, false /* isNotAWord */);
} else { // bigram
- if (FusionDictionary.findWordInTree(fusionDict.mRoot, word1) == null) {
+ if (FusionDictionary.findWordInTree(fusionDict.mRootNodeArray, word1) == null) {
fusionDict.add(word1, 2, null, false /* isNotAWord */);
}
fusionDict.setBigram(word1, word2, freq);
@@ -118,14 +118,13 @@ public final class UserHistoryDictIOUtils {
/**
* Reads dictionary from file.
*/
- public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer,
+ public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder,
final OnAddWordListener dict) {
- final Map<Integer, String> unigrams = CollectionUtils.newTreeMap();
- final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
- final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
+ final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap();
+ final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
+ final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
try {
- BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
- bigrams);
+ dictDecoder.readUnigramsAndBigramsBinary(unigrams, frequencies, bigrams);
} catch (IOException e) {
Log.e(TAG, "IO exception while reading file", e);
} catch (UnsupportedFormatException e) {
@@ -140,10 +139,11 @@ public final class UserHistoryDictIOUtils {
* Adds all unigrams and bigrams in maps to OnAddWordListener.
*/
@UsedForTesting
- static void addWordsFromWordMap(final Map<Integer, String> unigrams,
- final Map<Integer, Integer> frequencies,
- final Map<Integer, ArrayList<PendingAttribute>> bigrams, final OnAddWordListener to) {
- for (Map.Entry<Integer, String> entry : unigrams.entrySet()) {
+ static void addWordsFromWordMap(final TreeMap<Integer, String> unigrams,
+ final TreeMap<Integer, Integer> frequencies,
+ final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams,
+ final OnAddWordListener to) {
+ for (Entry<Integer, String> entry : unigrams.entrySet()) {
final String word1 = entry.getValue();
final int unigramFrequency = frequencies.get(entry.getKey());
to.setUnigram(word1, null, unigramFrequency);
@@ -156,7 +156,7 @@ public final class UserHistoryDictIOUtils {
continue;
}
to.setBigram(word1, word2,
- BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency,
+ BinaryDictIOUtils.reconstructBigramFrequency(unigramFrequency,
attr.mFrequency));
}
}
diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java
index 713a45bda..1992b2f5d 100644
--- a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java
+++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java
@@ -23,7 +23,9 @@ import java.util.concurrent.TimeUnit;
public final class UserHistoryForgettingCurveUtils {
private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName();
private static final boolean DEBUG = false;
- private static final int FC_FREQ_MAX = 127;
+ private static final int DEFAULT_FC_FREQ = 127;
+ private static final int BOOSTED_FC_FREQ = 200;
+ private static int FC_FREQ_MAX = DEFAULT_FC_FREQ;
/* package */ static final int COUNT_MAX = 3;
private static final int FC_LEVEL_MAX = 3;
/* package */ static final int ELAPSED_TIME_MAX = 15;
@@ -33,6 +35,14 @@ public final class UserHistoryForgettingCurveUtils {
private static final int HALF_LIFE_HOURS = 48;
private static final int MAX_PUSH_ELAPSED = (FC_LEVEL_MAX + 1) * (ELAPSED_TIME_MAX + 1);
+ public static void boostMaxFreqForDebug() {
+ FC_FREQ_MAX = BOOSTED_FC_FREQ;
+ }
+
+ public static void resetMaxFreqForDebug() {
+ FC_FREQ_MAX = DEFAULT_FC_FREQ;
+ }
+
private UserHistoryForgettingCurveUtils() {
// This utility class is not publicly instantiable.
}
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-06-02 02:35:35 +0000