diff options
64 files changed, 1535 insertions, 180 deletions
diff --git a/common/src/com/android/inputmethod/latin/common/LocaleUtils.java b/common/src/com/android/inputmethod/latin/common/LocaleUtils.java index 14b3d220d..7f2333be5 100644 --- a/common/src/com/android/inputmethod/latin/common/LocaleUtils.java +++ b/common/src/com/android/inputmethod/latin/common/LocaleUtils.java @@ -17,8 +17,12 @@ package com.android.inputmethod.latin.common; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + /** * A class to help with handling Locales in string form. * @@ -160,26 +164,49 @@ public final class LocaleUtils { /** * Creates a locale from a string specification. + * @param localeString a string specification of a locale, in a format of "ll_cc_variant" where + * "ll" is a language code, "cc" is a country code. */ - public static Locale constructLocaleFromString(final String localeStr) { - if (localeStr == null) + @Nullable + public static Locale constructLocaleFromString(@Nullable final String localeString) { + if (localeString == null) { return null; + } synchronized (sLocaleCache) { - if (sLocaleCache.containsKey(localeStr)) - return sLocaleCache.get(localeStr); - Locale retval = null; - String[] localeParams = localeStr.split("_", 3); - if (localeParams.length == 1) { - retval = new Locale(localeParams[0]); - } else if (localeParams.length == 2) { - retval = new Locale(localeParams[0], localeParams[1]); - } else if (localeParams.length == 3) { - retval = new Locale(localeParams[0], localeParams[1], localeParams[2]); + if (sLocaleCache.containsKey(localeString)) { + return sLocaleCache.get(localeString); } - if (retval != null) { - sLocaleCache.put(localeStr, retval); + final String[] elements = localeString.split("_", 3); + final Locale locale; + if (elements.length == 1) { + locale = new Locale(elements[0] /* language */); + } else if (elements.length == 2) { + locale = new Locale(elements[0] /* language */, elements[1] /* country */); + } else { // localeParams.length == 3 + locale = new Locale(elements[0] /* language */, elements[1] /* country */, + elements[2] /* variant */); } - return retval; + sLocaleCache.put(localeString, locale); + return locale; } } + + // TODO: Get this information from the framework instead of maintaining here by ourselves. + private static final HashSet<String> sRtlLanguageCodes = new HashSet<>(); + static { + // List of known Right-To-Left language codes. + sRtlLanguageCodes.add("ar"); // Arabic + sRtlLanguageCodes.add("fa"); // Persian + sRtlLanguageCodes.add("iw"); // Hebrew + sRtlLanguageCodes.add("ku"); // Kurdish + sRtlLanguageCodes.add("ps"); // Pashto + sRtlLanguageCodes.add("sd"); // Sindhi + sRtlLanguageCodes.add("ug"); // Uyghur + sRtlLanguageCodes.add("ur"); // Urdu + sRtlLanguageCodes.add("yi"); // Yiddish + } + + public static boolean isRtlLanguage(@Nonnull final Locale locale) { + return sRtlLanguageCodes.contains(locale.getLanguage()); + } } diff --git a/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java b/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java index f80625644..43675682d 100644 --- a/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java +++ b/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java @@ -48,4 +48,10 @@ public final class ProductionFlags { * When {@code true}, personal dictionary sync feature is ready to be enabled. */ public static final boolean ENABLE_PERSONAL_DICTIONARY_SYNC = ENABLE_ACCOUNT_SIGN_IN && false; + + /** + * When {@code true}, the IME maintains per account {@link UserHistoryDictionary}. + */ + public static final boolean ENABLE_PER_ACCOUNT_USER_HISTORY_DICTIONARY = + ENABLE_ACCOUNT_SIGN_IN && false; } diff --git a/java/res/drawable/btn_keyboard_key_ics.xml b/java/res/drawable/btn_keyboard_key_ics.xml index 0bb098d23..bacd5d78f 100644 --- a/java/res/drawable/btn_keyboard_key_ics.xml +++ b/java/res/drawable/btn_keyboard_key_ics.xml @@ -32,6 +32,8 @@ android:drawable="@drawable/btn_keyboard_key_normal_off_holo_dark" /> <!-- Empty background keys. --> + <item android:state_empty="true" android:state_pressed="true" + android:drawable="@drawable/btn_keyboard_key_pressed_ics_light" /> <item android:state_empty="true" android:drawable="@android:color/transparent" /> diff --git a/java/res/drawable/btn_keyboard_key_klp.xml b/java/res/drawable/btn_keyboard_key_klp.xml index 2a202a179..e2f208585 100644 --- a/java/res/drawable/btn_keyboard_key_klp.xml +++ b/java/res/drawable/btn_keyboard_key_klp.xml @@ -32,6 +32,8 @@ android:drawable="@drawable/btn_keyboard_key_normal_off_holo_dark" /> <!-- Empty background keys. --> + <item android:state_empty="true" android:state_pressed="true" + android:drawable="@drawable/btn_keyboard_key_pressed_klp_light" /> <item android:state_empty="true" android:drawable="@android:color/transparent" /> diff --git a/java/res/drawable/btn_keyboard_key_lxx_dark.xml b/java/res/drawable/btn_keyboard_key_lxx_dark.xml index bb1789ae3..161592d3d 100644 --- a/java/res/drawable/btn_keyboard_key_lxx_dark.xml +++ b/java/res/drawable/btn_keyboard_key_lxx_dark.xml @@ -32,8 +32,10 @@ android:drawable="@drawable/btn_keyboard_key_normal_off_lxx_dark" /> <!-- Empty background keys. --> + <item android:state_empty="true" android:state_pressed="true" + android:drawable="@color/key_background_pressed_lxx_dark" /> <item android:state_empty="true" - android:drawable="@color/key_background_lxx_dark" /> + android:drawable="@android:color/transparent" /> <!-- Normal keys. --> <item android:state_pressed="true" diff --git a/java/res/drawable/btn_keyboard_key_lxx_light.xml b/java/res/drawable/btn_keyboard_key_lxx_light.xml index 60fe02dd2..0154d75b4 100644 --- a/java/res/drawable/btn_keyboard_key_lxx_light.xml +++ b/java/res/drawable/btn_keyboard_key_lxx_light.xml @@ -32,8 +32,10 @@ android:drawable="@drawable/btn_keyboard_key_normal_off_lxx_light" /> <!-- Empty background keys. --> + <item android:state_empty="true" android:state_pressed="true" + android:drawable="@color/key_background_pressed_lxx_light" /> <item android:state_empty="true" - android:drawable="@color/key_background_lxx_light" /> + android:drawable="@android:color/transparent" /> <!-- Normal keys. --> <item android:state_pressed="true" diff --git a/java/res/values-eu-rES/strings-emoji-descriptions.xml b/java/res/values-eu-rES/strings-emoji-descriptions.xml index 2faec968a..c774ae1b8 100644 --- a/java/res/values-eu-rES/strings-emoji-descriptions.xml +++ b/java/res/values-eu-rES/strings-emoji-descriptions.xml @@ -846,6 +846,6 @@ <string name="spoken_emoji_1F6C1" msgid="2845056048320031158">"Bainuontzia"</string> <string name="spoken_emoji_1F6C2" msgid="8117262514698011877">"Pasaporte-kontrola"</string> <string name="spoken_emoji_1F6C3" msgid="1176342001834630675">"Aduana"</string> - <string name="spoken_emoji_1F6C4" msgid="1477622834179978886">"Maleta-erreklamazioa"</string> + <string name="spoken_emoji_1F6C4" msgid="1477622834179978886">"Ekipaje-erreklamazioa"</string> <string name="spoken_emoji_1F6C5" msgid="2495834050856617451">"Ahaztutako maletak"</string> </resources> diff --git a/java/res/values-km-rKH/strings-emoji-descriptions.xml b/java/res/values-km-rKH/strings-emoji-descriptions.xml index 757df50e7..e9b8780a5 100644 --- a/java/res/values-km-rKH/strings-emoji-descriptions.xml +++ b/java/res/values-km-rKH/strings-emoji-descriptions.xml @@ -267,7 +267,7 @@ <string name="spoken_emoji_1F36A" msgid="2726271795913042295">"ខូគី"</string> <string name="spoken_emoji_1F36B" msgid="6342163604299875931">"សូកូឡា"</string> <string name="spoken_emoji_1F36C" msgid="2168934753998218790">"ស្ករគ្រាប់"</string> - <string name="spoken_emoji_1F36D" msgid="3671507903799975792">"ស្ករគ្រាប់មានដងកាន់"</string> + <string name="spoken_emoji_1F36D" msgid="3671507903799975792">"Lollipop"</string> <string name="spoken_emoji_1F36E" msgid="4630541402785165902">"សង់ខ្យា"</string> <string name="spoken_emoji_1F36F" msgid="5577915387425169439">"ថូ"</string> <string name="spoken_emoji_1F370" msgid="7243244547866114951">"នំខេក"</string> diff --git a/java/res/values-pt/strings.xml b/java/res/values-pt/strings.xml index 2e0cd3b55..2cc86e12f 100644 --- a/java/res/values-pt/strings.xml +++ b/java/res/values-pt/strings.xml @@ -102,8 +102,7 @@ <string name="subtype_with_layout_hi_ZZ" msgid="6827402953860547044">"Híndi-inglês (<xliff:g id="KEYBOARD_LAYOUT">%s</xliff:g>)"</string> <string name="subtype_with_layout_sr_ZZ" msgid="2859024772719772407">"Sérvio (<xliff:g id="KEYBOARD_LAYOUT">%s</xliff:g>)"</string> <string name="subtype_generic_traditional" msgid="8584594350973800586">"<xliff:g id="LANGUAGE_NAME">%s</xliff:g> (tradicional)"</string> - <!-- no translation found for subtype_generic_compact (3353673321203202922) --> - <skip /> + <string name="subtype_generic_compact" msgid="3353673321203202922">"<xliff:g id="LANGUAGE_NAME">%s</xliff:g> (compacto)"</string> <string name="subtype_no_language" msgid="7137390094240139495">"Nenhum idioma (alfabeto)"</string> <string name="subtype_no_language_qwerty" msgid="244337630616742604">"Alfabeto (QWERTY)"</string> <string name="subtype_no_language_qwertz" msgid="443066912507547976">"Alfabeto (QWERTZ)"</string> diff --git a/java/res/xml/kbd_emoji_category1.xml b/java/res/xml/kbd_emoji_category1.xml index 2770cfb15..5145ea9d3 100644 --- a/java/res/xml/kbd_emoji_category1.xml +++ b/java/res/xml/kbd_emoji_category1.xml @@ -27,5 +27,5 @@ <GridRows latin:codesArray="@array/emoji_faces" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/res/xml/kbd_emoji_category2.xml b/java/res/xml/kbd_emoji_category2.xml index d547056e1..ac8784f4b 100644 --- a/java/res/xml/kbd_emoji_category2.xml +++ b/java/res/xml/kbd_emoji_category2.xml @@ -27,5 +27,5 @@ <GridRows latin:codesArray="@array/emoji_objects" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/res/xml/kbd_emoji_category3.xml b/java/res/xml/kbd_emoji_category3.xml index 2172d9880..88c4db92b 100644 --- a/java/res/xml/kbd_emoji_category3.xml +++ b/java/res/xml/kbd_emoji_category3.xml @@ -27,5 +27,5 @@ <GridRows latin:codesArray="@array/emoji_nature" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/res/xml/kbd_emoji_category4.xml b/java/res/xml/kbd_emoji_category4.xml index 46b6d46e8..262384d80 100644 --- a/java/res/xml/kbd_emoji_category4.xml +++ b/java/res/xml/kbd_emoji_category4.xml @@ -27,5 +27,5 @@ <GridRows latin:codesArray="@array/emoji_places" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/res/xml/kbd_emoji_category5.xml b/java/res/xml/kbd_emoji_category5.xml index 4304701d4..bf823f978 100644 --- a/java/res/xml/kbd_emoji_category5.xml +++ b/java/res/xml/kbd_emoji_category5.xml @@ -27,5 +27,5 @@ <GridRows latin:codesArray="@array/emoji_symbols" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/res/xml/kbd_emoji_category6.xml b/java/res/xml/kbd_emoji_category6.xml index 516ed7a42..edb82fc64 100644 --- a/java/res/xml/kbd_emoji_category6.xml +++ b/java/res/xml/kbd_emoji_category6.xml @@ -28,5 +28,5 @@ <GridRows latin:textsArray="@array/emoji_emoticons" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/res/xml/kbd_emoji_recents.xml b/java/res/xml/kbd_emoji_recents.xml index 4953c1079..edf3872c1 100644 --- a/java/res/xml/kbd_emoji_recents.xml +++ b/java/res/xml/kbd_emoji_recents.xml @@ -28,5 +28,5 @@ <GridRows latin:codesArray="@array/emoji_recents" latin:keyLabelFlags="fontNormal" - latin:backgroundType="normal" /> + latin:backgroundType="empty" /> </Keyboard> diff --git a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java index db4315f8f..ce23eb752 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java +++ b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java @@ -47,7 +47,8 @@ public class MetadataDbHelper extends SQLiteOpenHelper { // used to identify the versions for upgrades. This should never change going forward. private static final int METADATA_DATABASE_VERSION_WITH_CLIENTID = 6; // The current database version. - private static final int CURRENT_METADATA_DATABASE_VERSION = 10; + // This MUST be increased every time the dictionary pack metadata URL changes. + private static final int CURRENT_METADATA_DATABASE_VERSION = 11; private final static long NOT_A_DOWNLOAD_ID = -1; diff --git a/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java b/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java index bd6152119..aeb666704 100644 --- a/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java +++ b/java/src/com/android/inputmethod/dictionarypack/UpdateHandler.java @@ -40,6 +40,7 @@ import com.android.inputmethod.compat.DownloadManagerCompatUtils; import com.android.inputmethod.compat.NotificationCompatUtils; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.common.LocaleUtils; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.utils.ApplicationUtils; import com.android.inputmethod.latin.utils.DebugLogUtils; @@ -79,7 +80,8 @@ public final class UpdateHandler { // DownloadManager uses as an ID numbers returned out of an AUTOINCREMENT column // in SQLite, so it should never return anything < 0. public static final int NOT_AN_ID = -1; - public static final int MAXIMUM_SUPPORTED_FORMAT_VERSION = 2; + public static final int MAXIMUM_SUPPORTED_FORMAT_VERSION = + FormatSpec.MAXIMUM_SUPPORTED_STATIC_VERSION; // Arbitrary. Probably good if it's a power of 2, and a couple thousand bytes long. private static final int FILE_COPY_BUFFER_SIZE = 8192; diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java index 51f37fdc6..b1051385d 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java @@ -294,7 +294,7 @@ public final class KeyboardLayoutSet { : subtype; mParams.mSubtype = keyboardSubtype; mParams.mKeyboardLayoutSetName = KEYBOARD_LAYOUT_SET_RESOURCE_PREFIX - + SubtypeLocaleUtils.getKeyboardLayoutSetName(keyboardSubtype); + + keyboardSubtype.getKeyboardLayoutSetName(); return this; } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java index b07693c76..01980ea6e 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java @@ -25,7 +25,6 @@ import android.graphics.Paint; import android.graphics.Paint.Align; import android.graphics.PorterDuff; import android.graphics.Rect; -import android.graphics.Region; import android.graphics.Typeface; import android.graphics.drawable.Drawable; import android.graphics.drawable.NinePatchDrawable; @@ -41,6 +40,7 @@ import com.android.inputmethod.latin.utils.TypefaceUtils; import java.util.HashSet; +import javax.annotation.Nonnull; import javax.annotation.Nullable; /** @@ -100,6 +100,8 @@ public class KeyboardView extends View { private static final float MAX_LABEL_RATIO = 0.90f; // Main keyboard + // TODO: Consider having a dummy keyboard object to make this @Nonnull + @Nullable private Keyboard mKeyboard; protected final KeyDrawParams mKeyDrawParams = new KeyDrawParams(); @@ -108,14 +110,14 @@ public class KeyboardView extends View { private boolean mInvalidateAllKeys; /** The keys that should be drawn */ private final HashSet<Key> mInvalidatedKeys = new HashSet<>(); - /** The working rectangle variable */ - private final Rect mWorkingRect = new Rect(); + /** The working rectangle for clipping */ + private final Rect mClipRect = new Rect(); /** The keyboard bitmap buffer for faster updates */ - /** The clip region to draw keys */ - private final Region mClipRegion = new Region(); private Bitmap mOffscreenBuffer; /** The canvas for the above mutable keyboard bitmap */ + @Nonnull private final Canvas mOffscreenCanvas = new Canvas(); + @Nonnull private final Paint mPaint = new Paint(); private final Paint.FontMetrics mFontMetrics = new Paint.FontMetrics(); public KeyboardView(final Context context, final AttributeSet attrs) { @@ -161,11 +163,12 @@ public class KeyboardView extends View { mPaint.setAntiAlias(true); } + @Nullable public KeyVisualAttributes getKeyVisualAttribute() { return mKeyVisualAttributes; } - private static void blendAlpha(final Paint paint, final int alpha) { + private static void blendAlpha(@Nonnull final Paint paint, final int alpha) { final int color = paint.getColor(); paint.setARGB((paint.getAlpha() * alpha) / Constants.Color.ALPHA_OPAQUE, Color.red(color), Color.green(color), Color.blue(color)); @@ -184,7 +187,7 @@ public class KeyboardView extends View { * @see #getKeyboard() * @param keyboard the keyboard to display in this view */ - public void setKeyboard(final Keyboard keyboard) { + public void setKeyboard(@Nonnull final Keyboard keyboard) { mKeyboard = keyboard; final int keyHeight = keyboard.mMostCommonKeyHeight - keyboard.mVerticalGap; mKeyDrawParams.updateParams(keyHeight, mKeyVisualAttributes); @@ -198,6 +201,7 @@ public class KeyboardView extends View { * @return the currently attached keyboard * @see #setKeyboard(Keyboard) */ + @Nullable public Keyboard getKeyboard() { return mKeyboard; } @@ -212,13 +216,14 @@ public class KeyboardView extends View { @Override protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { - if (mKeyboard == null) { + final Keyboard keyboard = getKeyboard(); + if (keyboard == null) { super.onMeasure(widthMeasureSpec, heightMeasureSpec); return; } // The main keyboard expands to the entire this {@link KeyboardView}. - final int width = mKeyboard.mOccupiedWidth + getPaddingLeft() + getPaddingRight(); - final int height = mKeyboard.mOccupiedHeight + getPaddingTop() + getPaddingBottom(); + final int width = keyboard.mOccupiedWidth + getPaddingLeft() + getPaddingRight(); + final int height = keyboard.mOccupiedHeight + getPaddingTop() + getPaddingBottom(); setMeasuredDimension(width, height); } @@ -266,52 +271,45 @@ public class KeyboardView extends View { } } - private void onDrawKeyboard(final Canvas canvas) { - if (mKeyboard == null) return; + private void onDrawKeyboard(@Nonnull final Canvas canvas) { + final Keyboard keyboard = getKeyboard(); + if (keyboard == null) { + return; + } - final int width = getWidth(); - final int height = getHeight(); final Paint paint = mPaint; - + final Drawable background = getBackground(); // Calculate clip region and set. final boolean drawAllKeys = mInvalidateAllKeys || mInvalidatedKeys.isEmpty(); final boolean isHardwareAccelerated = canvas.isHardwareAccelerated(); // TODO: Confirm if it's really required to draw all keys when hardware acceleration is on. if (drawAllKeys || isHardwareAccelerated) { - mClipRegion.set(0, 0, width, height); - } else { - mClipRegion.setEmpty(); - for (final Key key : mInvalidatedKeys) { - if (mKeyboard.hasKey(key)) { - final int x = key.getX() + getPaddingLeft(); - final int y = key.getY() + getPaddingTop(); - mWorkingRect.set(x, y, x + key.getWidth(), y + key.getHeight()); - mClipRegion.union(mWorkingRect); - } - } - } - if (!isHardwareAccelerated) { - canvas.clipRegion(mClipRegion, Region.Op.REPLACE); - // Draw keyboard background. - canvas.drawColor(Color.BLACK, PorterDuff.Mode.CLEAR); - final Drawable background = getBackground(); - if (background != null) { + if (!isHardwareAccelerated && background != null) { + // Need to draw keyboard background on {@link #mOffscreenBuffer}. + canvas.drawColor(Color.BLACK, PorterDuff.Mode.CLEAR); background.draw(canvas); } - } - - // TODO: Confirm if it's really required to draw all keys when hardware acceleration is on. - if (drawAllKeys || isHardwareAccelerated) { // Draw all keys. - for (final Key key : mKeyboard.getSortedKeys()) { + for (final Key key : keyboard.getSortedKeys()) { onDrawKey(key, canvas, paint); } } else { - // Draw invalidated keys. for (final Key key : mInvalidatedKeys) { - if (mKeyboard.hasKey(key)) { - onDrawKey(key, canvas, paint); + if (!keyboard.hasKey(key)) { + continue; } + if (background != null) { + // Need to redraw key's background on {@link #mOffscreenBuffer}. + final int x = key.getX() + getPaddingLeft(); + final int y = key.getY() + getPaddingTop(); + mClipRect.set(x, y, x + key.getWidth(), y + key.getHeight()); + canvas.save(); + canvas.clipRect(mClipRect); + canvas.drawColor(Color.BLACK, PorterDuff.Mode.CLEAR); + background.draw(canvas); + canvas.restore(); + } + onDrawKey(key, canvas, paint); } } @@ -319,20 +317,22 @@ public class KeyboardView extends View { mInvalidateAllKeys = false; } - private void onDrawKey(final Key key, final Canvas canvas, final Paint paint) { + private void onDrawKey(@Nonnull final Key key, @Nonnull final Canvas canvas, + @Nonnull final Paint paint) { final int keyDrawX = key.getDrawX() + getPaddingLeft(); final int keyDrawY = key.getY() + getPaddingTop(); canvas.translate(keyDrawX, keyDrawY); - final int keyHeight = mKeyboard.mMostCommonKeyHeight - mKeyboard.mVerticalGap; final KeyVisualAttributes attr = key.getVisualAttributes(); - final KeyDrawParams params = mKeyDrawParams.mayCloneAndUpdateParams(keyHeight, attr); + final KeyDrawParams params = mKeyDrawParams.mayCloneAndUpdateParams(key.getHeight(), attr); params.mAnimAlpha = Constants.Color.ALPHA_OPAQUE; if (!key.isSpacer()) { final Drawable background = key.selectBackgroundDrawable( mKeyBackground, mFunctionalKeyBackground, mSpacebarBackground); - onDrawKeyBackground(key, canvas, background); + if (background != null) { + onDrawKeyBackground(key, canvas, background); + } } onDrawKeyTopVisuals(key, canvas, paint, params); @@ -340,8 +340,8 @@ public class KeyboardView extends View { } // Draw key background. - protected void onDrawKeyBackground(final Key key, final Canvas canvas, - final Drawable background) { + protected void onDrawKeyBackground(@Nonnull final Key key, @Nonnull final Canvas canvas, + @Nonnull final Drawable background) { final int keyWidth = key.getDrawWidth(); final int keyHeight = key.getHeight(); final int bgWidth, bgHeight, bgX, bgY; @@ -373,15 +373,17 @@ public class KeyboardView extends View { } // Draw key top visuals. - protected void onDrawKeyTopVisuals(final Key key, final Canvas canvas, final Paint paint, - final KeyDrawParams params) { + protected void onDrawKeyTopVisuals(@Nonnull final Key key, @Nonnull final Canvas canvas, + @Nonnull final Paint paint, @Nonnull final KeyDrawParams params) { final int keyWidth = key.getDrawWidth(); final int keyHeight = key.getHeight(); final float centerX = keyWidth * 0.5f; final float centerY = keyHeight * 0.5f; // Draw key label. - final Drawable icon = key.getIcon(mKeyboard.mIconsSet, params.mAnimAlpha); + final Keyboard keyboard = getKeyboard(); + final Drawable icon = (keyboard == null) ? null + : key.getIcon(keyboard.mIconsSet, params.mAnimAlpha); float labelX = centerX; float labelBaseline = centerY; final String label = key.getLabel(); @@ -500,8 +502,8 @@ public class KeyboardView extends View { } // Draw popup hint "..." at the bottom right corner of the key. - protected void drawKeyPopupHint(final Key key, final Canvas canvas, final Paint paint, - final KeyDrawParams params) { + protected void drawKeyPopupHint(@Nonnull final Key key, @Nonnull final Canvas canvas, + @Nonnull final Paint paint, @Nonnull final KeyDrawParams params) { if (TextUtils.isEmpty(mKeyPopupHintLetter)) { return; } @@ -518,15 +520,15 @@ public class KeyboardView extends View { canvas.drawText(mKeyPopupHintLetter, hintX, hintY, paint); } - protected static void drawIcon(final Canvas canvas, final Drawable icon, final int x, - final int y, final int width, final int height) { + protected static void drawIcon(@Nonnull final Canvas canvas,@Nonnull final Drawable icon, + final int x, final int y, final int width, final int height) { canvas.translate(x, y); icon.setBounds(0, 0, width, height); icon.draw(canvas); canvas.translate(-x, -y); } - public Paint newLabelPaint(final Key key) { + public Paint newLabelPaint(@Nullable final Key key) { final Paint paint = new Paint(); paint.setAntiAlias(true); if (key == null) { @@ -560,7 +562,7 @@ public class KeyboardView extends View { * @see #invalidateAllKeys */ public void invalidateKey(@Nullable final Key key) { - if (key == null || mInvalidateAllKeys) { + if (mInvalidateAllKeys || key == null) { return; } mInvalidatedKeys.add(key); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java index c739bf3e0..51f89c122 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java @@ -36,7 +36,6 @@ import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.common.Constants; import com.android.inputmethod.latin.common.StringUtils; import com.android.inputmethod.latin.utils.ResourceUtils; -import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; import com.android.inputmethod.latin.utils.XmlParseUtils; import com.android.inputmethod.latin.utils.XmlParseUtils.ParseException; @@ -648,7 +647,7 @@ public class KeyboardBuilder<KP extends KeyboardParams> { try { final boolean keyboardLayoutSetMatched = matchString(caseAttr, R.styleable.Keyboard_Case_keyboardLayoutSet, - SubtypeLocaleUtils.getKeyboardLayoutSetName(id.mSubtype)); + id.mSubtype.getKeyboardLayoutSetName()); final boolean keyboardLayoutSetElementMatched = matchTypedValue(caseAttr, R.styleable.Keyboard_Case_keyboardLayoutSetElement, id.mElementId, KeyboardId.elementIdToName(id.mElementId)); diff --git a/java/src/com/android/inputmethod/keyboard/internal/LanguageOnSpacebarHelper.java b/java/src/com/android/inputmethod/keyboard/internal/LanguageOnSpacebarHelper.java index 2a70ef51a..8ed80107a 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/LanguageOnSpacebarHelper.java +++ b/java/src/com/android/inputmethod/keyboard/internal/LanguageOnSpacebarHelper.java @@ -52,7 +52,7 @@ public final class LanguageOnSpacebarHelper { return FORMAT_TYPE_MULTIPLE; } final String keyboardLanguage = locales[0].getLanguage(); - final String keyboardLayout = SubtypeLocaleUtils.getKeyboardLayoutSetName(subtype); + final String keyboardLayout = subtype.getKeyboardLayoutSetName(); int sameLanguageAndLayoutCount = 0; for (final InputMethodSubtype ims : mEnabledSubtypes) { final String language = SubtypeLocaleUtils.getSubtypeLocale(ims).getLanguage(); diff --git a/java/src/com/android/inputmethod/latin/AssetFileAddress.java b/java/src/com/android/inputmethod/latin/AssetFileAddress.java index 923f43cbe..f8d02d6ea 100644 --- a/java/src/com/android/inputmethod/latin/AssetFileAddress.java +++ b/java/src/com/android/inputmethod/latin/AssetFileAddress.java @@ -62,4 +62,9 @@ public final class AssetFileAddress { public void deleteUnderlyingFile() { FileUtils.deleteRecursively(new File(mFilename)); } + + @Override + public String toString() { + return String.format("%s (offset=%d, length=%d)", mFilename, mOffset, mLength); + } } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 46c8d5562..5afb62b69 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -284,7 +284,8 @@ final public class BinaryDictionaryGetter { final AssetFileAddress afa = AssetFileAddress.makeFromFileName(f.getPath()); if (null != afa) fileList.add(afa); } else { - Log.e(TAG, "Found a cached dictionary file but cannot read or use it"); + Log.e(TAG, "Found a cached dictionary file for " + locale.toString() + + " but cannot read or use it"); } } diff --git a/java/src/com/android/inputmethod/latin/RichInputMethodSubtype.java b/java/src/com/android/inputmethod/latin/RichInputMethodSubtype.java index 03f6d60ab..ea8d4a210 100644 --- a/java/src/com/android/inputmethod/latin/RichInputMethodSubtype.java +++ b/java/src/com/android/inputmethod/latin/RichInputMethodSubtype.java @@ -135,13 +135,18 @@ public final class RichInputMethodSubtype { public boolean isRtlSubtype() { // The subtype is considered RTL if the language of the main subtype is RTL. - return SubtypeLocaleUtils.isRtlLanguage(mLocales[0]); + return LocaleUtils.isRtlLanguage(mLocales[0]); } // TODO: remove this method @Nonnull public InputMethodSubtype getRawSubtype() { return mSubtype; } + @Nonnull + public String getKeyboardLayoutSetName() { + return SubtypeLocaleUtils.getKeyboardLayoutSetName(mSubtype); + } + // Dummy no language QWERTY subtype. See {@link R.xml.method}. private static final int SUBTYPE_ID_OF_DUMMY_NO_LANGUAGE_SUBTYPE = 0xdde0bfd3; private static final String EXTRA_VALUE_OF_DUMMY_NO_LANGUAGE_SUBTYPE = diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 4ef504856..eba9654a5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -179,8 +179,8 @@ public final class FormatSpec { public static final int VERSION403 = 403; public static final int VERSION4 = VERSION403; public static final int VERSION4_DEV = VERSION403; - static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202; - static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION202; + public static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202; + public static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION202; static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4; static final int MAXIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4_DEV; diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index ac2fc07c2..8c5eb0aa7 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -28,32 +28,45 @@ import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Helps handle and manage personalized dictionaries such as {@link UserHistoryDictionary} and + * {@link PersonalizationDictionary}. + */ public class PersonalizationHelper { private static final String TAG = PersonalizationHelper.class.getSimpleName(); private static final boolean DEBUG = false; + private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>> sLangUserHistoryDictCache = new ConcurrentHashMap<>(); private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> sLangPersonalizationDictCache = new ConcurrentHashMap<>(); + @Nonnull public static UserHistoryDictionary getUserHistoryDictionary( - final Context context, final Locale locale) { - final String localeStr = locale.toString(); + final Context context, final Locale locale, @Nullable final String accountName) { + String lookupStr = locale.toString(); + if (accountName != null) { + lookupStr += "." + accountName; + } synchronized (sLangUserHistoryDictCache) { - if (sLangUserHistoryDictCache.containsKey(localeStr)) { + if (sLangUserHistoryDictCache.containsKey(lookupStr)) { final SoftReference<UserHistoryDictionary> ref = - sLangUserHistoryDictCache.get(localeStr); + sLangUserHistoryDictCache.get(lookupStr); final UserHistoryDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { - Log.w(TAG, "Use cached UserHistoryDictionary for " + locale); + Log.d(TAG, "Use cached UserHistoryDictionary for " + locale + + " & account" + accountName); } dict.reloadDictionaryIfRequired(); return dict; } } final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale); - sLangUserHistoryDictCache.put(localeStr, new SoftReference<>(dict)); + sLangUserHistoryDictCache.put(lookupStr, new SoftReference<>(dict)); return dict; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 58782c646..946835cbc 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -17,30 +17,73 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; +import android.content.SharedPreferences; +import android.preference.PreferenceManager; import com.android.inputmethod.annotations.ExternallyReferenced; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.common.Constants; +import com.android.inputmethod.latin.define.ProductionFlags; +import com.android.inputmethod.latin.settings.LocalSettingsConstants; import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.util.Locale; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * Locally gathers stats about the words user types and various other signals like auto-correction * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. */ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase { - /* package */ static final String NAME = UserHistoryDictionary.class.getSimpleName(); + static final String NAME = UserHistoryDictionary.class.getSimpleName(); // TODO: Make this constructor private - /* package */ UserHistoryDictionary(final Context context, final Locale locale) { - super(context, getDictName(NAME, locale, null /* dictFile */), locale, - Dictionary.TYPE_USER_HISTORY, null /* dictFile */); + UserHistoryDictionary(final Context context, final Locale locale) { + super(context, + getUserHistoryDictName( + NAME, + locale, + null /* dictFile */, + context), + locale, + Dictionary.TYPE_USER_HISTORY, + null /* dictFile */); + } + + /** + * @returns the name of the {@link UserHistoryDictionary}. + */ + @UsedForTesting + static String getUserHistoryDictName(final String name, final Locale locale, + @Nullable final File dictFile, final Context context) { + if (!ProductionFlags.ENABLE_PER_ACCOUNT_USER_HISTORY_DICTIONARY) { + return getDictName(name, locale, dictFile); + } + return getUserHistoryDictNamePerAccount(name, locale, dictFile, context); + } + + /** + * Uses the currently signed in account to determine the dictionary name. + */ + private static String getUserHistoryDictNamePerAccount(final String name, final Locale locale, + @Nullable final File dictFile, final Context context) { + if (dictFile != null) { + return dictFile.getName(); + } + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); + final String account = prefs.getString(LocalSettingsConstants.PREF_ACCOUNT_NAME, + null /* default */); + String dictName = name + "." + locale.toString(); + if (account != null) { + dictName += "." + account; + } + return dictName; } // Note: This method is called by {@link DictionaryFacilitator} using Java reflection. @@ -48,7 +91,14 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas @ExternallyReferenced public static UserHistoryDictionary getDictionary(final Context context, final Locale locale, final File dictFile, final String dictNamePrefix) { - return PersonalizationHelper.getUserHistoryDictionary(context, locale); + final String account; + if (ProductionFlags.ENABLE_PER_ACCOUNT_USER_HISTORY_DICTIONARY) { + account = PreferenceManager.getDefaultSharedPreferences(context) + .getString(LocalSettingsConstants.PREF_ACCOUNT_NAME, null /* default */); + } else { + account = null; + } + return PersonalizationHelper.getUserHistoryDictionary(context, locale, account); } /** diff --git a/java/src/com/android/inputmethod/latin/settings/CustomInputStylePreference.java b/java/src/com/android/inputmethod/latin/settings/CustomInputStylePreference.java index 01398f467..b749aa51a 100644 --- a/java/src/com/android/inputmethod/latin/settings/CustomInputStylePreference.java +++ b/java/src/com/android/inputmethod/latin/settings/CustomInputStylePreference.java @@ -346,8 +346,10 @@ final class CustomInputStylePreference extends DialogPreference super(context, android.R.layout.simple_spinner_item); setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + final String[] predefinedKeyboardLayoutSet = context.getResources().getStringArray( + R.array.predefined_layouts); // TODO: Should filter out already existing combinations of locale and layout. - for (final String layout : SubtypeLocaleUtils.getPredefinedKeyboardLayoutSet()) { + for (final String layout : predefinedKeyboardLayoutSet) { // This is a dummy subtype with NO_LANGUAGE, only for display. final InputMethodSubtype subtype = AdditionalSubtypeUtils.createDummyAdditionalSubtype( diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index 27a0f62ff..7991a2473 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -50,10 +50,10 @@ import com.android.inputmethod.latin.PunctuationSuggestions; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.SuggestedWords; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.common.LocaleUtils; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.settings.SettingsValues; import com.android.inputmethod.latin.utils.ResourceUtils; -import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; import com.android.inputmethod.latin.utils.ViewLayoutUtils; import java.util.ArrayList; @@ -570,8 +570,7 @@ final class SuggestionStripLayoutHelper { final boolean isRtlLanguage = (ViewCompat.getLayoutDirection(addToDictionaryStrip) == ViewCompat.LAYOUT_DIRECTION_RTL); final String arrow = isRtlLanguage ? RIGHTWARDS_ARROW : LEFTWARDS_ARROW; - final boolean isRtlSystem = SubtypeLocaleUtils.isRtlLanguage( - res.getConfiguration().locale); + final boolean isRtlSystem = LocaleUtils.isRtlLanguage(res.getConfiguration().locale); final CharSequence hint = res.getText(R.string.hint_add_to_dictionary); hintText = (isRtlLanguage == isRtlSystem) ? (arrow + hint) : (hint + arrow); hintWidth = width - wordWidth; diff --git a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java index b36168b6c..013f024c0 100644 --- a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java @@ -27,11 +27,9 @@ import android.util.Log; import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.latin.R; -import com.android.inputmethod.latin.RichInputMethodSubtype; import com.android.inputmethod.latin.common.LocaleUtils; import com.android.inputmethod.latin.common.StringUtils; -import java.util.Arrays; import java.util.HashMap; import java.util.Locale; @@ -56,7 +54,6 @@ public final class SubtypeLocaleUtils { private static volatile boolean sInitialized = false; private static final Object sInitializeLock = new Object(); private static Resources sResources; - private static String[] sPredefinedKeyboardLayoutSet; // Keyboard layout to its display name map. private static final HashMap<String, String> sKeyboardLayoutToDisplayNameMap = new HashMap<>(); // Keyboard layout to subtype name resource id map. @@ -103,7 +100,6 @@ public final class SubtypeLocaleUtils { sResources = res; final String[] predefinedLayoutSet = res.getStringArray(R.array.predefined_layouts); - sPredefinedKeyboardLayoutSet = predefinedLayoutSet; final String[] layoutDisplayNames = res.getStringArray( R.array.predefined_layout_display_names); for (int i = 0; i < predefinedLayoutSet.length; i++) { @@ -152,10 +148,6 @@ public final class SubtypeLocaleUtils { } } - public static String[] getPredefinedKeyboardLayoutSet() { - return sPredefinedKeyboardLayoutSet; - } - public static boolean isExceptionalLocale(final String localeString) { return sExceptionalLocaleToNameIdsMap.containsKey(localeString); } @@ -334,10 +326,6 @@ public final class SubtypeLocaleUtils { } @Nonnull - public static String getKeyboardLayoutSetName(@Nonnull final RichInputMethodSubtype subtype) { - return getKeyboardLayoutSetName(subtype.getRawSubtype()); - } - public static String getKeyboardLayoutSetName(final InputMethodSubtype subtype) { String keyboardLayoutSet = subtype.getExtraValueOf(KEYBOARD_LAYOUT_SET); if (keyboardLayoutSet == null) { @@ -357,22 +345,6 @@ public final class SubtypeLocaleUtils { return keyboardLayoutSet; } - // TODO: Get this information from the framework instead of maintaining here by ourselves. - // Sorted list of known Right-To-Left language codes. - private static final String[] SORTED_RTL_LANGUAGES = { - "ar", // Arabic - "fa", // Persian - "iw", // Hebrew - }; - static { - Arrays.sort(SORTED_RTL_LANGUAGES); - } - - public static boolean isRtlLanguage(final Locale locale) { - final String language = locale.getLanguage(); - return Arrays.binarySearch(SORTED_RTL_LANGUAGES, language) >= 0; - } - public static String getCombiningRulesExtraValue(final InputMethodSubtype subtype) { return subtype.getExtraValueOf(COMBINING_RULES); } diff --git a/native/dicttoolkit/NativeFileList.mk b/native/dicttoolkit/NativeFileList.mk index 925ec458b..d2c8c3a2c 100644 --- a/native/dicttoolkit/NativeFileList.mk +++ b/native/dicttoolkit/NativeFileList.mk @@ -22,8 +22,22 @@ LATIN_IME_DICT_TOOLKIT_SRC_FILES := \ help_executor.cpp \ info_executor.cpp \ makedict_executor.cpp) \ - utils/command_utils.cpp + $(addprefix offdevice_intermediate_dict/, \ + offdevice_intermediate_dict.cpp) \ + $(addprefix utils/, \ + arguments_parser.cpp \ + command_utils.cpp \ + utf8_utils.cpp) LATIN_IME_DICT_TOOLKIT_TEST_FILES := \ + $(addprefix command_executors/, \ + diff_executor_test.cpp \ + header_executor_test.cpp \ + info_executor_test.cpp \ + makedict_executor_test.cpp) \ dict_toolkit_defines_test.cpp \ - utils/command_utils_test.cpp + $(addprefix offdevice_intermediate_dict/, \ + offdevice_intermediate_dict_test.cpp) \ + $(addprefix utils/, \ + command_utils_test.cpp \ + utf8_utils_test.cpp) diff --git a/native/dicttoolkit/src/command_executors/diff_executor.cpp b/native/dicttoolkit/src/command_executors/diff_executor.cpp index 077a40090..bf6830686 100644 --- a/native/dicttoolkit/src/command_executors/diff_executor.cpp +++ b/native/dicttoolkit/src/command_executors/diff_executor.cpp @@ -30,8 +30,19 @@ const char *const DiffExecutor::COMMAND_NAME = "diff"; /* static */ void DiffExecutor::printUsage() { printf("*** %s\n", COMMAND_NAME); - printf("Usage: %s\n", COMMAND_NAME); - printf("Shows differences between two dictionaries.\n\n"); + getArgumentsParser().printUsage(COMMAND_NAME, "Shows differences between two dictionaries."); +} + +/* static */ const ArgumentsParser DiffExecutor::getArgumentsParser() { + std::unordered_map<std::string, OptionSpec> optionSpecs; + optionSpecs["p"] = OptionSpec::switchOption("(plumbing) produce output suitable for a script"); + + const std::vector<ArgumentSpec> argumentSpecs = { + ArgumentSpec::singleArgument("dict1", "dictionary file"), + ArgumentSpec::singleArgument("dict2", "dictionary file") + }; + + return ArgumentsParser(std::move(optionSpecs), std::move(argumentSpecs)); } } // namespace dicttoolkit diff --git a/native/dicttoolkit/src/command_executors/diff_executor.h b/native/dicttoolkit/src/command_executors/diff_executor.h index fc8dc0d8f..f92ae49d5 100644 --- a/native/dicttoolkit/src/command_executors/diff_executor.h +++ b/native/dicttoolkit/src/command_executors/diff_executor.h @@ -18,6 +18,7 @@ #define LATINIME_DICT_TOOLKIT_DIFF_EXECUTOR_H #include "dict_toolkit_defines.h" +#include "utils/arguments_parser.h" namespace latinime { namespace dicttoolkit { @@ -28,6 +29,7 @@ class DiffExecutor final { static int run(const int argc, char **argv); static void printUsage(); + static const ArgumentsParser getArgumentsParser(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DiffExecutor); diff --git a/native/dicttoolkit/src/command_executors/header_executor.cpp b/native/dicttoolkit/src/command_executors/header_executor.cpp index 068a62c31..b3d273b4e 100644 --- a/native/dicttoolkit/src/command_executors/header_executor.cpp +++ b/native/dicttoolkit/src/command_executors/header_executor.cpp @@ -30,9 +30,19 @@ const char *const HeaderExecutor::COMMAND_NAME = "header"; /* static */ void HeaderExecutor::printUsage() { printf("*** %s\n", COMMAND_NAME); - printf("Usage: %s\n", COMMAND_NAME); - printf("Prints the header contents of a dictionary file.\n\n"); + getArgumentsParser().printUsage(COMMAND_NAME, + "Prints the header contents of a dictionary file."); } +/* static */ const ArgumentsParser HeaderExecutor::getArgumentsParser() { + std::unordered_map<std::string, OptionSpec> optionSpecs; + optionSpecs["p"] = OptionSpec::switchOption("(plumbing) produce output suitable for a script"); + + const std::vector<ArgumentSpec> argumentSpecs = { + ArgumentSpec::singleArgument("dict", "prints the header contents of a dictionary file") + }; + + return ArgumentsParser(std::move(optionSpecs), std::move(argumentSpecs)); +} } // namespace dicttoolkit } // namespace latinime diff --git a/native/dicttoolkit/src/command_executors/header_executor.h b/native/dicttoolkit/src/command_executors/header_executor.h index 4cdeb1a99..44cc9cfc4 100644 --- a/native/dicttoolkit/src/command_executors/header_executor.h +++ b/native/dicttoolkit/src/command_executors/header_executor.h @@ -18,6 +18,7 @@ #define LATINIME_DICT_TOOLKIT_HEADER_EXECUTOR_H #include "dict_toolkit_defines.h" +#include "utils/arguments_parser.h" namespace latinime { namespace dicttoolkit { @@ -28,6 +29,7 @@ class HeaderExecutor final { static int run(const int argc, char **argv); static void printUsage(); + static const ArgumentsParser getArgumentsParser(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderExecutor); diff --git a/native/dicttoolkit/src/command_executors/info_executor.cpp b/native/dicttoolkit/src/command_executors/info_executor.cpp index c4d84cab3..351da4aff 100644 --- a/native/dicttoolkit/src/command_executors/info_executor.cpp +++ b/native/dicttoolkit/src/command_executors/info_executor.cpp @@ -17,6 +17,9 @@ #include "command_executors/info_executor.h" #include <cstdio> +#include <string> +#include <unordered_map> +#include <vector> namespace latinime { namespace dicttoolkit { @@ -30,8 +33,21 @@ const char *const InfoExecutor::COMMAND_NAME = "info"; /* static */ void InfoExecutor::printUsage() { printf("*** %s\n", COMMAND_NAME); - printf("Usage: %s\n", COMMAND_NAME); - printf("Prints various information about a dictionary file.\n\n"); + getArgumentsParser().printUsage(COMMAND_NAME, + "Prints various information about a dictionary file."); +} + +/* static */const ArgumentsParser InfoExecutor::getArgumentsParser() { + std::unordered_map<std::string, OptionSpec> optionSpecs; + optionSpecs["p"] = OptionSpec::switchOption("(plumbing) produce output suitable for a script"); + + const std::vector<ArgumentSpec> argumentSpecs = { + ArgumentSpec::singleArgument("dict", "dictionary file name"), + ArgumentSpec::variableLengthArguments("word", 0 /* minCount */, + ArgumentSpec::UNLIMITED_COUNT, "word to show information") + }; + + return ArgumentsParser(std::move(optionSpecs), std::move(argumentSpecs)); } } // namespace dicttoolkit diff --git a/native/dicttoolkit/src/command_executors/info_executor.h b/native/dicttoolkit/src/command_executors/info_executor.h index 4ffa74fb0..d4106d59f 100644 --- a/native/dicttoolkit/src/command_executors/info_executor.h +++ b/native/dicttoolkit/src/command_executors/info_executor.h @@ -18,6 +18,7 @@ #define LATINIME_DICT_TOOLKIT_INFO_EXECUTOR_H #include "dict_toolkit_defines.h" +#include "utils/arguments_parser.h" namespace latinime { namespace dicttoolkit { @@ -28,6 +29,7 @@ class InfoExecutor final { static int run(const int argc, char **argv); static void printUsage(); + static const ArgumentsParser getArgumentsParser(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(InfoExecutor); diff --git a/native/dicttoolkit/src/command_executors/makedict_executor.cpp b/native/dicttoolkit/src/command_executors/makedict_executor.cpp index ea62e3c37..8a84e8069 100644 --- a/native/dicttoolkit/src/command_executors/makedict_executor.cpp +++ b/native/dicttoolkit/src/command_executors/makedict_executor.cpp @@ -30,10 +30,25 @@ const char *const MakedictExecutor::COMMAND_NAME = "makedict"; /* static */ void MakedictExecutor::printUsage() { printf("*** %s\n", COMMAND_NAME); - printf("Usage: %s\n", COMMAND_NAME); - printf("Converts a source dictionary file to one or several outputs.\n" + getArgumentsParser().printUsage(COMMAND_NAME, + "Converts a source dictionary file to one or several outputs.\n" "Source can be a binary dictionary file or a combined format file.\n" - "Binary version 2 (Jelly Bean), 4, and combined format outputs are supported.\n\n"); + "Binary version 2 (Jelly Bean), 4, and combined format outputs are supported."); +} + +/* static */const ArgumentsParser MakedictExecutor::getArgumentsParser() { + std::unordered_map<std::string, OptionSpec> optionSpecs; + optionSpecs["o"] = OptionSpec::keyValueOption("format", "2", + "output format version: 2/4/combined"); + optionSpecs["t"] = OptionSpec::keyValueOption("mode", "off", + "code point table switch: on/off/auto"); + + const std::vector<ArgumentSpec> argumentSpecs = { + ArgumentSpec::singleArgument("src_dict", "source dictionary file"), + ArgumentSpec::singleArgument("dest_dict", "output dictionary file") + }; + + return ArgumentsParser(std::move(optionSpecs), std::move(argumentSpecs)); } } // namespace dicttoolkit diff --git a/native/dicttoolkit/src/command_executors/makedict_executor.h b/native/dicttoolkit/src/command_executors/makedict_executor.h index ae1309f60..c3de977a3 100644 --- a/native/dicttoolkit/src/command_executors/makedict_executor.h +++ b/native/dicttoolkit/src/command_executors/makedict_executor.h @@ -18,6 +18,7 @@ #define LATINIME_DICT_TOOLKIT_MAKEDICT_EXECUTOR_H #include "dict_toolkit_defines.h" +#include "utils/arguments_parser.h" namespace latinime { namespace dicttoolkit { @@ -28,6 +29,7 @@ class MakedictExecutor final { static int run(const int argc, char **argv); static void printUsage(); + static const ArgumentsParser getArgumentsParser(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(MakedictExecutor); diff --git a/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict.cpp b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict.cpp new file mode 100644 index 000000000..af28131cf --- /dev/null +++ b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "offdevice_intermediate_dict/offdevice_intermediate_dict.h" + +#include "offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node.h" + +namespace latinime { +namespace dicttoolkit { + +bool OffdeviceIntermediateDict::addWord(const WordProperty &wordProperty) { + const CodePointArrayView codePoints = wordProperty.getCodePoints(); + if (codePoints.empty() || codePoints.size() > MAX_WORD_LENGTH) { + return false; + } + return addWordInner(codePoints, wordProperty, mRootPtNodeArray); +} + +bool OffdeviceIntermediateDict::addWordInner(const CodePointArrayView codePoints, + const WordProperty &wordProperty, OffdeviceIntermediateDictPtNodeArray &ptNodeArray) { + auto ptNodeList = ptNodeArray.getMutablePtNodeList(); + auto ptNodeIt = ptNodeList->begin(); + for (; ptNodeIt != ptNodeList->end(); ++ptNodeIt) { + const auto &ptNode = *ptNodeIt; + const CodePointArrayView ptNodeCodePoints = ptNode->getPtNodeCodePoints(); + if (codePoints[0] < ptNodeCodePoints[0]) { + continue; + } + if (codePoints[0] > ptNodeCodePoints[0]) { + break; + } + size_t i = 1; + for (; i < codePoints.size(); ++i) { + if (i >= ptNodeCodePoints.size()) { + // Add new child. + return addWordInner(codePoints.skip(i), wordProperty, + ptNode->getChildrenPtNodeArray()); + } + if (codePoints[i] != ptNodeCodePoints[i]) { + break; + } + } + if (codePoints.size() == i && codePoints.size() == ptNodeCodePoints.size()) { + // All code points matched. + if (ptNode->getWordProperty()) { + // Adding the same word multiple times is not supported. + return false; + } + ptNodeList->insert(ptNodeIt, + std::make_shared<OffdeviceIntermediateDictPtNode>(wordProperty, *ptNode)); + ptNodeList->erase(ptNodeIt); + return true; + } + // The (i+1)-th elements are different. + // Create and Add new parent ptNode for the common part. + auto newPtNode = codePoints.size() == i + ? std::make_shared<OffdeviceIntermediateDictPtNode>(codePoints, wordProperty) + : std::make_shared<OffdeviceIntermediateDictPtNode>(codePoints.limit(i)); + ptNodeList->insert(ptNodeIt, newPtNode); + OffdeviceIntermediateDictPtNodeArray &childrenPtNodeArray = + newPtNode->getChildrenPtNodeArray(); + // Add new child for the existing ptNode. + childrenPtNodeArray.getMutablePtNodeList()->push_back( + std::make_shared<OffdeviceIntermediateDictPtNode>( + ptNodeCodePoints.skip(i), *ptNode)); + ptNodeList->erase(ptNodeIt); + if (codePoints.size() != i) { + // Add a child for the new word. + return addWordInner(codePoints.skip(i), wordProperty, childrenPtNodeArray); + } + return true; + } + ptNodeList->insert(ptNodeIt, + std::make_shared<OffdeviceIntermediateDictPtNode>(codePoints, wordProperty)); + return true; +} + +const WordProperty *OffdeviceIntermediateDict::getWordProperty( + const CodePointArrayView codePoints) const { + const OffdeviceIntermediateDictPtNodeArray *ptNodeArray = &mRootPtNodeArray; + for (size_t i = 0; i < codePoints.size();) { + bool foundNext = false; + for (const auto ptNode : ptNodeArray->getPtNodeList()) { + const CodePointArrayView ptNodeCodePoints = ptNode->getPtNodeCodePoints(); + if (codePoints[i] < ptNodeCodePoints[0]) { + continue; + } + if (codePoints[i] > ptNodeCodePoints[0] + || codePoints.size() < ptNodeCodePoints.size()) { + return nullptr; + } + for (size_t j = 1; j < ptNodeCodePoints.size(); ++j) { + if (codePoints[i + j] != ptNodeCodePoints[j]) { + return nullptr; + } + } + i += ptNodeCodePoints.size(); + if (i == codePoints.size()) { + return ptNode->getWordProperty(); + } + ptNodeArray = &ptNode->getChildrenPtNodeArray(); + foundNext = true; + break; + } + if (!foundNext) { + break; + } + } + return nullptr; +} + +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict.h b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict.h new file mode 100644 index 000000000..13d26ba91 --- /dev/null +++ b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_H +#define LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_H + +#include "dict_toolkit_defines.h" +#include "offdevice_intermediate_dict/offdevice_intermediate_dict_header.h" +#include "offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node_array.h" +#include "suggest/core/dictionary/property/word_property.h" +#include "utils/int_array_view.h" + +namespace latinime { +namespace dicttoolkit { + +/** + * On memory patricia trie to represent a dictionary. + */ +class OffdeviceIntermediateDict final { + public: + OffdeviceIntermediateDict(const OffdeviceIntermediateDictHeader &header) + : mHeader(header), mRootPtNodeArray() {} + + bool addWord(const WordProperty &wordProperty); + // The returned value will be invalid after modifying the dictionary. e.g. calling addWord(). + const WordProperty *getWordProperty(const CodePointArrayView codePoints) const; + const OffdeviceIntermediateDictHeader &getHeader() const { return mHeader; } + + private: + DISALLOW_ASSIGNMENT_OPERATOR(OffdeviceIntermediateDict); + + const OffdeviceIntermediateDictHeader mHeader; + OffdeviceIntermediateDictPtNodeArray mRootPtNodeArray; + + bool addWordInner(const CodePointArrayView codePoints, const WordProperty &wordProperty, + OffdeviceIntermediateDictPtNodeArray &ptNodeArray); +}; + +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_H diff --git a/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_header.h b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_header.h new file mode 100644 index 000000000..440627a79 --- /dev/null +++ b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_header.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_HEADER_H +#define LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_HEADER_H + +#include <map> +#include <vector> + +#include "dict_toolkit_defines.h" + +namespace latinime { +namespace dicttoolkit { + +class OffdeviceIntermediateDictHeader final { + public: + using AttributeMap = std::map<std::vector<int>, std::vector<int>>; + + OffdeviceIntermediateDictHeader(const AttributeMap &attributesMap) + : mAttributeMap(attributesMap) {} + + private: + DISALLOW_DEFAULT_CONSTRUCTOR(OffdeviceIntermediateDictHeader); + DISALLOW_ASSIGNMENT_OPERATOR(OffdeviceIntermediateDictHeader); + + const AttributeMap mAttributeMap; +}; + +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_HEADER_H diff --git a/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node.h b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node.h new file mode 100644 index 000000000..721ccd778 --- /dev/null +++ b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_PT_NODE_H +#define LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_PT_NODE_H + +#include <memory> + +#include "dict_toolkit_defines.h" +#include "offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node_array.h" +#include "suggest/core/dictionary/property/word_property.h" +#include "utils/int_array_view.h" + +namespace latinime { +namespace dicttoolkit { + +class OffdeviceIntermediateDictPtNode final { + public: + // Non-terminal + OffdeviceIntermediateDictPtNode(const CodePointArrayView ptNodeCodePoints) + : mPtNodeCodePoints(ptNodeCodePoints.toVector()), mChildrenPtNodeArray(), + mWortProperty(nullptr) {} + + // Terminal + OffdeviceIntermediateDictPtNode(const CodePointArrayView ptNodeCodePoints, + const WordProperty &wordProperty) + : mPtNodeCodePoints(ptNodeCodePoints.toVector()), mChildrenPtNodeArray(), + mWortProperty(new WordProperty(wordProperty)) {} + + // Replacing PtNodeCodePoints. + OffdeviceIntermediateDictPtNode(const CodePointArrayView ptNodeCodePoints, + const OffdeviceIntermediateDictPtNode &ptNode) + : mPtNodeCodePoints(ptNodeCodePoints.toVector()), + mChildrenPtNodeArray(ptNode.mChildrenPtNodeArray), + mWortProperty(new WordProperty(*ptNode.mWortProperty)) {} + + // Replacing WordProperty. + OffdeviceIntermediateDictPtNode(const WordProperty &wordProperty, + const OffdeviceIntermediateDictPtNode &ptNode) + : mPtNodeCodePoints(ptNode.mPtNodeCodePoints), + mChildrenPtNodeArray(ptNode.mChildrenPtNodeArray), + mWortProperty(new WordProperty(wordProperty)) {} + + const WordProperty *getWordProperty() const { + return mWortProperty.get(); + } + + const CodePointArrayView getPtNodeCodePoints() const { + return CodePointArrayView(mPtNodeCodePoints); + } + + OffdeviceIntermediateDictPtNodeArray &getChildrenPtNodeArray() { + return mChildrenPtNodeArray; + } + + private: + DISALLOW_COPY_AND_ASSIGN(OffdeviceIntermediateDictPtNode); + + const std::vector<int> mPtNodeCodePoints; + OffdeviceIntermediateDictPtNodeArray mChildrenPtNodeArray; + const std::unique_ptr<WordProperty> mWortProperty; +}; + +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_PT_NODE_H diff --git a/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node_array.h b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node_array.h new file mode 100644 index 000000000..f87456ce0 --- /dev/null +++ b/native/dicttoolkit/src/offdevice_intermediate_dict/offdevice_intermediate_dict_pt_node_array.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_PT_NODE_ARRAY_H +#define LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_PT_NODE_ARRAY_H + +#include <list> +#include <memory> + +#include "dict_toolkit_defines.h" + +namespace latinime { +namespace dicttoolkit { + +class OffdeviceIntermediateDictPtNode; + +class OffdeviceIntermediateDictPtNodeArray final { + public: + const std::list<std::shared_ptr<OffdeviceIntermediateDictPtNode>> &getPtNodeList() const { + return mPtNodes; + } + + std::list<std::shared_ptr<OffdeviceIntermediateDictPtNode>> *getMutablePtNodeList() { + return &mPtNodes; + } + + private: + DISALLOW_ASSIGNMENT_OPERATOR(OffdeviceIntermediateDictPtNodeArray); + + std::list<std::shared_ptr<OffdeviceIntermediateDictPtNode>> mPtNodes; +}; + +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_OFFDEVICE_INTERMEDIATE_DICT_PT_NODE_ARRAY_H diff --git a/native/dicttoolkit/src/utils/arguments_and_options.h b/native/dicttoolkit/src/utils/arguments_and_options.h new file mode 100644 index 000000000..d8f5985e5 --- /dev/null +++ b/native/dicttoolkit/src/utils/arguments_and_options.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_ARGUMENTS_AND_OPTIONS_H +#define LATINIME_DICT_TOOLKIT_ARGUMENTS_AND_OPTIONS_H + +#include <string> +#include <unordered_map> +#include <vector> + +#include "dict_toolkit_defines.h" + +namespace latinime { +namespace dicttoolkit { + +class ArgumentsAndOptions { + public: + ArgumentsAndOptions() : mIsValid(false), mOptions(), mArguments() {} + + ArgumentsAndOptions(std::unordered_map<std::string, std::string> &&options, + std::unordered_map<std::string, std::vector<std::string>> &&arguments) + : mIsValid(true), mOptions(std::move(options)), mArguments(std::move(arguments)) {} + + bool isValid() const { + return mIsValid; + } + + bool hasOption(const std::string &optionName) const { + return mOptions.find(optionName) != mOptions.end(); + } + + private: + DISALLOW_ASSIGNMENT_OPERATOR(ArgumentsAndOptions); + + const bool mIsValid; + const std::unordered_map<std::string, std::string> mOptions; + const std::unordered_map<std::string, std::vector<std::string>> mArguments; +}; +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_ARGUMENTS_AND_OPTIONS_H diff --git a/native/dicttoolkit/src/utils/arguments_parser.cpp b/native/dicttoolkit/src/utils/arguments_parser.cpp new file mode 100644 index 000000000..039dae35b --- /dev/null +++ b/native/dicttoolkit/src/utils/arguments_parser.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/arguments_parser.h" + +namespace latinime { +namespace dicttoolkit { + +const int ArgumentSpec::UNLIMITED_COUNT = -1; + +bool ArgumentsParser::validateSpecs() const { + for (size_t i = 0; i < mArgumentSpecs.size() ; ++i) { + if (mArgumentSpecs[i].getMinCount() != mArgumentSpecs[i].getMaxCount() + && i != mArgumentSpecs.size() - 1) { + AKLOGE("Variable length argument must be at the end."); + return false; + } + } + return true; +} + +void ArgumentsParser::printUsage(const std::string &commandName, + const std::string &description) const { + printf("Usage: %s", commandName.c_str()); + for (const auto &option : mOptionSpecs) { + const std::string &optionName = option.first; + const OptionSpec &spec = option.second; + printf(" [-%s", optionName.c_str()); + if (spec.takeValue()) { + printf(" <%s>", spec.getValueName().c_str()); + } + printf("]"); + } + for (const auto &argSpec : mArgumentSpecs) { + if (argSpec.getMinCount() == 0 && argSpec.getMaxCount() == 1) { + printf(" [<%s>]", argSpec.getName().c_str()); + } else if (argSpec.getMinCount() == 1 && argSpec.getMaxCount() == 1) { + printf(" <%s>", argSpec.getName().c_str()); + } else if (argSpec.getMinCount() == 0) { + printf(" [<%s>...]", argSpec.getName().c_str()); + } else if (argSpec.getMinCount() == 1) { + printf(" <%s>...", argSpec.getName().c_str()); + } + } + printf("\n%s\n\n", description.c_str()); + for (const auto &option : mOptionSpecs) { + const std::string &optionName = option.first; + const OptionSpec &spec = option.second; + printf(" -%s", optionName.c_str()); + if (spec.takeValue()) { + printf(" <%s>", spec.getValueName().c_str()); + } + printf("\t\t\t%s", spec.getDescription().c_str()); + if (spec.takeValue() && !spec.getDefaultValue().empty()) { + printf("\tdefault: %s", spec.getDefaultValue().c_str()); + } + printf("\n"); + } + for (const auto &argSpec : mArgumentSpecs) { + printf(" <%s>\t\t\t%s\n", argSpec.getName().c_str(), argSpec.getDescription().c_str()); + } + printf("\n\n"); +} + +const ArgumentsAndOptions ArgumentsParser::parseArguments(const int argc, char **argv) const { + // TODO: Implement + return ArgumentsAndOptions(); +} + +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/src/utils/arguments_parser.h b/native/dicttoolkit/src/utils/arguments_parser.h new file mode 100644 index 000000000..be2dd8749 --- /dev/null +++ b/native/dicttoolkit/src/utils/arguments_parser.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_ARGUMENTS_PARSER_H +#define LATINIME_DICT_TOOLKIT_ARGUMENTS_PARSER_H + +#include <string> +#include <unordered_map> +#include <vector> + +#include "dict_toolkit_defines.h" +#include "utils/arguments_and_options.h" + +namespace latinime { +namespace dicttoolkit { + +class OptionSpec { + public: + // Default constructor and assignment operator is enabled to be used with std::unordered_map. + OptionSpec() = default; + OptionSpec &operator=(const OptionSpec &) = default; + + static OptionSpec keyValueOption(const std::string &valueName, const std::string &defaultValue, + const std::string &description) { + return OptionSpec(true /* takeValue */, valueName, defaultValue, description); + } + + static OptionSpec switchOption(const std::string &description) { + return OptionSpec(false /* takeValue */, "" /* valueName */, "" /* defaultValue */, + description); + } + + bool takeValue() const { return mTakeValue; } + const std::string &getValueName() const { return mValueName; } + const std::string &getDefaultValue() const { return mDefaultValue; } + const std::string &getDescription() const { return mDescription; } + + private: + OptionSpec(const bool takeValue, const std::string &valueName, const std::string &defaultValue, + const std::string &description) + : mTakeValue(takeValue), mValueName(valueName), mDefaultValue(defaultValue), + mDescription(description) {} + + // Whether the option have to be used with a value or just a switch. + // e.g. 'f' in "command -f /path/to/file" is mTakeValue == true. + // 'f' in "command -f -t" is mTakeValue == false. + bool mTakeValue; + // Name of the value used to show usage. + std::string mValueName; + std::string mDefaultValue; + std::string mDescription; +}; + +class ArgumentSpec { + public: + static const int UNLIMITED_COUNT; + + static ArgumentSpec singleArgument(const std::string &name, const std::string &description) { + return ArgumentSpec(name, 1 /* minCount */, 1 /* maxCount */, description); + } + + static ArgumentSpec variableLengthArguments(const std::string &name, const int minCount, + const int maxCount, const std::string &description) { + return ArgumentSpec(name, minCount, maxCount, description); + } + + const std::string &getName() const { return mName; } + int getMinCount() const { return mMinCount; } + int getMaxCount() const { return mMaxCount; } + const std::string &getDescription() const { return mDescription; } + + private: + DISALLOW_DEFAULT_CONSTRUCTOR(ArgumentSpec); + + ArgumentSpec(const std::string &name, const int minCount, const int maxCount, + const std::string &description) + : mName(name), mMinCount(minCount), mMaxCount(maxCount), mDescription(description) {} + + const std::string mName; + const int mMinCount; + const int mMaxCount; + const std::string mDescription; +}; + +class ArgumentsParser { + public: + ArgumentsParser(std::unordered_map<std::string, OptionSpec> &&optionSpecs, + std::vector<ArgumentSpec> &&argumentSpecs) + : mOptionSpecs(std::move(optionSpecs)), mArgumentSpecs(std::move(argumentSpecs)) {} + + const ArgumentsAndOptions parseArguments(const int argc, char **argv) const; + bool validateSpecs() const; + void printUsage(const std::string &commandName, const std::string &description) const; + + private: + DISALLOW_DEFAULT_CONSTRUCTOR(ArgumentsParser); + DISALLOW_ASSIGNMENT_OPERATOR(ArgumentsParser); + + const std::unordered_map<std::string, OptionSpec> mOptionSpecs; + const std::vector<ArgumentSpec> mArgumentSpecs; +}; + +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_ARGUMENTS_PARSER_H diff --git a/native/dicttoolkit/src/utils/utf8_utils.cpp b/native/dicttoolkit/src/utils/utf8_utils.cpp new file mode 100644 index 000000000..0f349f512 --- /dev/null +++ b/native/dicttoolkit/src/utils/utf8_utils.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/utf8_utils.h" + +#include "utils/char_utils.h" + +namespace latinime { +namespace dicttoolkit { + +const size_t Utf8Utils::MAX_SEQUENCE_SIZE_FOR_A_CODE_POINT = 4; +const uint8_t Utf8Utils::FIRST_BYTE_MARKER_MASKS[] = {0, 0x80, 0xE0, 0xF0, 0xF8}; +const uint8_t Utf8Utils::FIRST_BYTE_MARKERS[] = {0, 0x00, 0xC0, 0xE0, 0xF0}; +const uint8_t Utf8Utils::FIRST_BYTE_CODE_POINT_BITS_MASKS[] = {0, 0x7F, 0x1F, 0x0F, 0x03}; +const int Utf8Utils::MAX_ENCODED_CODE_POINT_VALUES[] = {-1, 0x7F, 0x7FF, 0xFFFF, 0x10FFFF}; + +const uint8_t Utf8Utils::TRAILING_BYTE_CODE_POINT_BITS_MASK = 0x3F; +const uint8_t Utf8Utils::TRAILING_BYTE_MARKER = 0x80; +const size_t Utf8Utils::CODE_POINT_BIT_COUNT_IN_TRAILING_BYTE = 6; + +/* static */ std::vector<int> Utf8Utils::getCodePoints(const std::string &utf8Str) { + std::vector<int> codePoints; + int remainingByteCountForCurrentCodePoint = 0; + int currentCodePointSequenceSize = 0; + int codePoint = 0; + for (const char c : utf8Str) { + if (remainingByteCountForCurrentCodePoint == 0) { + currentCodePointSequenceSize = getSequenceSizeByCheckingFirstByte(c); + if (currentCodePointSequenceSize <= 0) { + AKLOGE("%x is an invalid utf8 first byte value.", c); + return std::vector<int>(); + } + remainingByteCountForCurrentCodePoint = currentCodePointSequenceSize; + codePoint = maskFirstByte(c, remainingByteCountForCurrentCodePoint); + } else { + codePoint <<= CODE_POINT_BIT_COUNT_IN_TRAILING_BYTE; + codePoint += maskTrailingByte(c); + } + remainingByteCountForCurrentCodePoint--; + if (remainingByteCountForCurrentCodePoint == 0) { + if (codePoint <= MAX_ENCODED_CODE_POINT_VALUES[currentCodePointSequenceSize - 1]) { + AKLOGE("%d bytes encode for codePoint(%x) is a redundant UTF-8 sequence.", + currentCodePointSequenceSize, codePoint); + return std::vector<int>(); + } + codePoints.push_back(codePoint); + } + } + return codePoints; +} + +/* static */ int Utf8Utils::getSequenceSizeByCheckingFirstByte(const uint8_t firstByte) { + for (size_t i = 1; i <= MAX_SEQUENCE_SIZE_FOR_A_CODE_POINT; ++i) { + if ((firstByte & FIRST_BYTE_MARKER_MASKS[i]) == FIRST_BYTE_MARKERS[i]) { + return i; + } + } + // Not a valid utf8 char first byte. + return -1; +} + +/* static */ AK_FORCE_INLINE int Utf8Utils::maskFirstByte(const uint8_t firstByte, + const int sequenceSize) { + return firstByte & FIRST_BYTE_CODE_POINT_BITS_MASKS[sequenceSize]; +} + +/* static */ AK_FORCE_INLINE int Utf8Utils::maskTrailingByte(const uint8_t secondOrLaterByte) { + return secondOrLaterByte & TRAILING_BYTE_CODE_POINT_BITS_MASK; +} + +/* static */ std::string Utf8Utils::getUtf8String(const CodePointArrayView codePoints) { + std::string utf8String; + for (const int codePoint : codePoints) { + const int sequenceSize = getSequenceSizeToEncodeCodePoint(codePoint); + if (sequenceSize <= 0) { + AKLOGE("Cannot encode code point (%d).", codePoint); + return std::string(); + } + const int trailingByteCount = sequenceSize - 1; + // Output first byte. + const int value = codePoint >> (trailingByteCount * CODE_POINT_BIT_COUNT_IN_TRAILING_BYTE); + utf8String.push_back(static_cast<char>(value | FIRST_BYTE_MARKERS[sequenceSize])); + // Output second and later bytes. + for (int i = 1; i < sequenceSize; ++i) { + const int shiftAmount = (trailingByteCount - i) * CODE_POINT_BIT_COUNT_IN_TRAILING_BYTE; + const int value = (codePoint >> shiftAmount) & TRAILING_BYTE_CODE_POINT_BITS_MASK; + utf8String.push_back(static_cast<char>(value | TRAILING_BYTE_MARKER)); + } + } + return utf8String; +} + +/* static */ int Utf8Utils::getSequenceSizeToEncodeCodePoint(const int codePoint) { + if (codePoint < 0) { + return -1; + } + for (size_t i = 1; i <= MAX_SEQUENCE_SIZE_FOR_A_CODE_POINT; ++i) { + if (codePoint <= MAX_ENCODED_CODE_POINT_VALUES[i]) { + return i; + } + } + return -1; +} + +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/src/utils/utf8_utils.h b/native/dicttoolkit/src/utils/utf8_utils.h new file mode 100644 index 000000000..35818e56c --- /dev/null +++ b/native/dicttoolkit/src/utils/utf8_utils.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_TOOLKIT_UTF8_UTILS_H +#define LATINIME_DICT_TOOLKIT_UTF8_UTILS_H + +#include <cstdint> +#include <string> +#include <vector> + +#include "dict_toolkit_defines.h" +#include "utils/int_array_view.h" + +namespace latinime { +namespace dicttoolkit { + +class Utf8Utils { +public: + static std::vector<int> getCodePoints(const std::string &utf8Str); + static std::string getUtf8String(const CodePointArrayView codePoints); + +private: + DISALLOW_IMPLICIT_CONSTRUCTORS(Utf8Utils); + + // Values indexed by sequence size. + static const size_t MAX_SEQUENCE_SIZE_FOR_A_CODE_POINT; + static const uint8_t FIRST_BYTE_MARKER_MASKS[]; + static const uint8_t FIRST_BYTE_MARKERS[]; + static const uint8_t FIRST_BYTE_CODE_POINT_BITS_MASKS[]; + static const int MAX_ENCODED_CODE_POINT_VALUES[]; + + static const uint8_t TRAILING_BYTE_CODE_POINT_BITS_MASK; + static const uint8_t TRAILING_BYTE_MARKER; + static const size_t CODE_POINT_BIT_COUNT_IN_TRAILING_BYTE; + + static int getSequenceSizeByCheckingFirstByte(const uint8_t firstByte); + static int maskFirstByte(const uint8_t firstByte, const int encodeSize); + static int maskTrailingByte(const uint8_t secondOrLaterByte); + static int getSequenceSizeToEncodeCodePoint(const int codePoint); +}; +} // namespace dicttoolkit +} // namespace latinime +#endif // LATINIME_DICT_TOOLKIT_UTF8_UTILS_H diff --git a/native/dicttoolkit/tests/command_executors/diff_executor_test.cpp b/native/dicttoolkit/tests/command_executors/diff_executor_test.cpp new file mode 100644 index 000000000..444141427 --- /dev/null +++ b/native/dicttoolkit/tests/command_executors/diff_executor_test.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "command_executors/diff_executor.h" + +#include <gtest/gtest.h> + +namespace latinime { +namespace dicttoolkit { +namespace { + +TEST(DiffExecutorTests, TestArguemntSpecs) { + EXPECT_TRUE(DiffExecutor::getArgumentsParser().validateSpecs()); +} + +} // namespace +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/tests/command_executors/header_executor_test.cpp b/native/dicttoolkit/tests/command_executors/header_executor_test.cpp new file mode 100644 index 000000000..a94150b01 --- /dev/null +++ b/native/dicttoolkit/tests/command_executors/header_executor_test.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "command_executors/header_executor.h" + +#include <gtest/gtest.h> + +namespace latinime { +namespace dicttoolkit { +namespace { + +TEST(HeaderExecutorTests, TestArguemntSpecs) { + EXPECT_TRUE(HeaderExecutor::getArgumentsParser().validateSpecs()); +} + +} // namespace +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/tests/command_executors/info_executor_test.cpp b/native/dicttoolkit/tests/command_executors/info_executor_test.cpp new file mode 100644 index 000000000..debe8c601 --- /dev/null +++ b/native/dicttoolkit/tests/command_executors/info_executor_test.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "command_executors/info_executor.h" + +#include <gtest/gtest.h> + +namespace latinime { +namespace dicttoolkit { +namespace { + +TEST(InfoExecutorTests, TestArguemntSpecs) { + EXPECT_TRUE(InfoExecutor::getArgumentsParser().validateSpecs()); +} + +} // namespace +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/tests/command_executors/makedict_executor_test.cpp b/native/dicttoolkit/tests/command_executors/makedict_executor_test.cpp new file mode 100644 index 000000000..44eb3dc1b --- /dev/null +++ b/native/dicttoolkit/tests/command_executors/makedict_executor_test.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "command_executors/makedict_executor.h" + +#include <gtest/gtest.h> + +namespace latinime { +namespace dicttoolkit { +namespace { + +TEST(MakedictExecutorTests, TestArguemntSpecs) { + EXPECT_TRUE(MakedictExecutor::getArgumentsParser().validateSpecs()); +} + +} // namespace +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/tests/offdevice_intermediate_dict/offdevice_intermediate_dict_test.cpp b/native/dicttoolkit/tests/offdevice_intermediate_dict/offdevice_intermediate_dict_test.cpp new file mode 100644 index 000000000..f2e24ab5f --- /dev/null +++ b/native/dicttoolkit/tests/offdevice_intermediate_dict/offdevice_intermediate_dict_test.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "offdevice_intermediate_dict/offdevice_intermediate_dict.h" + +#include <gtest/gtest.h> + +#include <vector> + +#include "suggest/core/dictionary/property/word_property.h" +#include "utils/int_array_view.h" + +namespace latinime { +namespace dicttoolkit { +namespace { + +const std::vector<int> getCodePointVector(const char *str) { + std::vector<int> codePoints; + while (*str) { + codePoints.push_back(*str); + ++str; + } + return codePoints; +} + +const WordProperty getDummpWordProperty(const std::vector<int> &&codePoints) { + return WordProperty(std::move(codePoints), UnigramProperty(), std::vector<NgramProperty>()); +} + +TEST(OffdeviceIntermediateDictTest, TestAddWordProperties) { + OffdeviceIntermediateDict dict = OffdeviceIntermediateDict( + OffdeviceIntermediateDictHeader(OffdeviceIntermediateDictHeader::AttributeMap())); + EXPECT_EQ(nullptr, dict.getWordProperty(CodePointArrayView())); + + const WordProperty wordProperty0 = getDummpWordProperty(getCodePointVector("abcd")); + EXPECT_TRUE(dict.addWord(wordProperty0)); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty0.getCodePoints())); + + const WordProperty wordProperty1 = getDummpWordProperty(getCodePointVector("efgh")); + EXPECT_TRUE(dict.addWord(wordProperty1)); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty1.getCodePoints())); + + const WordProperty wordProperty2 = getDummpWordProperty(getCodePointVector("ab")); + EXPECT_TRUE(dict.addWord(wordProperty2)); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty2.getCodePoints())); + + const WordProperty wordProperty3 = getDummpWordProperty(getCodePointVector("abcdefg")); + EXPECT_TRUE(dict.addWord(wordProperty3)); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty3.getCodePoints())); + + const WordProperty wordProperty4 = getDummpWordProperty(getCodePointVector("efef")); + EXPECT_TRUE(dict.addWord(wordProperty4)); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty4.getCodePoints())); + + const WordProperty wordProperty5 = getDummpWordProperty(getCodePointVector("ef")); + EXPECT_TRUE(dict.addWord(wordProperty5)); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty5.getCodePoints())); + + const WordProperty wordProperty6 = getDummpWordProperty(getCodePointVector("abcd")); + EXPECT_FALSE(dict.addWord(wordProperty6)) << "Adding the same word multiple times should fail."; + + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty0.getCodePoints())); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty1.getCodePoints())); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty2.getCodePoints())); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty3.getCodePoints())); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty4.getCodePoints())); + EXPECT_NE(nullptr, dict.getWordProperty(wordProperty5.getCodePoints())); +} + +} // namespace +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/dicttoolkit/tests/utils/utf8_utils_test.cpp b/native/dicttoolkit/tests/utils/utf8_utils_test.cpp new file mode 100644 index 000000000..9c59a8b05 --- /dev/null +++ b/native/dicttoolkit/tests/utils/utf8_utils_test.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/utf8_utils.h" + +#include <gtest/gtest.h> + +#include <vector> + +#include "utils/int_array_view.h" + +namespace latinime { +namespace dicttoolkit { +namespace { + +TEST(Utf8UtilsTests, TestGetCodePoints) { + { + const std::vector<int> codePoints = Utf8Utils::getCodePoints(""); + EXPECT_EQ(0u, codePoints.size()); + } + { + const std::vector<int> codePoints = Utf8Utils::getCodePoints("test"); + EXPECT_EQ(4u, codePoints.size()); + EXPECT_EQ('t', codePoints[0]); + EXPECT_EQ('e', codePoints[1]); + EXPECT_EQ('s', codePoints[2]); + EXPECT_EQ('t', codePoints[3]); + } + { + const std::vector<int> codePoints = Utf8Utils::getCodePoints(u8"\u3042a\u03C2\u0410"); + EXPECT_EQ(4u, codePoints.size()); + EXPECT_EQ(0x3042, codePoints[0]); // HIRAGANA LETTER A + EXPECT_EQ('a', codePoints[1]); + EXPECT_EQ(0x03C2, codePoints[2]); // CYRILLIC CAPITAL LETTER A + EXPECT_EQ(0x0410, codePoints[3]); // GREEK SMALL LETTER FINAL SIGMA + } + { + const std::vector<int> codePoints = Utf8Utils::getCodePoints(u8"\U0001F36A?\U0001F752"); + EXPECT_EQ(3u, codePoints.size()); + EXPECT_EQ(0x1F36A, codePoints[0]); // COOKIE + EXPECT_EQ('?', codePoints[1]); + EXPECT_EQ(0x1F752, codePoints[2]); // ALCHEMICAL SYMBOL FOR STARRED TRIDENT + } + + // Redundant UTF-8 sequences must be rejected. + EXPECT_TRUE(Utf8Utils::getCodePoints("\xC0\xAF").empty()); + EXPECT_TRUE(Utf8Utils::getCodePoints("\xE0\x80\xAF").empty()); + EXPECT_TRUE(Utf8Utils::getCodePoints("\xF0\x80\x80\xAF").empty()); +} + +TEST(Utf8UtilsTests, TestGetUtf8String) { + { + const std::vector<int> codePoints = {'t', 'e', 's', 't'}; + EXPECT_EQ("test", Utf8Utils::getUtf8String(CodePointArrayView(codePoints))); + } + { + const std::vector<int> codePoints = { + 0x00E0 /* LATIN SMALL LETTER A WITH GRAVE */, + 0x03C2 /* GREEK SMALL LETTER FINAL SIGMA */, + 0x0430 /* CYRILLIC SMALL LETTER A */, + 0x3042 /* HIRAGANA LETTER A */, + 0x1F36A /* COOKIE */, + 0x1F752 /* ALCHEMICAL SYMBOL FOR STARRED TRIDENT */ + }; + EXPECT_EQ(u8"\u00E0\u03C2\u0430\u3042\U0001F36A\U0001F752", + Utf8Utils::getUtf8String(CodePointArrayView(codePoints))); + } +} + +} // namespace +} // namespace dicttoolkit +} // namespace latinime diff --git a/native/jni/src/utils/int_array_view.h b/native/jni/src/utils/int_array_view.h index 408373176..e0f671056 100644 --- a/native/jni/src/utils/int_array_view.h +++ b/native/jni/src/utils/int_array_view.h @@ -133,6 +133,29 @@ class IntArrayView { return std::vector<int>(begin(), end()); } + std::vector<IntArrayView> split(const int separator, const int limit = S_INT_MAX) const { + if (limit <= 0) { + return std::vector<IntArrayView>(); + } + std::vector<IntArrayView> result; + if (limit == 1) { + result.emplace_back(mPtr, mSize); + return result; + } + size_t startIndex = 0; + for (size_t i = 0; i < mSize; ++i) { + if (mPtr[i] == separator) { + result.emplace_back(mPtr + startIndex, i - startIndex); + startIndex = i + 1; + if (result.size() >= static_cast<size_t>(limit - 1)) { + break; + } + } + } + result.emplace_back(mPtr + startIndex, mSize - startIndex); + return result; + } + private: DISALLOW_ASSIGNMENT_OPERATOR(IntArrayView); diff --git a/native/jni/tests/utils/int_array_view_test.cpp b/native/jni/tests/utils/int_array_view_test.cpp index 4757a416b..2fce633f5 100644 --- a/native/jni/tests/utils/int_array_view_test.cpp +++ b/native/jni/tests/utils/int_array_view_test.cpp @@ -151,5 +151,52 @@ TEST(IntArrayViewTest, TestToVector) { EXPECT_EQ(std::vector<int>(), CodePointArrayView().toVector()); } +TEST(IntArrayViewTest, TestSplit) { + EXPECT_TRUE(IntArrayView().split(0, 0).empty()); + { + const auto intArrayViews = IntArrayView().split(0, 1); + EXPECT_EQ(1u, intArrayViews.size()); + EXPECT_TRUE(intArrayViews[0].empty()); + } + { + const auto intArrayViews = IntArrayView().split(0, 100); + EXPECT_EQ(1u, intArrayViews.size()); + EXPECT_TRUE(intArrayViews[0].empty()); + } + + const std::vector<int> intVector = {1, 2, 3, 3, 2, 3}; + const IntArrayView intArrayView(intVector); + { + const auto intArrayViews = intArrayView.split(2); + EXPECT_EQ(3u, intArrayViews.size()); + EXPECT_EQ(std::vector<int>({1}), intArrayViews[0].toVector()); + EXPECT_EQ(std::vector<int>({3, 3}), intArrayViews[1].toVector()); + EXPECT_EQ(std::vector<int>({3}), intArrayViews[2].toVector()); + } + { + const auto intArrayViews = intArrayView.split(2, 2); + EXPECT_EQ(2u, intArrayViews.size()); + EXPECT_EQ(std::vector<int>({1}), intArrayViews[0].toVector()); + EXPECT_EQ(std::vector<int>({3, 3, 2, 3}), intArrayViews[1].toVector()); + } + { + const auto intArrayViews = intArrayView.split(2, 1); + EXPECT_EQ(1u, intArrayViews.size()); + EXPECT_EQ(intVector, intArrayViews[0].toVector()); + } + { + const auto intArrayViews = intArrayView.split(2, 0); + EXPECT_EQ(0u, intArrayViews.size()); + } + { + const auto intArrayViews = intArrayView.split(3); + EXPECT_EQ(4u, intArrayViews.size()); + EXPECT_EQ(std::vector<int>({1, 2}), intArrayViews[0].toVector()); + EXPECT_EQ(std::vector<int>(), intArrayViews[1].toVector()); + EXPECT_EQ(std::vector<int>({2}), intArrayViews[2].toVector()); + EXPECT_EQ(std::vector<int>(), intArrayViews[3].toVector()); + } +} + } // namespace } // namespace latinime diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 6c60fdc0c..d833b9736 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -26,7 +26,6 @@ import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.common.CodePointUtils; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.DictionaryOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 813a71239..a84df28c9 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -16,6 +16,8 @@ package com.android.inputmethod.latin.personalization; +import android.content.SharedPreferences; +import android.preference.PreferenceManager; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; @@ -24,6 +26,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.NgramContext.WordInfo; import com.android.inputmethod.latin.common.FileUtils; +import com.android.inputmethod.latin.settings.LocalSettingsConstants; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.DistracterFilter; @@ -36,6 +39,8 @@ import java.util.Locale; import java.util.Random; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + /** * Unit tests for UserHistoryDictionary */ @@ -44,6 +49,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName(); private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000; private static final String TEST_LOCALE_PREFIX = "test_"; + private static final String TEST_ACCOUNT = "account@example.com"; private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", @@ -52,15 +58,18 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private int mCurrentTime = 0; + private SharedPreferences mPrefs; + private String mLastKnownAccount = null; + private void removeAllTestDictFiles() { final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX); - final String dictName = ExpandableBinaryDictionary.getDictName( - UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */); + final String dictName = UserHistoryDictionary.getUserHistoryDictName( + UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); final File dictFile = ExpandableBinaryDictionary.getDictFile( mContext, dictName, null /* dictFile */); final FilenameFilter filenameFilter = new FilenameFilter() { @Override - public boolean accept(File dir, String filename) { + public boolean accept(final File dir, final String filename) { return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX); } }; @@ -99,6 +108,12 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { @Override protected void setUp() throws Exception { super.setUp(); + + mPrefs = PreferenceManager.getDefaultSharedPreferences(getContext()); + // Keep track of the current account so that we restore it when the test finishes. + mLastKnownAccount = mPrefs.getString(LocalSettingsConstants.PREF_ACCOUNT_NAME, null); + updateAccountName(TEST_ACCOUNT); + resetCurrentTimeForTestMode(); removeAllTestDictFiles(); } @@ -107,6 +122,10 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { protected void tearDown() throws Exception { removeAllTestDictFiles(); stopTestModeInNativeCode(); + + // Restore the account that was present before running the test. + updateAccountName(mLastKnownAccount); + super.tearDown(); } @@ -115,6 +134,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); } + private void updateAccountName(@Nullable final String accountName) { + if (accountName == null) { + mPrefs.edit().remove(LocalSettingsConstants.PREF_ACCOUNT_NAME).apply(); + } else { + mPrefs.edit().putString(LocalSettingsConstants.PREF_ACCOUNT_NAME, accountName).apply(); + } + } + private void forcePassingShortTime() { // 3 days. final int timeToElapse = (int)TimeUnit.DAYS.toSeconds(3); @@ -142,7 +169,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { */ private static String generateWord(final int value) { final int lengthOfChars = CHARACTERS.length; - StringBuilder builder = new StringBuilder(); + final StringBuilder builder = new StringBuilder(); long lvalue = Math.abs((long)value); while (lvalue > 0) { builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]); @@ -162,7 +189,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private static void addToDict(final UserHistoryDictionary dict, final List<String> words, final int timestamp) { NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; - for (String word : words) { + for (final String word : words) { UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp, DistracterFilter.EMPTY_DISTRACTER_FILTER); ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); @@ -204,12 +231,12 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Log.d(TAG, "This test can be used for profiling."); Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); final Locale dummyLocale = getDummyLocale("random_words"); - final String dictName = ExpandableBinaryDictionary.getDictName( - UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */); + final String dictName = UserHistoryDictionary.getUserHistoryDictName( + UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); final File dictFile = ExpandableBinaryDictionary.getDictFile( mContext, dictName, null /* dictFile */); final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( - getContext(), dummyLocale); + getContext(), dummyLocale, TEST_ACCOUNT); final int numberOfWords = 1000; final Random random = new Random(123456); @@ -232,12 +259,12 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { final Locale dummyLocale = getDummyLocale("switching_languages" + i); - final String dictName = ExpandableBinaryDictionary.getDictName( - UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */); + final String dictName = UserHistoryDictionary.getUserHistoryDictName( + UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); dictFiles[i] = ExpandableBinaryDictionary.getDictFile( mContext, dictName, null /* dictFile */); dicts[i] = PersonalizationHelper.getUserHistoryDictionary(getContext(), - dummyLocale); + dummyLocale, TEST_ACCOUNT); clearHistory(dicts[i]); } @@ -262,14 +289,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { public void testAddManyWords() { final Locale dummyLocale = getDummyLocale("many_random_words"); - final String dictName = ExpandableBinaryDictionary.getDictName( - UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */); + final String dictName = UserHistoryDictionary.getUserHistoryDictName( + UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); final File dictFile = ExpandableBinaryDictionary.getDictFile( mContext, dictName, null /* dictFile */); final int numberOfWords = 10000; final Random random = new Random(123456); final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( - getContext(), dummyLocale); + getContext(), dummyLocale, TEST_ACCOUNT); clearHistory(dict); try { addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */); @@ -281,7 +308,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { public void testDecaying() { final Locale dummyLocale = getDummyLocale("decaying"); final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( - getContext(), dummyLocale); + getContext(), dummyLocale, TEST_ACCOUNT); final int numberOfWords = 5000; final Random random = new Random(123456); resetCurrentTimeForTestMode(); @@ -309,4 +336,9 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { assertFalse(dict.isInDictionary(word)); } } -} + + public void testRandomWords_NullAccount() { + updateAccountName(null); + testRandomWords(); + } +}
\ No newline at end of file diff --git a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java index 54f478f5a..03dcdfc78 100644 --- a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java @@ -434,8 +434,8 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { // locale layout | display name // ------ -------------- - ---------------------- // sr south_slavic F Српски - // sr_ZZ serbian_qwertz F српски (латиница) - // sr_ZZ qwerty T српски (QWERTY) + // sr_ZZ serbian_qwertz F Српски (латиница) + // sr_ZZ qwerty T Српски (QWERTY) public void testSerbianLatinSubtypesInSerbianSystemLocale() { final RunInLocale<Void> tests = new RunInLocale<Void>() { @@ -445,12 +445,10 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(SR)); // These are preliminary subtypes and may not exist. if (SR_LATN != null) { - // TODO: Uncommented because of the current translation of these strings - // in Seriban are described in Latin script. -// assertEquals("sr_ZZ", "српски (латиница)", -// SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(SR_LATN)); -// assertEquals("sr_ZZ", "српски (QWERTY)", -// SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(SR_LATN_QWERTY)); + assertEquals("sr_ZZ", "Српски (латиница)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(SR_LATN)); + assertEquals("sr_ZZ", "Српски (QWERTY)", + SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(SR_LATN_QWERTY)); } return null; } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index 3ec28f313..84c3956f7 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -186,9 +186,7 @@ public final class BinaryDictOffdeviceUtils { throw new UnsupportedFormatException("Header too large"); } final byte[] headerBuffer = new byte[totalHeaderSize - tmpBuffer.length]; - if (headerBuffer.length != input.read(headerBuffer)) { - throw new UnsupportedFormatException("File shorter than specified in the header"); - } + readStreamExhaustively(input, headerBuffer); final HashMap<String, String> attributes = BinaryDictDecoderUtils.decodeHeaderAttributes(headerBuffer); return new DictionaryHeader(totalHeaderSize, new DictionaryOptions(attributes), @@ -196,6 +194,20 @@ public final class BinaryDictOffdeviceUtils { } } + private static void readStreamExhaustively(final InputStream inputStream, + final byte[] outBuffer) throws IOException, UnsupportedFormatException { + int readBytes = 0; + int readBytesLastCycle = -1; + while (readBytes != outBuffer.length) { + readBytesLastCycle = inputStream.read(outBuffer, readBytes, + outBuffer.length - readBytes); + if (readBytesLastCycle == -1) + throw new UnsupportedFormatException("File shorter than specified in the header" + + " (expected " + outBuffer.length + ", read " + readBytes + ")"); + readBytes += readBytesLastCycle; + } + } + public static void copy(final InputStream input, final OutputStream output) throws IOException { final byte[] buffer = new byte[COPY_BUFFER_SIZE]; for (int readBytes = input.read(buffer); readBytes >= 0; readBytes = input.read(buffer)) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Header.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Header.java index 51efdec33..ba96c0aeb 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Header.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Header.java @@ -62,6 +62,7 @@ public class Header extends Dicttool.Command { System.out.println("Dictionary : " + dictFile.getAbsolutePath()); System.out.println("Size : " + dictFile.length() + " bytes"); System.out.println("Format : Binary dictionary format"); + System.out.println("Format version : " + header.mFormatOptions.mVersion); System.out.println("Packaging : " + spec.describeChain()); System.out.println("Header attributes :"); System.out.print(header.mDictionaryOptions.toString(2 /* indentCount */, plumbing)); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java index 4e5c0742e..3efa10a80 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java @@ -16,6 +16,8 @@ package com.android.inputmethod.latin.dicttool; +import com.android.inputmethod.latin.makedict.DictionaryHeader; + import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; @@ -77,16 +79,16 @@ public class Package { if (mArgs.length != 2) { throw new RuntimeException("Too many/too few arguments for command " + COMMAND); } - final BinaryDictOffdeviceUtils.DecoderChainSpec<File> decodedSpec = - BinaryDictOffdeviceUtils.getRawDictionaryOrNull(new File(mArgs[0])); + final BinaryDictOffdeviceUtils.DecoderChainSpec<DictionaryHeader> decodedSpec = + BinaryDictOffdeviceUtils.decodeDictionaryForProcess(new File(mArgs[0]), + new BinaryDictOffdeviceUtils.HeaderReaderProcessor()); if (null == decodedSpec) { System.out.println(mArgs[0] + " does not seem to be a dictionary"); return; } System.out.println("Packaging : " + decodedSpec.describeChain()); - System.out.println("Uncompressed size : " + decodedSpec.mResult.length()); try ( - final InputStream input = getFileInputStream(decodedSpec.mResult); + final InputStream input = decodedSpec.getStream(new File(mArgs[0])); final OutputStream output = new BufferedOutputStream( getFileOutputStreamOrStdOut(mArgs[1])) ) { |