diff options
33 files changed, 373 insertions, 1728 deletions
diff --git a/java/res/xml/rowkeys_armenian_phonetic2.xml b/java/res/xml/rowkeys_armenian_phonetic2.xml index 5dcabc301..3764d0dbb 100644 --- a/java/res/xml/rowkeys_armenian_phonetic2.xml +++ b/java/res/xml/rowkeys_armenian_phonetic2.xml @@ -34,6 +34,7 @@ <Key latin:keyLabel="ե" latin:moreKeys="և" + latin:keyHintLabel="և" latin:keyLabelFlags="fontNormal" /> <!-- U+057C: "ռ" ARMENIAN SMALL LETTER RA --> <Key diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index fb84f1d73..53b448515 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -187,7 +187,7 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { final MainKeyboardView keyboardView = mKeyboardView; final Keyboard oldKeyboard = keyboardView.getKeyboard(); keyboardView.setKeyboard(keyboard); - mCurrentInputView.setKeyboardGeometry(keyboard.mTopPadding); + mCurrentInputView.setKeyboardTopPadding(keyboard.mTopPadding); keyboardView.setKeyPreviewPopupEnabled( Settings.readKeyPreviewPopupEnabled(mPrefs, mResources), Settings.readKeyPreviewPopupDismissDelay(mPrefs, mResources)); diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java index 1400e05c8..4b6e12f8e 100644 --- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java @@ -932,11 +932,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack @Override public void onShowMoreKeysPanel(final MoreKeysPanel panel) { locatePreviewPlacerView(); - // TODO: Remove this check - if (panel.isShowingInParent()) { - panel.dismissMoreKeysPanel(); - } - mPreviewPlacerView.addView(panel.getContainerView()); + panel.showInParent(mPreviewPlacerView); mMoreKeysPanel = panel; dimEntireKeyboard(true /* dimmed */); } @@ -954,7 +950,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack public void onDismissMoreKeysPanel(final MoreKeysPanel panel) { dimEntireKeyboard(false /* dimmed */); if (isShowingMoreKeysPanel()) { - mPreviewPlacerView.removeView(mMoreKeysPanel.getContainerView()); + mMoreKeysPanel.removeFromParent(); mMoreKeysPanel = null; } } diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java index a7c468538..5b13e9a41 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java @@ -21,6 +21,7 @@ import android.content.res.Resources; import android.util.AttributeSet; import android.view.MotionEvent; import android.view.View; +import android.view.ViewGroup; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; @@ -216,12 +217,26 @@ public class MoreKeysKeyboardView extends KeyboardView implements MoreKeysPanel return true; } - @Override - public View getContainerView() { + private View getContainerView() { return (View)getParent(); } @Override + public void showInParent(final ViewGroup parentView) { + removeFromParent(); + parentView.addView(getContainerView()); + } + + @Override + public void removeFromParent() { + final View containerView = getContainerView(); + final ViewGroup currentParent = (ViewGroup)containerView.getParent(); + if (currentParent != null) { + currentParent.removeView(containerView); + } + } + + @Override public boolean isShowingInParent() { return (getContainerView().getParent() != null); } diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysPanel.java b/java/src/com/android/inputmethod/keyboard/MoreKeysPanel.java index 886c6286f..4a33e6536 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysPanel.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysPanel.java @@ -17,6 +17,7 @@ package com.android.inputmethod.keyboard; import android.view.View; +import android.view.ViewGroup; public interface MoreKeysPanel { public interface Controller { @@ -119,9 +120,16 @@ public interface MoreKeysPanel { public int translateY(int y); /** - * Return the view containing the more keys panel. + * Show this {@link MoreKeysPanel} in the parent view. + * + * @param parentView the {@link ViewGroup} that hosts this {@link MoreKeysPanel}. + */ + public void showInParent(ViewGroup parentView); + + /** + * Remove this {@link MoreKeysPanel} from the parent view. */ - public View getContainerView(); + public void removeFromParent(); /** * Return whether the panel is currently being shown. diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java index a2528e046..e23d5a773 100644 --- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java +++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java @@ -167,10 +167,10 @@ public final class PointerTracker implements PointerTrackerQueue.Element { private static boolean sInGesture = false; private static long sGestureFirstDownTime; private static TimeRecorder sTimeRecorder; - private static final InputPointers sAggregratedPointers = new InputPointers( + private static final InputPointers sAggregatedPointers = new InputPointers( GestureStroke.DEFAULT_CAPACITY); - private static int sLastRecognitionPointSize = 0; // synchronized using sAggregratedPointers - private static long sLastRecognitionTime = 0; // synchronized using sAggregratedPointers + private static int sLastRecognitionPointSize = 0; // synchronized using sAggregatedPointers + private static long sLastRecognitionTime = 0; // synchronized using sAggregatedPointers static final class BogusMoveEventDetector { // Move these thresholds to resource. @@ -737,8 +737,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { Log.d(TAG, String.format("[%d] onStartBatchInput", mPointerId)); } sInGesture = true; - synchronized (sAggregratedPointers) { - sAggregratedPointers.reset(); + synchronized (sAggregatedPointers) { + sAggregatedPointers.reset(); sLastRecognitionPointSize = 0; sLastRecognitionTime = 0; sListener.onStartBatchInput(); @@ -769,10 +769,10 @@ public final class PointerTracker implements PointerTrackerQueue.Element { } private void updateBatchInput(final long eventTime) { - synchronized (sAggregratedPointers) { + synchronized (sAggregatedPointers) { final GestureStroke stroke = mGestureStrokeWithPreviewPoints; - stroke.appendIncrementalBatchPoints(sAggregratedPointers); - final int size = sAggregratedPointers.getPointerSize(); + stroke.appendIncrementalBatchPoints(sAggregatedPointers); + final int size = sAggregatedPointers.getPointerSize(); if (size > sLastRecognitionPointSize && stroke.hasRecognitionTimePast(eventTime, sLastRecognitionTime)) { if (DEBUG_LISTENER) { @@ -780,18 +780,18 @@ public final class PointerTracker implements PointerTrackerQueue.Element { size)); } sTimerProxy.startUpdateBatchInputTimer(this); - sListener.onUpdateBatchInput(sAggregratedPointers); + sListener.onUpdateBatchInput(sAggregatedPointers); // The listener may change the size of the pointers (when auto-committing // for example), so we need to get the size from the pointers again. - sLastRecognitionPointSize = sAggregratedPointers.getPointerSize(); + sLastRecognitionPointSize = sAggregatedPointers.getPointerSize(); sLastRecognitionTime = eventTime; } } } private void mayEndBatchInput(final long eventTime) { - synchronized (sAggregratedPointers) { - mGestureStrokeWithPreviewPoints.appendAllBatchPoints(sAggregratedPointers); + synchronized (sAggregatedPointers) { + mGestureStrokeWithPreviewPoints.appendAllBatchPoints(sAggregatedPointers); if (getActivePointerTrackerCount() == 1) { sInGesture = false; sTimeRecorder.onEndBatchInput(eventTime); @@ -799,9 +799,9 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (!mIsTrackingForActionDisabled) { if (DEBUG_LISTENER) { Log.d(TAG, String.format("[%d] onEndBatchInput : batchPoints=%d", - mPointerId, sAggregratedPointers.getPointerSize())); + mPointerId, sAggregatedPointers.getPointerSize())); } - sListener.onEndBatchInput(sAggregratedPointers); + sListener.onEndBatchInput(sAggregatedPointers); } } } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index f708c2cdd..9f33fcc0a 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -1811,7 +1811,7 @@ public final class KeyboardTextsSet { // U+055A: "՚" ARMENIAN APOSTROPHE // U+055B: "՛" ARMENIAN EMPHASIS MARK // U+055F: "՟" ARMENIAN ABBREVIATION MARK - /* 59 */ "!fixedColumnOrder!8,!,?,\\,,.,\u058A,\u055C,\u055D,\u055E,:,;,@,\u0559,\u055A,\u055B,\u055F", + /* 59 */ "!fixedColumnOrder!8,!,?,\u0559,\u055A,.,\u055C,\\,,\u055E,:,;,\u055F,\u00AB,\u00BB,\u058A,\u055D,\u055B", /* 60~ */ null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, diff --git a/java/src/com/android/inputmethod/latin/InputView.java b/java/src/com/android/inputmethod/latin/InputView.java index 81ccf83d8..76b0912f6 100644 --- a/java/src/com/android/inputmethod/latin/InputView.java +++ b/java/src/com/android/inputmethod/latin/InputView.java @@ -23,87 +23,203 @@ import android.view.MotionEvent; import android.view.View; import android.widget.LinearLayout; -public final class InputView extends LinearLayout { - private View mSuggestionStripView; - private View mKeyboardView; - private int mKeyboardTopPadding; +import com.android.inputmethod.keyboard.MainKeyboardView; +import com.android.inputmethod.latin.suggestions.MoreSuggestionsView; +import com.android.inputmethod.latin.suggestions.SuggestionStripView; - private boolean mIsForwardingEvent; +public final class InputView extends LinearLayout { private final Rect mInputViewRect = new Rect(); - private final Rect mEventForwardingRect = new Rect(); - private final Rect mEventReceivingRect = new Rect(); + private KeyboardTopPaddingForwarder mKeyboardTopPaddingForwarder; + private MoreSuggestionsViewCanceler mMoreSuggestionsViewCanceler; public InputView(final Context context, final AttributeSet attrs) { super(context, attrs, 0); } - public void setKeyboardGeometry(final int keyboardTopPadding) { - mKeyboardTopPadding = keyboardTopPadding; - } - @Override protected void onFinishInflate() { - mSuggestionStripView = findViewById(R.id.suggestion_strip_view); - mKeyboardView = findViewById(R.id.keyboard_view); + final SuggestionStripView suggestionStripView = + (SuggestionStripView)findViewById(R.id.suggestion_strip_view); + final MainKeyboardView mainKeyboardView = + (MainKeyboardView)findViewById(R.id.keyboard_view); + mKeyboardTopPaddingForwarder = new KeyboardTopPaddingForwarder( + mainKeyboardView, suggestionStripView); + mMoreSuggestionsViewCanceler = new MoreSuggestionsViewCanceler( + mainKeyboardView, suggestionStripView); + } + + public void setKeyboardTopPadding(final int keyboardTopPadding) { + mKeyboardTopPaddingForwarder.setKeyboardTopPadding(keyboardTopPadding); } @Override public boolean dispatchTouchEvent(final MotionEvent me) { - if (mSuggestionStripView.getVisibility() != VISIBLE - || mKeyboardView.getVisibility() != VISIBLE) { - return super.dispatchTouchEvent(me); - } - - // The touch events that hit the top padding of keyboard should be forwarded to - // {@link SuggestionStripView}. final Rect rect = mInputViewRect; - this.getGlobalVisibleRect(rect); + getGlobalVisibleRect(rect); final int x = (int)me.getX() + rect.left; final int y = (int)me.getY() + rect.top; - final Rect forwardingRect = mEventForwardingRect; - mKeyboardView.getGlobalVisibleRect(forwardingRect); - if (!mIsForwardingEvent && !forwardingRect.contains(x, y)) { - return super.dispatchTouchEvent(me); + // The touch events that hit the top padding of keyboard should be + // forwarded to {@link SuggestionStripView}. + if (mKeyboardTopPaddingForwarder.dispatchTouchEvent(x, y, me)) { + return true; + } + // To cancel {@link MoreSuggestionsView}, we should intercept a touch event to + // {@link MainKeyboardView} and dismiss the {@link MoreSuggestionsView}. + if (mMoreSuggestionsViewCanceler.dispatchTouchEvent(x, y, me)) { + return true; + } + return super.dispatchTouchEvent(me); + } + + /** + * This class forwards series of {@link MotionEvent}s from <code>Forwarder</code> view to + * <code>Receiver</code> view. + * + * @param <Sender> a {@link View} that may send a {@link MotionEvent} to <Receiver>. + * @param <Receiver> a {@link View} that receives forwarded {@link MotionEvent} from + * <Forwarder>. + */ + private static abstract class MotionEventForwarder<Sender extends View, Receiver extends View> { + protected final Sender mSenderView; + protected final Receiver mReceiverView; + + private boolean mIsForwardingEvent; + protected final Rect mEventSendingRect = new Rect(); + protected final Rect mEventReceivingRect = new Rect(); + + public MotionEventForwarder(final Sender senderView, final Receiver receiverView) { + mSenderView = senderView; + mReceiverView = receiverView; } - final int forwardingLimitY = forwardingRect.top + mKeyboardTopPadding; - boolean sendToTarget = false; + // Return true if a touch event of global coordinate x, y needs to be forwarded. + protected abstract boolean needsToForward(final int x, final int y); - switch (me.getAction()) { - case MotionEvent.ACTION_DOWN: - if (y < forwardingLimitY) { - // This down event and further move and up events should be forwarded to the target. - mIsForwardingEvent = true; - sendToTarget = true; + // Translate global x-coordinate to <code>Receiver</code> local coordinate. + protected int translateX(final int x) { + return x - mEventReceivingRect.left; + } + + // Translate global y-coordinate to <code>Receiver</code> local coordinate. + protected int translateY(final int y) { + return y - mEventReceivingRect.top; + } + + // Callback when a {@link MotionEvent} is forwarded. + protected void onForwardingEvent(final MotionEvent me) {} + + // Dispatches a {@link MotioneEvent} to <code>Receiver</code> if needed and returns true. + // Otherwise returns false. + public boolean dispatchTouchEvent(final int x, final int y, final MotionEvent me) { + // Forwards a {link MotionEvent} only if both <code>Sender</code> and + // <code>Receiver</code> are visible. + if (mSenderView.getVisibility() != View.VISIBLE || + mReceiverView.getVisibility() != View.VISIBLE) { + return false; + } + final Rect sendingRect = mEventSendingRect; + mSenderView.getGlobalVisibleRect(sendingRect); + if (!mIsForwardingEvent && !sendingRect.contains(x, y)) { + return false; } - break; - case MotionEvent.ACTION_MOVE: - sendToTarget = mIsForwardingEvent; - break; - case MotionEvent.ACTION_UP: - case MotionEvent.ACTION_CANCEL: - sendToTarget = mIsForwardingEvent; - mIsForwardingEvent = false; - break; + + boolean shouldForwardToReceiver = false; + + switch (me.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + // If the down event happens in the forwarding area, successive {@link MotionEvent}s + // should be forwarded. + if (needsToForward(x, y)) { + mIsForwardingEvent = true; + shouldForwardToReceiver = true; + } + break; + case MotionEvent.ACTION_MOVE: + shouldForwardToReceiver = mIsForwardingEvent; + break; + case MotionEvent.ACTION_UP: + case MotionEvent.ACTION_CANCEL: + shouldForwardToReceiver = mIsForwardingEvent; + mIsForwardingEvent = false; + break; + } + + if (!shouldForwardToReceiver) { + return false; + } + + final Rect receivingRect = mEventReceivingRect; + mReceiverView.getGlobalVisibleRect(receivingRect); + // Translate global coordinates to <code>Receiver</code> local coordinates. + me.setLocation(translateX(x), translateY(y)); + mReceiverView.dispatchTouchEvent(me); + onForwardingEvent(me); + return true; } + } + + /** + * This class forwards {@link MotionEvent}s happened in the top padding of + * {@link MainKeyboardView} to {@link SuggestionStripView}. + */ + private static class KeyboardTopPaddingForwarder + extends MotionEventForwarder<MainKeyboardView, SuggestionStripView> { + private int mKeyboardTopPadding; - if (!sendToTarget) { - return super.dispatchTouchEvent(me); + public KeyboardTopPaddingForwarder(final MainKeyboardView mainKeyboardView, + final SuggestionStripView suggestionStripView) { + super(mainKeyboardView, suggestionStripView); } - final Rect receivingRect = mEventReceivingRect; - mSuggestionStripView.getGlobalVisibleRect(receivingRect); - final int translatedX = x - receivingRect.left; - final int translatedY; - if (y < forwardingLimitY) { - // The forwarded event should have coordinates that are inside of the target. - translatedY = Math.min(y - receivingRect.top, receivingRect.height() - 1); - } else { - translatedY = y - receivingRect.top; + public void setKeyboardTopPadding(final int keyboardTopPadding) { + mKeyboardTopPadding = keyboardTopPadding; + } + + private boolean isInKeyboardTopPadding(final int y) { + return y < mEventSendingRect.top + mKeyboardTopPadding; + } + + @Override + protected boolean needsToForward(final int x, final int y) { + return isInKeyboardTopPadding(y); + } + + @Override + protected int translateY(final int y) { + final int translatedY = super.translateY(y); + if (isInKeyboardTopPadding(y)) { + // The forwarded event should have coordinates that are inside of + // the target. + return Math.min(translatedY, mEventReceivingRect.height() - 1); + } + return translatedY; + } + } + + /** + * This class forwards {@link MotionEvent}s happened in the {@link MainKeyboardView} to + * {@link SuggestionStripView} when the {@link MoreSuggestionsView} is showing. + * {@link SuggestionStripView} dismisses {@link MoreSuggestionsView} when it receives those + * events. + */ + private static class MoreSuggestionsViewCanceler + extends MotionEventForwarder<MainKeyboardView, SuggestionStripView> { + public MoreSuggestionsViewCanceler(final MainKeyboardView mainKeyboardView, + final SuggestionStripView suggestionStripView) { + super(mainKeyboardView, suggestionStripView); + } + + @Override + protected boolean needsToForward(final int x, final int y) { + return mReceiverView.isShowingMoreSuggestionPanel() && mEventSendingRect.contains(x, y); + } + + @Override + protected void onForwardingEvent(final MotionEvent me) { + if (me.getActionMasked() == MotionEvent.ACTION_DOWN) { + mReceiverView.dismissMoreSuggestionsPanel(); + } } - me.setLocation(translatedX, translatedY); - mSuggestionStripView.dispatchTouchEvent(me); - return true; } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 770ea49e6..67b570277 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1414,7 +1414,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.latinIME_maybeDoubleSpacePeriod(textToInsert, false /* isBatchMode */); } - mWordComposer.doubleSpacePeriod(); + mWordComposer.discardPreviousWordForSuggestion(); mKeyboardSwitcher.updateShiftState(); return true; } @@ -2510,29 +2510,23 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final SettingsValues currentSettings = mSettings.getCurrent(); final int[] additionalFeaturesOptions = currentSettings.mAdditionalFeaturesSettingValues; - final String previousWord; - if (mWordComposer.isComposingWord() || mWordComposer.isBatchMode()) { - previousWord = mWordComposer.getPreviousWord(); - } else { - // Not composing: this is for prediction. - // TODO: read the previous word earlier for prediction, like we are doing for - // normal suggestions. - previousWord = getNthPreviousWordForSuggestion(currentSettings, 1 /* nthPreviousWord*/); - } if (DEBUG) { - // TODO: this is for checking consistency with older versions. Remove this when - // we are confident this is stable. - // We're checking the previous word in the text field against the memorized previous - // word. If we are composing a word we should have the second word before the cursor - // memorized, otherwise we should have the first. - final String rereadPrevWord = getNthPreviousWordForSuggestion(currentSettings, - mWordComposer.isComposingWord() ? 2 : 1); - if (!TextUtils.equals(previousWord, rereadPrevWord)) { - throw new RuntimeException("Unexpected previous word: " - + previousWord + " <> " + rereadPrevWord); - } - } - suggest.getSuggestedWords(mWordComposer, mWordComposer.getPreviousWord(), + if (mWordComposer.isComposingWord() || mWordComposer.isBatchMode()) { + final String previousWord = mWordComposer.getPreviousWordForSuggestion(); + // TODO: this is for checking consistency with older versions. Remove this when + // we are confident this is stable. + // We're checking the previous word in the text field against the memorized previous + // word. If we are composing a word we should have the second word before the cursor + // memorized, otherwise we should have the first. + final String rereadPrevWord = getNthPreviousWordForSuggestion(currentSettings, + mWordComposer.isComposingWord() ? 2 : 1); + if (!TextUtils.equals(previousWord, rereadPrevWord)) { + throw new RuntimeException("Unexpected previous word: " + + previousWord + " <> " + rereadPrevWord); + } + } + } + suggest.getSuggestedWords(mWordComposer, mWordComposer.getPreviousWordForSuggestion(), keyboard.getProximityInfo(), currentSettings.mBlockPotentiallyOffensive, currentSettings.mCorrectionEnabled, additionalFeaturesOptions, sessionId, sequenceNumber, callback); @@ -2775,6 +2769,19 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // strings. mLastComposedWord = mWordComposer.commitWord(commitType, chosenWord, separatorString, prevWord); + final boolean shouldDiscardPreviousWordForSuggestion; + if (0 == StringUtils.codePointCount(separatorString)) { + // Separator is 0-length. Discard the word only if the current language has spaces. + shouldDiscardPreviousWordForSuggestion = + mSettings.getCurrent().mCurrentLanguageHasSpaces; + } else { + // Otherwise, we discard if the separator contains any non-whitespace. + shouldDiscardPreviousWordForSuggestion = + !StringUtils.containsOnlyWhitespace(separatorString); + } + if (shouldDiscardPreviousWordForSuggestion) { + mWordComposer.discardPreviousWordForSuggestion(); + } } private void setPunctuationSuggestions() { diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index 5ecfc67b2..7da97e57a 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -50,8 +50,9 @@ public final class WordComposer { private final StringBuilder mTypedWord; // The previous word (before the composing word). Used as context for suggestions. May be null // after resetting and before starting a new composing word, or when there is no context like - // at the start of text for example. - private String mPreviousWord; + // at the start of text for example. It can also be set to null externally when the user + // enters a separator that does not let bigrams across, like a period or a comma. + private String mPreviousWordForSuggestion; private String mAutoCorrection; private boolean mIsResumed; private boolean mIsBatchMode; @@ -89,7 +90,7 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPreviousWord = null; + mPreviousWordForSuggestion = null; refreshSize(); } @@ -106,7 +107,7 @@ public final class WordComposer { mIsBatchMode = source.mIsBatchMode; mCursorPositionWithinWord = source.mCursorPositionWithinWord; mRejectedBatchModeSuggestion = source.mRejectedBatchModeSuggestion; - mPreviousWord = source.mPreviousWord; + mPreviousWordForSuggestion = source.mPreviousWordForSuggestion; refreshSize(); } @@ -124,7 +125,7 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPreviousWord = null; + mPreviousWordForSuggestion = null; refreshSize(); } @@ -305,7 +306,7 @@ public final class WordComposer { addKeyInfo(codePoint, keyboard); } mIsResumed = true; - mPreviousWord = previousWord; + mPreviousWordForSuggestion = previousWord; } /** @@ -356,8 +357,8 @@ public final class WordComposer { return mTypedWord.toString(); } - public String getPreviousWord() { - return mPreviousWord; + public String getPreviousWordForSuggestion() { + return mPreviousWordForSuggestion; } /** @@ -419,7 +420,7 @@ public final class WordComposer { public void setCapitalizedModeAndPreviousWordAtStartComposingTime(final int mode, final String previousWord) { mCapitalizedMode = mode; - mPreviousWord = previousWord; + mPreviousWordForSuggestion = previousWord; } /** @@ -471,12 +472,7 @@ public final class WordComposer { mCapsCount = 0; mDigitsCount = 0; mIsBatchMode = false; - final boolean isWhitespace = 1 == StringUtils.codePointCount(separatorString) - && Character.isWhitespace(separatorString.codePointAt(0)); - // If not whitespace, we don't use the previous word for suggestion. This is consistent - // with how we get the previous word for suggestion: see RichInputConnection#spaceRegex and - // LatinIME#getNthPreviousWordForSuggestion. - mPreviousWord = isWhitespace ? mTypedWord.toString() : null; + mPreviousWordForSuggestion = mTypedWord.toString(); mTypedWord.setLength(0); mCodePointSize = 0; mTrailingSingleQuotesCount = 0; @@ -490,11 +486,11 @@ public final class WordComposer { return lastComposedWord; } - public void doubleSpacePeriod() { - // When a period was entered with a double space, the separator we got has been - // changed by a period (see #commitWord). We should not use the previous word for - // suggestion. - mPreviousWord = null; + // Call this when the recorded previous word should be discarded. This is typically called + // when the user inputs a separator that's not whitespace (including the case of the + // double-space-to-period feature). + public void discardPreviousWordForSuggestion() { + mPreviousWordForSuggestion = null; } public void resumeSuggestionOnLastComposedWord(final LastComposedWord lastComposedWord, @@ -509,7 +505,7 @@ public final class WordComposer { mCursorPositionWithinWord = mCodePointSize; mRejectedBatchModeSuggestion = null; mIsResumed = true; - mPreviousWord = previousWord; + mPreviousWordForSuggestion = previousWord; } public boolean isBatchMode() { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 7f0aa777f..2dbb5eb93 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -58,6 +58,7 @@ public final class BinaryDictDecoderUtils { public int readInt(); public int position(); public void position(int newPosition); + @UsedForTesting public void put(final byte b); public int limit(); @UsedForTesting @@ -166,6 +167,7 @@ public final class BinaryDictDecoderUtils { return size; } + @UsedForTesting static int getCharArraySize(final int[] chars, final int start, final int end) { int size = 0; for (int i = start; i < end; ++i) { @@ -262,6 +264,7 @@ public final class BinaryDictDecoderUtils { */ // TODO: Merge this method with writeCharArray and rename the various write* methods to // make the difference clear. + @UsedForTesting static int writeCodePoints(final OutputStream stream, final int[] codePoints, final int startIndex, final int endIndex) throws IOException { @@ -436,7 +439,7 @@ public final class BinaryDictDecoderUtils { final FormatOptions options) { dictDecoder.setPosition(headerSize); final int count = dictDecoder.readPtNodeCount(); - int groupPos = headerSize + BinaryDictIOUtils.getPtNodeCountSize(count); + int groupPos = dictDecoder.getPosition(); final StringBuilder builder = new StringBuilder(); WeightedString result = null; @@ -498,9 +501,9 @@ public final class BinaryDictDecoderUtils { do { // Scan the linked-list node. final int nodeArrayHeadPos = dictDecoder.getPosition(); final int count = dictDecoder.readPtNodeCount(); - int groupOffsetPos = nodeArrayHeadPos + BinaryDictIOUtils.getPtNodeCountSize(count); + int groupPos = dictDecoder.getPosition(); for (int i = count; i > 0; --i) { // Scan the array of PtNode. - PtNodeInfo info = dictDecoder.readPtNode(groupOffsetPos, options); + PtNodeInfo info = dictDecoder.readPtNode(groupPos, options); if (BinaryDictIOUtils.isMovedPtNode(info.mFlags, options)) continue; ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; ArrayList<WeightedString> bigrams = null; @@ -536,7 +539,7 @@ public final class BinaryDictDecoderUtils { 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); } - groupOffsetPos = info.mEndAddress; + groupPos = info.mEndAddress; } // reach the end of the array. diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 8ba0797de..bb40e0dd5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin.makedict; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; @@ -245,6 +246,7 @@ public class BinaryDictEncoderUtils { } } + @UsedForTesting static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value, final int size) { switch(size) { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 640d778bb..07ba777c7 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -23,7 +23,6 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; @@ -32,7 +31,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Iterator; import java.util.Map; import java.util.Stack; @@ -87,7 +85,7 @@ public final class BinaryDictIOUtils { if (p.mNumOfPtNode == Position.NOT_READ_PTNODE_COUNT) { p.mNumOfPtNode = dictDecoder.readPtNodeCount(); - p.mAddress += getPtNodeCountSize(p.mNumOfPtNode); + p.mAddress = dictDecoder.getPosition(); p.mPosition = 0; } if (p.mNumOfPtNode == 0) { @@ -245,6 +243,7 @@ public final class BinaryDictIOUtils { /** * @return the size written, in bytes. Always 3 bytes. */ + @UsedForTesting static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) { final int absValue = Math.abs(value); dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); @@ -256,6 +255,7 @@ public final class BinaryDictIOUtils { /** * @return the size written, in bytes. Always 3 bytes. */ + @UsedForTesting static int writeSInt24ToStream(final OutputStream destination, final int value) throws IOException { final int absValue = Math.abs(value); @@ -265,28 +265,7 @@ public final class BinaryDictIOUtils { return 3; } - /** - * @return the size written, in bytes. 1, 2, or 3 bytes. - */ - private static int writeVariableAddress(final OutputStream destination, final int value) - throws IOException { - switch (BinaryDictEncoderUtils.getByteSize(value)) { - case 1: - destination.write((byte)value); - break; - case 2: - destination.write((byte)(0xFF & (value >> 8))); - destination.write((byte)(0xFF & value)); - break; - case 3: - destination.write((byte)(0xFF & (value >> 16))); - destination.write((byte)(0xFF & (value >> 8))); - destination.write((byte)(0xFF & value)); - break; - } - return BinaryDictEncoderUtils.getByteSize(value); - } - + @UsedForTesting static void skipString(final DictBuffer dictBuffer, final boolean hasMultipleChars) { if (hasMultipleChars) { @@ -300,127 +279,13 @@ public final class BinaryDictIOUtils { } /** - * Write a PtNode to an output stream from a PtNodeInfo. - * A PtNode is an in-memory representation of a node in the patricia trie. - * A PtNode info is a container for low-level information about how the - * PtNode is stored in the binary format. - * - * @param destination the stream to write. - * @param info the PtNode info to be written. - * @return the size written, in bytes. - */ - private static int writePtNode(final OutputStream destination, final PtNodeInfo info) - throws IOException { - int size = FormatSpec.PTNODE_FLAGS_SIZE; - destination.write((byte)info.mFlags); - final int parentOffset = info.mParentAddress == FormatSpec.NO_PARENT_ADDRESS ? - FormatSpec.NO_PARENT_ADDRESS : info.mParentAddress - info.mOriginalAddress; - size += writeSInt24ToStream(destination, parentOffset); - - for (int i = 0; i < info.mCharacters.length; ++i) { - if (CharEncoding.getCharSize(info.mCharacters[i]) == 1) { - destination.write((byte)info.mCharacters[i]); - size++; - } else { - size += writeSInt24ToStream(destination, info.mCharacters[i]); - } - } - if (info.mCharacters.length > 1) { - destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); - size++; - } - - if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - destination.write((byte)info.mFrequency); - size++; - } - - if (DBG) { - MakedictLog.d("writePtNode origin=" + info.mOriginalAddress + ", size=" + size - + ", child=" + info.mChildrenAddress + ", characters =" - + new String(info.mCharacters, 0, info.mCharacters.length)); - } - final int childrenOffset = info.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS ? - 0 : info.mChildrenAddress - (info.mOriginalAddress + size); - writeSInt24ToStream(destination, childrenOffset); - size += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - - if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { - final int shortcutListSize = - BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); - destination.write((byte)(shortcutListSize >> 8)); - destination.write((byte)(shortcutListSize & 0xFF)); - size += 2; - final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), target.mFrequency)); - size++; - size += CharEncoding.writeString(destination, target.mWord); - } - } - - if (info.mBigrams != null) { - // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictEncoderUtils#computeActualNodeArraySize - for (int i = 0; i < info.mBigrams.size(); ++i) { - - final int bigramFrequency = info.mBigrams.get(i).mFrequency; - int bigramFlags = (i < info.mBigrams.size() - 1) - ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0; - size++; - final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress - + size); - bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0; - switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { - case 1: - bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; - break; - } - bigramFlags |= bigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; - destination.write((byte)bigramFlags); - size += writeVariableAddress(destination, Math.abs(bigramOffset)); - } - } - return size; - } - - /** - * Compute the size of the PtNode. - */ - static int computePtNodeSize(final PtNodeInfo info, final FormatOptions formatOptions) { - int size = FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictEncoderUtils.getPtNodeCharactersSize(info.mCharacters) - + getChildrenAddressSize(info.mFlags, formatOptions); - if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - size += FormatSpec.PTNODE_FREQUENCY_SIZE; - } - if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); - } - if (info.mBigrams != null) { - for (final PendingAttribute attr : info.mBigrams) { - size += FormatSpec.PTNODE_FLAGS_SIZE; - size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); - } - } - return size; - } - - /** * Writes a PtNodeCount to the stream. * * @param destination the stream to write. * @param ptNodeCount the count. * @return the size written in bytes. */ + @UsedForTesting static int writePtNodeCount(final OutputStream destination, final int ptNodeCount) throws IOException { final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); @@ -435,22 +300,6 @@ public final class BinaryDictIOUtils { return countSize; } - /** - * Write a node array to the stream. - * - * @param destination the stream to write. - * @param infos an array of PtNodeInfo to be written. - * @return the size written, in bytes. - * @throws IOException - */ - static int writeNodes(final OutputStream destination, final PtNodeInfo[] infos) - throws IOException { - int size = writePtNodeCount(destination, infos.length); - for (final PtNodeInfo info : infos) size += writePtNode(destination, info); - writeSInt24ToStream(destination, FormatSpec.NO_FORWARD_LINK_ADDRESS); - return size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - private static final int HEADER_READING_BUFFER_SIZE = 16384; /** * Convenience method to read the header of a binary file. diff --git a/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java deleted file mode 100644 index c4f7ec91f..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; - -import java.io.IOException; -import java.util.ArrayList; - -/** - * An interface of a binary dictionary updater. - */ -@UsedForTesting -public interface DictUpdater extends DictDecoder { - - /** - * Deletes the word from the binary dictionary. - * - * @param word the word to be deleted. - */ - @UsedForTesting - public void deleteWord(final String word) throws IOException, UnsupportedFormatException; - - /** - * Inserts a word into a binary dictionary. - * - * @param word the word to be inserted. - * @param frequency the frequency of the new word. - * @param bigramStrings bigram list, or null if none. - * @param shortcuts shortcut list, or null if none. - * @param isBlackListEntry whether this should be a blacklist entry. - */ - // TODO: Support batch insertion. - @UsedForTesting - public void insertWord(final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, - final boolean isBlackListEntry) throws IOException, UnsupportedFormatException; -} diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index af54805f7..437fa942b 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -188,7 +188,7 @@ public final class FormatSpec { // us to change the format during development while having testing devices remove // older files with each upgrade, while still having a readable versioning scheme. public static final int VERSION2 = 2; - public static final int VERSION4 = 400; + public static final int VERSION4 = 401; static final int MINIMUM_SUPPORTED_VERSION = VERSION2; static final int MAXIMUM_SUPPORTED_VERSION = VERSION4; diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java index 06088b651..63e1f56f5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java @@ -58,17 +58,17 @@ public class SparseTableContentReader { * @param blockSize the block size of the content table. * @param baseDir the directory which contains the files of the content table. * @param contentFilenames the file names of content files. - * @param contentIds the ids of contents. These ids are used for a suffix of a name of + * @param contentSuffixes the ids of contents. These ids are used for a suffix of a name of * address files and content files. * @param factory the DictionaryBufferFactory which is used for opening the files. */ public SparseTableContentReader(final String name, final int blockSize, final File baseDir, - final String[] contentFilenames, final String[] contentIds, + final String[] contentFilenames, final String[] contentSuffixes, final DictionaryBufferFactory factory) { - if (contentFilenames.length != contentIds.length) { + if (contentFilenames.length != contentSuffixes.length) { throw new RuntimeException("The length of contentFilenames and the length of" - + " contentIds are different " + contentFilenames.length + ", " - + contentIds.length); + + " contentSuffixes are different " + contentFilenames.length + ", " + + contentSuffixes.length); } mBlockSize = blockSize; mBaseDir = baseDir; @@ -79,8 +79,8 @@ public class SparseTableContentReader { mContentFiles = new File[mContentCount]; for (int i = 0; i < mContentCount; ++i) { mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentSuffixes[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentSuffixes[i]); } mAddressTableBuffers = new DictBuffer[mContentCount]; mContentBuffers = new DictBuffer[mContentCount]; @@ -94,27 +94,33 @@ public class SparseTableContentReader { } } - protected void read(final int contentIndex, final int index, + /** + * Calls the read() callback of the reader with the appropriate buffer appropriately positioned. + * @param contentNumber the index in the original contentFilenames[] array. + * @param terminalId the terminal ID to read. + * @param reader the reader on which to call the callback. + */ + protected void read(final int contentNumber, final int terminalId, final SparseTableContentReaderInterface reader) { - if (index < 0 || (index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES + if (terminalId < 0 || (terminalId / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES >= mLookupTableBuffer.limit()) { return; } - mLookupTableBuffer.position((index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - final int posInAddressTable = mLookupTableBuffer.readInt(); - if (posInAddressTable == SparseTable.NOT_EXIST) { + mLookupTableBuffer.position((terminalId / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); + final int indexInAddressTable = mLookupTableBuffer.readInt(); + if (indexInAddressTable == SparseTable.NOT_EXIST) { return; } - mAddressTableBuffers[contentIndex].position( - (posInAddressTable + index % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - final int address = mAddressTableBuffers[contentIndex].readInt(); + mAddressTableBuffers[contentNumber].position(SparseTable.SIZE_OF_INT_IN_BYTES + * ((indexInAddressTable * mBlockSize) + (terminalId % mBlockSize))); + final int address = mAddressTableBuffers[contentNumber].readInt(); if (address == SparseTable.NOT_EXIST) { return; } - mContentBuffers[contentIndex].position(address); - reader.read(mContentBuffers[contentIndex]); + mContentBuffers[contentNumber].position(address); + reader.read(mContentBuffers[contentNumber]); } }
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java deleted file mode 100644 index 4518f21b9..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * An auxiliary class for updating data associated with SparseTable. - */ -public class SparseTableContentUpdater extends SparseTableContentReader { - protected OutputStream mLookupTableOutStream; - protected OutputStream[] mAddressTableOutStreams; - protected OutputStream[] mContentOutStreams; - - public SparseTableContentUpdater(final String name, final int blockSize, - final File baseDir, final String[] contentFilenames, final String[] contentIds, - final DictionaryBufferFactory factory) { - super(name, blockSize, baseDir, contentFilenames, contentIds, factory); - mAddressTableOutStreams = new OutputStream[mContentCount]; - mContentOutStreams = new OutputStream[mContentCount]; - } - - protected void openStreamsAndBuffers() throws IOException { - openBuffers(); - mLookupTableOutStream = new FileOutputStream(mLookupTableFile, true /* append */); - for (int i = 0; i < mContentCount; ++i) { - mAddressTableOutStreams[i] = new FileOutputStream(mAddressTableFiles[i], - true /* append */); - mContentOutStreams[i] = new FileOutputStream(mContentFiles[i], true /* append */); - } - } - - /** - * Set the contentIndex-th elements of contentId-th table. - * - * @param contentId the id of the content table. - * @param contentIndex the index where to set the valie. - * @param value the value to set. - */ - protected void setContentValue(final int contentId, final int contentIndex, final int value) - throws IOException { - if ((contentIndex / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES - >= mLookupTableBuffer.limit()) { - // Need to extend the lookup table - final int currentSize = mLookupTableBuffer.limit() - / SparseTable.SIZE_OF_INT_IN_BYTES; - final int target = contentIndex / mBlockSize + 1; - for (int i = currentSize; i < target; ++i) { - BinaryDictEncoderUtils.writeUIntToStream(mLookupTableOutStream, - SparseTable.NOT_EXIST, SparseTable.SIZE_OF_INT_IN_BYTES); - } - // We need to reopen the byte buffer of the lookup table because a MappedByteBuffer in - // Java isn't expanded automatically when the underlying file is expanded. - reopenLookupTable(); - } - - mLookupTableBuffer.position((contentIndex / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - int posInAddressTable = mLookupTableBuffer.readInt(); - if (posInAddressTable == SparseTable.NOT_EXIST) { - // Need to extend the address table - mLookupTableBuffer.position(mLookupTableBuffer.position() - - SparseTable.SIZE_OF_INT_IN_BYTES); - posInAddressTable = mAddressTableBuffers[0].limit() / mBlockSize; - BinaryDictEncoderUtils.writeUIntToDictBuffer(mLookupTableBuffer, - posInAddressTable, SparseTable.SIZE_OF_INT_IN_BYTES); - for (int i = 0; i < mContentCount; ++i) { - for (int j = 0; j < mBlockSize; ++j) { - BinaryDictEncoderUtils.writeUIntToStream(mAddressTableOutStreams[i], - SparseTable.NOT_EXIST, SparseTable.SIZE_OF_INT_IN_BYTES); - } - } - // We need to reopen the byte buffers of the address tables because a MappedByteBuffer - // in Java isn't expanded automatically when the underlying file is expanded. - reopenAddressTables(); - } - posInAddressTable += (contentIndex % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES; - - mAddressTableBuffers[contentId].position(posInAddressTable); - BinaryDictEncoderUtils.writeUIntToDictBuffer(mAddressTableBuffers[contentId], - value, SparseTable.SIZE_OF_INT_IN_BYTES); - } - - private void reopenLookupTable() throws IOException { - mLookupTableOutStream.flush(); - mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); - } - - private void reopenAddressTables() throws IOException { - for (int i = 0; i < mContentCount; ++i) { - mAddressTableOutStreams[i].flush(); - mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); - } - } - - protected void close() throws IOException { - mLookupTableOutStream.close(); - for (final OutputStream stream : mAddressTableOutStreams) { - stream.close(); - } - for (final OutputStream stream : mContentOutStreams) { - stream.close(); - } - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java index e9667ab0b..ea0a2c6c2 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -23,7 +23,6 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.JniUtils; import android.util.Log; diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 3be62f066..7071893d2 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -440,6 +440,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder { } @Override + @UsedForTesting public void skipPtNode(final FormatOptions formatOptions) { final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); PtNodeReader.readParentAddress(mDictBuffer, formatOptions); diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 8eaee4d9f..d34aa171e 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -91,9 +91,9 @@ public class Ver4DictEncoder implements DictEncoder { for (final WeightedString word1 : word0.mBigrams) { binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.mFrequency, 0 /* timestamp */); - } - if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDict.flushWithGC(); + if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDict.flushWithGC(); + } } } binaryDict.flushWithGC(); diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java deleted file mode 100644 index 6298295c6..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java +++ /dev/null @@ -1,794 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import android.util.Log; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; - -/** - * An implementation of DictUpdater for version 4 binary dictionary. - */ -@UsedForTesting -public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { - private static final String TAG = Ver4DictUpdater.class.getSimpleName(); - private static final int MAX_JUMPS = 10000; - - private OutputStream mDictStream; - private final File mFrequencyFile; - - @UsedForTesting - public Ver4DictUpdater(final File dictDirectory, final int factoryType) - throws UnsupportedFormatException { - // DictUpdater must have an updatable DictBuffer. - super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) - ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); - mFrequencyFile = getFile(FILETYPE_FREQUENCY); - } - - private static class BigramContentUpdater extends SparseTableContentUpdater { - public BigramContentUpdater(final String name, final File baseDir, - final boolean hasTimestamp) { - super(name + FormatSpec.BIGRAM_FILE_EXTENSION, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - BigramContentReader.getContentFilenames(name, hasTimestamp), - BigramContentReader.getContentIds(hasTimestamp), - new DictionaryBufferFromWritableByteBufferFactory()); - } - - public void insertBigramEntries(final int terminalId, final int frequency, - final ArrayList<PendingAttribute> entries) throws IOException { - if (terminalId < 0) { - throw new RuntimeException("Invalid terminal id : " + terminalId); - } - openStreamsAndBuffers(); - - if (entries == null || entries.isEmpty()) { - setContentValue(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, - SparseTable.NOT_EXIST); - return; - } - final int positionOfEntries = - (int) mContentFiles[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX].length(); - setContentValue(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, positionOfEntries); - - final Iterator<PendingAttribute> bigramIterator = entries.iterator(); - while (bigramIterator.hasNext()) { - final PendingAttribute entry = bigramIterator.next(); - final int flags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), - 0 /* offset */, entry.mFrequency, frequency, "" /* word */); - BinaryDictEncoderUtils.writeUIntToStream( - mContentOutStreams[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX], flags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - BinaryDictEncoderUtils.writeUIntToStream( - mContentOutStreams[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX], entry.mAddress, - FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); - } - close(); - } - } - - private static class ShortcutContentUpdater extends SparseTableContentUpdater { - public ShortcutContentUpdater(final String name, final File baseDir) { - super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, - FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, - new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, - new DictionaryBufferFromWritableByteBufferFactory()); - } - - public void insertShortcuts(final int terminalId, - final ArrayList<WeightedString> shortcuts) throws IOException { - if (terminalId < 0) { - throw new RuntimeException("Invalid terminal id : " + terminalId); - } - openStreamsAndBuffers(); - if (shortcuts == null || shortcuts.isEmpty()) { - setContentValue(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, - SparseTable.NOT_EXIST); - return; - } - - final int positionOfShortcuts = - (int) mContentFiles[FormatSpec.SHORTCUT_CONTENT_INDEX].length(); - setContentValue(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, positionOfShortcuts); - - final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), target.mFrequency); - BinaryDictEncoderUtils.writeUIntToStream( - mContentOutStreams[FormatSpec.SHORTCUT_CONTENT_INDEX], shortcutFlags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - CharEncoding.writeString(mContentOutStreams[FormatSpec.SHORTCUT_CONTENT_INDEX], - target.mWord); - } - close(); - } - } - - @Override - public void deleteWord(final String word) throws IOException, UnsupportedFormatException { - if (mDictBuffer == null) { - openDictBuffer(); - readHeader(); - } - final int wordPos = getTerminalPosition(word); - if (wordPos != FormatSpec.NOT_VALID_WORD) { - mDictBuffer.position(wordPos); - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - mDictBuffer.position(wordPos); - mDictBuffer.put((byte)markAsDeleted(flags)); - } - } - - private int getNewTerminalId() { - // The size of frequency file is FormatSpec.FREQUENCY_AND_FLAGS_SIZE * number of terminals - // because each terminal always has a frequency. - // So we can get a fresh terminal id by this logic. - // CAVEAT: we are reading the file size from the disk each time: beware of race conditions, - // even on one thread. - return (int) (mFrequencyFile.length() / FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - } - - private void updateParentPosIfNotMoved(final int nodePos, final int newParentPos, - final FormatOptions formatOptions) { - final int originalPos = getPosition(); - setPosition(nodePos); - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - if (!BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { - final int parentOffset = newParentPos - nodePos; - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, parentOffset); - } - setPosition(originalPos); - } - - private void updateParentPositions(final int nodeArrayPos, final int newParentPos, - final FormatOptions formatOptions) { - final int originalPos = mDictBuffer.position(); - mDictBuffer.position(nodeArrayPos); - int jumpCount = 0; - do { - final int count = readPtNodeCount(); - for (int i = 0; i < count; ++i) { - updateParentPosIfNotMoved(getPosition(), newParentPos, formatOptions); - skipPtNode(formatOptions); - } - if (!readAndFollowForwardLink()) break; - } while (jumpCount++ < MAX_JUMPS); - setPosition(originalPos); - } - - private void updateChildrenPos(final int nodePos, final int newChildrenPos, - final FormatOptions options) { - final int originalPos = getPosition(); - setPosition(nodePos); - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - PtNodeReader.readParentAddress(mDictBuffer, options); - BinaryDictIOUtils.skipString(mDictBuffer, - (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer); - final int basePos = getPosition(); - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, newChildrenPos - basePos); - setPosition(originalPos); - } - - private void updateTerminalPosition(final int terminalId, final int position) { - if (terminalId == PtNode.NOT_A_TERMINAL - || terminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE - >= mTerminalAddressTableBuffer.limit()) return; - mTerminalAddressTableBuffer.position(terminalId - * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - BinaryDictEncoderUtils.writeUIntToDictBuffer(mTerminalAddressTableBuffer, position, - FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - } - - private void updateForwardLink(final int nodeArrayPos, final int newForwardLink, - final FormatOptions formatOptions) { - final int originalPos = getPosition(); - setPosition(nodeArrayPos); - int jumpCount = 0; - while (jumpCount++ < MAX_JUMPS) { - final int ptNodeCount = readPtNodeCount(); - for (int i = 0; i < ptNodeCount; ++i) { - skipPtNode(formatOptions); - } - final int forwardLinkPos = getPosition(); - if (!readAndFollowForwardLink()) { - setPosition(forwardLinkPos); - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, newForwardLink - forwardLinkPos); - break; - } - } - setPosition(originalPos); - } - - private void markPtNodeAsMoved(final int nodePos, final int newNodePos, - final FormatOptions options) { - final int originalPos = getPosition(); - updateParentPosIfNotMoved(nodePos, newNodePos, options); - setPosition(nodePos); - final int currentFlags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - setPosition(nodePos); - mDictBuffer.put((byte) (FormatSpec.FLAG_IS_MOVED - | (currentFlags & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); - final int offset = newNodePos - nodePos; - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, offset); - setPosition(originalPos); - } - - /** - * Writes a PtNode to an output stream from a Ver4PtNodeInfo. - * - * @param nodePos the position of the head of the PtNode. - * @param info the PtNode info to be written. - * @return the size written, in bytes. - */ - private int writePtNode(final int nodePos, final Ver4PtNodeInfo info) throws IOException { - int written = 0; - - // Write flags. - mDictStream.write((byte) (info.mFlags & 0xFF)); - written += FormatSpec.PTNODE_FLAGS_SIZE; - - // Write the parent position. - final int parentOffset = info.mParentPos == FormatSpec.NO_PARENT_ADDRESS ? - FormatSpec.NO_PARENT_ADDRESS : info.mParentPos - nodePos; - BinaryDictIOUtils.writeSInt24ToStream(mDictStream, parentOffset); - written += FormatSpec.PARENT_ADDRESS_SIZE; - - // Write a string. - if (((info.mFlags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0) - != (info.mEndIndexOfCharacters - info.mStartIndexOfCharacters > 1)) { - throw new RuntimeException("Inconsistent flags : hasMultipleChars = " - + ((info.mFlags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0) + ", length = " - + (info.mEndIndexOfCharacters - info.mStartIndexOfCharacters)); - } - written += CharEncoding.writeCodePoints(mDictStream, info.mCharacters, - info.mStartIndexOfCharacters, info.mEndIndexOfCharacters); - - // Write the terminal id. - if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - BinaryDictEncoderUtils.writeUIntToStream(mDictStream, info.mTerminalId, - FormatSpec.PTNODE_TERMINAL_ID_SIZE); - written += FormatSpec.PTNODE_TERMINAL_ID_SIZE; - } - - // Write the children position. - final int childrenOffset = info.mChildrenPos == FormatSpec.NO_CHILDREN_ADDRESS - ? 0 : info.mChildrenPos - (nodePos + written); - BinaryDictIOUtils.writeSInt24ToStream(mDictStream, childrenOffset); - written += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - - return written; - } - - /** - * Helper method to split and move PtNode. - * - * @param ptNodeArrayPos the position of PtNodeArray which contains the split and moved PtNode. - * @param splittedPtNodeToMovePos the position of the split and moved PtNode. - * @param newParent the parent PtNode after splitting. - * @param newChildren the children PtNodes after splitting. - * @param newParentStartPos where to write the new parent. - * @param formatOptions the format options. - */ - private void writeSplittedPtNodes(final int ptNodeArrayPos, final int splittedPtNodeToMovePos, - final Ver4PtNodeInfo newParent, final Ver4PtNodeInfo[] newChildren, - final int newParentStartPos, - final FormatOptions formatOptions) throws IOException { - updateTerminalPosition(newParent.mTerminalId, - newParentStartPos + 1 /* size of PtNodeCount */); - int written = writePtNodeArray(newParentStartPos, new Ver4PtNodeInfo[] { newParent }, - FormatSpec.NO_FORWARD_LINK_ADDRESS); - final int childrenStartPos = newParentStartPos + written; - writePtNodeArray(childrenStartPos, newChildren, FormatSpec.NO_FORWARD_LINK_ADDRESS); - int childrenNodePos = childrenStartPos + 1 /* size of PtNodeCount */; - for (final Ver4PtNodeInfo info : newChildren) { - updateTerminalPosition(info.mTerminalId, childrenNodePos); - childrenNodePos += computePtNodeSize(info.mCharacters, info.mStartIndexOfCharacters, - info.mEndIndexOfCharacters, - (info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0); - } - - // Mark as moved. - markPtNodeAsMoved(splittedPtNodeToMovePos, newParentStartPos + 1 /* size of PtNodeCount */, - formatOptions); - updateForwardLink(ptNodeArrayPos, newParentStartPos, formatOptions); - } - - /** - * Writes a node array to the stream. - * - * @param nodeArrayPos the position of the head of the node array. - * @param infos an array of Ver4PtNodeInfo to be written. - * @return the written length in bytes. - */ - private int writePtNodeArray(final int nodeArrayPos, final Ver4PtNodeInfo[] infos, - final int forwardLink) throws IOException { - int written = BinaryDictIOUtils.writePtNodeCount(mDictStream, infos.length); - for (int i = 0; i < infos.length; ++i) { - written += writePtNode(nodeArrayPos + written, infos[i]); - } - BinaryDictIOUtils.writeSInt24ToStream(mDictStream, forwardLink); - written += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - return written; - } - - private int computePtNodeSize(final int[] codePoints, final int startIndex, final int endIndex, - final boolean isTerminal) { - return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + CharEncoding.getCharArraySize(codePoints, startIndex, endIndex) - + (endIndex - startIndex > 1 ? FormatSpec.PTNODE_TERMINATOR_SIZE : 0) - + (isTerminal ? FormatSpec.PTNODE_TERMINAL_ID_SIZE : 0) - + FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } - - private void writeNewSinglePtNodeWithAttributes(final int[] codePoints, - final boolean hasShortcuts, final int terminalId, final boolean hasBigrams, - final boolean isNotAWord, final boolean isBlackListEntry, final int parentPos, - final FormatOptions formatOptions) throws IOException { - final int newNodeArrayPos = mDictBuffer.limit(); - final int newNodeFlags = BinaryDictEncoderUtils.makePtNodeFlags(codePoints.length > 1, - terminalId != PtNode.NOT_A_TERMINAL, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, - hasBigrams, isNotAWord, isBlackListEntry, formatOptions); - final Ver4PtNodeInfo info = new Ver4PtNodeInfo(newNodeFlags, codePoints, terminalId, - FormatSpec.NO_CHILDREN_ADDRESS, parentPos, 0 /* nodeSize */); - writePtNodeArray(newNodeArrayPos, new Ver4PtNodeInfo[] { info }, - FormatSpec.NO_FORWARD_LINK_ADDRESS); - } - - private int setMultipleCharsInFlags(final int currentFlags, final boolean hasMultipleChars) { - final int flags; - if (hasMultipleChars) { - flags = currentFlags | FormatSpec.FLAG_HAS_MULTIPLE_CHARS; - } else { - flags = currentFlags & (~FormatSpec.FLAG_HAS_MULTIPLE_CHARS); - } - return flags; - } - - private int setIsNotAWordInFlags(final int currentFlags, final boolean isNotAWord) { - final int flags; - if (isNotAWord) { - flags = currentFlags | FormatSpec.FLAG_IS_NOT_A_WORD; - } else { - flags = currentFlags & (~FormatSpec.FLAG_IS_NOT_A_WORD); - } - return flags; - } - - private int setIsBlackListEntryInFlags(final int currentFlags, final boolean isBlackListEntry) { - final int flags; - if (isBlackListEntry) { - flags = currentFlags | FormatSpec.FLAG_IS_BLACKLISTED; - } else { - flags = currentFlags & (~FormatSpec.FLAG_IS_BLACKLISTED); - } - return flags; - } - - /** - * Splits a PtNode. - * - * abcd - ef - * - * -> inserting "abc" - * - * abc - d - ef - * - * @param nodeArrayToSplitPos the position of PtNodeArray which contains the PtNode to split. - * @param nodeToSplitPos the position of the PtNode to split. - * @param nodeToSplitInfo the information of the PtNode to split. - * @param indexToSplit the index where to split in the code points array. - * @param parentOfNodeToSplitPos the absolute position of a parent of the node to split. - * @param newTerminalId the terminal id of the inserted node (corresponds to "d"). - * @param hasShortcuts whether the inserted word should have shortcuts. - * @param hasBigrams whether the inserted word should have bigrams. - * @param isNotAWord whether the inserted word should be not a word. - * @param isBlackListEntry whether the inserted word should be a black list entry. - * @param formatOptions the format options. - */ - private void splitOnly(final int nodeArrayToSplitPos, final int nodeToSplitPos, - final Ver4PtNodeInfo nodeToSplitInfo, final int indexToSplit, - final int parentOfNodeToSplitPos, final int newTerminalId, final boolean hasShortcuts, - final boolean hasBigrams, final boolean isNotAWord, final boolean isBlackListEntry, - final FormatOptions formatOptions) throws IOException { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - final int parentNodeStartPos = parentNodeArrayStartPos + 1 /* size of PtNodeCount */; - final int parentFlags = BinaryDictEncoderUtils.makePtNodeFlags(indexToSplit > 1, - true /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, formatOptions); - final Ver4PtNodeInfo parentInfo = new Ver4PtNodeInfo(parentFlags, - nodeToSplitInfo.mCharacters, newTerminalId, parentNodeStartPos - + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, true) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, - parentOfNodeToSplitPos, 0 /* nodeSize */); - parentInfo.mStartIndexOfCharacters = 0; - parentInfo.mEndIndexOfCharacters = indexToSplit; - - // Write the child. - final int childrenFlags = setMultipleCharsInFlags(nodeToSplitInfo.mFlags, - nodeToSplitInfo.mCharacters.length - indexToSplit > 1); - final Ver4PtNodeInfo childrenInfo = new Ver4PtNodeInfo(childrenFlags, - nodeToSplitInfo.mCharacters, nodeToSplitInfo.mTerminalId, - nodeToSplitInfo.mChildrenPos, parentNodeStartPos, 0 /* nodeSize */); - childrenInfo.mStartIndexOfCharacters = indexToSplit; - childrenInfo.mEndIndexOfCharacters = nodeToSplitInfo.mCharacters.length; - if (nodeToSplitInfo.mChildrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentPositions(nodeToSplitInfo.mChildrenPos, - parentInfo.mChildrenPos + 1 /* size of PtNodeCount */, formatOptions); - } - - writeSplittedPtNodes(nodeArrayToSplitPos, nodeToSplitPos, parentInfo, - new Ver4PtNodeInfo[] { childrenInfo }, parentNodeArrayStartPos, formatOptions); - } - - /** - * Split and branch a PtNode. - * - * ab - cd - * - * -> inserting "ac" - * - * a - b - cd - * | - * - c - * - * @param nodeArrayToSplitPos the position of PtNodeArray which contains the PtNode to split. - * @param nodeToSplitPos the position of the PtNode to split. - * @param nodeToSplitInfo the information of the PtNode to split. - * @param indexToSplit the index where to split in the code points array. - * @param parentOfNodeToSplitPos the absolute position of parent of the node to split. - * @param newWordSuffixCodePoints the suffix of the newly inserted word (corresponds to "c"). - * @param startIndexOfNewWordSuffixCodePoints the start index in newWordSuffixCodePoints where - * the suffix starts. - * @param newTerminalId the terminal id of the inserted node (correspond to "c"). - * @param hasShortcuts whether the inserted word should have shortcuts. - * @param hasBigrams whether the inserted word should have bigrams. - * @param isNotAWord whether the inserted word should be not a word. - * @param isBlackListEntry whether the inserted word should be a black list entry. - * @param formatOptions the format options. - */ - private void splitAndBranch(final int nodeArrayToSplitPos, final int nodeToSplitPos, - final Ver4PtNodeInfo nodeToSplitInfo, final int indexToSplit, - final int parentOfNodeToSplitPos, final int[] newWordSuffixCodePoints, - final int startIndexOfNewWordSuffixCodePoints, - final int newTerminalId, - final boolean hasShortcuts, final boolean hasBigrams, final boolean isNotAWord, - final boolean isBlackListEntry, final FormatOptions formatOptions) throws IOException { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - final int parentNodeStartPos = parentNodeArrayStartPos + 1 /* size of PtNodeCount */; - final int parentFlags = BinaryDictEncoderUtils.makePtNodeFlags( - indexToSplit > 1, - false /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, - false /* hasShortcut */, false /* hasBigrams */, - false /* isNotAWord */, false /* isBlackListEntry */, formatOptions); - final Ver4PtNodeInfo parentInfo = new Ver4PtNodeInfo(parentFlags, - nodeToSplitInfo.mCharacters, PtNode.NOT_A_TERMINAL, - parentNodeStartPos - + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, false) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, - parentOfNodeToSplitPos, 0 /* nodeSize */); - parentInfo.mStartIndexOfCharacters = 0; - parentInfo.mEndIndexOfCharacters = indexToSplit; - - final int childrenNodeArrayStartPos = parentNodeStartPos - + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, false) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - final int firstChildrenFlags = BinaryDictEncoderUtils.makePtNodeFlags( - newWordSuffixCodePoints.length - startIndexOfNewWordSuffixCodePoints > 1, - true /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, formatOptions); - final Ver4PtNodeInfo firstChildrenInfo = new Ver4PtNodeInfo(firstChildrenFlags, - newWordSuffixCodePoints, newTerminalId, - FormatSpec.NO_CHILDREN_ADDRESS, parentNodeStartPos, - 0 /* nodeSize */); - firstChildrenInfo.mStartIndexOfCharacters = startIndexOfNewWordSuffixCodePoints; - firstChildrenInfo.mEndIndexOfCharacters = newWordSuffixCodePoints.length; - - final int secondChildrenStartPos = childrenNodeArrayStartPos + 1 /* size of ptNodeCount */ - + computePtNodeSize(newWordSuffixCodePoints, startIndexOfNewWordSuffixCodePoints, - newWordSuffixCodePoints.length, true /* isTerminal */); - final int secondChildrenFlags = setMultipleCharsInFlags(nodeToSplitInfo.mFlags, - nodeToSplitInfo.mCharacters.length - indexToSplit > 1); - final Ver4PtNodeInfo secondChildrenInfo = new Ver4PtNodeInfo(secondChildrenFlags, - nodeToSplitInfo.mCharacters, nodeToSplitInfo.mTerminalId, - nodeToSplitInfo.mChildrenPos, parentNodeStartPos, 0 /* nodeSize */); - secondChildrenInfo.mStartIndexOfCharacters = indexToSplit; - secondChildrenInfo.mEndIndexOfCharacters = nodeToSplitInfo.mCharacters.length; - if (nodeToSplitInfo.mChildrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentPositions(nodeToSplitInfo.mChildrenPos, secondChildrenStartPos, - formatOptions); - } - - writeSplittedPtNodes(nodeArrayToSplitPos, nodeToSplitPos, parentInfo, - new Ver4PtNodeInfo[] { firstChildrenInfo, secondChildrenInfo }, - parentNodeArrayStartPos, formatOptions); - } - - /** - * Inserts a word into the trie file and returns the position of inserted terminal node. - * If the insertion is failed, returns FormatSpec.NOT_VALID_WORD. - */ - @UsedForTesting - private int insertWordToTrie(final String word, final int newTerminalId, - final boolean isNotAWord, final boolean isBlackListEntry, final boolean hasBigrams, - final boolean hasShortcuts) throws IOException, UnsupportedFormatException { - setPosition(0); - final FileHeader header = readHeader(); - - final int[] codePoints = FusionDictionary.getCodePoints(word); - final int wordLen = codePoints.length; - - int wordPos = 0; - for (int depth = 0; depth < FormatSpec.MAX_WORD_LENGTH; /* nop */) { - final int nodeArrayPos = getPosition(); - final int ptNodeCount = readPtNodeCount(); - boolean goToChildren = false; - int parentPos = FormatSpec.NO_PARENT_ADDRESS; - for (int i = 0; i < ptNodeCount; ++i) { - final int nodePos = getPosition(); - final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(nodePos, header.mFormatOptions); - if (BinaryDictIOUtils.isMovedPtNode(nodeInfo.mFlags, header.mFormatOptions)) { - continue; - } - if (nodeInfo.mParentPos != FormatSpec.NO_PARENT_ADDRESS) { - parentPos = nodePos + nodeInfo.mParentPos; - } - - final boolean firstCharacterMatched = - codePoints[wordPos] == nodeInfo.mCharacters[0]; - boolean allCharactersMatched = true; - int firstDifferentCharacterIndex = -1; - for (int p = 0; p < nodeInfo.mCharacters.length; ++p) { - if (wordPos + p >= codePoints.length) break; - if (codePoints[wordPos + p] != nodeInfo.mCharacters[p]) { - if (firstDifferentCharacterIndex == -1) { - firstDifferentCharacterIndex = p; - } - allCharactersMatched = false; - } - } - - if (!firstCharacterMatched) { - // Go to the next sibling node. - continue; - } - - if (!allCharactersMatched) { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - splitAndBranch(nodeArrayPos, nodePos, nodeInfo, firstDifferentCharacterIndex, - parentPos, codePoints, wordPos + firstDifferentCharacterIndex, - newTerminalId, hasShortcuts, hasBigrams, isNotAWord, - isBlackListEntry, header.mFormatOptions); - - return parentNodeArrayStartPos + computePtNodeSize(codePoints, wordPos, - wordPos + firstDifferentCharacterIndex, false) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE + 1 /* size of PtNodeCount */; - } - - if (wordLen - wordPos < nodeInfo.mCharacters.length) { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - splitOnly(nodeArrayPos, nodePos, nodeInfo, wordLen - wordPos, parentPos, - newTerminalId, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, - header.mFormatOptions); - - // Return the position of the inserted word. - return parentNodeArrayStartPos + 1 /* size of PtNodeCount */; - } - - wordPos += nodeInfo.mCharacters.length; - if (wordPos == wordLen) { - // This dictionary already contains the word. - Log.e(TAG, "Something went wrong. If the word is already contained, " - + " there is no need to insert new PtNode."); - return FormatSpec.NOT_VALID_WORD; - } - if (nodeInfo.mChildrenPos == FormatSpec.NO_CHILDREN_ADDRESS) { - // There are no children. - // We need to add a new node as a child of this node. - final int newNodeArrayPos = mDictBuffer.limit(); - final int[] newNodeCodePoints = Arrays.copyOfRange(codePoints, wordPos, - codePoints.length); - writeNewSinglePtNodeWithAttributes(newNodeCodePoints, hasShortcuts, - newTerminalId, hasBigrams, isNotAWord, isBlackListEntry, nodePos, - header.mFormatOptions); - updateChildrenPos(nodePos, newNodeArrayPos, header.mFormatOptions); - return newNodeArrayPos + 1 /* size of PtNodeCount */; - } else { - // Found the matched node. - // Go to the children of this node. - setPosition(nodeInfo.mChildrenPos); - goToChildren = true; - depth++; - break; - } - } - - if (goToChildren) continue; - if (!readAndFollowForwardLink()) { - // Add a new node that contains [wordPos, word.length()-1]. - // and update the forward link. - final int newNodeArrayPos = mDictBuffer.limit(); - final int[] newCodePoints = Arrays.copyOfRange(codePoints, wordPos, - codePoints.length); - writeNewSinglePtNodeWithAttributes(newCodePoints, hasShortcuts, newTerminalId, - hasBigrams, isNotAWord, isBlackListEntry, parentPos, header.mFormatOptions); - updateForwardLink(nodeArrayPos, newNodeArrayPos, header.mFormatOptions); - return newNodeArrayPos + 1 /* size of PtNodeCount */; - } - } - return FormatSpec.NOT_VALID_WORD; - } - - private void updateFrequency(final int terminalId, final int frequency) { - mFrequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - BinaryDictEncoderUtils.writeUIntToDictBuffer(mFrequencyBuffer, frequency, - FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - } - - private void insertFrequency(final int frequency) throws IOException { - final OutputStream frequencyStream = new FileOutputStream(mFrequencyFile, - true /* append */); - BinaryDictEncoderUtils.writeUIntToStream(frequencyStream, frequency, - FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - frequencyStream.close(); - } - - private void insertTerminalPosition(final int posOfTerminal) throws IOException, - UnsupportedFormatException { - final OutputStream terminalPosStream = new FileOutputStream( - getFile(FILETYPE_TERMINAL_ADDRESS_TABLE), true /* append */); - BinaryDictEncoderUtils.writeUIntToStream(terminalPosStream, posOfTerminal, - FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - terminalPosStream.close(); - } - - private void insertBigrams(final int terminalId, final int frequency, - final ArrayList<PendingAttribute> bigramAddresses) - throws IOException, UnsupportedFormatException { - openDictBuffer(); - final BigramContentUpdater updater = new BigramContentUpdater(mDictDirectory.getName(), - mDictDirectory, false); - - // Convert addresses to terminal ids. - final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); - mDictBuffer.position(0); - final FileHeader header = readHeader(); - for (PendingAttribute attr : bigramAddresses) { - mDictBuffer.position(attr.mAddress); - final Ver4PtNodeInfo info = readVer4PtNodeInfo(attr.mAddress, header.mFormatOptions); - if (info.mTerminalId == PtNode.NOT_A_TERMINAL) { - throw new RuntimeException("We can't have a bigram target that's not a terminal."); - } - bigrams.add(new PendingAttribute(frequency, info.mTerminalId)); - } - updater.insertBigramEntries(terminalId, frequency, bigrams); - close(); - } - - private void insertShortcuts(final int terminalId, final ArrayList<WeightedString> shortcuts) - throws IOException { - final ShortcutContentUpdater updater = new ShortcutContentUpdater(mDictDirectory.getName(), - mDictDirectory); - updater.insertShortcuts(terminalId, shortcuts); - } - - private void openBuffersAndStream() throws IOException, UnsupportedFormatException { - openDictBuffer(); - mDictStream = new FileOutputStream(getFile(FILETYPE_TRIE), true /* append */); - } - - private void close() throws IOException { - if (mDictStream != null) { - mDictStream.close(); - mDictStream = null; - } - mDictBuffer = null; - mFrequencyBuffer = null; - mTerminalAddressTableBuffer = null; - } - - private void updateAttributes(final int posOfWord, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, - final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { - mDictBuffer.position(0); - final FileHeader header = readHeader(); - mDictBuffer.position(posOfWord); - final Ver4PtNodeInfo info = readVer4PtNodeInfo(posOfWord, header.mFormatOptions); - final int terminalId = info.mTerminalId; - - // Update the flags. - final int newFlags = setIsNotAWordInFlags( - setIsBlackListEntryInFlags(info.mFlags, isBlackListEntry), isNotAWord); - mDictBuffer.position(posOfWord); - mDictBuffer.put((byte) newFlags); - - updateFrequency(terminalId, frequency); - insertBigrams(terminalId, frequency, resolveBigramPositions(this, bigramStrings)); - insertShortcuts(terminalId, shortcuts); - } - - @Override @UsedForTesting - public void insertWord(final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, - final boolean isNotAWord, final boolean isBlackListEntry) - throws IOException, UnsupportedFormatException { - final int newTerminalId = getNewTerminalId(); - - openBuffersAndStream(); - final int posOfWord = getTerminalPosition(word); - if (posOfWord != FormatSpec.NOT_VALID_WORD) { - // The word is already contained in the dictionary. - updateAttributes(posOfWord, frequency, bigramStrings, shortcuts, isNotAWord, - isBlackListEntry); - close(); - return; - } - - // Insert new PtNode into trie. - final int posOfTerminal = insertWordToTrie(word, newTerminalId, isNotAWord, - isBlackListEntry, bigramStrings != null && !bigramStrings.isEmpty(), - shortcuts != null && !shortcuts.isEmpty()); - insertFrequency(frequency); - insertTerminalPosition(posOfTerminal); - close(); - - insertBigrams(newTerminalId, frequency, resolveBigramPositions(this, bigramStrings)); - insertShortcuts(newTerminalId, shortcuts); - } - - /** - * Converts a list of WeightedString to a list of PendingAttribute. - */ - private static ArrayList<PendingAttribute> resolveBigramPositions(final DictUpdater dictUpdater, - final ArrayList<WeightedString> bigramStrings) - throws IOException, UnsupportedFormatException { - if (bigramStrings == null) return CollectionUtils.newArrayList(); - final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); - for (final WeightedString bigram : bigramStrings) { - final int pos = dictUpdater.getTerminalPosition(bigram.mWord); - if (pos == FormatSpec.NOT_VALID_WORD) { - // TODO: figure out what is the correct thing to do here. - } else { - bigrams.add(new PendingAttribute(bigram.mFrequency, pos)); - } - } - return bigrams; - } - - private static int markAsDeleted(final int flags) { - return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; - } -} diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java index aa87affa2..073148a50 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java @@ -162,19 +162,19 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick mSuggestionsStrip.removeAllViews(); removeAllViews(); addView(mSuggestionsStrip); - mMoreSuggestionsView.dismissMoreKeysPanel(); + dismissMoreSuggestionsPanel(); } private final MoreSuggestionsListener mMoreSuggestionsListener = new MoreSuggestionsListener() { @Override public void onSuggestionSelected(final int index, final SuggestedWordInfo wordInfo) { mListener.pickSuggestionManually(index, wordInfo); - mMoreSuggestionsView.dismissMoreKeysPanel(); + dismissMoreSuggestionsPanel(); } @Override public void onCancelInput() { - mMoreSuggestionsView.dismissMoreKeysPanel(); + dismissMoreSuggestionsPanel(); } }; @@ -192,10 +192,18 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick @Override public void onCancelMoreKeysPanel(final MoreKeysPanel panel) { - mMoreSuggestionsView.dismissMoreKeysPanel(); + dismissMoreSuggestionsPanel(); } }; + public boolean isShowingMoreSuggestionPanel() { + return mMoreSuggestionsView.isShowingInParent(); + } + + public void dismissMoreSuggestionsPanel() { + mMoreSuggestionsView.dismissMoreKeysPanel(); + } + @Override public boolean onLongClick(final View view) { AudioAndHapticFeedbackManager.getInstance().performHapticAndAudioFeedback( @@ -322,6 +330,6 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick @Override protected void onDetachedFromWindow() { super.onDetachedFromWindow(); - mMoreSuggestionsView.dismissMoreKeysPanel(); + dismissMoreSuggestionsPanel(); } } diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index df420417d..85f44541e 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -250,6 +250,24 @@ public final class StringUtils { return true; } + /** + * Returns true if all code points in text are whitespace, false otherwise. Empty is true. + */ + // Interestingly enough, U+00A0 NO-BREAK SPACE and U+200B ZERO-WIDTH SPACE are not considered + // whitespace, while EN SPACE, EM SPACE and IDEOGRAPHIC SPACES are. + public static boolean containsOnlyWhitespace(final String text) { + final int length = text.length(); + int i = 0; + while (i < length) { + final int codePoint = text.codePointAt(i); + if (!Character.isWhitespace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + @UsedForTesting public static boolean looksValidForDictionaryInsertion(final CharSequence text, final SettingsValues settings) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index 9afb5f221..deed010cd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -48,9 +48,9 @@ const int Ver4DictConstants::TIME_STAMP_FIELD_SIZE = 4; const int Ver4DictConstants::WORD_LEVEL_FIELD_SIZE = 1; const int Ver4DictConstants::WORD_COUNT_FIELD_SIZE = 1; -const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4; +const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 16; const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE = 4; -const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 16; +const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64; const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE = 4; const int Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE = 3; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index eb2227d60..7c6a21dd7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -31,7 +31,7 @@ class FormatUtils { enum FORMAT_VERSION { // These MUST have the same values as the relevant constants in FormatSpec.java. VERSION_2 = 2, - VERSION_4 = 400, + VERSION_4 = 401, UNKNOWN_VERSION = -1 }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.cpp index 4ad82f9f7..c380429d9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.cpp @@ -34,7 +34,8 @@ uint32_t SparseTable::get(const int id) const { const int indexTableReadingPos = getPosInIndexTable(id); const int index = mIndexTableBuffer->readUint(INDEX_SIZE, indexTableReadingPos); const int contentTableReadingPos = getPosInContentTable(id, index); - return mContentTableBuffer->readUint(mDataSize, contentTableReadingPos); + const int contentValue = mContentTableBuffer->readUint(mDataSize, contentTableReadingPos); + return contentValue == NOT_EXIST ? NOT_A_DICT_POS : contentValue; } bool SparseTable::set(const int id, const uint32_t value) { @@ -70,7 +71,7 @@ bool SparseTable::set(const int id, const uint32_t value) { // Write a new block that containing the entry to be set. int writingPos = getPosInContentTable(0 /* id */, index); for (int i = 0; i < mBlockSize; ++i) { - if (!mContentTableBuffer->writeUintAndAdvancePosition(NOT_A_DICT_POS, mDataSize, + if (!mContentTableBuffer->writeUintAndAdvancePosition(NOT_EXIST, mDataSize, &writingPos)) { AKLOGE("cannot write content table to extend. writingPos: %d, tailPos: %d, " "mDataSize: %d", writingPos, mContentTableBuffer->getTailPosition(), mDataSize); diff --git a/tests/src/com/android/inputmethod/latin/WordComposerTests.java b/tests/src/com/android/inputmethod/latin/WordComposerTests.java index a67f6a4ac..1336c6d1a 100644 --- a/tests/src/com/android/inputmethod/latin/WordComposerTests.java +++ b/tests/src/com/android/inputmethod/latin/WordComposerTests.java @@ -51,14 +51,14 @@ public class WordComposerTests extends AndroidTestCase { assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); // Check the previous word is still there - assertEquals(PREVWORD, wc.getPreviousWord()); + assertEquals(PREVWORD, wc.getPreviousWordForSuggestion()); // Move the cursor past the end of the word assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(15)); // Do what LatinIME does when the cursor is moved outside of the word, // and check the behavior is correct. wc.reset(); - assertNull(wc.getPreviousWord()); + assertNull(wc.getPreviousWordForSuggestion()); // \uD861\uDED7 is 𨛗, a character outside the BMP final String STR_WITH_SUPPLEMENTARY_CHAR = "abcde\uD861\uDED7fgh"; @@ -73,35 +73,35 @@ public class WordComposerTests extends AndroidTestCase { assertTrue(wc.isCursorFrontOrMiddleOfComposingWord()); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); - assertNull(wc.getPreviousWord()); + assertNull(wc.getPreviousWordForSuggestion()); wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, STR_WITHIN_BMP, null /* keyboard */); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); - assertEquals(STR_WITHIN_BMP, wc.getPreviousWord()); + assertEquals(STR_WITHIN_BMP, wc.getPreviousWordForSuggestion()); wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, STR_WITH_SUPPLEMENTARY_CHAR, null /* keyboard */); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); - assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPreviousWord()); + assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPreviousWordForSuggestion()); wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, STR_WITHIN_BMP, null /* keyboard */); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-3)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-1)); - assertEquals(STR_WITHIN_BMP, wc.getPreviousWord()); + assertEquals(STR_WITHIN_BMP, wc.getPreviousWordForSuggestion()); wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null /* previousWord */, null /* keyboard */); wc.setCursorPositionWithinWord(3); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-9)); - assertNull(wc.getPreviousWord()); + assertNull(wc.getPreviousWordForSuggestion()); wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, STR_WITH_SUPPLEMENTARY_CHAR, null /* keyboard */); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-10)); - assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPreviousWord()); + assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPreviousWordForSuggestion()); wc.setComposingWord(STR_WITH_SUPPLEMENTARY_CHAR, null /* previousWord */, null /* keyboard */); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index f52f5e73c..b5a71f0bf 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -34,7 +34,6 @@ import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -356,25 +355,25 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } // check bigrams - final HashMap<String, List<String>> expBigrams = new HashMap<String, List<String>>(); + final HashMap<String, Set<String>> expBigrams = new HashMap<String, Set<String>>(); for (int i = 0; i < expectedBigrams.size(); ++i) { final String word1 = expectedWords.get(expectedBigrams.keyAt(i)); for (int w2 : expectedBigrams.valueAt(i)) { if (expBigrams.get(word1) == null) { - expBigrams.put(word1, new ArrayList<String>()); + expBigrams.put(word1, new HashSet<String>()); } expBigrams.get(word1).add(expectedWords.get(w2)); } } - final HashMap<String, List<String>> actBigrams = new HashMap<String, List<String>>(); + final HashMap<String, Set<String>> actBigrams = new HashMap<String, Set<String>>(); for (Entry<Integer, ArrayList<PendingAttribute>> entry : resultBigrams.entrySet()) { final String word1 = resultWords.get(entry.getKey()); final int unigramFreq = resultFrequencies.get(entry.getKey()); for (PendingAttribute attr : entry.getValue()) { final String word2 = resultWords.get(attr.mAddress); if (actBigrams.get(word1) == null) { - actBigrams.put(word1, new ArrayList<String>()); + actBigrams.put(word1, new HashSet<String>()); } actBigrams.get(word1).add(word2); @@ -383,7 +382,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); } } - assertEquals(actBigrams, expBigrams); } @@ -594,35 +592,4 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { Log.d(TAG, result); } } - - private void runTestDeleteWord(final FormatOptions formatOptions) - throws IOException, UnsupportedFormatException { - final String dictName = "testDeleteWord"; - final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, - getContext().getCacheDir()); - - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion)); - addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); - timeWritingDictToFile(file, dict, formatOptions); - - final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(0))); - dictUpdater.deleteWord(sWords.get(0)); - assertEquals(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(0))); - - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(5))); - dictUpdater.deleteWord(sWords.get(5)); - assertEquals(FormatSpec.NOT_VALID_WORD, - dictUpdater.getTerminalPosition(sWords.get(5))); - } - - public void testDeleteWord() throws IOException, UnsupportedFormatException { - runTestDeleteWord(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runTestDeleteWord(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java deleted file mode 100644 index 9ed50c4b3..000000000 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import android.test.AndroidTestCase; -import android.test.MoreAsserts; -import android.test.suitebuilder.annotation.LargeTest; -import android.util.Log; - -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Random; - -@LargeTest -public class BinaryDictIOUtilsTests extends AndroidTestCase { - private static final String TAG = BinaryDictIOUtilsTests.class.getSimpleName(); - - private static final ArrayList<String> sWords = CollectionUtils.newArrayList(); - public static final int DEFAULT_MAX_UNIGRAMS = 1500; - private final int mMaxUnigrams; - - private static final String[] CHARACTERS = { - "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", - "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", - "\u00FC" /* ü */, "\u00E2" /* â */, "\u00F1" /* ñ */, // accented characters - "\u4E9C" /* 亜 */, "\u4F0A" /* 伊 */, "\u5B87" /* 宇 */, // kanji - "\uD841\uDE28" /* 𠘨 */, "\uD840\uDC0B" /* 𠀋 */, "\uD861\uDED7" /* 𨛗 */ // surrogate pair - }; - - public BinaryDictIOUtilsTests() { - // 1500 is the default max unigrams - this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); - } - - public BinaryDictIOUtilsTests(final long seed, final int maxUnigrams) { - super(); - Log.d(TAG, "Seed for test is " + seed + ", maxUnigrams is " + maxUnigrams); - mMaxUnigrams = maxUnigrams; - final Random random = new Random(seed); - sWords.clear(); - for (int i = 0; i < maxUnigrams; ++i) { - sWords.add(generateWord(random.nextInt())); - } - } - - // Utilities for test - private String generateWord(final int value) { - final int lengthOfChars = CHARACTERS.length; - StringBuilder builder = new StringBuilder(""); - long lvalue = Math.abs((long)value); - while (lvalue > 0) { - builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]); - lvalue /= lengthOfChars; - } - if (builder.toString().equals("")) return "a"; - return builder.toString(); - } - - private static void printPtNode(final PtNodeInfo info) { - Log.d(TAG, " PtNode at " + info.mOriginalAddress); - Log.d(TAG, " flags = " + info.mFlags); - Log.d(TAG, " parentAddress = " + info.mParentAddress); - Log.d(TAG, " characters = " + new String(info.mCharacters, 0, - info.mCharacters.length)); - if (info.mFrequency != -1) Log.d(TAG, " frequency = " + info.mFrequency); - if (info.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { - Log.d(TAG, " children address = no children address"); - } else { - Log.d(TAG, " children address = " + info.mChildrenAddress); - } - if (info.mShortcutTargets != null) { - for (final WeightedString ws : info.mShortcutTargets) { - Log.d(TAG, " shortcuts = " + ws.mWord); - } - } - if (info.mBigrams != null) { - for (final PendingAttribute attr : info.mBigrams) { - Log.d(TAG, " bigram = " + attr.mAddress); - } - } - Log.d(TAG, " end address = " + info.mEndAddress); - } - - private static void printNode(final Ver2DictDecoder dictDecoder, - final FormatSpec.FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - Log.d(TAG, "Node at " + dictBuffer.position()); - final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); - Log.d(TAG, " ptNodeCount = " + count); - for (int i = 0; i < count; ++i) { - final PtNodeInfo currentInfo = dictDecoder.readPtNode(dictBuffer.position(), - formatOptions); - printPtNode(currentInfo); - } - if (formatOptions.supportsDynamicUpdate()) { - final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); - Log.d(TAG, " forwardLinkAddress = " + forwardLinkAddress); - } - } - - @SuppressWarnings("unused") - private static void printBinaryFile(final Ver2DictDecoder dictDecoder) - throws IOException, UnsupportedFormatException { - final FileHeader fileHeader = dictDecoder.readHeader(); - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - while (dictBuffer.position() < dictBuffer.limit()) { - printNode(dictDecoder, fileHeader.mFormatOptions); - } - } - - private int getWordPosition(final File file, final String word) { - int position = FormatSpec.NOT_VALID_WORD; - - try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, - DictDecoder.USE_READONLY_BYTEBUFFER); - position = dictDecoder.getTerminalPosition(word); - } catch (IOException e) { - } catch (UnsupportedFormatException e) { - } - return position; - } - - /** - * Find a word using the DictDecoder. - * - * @param dictDecoder the dict decoder - * @param word the word searched - * @return the found ptNodeInfo - * @throws IOException - * @throws UnsupportedFormatException - */ - private static PtNodeInfo findWordByDictDecoder(final DictDecoder dictDecoder, - final String word) throws IOException, UnsupportedFormatException { - int position = dictDecoder.getTerminalPosition(word); - if (position != FormatSpec.NOT_VALID_WORD) { - dictDecoder.setPosition(0); - final FileHeader header = dictDecoder.readHeader(); - dictDecoder.setPosition(position); - return dictDecoder.readPtNode(position, header.mFormatOptions); - } - return null; - } - - private PtNodeInfo findWordFromFile(final File file, final String word) { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); - PtNodeInfo info = null; - try { - dictDecoder.openDictBuffer(); - info = findWordByDictDecoder(dictDecoder, word); - } catch (IOException e) { - } catch (UnsupportedFormatException e) { - } - return info; - } - - // return amount of time to insert a word - private long insertAndCheckWord(final File file, final String word, final int frequency, - final boolean exist, final ArrayList<WeightedString> bigrams, - final ArrayList<WeightedString> shortcuts, final FormatOptions formatOptions) { - long amountOfTime = -1; - try { - final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); - - if (!exist) { - assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - } - final long now = System.nanoTime(); - dictUpdater.insertWord(word, frequency, bigrams, shortcuts, false, false); - amountOfTime = System.nanoTime() - now; - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - } catch (IOException e) { - Log.e(TAG, "Raised an IOException while inserting a word", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e); - } - return amountOfTime; - } - - private void deleteWord(final File file, final String word, final FormatOptions formatOptions) { - try { - final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); - dictUpdater.deleteWord(word); - } catch (IOException e) { - Log.e(TAG, "Raised an IOException while deleting a word", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Raised an UnsupportedFormatException while deleting a word", e); - } - } - - private void checkReverseLookup(final File file, final String word, final int position) { - - try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); - final FileHeader fileHeader = dictDecoder.readHeader(); - assertEquals(word, - BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset, - position, fileHeader.mFormatOptions).mWord); - } catch (IOException e) { - Log.e(TAG, "Raised an IOException while looking up a word", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Raised an UnsupportedFormatException error while looking up a word", e); - } - } - - private void runTestInsertWord(final FormatOptions formatOptions) { - final String testName = "testInsertWord"; - final String version = Long.toString(System.currentTimeMillis()); - final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, - getContext().getCacheDir()); - - // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - BinaryDictUtils.makeDictionaryOptions(testName, version)); - dict.add("abcd", 10, null, false); - - try { - final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions); - dictEncoder.writeDictionary(dict, formatOptions); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); - } - - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - insertAndCheckWord(file, "abcde", 10, false, null, null, formatOptions); - checkReverseLookup(file, "abcde", getWordPosition(file, "abcde")); - - insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatOptions); - checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - - insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatOptions); - checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); - - // update the existing word. - insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatOptions); - checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); - - // Testing splitOnly - insertAndCheckWord(file, "ab", 20, false, null, null, formatOptions); - checkReverseLookup(file, "ab", getWordPosition(file, "ab")); - checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); - checkReverseLookup(file, "abcde", getWordPosition(file, "abcde")); - checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - - // Testing splitAndBranch - insertAndCheckWord(file, "ami", 30, false, null, null, formatOptions); - checkReverseLookup(file, "ami", getWordPosition(file, "ami")); - checkReverseLookup(file, "ab", getWordPosition(file, "ab")); - checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); - checkReverseLookup(file, "abcde", getWordPosition(file, "abcde")); - checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - checkReverseLookup(file, "ami", getWordPosition(file, "ami")); - - insertAndCheckWord(file, "abcdefzzzz", 40, false, null, null, formatOptions); - checkReverseLookup(file, "abcdefzzzz", getWordPosition(file, "abcdefzzzz")); - - deleteWord(file, "ami", formatOptions); - assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami")); - - insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatOptions); - - deleteWord(file, "abcd", formatOptions); - assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - } - - public void testInsertWord() { - runTestInsertWord(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runTestInsertWord(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - } - - private void runTestInsertWordWithBigrams(final FormatOptions formatOptions) { - final String testName = "testInsertWordWithBigrams"; - final String version = Long.toString(System.currentTimeMillis()); - File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, - getContext().getCacheDir()); - - // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - BinaryDictUtils.makeDictionaryOptions(testName, version)); - dict.add("abcd", 10, null, false); - dict.add("efgh", 15, null, false); - - try { - final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions); - dictEncoder.writeDictionary(dict, formatOptions); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); - } - - final ArrayList<WeightedString> banana = new ArrayList<WeightedString>(); - banana.add(new WeightedString("banana", 10)); - - insertAndCheckWord(file, "banana", 0, false, null, null, formatOptions); - insertAndCheckWord(file, "recursive", 60, true, banana, null, formatOptions); - - final PtNodeInfo info = findWordFromFile(file, "recursive"); - int bananaPos = getWordPosition(file, "banana"); - assertNotNull(info.mBigrams); - assertEquals(info.mBigrams.size(), 1); - assertEquals(info.mBigrams.get(0).mAddress, bananaPos); - } - - public void testInsertWordWithBigrams() { - runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - } - - private void runTestRandomWords(final FormatOptions formatOptions) { - final String testName = "testRandomWord"; - final String version = Long.toString(System.currentTimeMillis()); - final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, - getContext().getCacheDir()); - - // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - BinaryDictUtils.makeDictionaryOptions(testName, version)); - dict.add("initial", 10, null, false); - - try { - final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions); - dictEncoder.writeDictionary(dict, formatOptions); - } catch (IOException e) { - assertTrue(false); - } catch (UnsupportedFormatException e) { - assertTrue(false); - } - - long maxTimeToInsert = 0, sum = 0; - long minTimeToInsert = 100000000; // 1000000000 is an upper bound for minTimeToInsert. - int cnt = 0; - for (final String word : sWords) { - final long diff = insertAndCheckWord(file, word, - cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatOptions); - maxTimeToInsert = Math.max(maxTimeToInsert, diff); - minTimeToInsert = Math.min(minTimeToInsert, diff); - sum += diff; - cnt++; - } - cnt = 0; - for (final String word : sWords) { - MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - } - - Log.d(TAG, "Test version " + formatOptions.mVersion); - Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms."); - Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms."); - Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms."); - } - - public void testRandomWords() { - runTestRandomWords(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runTestRandomWords(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - } -} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java index 67d77e05a..f7a808c1e 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java @@ -69,14 +69,4 @@ public class BinaryDictUtils { + formatOptions.mVersion); } } - - public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions) - throws UnsupportedFormatException { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new UnsupportedFormatException("The format option has a wrong version : " - + formatOptions.mVersion); - } - } } diff --git a/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java index 2123e84e8..0c88f34f0 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringAndJsonUtilsTests.java @@ -292,6 +292,20 @@ public class StringAndJsonUtilsTests extends AndroidTestCase { assertTrue(bytesStr.equals(bytesStr2)); } + public void testContainsOnlyWhitespace() { + assertTrue(StringUtils.containsOnlyWhitespace(" ")); + assertTrue(StringUtils.containsOnlyWhitespace("")); + assertTrue(StringUtils.containsOnlyWhitespace(" \n\t\t")); + // U+2002 : EN SPACE + // U+2003 : EM SPACE + // U+3000 : IDEOGRAPHIC SPACE (commonly "double-width space") + assertTrue(StringUtils.containsOnlyWhitespace("\u2002\u2003\u3000")); + assertFalse(StringUtils.containsOnlyWhitespace(" a ")); + assertFalse(StringUtils.containsOnlyWhitespace(". ")); + assertFalse(StringUtils.containsOnlyWhitespace(".")); + assertTrue(StringUtils.containsOnlyWhitespace("")); + } + public void testJsonUtils() { final Object[] objs = new Object[] { 1, "aaa", "bbb", 3 }; final List<Object> objArray = Arrays.asList(objs); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java index 9174238da..48817b1b1 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java @@ -18,7 +18,6 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests; import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests; -import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests; import com.android.inputmethod.latin.makedict.FusionDictionaryTest; import java.lang.reflect.Constructor; @@ -31,15 +30,15 @@ import java.util.ArrayList; */ public class Test extends Dicttool.Command { public static final String COMMAND = "test"; + private static final int DEFAULT_MAX_UNIGRAMS = 1500; private long mSeed = System.currentTimeMillis(); - private int mMaxUnigrams = BinaryDictIOUtilsTests.DEFAULT_MAX_UNIGRAMS; + private int mMaxUnigrams = DEFAULT_MAX_UNIGRAMS; private static final Class<?>[] sClassesToTest = { BinaryDictOffdeviceUtilsTests.class, FusionDictionaryTest.class, BinaryDictDecoderEncoderTests.class, BinaryDictEncoderFlattenTreeTests.class, - BinaryDictIOUtilsTests.class }; private ArrayList<Method> mAllTestMethods = new ArrayList<Method>(); private ArrayList<String> mUsedTestMethods = new ArrayList<String>(); diff --git a/tools/make-keyboard-text/res/values-hy-rAM/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-hy-rAM/donottranslate-more-keys.xml index a94d50e53..73fe927ab 100644 --- a/tools/make-keyboard-text/res/values-hy-rAM/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values-hy-rAM/donottranslate-more-keys.xml @@ -26,7 +26,7 @@ <!-- U+055A: "՚" ARMENIAN APOSTROPHE --> <!-- U+055B: "՛" ARMENIAN EMPHASIS MARK --> <!-- U+055F: "՟" ARMENIAN ABBREVIATION MARK --> - <string name="more_keys_for_punctuation">"!fixedColumnOrder!8,!,?,\\,,.,֊,՜,՝,՞,:,;,\@,ՙ,՚,՛,՟"</string> + <string name="more_keys_for_punctuation">"!fixedColumnOrder!8,!,?,ՙ,՚,.,՜,\\,,՞,:,;,՟,«,»,֊,՝,՛"</string> <!-- U+055E: "՞" ARMENIAN QUESTION MARK --> <!-- U+00BF: "¿" INVERTED QUESTION MARK --> <string name="more_keys_for_question">՞,¿</string> |