diff options
22 files changed, 540 insertions, 430 deletions
diff --git a/java/src/com/android/inputmethod/event/InputTransaction.java b/java/src/com/android/inputmethod/event/InputTransaction.java index cdff265c6..b18bf5638 100644 --- a/java/src/com/android/inputmethod/event/InputTransaction.java +++ b/java/src/com/android/inputmethod/event/InputTransaction.java @@ -33,7 +33,7 @@ public class InputTransaction { // Initial conditions public final SettingsValues mSettingsValues; - public final Event mEvent; + private final Event mEvent; public final long mTimestamp; public final int mSpaceState; public final int mShiftState; @@ -42,6 +42,7 @@ public class InputTransaction { private int mRequiredShiftUpdate = SHIFT_NO_UPDATE; private boolean mRequiresUpdateSuggestions = false; private boolean mDidAffectContents = false; + private boolean mDidAutoCorrect = false; public InputTransaction(final SettingsValues settingsValues, final Event event, final long timestamp, final int spaceState, final int shiftState) { @@ -97,4 +98,19 @@ public class InputTransaction { public boolean didAffectContents() { return mDidAffectContents; } + + /** + * Indicate that this transaction performed an auto-correction. + */ + public void setDidAutoCorrect() { + mDidAutoCorrect = true; + } + + /** + * Find out whether this transaction performed an auto-correction. + * @return Whether this transaction performed an auto-correction. + */ + public boolean didAutoCorrect() { + return mDidAutoCorrect; + } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 719aeac77..4adc28d7a 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -248,7 +248,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen break; case MSG_RESET_CACHES: final SettingsValues settingsValues = latinIme.mSettings.getCurrent(); - if (latinIme.mInputLogic.retryResetCachesAndReturnSuccess(settingsValues, + if (latinIme.mInputLogic.retryResetCachesAndReturnSuccess( msg.arg1 == 1 /* tryResumeSuggestions */, msg.arg2 /* remainingTries */, this /* handler */)) { // If we were able to reset the caches, then we can reload the keyboard. @@ -752,8 +752,30 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen loadKeyboard(); } + /** + * A class that holds information to pass from onStartInputInternal to onStartInputViewInternal + * + * OnStartInput needs to reload the settings and that will prevent onStartInputViewInternal + * from comparing the old settings with the new ones, so we use this memory to pass the + * necessary information along. + */ + private static class EditorChangeInfo { + public final boolean mIsSameInputType; + public final boolean mHasSameOrientation; + public EditorChangeInfo(final boolean isSameInputType, final boolean hasSameOrientation) { + mIsSameInputType = isSameInputType; + mHasSameOrientation = hasSameOrientation; + } + } + + private EditorChangeInfo mLastEditorChangeInfo; + private void onStartInputInternal(final EditorInfo editorInfo, final boolean restarting) { super.onStartInput(editorInfo, restarting); + SettingsValues currentSettingsValues = mSettings.getCurrent(); + mLastEditorChangeInfo = new EditorChangeInfo( + currentSettingsValues.isSameInputType(editorInfo), + currentSettingsValues.hasSameOrientation(getResources().getConfiguration())); } @SuppressWarnings("deprecation") @@ -763,9 +785,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final KeyboardSwitcher switcher = mKeyboardSwitcher; switcher.updateKeyboardTheme(); final MainKeyboardView mainKeyboardView = switcher.getMainKeyboardView(); - // If we are starting input in a different text field from before, we'll have to reload - // settings, so currentSettingsValues can't be final. - SettingsValues currentSettingsValues = mSettings.getCurrent(); if (editorInfo == null) { Log.e(TAG, "Null EditorInfo in onStartInputView()"); @@ -808,7 +827,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen accessUtils.onStartInputViewInternal(mainKeyboardView, editorInfo, restarting); } - final boolean inputTypeChanged = !currentSettingsValues.isSameInputType(editorInfo); + final boolean inputTypeChanged = !mLastEditorChangeInfo.mIsSameInputType; final boolean isDifferentTextField = !restarting || inputTypeChanged; if (isDifferentTextField) { mSubtypeSwitcher.updateParametersOnStartInputView(); @@ -853,13 +872,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen canReachInputConnection = true; } - if (isDifferentTextField || - !currentSettingsValues.hasSameOrientation(getResources().getConfiguration())) { + if (isDifferentTextField || !mLastEditorChangeInfo.mHasSameOrientation) { loadSettings(); } + final SettingsValues currentSettingsValues = mSettings.getCurrent(); if (isDifferentTextField) { mainKeyboardView.closing(); - currentSettingsValues = mSettings.getCurrent(); if (currentSettingsValues.mAutoCorrectionEnabledPerUserSettings) { suggest.setAutoCorrectionThreshold( diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index dee7cd466..5c719edb9 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -426,121 +426,15 @@ public final class InputLogic { cancelDoubleSpacePeriodCountdown(); } - boolean didAutoCorrect = false; if (processedEvent.isConsumed()) { - // A consumed event may have text to commit and an update to the composing state, so - // we evaluate both. With some combiners, it's possible than an event contains both - // and we enter both of the following if clauses. - final CharSequence textToCommit = processedEvent.getTextToCommit(); - if (!TextUtils.isEmpty(textToCommit)) { - mConnection.commitText(textToCommit, 1); - inputTransaction.setDidAffectContents(); - } - if (mWordComposer.isComposingWord()) { - mConnection.setComposingText(mWordComposer.getTypedWord(), 1); - inputTransaction.setDidAffectContents(); - inputTransaction.setRequiresUpdateSuggestions(); - } + handleConsumedEvent(processedEvent, inputTransaction); } else if (processedEvent.isFunctionalKeyEvent()) { - // A special key, like delete, shift, emoji, or the settings key. - switch (processedEvent.mKeyCode) { - case Constants.CODE_DELETE: - handleBackspace(inputTransaction, currentKeyboardScriptId); - // Backspace is a functional key, but it affects the contents of the editor. - inputTransaction.setDidAffectContents(); - break; - case Constants.CODE_SHIFT: - performRecapitalization(inputTransaction.mSettingsValues); - inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW); - if (mSuggestedWords.mIsPrediction) { - inputTransaction.setRequiresUpdateSuggestions(); - } - break; - case Constants.CODE_CAPSLOCK: - // Note: Changing keyboard to shift lock state is handled in - // {@link KeyboardSwitcher#onCodeInput(int)}. - break; - case Constants.CODE_SYMBOL_SHIFT: - // Note: Calling back to the keyboard on the symbol Shift key is handled in - // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}. - break; - case Constants.CODE_SWITCH_ALPHA_SYMBOL: - // Note: Calling back to the keyboard on symbol key is handled in - // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}. - break; - case Constants.CODE_SETTINGS: - onSettingsKeyPressed(); - break; - case Constants.CODE_SHORTCUT: - // We need to switch to the shortcut IME. This is handled by LatinIME since the - // input logic has no business with IME switching. - break; - case Constants.CODE_ACTION_NEXT: - performEditorAction(EditorInfo.IME_ACTION_NEXT); - break; - case Constants.CODE_ACTION_PREVIOUS: - performEditorAction(EditorInfo.IME_ACTION_PREVIOUS); - break; - case Constants.CODE_LANGUAGE_SWITCH: - handleLanguageSwitchKey(); - break; - case Constants.CODE_EMOJI: - // Note: Switching emoji keyboard is being handled in - // {@link KeyboardState#onCodeInput(int,int)}. - break; - case Constants.CODE_ALPHA_FROM_EMOJI: - // Note: Switching back from Emoji keyboard to the main keyboard is being - // handled in {@link KeyboardState#onCodeInput(int,int)}. - break; - case Constants.CODE_SHIFT_ENTER: - // TODO: remove this object - final Event tmpEvent = Event.createSoftwareKeypressEvent(Constants.CODE_ENTER, - processedEvent.mKeyCode, processedEvent.mX, processedEvent.mY, - processedEvent.isKeyRepeat()); - final InputTransaction tmpTransaction = new InputTransaction( - inputTransaction.mSettingsValues, tmpEvent, - inputTransaction.mTimestamp, inputTransaction.mSpaceState, - inputTransaction.mShiftState); - didAutoCorrect = handleNonSpecialCharacter(tmpTransaction, handler); - // Shift + Enter is treated as a functional key but it results in adding a new - // line, so that does affect the contents of the editor. - inputTransaction.setDidAffectContents(); - break; - default: - throw new RuntimeException("Unknown key code : " + processedEvent.mKeyCode); - } + handleFunctionalEvent(processedEvent, inputTransaction, currentKeyboardScriptId, + handler); } else { - inputTransaction.setDidAffectContents(); - switch (processedEvent.mCodePoint) { - case Constants.CODE_ENTER: - final EditorInfo editorInfo = getCurrentInputEditorInfo(); - final int imeOptionsActionId = - InputTypeUtils.getImeOptionsActionIdFromEditorInfo(editorInfo); - if (InputTypeUtils.IME_ACTION_CUSTOM_LABEL == imeOptionsActionId) { - // Either we have an actionLabel and we should performEditorAction with - // actionId regardless of its value. - performEditorAction(editorInfo.actionId); - } else if (EditorInfo.IME_ACTION_NONE != imeOptionsActionId) { - // We didn't have an actionLabel, but we had another action to execute. - // EditorInfo.IME_ACTION_NONE explicitly means no action. In contrast, - // EditorInfo.IME_ACTION_UNSPECIFIED is the default value for an action, so it - // means there should be an action and the app didn't bother to set a specific - // code for it - presumably it only handles one. It does not have to be treated - // in any specific way: anything that is not IME_ACTION_NONE should be sent to - // performEditorAction. - performEditorAction(imeOptionsActionId); - } else { - // No action label, and the action from imeOptions is NONE: this is a regular - // enter key that should input a carriage return. - didAutoCorrect = handleNonSpecialCharacter(inputTransaction, handler); - } - break; - default: - didAutoCorrect = handleNonSpecialCharacter(inputTransaction, handler); - break; - } + handleNonFunctionalEvent(processedEvent, inputTransaction, handler); } - if (!didAutoCorrect && processedEvent.mKeyCode != Constants.CODE_SHIFT + if (!inputTransaction.didAutoCorrect() && processedEvent.mKeyCode != Constants.CODE_SHIFT && processedEvent.mKeyCode != Constants.CODE_CAPSLOCK && processedEvent.mKeyCode != Constants.CODE_SWITCH_ALPHA_SYMBOL) mLastComposedWord.deactivate(); @@ -686,6 +580,153 @@ public final class InputLogic { } /** + * Handle a consumed event. + * + * Consumed events represent events that have already been consumed, typically by the + * combining chain. + * + * @param event The event to handle. + * @param inputTransaction The transaction in progress. + */ + private void handleConsumedEvent(final Event event, final InputTransaction inputTransaction) { + // A consumed event may have text to commit and an update to the composing state, so + // we evaluate both. With some combiners, it's possible than an event contains both + // and we enter both of the following if clauses. + final CharSequence textToCommit = event.getTextToCommit(); + if (!TextUtils.isEmpty(textToCommit)) { + mConnection.commitText(textToCommit, 1); + inputTransaction.setDidAffectContents(); + } + if (mWordComposer.isComposingWord()) { + mConnection.setComposingText(mWordComposer.getTypedWord(), 1); + inputTransaction.setDidAffectContents(); + inputTransaction.setRequiresUpdateSuggestions(); + } + } + + /** + * Handle a functional key event. + * + * A functional event is a special key, like delete, shift, emoji, or the settings key. + * Non-special keys are those that generate a single code point. + * This includes all letters, digits, punctuation, separators, emoji. It excludes keys that + * manage keyboard-related stuff like shift, language switch, settings, layout switch, or + * any key that results in multiple code points like the ".com" key. + * + * @param event The event to handle. + * @param inputTransaction The transaction in progress. + */ + private void handleFunctionalEvent(final Event event, final InputTransaction inputTransaction, + // TODO: remove these arguments + final int currentKeyboardScriptId, final LatinIME.UIHandler handler) { + switch (event.mKeyCode) { + case Constants.CODE_DELETE: + handleBackspaceEvent(event, inputTransaction, currentKeyboardScriptId); + // Backspace is a functional key, but it affects the contents of the editor. + inputTransaction.setDidAffectContents(); + break; + case Constants.CODE_SHIFT: + performRecapitalization(inputTransaction.mSettingsValues); + inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW); + if (mSuggestedWords.mIsPrediction) { + inputTransaction.setRequiresUpdateSuggestions(); + } + break; + case Constants.CODE_CAPSLOCK: + // Note: Changing keyboard to shift lock state is handled in + // {@link KeyboardSwitcher#onCodeInput(int)}. + break; + case Constants.CODE_SYMBOL_SHIFT: + // Note: Calling back to the keyboard on the symbol Shift key is handled in + // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}. + break; + case Constants.CODE_SWITCH_ALPHA_SYMBOL: + // Note: Calling back to the keyboard on symbol key is handled in + // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}. + break; + case Constants.CODE_SETTINGS: + onSettingsKeyPressed(); + break; + case Constants.CODE_SHORTCUT: + // We need to switch to the shortcut IME. This is handled by LatinIME since the + // input logic has no business with IME switching. + break; + case Constants.CODE_ACTION_NEXT: + performEditorAction(EditorInfo.IME_ACTION_NEXT); + break; + case Constants.CODE_ACTION_PREVIOUS: + performEditorAction(EditorInfo.IME_ACTION_PREVIOUS); + break; + case Constants.CODE_LANGUAGE_SWITCH: + handleLanguageSwitchKey(); + break; + case Constants.CODE_EMOJI: + // Note: Switching emoji keyboard is being handled in + // {@link KeyboardState#onCodeInput(int,int)}. + break; + case Constants.CODE_ALPHA_FROM_EMOJI: + // Note: Switching back from Emoji keyboard to the main keyboard is being + // handled in {@link KeyboardState#onCodeInput(int,int)}. + break; + case Constants.CODE_SHIFT_ENTER: + // TODO: remove this object + final Event tmpEvent = Event.createSoftwareKeypressEvent(Constants.CODE_ENTER, + event.mKeyCode, event.mX, event.mY, event.isKeyRepeat()); + handleNonSpecialCharacterEvent(tmpEvent, inputTransaction, handler); + // Shift + Enter is treated as a functional key but it results in adding a new + // line, so that does affect the contents of the editor. + inputTransaction.setDidAffectContents(); + break; + default: + throw new RuntimeException("Unknown key code : " + event.mKeyCode); + } + } + + /** + * Handle an event that is not a functional event. + * + * These events are generally events that cause input, but in some cases they may do other + * things like trigger an editor action. + * + * @param event The event to handle. + * @param inputTransaction The transaction in progress. + */ + private void handleNonFunctionalEvent(final Event event, + final InputTransaction inputTransaction, + // TODO: remove this argument + final LatinIME.UIHandler handler) { + inputTransaction.setDidAffectContents(); + switch (event.mCodePoint) { + case Constants.CODE_ENTER: + final EditorInfo editorInfo = getCurrentInputEditorInfo(); + final int imeOptionsActionId = + InputTypeUtils.getImeOptionsActionIdFromEditorInfo(editorInfo); + if (InputTypeUtils.IME_ACTION_CUSTOM_LABEL == imeOptionsActionId) { + // Either we have an actionLabel and we should performEditorAction with + // actionId regardless of its value. + performEditorAction(editorInfo.actionId); + } else if (EditorInfo.IME_ACTION_NONE != imeOptionsActionId) { + // We didn't have an actionLabel, but we had another action to execute. + // EditorInfo.IME_ACTION_NONE explicitly means no action. In contrast, + // EditorInfo.IME_ACTION_UNSPECIFIED is the default value for an action, so it + // means there should be an action and the app didn't bother to set a specific + // code for it - presumably it only handles one. It does not have to be treated + // in any specific way: anything that is not IME_ACTION_NONE should be sent to + // performEditorAction. + performEditorAction(imeOptionsActionId); + } else { + // No action label, and the action from imeOptions is NONE: this is a regular + // enter key that should input a carriage return. + handleNonSpecialCharacterEvent(event, inputTransaction, handler); + } + break; + default: + handleNonSpecialCharacterEvent(event, inputTransaction, handler); + break; + } + } + + /** * Handle inputting a code point to the editor. * * Non-special keys are those that generate a single code point. @@ -693,21 +734,19 @@ public final class InputLogic { * manage keyboard-related stuff like shift, language switch, settings, layout switch, or * any key that results in multiple code points like the ".com" key. * + * @param event The event to handle. * @param inputTransaction The transaction in progress. - * @return whether this caused an auto-correction to happen. */ - private boolean handleNonSpecialCharacter(final InputTransaction inputTransaction, + private void handleNonSpecialCharacterEvent(final Event event, + final InputTransaction inputTransaction, // TODO: remove this argument final LatinIME.UIHandler handler) { - final int codePoint = inputTransaction.mEvent.mCodePoint; + final int codePoint = event.mCodePoint; mSpaceState = SpaceState.NONE; - final boolean didAutoCorrect; if (inputTransaction.mSettingsValues.isWordSeparator(codePoint) || Character.getType(codePoint) == Character.OTHER_SYMBOL) { - didAutoCorrect = handleSeparator(inputTransaction, - inputTransaction.mEvent.isSuggestionStripPress(), handler); + handleSeparatorEvent(event, inputTransaction, handler); } else { - didAutoCorrect = false; if (SpaceState.PHANTOM == inputTransaction.mSpaceState) { if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection @@ -718,22 +757,23 @@ public final class InputLogic { commitTyped(inputTransaction.mSettingsValues, LastComposedWord.NOT_A_SEPARATOR); } } - handleNonSeparator(inputTransaction.mSettingsValues, inputTransaction); + handleNonSeparatorEvent(event, inputTransaction.mSettingsValues, inputTransaction); } - return didAutoCorrect; } /** * Handle a non-separator. + * @param event The event to handle. * @param settingsValues The current settings values. * @param inputTransaction The transaction in progress. */ - private void handleNonSeparator(final SettingsValues settingsValues, + private void handleNonSeparatorEvent(final Event event, final SettingsValues settingsValues, final InputTransaction inputTransaction) { - final int codePoint = inputTransaction.mEvent.mCodePoint; + final int codePoint = event.mCodePoint; // TODO: refactor this method to stop flipping isComposingWord around all the time, and - // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter - // which has the same name as other handle* methods but is not the same. + // make it shorter (possibly cut into several pieces). Also factor + // handleNonSpecialCharacterEvent which has the same name as other handle* methods but is + // not the same. boolean isComposingWord = mWordComposer.isComposingWord(); // TODO: remove isWordConnector() and use isUsuallyFollowedBySpace() instead. @@ -780,7 +820,7 @@ public final class InputLogic { resetComposingState(false /* alsoResetLastComposedWord */); } if (isComposingWord) { - mWordComposer.applyProcessedEvent(inputTransaction.mEvent); + mWordComposer.applyProcessedEvent(event); // If it's the first letter, make note of auto-caps state if (mWordComposer.isSingleLetter()) { mWordComposer.setCapitalizedModeAtStartComposingTime(inputTransaction.mShiftState); @@ -788,10 +828,10 @@ public final class InputLogic { mConnection.setComposingText(getTextWithUnderline( mWordComposer.getTypedWord()), 1); } else { - final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead( + final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(event, inputTransaction); - if (swapWeakSpace && trySwapSwapperAndSpace(inputTransaction)) { + if (swapWeakSpace && trySwapSwapperAndSpace(event, inputTransaction)) { mSpaceState = SpaceState.WEAK; } else { sendKeyCodePoint(settingsValues, codePoint); @@ -804,17 +844,14 @@ public final class InputLogic { /** * Handle input of a separator code point. + * @param event The event to handle. * @param inputTransaction The transaction in progress. - * @param isFromSuggestionStrip whether this code point comes from the suggestion strip. - * @return whether this caused an auto-correction to happen. */ - private boolean handleSeparator(final InputTransaction inputTransaction, - final boolean isFromSuggestionStrip, + private void handleSeparatorEvent(final Event event, final InputTransaction inputTransaction, // TODO: remove this argument final LatinIME.UIHandler handler) { - final int codePoint = inputTransaction.mEvent.mCodePoint; + final int codePoint = event.mCodePoint; final SettingsValues settingsValues = inputTransaction.mSettingsValues; - boolean didAutoCorrect = false; final boolean wasComposingWord = mWordComposer.isComposingWord(); // We avoid sending spaces in languages without spaces if we were composing. final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == codePoint @@ -832,14 +869,14 @@ public final class InputLogic { final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR : StringUtils.newSingleCodePointString(codePoint); commitCurrentAutoCorrection(settingsValues, separator, handler); - didAutoCorrect = true; + inputTransaction.setDidAutoCorrect(); } else { commitTyped(settingsValues, StringUtils.newSingleCodePointString(codePoint)); } } - final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead( + final boolean swapWeakSpace = tryStripSpaceAndReturnWhetherShouldSwapInstead(event, inputTransaction); final boolean isInsideDoubleQuoteOrAfterDigit = Constants.CODE_DOUBLE_QUOTE == codePoint @@ -864,10 +901,10 @@ public final class InputLogic { promotePhantomSpace(settingsValues); } - if (tryPerformDoubleSpacePeriod(inputTransaction)) { + if (tryPerformDoubleSpacePeriod(event, inputTransaction)) { mSpaceState = SpaceState.DOUBLE; inputTransaction.setRequiresUpdateSuggestions(); - } else if (swapWeakSpace && trySwapSwapperAndSpace(inputTransaction)) { + } else if (swapWeakSpace && trySwapSwapperAndSpace(event, inputTransaction)) { mSpaceState = SpaceState.SWAP_PUNCTUATION; mSuggestionStripViewAccessor.setNeutralSuggestionStrip(); } else if (Constants.CODE_SPACE == codePoint) { @@ -910,17 +947,16 @@ public final class InputLogic { } inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW); - return didAutoCorrect; } /** * Handle a press on the backspace key. + * @param event The event to handle. * @param inputTransaction The transaction in progress. */ - private void handleBackspace(final InputTransaction inputTransaction, + private void handleBackspaceEvent(final Event event, final InputTransaction inputTransaction, // TODO: remove this argument, put it into settingsValues final int currentKeyboardScriptId) { - final Event event = inputTransaction.mEvent; mSpaceState = SpaceState.NONE; mDeleteCount++; @@ -1071,16 +1107,18 @@ public final class InputLogic { * * This method will check that there are two characters before the cursor and that the first * one is a space before it does the actual swapping. + * @param event The event to handle. * @param inputTransaction The transaction in progress. * @return true if the swap has been performed, false if it was prevented by preliminary checks. */ - private boolean trySwapSwapperAndSpace(final InputTransaction inputTransaction) { + private boolean trySwapSwapperAndSpace(final Event event, + final InputTransaction inputTransaction) { final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); if (Constants.CODE_SPACE != codePointBeforeCursor) { return false; } mConnection.deleteSurroundingText(1, 0); - final String text = inputTransaction.mEvent.getTextToCommit() + " "; + final String text = event.getTextToCommit() + " "; mConnection.commitText(text, 1); inputTransaction.requireShiftUpdate(InputTransaction.SHIFT_UPDATE_NOW); return true; @@ -1088,13 +1126,14 @@ public final class InputLogic { /* * Strip a trailing space if necessary and returns whether it's a swap weak space situation. + * @param event The event to handle. * @param inputTransaction The transaction in progress. * @return whether we should swap the space instead of removing it. */ - private boolean tryStripSpaceAndReturnWhetherShouldSwapInstead( + private boolean tryStripSpaceAndReturnWhetherShouldSwapInstead(final Event event, final InputTransaction inputTransaction) { - final int codePoint = inputTransaction.mEvent.mCodePoint; - final boolean isFromSuggestionStrip = inputTransaction.mEvent.isSuggestionStripPress(); + final int codePoint = event.mCodePoint; + final boolean isFromSuggestionStrip = event.isSuggestionStripPress(); if (Constants.CODE_ENTER == codePoint && SpaceState.SWAP_PUNCTUATION == inputTransaction.mSpaceState) { mConnection.removeTrailingSpace(); @@ -1139,14 +1178,16 @@ public final class InputLogic { * these conditions are fulfilled, this method applies the transformation and returns true. * Otherwise, it does nothing and returns false. * + * @param event The event to handle. * @param inputTransaction The transaction in progress. * @return true if we applied the double-space-to-period transformation, false otherwise. */ - private boolean tryPerformDoubleSpacePeriod(final InputTransaction inputTransaction) { + private boolean tryPerformDoubleSpacePeriod(final Event event, + final InputTransaction inputTransaction) { // Check the setting, the typed character and the countdown. If any of the conditions is // not fulfilled, return false. if (!inputTransaction.mSettingsValues.mUseDoubleSpacePeriod - || Constants.CODE_SPACE != inputTransaction.mEvent.mCodePoint + || Constants.CODE_SPACE != event.mCodePoint || !isDoubleSpacePeriodCountdownActive(inputTransaction)) { return false; } @@ -1974,14 +2015,13 @@ public final class InputLogic { * This method handles the retry, and re-schedules a new retry if we still can't access. * We only retry up to 5 times before giving up. * - * @param settingsValues the current values of the settings. * @param tryResumeSuggestions Whether we should resume suggestions or not. * @param remainingTries How many times we may try again before giving up. * @return whether true if the caches were successfully reset, false otherwise. */ // TODO: make this private - public boolean retryResetCachesAndReturnSuccess(final SettingsValues settingsValues, - final boolean tryResumeSuggestions, final int remainingTries, + public boolean retryResetCachesAndReturnSuccess(final boolean tryResumeSuggestions, + final int remainingTries, // TODO: remove these arguments final LatinIME.UIHandler handler) { final boolean shouldFinishComposition = mConnection.hasSelection() diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk index 1e3775b0a..7a732a588 100644 --- a/native/jni/NativeFileList.mk +++ b/native/jni/NativeFileList.mk @@ -73,7 +73,6 @@ LATIN_IME_CORE_SRC_FILES := \ $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \ bigram_dict_content.cpp \ language_model_dict_content.cpp \ - probability_dict_content.cpp \ shortcut_dict_content.cpp \ sparse_table_dict_content.cpp \ terminal_position_lookup_table.cpp) \ @@ -126,6 +125,8 @@ LATIN_IME_CORE_TEST_FILES := \ defines_test.cpp \ suggest/core/layout/normal_distribution_2d_test.cpp \ suggest/core/dictionary/bloom_filter_test.cpp \ + suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp \ + suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp \ suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer_test.cpp \ suggest/policyimpl/dictionary/utils/trie_map_test.cpp \ utils/autocorrection_threshold_utils_test.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp index b165bf4b7..07e1051bc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp @@ -22,4 +22,63 @@ bool LanguageModelDictContent::save(FILE *const file) const { return mTrieMap.save(file); } +bool LanguageModelDictContent::runGC( + const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, + const LanguageModelDictContent *const originalContent, + int *const outNgramCount) { + return runGCInner(terminalIdMap, originalContent->mTrieMap.getEntriesInRootLevel(), + 0 /* nextLevelBitmapEntryIndex */, outNgramCount); +} + +ProbabilityEntry LanguageModelDictContent::getNgramProbabilityEntry( + const WordIdArrayView prevWordIds, const int wordId) const { + if (!prevWordIds.empty()) { + // TODO: Read n-gram entry. + return ProbabilityEntry(); + } + const TrieMap::Result result = mTrieMap.getRoot(wordId); + if (!result.mIsValid) { + // Not found. + return ProbabilityEntry(); + } + return ProbabilityEntry::decode(result.mValue, mHasHistoricalInfo); +} + +bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView prevWordIds, + const int terminalId, const ProbabilityEntry *const probabilityEntry) { + if (!prevWordIds.empty()) { + // TODO: Add n-gram entry. + return false; + } + return mTrieMap.putRoot(terminalId, probabilityEntry->encode(mHasHistoricalInfo)); +} + + +bool LanguageModelDictContent::runGCInner( + const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, + const TrieMap::TrieMapRange trieMapRange, + const int nextLevelBitmapEntryIndex, int *const outNgramCount) { + for (auto &entry : trieMapRange) { + const auto it = terminalIdMap->find(entry.key()); + if (it == terminalIdMap->end() || it->second == Ver4DictConstants::NOT_A_TERMINAL_ID) { + // The word has been removed. + continue; + } + if (!mTrieMap.put(it->second, entry.value(), nextLevelBitmapEntryIndex)) { + return false; + } + if (outNgramCount) { + *outNgramCount += 1; + } + if (entry.hasNextLevelMap()) { + if (!runGCInner(terminalIdMap, entry.getEntriesInNextLevel(), + mTrieMap.getNextLevelBitmapEntryIndex(it->second, nextLevelBitmapEntryIndex), + outNgramCount)) { + return false; + } + } + } + return true; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h index 263911380..f181dfeee 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h @@ -20,25 +20,62 @@ #include <cstdio> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/trie_map.h" #include "utils/byte_array_view.h" +#include "utils/int_array_view.h" namespace latinime { +/** + * Class representing language model. + * + * This class provides methods to get and store unigram/n-gram probability information and flags. + */ class LanguageModelDictContent { public: LanguageModelDictContent(const ReadWriteByteArrayView trieMapBuffer, const bool hasHistoricalInfo) - : mTrieMap(trieMapBuffer) {} + : mTrieMap(trieMapBuffer), mHasHistoricalInfo(hasHistoricalInfo) {} + + explicit LanguageModelDictContent(const bool hasHistoricalInfo) + : mTrieMap(), mHasHistoricalInfo(hasHistoricalInfo) {} - explicit LanguageModelDictContent(const bool hasHistoricalInfo) : mTrieMap() {} + bool isNearSizeLimit() const { + return mTrieMap.isNearSizeLimit(); + } bool save(FILE *const file) const; + bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, + const LanguageModelDictContent *const originalContent, + int *const outNgramCount); + + ProbabilityEntry getProbabilityEntry(const int wordId) const { + return getNgramProbabilityEntry(WordIdArrayView(), wordId); + } + + bool setProbabilityEntry(const int wordId, const ProbabilityEntry *const probabilityEntry) { + return setNgramProbabilityEntry(WordIdArrayView(), wordId, probabilityEntry); + } + + ProbabilityEntry getNgramProbabilityEntry(const WordIdArrayView prevWordIds, + const int wordId) const; + + bool setNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId, + const ProbabilityEntry *const probabilityEntry); + private: DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent); TrieMap mTrieMap; + const bool mHasHistoricalInfo; + + bool runGCInner(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, + const TrieMap::TrieMapRange trieMapRange, const int nextLevelBitmapEntryIndex, + int *const outNgramCount); }; } // namespace latinime #endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp deleted file mode 100644 index 2425b3b2f..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" - -#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" -#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" -#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" - -namespace latinime { - -const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const { - if (terminalId < 0 || terminalId >= mSize) { - // This method can be called with invalid terminal id during GC. - return ProbabilityEntry(0 /* flags */, NOT_A_PROBABILITY); - } - const BufferWithExtendableBuffer *const buffer = getBuffer(); - int entryPos = getEntryPos(terminalId); - const int flags = buffer->readUintAndAdvancePosition( - Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &entryPos); - const int probability = buffer->readUintAndAdvancePosition( - Ver4DictConstants::PROBABILITY_SIZE, &entryPos); - if (mHasHistoricalInfo) { - const int timestamp = buffer->readUintAndAdvancePosition( - Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &entryPos); - const int level = buffer->readUintAndAdvancePosition( - Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &entryPos); - const int count = buffer->readUintAndAdvancePosition( - Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &entryPos); - const HistoricalInfo historicalInfo(timestamp, level, count); - return ProbabilityEntry(flags, probability, &historicalInfo); - } else { - return ProbabilityEntry(flags, probability); - } -} - -bool ProbabilityDictContent::setProbabilityEntry(const int terminalId, - const ProbabilityEntry *const probabilityEntry) { - if (terminalId < 0) { - return false; - } - const int entryPos = getEntryPos(terminalId); - if (terminalId >= mSize) { - ProbabilityEntry dummyEntry; - // Write new entry. - int writingPos = getBuffer()->getTailPosition(); - while (writingPos <= entryPos) { - // Fulfilling with dummy entries until writingPos. - if (!writeEntry(&dummyEntry, writingPos)) { - AKLOGE("Cannot write dummy entry. pos: %d, mSize: %d", writingPos, mSize); - return false; - } - writingPos += getEntrySize(); - mSize++; - } - } - return writeEntry(probabilityEntry, entryPos); -} - -bool ProbabilityDictContent::flushToFile(FILE *const file) const { - if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { - ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo); - for (int i = 0; i < mSize; ++i) { - const ProbabilityEntry probabilityEntry = getProbabilityEntry(i); - if (!probabilityDictContentToWrite.setProbabilityEntry(i, &probabilityEntry)) { - AKLOGE("Cannot set probability entry in flushToFile. terminalId: %d", i); - return false; - } - } - return probabilityDictContentToWrite.flush(file); - } else { - return flush(file); - } -} - -bool ProbabilityDictContent::runGC( - const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, - const ProbabilityDictContent *const originalProbabilityDictContent) { - mSize = 0; - for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin(); - it != terminalIdMap->end(); ++it) { - const ProbabilityEntry probabilityEntry = - originalProbabilityDictContent->getProbabilityEntry(it->first); - if (!setProbabilityEntry(it->second, &probabilityEntry)) { - AKLOGE("Cannot set probability entry in runGC. terminalId: %d", it->second); - return false; - } - mSize++; - } - return true; -} - -int ProbabilityDictContent::getEntrySize() const { - if (mHasHistoricalInfo) { - return Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE - + Ver4DictConstants::PROBABILITY_SIZE - + Ver4DictConstants::TIME_STAMP_FIELD_SIZE - + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE - + Ver4DictConstants::WORD_COUNT_FIELD_SIZE; - } else { - return Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE - + Ver4DictConstants::PROBABILITY_SIZE; - } -} - -int ProbabilityDictContent::getEntryPos(const int terminalId) const { - return terminalId * getEntrySize(); -} - -bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilityEntry, - const int entryPos) { - BufferWithExtendableBuffer *const bufferToWrite = getWritableBuffer(); - int writingPos = entryPos; - if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getFlags(), - Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &writingPos)) { - AKLOGE("Cannot write flags in probability dict content. pos: %d", writingPos); - return false; - } - if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getProbability(), - Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) { - AKLOGE("Cannot write probability in probability dict content. pos: %d", writingPos); - return false; - } - if (mHasHistoricalInfo) { - const HistoricalInfo *const historicalInfo = probabilityEntry->getHistoricalInfo(); - if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(), - Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &writingPos)) { - AKLOGE("Cannot write timestamp in probability dict content. pos: %d", writingPos); - return false; - } - if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getLevel(), - Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &writingPos)) { - AKLOGE("Cannot write level in probability dict content. pos: %d", writingPos); - return false; - } - if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getCount(), - Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &writingPos)) { - AKLOGE("Cannot write count in probability dict content. pos: %d", writingPos); - return false; - } - } - return true; -} - -} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h deleted file mode 100644 index 80e992c1c..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2013, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_PROBABILITY_DICT_CONTENT_H -#define LATINIME_PROBABILITY_DICT_CONTENT_H - -#include <cstdint> -#include <cstdio> - -#include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" -#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" - -namespace latinime { - -class ProbabilityEntry; - -class ProbabilityDictContent : public SingleDictContent { - public: - ProbabilityDictContent(uint8_t *const buffer, const int bufferSize, - const bool hasHistoricalInfo) - : SingleDictContent(buffer, bufferSize), - mHasHistoricalInfo(hasHistoricalInfo), - mSize(getBuffer()->getTailPosition() / getEntrySize()) {} - - ProbabilityDictContent(const bool hasHistoricalInfo) - : mHasHistoricalInfo(hasHistoricalInfo), mSize(0) {} - - const ProbabilityEntry getProbabilityEntry(const int terminalId) const; - - bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry); - - bool flushToFile(FILE *const file) const; - - bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, - const ProbabilityDictContent *const originalProbabilityDictContent); - - private: - DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent); - - int getEntrySize() const; - - int getEntryPos(const int terminalId) const; - - bool writeEntry(const ProbabilityEntry *const probabilityEntry, const int entryPos); - - bool mHasHistoricalInfo; - int mSize; -}; -} // namespace latinime -#endif /* LATINIME_PROBABILITY_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h index 36ba82be1..feff6b57f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h @@ -17,6 +17,9 @@ #ifndef LATINIME_PROBABILITY_ENTRY_H #define LATINIME_PROBABILITY_ENTRY_H +#include <climits> +#include <cstdint> + #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/historical_info.h" @@ -67,6 +70,50 @@ class ProbabilityEntry { return &mHistoricalInfo; } + uint64_t encode(const bool hasHistoricalInfo) const { + uint64_t encodedEntry = static_cast<uint64_t>(mFlags); + if (hasHistoricalInfo) { + encodedEntry = (encodedEntry << (Ver4DictConstants::TIME_STAMP_FIELD_SIZE * CHAR_BIT)) + ^ static_cast<uint64_t>(mHistoricalInfo.getTimeStamp()); + encodedEntry = (encodedEntry << (Ver4DictConstants::WORD_LEVEL_FIELD_SIZE * CHAR_BIT)) + ^ static_cast<uint64_t>(mHistoricalInfo.getLevel()); + encodedEntry = (encodedEntry << (Ver4DictConstants::WORD_COUNT_FIELD_SIZE * CHAR_BIT)) + ^ static_cast<uint64_t>(mHistoricalInfo.getCount()); + } else { + encodedEntry = (encodedEntry << (Ver4DictConstants::PROBABILITY_SIZE * CHAR_BIT)) + ^ static_cast<uint64_t>(mProbability); + } + return encodedEntry; + } + + static ProbabilityEntry decode(const uint64_t encodedEntry, const bool hasHistoricalInfo) { + if (hasHistoricalInfo) { + const int flags = readFromEncodedEntry(encodedEntry, + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, + Ver4DictConstants::TIME_STAMP_FIELD_SIZE + + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE + + Ver4DictConstants::WORD_COUNT_FIELD_SIZE); + const int timestamp = readFromEncodedEntry(encodedEntry, + Ver4DictConstants::TIME_STAMP_FIELD_SIZE, + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE + + Ver4DictConstants::WORD_COUNT_FIELD_SIZE); + const int level = readFromEncodedEntry(encodedEntry, + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, + Ver4DictConstants::WORD_COUNT_FIELD_SIZE); + const int count = readFromEncodedEntry(encodedEntry, + Ver4DictConstants::WORD_COUNT_FIELD_SIZE, 0 /* pos */); + const HistoricalInfo historicalInfo(timestamp, level, count); + return ProbabilityEntry(flags, NOT_A_PROBABILITY, &historicalInfo); + } else { + const int flags = readFromEncodedEntry(encodedEntry, + Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, + Ver4DictConstants::PROBABILITY_SIZE); + const int probability = readFromEncodedEntry(encodedEntry, + Ver4DictConstants::PROBABILITY_SIZE, 0 /* pos */); + return ProbabilityEntry(flags, probability); + } + } + private: // Copy constructor is public to use this class as a type of return value. DISALLOW_ASSIGNMENT_OPERATOR(ProbabilityEntry); @@ -74,6 +121,11 @@ class ProbabilityEntry { const int mFlags; const int mProbability; const HistoricalInfo mHistoricalInfo; + + static int readFromEncodedEntry(const uint64_t encodedEntry, const int size, const int pos) { + return static_cast<int>( + (encodedEntry >> (pos * CHAR_BIT)) & ((1ull << (size * CHAR_BIT)) - 1)); + } }; } // namespace latinime #endif /* LATINIME_PROBABILITY_ENTRY_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index 125ae1711..3c8008dc4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -157,11 +157,6 @@ bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const { AKLOGE("Terminal position lookup table cannot be written."); return false; } - // Write probability dict content. - if (!mProbabilityDictContent.flushToFile(file)) { - AKLOGE("Probability dict content cannot be written."); - return false; - } // Write language model content. if (!mLanguageModelDictContent.save(file)) { AKLOGE("Language model dict content cannot be written."); @@ -196,10 +191,6 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX], contentBufferSizes[ Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX]), - mProbabilityDictContent( - contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], - contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], - mHeaderPolicy.hasHistoricalInfoOfWords()), mLanguageModelDictContent( ReadWriteByteArrayView( contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX], @@ -216,7 +207,6 @@ Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const i : mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(headerPolicy), mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableTrieBuffer(maxTrieSize), mTerminalPositionLookupTable(), - mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()), mLanguageModelDictContent(headerPolicy->hasHistoricalInfoOfWords()), mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(), mIsUpdatable(true) {} diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index 36671bc0f..68027dcb8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -24,7 +24,6 @@ #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" @@ -53,7 +52,7 @@ class Ver4DictBuffers { AK_FORCE_INLINE bool isNearSizeLimit() const { return mExpandableTrieBuffer.isNearSizeLimit() || mTerminalPositionLookupTable.isNearSizeLimit() - || mProbabilityDictContent.isNearSizeLimit() + || mLanguageModelDictContent.isNearSizeLimit() || mBigramDictContent.isNearSizeLimit() || mShortcutDictContent.isNearSizeLimit(); } @@ -82,12 +81,12 @@ class Ver4DictBuffers { return &mTerminalPositionLookupTable; } - AK_FORCE_INLINE ProbabilityDictContent *getMutableProbabilityDictContent() { - return &mProbabilityDictContent; + AK_FORCE_INLINE LanguageModelDictContent *getMutableLanguageModelDictContent() { + return &mLanguageModelDictContent; } - AK_FORCE_INLINE const ProbabilityDictContent *getProbabilityDictContent() const { - return &mProbabilityDictContent; + AK_FORCE_INLINE const LanguageModelDictContent *getLanguageModelDictContent() const { + return &mLanguageModelDictContent; } AK_FORCE_INLINE BigramDictContent *getMutableBigramDictContent() { @@ -136,7 +135,6 @@ class Ver4DictBuffers { BufferWithExtendableBuffer mExpandableHeaderBuffer; BufferWithExtendableBuffer mExpandableTrieBuffer; TerminalPositionLookupTable mTerminalPositionLookupTable; - ProbabilityDictContent mProbabilityDictContent; LanguageModelDictContent mLanguageModelDictContent; BigramDictContent mBigramDictContent; ShortcutDictContent mShortcutDictContent; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index e7e31e96f..93d4e562d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -27,19 +27,18 @@ const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024; // limited to 1MB to prevent from inefficient traversing. const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024; -// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie, TerminalAddressLookupTable and Probability. +// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie and TerminalAddressLookupTable. +// NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT for language model. // NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut. const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE = - NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3 + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 2 + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2; const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0; const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX = TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; -const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX = - TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX = - PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; + TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX = LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT; const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX = diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index e75db9f75..6950ca70f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -35,7 +35,6 @@ class Ver4DictConstants { static const size_t NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE; static const int TRIE_BUFFER_INDEX; static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX; - static const int PROBABILITY_BUFFER_INDEX; static const int LANGUAGE_MODEL_BUFFER_INDEX; static const int BIGRAM_BUFFERS_INDEX; static const int SHORTCUT_BUFFERS_INDEX; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 0a435e91c..731092efd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -18,7 +18,7 @@ #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -61,8 +61,9 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce terminalIdFieldPos += mBuffer->getOriginalBufferSize(); } terminalId = Ver4PatriciaTrieReadingUtils::getTerminalIdAndAdvancePosition(dictBuf, &pos); + // TODO: Quit reading probability here. const ProbabilityEntry probabilityEntry = - mProbabilityDictContent->getProbabilityEntry(terminalId); + mLanguageModelDictContent->getProbabilityEntry(terminalId); if (probabilityEntry.hasHistoricalInfo()) { probability = ForgettingCurveUtils::decodeProbability( probabilityEntry.getHistoricalInfo(), mHeaderPolicy); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h index 22ed4a6c0..a91ad5728 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h @@ -25,18 +25,18 @@ namespace latinime { class BufferWithExtendableBuffer; class HeaderPolicy; -class ProbabilityDictContent; +class LanguageModelDictContent; /* * This class is used for helping to read nodes of ver4 patricia trie. This class handles moved - * node and reads node attributes including probability form probabilityBuffer. + * node and reads node attributes including probability form language model. */ class Ver4PatriciaTrieNodeReader : public PtNodeReader { public: Ver4PatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer, - const ProbabilityDictContent *const probabilityDictContent, + const LanguageModelDictContent *const languageModelDictContent, const HeaderPolicy *const headerPolicy) - : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent), + : mBuffer(buffer), mLanguageModelDictContent(languageModelDictContent), mHeaderPolicy(headerPolicy) {} ~Ver4PatriciaTrieNodeReader() {} @@ -50,7 +50,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader { DISALLOW_COPY_AND_ASSIGN(Ver4PatriciaTrieNodeReader); const BufferWithExtendableBuffer *const mBuffer; - const ProbabilityDictContent *const mProbabilityDictContent; + const LanguageModelDictContent *const mLanguageModelDictContent; const HeaderPolicy *const mHeaderPolicy; const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 3d8da9173..1a311b156 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -143,11 +143,11 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeUnigramProperty( return false; } const ProbabilityEntry originalProbabilityEntry = - mBuffers->getProbabilityDictContent()->getProbabilityEntry( + mBuffers->getLanguageModelDictContent()->getProbabilityEntry( toBeUpdatedPtNodeParams->getTerminalId()); const ProbabilityEntry probabilityEntry = createUpdatedEntryFrom(&originalProbabilityEntry, unigramProperty); - return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( + return mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry( toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry); } @@ -158,14 +158,14 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA return false; } const ProbabilityEntry originalProbabilityEntry = - mBuffers->getProbabilityDictContent()->getProbabilityEntry( + mBuffers->getLanguageModelDictContent()->getProbabilityEntry( toBeUpdatedPtNodeParams->getTerminalId()); if (originalProbabilityEntry.hasHistoricalInfo()) { const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy); const ProbabilityEntry probabilityEntry = originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo); - if (!mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( + if (!mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry( toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry)) { AKLOGE("Cannot write updated probability entry. terminalId: %d", toBeUpdatedPtNodeParams->getTerminalId()); @@ -218,8 +218,8 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition( ProbabilityEntry newProbabilityEntry; const ProbabilityEntry probabilityEntryToWrite = createUpdatedEntryFrom( &newProbabilityEntry, unigramProperty); - return mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry(terminalId, - &probabilityEntryToWrite); + return mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry( + terminalId, &probabilityEntryToWrite); } bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry( diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 4bf8050e1..2b92d5bea 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -452,7 +452,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code std::vector<int> codePointVector(ptNodeParams.getCodePoints(), ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount()); const ProbabilityEntry probabilityEntry = - mBuffers->getProbabilityDictContent()->getProbabilityEntry( + mBuffers->getLanguageModelDictContent()->getProbabilityEntry( ptNodeParams.getTerminalId()); const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo(); // Fetch bigram information. diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 76b3404c4..faad4290d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -46,7 +46,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { mBuffers->getTerminalPositionLookupTable(), mHeaderPolicy), mShortcutPolicy(mBuffers->getMutableShortcutDictContent(), mBuffers->getTerminalPositionLookupTable()), - mNodeReader(mDictBuffer, mBuffers->getProbabilityDictContent(), mHeaderPolicy), + mNodeReader(mDictBuffer, mBuffers->getLanguageModelDictContent(), mHeaderPolicy), mPtNodeArrayReader(mDictBuffer), mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader, &mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 0e658f8e3..4220312e0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -75,7 +75,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount, int *const outBigramCount) { Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(), - mBuffers->getProbabilityDictContent(), headerPolicy); + mBuffers->getLanguageModelDictContent(), headerPolicy); Ver4PtNodeArrayReader ptNodeArrayReader(mBuffers->getTrieBuffer()); Ver4BigramListPolicy bigramPolicy(mBuffers->getMutableBigramDictContent(), mBuffers->getTerminalPositionLookupTable(), headerPolicy); @@ -138,7 +138,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, // Create policy instances for the GCed dictionary. Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(), - buffersToWrite->getProbabilityDictContent(), headerPolicy); + buffersToWrite->getLanguageModelDictContent(), headerPolicy); Ver4PtNodeArrayReader newPtNodeArrayreader(buffersToWrite->getTrieBuffer()); Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getMutableBigramDictContent(), buffersToWrite->getTerminalPositionLookupTable(), headerPolicy); @@ -154,8 +154,8 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, return false; } // Run GC for probability dict content. - if (!buffersToWrite->getMutableProbabilityDictContent()->runGC(&terminalIdMap, - mBuffers->getProbabilityDictContent())) { + if (!buffersToWrite->getMutableLanguageModelDictContent()->runGC(&terminalIdMap, + mBuffers->getLanguageModelDictContent(), nullptr /* outNgramCount */)) { return false; } // Run GC for bigram dict content. @@ -201,7 +201,7 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams( continue; } const ProbabilityEntry probabilityEntry = - mBuffers->getProbabilityDictContent()->getProbabilityEntry(i); + mBuffers->getLanguageModelDictContent()->getProbabilityEntry(i); const int probability = probabilityEntry.hasHistoricalInfo() ? ForgettingCurveUtils::decodeProbability( probabilityEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) : diff --git a/native/jni/src/utils/int_array_view.h b/native/jni/src/utils/int_array_view.h index 3ff01f5d0..4bc2487ae 100644 --- a/native/jni/src/utils/int_array_view.h +++ b/native/jni/src/utils/int_array_view.h @@ -61,6 +61,10 @@ class IntArrayView { return mPtr[index]; } + AK_FORCE_INLINE bool empty() const { + return size() == 0; + } + AK_FORCE_INLINE size_t size() const { return mSize; } @@ -76,5 +80,7 @@ class IntArrayView { const size_t mSize; }; +using WordIdArrayView = IntArrayView; + } // namespace latinime #endif // LATINIME_MEMORY_VIEW_H diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp new file mode 100644 index 000000000..6eef2040b --- /dev/null +++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h" + +#include <gtest/gtest.h> + +#include "utils/int_array_view.h" + +namespace latinime { +namespace { + +TEST(LanguageModelDictContentTest, TestUnigramProbability) { + LanguageModelDictContent LanguageModelDictContent(false /* useHistoricalInfo */); + + const int flag = 0xFF; + const int probability = 10; + const int wordId = 100; + const ProbabilityEntry probabilityEntry(flag, probability); + LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry); + const ProbabilityEntry entry = + LanguageModelDictContent.getProbabilityEntry(wordId); + EXPECT_EQ(flag, entry.getFlags()); + EXPECT_EQ(probability, entry.getProbability()); +} + +TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) { + LanguageModelDictContent LanguageModelDictContent(true /* useHistoricalInfo */); + + const int flag = 0xF0; + const int timestamp = 0x3FFFFFFF; + const int level = 3; + const int count = 10; + const int wordId = 100; + const HistoricalInfo historicalInfo(timestamp, level, count); + const ProbabilityEntry probabilityEntry(flag, NOT_A_PROBABILITY, &historicalInfo); + LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry); + const ProbabilityEntry entry = LanguageModelDictContent.getProbabilityEntry(wordId); + EXPECT_EQ(flag, entry.getFlags()); + EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimeStamp()); + EXPECT_EQ(level, entry.getHistoricalInfo()->getLevel()); + EXPECT_EQ(count, entry.getHistoricalInfo()->getCount()); +} + +} // namespace +} // namespace latinime diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp new file mode 100644 index 000000000..db94550ef --- /dev/null +++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/probability_entry_test.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" + +#include <gtest/gtest.h> + +#include "defines.h" + +namespace latinime { +namespace { + +TEST(ProbabilityEntryTest, TestEncodeDecode) { + const int flag = 0xFF; + const int probability = 10; + + const ProbabilityEntry entry(flag, probability); + const uint64_t encodedEntry = entry.encode(false /* hasHistoricalInfo */); + const ProbabilityEntry decodedEntry = + ProbabilityEntry::decode(encodedEntry, false /* hasHistoricalInfo */); + EXPECT_EQ(0xFF0Aull, encodedEntry); + EXPECT_EQ(flag, decodedEntry.getFlags()); + EXPECT_EQ(probability, decodedEntry.getProbability()); +} + +TEST(ProbabilityEntryTest, TestEncodeDecodeWithHistoricalInfo) { + const int flag = 0xF0; + const int timestamp = 0x3FFFFFFF; + const int level = 3; + const int count = 10; + + const HistoricalInfo historicalInfo(timestamp, level, count); + const ProbabilityEntry entry(flag, NOT_A_PROBABILITY, &historicalInfo); + + const uint64_t encodedEntry = entry.encode(true /* hasHistoricalInfo */); + EXPECT_EQ(0xF03FFFFFFF030Aull, encodedEntry); + const ProbabilityEntry decodedEntry = + ProbabilityEntry::decode(encodedEntry, true /* hasHistoricalInfo */); + + EXPECT_EQ(flag, decodedEntry.getFlags()); + EXPECT_EQ(timestamp, decodedEntry.getHistoricalInfo()->getTimeStamp()); + EXPECT_EQ(level, decodedEntry.getHistoricalInfo()->getLevel()); + EXPECT_EQ(count, decodedEntry.getHistoricalInfo()->getCount()); +} + +} // namespace +} // namespace latinime |