diff options
Diffstat (limited to 'java/src/com/android')
7 files changed, 48 insertions, 835 deletions
diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java index b975b9c70..c8c4d30ef 100644 --- a/java/src/com/android/inputmethod/keyboard/Key.java +++ b/java/src/com/android/inputmethod/keyboard/Key.java @@ -348,8 +348,7 @@ public class Key implements Comparable<Key> { if (StringUtils.codePointCount(mLabel) == 1) { // Use the first letter of the hint label if shiftedLetterActivated flag is // specified. - if (hasShiftedLetterHint() && isShiftedLetterActivated() - && !TextUtils.isEmpty(mHintLabel)) { + if (hasShiftedLetterHint() && isShiftedLetterActivated()) { mCode = mHintLabel.codePointAt(0); } else { mCode = mLabel.codePointAt(0); @@ -687,7 +686,8 @@ public class Key implements Comparable<Key> { } public final boolean hasShiftedLetterHint() { - return (mLabelFlags & LABEL_FLAGS_HAS_SHIFTED_LETTER_HINT) != 0; + return (mLabelFlags & LABEL_FLAGS_HAS_SHIFTED_LETTER_HINT) != 0 + && !TextUtils.isEmpty(mHintLabel); } public final boolean hasHintLabel() { @@ -710,8 +710,9 @@ public class Key implements Comparable<Key> { return (mLabelFlags & LABEL_FLAGS_AUTO_SCALE) == LABEL_FLAGS_AUTO_SCALE; } - public final boolean isShiftedLetterActivated() { - return (mLabelFlags & LABEL_FLAGS_SHIFTED_LETTER_ACTIVATED) != 0; + private final boolean isShiftedLetterActivated() { + return (mLabelFlags & LABEL_FLAGS_SHIFTED_LETTER_ACTIVATED) != 0 + && !TextUtils.isEmpty(mHintLabel); } public final int getMoreKeysColumn() { diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java index dd98c1703..0c80ce206 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java @@ -304,6 +304,7 @@ public final class KeyboardState { mSwitchActions.setSymbolsKeyboard(); mIsAlphabetMode = false; mIsSymbolShifted = false; + mRecapitalizeMode = RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE; // Reset alphabet shift state. mAlphabetShiftState.setShiftLocked(false); mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; @@ -316,6 +317,7 @@ public final class KeyboardState { mSwitchActions.setSymbolsShiftedKeyboard(); mIsAlphabetMode = false; mIsSymbolShifted = true; + mRecapitalizeMode = RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE; // Reset alphabet shift state. mAlphabetShiftState.setShiftLocked(false); mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; @@ -327,6 +329,7 @@ public final class KeyboardState { } mIsAlphabetMode = false; mIsEmojiMode = true; + mRecapitalizeMode = RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE; // Remember caps lock mode and reset alphabet shift state. mPrevMainKeyboardWasShiftLocked = mAlphabetShiftState.isShiftLocked(); mAlphabetShiftState.setShiftLocked(false); diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index d2f254468..8b466559c 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -100,7 +100,6 @@ import com.android.inputmethod.latin.utils.RecapitalizeStatus; import com.android.inputmethod.latin.utils.StringUtils; import com.android.inputmethod.latin.utils.TargetPackageInfoGetterTask; import com.android.inputmethod.latin.utils.TextRange; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; import com.android.inputmethod.research.ResearchLogger; import java.io.FileDescriptor; @@ -1019,11 +1018,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Only for debug mBoostPersonalizationDictionaryForDebug = currentSettingsValues.mBoostPersonalizationDictionaryForDebug; - if (mBoostPersonalizationDictionaryForDebug) { - UserHistoryForgettingCurveUtils.boostMaxFreqForDebug(); - } else { - UserHistoryForgettingCurveUtils.resetMaxFreqForDebug(); - } } } @@ -1800,10 +1794,19 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); if (Character.isLetterOrDigit(codePointBeforeCursor) || currentSettingsValues.isUsuallyFollowedBySpace(codePointBeforeCursor)) { + final boolean autoShiftHasBeenOverriden = mKeyboardSwitcher.getKeyboardShiftMode() != + getCurrentAutoCapsState(); mSpaceState = SPACE_STATE_PHANTOM; + if (!autoShiftHasBeenOverriden) { + // When we change the space state, we need to update the shift state of the + // keyboard unless it has been overridden manually. This is happening for example + // after typing some letters and a period, then gesturing; the keyboard is not in + // caps mode yet, but since a gesture is starting, it should go in caps mode, + // unless the user explictly said it should not. + mKeyboardSwitcher.updateShiftState(); + } } mConnection.endBatchEdit(); - mKeyboardSwitcher.updateShiftState(); mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime(getActualCapsMode(), // Prev word is 1st word before cursor getNthPreviousWordForSuggestion(currentSettingsValues, 1 /* nthPreviousWord */)); diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java deleted file mode 100644 index 97ad667a6..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ /dev/null @@ -1,492 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; - -/** - * The utility class to help dynamic updates on the binary dictionary. - * - * All the methods in this class are static. - */ -@UsedForTesting -public final class DynamicBinaryDictIOUtils { - private static final boolean DBG = false; - static final int MAX_JUMPS = 10000; - - private DynamicBinaryDictIOUtils() { - // This utility class is not publicly instantiable. - } - - /* package */ static int markAsDeleted(final int flags) { - return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; - } - - /** - * Update a parent address in a PtNode that is referred to by ptNodeOriginAddress. - * - * @param dictUpdater the DictUpdater to write. - * @param ptNodeOriginAddress the address of the PtNode. - * @param newParentAddress the absolute address of the parent. - * @param formatOptions file format options. - */ - private static void updateParentAddress(final Ver2DictUpdater dictUpdater, - final int ptNodeOriginAddress, final int newParentAddress, - final FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); - final int originalPosition = dictBuffer.position(); - dictBuffer.position(ptNodeOriginAddress); - if (!formatOptions.supportsDynamicUpdate()) { - throw new RuntimeException("this file format does not support parent addresses"); - } - final int flags = dictBuffer.readUnsignedByte(); - if (BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { - // If the node is moved, the parent address is stored in the destination node. - // We are guaranteed to process the destination node later, so there is no need to - // update anything here. - dictBuffer.position(originalPosition); - return; - } - if (DBG) { - MakedictLog.d("update parent address flags=" + flags + ", " + ptNodeOriginAddress); - } - final int parentOffset = newParentAddress - ptNodeOriginAddress; - BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, parentOffset); - dictBuffer.position(originalPosition); - } - - /** - * Update parent addresses in a node array stored at ptNodeOriginAddress. - * - * @param dictUpdater the DictUpdater to be modified. - * @param ptNodeOriginAddress the address of the node array to update. - * @param newParentAddress the address to be written. - * @param formatOptions file format options. - */ - private static void updateParentAddresses(final Ver2DictUpdater dictUpdater, - final int ptNodeOriginAddress, final int newParentAddress, - final FormatOptions formatOptions) { - final int originalPosition = dictUpdater.getPosition(); - dictUpdater.setPosition(ptNodeOriginAddress); - do { - final int count = dictUpdater.readPtNodeCount(); - for (int i = 0; i < count; ++i) { - updateParentAddress(dictUpdater, dictUpdater.getPosition(), newParentAddress, - formatOptions); - dictUpdater.skipPtNode(formatOptions); - } - if (!dictUpdater.readAndFollowForwardLink()) break; - if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break; - } while (formatOptions.supportsDynamicUpdate()); - dictUpdater.setPosition(originalPosition); - } - - /** - * Update a children address in a PtNode that is addressed by ptNodeOriginAddress. - * - * @param dictUpdater the DictUpdater to write. - * @param ptNodeOriginAddress the address of the PtNode. - * @param newChildrenAddress the absolute address of the child. - * @param formatOptions file format options. - */ - private static void updateChildrenAddress(final Ver2DictUpdater dictUpdater, - final int ptNodeOriginAddress, final int newChildrenAddress, - final FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); - final int originalPosition = dictBuffer.position(); - dictBuffer.position(ptNodeOriginAddress); - final int flags = dictBuffer.readUnsignedByte(); - BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); - BinaryDictIOUtils.skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); - final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS - ? FormatSpec.NO_CHILDREN_ADDRESS : newChildrenAddress - dictBuffer.position(); - BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, childrenOffset); - dictBuffer.position(originalPosition); - } - - /** - * Helper method to move a PtNode to the tail of the file. - */ - private static int movePtNode(final OutputStream destination, - final Ver2DictUpdater dictUpdater, final PtNodeInfo info, - final int nodeArrayOriginAddress, final int oldNodeAddress, - final FormatOptions formatOptions) throws IOException { - final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); - updateParentAddress(dictUpdater, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); - dictBuffer.position(oldNodeAddress); - final int currentFlags = dictBuffer.readUnsignedByte(); - dictBuffer.position(oldNodeAddress); - dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags - & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); - int size = FormatSpec.PTNODE_FLAGS_SIZE; - updateForwardLink(dictUpdater, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); - size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info }); - return size; - } - - @SuppressWarnings("unused") - private static void updateForwardLink(final Ver2DictUpdater dictUpdater, - final int nodeArrayOriginAddress, final int newNodeArrayAddress, - final FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); - dictUpdater.setPosition(nodeArrayOriginAddress); - int jumpCount = 0; - while (jumpCount++ < MAX_JUMPS) { - final int count = dictUpdater.readPtNodeCount(); - for (int i = 0; i < count; ++i) { - dictUpdater.readPtNode(dictUpdater.getPosition(), formatOptions); - } - final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); - if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { - dictBuffer.position(dictBuffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE); - BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeArrayAddress); - return; - } - dictBuffer.position(forwardLinkAddress); - } - if (DBG && jumpCount >= MAX_JUMPS) { - throw new RuntimeException("too many jumps, probably a bug."); - } - } - - /** - * Move a PtNode that is referred to by oldPtNodeOrigin to the tail of the file, and set the - * children address to the byte after the PtNode. - * - * @param fileEndAddress the address of the tail of the file. - * @param codePoints the characters to put inside the PtNode. - * @param length how many code points to read from codePoints. - * @param flags the flags for this PtNode. - * @param frequency the frequency of this terminal. - * @param parentAddress the address of the parent PtNode of this PtNode. - * @param shortcutTargets the shortcut targets for this PtNode. - * @param bigrams the bigrams for this PtNode. - * @param destination the stream representing the tail of the file. - * @param dictUpdater the DictUpdater. - * @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of. - * @param oldPtNodeOrigin the old origin where this PtNode used to be stored. - * @param formatOptions format options for this dictionary. - * @return the size written, in bytes. - * @throws IOException if the file can't be accessed - */ - private static int movePtNode(final int fileEndAddress, final int[] codePoints, - final int length, final int flags, final int frequency, final int parentAddress, - final ArrayList<WeightedString> shortcutTargets, - final ArrayList<PendingAttribute> bigrams, final OutputStream destination, - final Ver2DictUpdater dictUpdater, final int oldPtNodeArrayOrigin, - final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException { - int size = 0; - final int newPtNodeOrigin = fileEndAddress + 1; - final int[] writtenCharacters = Arrays.copyOfRange(codePoints, 0, length); - final PtNodeInfo tmpInfo = new PtNodeInfo(newPtNodeOrigin, -1 /* endAddress */, - flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, - shortcutTargets, bigrams); - size = BinaryDictIOUtils.computePtNodeSize(tmpInfo, formatOptions); - final PtNodeInfo newInfo = new PtNodeInfo(newPtNodeOrigin, newPtNodeOrigin + size, - flags, writtenCharacters, frequency, parentAddress, - fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, - bigrams); - movePtNode(destination, dictUpdater, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, - formatOptions); - return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - - /** - * Converts a list of WeightedString to a list of PendingAttribute. - */ - public static ArrayList<PendingAttribute> resolveBigramPositions(final DictUpdater dictUpdater, - final ArrayList<WeightedString> bigramStrings) - throws IOException, UnsupportedFormatException { - if (bigramStrings == null) return CollectionUtils.newArrayList(); - final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); - for (final WeightedString bigram : bigramStrings) { - final int pos = dictUpdater.getTerminalPosition(bigram.mWord); - if (pos == FormatSpec.NOT_VALID_WORD) { - // TODO: figure out what is the correct thing to do here. - } else { - bigrams.add(new PendingAttribute(bigram.mFrequency, pos)); - } - } - return bigrams; - } - - /** - * Insert a word into a binary dictionary. - * - * @param dictUpdater the dict updater. - * @param destination a stream to the underlying file, with the pointer at the end of the file. - * @param word the word to insert. - * @param frequency the frequency of the new word. - * @param bigramStrings bigram list, or null if none. - * @param shortcuts shortcut list, or null if none. - * @param isBlackListEntry whether this should be a blacklist entry. - * @throws IOException if the file can't be accessed. - * @throws UnsupportedFormatException if the existing dictionary is in an unexpected format. - */ - // TODO: Support batch insertion. - // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. - @UsedForTesting - public static void insertWord(final Ver2DictUpdater dictUpdater, - final OutputStream destination, final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, - final boolean isBlackListEntry) - throws IOException, UnsupportedFormatException { - final ArrayList<PendingAttribute> bigrams = resolveBigramPositions(dictUpdater, - bigramStrings); - final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); - - final boolean isTerminal = true; - final boolean hasBigrams = !bigrams.isEmpty(); - final boolean hasShortcuts = shortcuts != null && !shortcuts.isEmpty(); - - // find the insert position of the word. - if (dictBuffer.position() != 0) dictBuffer.position(0); - final FileHeader fileHeader = dictUpdater.readHeader(); - - int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position(); - final int[] codePoints = FusionDictionary.getCodePoints(word); - final int wordLen = codePoints.length; - - for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { - if (wordPos >= wordLen) break; - nodeOriginAddress = dictBuffer.position(); - int nodeParentAddress = -1; - final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); - boolean foundNextNode = false; - - for (int i = 0; i < ptNodeCount; ++i) { - address = dictBuffer.position(); - final PtNodeInfo currentInfo = dictUpdater.readPtNode(address, - fileHeader.mFormatOptions); - final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, - fileHeader.mFormatOptions); - if (isMovedNode) continue; - nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) - ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; - boolean matched = true; - for (int p = 0; p < currentInfo.mCharacters.length; ++p) { - if (wordPos + p >= wordLen) { - /* - * splitting - * before - * abcd - ef - * - * insert "abc" - * - * after - * abc - d - ef - */ - final int newNodeAddress = dictBuffer.limit(); - final int flags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, - isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, - false /* isBlackListEntry */, fileHeader.mFormatOptions); - int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags, - frequency, nodeParentAddress, shortcuts, bigrams, destination, - dictUpdater, nodeOriginAddress, address, fileHeader.mFormatOptions); - - final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, - currentInfo.mCharacters.length); - if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, fileHeader.mFormatOptions); - } - final PtNodeInfo newInfo2 = new PtNodeInfo( - newNodeAddress + written + 1, -1 /* endAddress */, - currentInfo.mFlags, characters2, currentInfo.mFrequency, - newNodeAddress + 1, currentInfo.mChildrenAddress, - currentInfo.mShortcutTargets, currentInfo.mBigrams); - BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo2 }); - return; - } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { - if (p > 0) { - /* - * splitting - * before - * ab - cd - * - * insert "ac" - * - * after - * a - b - cd - * | - * - c - */ - - final int newNodeAddress = dictBuffer.limit(); - final int childrenAddress = currentInfo.mChildrenAddress; - - // move prefix - final int prefixFlags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, - false /* isTerminal */, 0 /* childrenAddressSize*/, - false /* hasShortcut */, false /* hasBigrams */, - false /* isNotAWord */, false /* isBlackListEntry */, - fileHeader.mFormatOptions); - int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, - prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, - destination, dictUpdater, nodeOriginAddress, address, - fileHeader.mFormatOptions); - - final int[] suffixCharacters = Arrays.copyOfRange( - currentInfo.mCharacters, p, currentInfo.mCharacters.length); - if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, fileHeader.mFormatOptions); - } - final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags( - suffixCharacters.length > 1, - (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, - 0 /* childrenAddressSize */, - (currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) - != 0, - (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, - isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final PtNodeInfo suffixInfo = new PtNodeInfo( - newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, - suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, - currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, - currentInfo.mBigrams); - written += BinaryDictIOUtils.computePtNodeSize(suffixInfo, - fileHeader.mFormatOptions) + 1; - - final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, - codePoints.length); - final int flags = BinaryDictEncoderUtils.makePtNodeFlags( - newCharacters.length > 1, isTerminal, - 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final PtNodeInfo newInfo = new PtNodeInfo( - newNodeAddress + written, -1 /* endAddress */, flags, - newCharacters, frequency, newNodeAddress + 1, - FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNodes(destination, - new PtNodeInfo[] { suffixInfo, newInfo }); - return; - } - matched = false; - break; - } - } - - if (matched) { - if (wordPos + currentInfo.mCharacters.length == wordLen) { - // the word exists in the dictionary. - // only update the PtNode. - final int newNodeAddress = dictBuffer.limit(); - final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, - -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, - nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, - bigrams); - movePtNode(destination, dictUpdater, newInfo, nodeOriginAddress, address, - fileHeader.mFormatOptions); - return; - } - wordPos += currentInfo.mCharacters.length; - if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { - /* - * found the prefix of the word. - * make new PtNode and link to the PtNode from this PtNode. - * - * before - * ab - cd - * - * insert "abcde" - * - * after - * ab - cd - e - */ - final int newNodeArrayAddress = dictBuffer.limit(); - updateChildrenAddress(dictUpdater, address, newNodeArrayAddress, - fileHeader.mFormatOptions); - final int newNodeAddress = newNodeArrayAddress + 1; - final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress, -1, flags, - characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, - shortcuts, bigrams); - BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo }); - return; - } - dictBuffer.position(currentInfo.mChildrenAddress); - foundNextNode = true; - break; - } - } - - if (foundNextNode) continue; - - // reached the end of the array. - final int linkAddressPosition = dictBuffer.position(); - int nextLink = dictBuffer.readUnsignedInt24(); - if ((nextLink & FormatSpec.MSB24) != 0) { - nextLink = -(nextLink & FormatSpec.SINT24_MAX); - } - if (nextLink == FormatSpec.NO_FORWARD_LINK_ADDRESS) { - /* - * expand this node. - * - * before - * ab - cd - * - * insert "abef" - * - * after - * ab - cd - * | - * - ef - */ - - // change the forward link address. - final int newNodeAddress = dictBuffer.limit(); - dictBuffer.position(linkAddressPosition); - BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); - - final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictEncoderUtils.makePtNodeFlags(characters.length > 1, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, - -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, - FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[]{ newInfo }); - return; - } else { - depth--; - dictBuffer.position(nextLink); - } - } - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictUpdater.java deleted file mode 100644 index 6419340ff..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictUpdater.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; - -/** - * An implementation of DictUpdater for version 2 binary dictionary. - */ -@UsedForTesting -public class Ver2DictUpdater extends Ver2DictDecoder implements DictUpdater { - private OutputStream mOutStream; - - @UsedForTesting - public Ver2DictUpdater(final File dictFile, final int factoryType) { - // DictUpdater must have an updatable DictBuffer. - super(dictFile, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) - ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); - mOutStream = null; - } - - private void openStreamAndBuffer() throws FileNotFoundException, IOException { - super.openDictBuffer(); - mOutStream = new FileOutputStream(mDictionaryBinaryFile, true /* append */); - } - - private void close() throws IOException { - if (mOutStream != null) { - mOutStream.close(); - mOutStream = null; - } - } - - @Override @UsedForTesting - public void deleteWord(final String word) throws IOException, UnsupportedFormatException { - if (mOutStream == null) openStreamAndBuffer(); - mDictBuffer.position(0); - readHeader(); - final int wordPos = getTerminalPosition(word); - if (wordPos != FormatSpec.NOT_VALID_WORD) { - mDictBuffer.position(wordPos); - final int flags = mDictBuffer.readUnsignedByte(); - mDictBuffer.position(wordPos); - mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags)); - } - close(); - } - - @Override @UsedForTesting - public void insertWord(final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, - final boolean isNotAWord, final boolean isBlackListEntry) - throws IOException, UnsupportedFormatException { - if (mOutStream == null) openStreamAndBuffer(); - DynamicBinaryDictIOUtils.insertWord(this, mOutStream, word, frequency, bigramStrings, - shortcuts, isNotAWord, isBlackListEntry); - close(); - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java index c46bc36bb..6298295c6 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -40,6 +40,7 @@ import java.util.Iterator; @UsedForTesting public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { private static final String TAG = Ver4DictUpdater.class.getSimpleName(); + private static final int MAX_JUMPS = 10000; private OutputStream mDictStream; private final File mFrequencyFile; @@ -146,7 +147,7 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { mDictBuffer.position(wordPos); final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); mDictBuffer.position(wordPos); - mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags)); + mDictBuffer.put((byte)markAsDeleted(flags)); } } @@ -183,7 +184,7 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { skipPtNode(formatOptions); } if (!readAndFollowForwardLink()) break; - } while (jumpCount++ < DynamicBinaryDictIOUtils.MAX_JUMPS); + } while (jumpCount++ < MAX_JUMPS); setPosition(originalPos); } @@ -216,7 +217,7 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { final int originalPos = getPosition(); setPosition(nodeArrayPos); int jumpCount = 0; - while (jumpCount++ < DynamicBinaryDictIOUtils.MAX_JUMPS) { + while (jumpCount++ < MAX_JUMPS) { final int ptNodeCount = readPtNodeCount(); for (int i = 0; i < ptNodeCount; ++i) { skipPtNode(formatOptions); @@ -735,8 +736,7 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { mDictBuffer.put((byte) newFlags); updateFrequency(terminalId, frequency); - insertBigrams(terminalId, frequency, - DynamicBinaryDictIOUtils.resolveBigramPositions(this, bigramStrings)); + insertBigrams(terminalId, frequency, resolveBigramPositions(this, bigramStrings)); insertShortcuts(terminalId, shortcuts); } @@ -765,8 +765,30 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { insertTerminalPosition(posOfTerminal); close(); - insertBigrams(newTerminalId, frequency, - DynamicBinaryDictIOUtils.resolveBigramPositions(this, bigramStrings)); + insertBigrams(newTerminalId, frequency, resolveBigramPositions(this, bigramStrings)); insertShortcuts(newTerminalId, shortcuts); } + + /** + * Converts a list of WeightedString to a list of PendingAttribute. + */ + private static ArrayList<PendingAttribute> resolveBigramPositions(final DictUpdater dictUpdater, + final ArrayList<WeightedString> bigramStrings) + throws IOException, UnsupportedFormatException { + if (bigramStrings == null) return CollectionUtils.newArrayList(); + final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); + for (final WeightedString bigram : bigramStrings) { + final int pos = dictUpdater.getTerminalPosition(bigram.mWord); + if (pos == FormatSpec.NOT_VALID_WORD) { + // TODO: figure out what is the correct thing to do here. + } else { + bigrams.add(new PendingAttribute(bigram.mFrequency, pos)); + } + } + return bigrams; + } + + private static int markAsDeleted(final int flags) { + return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; + } } diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java deleted file mode 100644 index 677035ed6..000000000 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.util.Log; - -import java.util.concurrent.TimeUnit; - -import com.android.inputmethod.annotations.UsedForTesting; - -@UsedForTesting -public final class UserHistoryForgettingCurveUtils { - private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); - private static final boolean DEBUG = false; - private static final int DEFAULT_FC_FREQ = 127; - private static final int BOOSTED_FC_FREQ = 200; - private static int FC_FREQ_MAX = DEFAULT_FC_FREQ; - /* package */ static final int COUNT_MAX = 3; - private static final int FC_LEVEL_MAX = 3; - /* package */ static final int ELAPSED_TIME_MAX = 15; - private static final int ELAPSED_TIME_INTERVAL_HOURS = 6; - private static final long ELAPSED_TIME_INTERVAL_MILLIS = - TimeUnit.HOURS.toMillis(ELAPSED_TIME_INTERVAL_HOURS); - private static final int HALF_LIFE_HOURS = 48; - private static final int MAX_PUSH_ELAPSED = (FC_LEVEL_MAX + 1) * (ELAPSED_TIME_MAX + 1); - - public static void boostMaxFreqForDebug() { - FC_FREQ_MAX = BOOSTED_FC_FREQ; - } - - public static void resetMaxFreqForDebug() { - FC_FREQ_MAX = DEFAULT_FC_FREQ; - } - - private UserHistoryForgettingCurveUtils() { - // This utility class is not publicly instantiable. - } - - public static final class ForgettingCurveParams { - private byte mFc; - long mLastTouchedTime = 0; - private final boolean mIsValid; - - private void updateLastTouchedTime() { - mLastTouchedTime = System.currentTimeMillis(); - } - - public ForgettingCurveParams(boolean isValid) { - this(System.currentTimeMillis(), isValid); - } - - private ForgettingCurveParams(long now, boolean isValid) { - this(pushCount((byte)0, isValid), now, now, isValid); - } - - /** This constructor is called when the user history bigram dictionary is being restored. */ - public ForgettingCurveParams(int fc, long now, long last) { - // All words with level >= 1 had been saved. - // Invalid words with level == 0 had been saved. - // Valid words words with level == 0 had *not* been saved. - this(fc, now, last, fcToLevel((byte)fc) > 0); - } - - private ForgettingCurveParams(int fc, long now, long last, boolean isValid) { - mIsValid = isValid; - mFc = (byte)fc; - mLastTouchedTime = last; - updateElapsedTime(now); - } - - public boolean isValid() { - return mIsValid; - } - - public byte getFc() { - updateElapsedTime(System.currentTimeMillis()); - return mFc; - } - - public int getFrequency() { - updateElapsedTime(System.currentTimeMillis()); - return UserHistoryForgettingCurveUtils.fcToFreq(mFc); - } - - public int notifyTypedAgainAndGetFrequency() { - updateLastTouchedTime(); - // TODO: Check whether this word is valid or not - mFc = pushCount(mFc, false); - return UserHistoryForgettingCurveUtils.fcToFreq(mFc); - } - - private void updateElapsedTime(long now) { - final int elapsedTimeCount = - (int)((now - mLastTouchedTime) / ELAPSED_TIME_INTERVAL_MILLIS); - if (elapsedTimeCount <= 0) { - return; - } - if (elapsedTimeCount >= MAX_PUSH_ELAPSED) { - mLastTouchedTime = now; - mFc = 0; - return; - } - for (int i = 0; i < elapsedTimeCount; ++i) { - mLastTouchedTime += ELAPSED_TIME_INTERVAL_MILLIS; - mFc = pushElapsedTime(mFc); - } - } - } - - @UsedForTesting - /* package */ static int fcToElapsedTime(byte fc) { - return fc & 0x0F; - } - - @UsedForTesting - /* package */ static int fcToCount(byte fc) { - return (fc >> 4) & 0x03; - } - - @UsedForTesting - /* package */ static int fcToLevel(byte fc) { - return (fc >> 6) & 0x03; - } - - @UsedForTesting - private static int calcFreq(int elapsedTime, int count, int level) { - if (level <= 0) { - // Reserved words, just return -1 - return -1; - } - if (count == COUNT_MAX) { - // Temporary promote because it's frequently typed recently - ++level; - } - final int et = Math.min(FC_FREQ_MAX, Math.max(0, elapsedTime)); - final int l = Math.min(FC_LEVEL_MAX, Math.max(0, level)); - return MathUtils.SCORE_TABLE[l - 1][et]; - } - - /* pakcage */ static byte calcFc(int elapsedTime, int count, int level) { - final int et = Math.min(FC_FREQ_MAX, Math.max(0, elapsedTime)); - final int c = Math.min(COUNT_MAX, Math.max(0, count)); - final int l = Math.min(FC_LEVEL_MAX, Math.max(0, level)); - return (byte)(et | (c << 4) | (l << 6)); - } - - public static int fcToFreq(byte fc) { - final int elapsedTime = fcToElapsedTime(fc); - final int count = fcToCount(fc); - final int level = fcToLevel(fc); - return calcFreq(elapsedTime, count, level); - } - - @UsedForTesting - public static byte pushElapsedTime(byte fc) { - int elapsedTime = fcToElapsedTime(fc); - int count = fcToCount(fc); - int level = fcToLevel(fc); - if (elapsedTime >= ELAPSED_TIME_MAX) { - // Downgrade level - elapsedTime = 0; - count = COUNT_MAX; - --level; - } else { - ++elapsedTime; - } - return calcFc(elapsedTime, count, level); - } - - @UsedForTesting - public static byte pushCount(byte fc, boolean isValid) { - final int elapsedTime = fcToElapsedTime(fc); - int count = fcToCount(fc); - int level = fcToLevel(fc); - if ((elapsedTime == 0 && count >= COUNT_MAX) || (isValid && level == 0)) { - // Upgrade level - ++level; - count = 0; - if (DEBUG) { - Log.d(TAG, "Upgrade level."); - } - } else { - ++count; - } - return calcFc(0, count, level); - } - - // TODO: isValid should be false for a word whose frequency is 0, - // or that is not in the dictionary. - /** - * Check wheather we should save the bigram to the SQL DB or not - */ - public static boolean needsToSave(byte fc, boolean isValid, boolean addLevel0Bigram) { - int level = fcToLevel(fc); - if (level == 0) { - if (isValid || !addLevel0Bigram) { - return false; - } - } - final int elapsedTime = fcToElapsedTime(fc); - return (elapsedTime < ELAPSED_TIME_MAX - 1 || level > 0); - } - - private static final class MathUtils { - public static final int[][] SCORE_TABLE = new int[FC_LEVEL_MAX][ELAPSED_TIME_MAX + 1]; - static { - for (int i = 0; i < FC_LEVEL_MAX; ++i) { - final float initialFreq; - if (i >= 2) { - initialFreq = FC_FREQ_MAX; - } else if (i == 1) { - initialFreq = FC_FREQ_MAX / 2; - } else if (i == 0) { - initialFreq = FC_FREQ_MAX / 4; - } else { - continue; - } - for (int j = 0; j < ELAPSED_TIME_MAX; ++j) { - final float elapsedHours = j * ELAPSED_TIME_INTERVAL_HOURS; - final float freq = initialFreq - * (float)Math.pow(initialFreq, elapsedHours / HALF_LIFE_HOURS); - final int intFreq = Math.min(FC_FREQ_MAX, Math.max(0, (int)freq)); - SCORE_TABLE[i][j] = intFreq; - } - } - } - } -} |