diff options
author | 2015-03-20 11:16:46 -0700 | |
---|---|---|
committer | 2015-03-20 11:16:46 -0700 | |
commit | c15bbb52a37be751fed2ba7e765dfd7727306308 (patch) | |
tree | 9fcbae84e121d0cdef5a3d74c6c5e2680dbc003a /tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java | |
parent | fe716f0f73ee551ce42b523983f92efccce79dc8 (diff) | |
download | latinime-c15bbb52a37be751fed2ba7e765dfd7727306308.tar.gz latinime-c15bbb52a37be751fed2ba7e765dfd7727306308.tar.xz latinime-c15bbb52a37be751fed2ba7e765dfd7727306308.zip |
Retire Delight2 migration code to speed up tests.
We're waiting 10 minutes for tests to run, and half of that time is spent in
depreacted code related to migration of Delight2 dictionary files.
LatinIME will never migrate another Delight2 dictionary file again, so we can
delete this code.
Change-Id: I05c7d8429e8d9a26139456763c77997340fea8c2
Diffstat (limited to 'tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java')
-rw-r--r-- | tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java | 319 |
1 files changed, 0 insertions, 319 deletions
diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java deleted file mode 100644 index 7ee1df92b..000000000 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.BinaryDictionary; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; - -/** - * An implementation of DictDecoder for version 2 binary dictionary. - */ -// TODO: Separate logics that are used only for testing. -@UsedForTesting -public class Ver2DictDecoder extends AbstractDictDecoder { - /** - * A utility class for reading a PtNode. - */ - static class PtNodeReader { - static ProbabilityInfo readProbabilityInfo(final DictBuffer dictBuffer) { - // Ver2 dicts don't contain historical information. - return new ProbabilityInfo(dictBuffer.readUnsignedByte()); - } - - static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedByte(); - } - - static int readChildrenAddress(final DictBuffer dictBuffer, - final int ptNodeFlags) { - switch (ptNodeFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: - return dictBuffer.readUnsignedByte(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: - return dictBuffer.readUnsignedShort(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: - return dictBuffer.readUnsignedInt24(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - - // Reads shortcuts and returns the read length. - static int readShortcut(final DictBuffer dictBuffer, - final ArrayList<WeightedString> shortcutTargets) { - final int pointerBefore = dictBuffer.position(); - dictBuffer.readUnsignedShort(); // skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return dictBuffer.position() - pointerBefore; - } - - static int readBigramAddresses(final DictBuffer dictBuffer, - final ArrayList<PendingAttribute> bigrams, final int baseAddress) { - int readLength = 0; - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++readLength; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = baseAddress + readLength; - switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - readLength += 1; - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - readLength += 2; - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: - bigramAddress += sign * dictBuffer.readUnsignedInt24(); - readLength += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute( - bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return readLength; - } - } - - protected final File mDictionaryBinaryFile; - protected final long mOffset; - protected final long mLength; - // TODO: Remove mBufferFactory and mDictBuffer from this class members because they are now - // used only for testing. - private final DictionaryBufferFactory mBufferFactory; - protected DictBuffer mDictBuffer; - - @UsedForTesting - /* package */ Ver2DictDecoder(final File file, final long offset, final long length, - final int factoryFlag) { - mDictionaryBinaryFile = file; - mOffset = offset; - mLength = length; - mDictBuffer = null; - if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { - mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); - } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { - mBufferFactory = new DictionaryBufferFromByteArrayFactory(); - } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { - mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); - } else { - mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); - } - } - - /* package */ Ver2DictDecoder(final File file, final long offset, final long length, - final DictionaryBufferFactory factory) { - mDictionaryBinaryFile = file; - mOffset = offset; - mLength = length; - mBufferFactory = factory; - } - - @Override - public void openDictBuffer() throws FileNotFoundException, IOException { - mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); - } - - @Override - public boolean isDictBufferOpen() { - return mDictBuffer != null; - } - - /* package */ DictBuffer getDictBuffer() { - return mDictBuffer; - } - - @UsedForTesting - /* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException { - openDictBuffer(); - return getDictBuffer(); - } - - @Override - public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { - // dictType is not being used in dicttool. Passing an empty string. - final BinaryDictionary binaryDictionary = new BinaryDictionary( - mDictionaryBinaryFile.getAbsolutePath(), mOffset, mLength, - true /* useFullEditDistance */, null /* locale */, "" /* dictType */, - false /* isUpdatable */); - final DictionaryHeader header = binaryDictionary.getHeader(); - binaryDictionary.close(); - if (header == null) { - throw new IOException("Cannot read the dictionary header."); - } - if (header.mFormatOptions.mVersion != FormatSpec.VERSION2 && - header.mFormatOptions.mVersion != FormatSpec.VERSION201 && - header.mFormatOptions.mVersion != FormatSpec.VERSION202) { - throw new UnsupportedFormatException("File header has a wrong version : " - + header.mFormatOptions.mVersion); - } - if (!isDictBufferOpen()) { - openDictBuffer(); - } - // Advance buffer reading position to the head of dictionary body. - setPosition(header.mBodyOffset); - return header; - } - - // TODO: Make this buffer multi thread safe. - private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; - @Override - public PtNodeInfo readPtNode(final int ptNodePos) { - int addressPointer = ptNodePos; - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character, null); - while (FormatSpec.INVALID_CHARACTER != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of mCharacterBuffer. - mCharacterBuffer[index++] = character; - character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character, null); - } - characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); - } else { - final int character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character, null); - characters = new int[] { character }; - } - final ProbabilityInfo probabilityInfo; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - probabilityInfo = PtNodeReader.readProbabilityInfo(mDictBuffer); - addressPointer += FormatSpec.PTNODE_FREQUENCY_SIZE; - } else { - probabilityInfo = null; - } - int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags); - final ArrayList<WeightedString> shortcutTargets; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - // readShortcut will add shortcuts to shortcutTargets. - shortcutTargets = new ArrayList<>(); - addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); - } else { - shortcutTargets = null; - } - - final ArrayList<PendingAttribute> bigrams; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<>(); - addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, - addressPointer); - if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() - + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); - } - } else { - bigrams = null; - } - return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo, - childrenAddress, shortcutTargets, bigrams); - } - - @Override - public FusionDictionary readDictionaryBinary(final boolean deleteDictIfBroken) - throws FileNotFoundException, IOException, UnsupportedFormatException { - // dictType is not being used in dicttool. Passing an empty string. - final BinaryDictionary binaryDictionary = new BinaryDictionary( - mDictionaryBinaryFile.getAbsolutePath(), 0 /* offset */, - mDictionaryBinaryFile.length() /* length */, true /* useFullEditDistance */, - null /* locale */, "" /* dictType */, false /* isUpdatable */); - final DictionaryHeader header = readHeader(); - final FusionDictionary fusionDict = - new FusionDictionary(new FusionDictionary.PtNodeArray(), header.mDictionaryOptions); - int token = 0; - final ArrayList<WordProperty> wordProperties = new ArrayList<>(); - do { - final BinaryDictionary.GetNextWordPropertyResult result = - binaryDictionary.getNextWordProperty(token); - final WordProperty wordProperty = result.mWordProperty; - if (wordProperty == null) { - binaryDictionary.close(); - if (deleteDictIfBroken) { - mDictionaryBinaryFile.delete(); - } - return null; - } - wordProperties.add(wordProperty); - token = result.mNextToken; - } while (token != 0); - - // Insert unigrams into the fusion dictionary. - for (final WordProperty wordProperty : wordProperties) { - fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, - wordProperty.mIsNotAWord, - wordProperty.mIsPossiblyOffensive); - } - // Insert bigrams into the fusion dictionary. - for (final WordProperty wordProperty : wordProperties) { - if (!wordProperty.mHasNgrams) { - continue; - } - final String word0 = wordProperty.mWord; - for (final WeightedString bigram : wordProperty.getBigrams()) { - fusionDict.setBigram(word0, bigram.mWord, bigram.mProbabilityInfo); - } - } - binaryDictionary.close(); - return fusionDict; - } - - @Override - public void setPosition(int newPos) { - mDictBuffer.position(newPos); - } - - @Override - public int getPosition() { - return mDictBuffer.position(); - } - - @Override - public int readPtNodeCount() { - return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); - } -} |