Include a code point table in the binary dictionary.

Bug:17097992
Change-Id: I677a5eb3a704e4386f6573360e44ca335d81d2df

1 files changed, 40 insertions, 18 deletions

diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
index 96604a197..1f3ee19af 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
@@ -17,11 +17,11 @@
 package com.android.inputmethod.latin.makedict;
 
 import com.android.inputmethod.annotations.UsedForTesting;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
+import java.util.HashMap;
 
 /**
  * Decodes binary files for a FusionDictionary.
@@ -109,15 +109,19 @@ public final class BinaryDictDecoderUtils {
      * A class grouping utility function for our specific character encoding.
      */
     static final class CharEncoding {
-        private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
-        private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
 
         /**
          * Helper method to find out whether this code fits on one byte
          */
-        private static boolean fitsOnOneByte(final int character) {
-            return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
-                    && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
+        private static boolean fitsOnOneByte(int character,
+                final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
+            if (codePointToOneByteCodeMap != null) {
+                if (codePointToOneByteCodeMap.containsKey(character)) {
+                    character = codePointToOneByteCodeMap.get(character);
+                }
+            }
+            return character >= FormatSpec.MINIMAL_ONE_BYTE_CHARACTER_VALUE
+                    && character <= FormatSpec.MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
         }
 
         /**
@@ -137,9 +141,10 @@ public final class BinaryDictDecoderUtils {
          * @param character the character code.
          * @return the size in binary encoded-form, either 1 or 3 bytes.
          */
-        static int getCharSize(final int character) {
+        static int getCharSize(final int character,
+                final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
             // See char encoding in FusionDictionary.java
-            if (fitsOnOneByte(character)) return 1;
+            if (fitsOnOneByte(character, codePointToOneByteCodeMap)) return 1;
             if (FormatSpec.INVALID_CHARACTER == character) return 1;
             return 3;
         }
@@ -147,9 +152,10 @@ public final class BinaryDictDecoderUtils {
         /**
          * Compute the byte size of a character array.
          */
-        static int getCharArraySize(final int[] chars) {
+        static int getCharArraySize(final int[] chars,
+                final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
             int size = 0;
-            for (int character : chars) size += getCharSize(character);
+            for (int character : chars) size += getCharSize(character, codePointToOneByteCodeMap);
             return size;
         }
 
@@ -159,11 +165,19 @@ public final class BinaryDictDecoderUtils {
          * @param codePoints the code point array to write.
          * @param buffer the byte buffer to write to.
          * @param index the index in buffer to write the character array to.
+         * @param codePointToOneByteCodeMap the map to convert the code point.
          * @return the index after the last character.
          */
-        static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
+        static int writeCharArray(final int[] codePoints, final byte[] buffer, int index,
+                final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
             for (int codePoint : codePoints) {
-                if (1 == getCharSize(codePoint)) {
+                if (codePointToOneByteCodeMap != null) {
+                    if (codePointToOneByteCodeMap.containsKey(codePoint)) {
+                        // Convert code points
+                        codePoint = codePointToOneByteCodeMap.get(codePoint);
+                    }
+                }
+                if (1 == getCharSize(codePoint, codePointToOneByteCodeMap)) {
                     buffer[index++] = (byte)codePoint;
                 } else {
                     buffer[index++] = (byte)(0xFF & (codePoint >> 16));
@@ -184,12 +198,19 @@ public final class BinaryDictDecoderUtils {
          * @param word the string to write.
          * @return the size written, in bytes.
          */
-        static int writeString(final byte[] buffer, final int origin, final String word) {
+        static int writeString(final byte[] buffer, final int origin, final String word,
+                final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
             final int length = word.length();
             int index = origin;
             for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
-                final int codePoint = word.codePointAt(i);
-                if (1 == getCharSize(codePoint)) {
+                int codePoint = word.codePointAt(i);
+                if (codePointToOneByteCodeMap != null) {
+                    if (codePointToOneByteCodeMap.containsKey(codePoint)) {
+                        // Convert code points
+                        codePoint = codePointToOneByteCodeMap.get(codePoint);
+                    }
+                }
+                if (1 == getCharSize(codePoint, codePointToOneByteCodeMap)) {
                     buffer[index++] = (byte)codePoint;
                 } else {
                     buffer[index++] = (byte)(0xFF & (codePoint >> 16));
@@ -210,12 +231,13 @@ public final class BinaryDictDecoderUtils {
          * @param word the string to write.
          * @return the size written, in bytes.
          */
-        static int writeString(final OutputStream stream, final String word) throws IOException {
+        static int writeString(final OutputStream stream, final String word,
+                final HashMap<Integer, Integer> codePointToOneByteCodeMap) throws IOException {
             final int length = word.length();
             int written = 0;
             for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
                 final int codePoint = word.codePointAt(i);
-                final int charSize = getCharSize(codePoint);
+                final int charSize = getCharSize(codePoint, codePointToOneByteCodeMap);
                 if (1 == charSize) {
                     stream.write((byte) codePoint);
                 } else {
@@ -253,7 +275,7 @@ public final class BinaryDictDecoderUtils {
          */
         static int readChar(final DictBuffer dictBuffer) {
             int character = dictBuffer.readUnsignedByte();
-            if (!fitsOnOneByte(character)) {
+            if (!fitsOnOneByte(character, null)) {
                 if (FormatSpec.PTNODE_CHARACTERS_TERMINATOR == character) {
                     return FormatSpec.INVALID_CHARACTER;
                 }