Have dicttool use the native library to generate v4 dicts.

Yay !

Change-Id: Iea8ced9e81031b9ab7eff05ad9ef7215be248de9

10 files changed, 139 insertions, 170 deletions

diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index e11e706f3..7827db302 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -39,87 +39,7 @@ endif # TARGET_ARCH
 # To suppress compiler warnings for unused variables/functions used for debug features etc.
 LOCAL_CFLAGS += -Wno-unused-parameter -Wno-unused-function
 
-LATIN_IME_JNI_SRC_FILES := \
-    com_android_inputmethod_keyboard_ProximityInfo.cpp \
-    com_android_inputmethod_latin_BinaryDictionary.cpp \
-    com_android_inputmethod_latin_DicTraverseSession.cpp \
-    com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \
-    jni_common.cpp
-
-LATIN_IME_CORE_SRC_FILES := \
-    suggest/core/suggest.cpp \
-    $(addprefix suggest/core/dicnode/, \
-        dic_node.cpp \
-        dic_node_utils.cpp \
-        dic_nodes_cache.cpp) \
-    $(addprefix suggest/core/dictionary/, \
-        bigram_dictionary.cpp \
-        bloom_filter.cpp \
-        dictionary.cpp \
-        digraph_utils.cpp \
-        error_type_utils.cpp \
-        multi_bigram_map.cpp \
-        unigram_property.cpp) \
-    $(addprefix suggest/core/layout/, \
-        additional_proximity_chars.cpp \
-        proximity_info.cpp \
-        proximity_info_params.cpp \
-        proximity_info_state.cpp \
-        proximity_info_state_utils.cpp) \
-    suggest/core/policy/weighting.cpp \
-    suggest/core/session/dic_traverse_session.cpp \
-    $(addprefix suggest/policyimpl/dictionary/, \
-        header/header_policy.cpp \
-        header/header_read_write_utils.cpp \
-        shortcut/shortcut_list_reading_utils.cpp \
-        structure/dictionary_structure_with_buffer_policy_factory.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/bigram/, \
-        bigram_list_read_write_utils.cpp \
-        ver4_bigram_list_policy.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
-        dynamic_pt_gc_event_listeners.cpp \
-        dynamic_pt_reading_helper.cpp \
-        dynamic_pt_reading_utils.cpp \
-        dynamic_pt_updating_helper.cpp \
-        dynamic_pt_writing_utils.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/v2/, \
-        patricia_trie_policy.cpp \
-        patricia_trie_reading_utils.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/v4/, \
-        ver4_dict_buffers.cpp \
-        ver4_dict_constants.cpp \
-        ver4_patricia_trie_node_reader.cpp \
-        ver4_patricia_trie_node_writer.cpp \
-        ver4_patricia_trie_policy.cpp \
-        ver4_patricia_trie_reading_utils.cpp \
-        ver4_patricia_trie_writing_helper.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
-        bigram_dict_content.cpp \
-        probability_dict_content.cpp \
-        shortcut_dict_content.cpp \
-        sparse_table_dict_content.cpp \
-        terminal_position_lookup_table.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/utils/, \
-        buffer_with_extendable_buffer.cpp \
-        byte_array_utils.cpp \
-        dict_file_writing_utils.cpp \
-        file_utils.cpp \
-        forgetting_curve_utils.cpp \
-        format_utils.cpp \
-        mmapped_buffer.cpp \
-        sparse_table.cpp) \
-    suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
-    $(addprefix suggest/policyimpl/typing/, \
-        scoring_params.cpp \
-        typing_scoring.cpp \
-        typing_suggest_policy.cpp \
-        typing_traversal.cpp \
-        typing_weighting.cpp) \
-    $(addprefix utils/, \
-        autocorrection_threshold_utils.cpp \
-        char_utils.cpp \
-        log_utils.cpp \
-        time_keeper.cpp)
+include $(LOCAL_PATH)/NativeFileList.mk
 
 LOCAL_SRC_FILES := \
     $(LATIN_IME_JNI_SRC_FILES) \
@@ -172,6 +92,4 @@ LOCAL_LDFLAGS += -ldl
 include $(BUILD_SHARED_LIBRARY)
 
 #################### Clean up the tmp vars
-LATIN_IME_CORE_SRC_FILES :=
-LATIN_IME_JNI_SRC_FILES :=
-LATIN_IME_SRC_DIR :=
+include $(LOCAL_PATH)/CleanupNativeFileList.mk
diff --git a/native/jni/CleanupNativeFileList.mk b/native/jni/CleanupNativeFileList.mk
new file mode 100644
index 000000000..5420f16f9
--- /dev/null
+++ b/native/jni/CleanupNativeFileList.mk
@@ -0,0 +1,17 @@
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LATIN_IME_CORE_SRC_FILES :=
+LATIN_IME_JNI_SRC_FILES :=
+LATIN_IME_SRC_DIR :=
diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk
new file mode 100644
index 000000000..d80a1115f
--- /dev/null
+++ b/native/jni/NativeFileList.mk
@@ -0,0 +1,94 @@
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LATIN_IME_JNI_SRC_FILES := \
+    com_android_inputmethod_keyboard_ProximityInfo.cpp \
+    com_android_inputmethod_latin_BinaryDictionary.cpp \
+    com_android_inputmethod_latin_DicTraverseSession.cpp \
+    jni_common.cpp
+
+LATIN_IME_CORE_SRC_FILES := \
+    suggest/core/suggest.cpp \
+    $(addprefix suggest/core/dicnode/, \
+        dic_node.cpp \
+        dic_node_utils.cpp \
+        dic_nodes_cache.cpp) \
+    $(addprefix suggest/core/dictionary/, \
+        bigram_dictionary.cpp \
+        bloom_filter.cpp \
+        dictionary.cpp \
+        digraph_utils.cpp \
+        error_type_utils.cpp \
+        multi_bigram_map.cpp \
+        unigram_property.cpp) \
+    $(addprefix suggest/core/layout/, \
+        additional_proximity_chars.cpp \
+        proximity_info.cpp \
+        proximity_info_params.cpp \
+        proximity_info_state.cpp \
+        proximity_info_state_utils.cpp) \
+    suggest/core/policy/weighting.cpp \
+    suggest/core/session/dic_traverse_session.cpp \
+    $(addprefix suggest/policyimpl/dictionary/, \
+        header/header_policy.cpp \
+        header/header_read_write_utils.cpp \
+        shortcut/shortcut_list_reading_utils.cpp \
+        structure/dictionary_structure_with_buffer_policy_factory.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/bigram/, \
+        bigram_list_read_write_utils.cpp \
+        ver4_bigram_list_policy.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
+        dynamic_pt_gc_event_listeners.cpp \
+        dynamic_pt_reading_helper.cpp \
+        dynamic_pt_reading_utils.cpp \
+        dynamic_pt_updating_helper.cpp \
+        dynamic_pt_writing_utils.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/v2/, \
+        patricia_trie_policy.cpp \
+        patricia_trie_reading_utils.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/v4/, \
+        ver4_dict_buffers.cpp \
+        ver4_dict_constants.cpp \
+        ver4_patricia_trie_node_reader.cpp \
+        ver4_patricia_trie_node_writer.cpp \
+        ver4_patricia_trie_policy.cpp \
+        ver4_patricia_trie_reading_utils.cpp \
+        ver4_patricia_trie_writing_helper.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
+        bigram_dict_content.cpp \
+        probability_dict_content.cpp \
+        shortcut_dict_content.cpp \
+        sparse_table_dict_content.cpp \
+        terminal_position_lookup_table.cpp) \
+    $(addprefix suggest/policyimpl/dictionary/utils/, \
+        buffer_with_extendable_buffer.cpp \
+        byte_array_utils.cpp \
+        dict_file_writing_utils.cpp \
+        file_utils.cpp \
+        forgetting_curve_utils.cpp \
+        format_utils.cpp \
+        mmapped_buffer.cpp \
+        sparse_table.cpp) \
+    suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
+    $(addprefix suggest/policyimpl/typing/, \
+        scoring_params.cpp \
+        typing_scoring.cpp \
+        typing_suggest_policy.cpp \
+        typing_traversal.cpp \
+        typing_weighting.cpp) \
+    $(addprefix utils/, \
+        autocorrection_threshold_utils.cpp \
+        char_utils.cpp \
+        log_utils.cpp \
+        time_keeper.cpp)
diff --git a/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp b/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp
deleted file mode 100644
index 15088b65a..000000000
--- a/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define LOG_TAG "LatinIME: jni: Ver3DictDecoder"
-
-#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h"
-
-#include "defines.h"
-#include "jni.h"
-#include "jni_common.h"
-
-namespace latinime {
-static int latinime_Ver3DictDecoder_doNothing(JNIEnv *env, jclass clazz) {
-    // This is a phony method for test - it does nothing. It just returns some value
-    // unlikely to be in memory by chance for testing purposes.
-    // TODO: remove this method.
-    return 2097;
-}
-
-static const JNINativeMethod sMethods[] = {
-    {
-        // TODO: remove this entry when we have one useful method in here
-        const_cast<char *>("doNothing"),
-        const_cast<char *>("()I"),
-        reinterpret_cast<void *>(latinime_Ver3DictDecoder_doNothing)
-    },
-};
-
-int register_Ver3DictDecoder(JNIEnv *env) {
-    const char *const kClassPathName =
-            "com/android/inputmethod/latin/makedict/Ver3DictDecoder";
-    return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
-}
-} // namespace latinime
diff --git a/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.h b/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.h
deleted file mode 100644
index 07e80f1d8..000000000
--- a/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
-#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
-
-#include "jni.h"
-
-namespace latinime {
-int register_Ver3DictDecoder(JNIEnv *env);
-} // namespace latinime
-#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H
diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp
index 3a8f4362d..f2867d7c3 100644
--- a/native/jni/jni_common.cpp
+++ b/native/jni/jni_common.cpp
@@ -18,12 +18,9 @@
 
 #include "jni_common.h"
 
-#ifndef HOST_TOOL
 #include "com_android_inputmethod_keyboard_ProximityInfo.h"
 #include "com_android_inputmethod_latin_BinaryDictionary.h"
 #include "com_android_inputmethod_latin_DicTraverseSession.h"
-#endif
-#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h"
 #include "defines.h"
 
 /*
@@ -41,7 +38,6 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
         AKLOGE("ERROR: JNIEnv is invalid");
         return -1;
     }
-#ifndef HOST_TOOL
     if (!latinime::register_BinaryDictionary(env)) {
         AKLOGE("ERROR: BinaryDictionary native registration failed");
         return -1;
@@ -54,11 +50,6 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
         AKLOGE("ERROR: ProximityInfo native registration failed");
         return -1;
     }
-#endif
-    if (!latinime::register_Ver3DictDecoder(env)) {
-        AKLOGE("ERROR: Ver3DictDecoder native registration failed");
-        return -1;
-    }
     /* success -- return valid version number */
     return JNI_VERSION_1_6;
 }
diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h
index 9a26fe051..1969ebae0 100644
--- a/native/jni/src/defines.h
+++ b/native/jni/src/defines.h
@@ -87,12 +87,21 @@ AK_FORCE_INLINE static int intArrayToCharArray(const int *const source, const in
 }
 
 #if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG)
+#if defined(__ANDROID__)
 #include <android/log.h>
+#endif // defined(__ANDROID__)
 #ifndef LOG_TAG
 #define LOG_TAG "LatinIME: "
 #endif // LOG_TAG
+
+#if defined(HOST_TOOL)
+#include <stdio.h>
+#define AKLOGE(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
+#define AKLOGI(fmt, ...) printf(fmt "\n", ##__VA_ARGS__)
+#else // defined(HOST_TOOL)
 #define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__)
 #define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__)
+#endif // defined(HOST_TOOL)
 
 #define DUMP_RESULT(words, frequencies) do { dumpResult(words, frequencies); } while (0)
 #define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0)
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
index 34fecc25f..9afb5f221 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
@@ -33,8 +33,7 @@ const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
         ".shortcut_index_shortcut";
 
 // Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
-// TODO: Make MAX_DICTIONARY_SIZE 8MB.
-const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
+const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
 // Extended region size, which is not GCed region size in dict file + additional buffer size, is
 // limited to 1MB to prevent from inefficient traversing.
 const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
index 1f25cfa1e..9441a75fc 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
@@ -53,6 +53,11 @@ namespace latinime {
 
 // Remove a directory and all files in the directory.
 /* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
+    return removeDirAndFiles(dirPath, 5 /* maxTries */);
+}
+
+// Remove a directory and all files in the directory, trying up to maxTimes.
+/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath, const int maxTries) {
     DIR *const dir = opendir(dirPath);
     if (dir == NULL) {
         AKLOGE("Cannot open dir %s.", dirPath);
@@ -60,7 +65,7 @@ namespace latinime {
     }
     struct dirent *dirent;
     while ((dirent = readdir(dir)) != NULL) {
-        if (dirent->d_type != DT_REG) {
+        if (dirent->d_type == DT_DIR) {
             continue;
         }
         const int filePathBufSize = getFilePathBufSize(dirPath, dirent->d_name);
@@ -74,8 +79,14 @@ namespace latinime {
     }
     closedir(dir);
     if (remove(dirPath) != 0) {
-        AKLOGE("Cannot remove directory %s.", dirPath);
-        return false;
+        if (maxTries > 0) {
+            // On NFS, deleting files sometimes creates new files. I'm not sure what the
+            // correct way of dealing with this is, but for the time being, this seems to work.
+            removeDirAndFiles(dirPath, maxTries - 1);
+        } else {
+            AKLOGE("Cannot remove directory %s.", dirPath);
+            return false;
+        }
     }
     return true;
 }
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
index 3e84a3038..4f1b93a6a 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
@@ -53,6 +53,8 @@ class FileUtils {
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(FileUtils);
+
+    static bool removeDirAndFiles(const char *const dirPath, const int maxTries);
 };
 } // namespace latinime
 #endif /* LATINIME_FILE_UTILS_H */