aboutsummaryrefslogtreecommitdiffstats
path: root/native/src
diff options
context:
space:
mode:
Diffstat (limited to 'native/src')
-rw-r--r--native/src/basechars.h172
-rw-r--r--native/src/bigram_dictionary.cpp265
-rw-r--r--native/src/bigram_dictionary.h56
-rw-r--r--native/src/char_utils.cpp899
-rw-r--r--native/src/char_utils.h26
-rw-r--r--native/src/debug.h72
-rw-r--r--native/src/defines.h170
-rw-r--r--native/src/dictionary.cpp68
-rw-r--r--native/src/dictionary.h161
-rw-r--r--native/src/proximity_info.cpp66
-rw-r--r--native/src/proximity_info.h47
-rw-r--r--native/src/unigram_dictionary.cpp1127
-rw-r--r--native/src/unigram_dictionary.h152
13 files changed, 0 insertions, 3281 deletions
diff --git a/native/src/basechars.h b/native/src/basechars.h
deleted file mode 100644
index 5a4406606..000000000
--- a/native/src/basechars.h
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Table mapping most combined Latin, Greek, and Cyrillic characters
- * to their base characters. If c is in range, BASE_CHARS[c] == c
- * if c is not a combined character, or the base character if it
- * is combined.
- */
-static unsigned short BASE_CHARS[] = {
- 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
- 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
- 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
- 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f,
- 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
- 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
- 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
- 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
- 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
- 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
- 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
- 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
- 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
- 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
- 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
- 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f,
- 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
- 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f,
- 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
- 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f,
- 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7,
- 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0020,
- 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7,
- 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf,
- 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x00c6, 0x0043,
- 0x0045, 0x0045, 0x0045, 0x0045, 0x0049, 0x0049, 0x0049, 0x0049,
- 0x00d0, 0x004e, 0x004f, 0x004f, 0x004f, 0x004f, 0x004f, 0x00d7,
- 0x004f, 0x0055, 0x0055, 0x0055, 0x0055, 0x0059, 0x00de, 0x0073, // Manually changed d8 to 4f
- // Manually changed df to 73
- 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063,
- 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069,
- 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7,
- 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0079, // Manually changed f8 to 6f
- 0x0041, 0x0061, 0x0041, 0x0061, 0x0041, 0x0061, 0x0043, 0x0063,
- 0x0043, 0x0063, 0x0043, 0x0063, 0x0043, 0x0063, 0x0044, 0x0064,
- 0x0110, 0x0111, 0x0045, 0x0065, 0x0045, 0x0065, 0x0045, 0x0065,
- 0x0045, 0x0065, 0x0045, 0x0065, 0x0047, 0x0067, 0x0047, 0x0067,
- 0x0047, 0x0067, 0x0047, 0x0067, 0x0048, 0x0068, 0x0126, 0x0127,
- 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069,
- 0x0049, 0x0131, 0x0049, 0x0069, 0x004a, 0x006a, 0x004b, 0x006b,
- 0x0138, 0x004c, 0x006c, 0x004c, 0x006c, 0x004c, 0x006c, 0x004c,
- 0x006c, 0x0141, 0x0142, 0x004e, 0x006e, 0x004e, 0x006e, 0x004e,
- 0x006e, 0x02bc, 0x014a, 0x014b, 0x004f, 0x006f, 0x004f, 0x006f,
- 0x004f, 0x006f, 0x0152, 0x0153, 0x0052, 0x0072, 0x0052, 0x0072,
- 0x0052, 0x0072, 0x0053, 0x0073, 0x0053, 0x0073, 0x0053, 0x0073,
- 0x0053, 0x0073, 0x0054, 0x0074, 0x0054, 0x0074, 0x0166, 0x0167,
- 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075,
- 0x0055, 0x0075, 0x0055, 0x0075, 0x0057, 0x0077, 0x0059, 0x0079,
- 0x0059, 0x005a, 0x007a, 0x005a, 0x007a, 0x005a, 0x007a, 0x0073,
- 0x0180, 0x0181, 0x0182, 0x0183, 0x0184, 0x0185, 0x0186, 0x0187,
- 0x0188, 0x0189, 0x018a, 0x018b, 0x018c, 0x018d, 0x018e, 0x018f,
- 0x0190, 0x0191, 0x0192, 0x0193, 0x0194, 0x0195, 0x0196, 0x0197,
- 0x0198, 0x0199, 0x019a, 0x019b, 0x019c, 0x019d, 0x019e, 0x019f,
- 0x004f, 0x006f, 0x01a2, 0x01a3, 0x01a4, 0x01a5, 0x01a6, 0x01a7,
- 0x01a8, 0x01a9, 0x01aa, 0x01ab, 0x01ac, 0x01ad, 0x01ae, 0x0055,
- 0x0075, 0x01b1, 0x01b2, 0x01b3, 0x01b4, 0x01b5, 0x01b6, 0x01b7,
- 0x01b8, 0x01b9, 0x01ba, 0x01bb, 0x01bc, 0x01bd, 0x01be, 0x01bf,
- 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0044, 0x0044, 0x0064, 0x004c,
- 0x004c, 0x006c, 0x004e, 0x004e, 0x006e, 0x0041, 0x0061, 0x0049,
- 0x0069, 0x004f, 0x006f, 0x0055, 0x0075, 0x00dc, 0x00fc, 0x00dc,
- 0x00fc, 0x00dc, 0x00fc, 0x00dc, 0x00fc, 0x01dd, 0x00c4, 0x00e4,
- 0x0226, 0x0227, 0x00c6, 0x00e6, 0x01e4, 0x01e5, 0x0047, 0x0067,
- 0x004b, 0x006b, 0x004f, 0x006f, 0x01ea, 0x01eb, 0x01b7, 0x0292,
- 0x006a, 0x0044, 0x0044, 0x0064, 0x0047, 0x0067, 0x01f6, 0x01f7,
- 0x004e, 0x006e, 0x00c5, 0x00e5, 0x00c6, 0x00e6, 0x00d8, 0x00f8,
- 0x0041, 0x0061, 0x0041, 0x0061, 0x0045, 0x0065, 0x0045, 0x0065,
- 0x0049, 0x0069, 0x0049, 0x0069, 0x004f, 0x006f, 0x004f, 0x006f,
- 0x0052, 0x0072, 0x0052, 0x0072, 0x0055, 0x0075, 0x0055, 0x0075,
- 0x0053, 0x0073, 0x0054, 0x0074, 0x021c, 0x021d, 0x0048, 0x0068,
- 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0041, 0x0061,
- 0x0045, 0x0065, 0x00d6, 0x00f6, 0x00d5, 0x00f5, 0x004f, 0x006f,
- 0x022e, 0x022f, 0x0059, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237,
- 0x0238, 0x0239, 0x023a, 0x023b, 0x023c, 0x023d, 0x023e, 0x023f,
- 0x0240, 0x0241, 0x0242, 0x0243, 0x0244, 0x0245, 0x0246, 0x0247,
- 0x0248, 0x0249, 0x024a, 0x024b, 0x024c, 0x024d, 0x024e, 0x024f,
- 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257,
- 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f,
- 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267,
- 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f,
- 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277,
- 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f,
- 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287,
- 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f,
- 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297,
- 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f,
- 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7,
- 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af,
- 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077,
- 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf,
- 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7,
- 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf,
- 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7,
- 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df,
- 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7,
- 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef,
- 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7,
- 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff,
- 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307,
- 0x0308, 0x0309, 0x030a, 0x030b, 0x030c, 0x030d, 0x030e, 0x030f,
- 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317,
- 0x0318, 0x0319, 0x031a, 0x031b, 0x031c, 0x031d, 0x031e, 0x031f,
- 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327,
- 0x0328, 0x0329, 0x032a, 0x032b, 0x032c, 0x032d, 0x032e, 0x032f,
- 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337,
- 0x0338, 0x0339, 0x033a, 0x033b, 0x033c, 0x033d, 0x033e, 0x033f,
- 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x0345, 0x0346, 0x0347,
- 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x034f,
- 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357,
- 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f,
- 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367,
- 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f,
- 0x0370, 0x0371, 0x0372, 0x0373, 0x02b9, 0x0375, 0x0376, 0x0377,
- 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f,
- 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x00a8, 0x0391, 0x00b7,
- 0x0395, 0x0397, 0x0399, 0x038b, 0x039f, 0x038d, 0x03a5, 0x03a9,
- 0x03ca, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
- 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f,
- 0x03a0, 0x03a1, 0x03a2, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7,
- 0x03a8, 0x03a9, 0x0399, 0x03a5, 0x03b1, 0x03b5, 0x03b7, 0x03b9,
- 0x03cb, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7,
- 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf,
- 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7,
- 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03cf,
- 0x03b2, 0x03b8, 0x03a5, 0x03d2, 0x03d2, 0x03c6, 0x03c0, 0x03d7,
- 0x03d8, 0x03d9, 0x03da, 0x03db, 0x03dc, 0x03dd, 0x03de, 0x03df,
- 0x03e0, 0x03e1, 0x03e2, 0x03e3, 0x03e4, 0x03e5, 0x03e6, 0x03e7,
- 0x03e8, 0x03e9, 0x03ea, 0x03eb, 0x03ec, 0x03ed, 0x03ee, 0x03ef,
- 0x03ba, 0x03c1, 0x03c2, 0x03f3, 0x0398, 0x03b5, 0x03f6, 0x03f7,
- 0x03f8, 0x03a3, 0x03fa, 0x03fb, 0x03fc, 0x03fd, 0x03fe, 0x03ff,
- 0x0415, 0x0415, 0x0402, 0x0413, 0x0404, 0x0405, 0x0406, 0x0406,
- 0x0408, 0x0409, 0x040a, 0x040b, 0x041a, 0x0418, 0x0423, 0x040f,
- 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
- 0x0418, 0x0418, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
- 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427,
- 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
- 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437,
- 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
- 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447,
- 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f,
- 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456,
- 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f,
- 0x0460, 0x0461, 0x0462, 0x0463, 0x0464, 0x0465, 0x0466, 0x0467,
- 0x0468, 0x0469, 0x046a, 0x046b, 0x046c, 0x046d, 0x046e, 0x046f,
- 0x0470, 0x0471, 0x0472, 0x0473, 0x0474, 0x0475, 0x0474, 0x0475,
- 0x0478, 0x0479, 0x047a, 0x047b, 0x047c, 0x047d, 0x047e, 0x047f,
- 0x0480, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487,
- 0x0488, 0x0489, 0x048a, 0x048b, 0x048c, 0x048d, 0x048e, 0x048f,
- 0x0490, 0x0491, 0x0492, 0x0493, 0x0494, 0x0495, 0x0496, 0x0497,
- 0x0498, 0x0499, 0x049a, 0x049b, 0x049c, 0x049d, 0x049e, 0x049f,
- 0x04a0, 0x04a1, 0x04a2, 0x04a3, 0x04a4, 0x04a5, 0x04a6, 0x04a7,
- 0x04a8, 0x04a9, 0x04aa, 0x04ab, 0x04ac, 0x04ad, 0x04ae, 0x04af,
- 0x04b0, 0x04b1, 0x04b2, 0x04b3, 0x04b4, 0x04b5, 0x04b6, 0x04b7,
- 0x04b8, 0x04b9, 0x04ba, 0x04bb, 0x04bc, 0x04bd, 0x04be, 0x04bf,
- 0x04c0, 0x0416, 0x0436, 0x04c3, 0x04c4, 0x04c5, 0x04c6, 0x04c7,
- 0x04c8, 0x04c9, 0x04ca, 0x04cb, 0x04cc, 0x04cd, 0x04ce, 0x04cf,
- 0x0410, 0x0430, 0x0410, 0x0430, 0x04d4, 0x04d5, 0x0415, 0x0435,
- 0x04d8, 0x04d9, 0x04d8, 0x04d9, 0x0416, 0x0436, 0x0417, 0x0437,
- 0x04e0, 0x04e1, 0x0418, 0x0438, 0x0418, 0x0438, 0x041e, 0x043e,
- 0x04e8, 0x04e9, 0x04e8, 0x04e9, 0x042d, 0x044d, 0x0423, 0x0443,
- 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04f6, 0x04f7,
- 0x042b, 0x044b, 0x04fa, 0x04fb, 0x04fc, 0x04fd, 0x04fe, 0x04ff,
-};
-
-// generated with:
-// cat UnicodeData.txt | perl -e 'while (<>) { @foo = split(/;/); $foo[5] =~ s/<.*> //; $base[hex($foo[0])] = hex($foo[5]);} for ($i = 0; $i < 0x500; $i += 8) { for ($j = $i; $j < $i + 8; $j++) { printf("0x%04x, ", $base[$j] ? $base[$j] : $j)}; print "\n"; }'
diff --git a/native/src/bigram_dictionary.cpp b/native/src/bigram_dictionary.cpp
deleted file mode 100644
index 11e6dc250..000000000
--- a/native/src/bigram_dictionary.cpp
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
-**
-** Copyright 2010, The Android Open Source Project
-**
-** Licensed under the Apache License, Version 2.0 (the "License");
-** you may not use this file except in compliance with the License.
-** You may obtain a copy of the License at
-**
-** http://www.apache.org/licenses/LICENSE-2.0
-**
-** Unless required by applicable law or agreed to in writing, software
-** distributed under the License is distributed on an "AS IS" BASIS,
-** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-** See the License for the specific language governing permissions and
-** limitations under the License.
-*/
-
-#include <string.h>
-
-#define LOG_TAG "LatinIME: bigram_dictionary.cpp"
-
-#include "bigram_dictionary.h"
-#include "dictionary.h"
-
-namespace latinime {
-
-BigramDictionary::BigramDictionary(const unsigned char *dict, int maxWordLength,
- int maxAlternatives, const bool isLatestDictVersion, const bool hasBigram,
- Dictionary *parentDictionary)
- : DICT(dict), MAX_WORD_LENGTH(maxWordLength),
- MAX_ALTERNATIVES(maxAlternatives), IS_LATEST_DICT_VERSION(isLatestDictVersion),
- HAS_BIGRAM(hasBigram), mParentDictionary(parentDictionary) {
- if (DEBUG_DICT) {
- LOGI("BigramDictionary - constructor");
- LOGI("Has Bigram : %d", hasBigram);
- }
-}
-
-BigramDictionary::~BigramDictionary() {
-}
-
-bool BigramDictionary::addWordBigram(unsigned short *word, int length, int frequency) {
- word[length] = 0;
- if (DEBUG_DICT) {
- char s[length + 1];
- for (int i = 0; i <= length; i++) s[i] = word[i];
- LOGI("Bigram: Found word = %s, freq = %d :", s, frequency);
- }
-
- // Find the right insertion point
- int insertAt = 0;
- while (insertAt < mMaxBigrams) {
- if (frequency > mBigramFreq[insertAt] || (mBigramFreq[insertAt] == frequency
- && length < Dictionary::wideStrLen(mBigramChars + insertAt * MAX_WORD_LENGTH))) {
- break;
- }
- insertAt++;
- }
- if (DEBUG_DICT) {
- LOGI("Bigram: InsertAt -> %d maxBigrams: %d", insertAt, mMaxBigrams);
- }
- if (insertAt < mMaxBigrams) {
- memmove((char*) mBigramFreq + (insertAt + 1) * sizeof(mBigramFreq[0]),
- (char*) mBigramFreq + insertAt * sizeof(mBigramFreq[0]),
- (mMaxBigrams - insertAt - 1) * sizeof(mBigramFreq[0]));
- mBigramFreq[insertAt] = frequency;
- memmove((char*) mBigramChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short),
- (char*) mBigramChars + (insertAt ) * MAX_WORD_LENGTH * sizeof(short),
- (mMaxBigrams - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH);
- unsigned short *dest = mBigramChars + (insertAt ) * MAX_WORD_LENGTH;
- while (length--) {
- *dest++ = *word++;
- }
- *dest = 0; // NULL terminate
- if (DEBUG_DICT) {
- LOGI("Bigram: Added word at %d", insertAt);
- }
- return true;
- }
- return false;
-}
-
-int BigramDictionary::getBigramAddress(int *pos, bool advance) {
- int address = 0;
-
- address += (DICT[*pos] & 0x3F) << 16;
- address += (DICT[*pos + 1] & 0xFF) << 8;
- address += (DICT[*pos + 2] & 0xFF);
-
- if (advance) {
- *pos += 3;
- }
-
- return address;
-}
-
-int BigramDictionary::getBigramFreq(int *pos) {
- int freq = DICT[(*pos)++] & FLAG_BIGRAM_FREQ;
-
- return freq;
-}
-
-
-int BigramDictionary::getBigrams(unsigned short *prevWord, int prevWordLength, int *codes,
- int codesSize, unsigned short *bigramChars, int *bigramFreq, int maxWordLength,
- int maxBigrams, int maxAlternatives) {
- mBigramFreq = bigramFreq;
- mBigramChars = bigramChars;
- mInputCodes = codes;
- mInputLength = codesSize;
- mMaxBigrams = maxBigrams;
-
- if (HAS_BIGRAM && IS_LATEST_DICT_VERSION) {
- int pos = mParentDictionary->getBigramPosition(prevWord, prevWordLength);
- if (DEBUG_DICT) {
- LOGI("Pos -> %d", pos);
- }
- if (pos < 0) {
- return 0;
- }
-
- int bigramCount = 0;
- int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0 && bigramCount < maxBigrams) {
- int bigramAddress = getBigramAddress(&pos, true);
- int frequency = (FLAG_BIGRAM_FREQ & DICT[pos]);
- // search for all bigrams and store them
- searchForTerminalNode(bigramAddress, frequency);
- nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED);
- bigramCount++;
- }
- }
-
- return bigramCount;
- }
- return 0;
-}
-
-void BigramDictionary::searchForTerminalNode(int addressLookingFor, int frequency) {
- // track word with such address and store it in an array
- unsigned short word[MAX_WORD_LENGTH];
-
- int pos;
- int followDownBranchAddress = DICTIONARY_HEADER_SIZE;
- bool found = false;
- char followingChar = ' ';
- int depth = -1;
-
- while(!found) {
- bool followDownAddressSearchStop = false;
- bool firstAddress = true;
- bool haveToSearchAll = true;
-
- if (depth < MAX_WORD_LENGTH && depth >= 0) {
- word[depth] = (unsigned short) followingChar;
- }
- pos = followDownBranchAddress; // pos start at count
- int count = DICT[pos] & 0xFF;
- if (DEBUG_DICT) {
- LOGI("count - %d",count);
- }
- pos++;
- for (int i = 0; i < count; i++) {
- // pos at data
- pos++;
- // pos now at flag
- if (!getFirstBitOfByte(&pos)) { // non-terminal
- if (!followDownAddressSearchStop) {
- int addr = getBigramAddress(&pos, false);
- if (addr > addressLookingFor) {
- followDownAddressSearchStop = true;
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- } else if (!haveToSearchAll) {
- break;
- }
- } else {
- followDownBranchAddress = addr;
- followingChar = (char)(0xFF & DICT[pos-1]);
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = false;
- }
- }
- }
- pos += 3;
- } else if (getFirstBitOfByte(&pos)) { // terminal
- if (addressLookingFor == (pos-1)) { // found !!
- depth++;
- word[depth] = (0xFF & DICT[pos-1]);
- found = true;
- break;
- }
- if (getSecondBitOfByte(&pos)) { // address + freq (4 byte)
- if (!followDownAddressSearchStop) {
- int addr = getBigramAddress(&pos, false);
- if (addr > addressLookingFor) {
- followDownAddressSearchStop = true;
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- } else if (!haveToSearchAll) {
- break;
- }
- } else {
- followDownBranchAddress = addr;
- followingChar = (char)(0xFF & DICT[pos-1]);
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- }
- }
- }
- pos += 4;
- } else { // freq only (2 byte)
- pos += 2;
- }
-
- // skipping bigram
- int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0) {
- pos += 3;
- nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED);
- }
- } else {
- pos++;
- }
- }
- }
- depth++;
- if (followDownBranchAddress == 0) {
- if (DEBUG_DICT) {
- LOGI("ERROR!!! Cannot find bigram!!");
- }
- break;
- }
- }
- if (checkFirstCharacter(word)) {
- addWordBigram(word, depth, frequency);
- }
-}
-
-bool BigramDictionary::checkFirstCharacter(unsigned short *word) {
- // Checks whether this word starts with same character or neighboring characters of
- // what user typed.
-
- int *inputCodes = mInputCodes;
- int maxAlt = MAX_ALTERNATIVES;
- while (maxAlt > 0) {
- if ((unsigned int) *inputCodes == (unsigned int) *word) {
- return true;
- }
- inputCodes++;
- maxAlt--;
- }
- return false;
-}
-
-// TODO: Move functions related to bigram to here
-} // namespace latinime
diff --git a/native/src/bigram_dictionary.h b/native/src/bigram_dictionary.h
deleted file mode 100644
index c07458a38..000000000
--- a/native/src/bigram_dictionary.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_BIGRAM_DICTIONARY_H
-#define LATINIME_BIGRAM_DICTIONARY_H
-
-namespace latinime {
-
-class Dictionary;
-class BigramDictionary {
-public:
- BigramDictionary(const unsigned char *dict, int maxWordLength, int maxAlternatives,
- const bool isLatestDictVersion, const bool hasBigram, Dictionary *parentDictionary);
- int getBigrams(unsigned short *word, int length, int *codes, int codesSize,
- unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams,
- int maxAlternatives);
- ~BigramDictionary();
-private:
- bool addWordBigram(unsigned short *word, int length, int frequency);
- int getBigramAddress(int *pos, bool advance);
- int getBigramFreq(int *pos);
- void searchForTerminalNode(int addressLookingFor, int frequency);
- bool getFirstBitOfByte(int *pos) { return (DICT[*pos] & 0x80) > 0; }
- bool getSecondBitOfByte(int *pos) { return (DICT[*pos] & 0x40) > 0; }
- bool checkFirstCharacter(unsigned short *word);
-
- const unsigned char *DICT;
- const int MAX_WORD_LENGTH;
- const int MAX_ALTERNATIVES;
- const bool IS_LATEST_DICT_VERSION;
- const bool HAS_BIGRAM;
-
- Dictionary *mParentDictionary;
- int *mBigramFreq;
- int mMaxBigrams;
- unsigned short *mBigramChars;
- int *mInputCodes;
- int mInputLength;
-};
-
-} // namespace latinime
-
-#endif // LATINIME_BIGRAM_DICTIONARY_H
diff --git a/native/src/char_utils.cpp b/native/src/char_utils.cpp
deleted file mode 100644
index a31a0632c..000000000
--- a/native/src/char_utils.cpp
+++ /dev/null
@@ -1,899 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <stdlib.h>
-
-namespace latinime {
-
-struct LatinCapitalSmallPair {
- unsigned short capital;
- unsigned short small;
-};
-
-// Generated from http://unicode.org/Public/UNIDATA/UnicodeData.txt
-//
-// 1. Run the following code. Bascially taken from
-// Dictionary::toLowerCase(unsigned short c) in dictionary.cpp.
-// Then, get the list of chars where cc != ccc.
-//
-// unsigned short c, cc, ccc, ccc2;
-// for (c = 0; c < 0xFFFF ; c++) {
-// if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
-// cc = BASE_CHARS[c];
-// } else {
-// cc = c;
-// }
-//
-// // tolower
-// int isBase = 0;
-// if (cc >='A' && cc <= 'Z') {
-// ccc = (cc | 0x20);
-// ccc2 = ccc;
-// isBase = 1;
-// } else if (cc > 0x7F) {
-// ccc = u_tolower(cc);
-// ccc2 = latin_tolower(cc);
-// } else {
-// ccc = cc;
-// ccc2 = ccc;
-// }
-// if (!isBase && cc != ccc) {
-// wprintf(L" 0x%04X => 0x%04X => 0x%04X %lc => %lc => %lc \n",
-// c, cc, ccc, c, cc, ccc);
-// //assert(ccc == ccc2);
-// }
-// }
-//
-// Initially, started with an empty latin_tolower() as below.
-//
-// unsigned short latin_tolower(unsigned short c) {
-// return c;
-// }
-//
-//
-// 2. Process the list obtained by 1 by the following perl script and apply
-// 'sort -u' as well. Get the SORTED_CHAR_MAP[].
-// Note that '$1' in the perl script is 'cc' in the above C code.
-//
-// while(<>) {
-// / 0x\w* => 0x(\w*) =/;
-// open(HDL, "grep -iw ^" . $1 . " UnicodeData.txt | ");
-// $line = <HDL>;
-// chomp $line;
-// @cols = split(/;/, $line);
-// print " { 0x$1, 0x$cols[13] }, // $cols[1]\n";
-// }
-//
-//
-// 3. Update the latin_tolower() function above with SORTED_CHAR_MAP. Enable
-// the assert(ccc == ccc2) above and confirm the function exits successfully.
-//
-static const struct LatinCapitalSmallPair SORTED_CHAR_MAP[] = {
- { 0x00C4, 0x00E4 }, // LATIN CAPITAL LETTER A WITH DIAERESIS
- { 0x00C5, 0x00E5 }, // LATIN CAPITAL LETTER A WITH RING ABOVE
- { 0x00C6, 0x00E6 }, // LATIN CAPITAL LETTER AE
- { 0x00D0, 0x00F0 }, // LATIN CAPITAL LETTER ETH
- { 0x00D5, 0x00F5 }, // LATIN CAPITAL LETTER O WITH TILDE
- { 0x00D6, 0x00F6 }, // LATIN CAPITAL LETTER O WITH DIAERESIS
- { 0x00D8, 0x00F8 }, // LATIN CAPITAL LETTER O WITH STROKE
- { 0x00DC, 0x00FC }, // LATIN CAPITAL LETTER U WITH DIAERESIS
- { 0x00DE, 0x00FE }, // LATIN CAPITAL LETTER THORN
- { 0x0110, 0x0111 }, // LATIN CAPITAL LETTER D WITH STROKE
- { 0x0126, 0x0127 }, // LATIN CAPITAL LETTER H WITH STROKE
- { 0x0141, 0x0142 }, // LATIN CAPITAL LETTER L WITH STROKE
- { 0x014A, 0x014B }, // LATIN CAPITAL LETTER ENG
- { 0x0152, 0x0153 }, // LATIN CAPITAL LIGATURE OE
- { 0x0166, 0x0167 }, // LATIN CAPITAL LETTER T WITH STROKE
- { 0x0181, 0x0253 }, // LATIN CAPITAL LETTER B WITH HOOK
- { 0x0182, 0x0183 }, // LATIN CAPITAL LETTER B WITH TOPBAR
- { 0x0184, 0x0185 }, // LATIN CAPITAL LETTER TONE SIX
- { 0x0186, 0x0254 }, // LATIN CAPITAL LETTER OPEN O
- { 0x0187, 0x0188 }, // LATIN CAPITAL LETTER C WITH HOOK
- { 0x0189, 0x0256 }, // LATIN CAPITAL LETTER AFRICAN D
- { 0x018A, 0x0257 }, // LATIN CAPITAL LETTER D WITH HOOK
- { 0x018B, 0x018C }, // LATIN CAPITAL LETTER D WITH TOPBAR
- { 0x018E, 0x01DD }, // LATIN CAPITAL LETTER REVERSED E
- { 0x018F, 0x0259 }, // LATIN CAPITAL LETTER SCHWA
- { 0x0190, 0x025B }, // LATIN CAPITAL LETTER OPEN E
- { 0x0191, 0x0192 }, // LATIN CAPITAL LETTER F WITH HOOK
- { 0x0193, 0x0260 }, // LATIN CAPITAL LETTER G WITH HOOK
- { 0x0194, 0x0263 }, // LATIN CAPITAL LETTER GAMMA
- { 0x0196, 0x0269 }, // LATIN CAPITAL LETTER IOTA
- { 0x0197, 0x0268 }, // LATIN CAPITAL LETTER I WITH STROKE
- { 0x0198, 0x0199 }, // LATIN CAPITAL LETTER K WITH HOOK
- { 0x019C, 0x026F }, // LATIN CAPITAL LETTER TURNED M
- { 0x019D, 0x0272 }, // LATIN CAPITAL LETTER N WITH LEFT HOOK
- { 0x019F, 0x0275 }, // LATIN CAPITAL LETTER O WITH MIDDLE TILDE
- { 0x01A2, 0x01A3 }, // LATIN CAPITAL LETTER OI
- { 0x01A4, 0x01A5 }, // LATIN CAPITAL LETTER P WITH HOOK
- { 0x01A6, 0x0280 }, // LATIN LETTER YR
- { 0x01A7, 0x01A8 }, // LATIN CAPITAL LETTER TONE TWO
- { 0x01A9, 0x0283 }, // LATIN CAPITAL LETTER ESH
- { 0x01AC, 0x01AD }, // LATIN CAPITAL LETTER T WITH HOOK
- { 0x01AE, 0x0288 }, // LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
- { 0x01B1, 0x028A }, // LATIN CAPITAL LETTER UPSILON
- { 0x01B2, 0x028B }, // LATIN CAPITAL LETTER V WITH HOOK
- { 0x01B3, 0x01B4 }, // LATIN CAPITAL LETTER Y WITH HOOK
- { 0x01B5, 0x01B6 }, // LATIN CAPITAL LETTER Z WITH STROKE
- { 0x01B7, 0x0292 }, // LATIN CAPITAL LETTER EZH
- { 0x01B8, 0x01B9 }, // LATIN CAPITAL LETTER EZH REVERSED
- { 0x01BC, 0x01BD }, // LATIN CAPITAL LETTER TONE FIVE
- { 0x01E4, 0x01E5 }, // LATIN CAPITAL LETTER G WITH STROKE
- { 0x01EA, 0x01EB }, // LATIN CAPITAL LETTER O WITH OGONEK
- { 0x01F6, 0x0195 }, // LATIN CAPITAL LETTER HWAIR
- { 0x01F7, 0x01BF }, // LATIN CAPITAL LETTER WYNN
- { 0x021C, 0x021D }, // LATIN CAPITAL LETTER YOGH
- { 0x0220, 0x019E }, // LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
- { 0x0222, 0x0223 }, // LATIN CAPITAL LETTER OU
- { 0x0224, 0x0225 }, // LATIN CAPITAL LETTER Z WITH HOOK
- { 0x0226, 0x0227 }, // LATIN CAPITAL LETTER A WITH DOT ABOVE
- { 0x022E, 0x022F }, // LATIN CAPITAL LETTER O WITH DOT ABOVE
- { 0x023A, 0x2C65 }, // LATIN CAPITAL LETTER A WITH STROKE
- { 0x023B, 0x023C }, // LATIN CAPITAL LETTER C WITH STROKE
- { 0x023D, 0x019A }, // LATIN CAPITAL LETTER L WITH BAR
- { 0x023E, 0x2C66 }, // LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
- { 0x0241, 0x0242 }, // LATIN CAPITAL LETTER GLOTTAL STOP
- { 0x0243, 0x0180 }, // LATIN CAPITAL LETTER B WITH STROKE
- { 0x0244, 0x0289 }, // LATIN CAPITAL LETTER U BAR
- { 0x0245, 0x028C }, // LATIN CAPITAL LETTER TURNED V
- { 0x0246, 0x0247 }, // LATIN CAPITAL LETTER E WITH STROKE
- { 0x0248, 0x0249 }, // LATIN CAPITAL LETTER J WITH STROKE
- { 0x024A, 0x024B }, // LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
- { 0x024C, 0x024D }, // LATIN CAPITAL LETTER R WITH STROKE
- { 0x024E, 0x024F }, // LATIN CAPITAL LETTER Y WITH STROKE
- { 0x0370, 0x0371 }, // GREEK CAPITAL LETTER HETA
- { 0x0372, 0x0373 }, // GREEK CAPITAL LETTER ARCHAIC SAMPI
- { 0x0376, 0x0377 }, // GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
- { 0x0391, 0x03B1 }, // GREEK CAPITAL LETTER ALPHA
- { 0x0392, 0x03B2 }, // GREEK CAPITAL LETTER BETA
- { 0x0393, 0x03B3 }, // GREEK CAPITAL LETTER GAMMA
- { 0x0394, 0x03B4 }, // GREEK CAPITAL LETTER DELTA
- { 0x0395, 0x03B5 }, // GREEK CAPITAL LETTER EPSILON
- { 0x0396, 0x03B6 }, // GREEK CAPITAL LETTER ZETA
- { 0x0397, 0x03B7 }, // GREEK CAPITAL LETTER ETA
- { 0x0398, 0x03B8 }, // GREEK CAPITAL LETTER THETA
- { 0x0399, 0x03B9 }, // GREEK CAPITAL LETTER IOTA
- { 0x039A, 0x03BA }, // GREEK CAPITAL LETTER KAPPA
- { 0x039B, 0x03BB }, // GREEK CAPITAL LETTER LAMDA
- { 0x039C, 0x03BC }, // GREEK CAPITAL LETTER MU
- { 0x039D, 0x03BD }, // GREEK CAPITAL LETTER NU
- { 0x039E, 0x03BE }, // GREEK CAPITAL LETTER XI
- { 0x039F, 0x03BF }, // GREEK CAPITAL LETTER OMICRON
- { 0x03A0, 0x03C0 }, // GREEK CAPITAL LETTER PI
- { 0x03A1, 0x03C1 }, // GREEK CAPITAL LETTER RHO
- { 0x03A3, 0x03C3 }, // GREEK CAPITAL LETTER SIGMA
- { 0x03A4, 0x03C4 }, // GREEK CAPITAL LETTER TAU
- { 0x03A5, 0x03C5 }, // GREEK CAPITAL LETTER UPSILON
- { 0x03A6, 0x03C6 }, // GREEK CAPITAL LETTER PHI
- { 0x03A7, 0x03C7 }, // GREEK CAPITAL LETTER CHI
- { 0x03A8, 0x03C8 }, // GREEK CAPITAL LETTER PSI
- { 0x03A9, 0x03C9 }, // GREEK CAPITAL LETTER OMEGA
- { 0x03CF, 0x03D7 }, // GREEK CAPITAL KAI SYMBOL
- { 0x03D8, 0x03D9 }, // GREEK LETTER ARCHAIC KOPPA
- { 0x03DA, 0x03DB }, // GREEK LETTER STIGMA
- { 0x03DC, 0x03DD }, // GREEK LETTER DIGAMMA
- { 0x03DE, 0x03DF }, // GREEK LETTER KOPPA
- { 0x03E0, 0x03E1 }, // GREEK LETTER SAMPI
- { 0x03E2, 0x03E3 }, // COPTIC CAPITAL LETTER SHEI
- { 0x03E4, 0x03E5 }, // COPTIC CAPITAL LETTER FEI
- { 0x03E6, 0x03E7 }, // COPTIC CAPITAL LETTER KHEI
- { 0x03E8, 0x03E9 }, // COPTIC CAPITAL LETTER HORI
- { 0x03EA, 0x03EB }, // COPTIC CAPITAL LETTER GANGIA
- { 0x03EC, 0x03ED }, // COPTIC CAPITAL LETTER SHIMA
- { 0x03EE, 0x03EF }, // COPTIC CAPITAL LETTER DEI
- { 0x03F7, 0x03F8 }, // GREEK CAPITAL LETTER SHO
- { 0x03FA, 0x03FB }, // GREEK CAPITAL LETTER SAN
- { 0x03FD, 0x037B }, // GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
- { 0x03FE, 0x037C }, // GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
- { 0x03FF, 0x037D }, // GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
- { 0x0402, 0x0452 }, // CYRILLIC CAPITAL LETTER DJE
- { 0x0404, 0x0454 }, // CYRILLIC CAPITAL LETTER UKRAINIAN IE
- { 0x0405, 0x0455 }, // CYRILLIC CAPITAL LETTER DZE
- { 0x0406, 0x0456 }, // CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
- { 0x0408, 0x0458 }, // CYRILLIC CAPITAL LETTER JE
- { 0x0409, 0x0459 }, // CYRILLIC CAPITAL LETTER LJE
- { 0x040A, 0x045A }, // CYRILLIC CAPITAL LETTER NJE
- { 0x040B, 0x045B }, // CYRILLIC CAPITAL LETTER TSHE
- { 0x040F, 0x045F }, // CYRILLIC CAPITAL LETTER DZHE
- { 0x0410, 0x0430 }, // CYRILLIC CAPITAL LETTER A
- { 0x0411, 0x0431 }, // CYRILLIC CAPITAL LETTER BE
- { 0x0412, 0x0432 }, // CYRILLIC CAPITAL LETTER VE
- { 0x0413, 0x0433 }, // CYRILLIC CAPITAL LETTER GHE
- { 0x0414, 0x0434 }, // CYRILLIC CAPITAL LETTER DE
- { 0x0415, 0x0435 }, // CYRILLIC CAPITAL LETTER IE
- { 0x0416, 0x0436 }, // CYRILLIC CAPITAL LETTER ZHE
- { 0x0417, 0x0437 }, // CYRILLIC CAPITAL LETTER ZE
- { 0x0418, 0x0438 }, // CYRILLIC CAPITAL LETTER I
- { 0x041A, 0x043A }, // CYRILLIC CAPITAL LETTER KA
- { 0x041B, 0x043B }, // CYRILLIC CAPITAL LETTER EL
- { 0x041C, 0x043C }, // CYRILLIC CAPITAL LETTER EM
- { 0x041D, 0x043D }, // CYRILLIC CAPITAL LETTER EN
- { 0x041E, 0x043E }, // CYRILLIC CAPITAL LETTER O
- { 0x041F, 0x043F }, // CYRILLIC CAPITAL LETTER PE
- { 0x0420, 0x0440 }, // CYRILLIC CAPITAL LETTER ER
- { 0x0421, 0x0441 }, // CYRILLIC CAPITAL LETTER ES
- { 0x0422, 0x0442 }, // CYRILLIC CAPITAL LETTER TE
- { 0x0423, 0x0443 }, // CYRILLIC CAPITAL LETTER U
- { 0x0424, 0x0444 }, // CYRILLIC CAPITAL LETTER EF
- { 0x0425, 0x0445 }, // CYRILLIC CAPITAL LETTER HA
- { 0x0426, 0x0446 }, // CYRILLIC CAPITAL LETTER TSE
- { 0x0427, 0x0447 }, // CYRILLIC CAPITAL LETTER CHE
- { 0x0428, 0x0448 }, // CYRILLIC CAPITAL LETTER SHA
- { 0x0429, 0x0449 }, // CYRILLIC CAPITAL LETTER SHCHA
- { 0x042A, 0x044A }, // CYRILLIC CAPITAL LETTER HARD SIGN
- { 0x042B, 0x044B }, // CYRILLIC CAPITAL LETTER YERU
- { 0x042C, 0x044C }, // CYRILLIC CAPITAL LETTER SOFT SIGN
- { 0x042D, 0x044D }, // CYRILLIC CAPITAL LETTER E
- { 0x042E, 0x044E }, // CYRILLIC CAPITAL LETTER YU
- { 0x042F, 0x044F }, // CYRILLIC CAPITAL LETTER YA
- { 0x0460, 0x0461 }, // CYRILLIC CAPITAL LETTER OMEGA
- { 0x0462, 0x0463 }, // CYRILLIC CAPITAL LETTER YAT
- { 0x0464, 0x0465 }, // CYRILLIC CAPITAL LETTER IOTIFIED E
- { 0x0466, 0x0467 }, // CYRILLIC CAPITAL LETTER LITTLE YUS
- { 0x0468, 0x0469 }, // CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
- { 0x046A, 0x046B }, // CYRILLIC CAPITAL LETTER BIG YUS
- { 0x046C, 0x046D }, // CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
- { 0x046E, 0x046F }, // CYRILLIC CAPITAL LETTER KSI
- { 0x0470, 0x0471 }, // CYRILLIC CAPITAL LETTER PSI
- { 0x0472, 0x0473 }, // CYRILLIC CAPITAL LETTER FITA
- { 0x0474, 0x0475 }, // CYRILLIC CAPITAL LETTER IZHITSA
- { 0x0478, 0x0479 }, // CYRILLIC CAPITAL LETTER UK
- { 0x047A, 0x047B }, // CYRILLIC CAPITAL LETTER ROUND OMEGA
- { 0x047C, 0x047D }, // CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
- { 0x047E, 0x047F }, // CYRILLIC CAPITAL LETTER OT
- { 0x0480, 0x0481 }, // CYRILLIC CAPITAL LETTER KOPPA
- { 0x048A, 0x048B }, // CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
- { 0x048C, 0x048D }, // CYRILLIC CAPITAL LETTER SEMISOFT SIGN
- { 0x048E, 0x048F }, // CYRILLIC CAPITAL LETTER ER WITH TICK
- { 0x0490, 0x0491 }, // CYRILLIC CAPITAL LETTER GHE WITH UPTURN
- { 0x0492, 0x0493 }, // CYRILLIC CAPITAL LETTER GHE WITH STROKE
- { 0x0494, 0x0495 }, // CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
- { 0x0496, 0x0497 }, // CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
- { 0x0498, 0x0499 }, // CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
- { 0x049A, 0x049B }, // CYRILLIC CAPITAL LETTER KA WITH DESCENDER
- { 0x049C, 0x049D }, // CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
- { 0x049E, 0x049F }, // CYRILLIC CAPITAL LETTER KA WITH STROKE
- { 0x04A0, 0x04A1 }, // CYRILLIC CAPITAL LETTER BASHKIR KA
- { 0x04A2, 0x04A3 }, // CYRILLIC CAPITAL LETTER EN WITH DESCENDER
- { 0x04A4, 0x04A5 }, // CYRILLIC CAPITAL LIGATURE EN GHE
- { 0x04A6, 0x04A7 }, // CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
- { 0x04A8, 0x04A9 }, // CYRILLIC CAPITAL LETTER ABKHASIAN HA
- { 0x04AA, 0x04AB }, // CYRILLIC CAPITAL LETTER ES WITH DESCENDER
- { 0x04AC, 0x04AD }, // CYRILLIC CAPITAL LETTER TE WITH DESCENDER
- { 0x04AE, 0x04AF }, // CYRILLIC CAPITAL LETTER STRAIGHT U
- { 0x04B0, 0x04B1 }, // CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
- { 0x04B2, 0x04B3 }, // CYRILLIC CAPITAL LETTER HA WITH DESCENDER
- { 0x04B4, 0x04B5 }, // CYRILLIC CAPITAL LIGATURE TE TSE
- { 0x04B6, 0x04B7 }, // CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
- { 0x04B8, 0x04B9 }, // CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
- { 0x04BA, 0x04BB }, // CYRILLIC CAPITAL LETTER SHHA
- { 0x04BC, 0x04BD }, // CYRILLIC CAPITAL LETTER ABKHASIAN CHE
- { 0x04BE, 0x04BF }, // CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
- { 0x04C0, 0x04CF }, // CYRILLIC LETTER PALOCHKA
- { 0x04C3, 0x04C4 }, // CYRILLIC CAPITAL LETTER KA WITH HOOK
- { 0x04C5, 0x04C6 }, // CYRILLIC CAPITAL LETTER EL WITH TAIL
- { 0x04C7, 0x04C8 }, // CYRILLIC CAPITAL LETTER EN WITH HOOK
- { 0x04C9, 0x04CA }, // CYRILLIC CAPITAL LETTER EN WITH TAIL
- { 0x04CB, 0x04CC }, // CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
- { 0x04CD, 0x04CE }, // CYRILLIC CAPITAL LETTER EM WITH TAIL
- { 0x04D4, 0x04D5 }, // CYRILLIC CAPITAL LIGATURE A IE
- { 0x04D8, 0x04D9 }, // CYRILLIC CAPITAL LETTER SCHWA
- { 0x04E0, 0x04E1 }, // CYRILLIC CAPITAL LETTER ABKHASIAN DZE
- { 0x04E8, 0x04E9 }, // CYRILLIC CAPITAL LETTER BARRED O
- { 0x04F6, 0x04F7 }, // CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
- { 0x04FA, 0x04FB }, // CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
- { 0x04FC, 0x04FD }, // CYRILLIC CAPITAL LETTER HA WITH HOOK
- { 0x04FE, 0x04FF }, // CYRILLIC CAPITAL LETTER HA WITH STROKE
- { 0x0500, 0x0501 }, // CYRILLIC CAPITAL LETTER KOMI DE
- { 0x0502, 0x0503 }, // CYRILLIC CAPITAL LETTER KOMI DJE
- { 0x0504, 0x0505 }, // CYRILLIC CAPITAL LETTER KOMI ZJE
- { 0x0506, 0x0507 }, // CYRILLIC CAPITAL LETTER KOMI DZJE
- { 0x0508, 0x0509 }, // CYRILLIC CAPITAL LETTER KOMI LJE
- { 0x050A, 0x050B }, // CYRILLIC CAPITAL LETTER KOMI NJE
- { 0x050C, 0x050D }, // CYRILLIC CAPITAL LETTER KOMI SJE
- { 0x050E, 0x050F }, // CYRILLIC CAPITAL LETTER KOMI TJE
- { 0x0510, 0x0511 }, // CYRILLIC CAPITAL LETTER REVERSED ZE
- { 0x0512, 0x0513 }, // CYRILLIC CAPITAL LETTER EL WITH HOOK
- { 0x0514, 0x0515 }, // CYRILLIC CAPITAL LETTER LHA
- { 0x0516, 0x0517 }, // CYRILLIC CAPITAL LETTER RHA
- { 0x0518, 0x0519 }, // CYRILLIC CAPITAL LETTER YAE
- { 0x051A, 0x051B }, // CYRILLIC CAPITAL LETTER QA
- { 0x051C, 0x051D }, // CYRILLIC CAPITAL LETTER WE
- { 0x051E, 0x051F }, // CYRILLIC CAPITAL LETTER ALEUT KA
- { 0x0520, 0x0521 }, // CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
- { 0x0522, 0x0523 }, // CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
- { 0x0524, 0x0525 }, // CYRILLIC CAPITAL LETTER PE WITH DESCENDER
- { 0x0531, 0x0561 }, // ARMENIAN CAPITAL LETTER AYB
- { 0x0532, 0x0562 }, // ARMENIAN CAPITAL LETTER BEN
- { 0x0533, 0x0563 }, // ARMENIAN CAPITAL LETTER GIM
- { 0x0534, 0x0564 }, // ARMENIAN CAPITAL LETTER DA
- { 0x0535, 0x0565 }, // ARMENIAN CAPITAL LETTER ECH
- { 0x0536, 0x0566 }, // ARMENIAN CAPITAL LETTER ZA
- { 0x0537, 0x0567 }, // ARMENIAN CAPITAL LETTER EH
- { 0x0538, 0x0568 }, // ARMENIAN CAPITAL LETTER ET
- { 0x0539, 0x0569 }, // ARMENIAN CAPITAL LETTER TO
- { 0x053A, 0x056A }, // ARMENIAN CAPITAL LETTER ZHE
- { 0x053B, 0x056B }, // ARMENIAN CAPITAL LETTER INI
- { 0x053C, 0x056C }, // ARMENIAN CAPITAL LETTER LIWN
- { 0x053D, 0x056D }, // ARMENIAN CAPITAL LETTER XEH
- { 0x053E, 0x056E }, // ARMENIAN CAPITAL LETTER CA
- { 0x053F, 0x056F }, // ARMENIAN CAPITAL LETTER KEN
- { 0x0540, 0x0570 }, // ARMENIAN CAPITAL LETTER HO
- { 0x0541, 0x0571 }, // ARMENIAN CAPITAL LETTER JA
- { 0x0542, 0x0572 }, // ARMENIAN CAPITAL LETTER GHAD
- { 0x0543, 0x0573 }, // ARMENIAN CAPITAL LETTER CHEH
- { 0x0544, 0x0574 }, // ARMENIAN CAPITAL LETTER MEN
- { 0x0545, 0x0575 }, // ARMENIAN CAPITAL LETTER YI
- { 0x0546, 0x0576 }, // ARMENIAN CAPITAL LETTER NOW
- { 0x0547, 0x0577 }, // ARMENIAN CAPITAL LETTER SHA
- { 0x0548, 0x0578 }, // ARMENIAN CAPITAL LETTER VO
- { 0x0549, 0x0579 }, // ARMENIAN CAPITAL LETTER CHA
- { 0x054A, 0x057A }, // ARMENIAN CAPITAL LETTER PEH
- { 0x054B, 0x057B }, // ARMENIAN CAPITAL LETTER JHEH
- { 0x054C, 0x057C }, // ARMENIAN CAPITAL LETTER RA
- { 0x054D, 0x057D }, // ARMENIAN CAPITAL LETTER SEH
- { 0x054E, 0x057E }, // ARMENIAN CAPITAL LETTER VEW
- { 0x054F, 0x057F }, // ARMENIAN CAPITAL LETTER TIWN
- { 0x0550, 0x0580 }, // ARMENIAN CAPITAL LETTER REH
- { 0x0551, 0x0581 }, // ARMENIAN CAPITAL LETTER CO
- { 0x0552, 0x0582 }, // ARMENIAN CAPITAL LETTER YIWN
- { 0x0553, 0x0583 }, // ARMENIAN CAPITAL LETTER PIWR
- { 0x0554, 0x0584 }, // ARMENIAN CAPITAL LETTER KEH
- { 0x0555, 0x0585 }, // ARMENIAN CAPITAL LETTER OH
- { 0x0556, 0x0586 }, // ARMENIAN CAPITAL LETTER FEH
- { 0x10A0, 0x2D00 }, // GEORGIAN CAPITAL LETTER AN
- { 0x10A1, 0x2D01 }, // GEORGIAN CAPITAL LETTER BAN
- { 0x10A2, 0x2D02 }, // GEORGIAN CAPITAL LETTER GAN
- { 0x10A3, 0x2D03 }, // GEORGIAN CAPITAL LETTER DON
- { 0x10A4, 0x2D04 }, // GEORGIAN CAPITAL LETTER EN
- { 0x10A5, 0x2D05 }, // GEORGIAN CAPITAL LETTER VIN
- { 0x10A6, 0x2D06 }, // GEORGIAN CAPITAL LETTER ZEN
- { 0x10A7, 0x2D07 }, // GEORGIAN CAPITAL LETTER TAN
- { 0x10A8, 0x2D08 }, // GEORGIAN CAPITAL LETTER IN
- { 0x10A9, 0x2D09 }, // GEORGIAN CAPITAL LETTER KAN
- { 0x10AA, 0x2D0A }, // GEORGIAN CAPITAL LETTER LAS
- { 0x10AB, 0x2D0B }, // GEORGIAN CAPITAL LETTER MAN
- { 0x10AC, 0x2D0C }, // GEORGIAN CAPITAL LETTER NAR
- { 0x10AD, 0x2D0D }, // GEORGIAN CAPITAL LETTER ON
- { 0x10AE, 0x2D0E }, // GEORGIAN CAPITAL LETTER PAR
- { 0x10AF, 0x2D0F }, // GEORGIAN CAPITAL LETTER ZHAR
- { 0x10B0, 0x2D10 }, // GEORGIAN CAPITAL LETTER RAE
- { 0x10B1, 0x2D11 }, // GEORGIAN CAPITAL LETTER SAN
- { 0x10B2, 0x2D12 }, // GEORGIAN CAPITAL LETTER TAR
- { 0x10B3, 0x2D13 }, // GEORGIAN CAPITAL LETTER UN
- { 0x10B4, 0x2D14 }, // GEORGIAN CAPITAL LETTER PHAR
- { 0x10B5, 0x2D15 }, // GEORGIAN CAPITAL LETTER KHAR
- { 0x10B6, 0x2D16 }, // GEORGIAN CAPITAL LETTER GHAN
- { 0x10B7, 0x2D17 }, // GEORGIAN CAPITAL LETTER QAR
- { 0x10B8, 0x2D18 }, // GEORGIAN CAPITAL LETTER SHIN
- { 0x10B9, 0x2D19 }, // GEORGIAN CAPITAL LETTER CHIN
- { 0x10BA, 0x2D1A }, // GEORGIAN CAPITAL LETTER CAN
- { 0x10BB, 0x2D1B }, // GEORGIAN CAPITAL LETTER JIL
- { 0x10BC, 0x2D1C }, // GEORGIAN CAPITAL LETTER CIL
- { 0x10BD, 0x2D1D }, // GEORGIAN CAPITAL LETTER CHAR
- { 0x10BE, 0x2D1E }, // GEORGIAN CAPITAL LETTER XAN
- { 0x10BF, 0x2D1F }, // GEORGIAN CAPITAL LETTER JHAN
- { 0x10C0, 0x2D20 }, // GEORGIAN CAPITAL LETTER HAE
- { 0x10C1, 0x2D21 }, // GEORGIAN CAPITAL LETTER HE
- { 0x10C2, 0x2D22 }, // GEORGIAN CAPITAL LETTER HIE
- { 0x10C3, 0x2D23 }, // GEORGIAN CAPITAL LETTER WE
- { 0x10C4, 0x2D24 }, // GEORGIAN CAPITAL LETTER HAR
- { 0x10C5, 0x2D25 }, // GEORGIAN CAPITAL LETTER HOE
- { 0x1E00, 0x1E01 }, // LATIN CAPITAL LETTER A WITH RING BELOW
- { 0x1E02, 0x1E03 }, // LATIN CAPITAL LETTER B WITH DOT ABOVE
- { 0x1E04, 0x1E05 }, // LATIN CAPITAL LETTER B WITH DOT BELOW
- { 0x1E06, 0x1E07 }, // LATIN CAPITAL LETTER B WITH LINE BELOW
- { 0x1E08, 0x1E09 }, // LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
- { 0x1E0A, 0x1E0B }, // LATIN CAPITAL LETTER D WITH DOT ABOVE
- { 0x1E0C, 0x1E0D }, // LATIN CAPITAL LETTER D WITH DOT BELOW
- { 0x1E0E, 0x1E0F }, // LATIN CAPITAL LETTER D WITH LINE BELOW
- { 0x1E10, 0x1E11 }, // LATIN CAPITAL LETTER D WITH CEDILLA
- { 0x1E12, 0x1E13 }, // LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
- { 0x1E14, 0x1E15 }, // LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
- { 0x1E16, 0x1E17 }, // LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
- { 0x1E18, 0x1E19 }, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
- { 0x1E1A, 0x1E1B }, // LATIN CAPITAL LETTER E WITH TILDE BELOW
- { 0x1E1C, 0x1E1D }, // LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
- { 0x1E1E, 0x1E1F }, // LATIN CAPITAL LETTER F WITH DOT ABOVE
- { 0x1E20, 0x1E21 }, // LATIN CAPITAL LETTER G WITH MACRON
- { 0x1E22, 0x1E23 }, // LATIN CAPITAL LETTER H WITH DOT ABOVE
- { 0x1E24, 0x1E25 }, // LATIN CAPITAL LETTER H WITH DOT BELOW
- { 0x1E26, 0x1E27 }, // LATIN CAPITAL LETTER H WITH DIAERESIS
- { 0x1E28, 0x1E29 }, // LATIN CAPITAL LETTER H WITH CEDILLA
- { 0x1E2A, 0x1E2B }, // LATIN CAPITAL LETTER H WITH BREVE BELOW
- { 0x1E2C, 0x1E2D }, // LATIN CAPITAL LETTER I WITH TILDE BELOW
- { 0x1E2E, 0x1E2F }, // LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
- { 0x1E30, 0x1E31 }, // LATIN CAPITAL LETTER K WITH ACUTE
- { 0x1E32, 0x1E33 }, // LATIN CAPITAL LETTER K WITH DOT BELOW
- { 0x1E34, 0x1E35 }, // LATIN CAPITAL LETTER K WITH LINE BELOW
- { 0x1E36, 0x1E37 }, // LATIN CAPITAL LETTER L WITH DOT BELOW
- { 0x1E38, 0x1E39 }, // LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
- { 0x1E3A, 0x1E3B }, // LATIN CAPITAL LETTER L WITH LINE BELOW
- { 0x1E3C, 0x1E3D }, // LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
- { 0x1E3E, 0x1E3F }, // LATIN CAPITAL LETTER M WITH ACUTE
- { 0x1E40, 0x1E41 }, // LATIN CAPITAL LETTER M WITH DOT ABOVE
- { 0x1E42, 0x1E43 }, // LATIN CAPITAL LETTER M WITH DOT BELOW
- { 0x1E44, 0x1E45 }, // LATIN CAPITAL LETTER N WITH DOT ABOVE
- { 0x1E46, 0x1E47 }, // LATIN CAPITAL LETTER N WITH DOT BELOW
- { 0x1E48, 0x1E49 }, // LATIN CAPITAL LETTER N WITH LINE BELOW
- { 0x1E4A, 0x1E4B }, // LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
- { 0x1E4C, 0x1E4D }, // LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
- { 0x1E4E, 0x1E4F }, // LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
- { 0x1E50, 0x1E51 }, // LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
- { 0x1E52, 0x1E53 }, // LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
- { 0x1E54, 0x1E55 }, // LATIN CAPITAL LETTER P WITH ACUTE
- { 0x1E56, 0x1E57 }, // LATIN CAPITAL LETTER P WITH DOT ABOVE
- { 0x1E58, 0x1E59 }, // LATIN CAPITAL LETTER R WITH DOT ABOVE
- { 0x1E5A, 0x1E5B }, // LATIN CAPITAL LETTER R WITH DOT BELOW
- { 0x1E5C, 0x1E5D }, // LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
- { 0x1E5E, 0x1E5F }, // LATIN CAPITAL LETTER R WITH LINE BELOW
- { 0x1E60, 0x1E61 }, // LATIN CAPITAL LETTER S WITH DOT ABOVE
- { 0x1E62, 0x1E63 }, // LATIN CAPITAL LETTER S WITH DOT BELOW
- { 0x1E64, 0x1E65 }, // LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
- { 0x1E66, 0x1E67 }, // LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
- { 0x1E68, 0x1E69 }, // LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
- { 0x1E6A, 0x1E6B }, // LATIN CAPITAL LETTER T WITH DOT ABOVE
- { 0x1E6C, 0x1E6D }, // LATIN CAPITAL LETTER T WITH DOT BELOW
- { 0x1E6E, 0x1E6F }, // LATIN CAPITAL LETTER T WITH LINE BELOW
- { 0x1E70, 0x1E71 }, // LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
- { 0x1E72, 0x1E73 }, // LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
- { 0x1E74, 0x1E75 }, // LATIN CAPITAL LETTER U WITH TILDE BELOW
- { 0x1E76, 0x1E77 }, // LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
- { 0x1E78, 0x1E79 }, // LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
- { 0x1E7A, 0x1E7B }, // LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
- { 0x1E7C, 0x1E7D }, // LATIN CAPITAL LETTER V WITH TILDE
- { 0x1E7E, 0x1E7F }, // LATIN CAPITAL LETTER V WITH DOT BELOW
- { 0x1E80, 0x1E81 }, // LATIN CAPITAL LETTER W WITH GRAVE
- { 0x1E82, 0x1E83 }, // LATIN CAPITAL LETTER W WITH ACUTE
- { 0x1E84, 0x1E85 }, // LATIN CAPITAL LETTER W WITH DIAERESIS
- { 0x1E86, 0x1E87 }, // LATIN CAPITAL LETTER W WITH DOT ABOVE
- { 0x1E88, 0x1E89 }, // LATIN CAPITAL LETTER W WITH DOT BELOW
- { 0x1E8A, 0x1E8B }, // LATIN CAPITAL LETTER X WITH DOT ABOVE
- { 0x1E8C, 0x1E8D }, // LATIN CAPITAL LETTER X WITH DIAERESIS
- { 0x1E8E, 0x1E8F }, // LATIN CAPITAL LETTER Y WITH DOT ABOVE
- { 0x1E90, 0x1E91 }, // LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
- { 0x1E92, 0x1E93 }, // LATIN CAPITAL LETTER Z WITH DOT BELOW
- { 0x1E94, 0x1E95 }, // LATIN CAPITAL LETTER Z WITH LINE BELOW
- { 0x1E9E, 0x00DF }, // LATIN CAPITAL LETTER SHARP S
- { 0x1EA0, 0x1EA1 }, // LATIN CAPITAL LETTER A WITH DOT BELOW
- { 0x1EA2, 0x1EA3 }, // LATIN CAPITAL LETTER A WITH HOOK ABOVE
- { 0x1EA4, 0x1EA5 }, // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
- { 0x1EA6, 0x1EA7 }, // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
- { 0x1EA8, 0x1EA9 }, // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
- { 0x1EAA, 0x1EAB }, // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
- { 0x1EAC, 0x1EAD }, // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
- { 0x1EAE, 0x1EAF }, // LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
- { 0x1EB0, 0x1EB1 }, // LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
- { 0x1EB2, 0x1EB3 }, // LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
- { 0x1EB4, 0x1EB5 }, // LATIN CAPITAL LETTER A WITH BREVE AND TILDE
- { 0x1EB6, 0x1EB7 }, // LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
- { 0x1EB8, 0x1EB9 }, // LATIN CAPITAL LETTER E WITH DOT BELOW
- { 0x1EBA, 0x1EBB }, // LATIN CAPITAL LETTER E WITH HOOK ABOVE
- { 0x1EBC, 0x1EBD }, // LATIN CAPITAL LETTER E WITH TILDE
- { 0x1EBE, 0x1EBF }, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
- { 0x1EC0, 0x1EC1 }, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
- { 0x1EC2, 0x1EC3 }, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
- { 0x1EC4, 0x1EC5 }, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
- { 0x1EC6, 0x1EC7 }, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
- { 0x1EC8, 0x1EC9 }, // LATIN CAPITAL LETTER I WITH HOOK ABOVE
- { 0x1ECA, 0x1ECB }, // LATIN CAPITAL LETTER I WITH DOT BELOW
- { 0x1ECC, 0x1ECD }, // LATIN CAPITAL LETTER O WITH DOT BELOW
- { 0x1ECE, 0x1ECF }, // LATIN CAPITAL LETTER O WITH HOOK ABOVE
- { 0x1ED0, 0x1ED1 }, // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
- { 0x1ED2, 0x1ED3 }, // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
- { 0x1ED4, 0x1ED5 }, // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
- { 0x1ED6, 0x1ED7 }, // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
- { 0x1ED8, 0x1ED9 }, // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
- { 0x1EDA, 0x1EDB }, // LATIN CAPITAL LETTER O WITH HORN AND ACUTE
- { 0x1EDC, 0x1EDD }, // LATIN CAPITAL LETTER O WITH HORN AND GRAVE
- { 0x1EDE, 0x1EDF }, // LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
- { 0x1EE0, 0x1EE1 }, // LATIN CAPITAL LETTER O WITH HORN AND TILDE
- { 0x1EE2, 0x1EE3 }, // LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
- { 0x1EE4, 0x1EE5 }, // LATIN CAPITAL LETTER U WITH DOT BELOW
- { 0x1EE6, 0x1EE7 }, // LATIN CAPITAL LETTER U WITH HOOK ABOVE
- { 0x1EE8, 0x1EE9 }, // LATIN CAPITAL LETTER U WITH HORN AND ACUTE
- { 0x1EEA, 0x1EEB }, // LATIN CAPITAL LETTER U WITH HORN AND GRAVE
- { 0x1EEC, 0x1EED }, // LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
- { 0x1EEE, 0x1EEF }, // LATIN CAPITAL LETTER U WITH HORN AND TILDE
- { 0x1EF0, 0x1EF1 }, // LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
- { 0x1EF2, 0x1EF3 }, // LATIN CAPITAL LETTER Y WITH GRAVE
- { 0x1EF4, 0x1EF5 }, // LATIN CAPITAL LETTER Y WITH DOT BELOW
- { 0x1EF6, 0x1EF7 }, // LATIN CAPITAL LETTER Y WITH HOOK ABOVE
- { 0x1EF8, 0x1EF9 }, // LATIN CAPITAL LETTER Y WITH TILDE
- { 0x1EFA, 0x1EFB }, // LATIN CAPITAL LETTER MIDDLE-WELSH LL
- { 0x1EFC, 0x1EFD }, // LATIN CAPITAL LETTER MIDDLE-WELSH V
- { 0x1EFE, 0x1EFF }, // LATIN CAPITAL LETTER Y WITH LOOP
- { 0x1F08, 0x1F00 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI
- { 0x1F09, 0x1F01 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA
- { 0x1F0A, 0x1F02 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
- { 0x1F0B, 0x1F03 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
- { 0x1F0C, 0x1F04 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
- { 0x1F0D, 0x1F05 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
- { 0x1F0E, 0x1F06 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
- { 0x1F0F, 0x1F07 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
- { 0x1F18, 0x1F10 }, // GREEK CAPITAL LETTER EPSILON WITH PSILI
- { 0x1F19, 0x1F11 }, // GREEK CAPITAL LETTER EPSILON WITH DASIA
- { 0x1F1A, 0x1F12 }, // GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
- { 0x1F1B, 0x1F13 }, // GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
- { 0x1F1C, 0x1F14 }, // GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
- { 0x1F1D, 0x1F15 }, // GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
- { 0x1F28, 0x1F20 }, // GREEK CAPITAL LETTER ETA WITH PSILI
- { 0x1F29, 0x1F21 }, // GREEK CAPITAL LETTER ETA WITH DASIA
- { 0x1F2A, 0x1F22 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
- { 0x1F2B, 0x1F23 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
- { 0x1F2C, 0x1F24 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
- { 0x1F2D, 0x1F25 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
- { 0x1F2E, 0x1F26 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
- { 0x1F2F, 0x1F27 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
- { 0x1F38, 0x1F30 }, // GREEK CAPITAL LETTER IOTA WITH PSILI
- { 0x1F39, 0x1F31 }, // GREEK CAPITAL LETTER IOTA WITH DASIA
- { 0x1F3A, 0x1F32 }, // GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
- { 0x1F3B, 0x1F33 }, // GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
- { 0x1F3C, 0x1F34 }, // GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
- { 0x1F3D, 0x1F35 }, // GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
- { 0x1F3E, 0x1F36 }, // GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
- { 0x1F3F, 0x1F37 }, // GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
- { 0x1F48, 0x1F40 }, // GREEK CAPITAL LETTER OMICRON WITH PSILI
- { 0x1F49, 0x1F41 }, // GREEK CAPITAL LETTER OMICRON WITH DASIA
- { 0x1F4A, 0x1F42 }, // GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
- { 0x1F4B, 0x1F43 }, // GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
- { 0x1F4C, 0x1F44 }, // GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
- { 0x1F4D, 0x1F45 }, // GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
- { 0x1F59, 0x1F51 }, // GREEK CAPITAL LETTER UPSILON WITH DASIA
- { 0x1F5B, 0x1F53 }, // GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
- { 0x1F5D, 0x1F55 }, // GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
- { 0x1F5F, 0x1F57 }, // GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
- { 0x1F68, 0x1F60 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI
- { 0x1F69, 0x1F61 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA
- { 0x1F6A, 0x1F62 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
- { 0x1F6B, 0x1F63 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
- { 0x1F6C, 0x1F64 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
- { 0x1F6D, 0x1F65 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
- { 0x1F6E, 0x1F66 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
- { 0x1F6F, 0x1F67 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
- { 0x1F88, 0x1F80 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
- { 0x1F89, 0x1F81 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
- { 0x1F8A, 0x1F82 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
- { 0x1F8B, 0x1F83 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
- { 0x1F8C, 0x1F84 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
- { 0x1F8D, 0x1F85 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
- { 0x1F8E, 0x1F86 }, // GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
- { 0x1F8F, 0x1F87 }, // GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
- { 0x1F98, 0x1F90 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
- { 0x1F99, 0x1F91 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
- { 0x1F9A, 0x1F92 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
- { 0x1F9B, 0x1F93 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
- { 0x1F9C, 0x1F94 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
- { 0x1F9D, 0x1F95 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
- { 0x1F9E, 0x1F96 }, // GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
- { 0x1F9F, 0x1F97 }, // GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
- { 0x1FA8, 0x1FA0 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
- { 0x1FA9, 0x1FA1 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
- { 0x1FAA, 0x1FA2 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
- { 0x1FAB, 0x1FA3 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
- { 0x1FAC, 0x1FA4 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
- { 0x1FAD, 0x1FA5 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
- { 0x1FAE, 0x1FA6 }, // GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
- { 0x1FAF, 0x1FA7 }, // GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
- { 0x1FB8, 0x1FB0 }, // GREEK CAPITAL LETTER ALPHA WITH VRACHY
- { 0x1FB9, 0x1FB1 }, // GREEK CAPITAL LETTER ALPHA WITH MACRON
- { 0x1FBA, 0x1F70 }, // GREEK CAPITAL LETTER ALPHA WITH VARIA
- { 0x1FBB, 0x1F71 }, // GREEK CAPITAL LETTER ALPHA WITH OXIA
- { 0x1FBC, 0x1FB3 }, // GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
- { 0x1FC8, 0x1F72 }, // GREEK CAPITAL LETTER EPSILON WITH VARIA
- { 0x1FC9, 0x1F73 }, // GREEK CAPITAL LETTER EPSILON WITH OXIA
- { 0x1FCA, 0x1F74 }, // GREEK CAPITAL LETTER ETA WITH VARIA
- { 0x1FCB, 0x1F75 }, // GREEK CAPITAL LETTER ETA WITH OXIA
- { 0x1FCC, 0x1FC3 }, // GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
- { 0x1FD8, 0x1FD0 }, // GREEK CAPITAL LETTER IOTA WITH VRACHY
- { 0x1FD9, 0x1FD1 }, // GREEK CAPITAL LETTER IOTA WITH MACRON
- { 0x1FDA, 0x1F76 }, // GREEK CAPITAL LETTER IOTA WITH VARIA
- { 0x1FDB, 0x1F77 }, // GREEK CAPITAL LETTER IOTA WITH OXIA
- { 0x1FE8, 0x1FE0 }, // GREEK CAPITAL LETTER UPSILON WITH VRACHY
- { 0x1FE9, 0x1FE1 }, // GREEK CAPITAL LETTER UPSILON WITH MACRON
- { 0x1FEA, 0x1F7A }, // GREEK CAPITAL LETTER UPSILON WITH VARIA
- { 0x1FEB, 0x1F7B }, // GREEK CAPITAL LETTER UPSILON WITH OXIA
- { 0x1FEC, 0x1FE5 }, // GREEK CAPITAL LETTER RHO WITH DASIA
- { 0x1FF8, 0x1F78 }, // GREEK CAPITAL LETTER OMICRON WITH VARIA
- { 0x1FF9, 0x1F79 }, // GREEK CAPITAL LETTER OMICRON WITH OXIA
- { 0x1FFA, 0x1F7C }, // GREEK CAPITAL LETTER OMEGA WITH VARIA
- { 0x1FFB, 0x1F7D }, // GREEK CAPITAL LETTER OMEGA WITH OXIA
- { 0x1FFC, 0x1FF3 }, // GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
- { 0x2126, 0x03C9 }, // OHM SIGN
- { 0x212A, 0x006B }, // KELVIN SIGN
- { 0x212B, 0x00E5 }, // ANGSTROM SIGN
- { 0x2132, 0x214E }, // TURNED CAPITAL F
- { 0x2160, 0x2170 }, // ROMAN NUMERAL ONE
- { 0x2161, 0x2171 }, // ROMAN NUMERAL TWO
- { 0x2162, 0x2172 }, // ROMAN NUMERAL THREE
- { 0x2163, 0x2173 }, // ROMAN NUMERAL FOUR
- { 0x2164, 0x2174 }, // ROMAN NUMERAL FIVE
- { 0x2165, 0x2175 }, // ROMAN NUMERAL SIX
- { 0x2166, 0x2176 }, // ROMAN NUMERAL SEVEN
- { 0x2167, 0x2177 }, // ROMAN NUMERAL EIGHT
- { 0x2168, 0x2178 }, // ROMAN NUMERAL NINE
- { 0x2169, 0x2179 }, // ROMAN NUMERAL TEN
- { 0x216A, 0x217A }, // ROMAN NUMERAL ELEVEN
- { 0x216B, 0x217B }, // ROMAN NUMERAL TWELVE
- { 0x216C, 0x217C }, // ROMAN NUMERAL FIFTY
- { 0x216D, 0x217D }, // ROMAN NUMERAL ONE HUNDRED
- { 0x216E, 0x217E }, // ROMAN NUMERAL FIVE HUNDRED
- { 0x216F, 0x217F }, // ROMAN NUMERAL ONE THOUSAND
- { 0x2183, 0x2184 }, // ROMAN NUMERAL REVERSED ONE HUNDRED
- { 0x24B6, 0x24D0 }, // CIRCLED LATIN CAPITAL LETTER A
- { 0x24B7, 0x24D1 }, // CIRCLED LATIN CAPITAL LETTER B
- { 0x24B8, 0x24D2 }, // CIRCLED LATIN CAPITAL LETTER C
- { 0x24B9, 0x24D3 }, // CIRCLED LATIN CAPITAL LETTER D
- { 0x24BA, 0x24D4 }, // CIRCLED LATIN CAPITAL LETTER E
- { 0x24BB, 0x24D5 }, // CIRCLED LATIN CAPITAL LETTER F
- { 0x24BC, 0x24D6 }, // CIRCLED LATIN CAPITAL LETTER G
- { 0x24BD, 0x24D7 }, // CIRCLED LATIN CAPITAL LETTER H
- { 0x24BE, 0x24D8 }, // CIRCLED LATIN CAPITAL LETTER I
- { 0x24BF, 0x24D9 }, // CIRCLED LATIN CAPITAL LETTER J
- { 0x24C0, 0x24DA }, // CIRCLED LATIN CAPITAL LETTER K
- { 0x24C1, 0x24DB }, // CIRCLED LATIN CAPITAL LETTER L
- { 0x24C2, 0x24DC }, // CIRCLED LATIN CAPITAL LETTER M
- { 0x24C3, 0x24DD }, // CIRCLED LATIN CAPITAL LETTER N
- { 0x24C4, 0x24DE }, // CIRCLED LATIN CAPITAL LETTER O
- { 0x24C5, 0x24DF }, // CIRCLED LATIN CAPITAL LETTER P
- { 0x24C6, 0x24E0 }, // CIRCLED LATIN CAPITAL LETTER Q
- { 0x24C7, 0x24E1 }, // CIRCLED LATIN CAPITAL LETTER R
- { 0x24C8, 0x24E2 }, // CIRCLED LATIN CAPITAL LETTER S
- { 0x24C9, 0x24E3 }, // CIRCLED LATIN CAPITAL LETTER T
- { 0x24CA, 0x24E4 }, // CIRCLED LATIN CAPITAL LETTER U
- { 0x24CB, 0x24E5 }, // CIRCLED LATIN CAPITAL LETTER V
- { 0x24CC, 0x24E6 }, // CIRCLED LATIN CAPITAL LETTER W
- { 0x24CD, 0x24E7 }, // CIRCLED LATIN CAPITAL LETTER X
- { 0x24CE, 0x24E8 }, // CIRCLED LATIN CAPITAL LETTER Y
- { 0x24CF, 0x24E9 }, // CIRCLED LATIN CAPITAL LETTER Z
- { 0x2C00, 0x2C30 }, // GLAGOLITIC CAPITAL LETTER AZU
- { 0x2C01, 0x2C31 }, // GLAGOLITIC CAPITAL LETTER BUKY
- { 0x2C02, 0x2C32 }, // GLAGOLITIC CAPITAL LETTER VEDE
- { 0x2C03, 0x2C33 }, // GLAGOLITIC CAPITAL LETTER GLAGOLI
- { 0x2C04, 0x2C34 }, // GLAGOLITIC CAPITAL LETTER DOBRO
- { 0x2C05, 0x2C35 }, // GLAGOLITIC CAPITAL LETTER YESTU
- { 0x2C06, 0x2C36 }, // GLAGOLITIC CAPITAL LETTER ZHIVETE
- { 0x2C07, 0x2C37 }, // GLAGOLITIC CAPITAL LETTER DZELO
- { 0x2C08, 0x2C38 }, // GLAGOLITIC CAPITAL LETTER ZEMLJA
- { 0x2C09, 0x2C39 }, // GLAGOLITIC CAPITAL LETTER IZHE
- { 0x2C0A, 0x2C3A }, // GLAGOLITIC CAPITAL LETTER INITIAL IZHE
- { 0x2C0B, 0x2C3B }, // GLAGOLITIC CAPITAL LETTER I
- { 0x2C0C, 0x2C3C }, // GLAGOLITIC CAPITAL LETTER DJERVI
- { 0x2C0D, 0x2C3D }, // GLAGOLITIC CAPITAL LETTER KAKO
- { 0x2C0E, 0x2C3E }, // GLAGOLITIC CAPITAL LETTER LJUDIJE
- { 0x2C0F, 0x2C3F }, // GLAGOLITIC CAPITAL LETTER MYSLITE
- { 0x2C10, 0x2C40 }, // GLAGOLITIC CAPITAL LETTER NASHI
- { 0x2C11, 0x2C41 }, // GLAGOLITIC CAPITAL LETTER ONU
- { 0x2C12, 0x2C42 }, // GLAGOLITIC CAPITAL LETTER POKOJI
- { 0x2C13, 0x2C43 }, // GLAGOLITIC CAPITAL LETTER RITSI
- { 0x2C14, 0x2C44 }, // GLAGOLITIC CAPITAL LETTER SLOVO
- { 0x2C15, 0x2C45 }, // GLAGOLITIC CAPITAL LETTER TVRIDO
- { 0x2C16, 0x2C46 }, // GLAGOLITIC CAPITAL LETTER UKU
- { 0x2C17, 0x2C47 }, // GLAGOLITIC CAPITAL LETTER FRITU
- { 0x2C18, 0x2C48 }, // GLAGOLITIC CAPITAL LETTER HERU
- { 0x2C19, 0x2C49 }, // GLAGOLITIC CAPITAL LETTER OTU
- { 0x2C1A, 0x2C4A }, // GLAGOLITIC CAPITAL LETTER PE
- { 0x2C1B, 0x2C4B }, // GLAGOLITIC CAPITAL LETTER SHTA
- { 0x2C1C, 0x2C4C }, // GLAGOLITIC CAPITAL LETTER TSI
- { 0x2C1D, 0x2C4D }, // GLAGOLITIC CAPITAL LETTER CHRIVI
- { 0x2C1E, 0x2C4E }, // GLAGOLITIC CAPITAL LETTER SHA
- { 0x2C1F, 0x2C4F }, // GLAGOLITIC CAPITAL LETTER YERU
- { 0x2C20, 0x2C50 }, // GLAGOLITIC CAPITAL LETTER YERI
- { 0x2C21, 0x2C51 }, // GLAGOLITIC CAPITAL LETTER YATI
- { 0x2C22, 0x2C52 }, // GLAGOLITIC CAPITAL LETTER SPIDERY HA
- { 0x2C23, 0x2C53 }, // GLAGOLITIC CAPITAL LETTER YU
- { 0x2C24, 0x2C54 }, // GLAGOLITIC CAPITAL LETTER SMALL YUS
- { 0x2C25, 0x2C55 }, // GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
- { 0x2C26, 0x2C56 }, // GLAGOLITIC CAPITAL LETTER YO
- { 0x2C27, 0x2C57 }, // GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
- { 0x2C28, 0x2C58 }, // GLAGOLITIC CAPITAL LETTER BIG YUS
- { 0x2C29, 0x2C59 }, // GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
- { 0x2C2A, 0x2C5A }, // GLAGOLITIC CAPITAL LETTER FITA
- { 0x2C2B, 0x2C5B }, // GLAGOLITIC CAPITAL LETTER IZHITSA
- { 0x2C2C, 0x2C5C }, // GLAGOLITIC CAPITAL LETTER SHTAPIC
- { 0x2C2D, 0x2C5D }, // GLAGOLITIC CAPITAL LETTER TROKUTASTI A
- { 0x2C2E, 0x2C5E }, // GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
- { 0x2C60, 0x2C61 }, // LATIN CAPITAL LETTER L WITH DOUBLE BAR
- { 0x2C62, 0x026B }, // LATIN CAPITAL LETTER L WITH MIDDLE TILDE
- { 0x2C63, 0x1D7D }, // LATIN CAPITAL LETTER P WITH STROKE
- { 0x2C64, 0x027D }, // LATIN CAPITAL LETTER R WITH TAIL
- { 0x2C67, 0x2C68 }, // LATIN CAPITAL LETTER H WITH DESCENDER
- { 0x2C69, 0x2C6A }, // LATIN CAPITAL LETTER K WITH DESCENDER
- { 0x2C6B, 0x2C6C }, // LATIN CAPITAL LETTER Z WITH DESCENDER
- { 0x2C6D, 0x0251 }, // LATIN CAPITAL LETTER ALPHA
- { 0x2C6E, 0x0271 }, // LATIN CAPITAL LETTER M WITH HOOK
- { 0x2C6F, 0x0250 }, // LATIN CAPITAL LETTER TURNED A
- { 0x2C70, 0x0252 }, // LATIN CAPITAL LETTER TURNED ALPHA
- { 0x2C72, 0x2C73 }, // LATIN CAPITAL LETTER W WITH HOOK
- { 0x2C75, 0x2C76 }, // LATIN CAPITAL LETTER HALF H
- { 0x2C7E, 0x023F }, // LATIN CAPITAL LETTER S WITH SWASH TAIL
- { 0x2C7F, 0x0240 }, // LATIN CAPITAL LETTER Z WITH SWASH TAIL
- { 0x2C80, 0x2C81 }, // COPTIC CAPITAL LETTER ALFA
- { 0x2C82, 0x2C83 }, // COPTIC CAPITAL LETTER VIDA
- { 0x2C84, 0x2C85 }, // COPTIC CAPITAL LETTER GAMMA
- { 0x2C86, 0x2C87 }, // COPTIC CAPITAL LETTER DALDA
- { 0x2C88, 0x2C89 }, // COPTIC CAPITAL LETTER EIE
- { 0x2C8A, 0x2C8B }, // COPTIC CAPITAL LETTER SOU
- { 0x2C8C, 0x2C8D }, // COPTIC CAPITAL LETTER ZATA
- { 0x2C8E, 0x2C8F }, // COPTIC CAPITAL LETTER HATE
- { 0x2C90, 0x2C91 }, // COPTIC CAPITAL LETTER THETHE
- { 0x2C92, 0x2C93 }, // COPTIC CAPITAL LETTER IAUDA
- { 0x2C94, 0x2C95 }, // COPTIC CAPITAL LETTER KAPA
- { 0x2C96, 0x2C97 }, // COPTIC CAPITAL LETTER LAULA
- { 0x2C98, 0x2C99 }, // COPTIC CAPITAL LETTER MI
- { 0x2C9A, 0x2C9B }, // COPTIC CAPITAL LETTER NI
- { 0x2C9C, 0x2C9D }, // COPTIC CAPITAL LETTER KSI
- { 0x2C9E, 0x2C9F }, // COPTIC CAPITAL LETTER O
- { 0x2CA0, 0x2CA1 }, // COPTIC CAPITAL LETTER PI
- { 0x2CA2, 0x2CA3 }, // COPTIC CAPITAL LETTER RO
- { 0x2CA4, 0x2CA5 }, // COPTIC CAPITAL LETTER SIMA
- { 0x2CA6, 0x2CA7 }, // COPTIC CAPITAL LETTER TAU
- { 0x2CA8, 0x2CA9 }, // COPTIC CAPITAL LETTER UA
- { 0x2CAA, 0x2CAB }, // COPTIC CAPITAL LETTER FI
- { 0x2CAC, 0x2CAD }, // COPTIC CAPITAL LETTER KHI
- { 0x2CAE, 0x2CAF }, // COPTIC CAPITAL LETTER PSI
- { 0x2CB0, 0x2CB1 }, // COPTIC CAPITAL LETTER OOU
- { 0x2CB2, 0x2CB3 }, // COPTIC CAPITAL LETTER DIALECT-P ALEF
- { 0x2CB4, 0x2CB5 }, // COPTIC CAPITAL LETTER OLD COPTIC AIN
- { 0x2CB6, 0x2CB7 }, // COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE
- { 0x2CB8, 0x2CB9 }, // COPTIC CAPITAL LETTER DIALECT-P KAPA
- { 0x2CBA, 0x2CBB }, // COPTIC CAPITAL LETTER DIALECT-P NI
- { 0x2CBC, 0x2CBD }, // COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI
- { 0x2CBE, 0x2CBF }, // COPTIC CAPITAL LETTER OLD COPTIC OOU
- { 0x2CC0, 0x2CC1 }, // COPTIC CAPITAL LETTER SAMPI
- { 0x2CC2, 0x2CC3 }, // COPTIC CAPITAL LETTER CROSSED SHEI
- { 0x2CC4, 0x2CC5 }, // COPTIC CAPITAL LETTER OLD COPTIC SHEI
- { 0x2CC6, 0x2CC7 }, // COPTIC CAPITAL LETTER OLD COPTIC ESH
- { 0x2CC8, 0x2CC9 }, // COPTIC CAPITAL LETTER AKHMIMIC KHEI
- { 0x2CCA, 0x2CCB }, // COPTIC CAPITAL LETTER DIALECT-P HORI
- { 0x2CCC, 0x2CCD }, // COPTIC CAPITAL LETTER OLD COPTIC HORI
- { 0x2CCE, 0x2CCF }, // COPTIC CAPITAL LETTER OLD COPTIC HA
- { 0x2CD0, 0x2CD1 }, // COPTIC CAPITAL LETTER L-SHAPED HA
- { 0x2CD2, 0x2CD3 }, // COPTIC CAPITAL LETTER OLD COPTIC HEI
- { 0x2CD4, 0x2CD5 }, // COPTIC CAPITAL LETTER OLD COPTIC HAT
- { 0x2CD6, 0x2CD7 }, // COPTIC CAPITAL LETTER OLD COPTIC GANGIA
- { 0x2CD8, 0x2CD9 }, // COPTIC CAPITAL LETTER OLD COPTIC DJA
- { 0x2CDA, 0x2CDB }, // COPTIC CAPITAL LETTER OLD COPTIC SHIMA
- { 0x2CDC, 0x2CDD }, // COPTIC CAPITAL LETTER OLD NUBIAN SHIMA
- { 0x2CDE, 0x2CDF }, // COPTIC CAPITAL LETTER OLD NUBIAN NGI
- { 0x2CE0, 0x2CE1 }, // COPTIC CAPITAL LETTER OLD NUBIAN NYI
- { 0x2CE2, 0x2CE3 }, // COPTIC CAPITAL LETTER OLD NUBIAN WAU
- { 0x2CEB, 0x2CEC }, // COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI
- { 0x2CED, 0x2CEE }, // COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA
- { 0xA640, 0xA641 }, // CYRILLIC CAPITAL LETTER ZEMLYA
- { 0xA642, 0xA643 }, // CYRILLIC CAPITAL LETTER DZELO
- { 0xA644, 0xA645 }, // CYRILLIC CAPITAL LETTER REVERSED DZE
- { 0xA646, 0xA647 }, // CYRILLIC CAPITAL LETTER IOTA
- { 0xA648, 0xA649 }, // CYRILLIC CAPITAL LETTER DJERV
- { 0xA64A, 0xA64B }, // CYRILLIC CAPITAL LETTER MONOGRAPH UK
- { 0xA64C, 0xA64D }, // CYRILLIC CAPITAL LETTER BROAD OMEGA
- { 0xA64E, 0xA64F }, // CYRILLIC CAPITAL LETTER NEUTRAL YER
- { 0xA650, 0xA651 }, // CYRILLIC CAPITAL LETTER YERU WITH BACK YER
- { 0xA652, 0xA653 }, // CYRILLIC CAPITAL LETTER IOTIFIED YAT
- { 0xA654, 0xA655 }, // CYRILLIC CAPITAL LETTER REVERSED YU
- { 0xA656, 0xA657 }, // CYRILLIC CAPITAL LETTER IOTIFIED A
- { 0xA658, 0xA659 }, // CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
- { 0xA65A, 0xA65B }, // CYRILLIC CAPITAL LETTER BLENDED YUS
- { 0xA65C, 0xA65D }, // CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
- { 0xA65E, 0xA65F }, // CYRILLIC CAPITAL LETTER YN
- { 0xA662, 0xA663 }, // CYRILLIC CAPITAL LETTER SOFT DE
- { 0xA664, 0xA665 }, // CYRILLIC CAPITAL LETTER SOFT EL
- { 0xA666, 0xA667 }, // CYRILLIC CAPITAL LETTER SOFT EM
- { 0xA668, 0xA669 }, // CYRILLIC CAPITAL LETTER MONOCULAR O
- { 0xA66A, 0xA66B }, // CYRILLIC CAPITAL LETTER BINOCULAR O
- { 0xA66C, 0xA66D }, // CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O
- { 0xA680, 0xA681 }, // CYRILLIC CAPITAL LETTER DWE
- { 0xA682, 0xA683 }, // CYRILLIC CAPITAL LETTER DZWE
- { 0xA684, 0xA685 }, // CYRILLIC CAPITAL LETTER ZHWE
- { 0xA686, 0xA687 }, // CYRILLIC CAPITAL LETTER CCHE
- { 0xA688, 0xA689 }, // CYRILLIC CAPITAL LETTER DZZE
- { 0xA68A, 0xA68B }, // CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK
- { 0xA68C, 0xA68D }, // CYRILLIC CAPITAL LETTER TWE
- { 0xA68E, 0xA68F }, // CYRILLIC CAPITAL LETTER TSWE
- { 0xA690, 0xA691 }, // CYRILLIC CAPITAL LETTER TSSE
- { 0xA692, 0xA693 }, // CYRILLIC CAPITAL LETTER TCHE
- { 0xA694, 0xA695 }, // CYRILLIC CAPITAL LETTER HWE
- { 0xA696, 0xA697 }, // CYRILLIC CAPITAL LETTER SHWE
- { 0xA722, 0xA723 }, // LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF
- { 0xA724, 0xA725 }, // LATIN CAPITAL LETTER EGYPTOLOGICAL AIN
- { 0xA726, 0xA727 }, // LATIN CAPITAL LETTER HENG
- { 0xA728, 0xA729 }, // LATIN CAPITAL LETTER TZ
- { 0xA72A, 0xA72B }, // LATIN CAPITAL LETTER TRESILLO
- { 0xA72C, 0xA72D }, // LATIN CAPITAL LETTER CUATRILLO
- { 0xA72E, 0xA72F }, // LATIN CAPITAL LETTER CUATRILLO WITH COMMA
- { 0xA732, 0xA733 }, // LATIN CAPITAL LETTER AA
- { 0xA734, 0xA735 }, // LATIN CAPITAL LETTER AO
- { 0xA736, 0xA737 }, // LATIN CAPITAL LETTER AU
- { 0xA738, 0xA739 }, // LATIN CAPITAL LETTER AV
- { 0xA73A, 0xA73B }, // LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR
- { 0xA73C, 0xA73D }, // LATIN CAPITAL LETTER AY
- { 0xA73E, 0xA73F }, // LATIN CAPITAL LETTER REVERSED C WITH DOT
- { 0xA740, 0xA741 }, // LATIN CAPITAL LETTER K WITH STROKE
- { 0xA742, 0xA743 }, // LATIN CAPITAL LETTER K WITH DIAGONAL STROKE
- { 0xA744, 0xA745 }, // LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE
- { 0xA746, 0xA747 }, // LATIN CAPITAL LETTER BROKEN L
- { 0xA748, 0xA749 }, // LATIN CAPITAL LETTER L WITH HIGH STROKE
- { 0xA74A, 0xA74B }, // LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY
- { 0xA74C, 0xA74D }, // LATIN CAPITAL LETTER O WITH LOOP
- { 0xA74E, 0xA74F }, // LATIN CAPITAL LETTER OO
- { 0xA750, 0xA751 }, // LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER
- { 0xA752, 0xA753 }, // LATIN CAPITAL LETTER P WITH FLOURISH
- { 0xA754, 0xA755 }, // LATIN CAPITAL LETTER P WITH SQUIRREL TAIL
- { 0xA756, 0xA757 }, // LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER
- { 0xA758, 0xA759 }, // LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE
- { 0xA75A, 0xA75B }, // LATIN CAPITAL LETTER R ROTUNDA
- { 0xA75C, 0xA75D }, // LATIN CAPITAL LETTER RUM ROTUNDA
- { 0xA75E, 0xA75F }, // LATIN CAPITAL LETTER V WITH DIAGONAL STROKE
- { 0xA760, 0xA761 }, // LATIN CAPITAL LETTER VY
- { 0xA762, 0xA763 }, // LATIN CAPITAL LETTER VISIGOTHIC Z
- { 0xA764, 0xA765 }, // LATIN CAPITAL LETTER THORN WITH STROKE
- { 0xA766, 0xA767 }, // LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER
- { 0xA768, 0xA769 }, // LATIN CAPITAL LETTER VEND
- { 0xA76A, 0xA76B }, // LATIN CAPITAL LETTER ET
- { 0xA76C, 0xA76D }, // LATIN CAPITAL LETTER IS
- { 0xA76E, 0xA76F }, // LATIN CAPITAL LETTER CON
- { 0xA779, 0xA77A }, // LATIN CAPITAL LETTER INSULAR D
- { 0xA77B, 0xA77C }, // LATIN CAPITAL LETTER INSULAR F
- { 0xA77D, 0x1D79 }, // LATIN CAPITAL LETTER INSULAR G
- { 0xA77E, 0xA77F }, // LATIN CAPITAL LETTER TURNED INSULAR G
- { 0xA780, 0xA781 }, // LATIN CAPITAL LETTER TURNED L
- { 0xA782, 0xA783 }, // LATIN CAPITAL LETTER INSULAR R
- { 0xA784, 0xA785 }, // LATIN CAPITAL LETTER INSULAR S
- { 0xA786, 0xA787 }, // LATIN CAPITAL LETTER INSULAR T
- { 0xA78B, 0xA78C }, // LATIN CAPITAL LETTER SALTILLO
- { 0xFF21, 0xFF41 }, // FULLWIDTH LATIN CAPITAL LETTER A
- { 0xFF22, 0xFF42 }, // FULLWIDTH LATIN CAPITAL LETTER B
- { 0xFF23, 0xFF43 }, // FULLWIDTH LATIN CAPITAL LETTER C
- { 0xFF24, 0xFF44 }, // FULLWIDTH LATIN CAPITAL LETTER D
- { 0xFF25, 0xFF45 }, // FULLWIDTH LATIN CAPITAL LETTER E
- { 0xFF26, 0xFF46 }, // FULLWIDTH LATIN CAPITAL LETTER F
- { 0xFF27, 0xFF47 }, // FULLWIDTH LATIN CAPITAL LETTER G
- { 0xFF28, 0xFF48 }, // FULLWIDTH LATIN CAPITAL LETTER H
- { 0xFF29, 0xFF49 }, // FULLWIDTH LATIN CAPITAL LETTER I
- { 0xFF2A, 0xFF4A }, // FULLWIDTH LATIN CAPITAL LETTER J
- { 0xFF2B, 0xFF4B }, // FULLWIDTH LATIN CAPITAL LETTER K
- { 0xFF2C, 0xFF4C }, // FULLWIDTH LATIN CAPITAL LETTER L
- { 0xFF2D, 0xFF4D }, // FULLWIDTH LATIN CAPITAL LETTER M
- { 0xFF2E, 0xFF4E }, // FULLWIDTH LATIN CAPITAL LETTER N
- { 0xFF2F, 0xFF4F }, // FULLWIDTH LATIN CAPITAL LETTER O
- { 0xFF30, 0xFF50 }, // FULLWIDTH LATIN CAPITAL LETTER P
- { 0xFF31, 0xFF51 }, // FULLWIDTH LATIN CAPITAL LETTER Q
- { 0xFF32, 0xFF52 }, // FULLWIDTH LATIN CAPITAL LETTER R
- { 0xFF33, 0xFF53 }, // FULLWIDTH LATIN CAPITAL LETTER S
- { 0xFF34, 0xFF54 }, // FULLWIDTH LATIN CAPITAL LETTER T
- { 0xFF35, 0xFF55 }, // FULLWIDTH LATIN CAPITAL LETTER U
- { 0xFF36, 0xFF56 }, // FULLWIDTH LATIN CAPITAL LETTER V
- { 0xFF37, 0xFF57 }, // FULLWIDTH LATIN CAPITAL LETTER W
- { 0xFF38, 0xFF58 }, // FULLWIDTH LATIN CAPITAL LETTER X
- { 0xFF39, 0xFF59 }, // FULLWIDTH LATIN CAPITAL LETTER Y
- { 0xFF3A, 0xFF5A } // FULLWIDTH LATIN CAPITAL LETTER Z
-};
-
-static int compare_pair_capital(const void *a, const void *b) {
- return (int)(*(unsigned short *)a)
- - (int)((struct LatinCapitalSmallPair*)b)->capital;
-}
-
-unsigned short latin_tolower(unsigned short c) {
- struct LatinCapitalSmallPair *p =
- (struct LatinCapitalSmallPair *)bsearch(&c, SORTED_CHAR_MAP,
- sizeof(SORTED_CHAR_MAP) / sizeof(SORTED_CHAR_MAP[0]),
- sizeof(SORTED_CHAR_MAP[0]),
- compare_pair_capital);
- return p ? p->small : c;
-}
-
-} // namespace latinime
diff --git a/native/src/char_utils.h b/native/src/char_utils.h
deleted file mode 100644
index a69a35e7a..000000000
--- a/native/src/char_utils.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_CHAR_UTILS_H
-#define LATINIME_CHAR_UTILS_H
-
-namespace latinime {
-
-unsigned short latin_tolower(unsigned short c);
-
-} // namespace latinime
-
-#endif // LATINIME_CHAR_UTILS_H
diff --git a/native/src/debug.h b/native/src/debug.h
deleted file mode 100644
index 38b2f107a..000000000
--- a/native/src/debug.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-**
-** Copyright 2011, The Android Open Source Project
-**
-** Licensed under the Apache License, Version 2.0 (the "License");
-** you may not use this file except in compliance with the License.
-** You may obtain a copy of the License at
-**
-** http://www.apache.org/licenses/LICENSE-2.0
-**
-** Unless required by applicable law or agreed to in writing, software
-** distributed under the License is distributed on an "AS IS" BASIS,
-** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-** See the License for the specific language governing permissions and
-** limitations under the License.
-*/
-
-#ifndef LATINIME_DEBUG_H
-#define LATINIME_DEBUG_H
-
-#include "defines.h"
-
-static inline unsigned char* convertToUnibyteString(unsigned short* input, unsigned char* output,
- const unsigned int length) {
- int i = 0;
- for (; i <= length && input[i] != 0; ++i)
- output[i] = input[i] & 0xFF;
- output[i] = 0;
- return output;
-}
-
-static inline unsigned char* convertToUnibyteStringAndReplaceLastChar(unsigned short* input,
- unsigned char* output, const unsigned int length, unsigned char c) {
- int i = 0;
- for (; i <= length && input[i] != 0; ++i)
- output[i] = input[i] & 0xFF;
- output[i-1] = c;
- output[i] = 0;
- return output;
-}
-
-static inline void LOGI_S16(unsigned short* string, const unsigned int length) {
- unsigned char tmp_buffer[length];
- convertToUnibyteString(string, tmp_buffer, length);
- LOGI(">> %s", tmp_buffer);
- // The log facility is throwing out log that comes too fast. The following
- // is a dirty way of slowing down processing so that we can see all log.
- // TODO : refactor this in a blocking log or something.
- // usleep(10);
-}
-
-static inline void LOGI_S16_PLUS(unsigned short* string, const unsigned int length,
- unsigned char c) {
- unsigned char tmp_buffer[length+1];
- convertToUnibyteStringAndReplaceLastChar(string, tmp_buffer, length, c);
- LOGI(">> %s", tmp_buffer);
- // Likewise
- // usleep(10);
-}
-
-static inline void printDebug(const char* tag, int* codes, int codesSize, int MAX_PROXIMITY_CHARS) {
- unsigned char *buf = (unsigned char*)malloc((1 + codesSize) * sizeof(*buf));
-
- buf[codesSize] = 0;
- while (--codesSize >= 0)
- buf[codesSize] = (unsigned char)codes[codesSize * MAX_PROXIMITY_CHARS];
- LOGI("%s, WORD = %s", tag, buf);
-
- free(buf);
-}
-
-#endif // LATINIME_DEBUG_H
diff --git a/native/src/defines.h b/native/src/defines.h
deleted file mode 100644
index 0a3240507..000000000
--- a/native/src/defines.h
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
-**
-** Copyright 2010, The Android Open Source Project
-**
-** Licensed under the Apache License, Version 2.0 (the "License");
-** you may not use this file except in compliance with the License.
-** You may obtain a copy of the License at
-**
-** http://www.apache.org/licenses/LICENSE-2.0
-**
-** Unless required by applicable law or agreed to in writing, software
-** distributed under the License is distributed on an "AS IS" BASIS,
-** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-** See the License for the specific language governing permissions and
-** limitations under the License.
-*/
-
-#ifndef LATINIME_DEFINES_H
-#define LATINIME_DEFINES_H
-
-#ifdef FLAG_DBG
-#include <cutils/log.h>
-#ifndef LOG_TAG
-#define LOG_TAG "LatinIME: "
-#endif
-#define DEBUG_DICT true
-#define DEBUG_DICT_FULL false
-#define DEBUG_SHOW_FOUND_WORD DEBUG_DICT_FULL
-#define DEBUG_NODE DEBUG_DICT_FULL
-#define DEBUG_TRACE DEBUG_DICT_FULL
-#define DEBUG_PROXIMITY_INFO true
-
-// Profiler
-#include <time.h>
-#define PROF_BUF_SIZE 100
-static double profile_buf[PROF_BUF_SIZE];
-static double profile_old[PROF_BUF_SIZE];
-static unsigned int profile_counter[PROF_BUF_SIZE];
-
-#define PROF_RESET prof_reset()
-#define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id]
-#define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while(0)
-#define PROF_START(prof_buf_id) do { \
- PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while(0)
-#define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while(0)
-#define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id])
-#define PROF_CLOCKOUT(prof_buf_id) \
- LOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id]))
-#define PROF_OUTALL do { LOGI("--- %s ---", __FUNCTION__); prof_out(); } while(0)
-
-static void prof_reset(void) {
- for (int i = 0; i < PROF_BUF_SIZE; ++i) {
- profile_buf[i] = 0;
- profile_old[i] = 0;
- profile_counter[i] = 0;
- }
-}
-
-static void prof_out(void) {
- if (profile_counter[PROF_BUF_SIZE - 1] != 1) {
- LOGI("Error: You must call PROF_OPEN before PROF_CLOSE.");
- }
- LOGI("Total time is %6.3f ms.",
- profile_buf[PROF_BUF_SIZE - 1] * 1000 / (double)CLOCKS_PER_SEC);
- double all = 0;
- for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) {
- all += profile_buf[i];
- }
- if (all == 0) all = 1;
- for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) {
- if (profile_buf[i] != 0) {
- LOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.",
- i, (profile_buf[i] * 100 / all),
- profile_buf[i] * 1000 / (double)CLOCKS_PER_SEC, profile_counter[i]);
- }
- }
-}
-
-#else // FLAG_DBG
-#define LOGE(fmt, ...)
-#define LOGI(fmt, ...)
-#define DEBUG_DICT false
-#define DEBUG_DICT_FULL false
-#define DEBUG_SHOW_FOUND_WORD false
-#define DEBUG_NODE false
-#define DEBUG_TRACE false
-#define DEBUG_PROXIMITY_INFO false
-
-#define PROF_BUF_SIZE 0
-#define PROF_RESET
-#define PROF_COUNT(prof_buf_id)
-#define PROF_OPEN
-#define PROF_START(prof_buf_id)
-#define PROF_CLOSE
-#define PROF_END(prof_buf_id)
-#define PROF_CLOCK_OUT(prof_buf_id)
-#define PROF_CLOCKOUT(prof_buf_id)
-#define PROF_OUTALL
-
-#endif // FLAG_DBG
-
-#ifndef U_SHORT_MAX
-#define U_SHORT_MAX 1 << 16
-#endif
-#ifndef S_INT_MAX
-#define S_INT_MAX 2147483647 // ((1 << 31) - 1)
-#endif
-
-// Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap().
-// We measured and compared performance of both, and found mmap() is fairly good in terms of
-// loading time, and acceptable even for several initial lookups which involve page faults.
-#define USE_MMAP_FOR_DICTIONARY
-
-// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
-#define ADDRESS_MASK 0x3FFFFF
-
-// The bit that decides if an address follows in the next 22 bits
-#define FLAG_ADDRESS_MASK 0x40
-// The bit that decides if this is a terminal node for a word. The node could still have children,
-// if the word has other endings.
-#define FLAG_TERMINAL_MASK 0x80
-
-#define FLAG_BIGRAM_READ 0x80
-#define FLAG_BIGRAM_CHILDEXIST 0x40
-#define FLAG_BIGRAM_CONTINUED 0x80
-#define FLAG_BIGRAM_FREQ 0x7F
-
-#define DICTIONARY_VERSION_MIN 200
-#define DICTIONARY_HEADER_SIZE 2
-#define NOT_VALID_WORD -99
-
-#define KEYCODE_SPACE ' '
-
-#define SUGGEST_WORDS_WITH_MISSING_CHARACTER true
-#define SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER true
-#define SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER true
-#define SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS true
-#define SUGGEST_WORDS_WITH_SPACE_PROXIMITY true
-
-// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent.
-#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80
-#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12
-#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 67
-#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75
-#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75
-#define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 60
-#define FULL_MATCHED_WORDS_PROMOTION_RATE 120
-#define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90
-
-// This should be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
-// This is only used for the size of array. Not to be used in c functions.
-#define MAX_WORD_LENGTH_INTERNAL 48
-
-#define MAX_DEPTH_MULTIPLIER 3
-
-// TODO: Reduce this constant if possible; check the maximum number of umlauts in the same German
-// word in the dictionary
-#define DEFAULT_MAX_UMLAUT_SEARCH_DEPTH 5
-
-// Minimum suggest depth for one word for all cases except for missing space suggestions.
-#define MIN_SUGGEST_DEPTH 1
-#define MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION 3
-#define MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION 3
-
-// The size of next letters frequency array. Zero will disable the feature.
-#define NEXT_LETTERS_SIZE 0
-
-#define min(a,b) ((a)<(b)?(a):(b))
-
-#endif // LATINIME_DEFINES_H
diff --git a/native/src/dictionary.cpp b/native/src/dictionary.cpp
deleted file mode 100644
index 9e32ee80f..000000000
--- a/native/src/dictionary.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
-**
-** Copyright 2009, The Android Open Source Project
-**
-** Licensed under the Apache License, Version 2.0 (the "License");
-** you may not use this file except in compliance with the License.
-** You may obtain a copy of the License at
-**
-** http://www.apache.org/licenses/LICENSE-2.0
-**
-** Unless required by applicable law or agreed to in writing, software
-** distributed under the License is distributed on an "AS IS" BASIS,
-** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-** See the License for the specific language governing permissions and
-** limitations under the License.
-*/
-
-#include <stdio.h>
-
-#define LOG_TAG "LatinIME: dictionary.cpp"
-
-#include "dictionary.h"
-
-namespace latinime {
-
-// TODO: Change the type of all keyCodes to uint32_t
-Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust,
- int typedLetterMultiplier, int fullWordMultiplier,
- int maxWordLength, int maxWords, int maxAlternatives)
- : mDict((unsigned char*) dict), mDictSize(dictSize),
- mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust),
- // Checks whether it has the latest dictionary or the old dictionary
- IS_LATEST_DICT_VERSION((((unsigned char*) dict)[0] & 0xFF) >= DICTIONARY_VERSION_MIN) {
- if (DEBUG_DICT) {
- if (MAX_WORD_LENGTH_INTERNAL < maxWordLength) {
- LOGI("Max word length (%d) is greater than %d",
- maxWordLength, MAX_WORD_LENGTH_INTERNAL);
- LOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF));
- }
- }
- mUnigramDictionary = new UnigramDictionary(mDict, typedLetterMultiplier, fullWordMultiplier,
- maxWordLength, maxWords, maxAlternatives, IS_LATEST_DICT_VERSION);
- mBigramDictionary = new BigramDictionary(mDict, maxWordLength, maxAlternatives,
- IS_LATEST_DICT_VERSION, hasBigram(), this);
-}
-
-Dictionary::~Dictionary() {
- delete mUnigramDictionary;
- delete mBigramDictionary;
-}
-
-bool Dictionary::hasBigram() {
- return ((mDict[1] & 0xFF) == 1);
-}
-
-bool Dictionary::isValidWord(unsigned short *word, int length) {
- return mUnigramDictionary->isValidWord(word, length);
-}
-
-int Dictionary::getBigramPosition(unsigned short *word, int length) {
- if (IS_LATEST_DICT_VERSION) {
- return mUnigramDictionary->getBigramPosition(DICTIONARY_HEADER_SIZE, word, 0, length);
- } else {
- return mUnigramDictionary->getBigramPosition(0, word, 0, length);
- }
-}
-
-} // namespace latinime
diff --git a/native/src/dictionary.h b/native/src/dictionary.h
deleted file mode 100644
index 3dc577a56..000000000
--- a/native/src/dictionary.h
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright (C) 2009 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_DICTIONARY_H
-#define LATINIME_DICTIONARY_H
-
-#include "bigram_dictionary.h"
-#include "defines.h"
-#include "proximity_info.h"
-#include "unigram_dictionary.h"
-
-namespace latinime {
-
-class Dictionary {
-public:
- Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler,
- int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives);
- int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates,
- int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) {
- return mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
- codesSize, flags, outWords, frequencies);
- }
-
- // TODO: Call mBigramDictionary instead of mUnigramDictionary
- int getBigrams(unsigned short *word, int length, int *codes, int codesSize,
- unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams,
- int maxAlternatives) {
- return mBigramDictionary->getBigrams(word, length, codes, codesSize, outWords, frequencies,
- maxWordLength, maxBigrams, maxAlternatives);
- }
-
- bool isValidWord(unsigned short *word, int length);
- void *getDict() { return (void *)mDict; }
- int getDictSize() { return mDictSize; }
- int getMmapFd() { return mMmapFd; }
- int getDictBufAdjust() { return mDictBufAdjust; }
- ~Dictionary();
-
- // public static utility methods
- // static inline methods should be defined in the header file
- static unsigned short getChar(const unsigned char *dict, int *pos);
- static int getCount(const unsigned char *dict, int *pos);
- static bool getTerminal(const unsigned char *dict, int *pos);
- static int getAddress(const unsigned char *dict, int *pos);
- static int getFreq(const unsigned char *dict, const bool isLatestDictVersion, int *pos);
- static int wideStrLen(unsigned short *str);
- // returns next sibling's position
- static int setDictionaryValues(const unsigned char *dict, const bool isLatestDictVersion,
- const int pos, unsigned short *c, int *childrenPosition,
- bool *terminal, int *freq);
-
- // TODO: delete this
- int getBigramPosition(unsigned short *word, int length);
-
-private:
- bool hasBigram();
-
- const unsigned char *mDict;
-
- // Used only for the mmap version of dictionary loading, but we use these as dummy variables
- // also for the malloc version.
- const int mDictSize;
- const int mMmapFd;
- const int mDictBufAdjust;
-
- const bool IS_LATEST_DICT_VERSION;
- UnigramDictionary *mUnigramDictionary;
- BigramDictionary *mBigramDictionary;
-};
-
-// public static utility methods
-// static inline methods should be defined in the header file
-inline unsigned short Dictionary::getChar(const unsigned char *dict, int *pos) {
- unsigned short ch = (unsigned short) (dict[(*pos)++] & 0xFF);
- // If the code is 255, then actual 16 bit code follows (in big endian)
- if (ch == 0xFF) {
- ch = ((dict[*pos] & 0xFF) << 8) | (dict[*pos + 1] & 0xFF);
- (*pos) += 2;
- }
- return ch;
-}
-
-inline int Dictionary::getCount(const unsigned char *dict, int *pos) {
- return dict[(*pos)++] & 0xFF;
-}
-
-inline bool Dictionary::getTerminal(const unsigned char *dict, int *pos) {
- return (dict[*pos] & FLAG_TERMINAL_MASK) > 0;
-}
-
-inline int Dictionary::getAddress(const unsigned char *dict, int *pos) {
- int address = 0;
- if ((dict[*pos] & FLAG_ADDRESS_MASK) == 0) {
- *pos += 1;
- } else {
- address += (dict[*pos] & (ADDRESS_MASK >> 16)) << 16;
- address += (dict[*pos + 1] & 0xFF) << 8;
- address += (dict[*pos + 2] & 0xFF);
- *pos += 3;
- }
- return address;
-}
-
-inline int Dictionary::getFreq(const unsigned char *dict,
- const bool isLatestDictVersion, int *pos) {
- int freq = dict[(*pos)++] & 0xFF;
- if (isLatestDictVersion) {
- // skipping bigram
- int bigramExist = (dict[*pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0) {
- (*pos) += 3;
- nextBigramExist = (dict[(*pos)++] & FLAG_BIGRAM_CONTINUED);
- }
- } else {
- (*pos)++;
- }
- }
- return freq;
-}
-
-inline int Dictionary::wideStrLen(unsigned short *str) {
- if (!str) return 0;
- unsigned short *end = str;
- while (*end)
- end++;
- return end - str;
-}
-
-inline int Dictionary::setDictionaryValues(const unsigned char *dict,
- const bool isLatestDictVersion, const int pos, unsigned short *c,int *childrenPosition,
- bool *terminal, int *freq) {
- int position = pos;
- // -- at char
- *c = Dictionary::getChar(dict, &position);
- // -- at flag/add
- *terminal = Dictionary::getTerminal(dict, &position);
- *childrenPosition = Dictionary::getAddress(dict, &position);
- // -- after address or flag
- *freq = (*terminal) ? Dictionary::getFreq(dict, isLatestDictVersion, &position) : 1;
- // returns next sibling's position
- return position;
-}
-
-} // namespace latinime
-
-#endif // LATINIME_DICTIONARY_H
diff --git a/native/src/proximity_info.cpp b/native/src/proximity_info.cpp
deleted file mode 100644
index 209c31e6e..000000000
--- a/native/src/proximity_info.cpp
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <stdio.h>
-#include <string.h>
-
-#define LOG_TAG "LatinIME: proximity_info.cpp"
-
-#include "proximity_info.h"
-
-namespace latinime {
-
-ProximityInfo::ProximityInfo(const int maxProximityCharsSize, const int keyboardWidth,
- const int keyboardHeight, const int gridWidth, const int gridHeight,
- const uint32_t *proximityCharsArray)
- : MAX_PROXIMITY_CHARS_SIZE(maxProximityCharsSize), KEYBOARD_WIDTH(keyboardWidth),
- KEYBOARD_HEIGHT(keyboardHeight), GRID_WIDTH(gridWidth), GRID_HEIGHT(gridHeight),
- CELL_WIDTH((keyboardWidth + gridWidth - 1) / gridWidth),
- CELL_HEIGHT((keyboardHeight + gridHeight - 1) / gridHeight) {
- const int len = GRID_WIDTH * GRID_HEIGHT * MAX_PROXIMITY_CHARS_SIZE;
- mProximityCharsArray = new uint32_t[len];
- if (DEBUG_PROXIMITY_INFO) {
- LOGI("Create proximity info array %d", len);
- }
- memcpy(mProximityCharsArray, proximityCharsArray, len * sizeof(mProximityCharsArray[0]));
-}
-
-ProximityInfo::~ProximityInfo() {
- delete[] mProximityCharsArray;
-}
-
-inline int ProximityInfo::getStartIndexFromCoordinates(const int x, const int y) const {
- return ((y / CELL_HEIGHT) * GRID_WIDTH + (x / CELL_WIDTH))
- * MAX_PROXIMITY_CHARS_SIZE;
-}
-
-bool ProximityInfo::hasSpaceProximity(const int x, const int y) const {
- const int startIndex = getStartIndexFromCoordinates(x, y);
- if (DEBUG_PROXIMITY_INFO) {
- LOGI("hasSpaceProximity: index %d", startIndex);
- }
- for (int i = 0; i < MAX_PROXIMITY_CHARS_SIZE; ++i) {
- if (DEBUG_PROXIMITY_INFO) {
- LOGI("Index: %d", mProximityCharsArray[startIndex + i]);
- }
- if (mProximityCharsArray[startIndex + i] == KEYCODE_SPACE) {
- return true;
- }
- }
- return false;
-}
-
-} // namespace latinime
diff --git a/native/src/proximity_info.h b/native/src/proximity_info.h
deleted file mode 100644
index 327cd0940..000000000
--- a/native/src/proximity_info.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2011 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_PROXIMITY_INFO_H
-#define LATINIME_PROXIMITY_INFO_H
-
-#include <stdint.h>
-
-#include "defines.h"
-
-namespace latinime {
-
-class ProximityInfo {
-public:
- ProximityInfo(const int maxProximityCharsSize, const int keyboardWidth,
- const int keybaordHeight, const int gridWidth, const int gridHeight,
- const uint32_t *proximityCharsArray);
- ~ProximityInfo();
- bool hasSpaceProximity(const int x, const int y) const;
-private:
- int getStartIndexFromCoordinates(const int x, const int y) const;
- const int MAX_PROXIMITY_CHARS_SIZE;
- const int KEYBOARD_WIDTH;
- const int KEYBOARD_HEIGHT;
- const int GRID_WIDTH;
- const int GRID_HEIGHT;
- const int CELL_WIDTH;
- const int CELL_HEIGHT;
- uint32_t *mProximityCharsArray;
-};
-
-} // namespace latinime
-
-#endif // LATINIME_PROXIMITY_INFO_H
diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp
deleted file mode 100644
index e3296f12a..000000000
--- a/native/src/unigram_dictionary.cpp
+++ /dev/null
@@ -1,1127 +0,0 @@
-/*
-**
-** Copyright 2010, The Android Open Source Project
-**
-** Licensed under the Apache License, Version 2.0 (the "License");
-** you may not use this file except in compliance with the License.
-** You may obtain a copy of the License at
-**
-** http://www.apache.org/licenses/LICENSE-2.0
-**
-** Unless required by applicable law or agreed to in writing, software
-** distributed under the License is distributed on an "AS IS" BASIS,
-** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-** See the License for the specific language governing permissions and
-** limitations under the License.
-*/
-
-#include <assert.h>
-#include <string.h>
-
-#define LOG_TAG "LatinIME: unigram_dictionary.cpp"
-
-#include "basechars.h"
-#include "char_utils.h"
-#include "dictionary.h"
-#include "unigram_dictionary.h"
-
-namespace latinime {
-
-const UnigramDictionary::digraph_t UnigramDictionary::GERMAN_UMLAUT_DIGRAPHS[] =
- { { 'a', 'e' },
- { 'o', 'e' },
- { 'u', 'e' } };
-
-// TODO: check the header
-UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier,
- int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars,
- const bool isLatestDictVersion)
- : DICT_ROOT(streamStart),
- MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords),
- MAX_PROXIMITY_CHARS(maxProximityChars), IS_LATEST_DICT_VERSION(isLatestDictVersion),
- TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier),
- ROOT_POS(isLatestDictVersion ? DICTIONARY_HEADER_SIZE : 0),
- BYTES_IN_ONE_CHAR(MAX_PROXIMITY_CHARS * sizeof(*mInputCodes)),
- MAX_UMLAUT_SEARCH_DEPTH(DEFAULT_MAX_UMLAUT_SEARCH_DEPTH) {
- if (DEBUG_DICT) {
- LOGI("UnigramDictionary - constructor");
- }
-}
-
-UnigramDictionary::~UnigramDictionary() {}
-
-static inline unsigned int getCodesBufferSize(const int* codes, const int codesSize,
- const int MAX_PROXIMITY_CHARS) {
- return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize;
-}
-
-bool UnigramDictionary::isDigraph(const int* codes, const int i, const int codesSize) const {
-
- // There can't be a digraph if we don't have at least 2 characters to examine
- if (i + 2 > codesSize) return false;
-
- // Search for the first char of some digraph
- int lastDigraphIndex = -1;
- const int thisChar = codes[i * MAX_PROXIMITY_CHARS];
- for (lastDigraphIndex = sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0]) - 1;
- lastDigraphIndex >= 0; --lastDigraphIndex) {
- if (thisChar == GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].first) break;
- }
- // No match: return early
- if (lastDigraphIndex < 0) return false;
-
- // It's an interesting digraph if the second char matches too.
- return GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].second == codes[(i + 1) * MAX_PROXIMITY_CHARS];
-}
-
-// Mostly the same arguments as the non-recursive version, except:
-// codes is the original value. It points to the start of the work buffer, and gets passed as is.
-// codesSize is the size of the user input (thus, it is the size of codesSrc).
-// codesDest is the current point in the work buffer.
-// codesSrc is the current point in the user-input, original, content-unmodified buffer.
-// codesRemain is the remaining size in codesSrc.
-void UnigramDictionary::getWordWithDigraphSuggestionsRec(const ProximityInfo *proximityInfo,
- const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
- const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain,
- const int currentDepth, int* codesDest, unsigned short* outWords, int* frequencies) {
-
- if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) {
- for (int i = 0; i < codesRemain; ++i) {
- if (isDigraph(codesSrc, i, codesRemain)) {
- // Found a digraph. We will try both spellings. eg. the word is "pruefen"
-
- // Copy the word up to the first char of the digraph, then continue processing
- // on the remaining part of the word, skipping the second char of the digraph.
- // In our example, copy "pru" and continue running on "fen"
- // Make i the index of the second char of the digraph for simplicity. Forgetting
- // to do that results in an infinite recursion so take care!
- ++i;
- memcpy(codesDest, codesSrc, i * BYTES_IN_ONE_CHAR);
- getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
- codesBuffer, codesBufferSize, flags,
- codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1,
- currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, outWords,
- frequencies);
-
- // Copy the second char of the digraph in place, then continue processing on
- // the remaining part of the word.
- // In our example, after "pru" in the buffer copy the "e", and continue on "fen"
- memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS,
- BYTES_IN_ONE_CHAR);
- getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
- codesBuffer, codesBufferSize, flags, codesSrc + i * MAX_PROXIMITY_CHARS,
- codesRemain - i, currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS,
- outWords, frequencies);
- return;
- }
- }
- }
-
- // If we come here, we hit the end of the word: let's check it against the dictionary.
- // In our example, we'll come here once for "prufen" and then once for "pruefen".
- // If the word contains several digraphs, we'll come it for the product of them.
- // eg. if the word is "ueberpruefen" we'll test, in order, against
- // "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen".
- const unsigned int remainingBytes = BYTES_IN_ONE_CHAR * codesRemain;
- if (0 != remainingBytes)
- memcpy(codesDest, codesSrc, remainingBytes);
-
- getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
- (codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, outWords, frequencies);
-}
-
-int UnigramDictionary::getSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const int codesSize, const int flags,
- unsigned short *outWords, int *frequencies) {
-
- if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags)
- { // Incrementally tune the word and try all possibilities
- int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)];
- getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
- codesSize, flags, codes, codesSize, 0, codesBuffer, outWords, frequencies);
- } else { // Normal processing
- getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize,
- outWords, frequencies);
- }
-
- PROF_START(20);
- // Get the word count
- int suggestedWordsCount = 0;
- while (suggestedWordsCount < MAX_WORDS && mFrequencies[suggestedWordsCount] > 0) {
- suggestedWordsCount++;
- }
-
- if (DEBUG_DICT) {
- LOGI("Returning %d words", suggestedWordsCount);
- LOGI("Next letters: ");
- for (int k = 0; k < NEXT_LETTERS_SIZE; k++) {
- if (mNextLettersFrequency[k] > 0) {
- LOGI("%c = %d,", k, mNextLettersFrequency[k]);
- }
- }
- }
- PROF_END(20);
- PROF_CLOSE;
- return suggestedWordsCount;
-}
-
-void UnigramDictionary::getWordSuggestions(const ProximityInfo *proximityInfo,
- const int *xcoordinates, const int *ycoordinates, const int *codes, const int codesSize,
- unsigned short *outWords, int *frequencies) {
-
- PROF_OPEN;
- PROF_START(0);
- initSuggestions(codes, codesSize, outWords, frequencies);
- if (DEBUG_DICT) assert(codesSize == mInputLength);
-
- const int MAX_DEPTH = min(mInputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
- PROF_END(0);
-
- PROF_START(1);
- getSuggestionCandidates(-1, -1, -1, mNextLettersFrequency, NEXT_LETTERS_SIZE, MAX_DEPTH);
- PROF_END(1);
-
- PROF_START(2);
- // Suggestion with missing character
- if (SUGGEST_WORDS_WITH_MISSING_CHARACTER) {
- for (int i = 0; i < codesSize; ++i) {
- if (DEBUG_DICT) {
- LOGI("--- Suggest missing characters %d", i);
- }
- getSuggestionCandidates(i, -1, -1, NULL, 0, MAX_DEPTH);
- }
- }
- PROF_END(2);
-
- PROF_START(3);
- // Suggestion with excessive character
- if (SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER
- && mInputLength >= MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION) {
- for (int i = 0; i < codesSize; ++i) {
- if (DEBUG_DICT) {
- LOGI("--- Suggest excessive characters %d", i);
- }
- getSuggestionCandidates(-1, i, -1, NULL, 0, MAX_DEPTH);
- }
- }
- PROF_END(3);
-
- PROF_START(4);
- // Suggestion with transposed characters
- // Only suggest words that length is mInputLength
- if (SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS) {
- for (int i = 0; i < codesSize; ++i) {
- if (DEBUG_DICT) {
- LOGI("--- Suggest transposed characters %d", i);
- }
- getSuggestionCandidates(-1, -1, i, NULL, 0, mInputLength - 1);
- }
- }
- PROF_END(4);
-
- PROF_START(5);
- // Suggestions with missing space
- if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER
- && mInputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) {
- for (int i = 1; i < codesSize; ++i) {
- if (DEBUG_DICT) {
- LOGI("--- Suggest missing space characters %d", i);
- }
- getMissingSpaceWords(mInputLength, i);
- }
- }
- PROF_END(5);
-
- PROF_START(6);
- if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) {
- // The first and last "mistyped spaces" are taken care of by excessive character handling
- for (int i = 1; i < codesSize - 1; ++i) {
- if (DEBUG_DICT) {
- LOGI("--- Suggest words with proximity space %d", i);
- }
- const int x = xcoordinates[i];
- const int y = ycoordinates[i];
- if (DEBUG_PROXIMITY_INFO) {
- LOGI("Input[%d] x = %d, y = %d, has space proximity = %d",
- i, x, y, proximityInfo->hasSpaceProximity(x, y));
- }
- if (proximityInfo->hasSpaceProximity(x, y)) {
- getMistypedSpaceWords(mInputLength, i);
- }
- }
- }
- PROF_END(6);
-}
-
-void UnigramDictionary::initSuggestions(const int *codes, const int codesSize,
- unsigned short *outWords, int *frequencies) {
- if (DEBUG_DICT) {
- LOGI("initSuggest");
- }
- mFrequencies = frequencies;
- mOutputChars = outWords;
- mInputCodes = codes;
- mInputLength = codesSize;
- mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2;
-}
-
-static inline void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize) {
- if (c < nextLettersSize) {
- nextLetters[c]++;
- }
-}
-
-// TODO: We need to optimize addWord by using STL or something
-// TODO: This needs to take an const unsigned short* and not tinker with its contents
-bool UnigramDictionary::addWord(unsigned short *word, int length, int frequency) {
- word[length] = 0;
- if (DEBUG_DICT && DEBUG_SHOW_FOUND_WORD) {
- char s[length + 1];
- for (int i = 0; i <= length; i++) s[i] = word[i];
- LOGI("Found word = %s, freq = %d", s, frequency);
- }
- if (length > MAX_WORD_LENGTH) {
- if (DEBUG_DICT) {
- LOGI("Exceeded max word length.");
- }
- return false;
- }
-
- // Find the right insertion point
- int insertAt = 0;
- while (insertAt < MAX_WORDS) {
- // TODO: How should we sort words with the same frequency?
- if (frequency > mFrequencies[insertAt]) {
- break;
- }
- insertAt++;
- }
- if (insertAt < MAX_WORDS) {
- if (DEBUG_DICT) {
- char s[length + 1];
- for (int i = 0; i <= length; i++) s[i] = word[i];
- LOGI("Added word = %s, freq = %d, %d", s, frequency, S_INT_MAX);
- }
- memmove((char*) mFrequencies + (insertAt + 1) * sizeof(mFrequencies[0]),
- (char*) mFrequencies + insertAt * sizeof(mFrequencies[0]),
- (MAX_WORDS - insertAt - 1) * sizeof(mFrequencies[0]));
- mFrequencies[insertAt] = frequency;
- memmove((char*) mOutputChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short),
- (char*) mOutputChars + insertAt * MAX_WORD_LENGTH * sizeof(short),
- (MAX_WORDS - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH);
- unsigned short *dest = mOutputChars + insertAt * MAX_WORD_LENGTH;
- while (length--) {
- *dest++ = *word++;
- }
- *dest = 0; // NULL terminate
- if (DEBUG_DICT) {
- LOGI("Added word at %d", insertAt);
- }
- return true;
- }
- return false;
-}
-
-inline void UnigramDictionary::addWordAlternatesSpellings(const uint8_t* const root, int pos,
- int depth, int finalFreq) {
- // TODO: actually add alternates when the format supports it.
-}
-
-static inline bool hasAlternateSpellings(uint8_t flags) {
- // TODO: when the format supports it, return the actual value.
- return false;
-}
-
-static inline unsigned short toBaseLowerCase(unsigned short c) {
- if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
- c = BASE_CHARS[c];
- }
- if (c >='A' && c <= 'Z') {
- c |= 32;
- } else if (c > 127) {
- c = latin_tolower(c);
- }
- return c;
-}
-
-bool UnigramDictionary::sameAsTyped(const unsigned short *word, int length) const {
- if (length != mInputLength) {
- return false;
- }
- const int *inputCodes = mInputCodes;
- while (length--) {
- if ((unsigned int) *inputCodes != (unsigned int) *word) {
- return false;
- }
- inputCodes += MAX_PROXIMITY_CHARS;
- word++;
- }
- return true;
-}
-
-static const char QUOTE = '\'';
-static const char SPACE = ' ';
-
-void UnigramDictionary::getSuggestionCandidates(const int skipPos,
- const int excessivePos, const int transposedPos, int *nextLetters,
- const int nextLettersSize, const int maxDepth) {
- if (DEBUG_DICT) {
- LOGI("getSuggestionCandidates %d", maxDepth);
- assert(transposedPos + 1 < mInputLength);
- assert(excessivePos < mInputLength);
- assert(missingPos < mInputLength);
- }
- int rootPosition = ROOT_POS;
- // Get the number of child of root, then increment the position
- int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition);
- int depth = 0;
-
- mStackChildCount[0] = childCount;
- mStackTraverseAll[0] = (mInputLength <= 0);
- mStackNodeFreq[0] = 1;
- mStackInputIndex[0] = 0;
- mStackDiffs[0] = 0;
- mStackSiblingPos[0] = rootPosition;
- mStackOutputIndex[0] = 0;
-
- // Depth first search
- while (depth >= 0) {
- if (mStackChildCount[depth] > 0) {
- --mStackChildCount[depth];
- bool traverseAllNodes = mStackTraverseAll[depth];
- int matchWeight = mStackNodeFreq[depth];
- int inputIndex = mStackInputIndex[depth];
- int diffs = mStackDiffs[depth];
- int siblingPos = mStackSiblingPos[depth];
- int outputIndex = mStackOutputIndex[depth];
- int firstChildPos;
- // depth will never be greater than maxDepth because in that case,
- // needsToTraverseChildrenNodes should be false
- const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, outputIndex,
- maxDepth, traverseAllNodes, matchWeight, inputIndex, diffs, skipPos,
- excessivePos, transposedPos, nextLetters, nextLettersSize, &childCount,
- &firstChildPos, &traverseAllNodes, &matchWeight, &inputIndex, &diffs,
- &siblingPos, &outputIndex);
- // Update next sibling pos
- mStackSiblingPos[depth] = siblingPos;
- if (needsToTraverseChildrenNodes) {
- // Goes to child node
- ++depth;
- mStackChildCount[depth] = childCount;
- mStackTraverseAll[depth] = traverseAllNodes;
- mStackNodeFreq[depth] = matchWeight;
- mStackInputIndex[depth] = inputIndex;
- mStackDiffs[depth] = diffs;
- mStackSiblingPos[depth] = firstChildPos;
- mStackOutputIndex[depth] = outputIndex;
- }
- } else {
- // Goes to parent sibling node
- --depth;
- }
- }
-}
-
-static const int TWO_31ST_DIV_255 = S_INT_MAX / 255;
-static inline int capped255MultForFullMatchAccentsOrCapitalizationDifference(const int num) {
- return (num < TWO_31ST_DIV_255 ? 255 * num : S_INT_MAX);
-}
-
-static const int TWO_31ST_DIV_2 = S_INT_MAX / 2;
-inline static void multiplyIntCapped(const int multiplier, int *base) {
- const int temp = *base;
- if (temp != S_INT_MAX) {
- // Branch if multiplier == 2 for the optimization
- if (multiplier == 2) {
- *base = TWO_31ST_DIV_2 >= temp ? temp << 1 : S_INT_MAX;
- } else {
- const int tempRetval = temp * multiplier;
- *base = tempRetval >= temp ? tempRetval : S_INT_MAX;
- }
- }
-}
-
-inline static int powerIntCapped(const int base, const int n) {
- if (base == 2) {
- return n < 31 ? 1 << n : S_INT_MAX;
- } else {
- int ret = base;
- for (int i = 1; i < n; ++i) multiplyIntCapped(base, &ret);
- return ret;
- }
-}
-
-inline static void multiplyRate(const int rate, int *freq) {
- if (*freq != S_INT_MAX) {
- if (*freq > 1000000) {
- *freq /= 100;
- multiplyIntCapped(rate, freq);
- } else {
- multiplyIntCapped(rate, freq);
- *freq /= 100;
- }
- }
-}
-
-inline static int calcFreqForSplitTwoWords(
- const int typedLetterMultiplier, const int firstWordLength, const int secondWordLength,
- const int firstFreq, const int secondFreq, const bool isSpaceProximity) {
- if (firstWordLength == 0 || secondWordLength == 0) {
- return 0;
- }
- const int firstDemotionRate = 100 - 100 / (firstWordLength + 1);
- int tempFirstFreq = firstFreq;
- multiplyRate(firstDemotionRate, &tempFirstFreq);
-
- const int secondDemotionRate = 100 - 100 / (secondWordLength + 1);
- int tempSecondFreq = secondFreq;
- multiplyRate(secondDemotionRate, &tempSecondFreq);
-
- const int totalLength = firstWordLength + secondWordLength;
-
- // Promote pairFreq with multiplying by 2, because the word length is the same as the typed
- // length.
- int totalFreq = tempFirstFreq + tempSecondFreq;
-
- // This is a workaround to try offsetting the not-enough-demotion which will be done in
- // calcNormalizedScore in Utils.java.
- // In calcNormalizedScore the score will be demoted by (1 - 1 / length)
- // but we demoted only (1 - 1 / (length + 1)) so we will additionally adjust freq by
- // (1 - 1 / length) / (1 - 1 / (length + 1)) = (1 - 1 / (length * length))
- const int normalizedScoreNotEnoughDemotionAdjustment = 100 - 100 / (totalLength * totalLength);
- multiplyRate(normalizedScoreNotEnoughDemotionAdjustment, &totalFreq);
-
- // At this moment, totalFreq is calculated by the following formula:
- // (firstFreq * (1 - 1 / (firstWordLength + 1)) + secondFreq * (1 - 1 / (secondWordLength + 1)))
- // * (1 - 1 / totalLength) / (1 - 1 / (totalLength + 1))
-
- multiplyIntCapped(powerIntCapped(typedLetterMultiplier, totalLength), &totalFreq);
-
- // This is another workaround to offset the demotion which will be done in
- // calcNormalizedScore in Utils.java.
- // In calcNormalizedScore the score will be demoted by (1 - 1 / length) so we have to promote
- // the same amount because we already have adjusted the synthetic freq of this "missing or
- // mistyped space" suggestion candidate above in this method.
- const int normalizedScoreDemotionRateOffset = (100 + 100 / totalLength);
- multiplyRate(normalizedScoreDemotionRateOffset, &totalFreq);
-
- if (isSpaceProximity) {
- // A word pair with one space proximity correction
- if (DEBUG_DICT) {
- LOGI("Found a word pair with space proximity correction.");
- }
- multiplyIntCapped(typedLetterMultiplier, &totalFreq);
- multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &totalFreq);
- }
-
- multiplyRate(WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE, &totalFreq);
- return totalFreq;
-}
-
-bool UnigramDictionary::getMissingSpaceWords(const int inputLength, const int missingSpacePos) {
- return getSplitTwoWordsSuggestion(
- inputLength, 0, missingSpacePos, missingSpacePos, inputLength - missingSpacePos, false);
-}
-
-bool UnigramDictionary::getMistypedSpaceWords(const int inputLength, const int spaceProximityPos) {
- return getSplitTwoWordsSuggestion(
- inputLength, 0, spaceProximityPos, spaceProximityPos + 1,
- inputLength - spaceProximityPos - 1, true);
-}
-
-inline int UnigramDictionary::calculateFinalFreq(const int inputIndex, const int depth,
- const int matchWeight, const int skipPos, const int excessivePos, const int transposedPos,
- const int freq, const bool sameLength) const {
- // TODO: Demote by edit distance
- int finalFreq = freq * matchWeight;
- if (skipPos >= 0) {
- if (mInputLength >= 2) {
- const int demotionRate = WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE
- * (10 * mInputLength - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X)
- / (10 * mInputLength
- - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X + 10);
- if (DEBUG_DICT_FULL) {
- LOGI("Demotion rate for missing character is %d.", demotionRate);
- }
- multiplyRate(demotionRate, &finalFreq);
- } else {
- finalFreq = 0;
- }
- }
- if (transposedPos >= 0) multiplyRate(
- WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE, &finalFreq);
- if (excessivePos >= 0) {
- multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq);
- if (!existsAdjacentProximityChars(inputIndex, mInputLength)) {
- multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE, &finalFreq);
- }
- }
- int lengthFreq = TYPED_LETTER_MULTIPLIER;
- multiplyIntCapped(powerIntCapped(TYPED_LETTER_MULTIPLIER, depth), &lengthFreq);
- if (lengthFreq == matchWeight) {
- // Full exact match
- if (depth > 1) {
- if (DEBUG_DICT) {
- LOGI("Found full matched word.");
- }
- multiplyRate(FULL_MATCHED_WORDS_PROMOTION_RATE, &finalFreq);
- }
- if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0) {
- finalFreq = capped255MultForFullMatchAccentsOrCapitalizationDifference(finalFreq);
- }
- } else if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0 && depth > 0) {
- // A word with proximity corrections
- if (DEBUG_DICT) {
- LOGI("Found one proximity correction.");
- }
- multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &finalFreq);
- multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq);
- }
- if (DEBUG_DICT) {
- LOGI("calc: %d, %d", depth, sameLength);
- }
- if (sameLength) multiplyIntCapped(FULL_WORD_MULTIPLIER, &finalFreq);
- return finalFreq;
-}
-
-inline bool UnigramDictionary::needsToSkipCurrentNode(const unsigned short c,
- const int inputIndex, const int skipPos, const int depth) {
- const unsigned short userTypedChar = getInputCharsAt(inputIndex)[0];
- // Skip the ' or other letter and continue deeper
- return (c == QUOTE && userTypedChar != QUOTE) || skipPos == depth;
-}
-
-inline bool UnigramDictionary::existsAdjacentProximityChars(const int inputIndex,
- const int inputLength) const {
- if (inputIndex < 0 || inputIndex >= inputLength) return false;
- const int currentChar = *getInputCharsAt(inputIndex);
- const int leftIndex = inputIndex - 1;
- if (leftIndex >= 0) {
- const int *leftChars = getInputCharsAt(leftIndex);
- int i = 0;
- while (leftChars[i] > 0 && i < MAX_PROXIMITY_CHARS) {
- if (leftChars[i++] == currentChar) return true;
- }
- }
- const int rightIndex = inputIndex + 1;
- if (rightIndex < inputLength) {
- const int *rightChars = getInputCharsAt(rightIndex);
- int i = 0;
- while (rightChars[i] > 0 && i < MAX_PROXIMITY_CHARS) {
- if (rightChars[i++] == currentChar) return true;
- }
- }
- return false;
-}
-
-// In the following function, c is the current character of the dictionary word
-// currently examined.
-// currentChars is an array containing the keys close to the character the
-// user actually typed at the same position. We want to see if c is in it: if so,
-// then the word contains at that position a character close to what the user
-// typed.
-// What the user typed is actually the first character of the array.
-// Notice : accented characters do not have a proximity list, so they are alone
-// in their list. The non-accented version of the character should be considered
-// "close", but not the other keys close to the non-accented version.
-inline UnigramDictionary::ProximityType UnigramDictionary::getMatchedProximityId(
- const int *currentChars, const unsigned short c, const int skipPos,
- const int excessivePos, const int transposedPos) {
- const unsigned short baseLowerC = toBaseLowerCase(c);
-
- // The first char in the array is what user typed. If it matches right away,
- // that means the user typed that same char for this pos.
- if (currentChars[0] == baseLowerC || currentChars[0] == c)
- return SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR;
-
- // If one of those is true, we should not check for close characters at all.
- if (skipPos >= 0 || excessivePos >= 0 || transposedPos >= 0)
- return UNRELATED_CHAR;
-
- // If the non-accented, lowercased version of that first character matches c,
- // then we have a non-accented version of the accented character the user
- // typed. Treat it as a close char.
- if (toBaseLowerCase(currentChars[0]) == baseLowerC)
- return NEAR_PROXIMITY_CHAR;
-
- // Not an exact nor an accent-alike match: search the list of close keys
- int j = 1;
- while (currentChars[j] > 0 && j < MAX_PROXIMITY_CHARS) {
- const bool matched = (currentChars[j] == baseLowerC || currentChars[j] == c);
- if (matched) return NEAR_PROXIMITY_CHAR;
- ++j;
- }
-
- // Was not included, signal this as an unrelated character.
- return UNRELATED_CHAR;
-}
-
-inline void UnigramDictionary::onTerminal(unsigned short int* word, const int depth,
- const uint8_t* const root, const uint8_t flags, int pos,
- const int inputIndex, const int matchWeight, const int skipPos,
- const int excessivePos, const int transposedPos, const int freq, const bool sameLength,
- int* nextLetters, const int nextLettersSize) {
-
- const bool isSameAsTyped = sameLength ? sameAsTyped(word, depth + 1) : false;
- const bool hasAlternates = hasAlternateSpellings(flags);
- if (isSameAsTyped && !hasAlternates) return;
-
- if (depth >= MIN_SUGGEST_DEPTH) {
- const int finalFreq = calculateFinalFreq(inputIndex, depth, matchWeight, skipPos,
- excessivePos, transposedPos, freq, sameLength);
- if (!isSameAsTyped)
- addWord(word, depth + 1, finalFreq);
- if (hasAlternates)
- addWordAlternatesSpellings(DICT_ROOT, pos, flags, finalFreq);
- }
-
- if (sameLength && depth >= mInputLength && skipPos < 0) {
- registerNextLetter(word[mInputLength], nextLetters, nextLettersSize);
- }
-}
-
-#ifndef NEW_DICTIONARY_FORMAT
-// TODO: Don't forget to bring inline functions back to over where they are used.
-
-// The following functions will be entirely replaced with new implementations.
-void UnigramDictionary::getWordsOld(const int initialPos, const int inputLength, const int skipPos,
- const int excessivePos, const int transposedPos,int *nextLetters,
- const int nextLettersSize) {
- int initialPosition = initialPos;
- const int count = Dictionary::getCount(DICT_ROOT, &initialPosition);
- getWordsRec(count, initialPosition, 0,
- min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH),
- mInputLength <= 0, 1, 0, 0, skipPos, excessivePos, transposedPos, nextLetters,
- nextLettersSize);
-}
-
-void UnigramDictionary::getWordsRec(const int childrenCount, const int pos, const int depth,
- const int maxDepth, const bool traverseAllNodes, const int matchWeight,
- const int inputIndex, const int diffs, const int skipPos, const int excessivePos,
- const int transposedPos, int *nextLetters, const int nextLettersSize) {
- int siblingPos = pos;
- for (int i = 0; i < childrenCount; ++i) {
- int newCount;
- int newChildPosition;
- bool newTraverseAllNodes;
- int newMatchRate;
- int newInputIndex;
- int newDiffs;
- int newSiblingPos;
- int newOutputIndex;
- const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, depth, maxDepth,
- traverseAllNodes, matchWeight, inputIndex, diffs,
- skipPos, excessivePos, transposedPos,
- nextLetters, nextLettersSize,
- &newCount, &newChildPosition, &newTraverseAllNodes, &newMatchRate,
- &newInputIndex, &newDiffs, &newSiblingPos, &newOutputIndex);
- siblingPos = newSiblingPos;
-
- if (needsToTraverseChildrenNodes) {
- getWordsRec(newCount, newChildPosition, newOutputIndex, maxDepth, newTraverseAllNodes,
- newMatchRate, newInputIndex, newDiffs, skipPos, excessivePos, transposedPos,
- nextLetters, nextLettersSize);
- }
- }
-}
-
-inline int UnigramDictionary::getBestWordFreq(const int startInputIndex, const int inputLength,
- unsigned short *word) {
- int pos = ROOT_POS;
- int count = Dictionary::getCount(DICT_ROOT, &pos);
- int maxFreq = 0;
- int depth = 0;
- unsigned short newWord[MAX_WORD_LENGTH_INTERNAL];
- bool terminal = false;
-
- mStackChildCount[0] = count;
- mStackSiblingPos[0] = pos;
-
- while (depth >= 0) {
- if (mStackChildCount[depth] > 0) {
- --mStackChildCount[depth];
- int firstChildPos;
- int newFreq;
- int siblingPos = mStackSiblingPos[depth];
- const bool needsToTraverseChildrenNodes = processCurrentNodeForExactMatch(siblingPos,
- startInputIndex, depth, newWord, &firstChildPos, &count, &terminal, &newFreq,
- &siblingPos);
- mStackSiblingPos[depth] = siblingPos;
- if (depth == (inputLength - 1)) {
- // Traverse sibling node
- if (terminal) {
- if (newFreq > maxFreq) {
- for (int i = 0; i < inputLength; ++i) word[i] = newWord[i];
- if (DEBUG_DICT && DEBUG_NODE) {
- char s[inputLength + 1];
- for (int i = 0; i < inputLength; ++i) s[i] = word[i];
- s[inputLength] = 0;
- LOGI("New missing space word found: %d > %d (%s), %d, %d",
- newFreq, maxFreq, s, inputLength, depth);
- }
- maxFreq = newFreq;
- }
- }
- } else if (needsToTraverseChildrenNodes) {
- // Traverse children nodes
- ++depth;
- mStackChildCount[depth] = count;
- mStackSiblingPos[depth] = firstChildPos;
- }
- } else {
- // Traverse parent node
- --depth;
- }
- }
-
- word[inputLength] = 0;
- return maxFreq;
-}
-
-inline bool UnigramDictionary::processCurrentNodeForExactMatch(const int firstChildPos,
- const int startInputIndex, const int depth, unsigned short *word, int *newChildPosition,
- int *newCount, bool *newTerminal, int *newFreq, int *siblingPos) {
- const int inputIndex = startInputIndex + depth;
- const int *currentChars = getInputCharsAt(inputIndex);
- unsigned short c;
- *siblingPos = Dictionary::setDictionaryValues(DICT_ROOT, IS_LATEST_DICT_VERSION, firstChildPos,
- &c, newChildPosition, newTerminal, newFreq);
- const unsigned int inputC = currentChars[0];
- if (DEBUG_DICT) {
- assert(inputC <= U_SHORT_MAX);
- }
- const unsigned short baseLowerC = toBaseLowerCase(c);
- const bool matched = (inputC == baseLowerC || inputC == c);
- const bool hasChild = *newChildPosition != 0;
- if (matched) {
- word[depth] = c;
- if (DEBUG_DICT && DEBUG_NODE) {
- LOGI("Node(%c, %c)<%d>, %d, %d", inputC, c, matched, hasChild, *newFreq);
- if (*newTerminal) {
- LOGI("Terminal %d", *newFreq);
- }
- }
- if (hasChild) {
- *newCount = Dictionary::getCount(DICT_ROOT, newChildPosition);
- return true;
- } else {
- return false;
- }
- } else {
- // If this node is not user typed character, this method treats this word as unmatched.
- // Thus newTerminal shouldn't be true.
- *newTerminal = false;
- return false;
- }
-}
-
-// TODO: use uint32_t instead of unsigned short
-bool UnigramDictionary::isValidWord(unsigned short *word, int length) {
- if (IS_LATEST_DICT_VERSION) {
- return (getBigramPosition(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD);
- } else {
- return (getBigramPosition(0, word, 0, length) != NOT_VALID_WORD);
- }
-}
-
-
-// Require strict exact match.
-int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offset,
- int length) const {
- // returns address of bigram data of that word
- // return -99 if not found
-
- int count = Dictionary::getCount(DICT_ROOT, &pos);
- unsigned short currentChar = (unsigned short) word[offset];
- for (int j = 0; j < count; j++) {
- unsigned short c = Dictionary::getChar(DICT_ROOT, &pos);
- int terminal = Dictionary::getTerminal(DICT_ROOT, &pos);
- int childPos = Dictionary::getAddress(DICT_ROOT, &pos);
- if (c == currentChar) {
- if (offset == length - 1) {
- if (terminal) {
- return (pos+1);
- }
- } else {
- if (childPos != 0) {
- int t = getBigramPosition(childPos, word, offset + 1, length);
- if (t > 0) {
- return t;
- }
- }
- }
- }
- if (terminal) {
- Dictionary::getFreq(DICT_ROOT, IS_LATEST_DICT_VERSION, &pos);
- }
- // There could be two instances of each alphabet - upper and lower case. So continue
- // looking ...
- }
- return NOT_VALID_WORD;
-}
-
-
-// The following functions will be modified.
-bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength,
- const int firstWordStartPos, const int firstWordLength, const int secondWordStartPos,
- const int secondWordLength, const bool isSpaceProximity) {
- if (inputLength >= MAX_WORD_LENGTH) return false;
- if (0 >= firstWordLength || 0 >= secondWordLength || firstWordStartPos >= secondWordStartPos
- || firstWordStartPos < 0 || secondWordStartPos + secondWordLength > inputLength)
- return false;
- const int newWordLength = firstWordLength + secondWordLength + 1;
- // Allocating variable length array on stack
- unsigned short word[newWordLength];
- const int firstFreq = getBestWordFreq(firstWordStartPos, firstWordLength, mWord);
- if (DEBUG_DICT) {
- LOGI("First freq: %d", firstFreq);
- }
- if (firstFreq <= 0) return false;
-
- for (int i = 0; i < firstWordLength; ++i) {
- word[i] = mWord[i];
- }
-
- const int secondFreq = getBestWordFreq(secondWordStartPos, secondWordLength, mWord);
- if (DEBUG_DICT) {
- LOGI("Second freq: %d", secondFreq);
- }
- if (secondFreq <= 0) return false;
-
- word[firstWordLength] = SPACE;
- for (int i = (firstWordLength + 1); i < newWordLength; ++i) {
- word[i] = mWord[i - firstWordLength - 1];
- }
-
- int pairFreq = calcFreqForSplitTwoWords(TYPED_LETTER_MULTIPLIER, firstWordLength,
- secondWordLength, firstFreq, secondFreq, isSpaceProximity);
- if (DEBUG_DICT) {
- LOGI("Split two words: %d, %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength,
- TYPED_LETTER_MULTIPLIER);
- }
- addWord(word, newWordLength, pairFreq);
- return true;
-}
-
-inline bool UnigramDictionary::processCurrentNode(const int pos, const int depth,
- const int maxDepth, const bool traverseAllNodes, int matchWeight, int inputIndex,
- const int diffs, const int skipPos, const int excessivePos, const int transposedPos,
- int *nextLetters, const int nextLettersSize, int *newCount, int *newChildPosition,
- bool *newTraverseAllNodes, int *newMatchRate, int *newInputIndex, int *newDiffs,
- int *nextSiblingPosition, int *nextOutputIndex) {
- if (DEBUG_DICT) {
- int inputCount = 0;
- if (skipPos >= 0) ++inputCount;
- if (excessivePos >= 0) ++inputCount;
- if (transposedPos >= 0) ++inputCount;
- assert(inputCount <= 1);
- }
- unsigned short c;
- int childPosition;
- bool terminal;
- int freq;
- bool isSameAsUserTypedLength = false;
-
- const uint8_t flags = 0; // No flags for now
-
- if (excessivePos == depth && inputIndex < mInputLength - 1) ++inputIndex;
-
- *nextSiblingPosition = Dictionary::setDictionaryValues(DICT_ROOT, IS_LATEST_DICT_VERSION, pos,
- &c, &childPosition, &terminal, &freq);
- *nextOutputIndex = depth + 1;
-
- const bool needsToTraverseChildrenNodes = childPosition != 0;
-
- // If we are only doing traverseAllNodes, no need to look at the typed characters.
- if (traverseAllNodes || needsToSkipCurrentNode(c, inputIndex, skipPos, depth)) {
- mWord[depth] = c;
- if (traverseAllNodes && terminal) {
- onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos,
- excessivePos, transposedPos, freq, false, nextLetters, nextLettersSize);
- }
- if (!needsToTraverseChildrenNodes) return false;
- *newTraverseAllNodes = traverseAllNodes;
- *newMatchRate = matchWeight;
- *newDiffs = diffs;
- *newInputIndex = inputIndex;
- } else {
- const int *currentChars = getInputCharsAt(inputIndex);
-
- if (transposedPos >= 0) {
- if (inputIndex == transposedPos) currentChars += MAX_PROXIMITY_CHARS;
- if (inputIndex == (transposedPos + 1)) currentChars -= MAX_PROXIMITY_CHARS;
- }
-
- int matchedProximityCharId = getMatchedProximityId(currentChars, c, skipPos, excessivePos,
- transposedPos);
- if (UNRELATED_CHAR == matchedProximityCharId) return false;
- mWord[depth] = c;
- // If inputIndex is greater than mInputLength, that means there is no
- // proximity chars. So, we don't need to check proximity.
- if (SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR == matchedProximityCharId) {
- multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &matchWeight);
- }
- bool isSameAsUserTypedLength = mInputLength == inputIndex + 1
- || (excessivePos == mInputLength - 1 && inputIndex == mInputLength - 2);
- if (isSameAsUserTypedLength && terminal) {
- onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos,
- excessivePos, transposedPos, freq, true, nextLetters, nextLettersSize);
- }
- if (!needsToTraverseChildrenNodes) return false;
- // Start traversing all nodes after the index exceeds the user typed length
- *newTraverseAllNodes = isSameAsUserTypedLength;
- *newMatchRate = matchWeight;
- *newDiffs = diffs + ((NEAR_PROXIMITY_CHAR == matchedProximityCharId) ? 1 : 0);
- *newInputIndex = inputIndex + 1;
- }
- // Optimization: Prune out words that are too long compared to how much was typed.
- if (depth >= maxDepth || *newDiffs > mMaxEditDistance) {
- return false;
- }
-
- // If inputIndex is greater than mInputLength, that means there are no proximity chars.
- // TODO: Check if this can be isSameAsUserTypedLength only.
- if (isSameAsUserTypedLength || mInputLength <= *newInputIndex) {
- *newTraverseAllNodes = true;
- }
- // get the count of nodes and increment childAddress.
- *newCount = Dictionary::getCount(DICT_ROOT, &childPosition);
- *newChildPosition = childPosition;
- if (DEBUG_DICT) assert(needsToTraverseChildrenNodes);
- return needsToTraverseChildrenNodes;
-}
-
-#else // NEW_DICTIONARY_FORMAT
-
-bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength,
- const int firstWordStartPos, const int firstWordLength, const int secondWordStartPos,
- const int secondWordLength, const bool isSpaceProximity) {
- if (inputLength >= MAX_WORD_LENGTH) return false;
- if (0 >= firstWordLength || 0 >= secondWordLength || firstWordStartPos >= secondWordStartPos
- || firstWordStartPos < 0 || secondWordStartPos + secondWordLength > inputLength)
- return false;
- const int newWordLength = firstWordLength + secondWordLength + 1;
- // Allocating variable length array on stack
- unsigned short word[newWordLength];
- const int firstFreq = getBestWordFreq(firstWordStartPos, firstWordLength, mWord);
- if (DEBUG_DICT) {
- LOGI("First freq: %d", firstFreq);
- }
- if (firstFreq <= 0) return false;
-
- for (int i = 0; i < firstWordLength; ++i) {
- word[i] = mWord[i];
- }
-
- const int secondFreq = getBestWordFreq(secondWordStartPos, secondWordLength, mWord);
- if (DEBUG_DICT) {
- LOGI("Second freq: %d", secondFreq);
- }
- if (secondFreq <= 0) return false;
-
- word[firstWordLength] = SPACE;
- for (int i = (firstWordLength + 1); i < newWordLength; ++i) {
- word[i] = mWord[i - firstWordLength - 1];
- }
-
- int pairFreq = calcFreqForSplitTwoWords(TYPED_LETTER_MULTIPLIER, firstWordLength,
- secondWordLength, firstFreq, secondFreq, isSpaceProximity);
- if (DEBUG_DICT) {
- LOGI("Split two words: %d, %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength,
- TYPED_LETTER_MULTIPLIER);
- }
- addWord(word, newWordLength, pairFreq);
- return true;
-}
-
-inline bool UnigramDictionary::processCurrentNode(const int pos, const int depth,
- const int maxDepth, const bool traverseAllNodes, int matchWeight, int inputIndex,
- const int diffs, const int skipPos, const int excessivePos, const int transposedPos,
- int *nextLetters, const int nextLettersSize, int *newCount, int *newChildPosition,
- bool *newTraverseAllNodes, int *newMatchRate, int *newInputIndex, int *newDiffs,
- int *nextSiblingPosition, int *nextOutputIndex) {
- if (DEBUG_DICT) {
- int inputCount = 0;
- if (skipPos >= 0) ++inputCount;
- if (excessivePos >= 0) ++inputCount;
- if (transposedPos >= 0) ++inputCount;
- assert(inputCount <= 1);
- }
- unsigned short c;
- int childPosition;
- bool terminal;
- int freq;
- bool isSameAsUserTypedLength = false;
-
- const uint8_t flags = 0; // No flags for now
-
- if (excessivePos == depth && inputIndex < mInputLength - 1) ++inputIndex;
-
- *nextSiblingPosition = Dictionary::setDictionaryValues(DICT_ROOT, IS_LATEST_DICT_VERSION, pos,
- &c, &childPosition, &terminal, &freq);
- *nextOutputIndex = depth + 1;
-
- const bool needsToTraverseChildrenNodes = childPosition != 0;
-
- // If we are only doing traverseAllNodes, no need to look at the typed characters.
- if (traverseAllNodes || needsToSkipCurrentNode(c, inputIndex, skipPos, depth)) {
- mWord[depth] = c;
- if (traverseAllNodes && terminal) {
- onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos,
- excessivePos, transposedPos, freq, false, nextLetters, nextLettersSize);
- }
- if (!needsToTraverseChildrenNodes) return false;
- *newTraverseAllNodes = traverseAllNodes;
- *newMatchRate = matchWeight;
- *newDiffs = diffs;
- *newInputIndex = inputIndex;
- } else {
- const int *currentChars = getInputCharsAt(inputIndex);
-
- if (transposedPos >= 0) {
- if (inputIndex == transposedPos) currentChars += MAX_PROXIMITY_CHARS;
- if (inputIndex == (transposedPos + 1)) currentChars -= MAX_PROXIMITY_CHARS;
- }
-
- int matchedProximityCharId = getMatchedProximityId(currentChars, c, skipPos, excessivePos,
- transposedPos);
- if (UNRELATED_CHAR == matchedProximityCharId) return false;
- mWord[depth] = c;
- // If inputIndex is greater than mInputLength, that means there is no
- // proximity chars. So, we don't need to check proximity.
- if (SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR == matchedProximityCharId) {
- multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &matchWeight);
- }
- bool isSameAsUserTypedLength = mInputLength == inputIndex + 1
- || (excessivePos == mInputLength - 1 && inputIndex == mInputLength - 2);
- if (isSameAsUserTypedLength && terminal) {
- onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos,
- excessivePos, transposedPos, freq, true, nextLetters, nextLettersSize);
- }
- if (!needsToTraverseChildrenNodes) return false;
- // Start traversing all nodes after the index exceeds the user typed length
- *newTraverseAllNodes = isSameAsUserTypedLength;
- *newMatchRate = matchWeight;
- *newDiffs = diffs + ((NEAR_PROXIMITY_CHAR == matchedProximityCharId) ? 1 : 0);
- *newInputIndex = inputIndex + 1;
- }
- // Optimization: Prune out words that are too long compared to how much was typed.
- if (depth >= maxDepth || *newDiffs > mMaxEditDistance) {
- return false;
- }
-
- // If inputIndex is greater than mInputLength, that means there are no proximity chars.
- // TODO: Check if this can be isSameAsUserTypedLength only.
- if (isSameAsUserTypedLength || mInputLength <= *newInputIndex) {
- *newTraverseAllNodes = true;
- }
- // get the count of nodes and increment childAddress.
- *newCount = Dictionary::getCount(DICT_ROOT, &childPosition);
- *newChildPosition = childPosition;
- if (DEBUG_DICT) assert(needsToTraverseChildrenNodes);
- return needsToTraverseChildrenNodes;
-}
-
-#endif // NEW_DICTIONARY_FORMAT
-
-} // namespace latinime
diff --git a/native/src/unigram_dictionary.h b/native/src/unigram_dictionary.h
deleted file mode 100644
index 154ac9b36..000000000
--- a/native/src/unigram_dictionary.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_UNIGRAM_DICTIONARY_H
-#define LATINIME_UNIGRAM_DICTIONARY_H
-
-#include <stdint.h>
-#include "defines.h"
-#include "proximity_info.h"
-
-#ifndef NULL
-#define NULL 0
-#endif
-
-namespace latinime {
-
-class UnigramDictionary {
-
- typedef enum { // Used as a return value for character comparison
- SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR, // Same char, possibly with different case or accent
- NEAR_PROXIMITY_CHAR, // It is a char located nearby on the keyboard
- UNRELATED_CHAR // It is an unrelated char
- } ProximityType;
-
-public:
- UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler,
- int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars,
- const bool isLatestDictVersion);
- bool isValidWord(unsigned short *word, int length);
- int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
- int getSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const int codesSize, const int flags,
- unsigned short *outWords, int *frequencies);
- ~UnigramDictionary();
-
-private:
- void getWordSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates,
- const int *ycoordinates, const int *codes, const int codesSize,
- unsigned short *outWords, int *frequencies);
- bool isDigraph(const int* codes, const int i, const int codesSize) const;
- void getWordWithDigraphSuggestionsRec(const ProximityInfo *proximityInfo,
- const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
- const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain,
- const int currentDepth, int* codesDest, unsigned short* outWords, int* frequencies);
- void initSuggestions(const int *codes, const int codesSize, unsigned short *outWords,
- int *frequencies);
- void getSuggestionCandidates(const int skipPos, const int excessivePos,
- const int transposedPos, int *nextLetters, const int nextLettersSize,
- const int maxDepth);
- void getVersionNumber();
- bool checkIfDictVersionIsLatest();
- int getAddress(int *pos);
- int getFreq(int *pos);
- bool sameAsTyped(const unsigned short *word, int length) const;
- bool addWord(unsigned short *word, int length, int frequency);
- void addWordAlternatesSpellings(const uint8_t* const root, int pos, int depth, int finalFreq);
- void getWordsRec(const int childrenCount, const int pos, const int depth, const int maxDepth,
- const bool traverseAllNodes, const int snr, const int inputIndex, const int diffs,
- const int skipPos, const int excessivePos, const int transposedPos, int *nextLetters,
- const int nextLettersSize);
- bool getSplitTwoWordsSuggestion(const int inputLength,
- const int firstWordStartPos, const int firstWordLength,
- const int secondWordStartPos, const int secondWordLength, const bool isSpaceProximity);
- bool getMissingSpaceWords(const int inputLength, const int missingSpacePos);
- bool getMistypedSpaceWords(const int inputLength, const int spaceProximityPos);
- // Keep getWordsOld for comparing performance between getWords and getWordsOld
- void getWordsOld(const int initialPos, const int inputLength, const int skipPos,
- const int excessivePos, const int transposedPos, int *nextLetters,
- const int nextLettersSize);
- int calculateFinalFreq(const int inputIndex, const int depth, const int snr, const int skipPos,
- const int excessivePos, const int transposedPos, const int freq,
- const bool sameLength) const;
- void onTerminal(unsigned short int* word, const int depth,
- const uint8_t* const root, const uint8_t flags, int pos,
- const int inputIndex, const int matchWeight, const int skipPos,
- const int excessivePos, const int transposedPos, const int freq, const bool sameLength,
- int *nextLetters, const int nextLettersSize);
- bool needsToSkipCurrentNode(const unsigned short c,
- const int inputIndex, const int skipPos, const int depth);
- ProximityType getMatchedProximityId(const int *currentChars, const unsigned short c,
- const int skipPos, const int excessivePos, const int transposedPos);
- // Process a node by considering proximity, missing and excessive character
- bool processCurrentNode(const int pos, const int depth,
- const int maxDepth, const bool traverseAllNodes, const int snr, int inputIndex,
- const int diffs, const int skipPos, const int excessivePos, const int transposedPos,
- int *nextLetters, const int nextLettersSize, int *newCount, int *newChildPosition,
- bool *newTraverseAllNodes, int *newSnr, int*newInputIndex, int *newDiffs,
- int *nextSiblingPosition, int *nextOutputIndex);
- int getBestWordFreq(const int startInputIndex, const int inputLength, unsigned short *word);
- // Process a node by considering missing space
- bool processCurrentNodeForExactMatch(const int firstChildPos,
- const int startInputIndex, const int depth, unsigned short *word,
- int *newChildPosition, int *newCount, bool *newTerminal, int *newFreq, int *siblingPos);
- bool existsAdjacentProximityChars(const int inputIndex, const int inputLength) const;
- inline const int* getInputCharsAt(const int index) const {
- return mInputCodes + (index * MAX_PROXIMITY_CHARS);
- }
-
- const uint8_t* const DICT_ROOT;
- const int MAX_WORD_LENGTH;
- const int MAX_WORDS;
- const int MAX_PROXIMITY_CHARS;
- const bool IS_LATEST_DICT_VERSION;
- const int TYPED_LETTER_MULTIPLIER;
- const int FULL_WORD_MULTIPLIER;
- const int ROOT_POS;
- const unsigned int BYTES_IN_ONE_CHAR;
- const int MAX_UMLAUT_SEARCH_DEPTH;
-
- // Flags for special processing
- // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java
- // or something very bad (like, the apocalypse) will happen.
- // Please update both at the same time.
- enum {
- REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1
- };
- static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[];
-
- int *mFrequencies;
- unsigned short *mOutputChars;
- const int *mInputCodes;
- int mInputLength;
- // MAX_WORD_LENGTH_INTERNAL must be bigger than MAX_WORD_LENGTH
- unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];
- int mMaxEditDistance;
-
- int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];
- bool mStackTraverseAll[MAX_WORD_LENGTH_INTERNAL];
- int mStackNodeFreq[MAX_WORD_LENGTH_INTERNAL];
- int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];
- int mStackDiffs[MAX_WORD_LENGTH_INTERNAL];
- int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];
- int mStackOutputIndex[MAX_WORD_LENGTH_INTERNAL];
- int mNextLettersFrequency[NEXT_LETTERS_SIZE];
-};
-
-} // namespace latinime
-
-#endif // LATINIME_UNIGRAM_DICTIONARY_H
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-06-03 23:47:55 +0000