add generator script for NFC character properties

7 years ago · 0072632a88
parent 69a4c693b7
commit 0072632a88
5 changed files with 229 additions and 5 deletions
--- a/server/Makefile
+++ b/server/Makefile
@ -28,6 +28,7 @@ mt-server: main.o libjson-adapter.a
 libjson-adapter.a: json-adapter.o registry.o nfc.o json_rpc.o \
    function_map.o pep-types.o \
    security-token.o \
+    nfc_sets.o \
    json_spirit/json_spirit_reader.o json_spirit/json_spirit_value.o json_spirit/json_spirit_writer.o 
 	ar rcs $@ $^

--- a/server/nfc.cc
+++ b/server/nfc.cc
@ -4,11 +4,7 @@
 #include <cstdint>
 #include <set>

-// TODO: These sets should be filled by code generated automatically
-// from Unicode's DerivedNormalizationProps.txt
-extern const std::set<unsigned> NFC_No;
-extern const std::set<unsigned> NFC_Maybe;
-
+#include "nfc_sets.hh"

 namespace
 {
--- a/server/nfc_sets.cc
+++ b/server/nfc_sets.cc
@ -0,0 +1,137 @@
+// This file is generated by scripts/gen_sets.sh
+// DO NOT EDIT IT!
+
+#include "nfc_sets.hh"
+
+
+const unsigned NFC_No_Size = 1120;
+const std::set<unsigned> NFC_No = {
+	0x0340,0x0341,0x0343,0x0344,0x0374,0x037E,0x0387,0x0958,0x0959,0x095A,
+	0x095B,0x095C,0x095D,0x095E,0x095F,0x09DC,0x09DD,0x09DF,0x0A33,0x0A36,
+	0x0A59,0x0A5A,0x0A5B,0x0A5E,0x0B5C,0x0B5D,0x0F43,0x0F4D,0x0F52,0x0F57,
+	0x0F5C,0x0F69,0x0F73,0x0F75,0x0F76,0x0F78,0x0F81,0x0F93,0x0F9D,0x0FA2,
+	0x0FA7,0x0FAC,0x0FB9,0x1F71,0x1F73,0x1F75,0x1F77,0x1F79,0x1F7B,0x1F7D,
+	0x1FBB,0x1FBE,0x1FC9,0x1FCB,0x1FD3,0x1FDB,0x1FE3,0x1FEB,0x1FEE,0x1FEF,
+	0x1FF9,0x1FFB,0x1FFD,0x2000,0x2001,0x2126,0x212A,0x212B,0x2329,0x232A,
+	0x2ADC,0xF900,0xF901,0xF902,0xF903,0xF904,0xF905,0xF906,0xF907,0xF908,
+	0xF909,0xF90A,0xF90B,0xF90C,0xF90D,0xF90E,0xF90F,0xF910,0xF911,0xF912,
+	0xF913,0xF914,0xF915,0xF916,0xF917,0xF918,0xF919,0xF91A,0xF91B,0xF91C,
+	0xF91D,0xF91E,0xF91F,0xF920,0xF921,0xF922,0xF923,0xF924,0xF925,0xF926,
+	0xF927,0xF928,0xF929,0xF92A,0xF92B,0xF92C,0xF92D,0xF92E,0xF92F,0xF930,
+	0xF931,0xF932,0xF933,0xF934,0xF935,0xF936,0xF937,0xF938,0xF939,0xF93A,
+	0xF93B,0xF93C,0xF93D,0xF93E,0xF93F,0xF940,0xF941,0xF942,0xF943,0xF944,
+	0xF945,0xF946,0xF947,0xF948,0xF949,0xF94A,0xF94B,0xF94C,0xF94D,0xF94E,
+	0xF94F,0xF950,0xF951,0xF952,0xF953,0xF954,0xF955,0xF956,0xF957,0xF958,
+	0xF959,0xF95A,0xF95B,0xF95C,0xF95D,0xF95E,0xF95F,0xF960,0xF961,0xF962,
+	0xF963,0xF964,0xF965,0xF966,0xF967,0xF968,0xF969,0xF96A,0xF96B,0xF96C,
+	0xF96D,0xF96E,0xF96F,0xF970,0xF971,0xF972,0xF973,0xF974,0xF975,0xF976,
+	0xF977,0xF978,0xF979,0xF97A,0xF97B,0xF97C,0xF97D,0xF97E,0xF97F,0xF980,
+	0xF981,0xF982,0xF983,0xF984,0xF985,0xF986,0xF987,0xF988,0xF989,0xF98A,
+	0xF98B,0xF98C,0xF98D,0xF98E,0xF98F,0xF990,0xF991,0xF992,0xF993,0xF994,
+	0xF995,0xF996,0xF997,0xF998,0xF999,0xF99A,0xF99B,0xF99C,0xF99D,0xF99E,
+	0xF99F,0xF9A0,0xF9A1,0xF9A2,0xF9A3,0xF9A4,0xF9A5,0xF9A6,0xF9A7,0xF9A8,
+	0xF9A9,0xF9AA,0xF9AB,0xF9AC,0xF9AD,0xF9AE,0xF9AF,0xF9B0,0xF9B1,0xF9B2,
+	0xF9B3,0xF9B4,0xF9B5,0xF9B6,0xF9B7,0xF9B8,0xF9B9,0xF9BA,0xF9BB,0xF9BC,
+	0xF9BD,0xF9BE,0xF9BF,0xF9C0,0xF9C1,0xF9C2,0xF9C3,0xF9C4,0xF9C5,0xF9C6,
+	0xF9C7,0xF9C8,0xF9C9,0xF9CA,0xF9CB,0xF9CC,0xF9CD,0xF9CE,0xF9CF,0xF9D0,
+	0xF9D1,0xF9D2,0xF9D3,0xF9D4,0xF9D5,0xF9D6,0xF9D7,0xF9D8,0xF9D9,0xF9DA,
+	0xF9DB,0xF9DC,0xF9DD,0xF9DE,0xF9DF,0xF9E0,0xF9E1,0xF9E2,0xF9E3,0xF9E4,
+	0xF9E5,0xF9E6,0xF9E7,0xF9E8,0xF9E9,0xF9EA,0xF9EB,0xF9EC,0xF9ED,0xF9EE,
+	0xF9EF,0xF9F0,0xF9F1,0xF9F2,0xF9F3,0xF9F4,0xF9F5,0xF9F6,0xF9F7,0xF9F8,
+	0xF9F9,0xF9FA,0xF9FB,0xF9FC,0xF9FD,0xF9FE,0xF9FF,0xFA00,0xFA01,0xFA02,
+	0xFA03,0xFA04,0xFA05,0xFA06,0xFA07,0xFA08,0xFA09,0xFA0A,0xFA0B,0xFA0C,
+	0xFA0D,0xFA10,0xFA12,0xFA15,0xFA16,0xFA17,0xFA18,0xFA19,0xFA1A,0xFA1B,
+	0xFA1C,0xFA1D,0xFA1E,0xFA20,0xFA22,0xFA25,0xFA26,0xFA2A,0xFA2B,0xFA2C,
+	0xFA2D,0xFA2E,0xFA2F,0xFA30,0xFA31,0xFA32,0xFA33,0xFA34,0xFA35,0xFA36,
+	0xFA37,0xFA38,0xFA39,0xFA3A,0xFA3B,0xFA3C,0xFA3D,0xFA3E,0xFA3F,0xFA40,
+	0xFA41,0xFA42,0xFA43,0xFA44,0xFA45,0xFA46,0xFA47,0xFA48,0xFA49,0xFA4A,
+	0xFA4B,0xFA4C,0xFA4D,0xFA4E,0xFA4F,0xFA50,0xFA51,0xFA52,0xFA53,0xFA54,
+	0xFA55,0xFA56,0xFA57,0xFA58,0xFA59,0xFA5A,0xFA5B,0xFA5C,0xFA5D,0xFA5E,
+	0xFA5F,0xFA60,0xFA61,0xFA62,0xFA63,0xFA64,0xFA65,0xFA66,0xFA67,0xFA68,
+	0xFA69,0xFA6A,0xFA6B,0xFA6C,0xFA6D,0xFA70,0xFA71,0xFA72,0xFA73,0xFA74,
+	0xFA75,0xFA76,0xFA77,0xFA78,0xFA79,0xFA7A,0xFA7B,0xFA7C,0xFA7D,0xFA7E,
+	0xFA7F,0xFA80,0xFA81,0xFA82,0xFA83,0xFA84,0xFA85,0xFA86,0xFA87,0xFA88,
+	0xFA89,0xFA8A,0xFA8B,0xFA8C,0xFA8D,0xFA8E,0xFA8F,0xFA90,0xFA91,0xFA92,
+	0xFA93,0xFA94,0xFA95,0xFA96,0xFA97,0xFA98,0xFA99,0xFA9A,0xFA9B,0xFA9C,
+	0xFA9D,0xFA9E,0xFA9F,0xFAA0,0xFAA1,0xFAA2,0xFAA3,0xFAA4,0xFAA5,0xFAA6,
+	0xFAA7,0xFAA8,0xFAA9,0xFAAA,0xFAAB,0xFAAC,0xFAAD,0xFAAE,0xFAAF,0xFAB0,
+	0xFAB1,0xFAB2,0xFAB3,0xFAB4,0xFAB5,0xFAB6,0xFAB7,0xFAB8,0xFAB9,0xFABA,
+	0xFABB,0xFABC,0xFABD,0xFABE,0xFABF,0xFAC0,0xFAC1,0xFAC2,0xFAC3,0xFAC4,
+	0xFAC5,0xFAC6,0xFAC7,0xFAC8,0xFAC9,0xFACA,0xFACB,0xFACC,0xFACD,0xFACE,
+	0xFACF,0xFAD0,0xFAD1,0xFAD2,0xFAD3,0xFAD4,0xFAD5,0xFAD6,0xFAD7,0xFAD8,
+	0xFAD9,0xFB1D,0xFB1F,0xFB2A,0xFB2B,0xFB2C,0xFB2D,0xFB2E,0xFB2F,0xFB30,
+	0xFB31,0xFB32,0xFB33,0xFB34,0xFB35,0xFB36,0xFB38,0xFB39,0xFB3A,0xFB3B,
+	0xFB3C,0xFB3E,0xFB40,0xFB41,0xFB43,0xFB44,0xFB46,0xFB47,0xFB48,0xFB49,
+	0xFB4A,0xFB4B,0xFB4C,0xFB4D,0xFB4E,0x1D15E,0x1D15F,0x1D160,0x1D161,0x1D162,
+	0x1D163,0x1D164,0x1D1BB,0x1D1BC,0x1D1BD,0x1D1BE,0x1D1BF,0x1D1C0,0x2F800,0x2F801,
+	0x2F802,0x2F803,0x2F804,0x2F805,0x2F806,0x2F807,0x2F808,0x2F809,0x2F80A,0x2F80B,
+	0x2F80C,0x2F80D,0x2F80E,0x2F80F,0x2F810,0x2F811,0x2F812,0x2F813,0x2F814,0x2F815,
+	0x2F816,0x2F817,0x2F818,0x2F819,0x2F81A,0x2F81B,0x2F81C,0x2F81D,0x2F81E,0x2F81F,
+	0x2F820,0x2F821,0x2F822,0x2F823,0x2F824,0x2F825,0x2F826,0x2F827,0x2F828,0x2F829,
+	0x2F82A,0x2F82B,0x2F82C,0x2F82D,0x2F82E,0x2F82F,0x2F830,0x2F831,0x2F832,0x2F833,
+	0x2F834,0x2F835,0x2F836,0x2F837,0x2F838,0x2F839,0x2F83A,0x2F83B,0x2F83C,0x2F83D,
+	0x2F83E,0x2F83F,0x2F840,0x2F841,0x2F842,0x2F843,0x2F844,0x2F845,0x2F846,0x2F847,
+	0x2F848,0x2F849,0x2F84A,0x2F84B,0x2F84C,0x2F84D,0x2F84E,0x2F84F,0x2F850,0x2F851,
+	0x2F852,0x2F853,0x2F854,0x2F855,0x2F856,0x2F857,0x2F858,0x2F859,0x2F85A,0x2F85B,
+	0x2F85C,0x2F85D,0x2F85E,0x2F85F,0x2F860,0x2F861,0x2F862,0x2F863,0x2F864,0x2F865,
+	0x2F866,0x2F867,0x2F868,0x2F869,0x2F86A,0x2F86B,0x2F86C,0x2F86D,0x2F86E,0x2F86F,
+	0x2F870,0x2F871,0x2F872,0x2F873,0x2F874,0x2F875,0x2F876,0x2F877,0x2F878,0x2F879,
+	0x2F87A,0x2F87B,0x2F87C,0x2F87D,0x2F87E,0x2F87F,0x2F880,0x2F881,0x2F882,0x2F883,
+	0x2F884,0x2F885,0x2F886,0x2F887,0x2F888,0x2F889,0x2F88A,0x2F88B,0x2F88C,0x2F88D,
+	0x2F88E,0x2F88F,0x2F890,0x2F891,0x2F892,0x2F893,0x2F894,0x2F895,0x2F896,0x2F897,
+	0x2F898,0x2F899,0x2F89A,0x2F89B,0x2F89C,0x2F89D,0x2F89E,0x2F89F,0x2F8A0,0x2F8A1,
+	0x2F8A2,0x2F8A3,0x2F8A4,0x2F8A5,0x2F8A6,0x2F8A7,0x2F8A8,0x2F8A9,0x2F8AA,0x2F8AB,
+	0x2F8AC,0x2F8AD,0x2F8AE,0x2F8AF,0x2F8B0,0x2F8B1,0x2F8B2,0x2F8B3,0x2F8B4,0x2F8B5,
+	0x2F8B6,0x2F8B7,0x2F8B8,0x2F8B9,0x2F8BA,0x2F8BB,0x2F8BC,0x2F8BD,0x2F8BE,0x2F8BF,
+	0x2F8C0,0x2F8C1,0x2F8C2,0x2F8C3,0x2F8C4,0x2F8C5,0x2F8C6,0x2F8C7,0x2F8C8,0x2F8C9,
+	0x2F8CA,0x2F8CB,0x2F8CC,0x2F8CD,0x2F8CE,0x2F8CF,0x2F8D0,0x2F8D1,0x2F8D2,0x2F8D3,
+	0x2F8D4,0x2F8D5,0x2F8D6,0x2F8D7,0x2F8D8,0x2F8D9,0x2F8DA,0x2F8DB,0x2F8DC,0x2F8DD,
+	0x2F8DE,0x2F8DF,0x2F8E0,0x2F8E1,0x2F8E2,0x2F8E3,0x2F8E4,0x2F8E5,0x2F8E6,0x2F8E7,
+	0x2F8E8,0x2F8E9,0x2F8EA,0x2F8EB,0x2F8EC,0x2F8ED,0x2F8EE,0x2F8EF,0x2F8F0,0x2F8F1,
+	0x2F8F2,0x2F8F3,0x2F8F4,0x2F8F5,0x2F8F6,0x2F8F7,0x2F8F8,0x2F8F9,0x2F8FA,0x2F8FB,
+	0x2F8FC,0x2F8FD,0x2F8FE,0x2F8FF,0x2F900,0x2F901,0x2F902,0x2F903,0x2F904,0x2F905,
+	0x2F906,0x2F907,0x2F908,0x2F909,0x2F90A,0x2F90B,0x2F90C,0x2F90D,0x2F90E,0x2F90F,
+	0x2F910,0x2F911,0x2F912,0x2F913,0x2F914,0x2F915,0x2F916,0x2F917,0x2F918,0x2F919,
+	0x2F91A,0x2F91B,0x2F91C,0x2F91D,0x2F91E,0x2F91F,0x2F920,0x2F921,0x2F922,0x2F923,
+	0x2F924,0x2F925,0x2F926,0x2F927,0x2F928,0x2F929,0x2F92A,0x2F92B,0x2F92C,0x2F92D,
+	0x2F92E,0x2F92F,0x2F930,0x2F931,0x2F932,0x2F933,0x2F934,0x2F935,0x2F936,0x2F937,
+	0x2F938,0x2F939,0x2F93A,0x2F93B,0x2F93C,0x2F93D,0x2F93E,0x2F93F,0x2F940,0x2F941,
+	0x2F942,0x2F943,0x2F944,0x2F945,0x2F946,0x2F947,0x2F948,0x2F949,0x2F94A,0x2F94B,
+	0x2F94C,0x2F94D,0x2F94E,0x2F94F,0x2F950,0x2F951,0x2F952,0x2F953,0x2F954,0x2F955,
+	0x2F956,0x2F957,0x2F958,0x2F959,0x2F95A,0x2F95B,0x2F95C,0x2F95D,0x2F95E,0x2F95F,
+	0x2F960,0x2F961,0x2F962,0x2F963,0x2F964,0x2F965,0x2F966,0x2F967,0x2F968,0x2F969,
+	0x2F96A,0x2F96B,0x2F96C,0x2F96D,0x2F96E,0x2F96F,0x2F970,0x2F971,0x2F972,0x2F973,
+	0x2F974,0x2F975,0x2F976,0x2F977,0x2F978,0x2F979,0x2F97A,0x2F97B,0x2F97C,0x2F97D,
+	0x2F97E,0x2F97F,0x2F980,0x2F981,0x2F982,0x2F983,0x2F984,0x2F985,0x2F986,0x2F987,
+	0x2F988,0x2F989,0x2F98A,0x2F98B,0x2F98C,0x2F98D,0x2F98E,0x2F98F,0x2F990,0x2F991,
+	0x2F992,0x2F993,0x2F994,0x2F995,0x2F996,0x2F997,0x2F998,0x2F999,0x2F99A,0x2F99B,
+	0x2F99C,0x2F99D,0x2F99E,0x2F99F,0x2F9A0,0x2F9A1,0x2F9A2,0x2F9A3,0x2F9A4,0x2F9A5,
+	0x2F9A6,0x2F9A7,0x2F9A8,0x2F9A9,0x2F9AA,0x2F9AB,0x2F9AC,0x2F9AD,0x2F9AE,0x2F9AF,
+	0x2F9B0,0x2F9B1,0x2F9B2,0x2F9B3,0x2F9B4,0x2F9B5,0x2F9B6,0x2F9B7,0x2F9B8,0x2F9B9,
+	0x2F9BA,0x2F9BB,0x2F9BC,0x2F9BD,0x2F9BE,0x2F9BF,0x2F9C0,0x2F9C1,0x2F9C2,0x2F9C3,
+	0x2F9C4,0x2F9C5,0x2F9C6,0x2F9C7,0x2F9C8,0x2F9C9,0x2F9CA,0x2F9CB,0x2F9CC,0x2F9CD,
+	0x2F9CE,0x2F9CF,0x2F9D0,0x2F9D1,0x2F9D2,0x2F9D3,0x2F9D4,0x2F9D5,0x2F9D6,0x2F9D7,
+	0x2F9D8,0x2F9D9,0x2F9DA,0x2F9DB,0x2F9DC,0x2F9DD,0x2F9DE,0x2F9DF,0x2F9E0,0x2F9E1,
+	0x2F9E2,0x2F9E3,0x2F9E4,0x2F9E5,0x2F9E6,0x2F9E7,0x2F9E8,0x2F9E9,0x2F9EA,0x2F9EB,
+	0x2F9EC,0x2F9ED,0x2F9EE,0x2F9EF,0x2F9F0,0x2F9F1,0x2F9F2,0x2F9F3,0x2F9F4,0x2F9F5,
+	0x2F9F6,0x2F9F7,0x2F9F8,0x2F9F9,0x2F9FA,0x2F9FB,0x2F9FC,0x2F9FD,0x2F9FE,0x2F9FF,
+	0x2FA00,0x2FA01,0x2FA02,0x2FA03,0x2FA04,0x2FA05,0x2FA06,0x2FA07,0x2FA08,0x2FA09,
+	0x2FA0A,0x2FA0B,0x2FA0C,0x2FA0D,0x2FA0E,0x2FA0F,0x2FA10,0x2FA11,0x2FA12,0x2FA13,
+	0x2FA14,0x2FA15,0x2FA16,0x2FA17,0x2FA18,0x2FA19,0x2FA1A,0x2FA1B,0x2FA1C,0x2FA1D,
+	};
+
+const unsigned NFC_Maybe_Size = 110;
+const std::set<unsigned> NFC_Maybe = {
+	0x0300,0x0301,0x0302,0x0303,0x0304,0x0306,0x0307,0x0308,0x0309,0x030A,
+	0x030B,0x030C,0x030F,0x0311,0x0313,0x0314,0x031B,0x0323,0x0324,0x0325,
+	0x0326,0x0327,0x0328,0x032D,0x032E,0x0330,0x0331,0x0338,0x0342,0x0345,
+	0x0653,0x0654,0x0655,0x093C,0x09BE,0x09D7,0x0B3E,0x0B56,0x0B57,0x0BBE,
+	0x0BD7,0x0C56,0x0CC2,0x0CD5,0x0CD6,0x0D3E,0x0D57,0x0DCA,0x0DCF,0x0DDF,
+	0x102E,0x1161,0x1162,0x1163,0x1164,0x1165,0x1166,0x1167,0x1168,0x1169,
+	0x116A,0x116B,0x116C,0x116D,0x116E,0x116F,0x1170,0x1171,0x1172,0x1173,
+	0x1174,0x1175,0x11A8,0x11A9,0x11AA,0x11AB,0x11AC,0x11AD,0x11AE,0x11AF,
+	0x11B0,0x11B1,0x11B2,0x11B3,0x11B4,0x11B5,0x11B6,0x11B7,0x11B8,0x11B9,
+	0x11BA,0x11BB,0x11BC,0x11BD,0x11BE,0x11BF,0x11C0,0x11C1,0x11C2,0x1B35,
+	0x3099,0x309A,0x110BA,0x11127,0x1133E,0x11357,0x114B0,0x114BA,0x114BD,0x115AF,
+	};
+
--- a/server/nfc_sets.hh
+++ b/server/nfc_sets.hh
@ -0,0 +1,11 @@
+#ifndef NFC_SETS_HH
+#define NFC_SETS_HH
+
+#include <set>
+
+// These sets are filled by code generated automatically
+// from Unicode's DerivedNormalizationProps.txt
+extern const std::set<unsigned> NFC_No;
+extern const std::set<unsigned> NFC_Maybe;
+
+#endif // NFC_SETS_HH
--- a/server/scripts/gen_sets.sh
+++ b/server/scripts/gen_sets.sh
@ -0,0 +1,79 @@
+#!/bin/bash
+
+################################# ### ## # #  #    #
+#
+# Reads the file /usr/share/unicode/DerivedNormalizationProps.txt  (Debian package: unicode-data)
+# and generates C++ code for the std::set<> containing the normalization properties
+#
+################################# ### ## # #  #    #
+
+cat /usr/share/unicode/DerivedNormalizationProps.txt | sed -e 's/#.*//g' | grep NFC_QC | sed -e 's/; NFC_QC;//g' |
+
+(
+
+declare -a CHAR_NO
+declare -a CHAR_MAYBE
+
+echo -e '// This file is generated by scripts/gen_sets.sh\n// DO NOT EDIT IT!\n\n#include "nfc_sets.hh"\n\n'
+
+U=dummyvalue
+
+while [ -n "$U" ] ; do
+	read U V
+	if [ -n "$U" ] ; then 
+		
+		START=0x${U/..*/}
+		END=0x${U/*../}
+		
+		for i in `seq $START $END` ; do
+			case $V in
+				"N")
+					CHAR_NO+=($i)
+				;;
+				"M")
+					CHAR_MAYBE+=($i)
+				;;
+				*)
+					echo 'Unknown: V='$V
+					exit 2
+			esac
+		
+		done
+	fi
+done
+
+
+echo "const unsigned NFC_No_Size = ${#CHAR_NO[*]};"
+echo -en 'const std::set<unsigned> NFC_No = {'
+
+index=10
+for u in "${CHAR_NO[@]}"; do
+	if [ $index -ge 10 ] ; then
+		echo -en '\n\t'
+		index=0
+	fi
+	printf '0x%04X,' $u
+	index=$(( index + 1 ))
+done
+echo -en '\n\t};\n\n'
+
+
+echo "const unsigned NFC_Maybe_Size = ${#CHAR_MAYBE[*]};"
+echo -en 'const std::set<unsigned> NFC_Maybe = {'
+
+index=10
+for u in "${CHAR_MAYBE[@]}"; do
+	if [ $index -ge 10 ] ; then
+		echo -en '\n\t'
+		index=0
+	fi
+	printf '0x%04X,' $u
+	index=$(( index + 1 ))
+
+done
+echo -en '\n\t};\n\n'
+
+)
+
+
+# end of file