Commit | Line | Data |
---|---|---|
61507e3c RK |
1 | #! /usr/bin/perl -w |
2 | # | |
e5a5a138 RK |
3 | # This file is part of DisOrder. |
4 | # Copyright (C) 2007 Richard Kettlewell | |
5 | # | |
6 | # This program is free software; you can redistribute it and/or modify | |
7 | # it under the terms of the GNU General Public License as published by | |
8 | # the Free Software Foundation; either version 2 of the License, or | |
9 | # (at your option) any later version. | |
10 | # | |
11 | # This program is distributed in the hope that it will be useful, but | |
12 | # WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | # General Public License for more details. | |
15 | # | |
16 | # You should have received a copy of the GNU General Public License | |
17 | # along with this program; if not, write to the Free Software | |
18 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | |
19 | # USA | |
20 | # | |
21 | # | |
22 | # Generate Unicode support tables | |
23 | # | |
24 | # This script will download data from unicode.org if the required files | |
25 | # aren't in the current directory. | |
26 | # | |
27 | # After modifying this script you should run: | |
28 | # make -C lib rebuild-unicode check | |
29 | # | |
30 | # Things not supported yet: | |
31 | # - SpecialCasing.txt data for case mapping | |
32 | # - Title case offsets | |
33 | # - Some kind of hinting for composition | |
34 | # - Word boundary support | |
35 | # - ... | |
36 | # | |
37 | # NB the generated files DO NOT offer a stable ABI and so are not immediately | |
38 | # suitable for use in a general-purpose library. Things that would need to | |
39 | # be done: | |
40 | # - Hide unidata.h from applications; it will never be ABI- or even API-stable. | |
41 | # - Stablized General_Category values | |
42 | # - Extend the unicode.h API to general utility rather than just what | |
43 | # DisOrder needs. | |
44 | # - ... | |
45 | # | |
61507e3c RK |
46 | use strict; |
47 | ||
48 | sub out { | |
49 | print @_ or die "$!\n"; | |
50 | } | |
51 | ||
52 | sub key { | |
53 | my $d = shift; | |
54 | local $_; | |
55 | ||
56 | return join("-", map($d->{$_}, sort keys %$d)); | |
57 | } | |
58 | ||
e5a5a138 RK |
59 | # Size of a subtable |
60 | # | |
61 | # This can be varied to trade off the number of subtables against their size. | |
62 | our $modulus = 128; | |
63 | ||
61507e3c RK |
64 | my %cats = (); # known general categories |
65 | my %data = (); # mapping of codepoints to information | |
61507e3c | 66 | my $max = 0; # maximum codepoint |
e5a5a138 RK |
67 | my $maxccc = 0; # maximum combining class |
68 | my $maxud = 0; | |
69 | my $minud = 0; # max/min upper case offset | |
70 | my $maxld = 0; | |
71 | my $minld = 0; # max/min lower case offset | |
72 | ||
73 | # Make sure we have our desired input files. We explicitly specify a | |
74 | # Unicode standard version to make sure that a given version of DisOrder | |
75 | # supports a given version of Unicode. | |
76 | sub need_input { | |
77 | my $f = shift; | |
78 | if(!-e $f) { | |
79 | system("wget http://www.unicode.org/Public/5.0.0/ucd/$f"); | |
80 | chmod(0444, $f); | |
81 | } | |
82 | } | |
61507e3c | 83 | |
e5a5a138 RK |
84 | need_input("UnicodeData.txt"); |
85 | need_input("CaseFolding.txt"); | |
86 | ||
87 | # Read the main data file | |
88 | open(STDIN, "<UnicodeData.txt") or die "UnicodeData.txt: $!\n"; | |
61507e3c RK |
89 | while(<>) { |
90 | my @f = split(/;/, $_); | |
91 | my $c = hex($f[0]); # codepoint | |
92 | next if $c >= 0xE0000; # ignore various high-numbered stuff | |
e5a5a138 | 93 | # TODO justify this exclusion! |
61507e3c | 94 | my $name = $f[1]; |
e5a5a138 RK |
95 | my $gc = $f[2]; # General_Category |
96 | my $ccc = $f[3]; # Canonical_Combining_Class | |
97 | my $dm = $f[5]; # Decomposition_Type + Decomposition_Mapping | |
98 | my $sum = hex($f[12]) || $c; # Simple_Uppercase_Mapping | |
99 | my $slm = hex($f[13]) || $c; # Simple_Lowercase_Mapping | |
61507e3c RK |
100 | # recalculate the upper/lower case mappings as offsets |
101 | my $ud = $sum - $c; | |
102 | my $ld = $slm - $c; | |
e5a5a138 RK |
103 | # update bounds on various values |
104 | $maxccc = $ccc if $ccc > $maxccc; # assumed never to be -ve | |
105 | $minud = $ud if $ud < $minud; | |
106 | $maxud = $ud if $ud > $maxud; | |
107 | $minld = $ld if $ld < $minld; | |
108 | $maxld = $ld if $ld > $maxld; | |
61507e3c RK |
109 | $data{$c} = { |
110 | "gc" => $gc, | |
111 | "ccc" => $ccc, | |
112 | "ud" => $ud, | |
e5a5a138 | 113 | "ld" => $ld, |
61507e3c | 114 | }; |
e5a5a138 RK |
115 | if($dm ne '') { |
116 | if($dm !~ /</) { | |
117 | # This is a canonical decomposition | |
118 | $data{$c}->{canon} = $dm; | |
119 | $data{$c}->{compat} = $dm; | |
120 | } else { | |
121 | # This is only a compatibility decomposition | |
122 | $dm =~ s/^<.*>\s*//; | |
123 | $data{$c}->{compat} = $dm; | |
124 | } | |
125 | } | |
61507e3c RK |
126 | $cats{$gc} = 1; |
127 | $max = $c if $c > $max; | |
128 | } | |
129 | ||
e5a5a138 RK |
130 | # Round up the maximum value to a whole number of subtables |
131 | $max += ($modulus - 1) - ($max % $modulus); | |
61507e3c RK |
132 | |
133 | # Make sure there are no gaps | |
134 | for(my $c = 0; $c <= $max; ++$c) { | |
135 | if(!exists $data{$c}) { | |
136 | $data{$c} = { | |
137 | "gc" => "Cn", # not assigned | |
138 | "ccc" => 0, | |
139 | "ud" => 0, | |
140 | "ld" => 0 | |
141 | }; | |
142 | } | |
143 | } | |
144 | $cats{'Cn'} = 1; | |
145 | ||
e5a5a138 RK |
146 | # Read the casefolding data too |
147 | open(STDIN, "<CaseFolding.txt") or die "CaseFolding.txt: $!\n"; | |
148 | while(<>) { | |
149 | chomp; | |
150 | next if /^\#/ or $_ eq ''; | |
151 | my @f = split(/\s*;\s*/, $_); | |
152 | # Full case folding means use status C and F. | |
153 | # We discard status T, Turkish users may wish to change this. | |
154 | if($f[1] eq 'C' or $f[1] eq 'F') { | |
155 | my $c = hex($f[0]); | |
156 | $data{$c}->{casefold} = $f[2]; | |
157 | # We are particularly interest in combining characters that | |
158 | # case-fold to non-combining characters, or characters that | |
159 | # case-fold to sequences with combining characters in non-initial | |
160 | # positions, as these required decomposiiton before case-folding | |
161 | my @d = map(hex($_), split(/\s+/, $data{$c}->{casefold})); | |
162 | if($data{$c}->{ccc} != 0) { | |
163 | # This is a combining character | |
164 | if($data{$d[0]}->{ccc} == 0) { | |
165 | # The first character of its case-folded form is NOT | |
166 | # a combining character. The field name is the example | |
167 | # explicitly mentioned in the spec. | |
168 | $data{$c}->{ypogegrammeni} = 1; | |
169 | } | |
170 | } else { | |
171 | # This is a non-combining character; inspect the non-initial | |
172 | # code points of the case-folded sequence | |
173 | shift(@d); | |
174 | if(grep($data{$_}->{ccc} != 0, @d)) { | |
175 | # Some non-initial code point in the case-folded for is NOT a | |
176 | # a combining character. | |
177 | $data{$c}->{ypogegrammeni} = 1; | |
178 | } | |
179 | } | |
180 | } | |
181 | } | |
182 | ||
183 | # Generate the header file | |
61507e3c RK |
184 | open(STDOUT, ">unidata.h") or die "unidata.h: $!\n"; |
185 | ||
e5a5a138 RK |
186 | out("/* Automatically generated file, see scripts/make-unidata */\n", |
187 | "#ifndef UNIDATA_H\n", | |
61507e3c RK |
188 | "#define UNIDATA_H\n"); |
189 | ||
e5a5a138 | 190 | # TODO choose stable values for General_Category |
61507e3c RK |
191 | out("enum unicode_gc_cat {\n", |
192 | join(",\n", | |
193 | map(" unicode_gc_$_", sort keys %cats)), "\n};\n"); | |
e5a5a138 RK |
194 | |
195 | out("enum unicode_flags {\n", | |
196 | " unicode_normalize_before_casefold = 1\n", | |
197 | "};\n", | |
198 | "\n"); | |
199 | ||
200 | # Choose the narrowest type that will fit the required values | |
201 | sub choosetype { | |
202 | my ($min, $max) = @_; | |
203 | if($min >= 0) { | |
204 | return "char" if $max <= 127; | |
205 | return "unsigned char" if $max <= 255; | |
206 | return "int16_t" if $max < 32767; | |
207 | return "uint16_t" if $max < 65535; | |
208 | return "int32_t"; | |
209 | } else { | |
210 | return "char" if $min >= -127 && $max <= 127; | |
211 | return "int16_t" if $min >= -32767 && $max <= 32767; | |
212 | return "int32_t"; | |
213 | } | |
214 | } | |
215 | ||
61507e3c | 216 | out("struct unidata {\n", |
e5a5a138 RK |
217 | " const uint32_t *compat;\n", |
218 | " const uint32_t *canon;\n", | |
219 | " const uint32_t *casefold;\n", | |
220 | " ".choosetype($minud, $maxud)." upper_offset;\n", | |
221 | " ".choosetype($minld, $maxld)." lower_offset;\n", | |
222 | " ".choosetype(0, $maxccc)." ccc;\n", | |
223 | " char gc;\n", | |
224 | " uint8_t flags;\n", | |
61507e3c | 225 | "};\n"); |
e5a5a138 RK |
226 | # compat, canon and casefold do have have non-BMP characters, so we |
227 | # can't use a simple 16-bit table. We could use UTF-8 or UTF-16 | |
228 | # though, saving a bit of space (probably not that much...) at the | |
229 | # cost of marginally reduced performance and additional complexity | |
61507e3c RK |
230 | |
231 | out("extern const struct unidata *const unidata[];\n"); | |
232 | ||
233 | out("#define UNICODE_NCHARS ", ($max + 1), "\n"); | |
e5a5a138 | 234 | out("#define UNICODE_MODULUS $modulus\n"); |
61507e3c RK |
235 | |
236 | out("#endif\n"); | |
237 | ||
238 | close STDOUT or die "unidata.h: $!\n"; | |
239 | ||
240 | open(STDOUT, ">unidata.c") or die "unidata.c: $!\n"; | |
241 | ||
e5a5a138 RK |
242 | out("/* Automatically generated file, see scripts/make-unidata */\n", |
243 | "#include <config.h>\n", | |
244 | "#include \"types.h\"\n", | |
245 | "#include \"unidata.h\"\n"); | |
246 | ||
247 | # Short aliases for general category codes | |
248 | ||
249 | out(map(sprintf("#define %s unicode_gc_%s\n", $_, $_), sort keys %cats)); | |
250 | ||
251 | # Generate the decomposition mapping tables. We look out for duplicates | |
252 | # in order to save space and report this as decompsaved at the end. In | |
253 | # Unicode 5.0.0 this saves 1795 entries, which is at least 14Kbytes. | |
254 | my $decompnum = 0; | |
255 | my %decompnums = (); | |
256 | my $decompsaved = 0; | |
257 | out("static const uint32_t "); | |
258 | for(my $c = 0; $c <= $max; ++$c) { | |
259 | # If canon is set then compat will be too and will be identical. | |
260 | # If compat is set the canon might be clear. So we use the | |
261 | # compat version and fix up the symbols after. | |
262 | if(exists $data{$c}->{compat}) { | |
263 | my $s = join(",", | |
264 | (map(hex($_), split(/\s+/, $data{$c}->{compat})), 0)); | |
265 | if(!exists $decompnums{$s}) { | |
266 | out(",\n") if $decompnum != 0; | |
267 | out("cd$decompnum\[]={$s}"); | |
268 | $decompnums{$s} = $decompnum++; | |
269 | } else { | |
270 | ++$decompsaved; | |
271 | } | |
272 | $data{$c}->{compatsym} = "cd$decompnums{$s}"; | |
273 | if(exists $data{$c}->{canon}) { | |
274 | $data{$c}->{canonsym} = "cd$decompnums{$s}"; | |
275 | } | |
276 | } | |
277 | } | |
278 | out(";\n"); | |
279 | ||
280 | # ...and the case folding table. Again we compress equal entries to save | |
281 | # space. In Unicode 5.0.0 this saves 51 entries or at least 408 bytes. | |
282 | # This doesns't seem as worthwhile as the decomposition mapping saving above. | |
283 | my $cfnum = 0; | |
284 | my %cfnums = (); | |
285 | my $cfsaved = 0; | |
286 | out("static const uint32_t "); | |
287 | for(my $c = 0; $c <= $max; ++$c) { | |
288 | if(exists $data{$c}->{casefold}) { | |
289 | my $s = join(",", | |
290 | (map(hex($_), split(/\s+/, $data{$c}->{casefold})), 0)); | |
291 | if(!exists $cfnums{$s}) { | |
292 | out(",\n") if $cfnum != 0; | |
293 | out("cf$cfnum\[]={$s}"); | |
294 | $cfnums{$s} = $cfnum++; | |
295 | } else { | |
296 | ++$cfsaved; | |
297 | } | |
298 | $data{$c}->{cfsym} = "cf$cfnums{$s}"; | |
299 | } | |
300 | } | |
301 | out(";\n"); | |
302 | ||
303 | # Visit all the $modulus-character blocks in turn and generate the | |
304 | # required subtables. As above we spot duplicates to save space. In | |
305 | # Unicode 5.0.0 with $modulus=128 and current table data this saves | |
306 | # 1372 subtables or at least three and a half megabytes on 32-bit | |
307 | # platforms. | |
61507e3c | 308 | |
61507e3c RK |
309 | my %subtable = (); # base->subtable number |
310 | my %subtableno = (); # subtable number -> content | |
311 | my $subtablecounter = 0; # counter for subtable numbers | |
e5a5a138 RK |
312 | my $subtablessaved = 0; # number of tables saved |
313 | for(my $base = 0; $base <= $max; $base += $modulus) { | |
61507e3c | 314 | my @t; |
e5a5a138 | 315 | for(my $c = $base; $c < $base + $modulus; ++$c) { |
61507e3c | 316 | my $d = $data{$c}; |
e5a5a138 RK |
317 | my $canonsym = ($data{$c}->{canonsym} or "0"); |
318 | my $compatsym = ($data{$c}->{compatsym} or "0"); | |
319 | my $cfsym = ($data{$c}->{cfsym} or "0"); | |
320 | my $flags = ($data{$c}->{ypogegrammeni} | |
321 | ? "unicode_normalize_before_casefold" | |
322 | : 0); | |
323 | push(@t, "{". | |
324 | join(",", | |
325 | $compatsym, | |
326 | $canonsym, | |
327 | $cfsym, | |
328 | "$d->{ud}", | |
329 | "$d->{ld}", | |
330 | "$d->{ccc}", | |
331 | "$d->{gc}", | |
332 | $flags, | |
333 | )."}"); | |
61507e3c RK |
334 | } |
335 | my $t = join(",\n", @t); | |
336 | if(!exists $subtable{$t}) { | |
e5a5a138 | 337 | out("static const struct unidata st$subtablecounter\[] = {\n", |
61507e3c RK |
338 | "$t\n", |
339 | "};\n"); | |
340 | $subtable{$t} = $subtablecounter++; | |
e5a5a138 RK |
341 | } else { |
342 | ++$subtablessaved; | |
61507e3c RK |
343 | } |
344 | $subtableno{$base} = $subtable{$t}; | |
345 | } | |
346 | ||
e5a5a138 RK |
347 | out("const struct unidata*const unidata[]={\n"); |
348 | for(my $base = 0; $base <= $max; $base += $modulus) { | |
349 | out("st$subtableno{$base},\n"); | |
61507e3c RK |
350 | } |
351 | out("};\n"); | |
352 | ||
353 | close STDOUT or die "unidata.c: $!\n"; | |
354 | ||
e5a5a138 RK |
355 | print STDERR "max=$max, subtables=$subtablecounter, subtablessaved=$subtablessaved\n"; |
356 | print STDERR "decompsaved=$decompsaved cfsaved=$cfsaved\n"; |