Commit | Line | Data |
---|---|---|
d73e5302 | 1 | #!/usr/bin/perl -w |
99870f4d KW |
2 | |
3 | # !!!!!!!!!!!!!! IF YOU MODIFY THIS FILE !!!!!!!!!!!!!!!!!!!!!!!!! | |
4 | # Any files created or read by this program should be listed in 'mktables.lst' | |
5 | # Use -makelist to regenerate it. | |
6 | ||
23e33b60 KW |
7 | # Needs 'no overloading' to run faster on miniperl. Code commented out at the |
8 | # subroutine objaddr can be used instead to work as far back (untested) as | |
f998e60c KW |
9 | # 5.8: needs pack "U". But almost all occurrences of objaddr have been |
10 | # removed in favor of using 'no overloading'. You also would have to go | |
11 | # through and replace occurrences like: | |
ffe43484 | 12 | # my $addr = do { no overloading; pack 'J', $self; } |
f998e60c KW |
13 | # with |
14 | # my $addr = main::objaddr $self; | |
6c68572b | 15 | # (or reverse commit 9b01bafde4b022706c3d6f947a0963f821b2e50b |
051df77b NC |
16 | # that instituted the change to main::objaddr, and subsequent commits that |
17 | # changed 0+$self to pack 'J', $self.) | |
6c68572b | 18 | |
cdcef19a | 19 | my $start_time; |
98dc9551 | 20 | BEGIN { # Get the time the script started running; do it at compilation to |
cdcef19a KW |
21 | # get it as close as possible |
22 | $start_time= time; | |
23 | } | |
24 | ||
23e33b60 | 25 | require 5.010_001; |
d73e5302 | 26 | use strict; |
99870f4d | 27 | use warnings; |
cf25bb62 | 28 | use Carp; |
bd9ebcfd | 29 | use Config; |
99870f4d KW |
30 | use File::Find; |
31 | use File::Path; | |
d07a55ed | 32 | use File::Spec; |
99870f4d KW |
33 | use Text::Tabs; |
34 | ||
35 | sub DEBUG () { 0 } # Set to 0 for production; 1 for development | |
bd9ebcfd | 36 | my $debugging_build = $Config{"ccflags"} =~ /-DDEBUGGING/; |
99870f4d KW |
37 | |
38 | ########################################################################## | |
39 | # | |
40 | # mktables -- create the runtime Perl Unicode files (lib/unicore/.../*.pl), | |
41 | # from the Unicode database files (lib/unicore/.../*.txt), It also generates | |
42 | # a pod file and a .t file | |
43 | # | |
44 | # The structure of this file is: | |
45 | # First these introductory comments; then | |
46 | # code needed for everywhere, such as debugging stuff; then | |
47 | # code to handle input parameters; then | |
48 | # data structures likely to be of external interest (some of which depend on | |
49 | # the input parameters, so follows them; then | |
50 | # more data structures and subroutine and package (class) definitions; then | |
51 | # the small actual loop to process the input files and finish up; then | |
52 | # a __DATA__ section, for the .t tests | |
53 | # | |
5f7264c7 | 54 | # This program works on all releases of Unicode through at least 6.0. The |
99870f4d KW |
55 | # outputs have been scrutinized most intently for release 5.1. The others |
56 | # have been checked for somewhat more than just sanity. It can handle all | |
57 | # existing Unicode character properties in those releases. | |
58 | # | |
99870f4d KW |
59 | # This program is mostly about Unicode character (or code point) properties. |
60 | # A property describes some attribute or quality of a code point, like if it | |
61 | # is lowercase or not, its name, what version of Unicode it was first defined | |
62 | # in, or what its uppercase equivalent is. Unicode deals with these disparate | |
63 | # possibilities by making all properties into mappings from each code point | |
64 | # into some corresponding value. In the case of it being lowercase or not, | |
65 | # the mapping is either to 'Y' or 'N' (or various synonyms thereof). Each | |
66 | # property maps each Unicode code point to a single value, called a "property | |
67 | # value". (Hence each Unicode property is a true mathematical function with | |
68 | # exactly one value per code point.) | |
69 | # | |
70 | # When using a property in a regular expression, what is desired isn't the | |
71 | # mapping of the code point to its property's value, but the reverse (or the | |
72 | # mathematical "inverse relation"): starting with the property value, "Does a | |
73 | # code point map to it?" These are written in a "compound" form: | |
74 | # \p{property=value}, e.g., \p{category=punctuation}. This program generates | |
75 | # files containing the lists of code points that map to each such regular | |
76 | # expression property value, one file per list | |
77 | # | |
78 | # There is also a single form shortcut that Perl adds for many of the commonly | |
79 | # used properties. This happens for all binary properties, plus script, | |
80 | # general_category, and block properties. | |
81 | # | |
82 | # Thus the outputs of this program are files. There are map files, mostly in | |
83 | # the 'To' directory; and there are list files for use in regular expression | |
84 | # matching, all in subdirectories of the 'lib' directory, with each | |
85 | # subdirectory being named for the property that the lists in it are for. | |
86 | # Bookkeeping, test, and documentation files are also generated. | |
87 | ||
88 | my $matches_directory = 'lib'; # Where match (\p{}) files go. | |
89 | my $map_directory = 'To'; # Where map files go. | |
90 | ||
91 | # DATA STRUCTURES | |
92 | # | |
93 | # The major data structures of this program are Property, of course, but also | |
94 | # Table. There are two kinds of tables, very similar to each other. | |
95 | # "Match_Table" is the data structure giving the list of code points that have | |
96 | # a particular property value, mentioned above. There is also a "Map_Table" | |
97 | # data structure which gives the property's mapping from code point to value. | |
98 | # There are two structures because the match tables need to be combined in | |
99 | # various ways, such as constructing unions, intersections, complements, etc., | |
100 | # and the map ones don't. And there would be problems, perhaps subtle, if | |
101 | # a map table were inadvertently operated on in some of those ways. | |
102 | # The use of separate classes with operations defined on one but not the other | |
103 | # prevents accidentally confusing the two. | |
104 | # | |
105 | # At the heart of each table's data structure is a "Range_List", which is just | |
106 | # an ordered list of "Ranges", plus ancillary information, and methods to | |
107 | # operate on them. A Range is a compact way to store property information. | |
108 | # Each range has a starting code point, an ending code point, and a value that | |
109 | # is meant to apply to all the code points between the two end points, | |
110 | # inclusive. For a map table, this value is the property value for those | |
111 | # code points. Two such ranges could be written like this: | |
112 | # 0x41 .. 0x5A, 'Upper', | |
113 | # 0x61 .. 0x7A, 'Lower' | |
114 | # | |
115 | # Each range also has a type used as a convenience to classify the values. | |
116 | # Most ranges in this program will be Type 0, or normal, but there are some | |
117 | # ranges that have a non-zero type. These are used only in map tables, and | |
118 | # are for mappings that don't fit into the normal scheme of things. Mappings | |
119 | # that require a hash entry to communicate with utf8.c are one example; | |
120 | # another example is mappings for charnames.pm to use which indicate a name | |
121 | # that is algorithmically determinable from its code point (and vice-versa). | |
122 | # These are used to significantly compact these tables, instead of listing | |
123 | # each one of the tens of thousands individually. | |
124 | # | |
125 | # In a match table, the value of a range is irrelevant (and hence the type as | |
126 | # well, which will always be 0), and arbitrarily set to the null string. | |
127 | # Using the example above, there would be two match tables for those two | |
128 | # entries, one named Upper would contain the 0x41..0x5A range, and the other | |
129 | # named Lower would contain 0x61..0x7A. | |
130 | # | |
131 | # Actually, there are two types of range lists, "Range_Map" is the one | |
132 | # associated with map tables, and "Range_List" with match tables. | |
133 | # Again, this is so that methods can be defined on one and not the other so as | |
134 | # to prevent operating on them in incorrect ways. | |
135 | # | |
136 | # Eventually, most tables are written out to files to be read by utf8_heavy.pl | |
137 | # in the perl core. All tables could in theory be written, but some are | |
138 | # suppressed because there is no current practical use for them. It is easy | |
139 | # to change which get written by changing various lists that are near the top | |
140 | # of the actual code in this file. The table data structures contain enough | |
141 | # ancillary information to allow them to be treated as separate entities for | |
142 | # writing, such as the path to each one's file. There is a heading in each | |
143 | # map table that gives the format of its entries, and what the map is for all | |
144 | # the code points missing from it. (This allows tables to be more compact.) | |
678f13d5 | 145 | # |
99870f4d KW |
146 | # The Property data structure contains one or more tables. All properties |
147 | # contain a map table (except the $perl property which is a | |
148 | # pseudo-property containing only match tables), and any properties that | |
149 | # are usable in regular expression matches also contain various matching | |
150 | # tables, one for each value the property can have. A binary property can | |
151 | # have two values, True and False (or Y and N, which are preferred by Unicode | |
152 | # terminology). Thus each of these properties will have a map table that | |
153 | # takes every code point and maps it to Y or N (but having ranges cuts the | |
154 | # number of entries in that table way down), and two match tables, one | |
155 | # which has a list of all the code points that map to Y, and one for all the | |
156 | # code points that map to N. (For each of these, a third table is also | |
157 | # generated for the pseudo Perl property. It contains the identical code | |
158 | # points as the Y table, but can be written, not in the compound form, but in | |
159 | # a "single" form like \p{IsUppercase}.) Many properties are binary, but some | |
160 | # properties have several possible values, some have many, and properties like | |
161 | # Name have a different value for every named code point. Those will not, | |
162 | # unless the controlling lists are changed, have their match tables written | |
163 | # out. But all the ones which can be used in regular expression \p{} and \P{} | |
c12f2655 KW |
164 | # constructs will. Prior to 5.14, generally a property would have either its |
165 | # map table or its match tables written but not both. Again, what gets | |
166 | # written is controlled by lists which can easily be changed. Starting in | |
167 | # 5.14, advantage was taken of this, and all the map tables needed to | |
168 | # reconstruct the Unicode db are now written out, while suppressing the | |
169 | # Unicode .txt files that contain the data. Our tables are much more compact | |
170 | # than the .txt files, so a significant space savings was achieved. | |
171 | ||
172 | # Properties have a 'Type', like binary, or string, or enum depending on how | |
173 | # many match tables there are and the content of the maps. This 'Type' is | |
174 | # different than a range 'Type', so don't get confused by the two concepts | |
175 | # having the same name. | |
678f13d5 | 176 | # |
99870f4d KW |
177 | # For information about the Unicode properties, see Unicode's UAX44 document: |
178 | ||
179 | my $unicode_reference_url = 'http://www.unicode.org/reports/tr44/'; | |
180 | ||
181 | # As stated earlier, this program will work on any release of Unicode so far. | |
182 | # Most obvious problems in earlier data have NOT been corrected except when | |
183 | # necessary to make Perl or this program work reasonably. For example, no | |
c12f2655 KW |
184 | # folding information was given in early releases, so this program substitutes |
185 | # lower case instead, just so that a regular expression with the /i option | |
186 | # will do something that actually gives the right results in many cases. | |
187 | # There are also a couple other corrections for version 1.1.5, commented at | |
188 | # the point they are made. As an example of corrections that weren't made | |
189 | # (but could be) is this statement from DerivedAge.txt: "The supplementary | |
190 | # private use code points and the non-character code points were assigned in | |
191 | # version 2.0, but not specifically listed in the UCD until versions 3.0 and | |
192 | # 3.1 respectively." (To be precise it was 3.0.1 not 3.0.0) More information | |
193 | # on Unicode version glitches is further down in these introductory comments. | |
99870f4d | 194 | # |
5f7264c7 KW |
195 | # This program works on all non-provisional properties as of 6.0, though the |
196 | # files for some are suppressed from apparent lack of demand for them. You | |
197 | # can change which are output by changing lists in this program. | |
678f13d5 | 198 | # |
dc85bd38 | 199 | # The old version of mktables emphasized the term "Fuzzy" to mean Unicode's |
99870f4d KW |
200 | # loose matchings rules (from Unicode TR18): |
201 | # | |
202 | # The recommended names for UCD properties and property values are in | |
203 | # PropertyAliases.txt [Prop] and PropertyValueAliases.txt | |
204 | # [PropValue]. There are both abbreviated names and longer, more | |
205 | # descriptive names. It is strongly recommended that both names be | |
206 | # recognized, and that loose matching of property names be used, | |
207 | # whereby the case distinctions, whitespace, hyphens, and underbar | |
208 | # are ignored. | |
209 | # The program still allows Fuzzy to override its determination of if loose | |
210 | # matching should be used, but it isn't currently used, as it is no longer | |
211 | # needed; the calculations it makes are good enough. | |
678f13d5 | 212 | # |
99870f4d KW |
213 | # SUMMARY OF HOW IT WORKS: |
214 | # | |
215 | # Process arguments | |
216 | # | |
217 | # A list is constructed containing each input file that is to be processed | |
218 | # | |
219 | # Each file on the list is processed in a loop, using the associated handler | |
220 | # code for each: | |
221 | # The PropertyAliases.txt and PropValueAliases.txt files are processed | |
222 | # first. These files name the properties and property values. | |
223 | # Objects are created of all the property and property value names | |
224 | # that the rest of the input should expect, including all synonyms. | |
225 | # The other input files give mappings from properties to property | |
226 | # values. That is, they list code points and say what the mapping | |
227 | # is under the given property. Some files give the mappings for | |
228 | # just one property; and some for many. This program goes through | |
229 | # each file and populates the properties from them. Some properties | |
230 | # are listed in more than one file, and Unicode has set up a | |
231 | # precedence as to which has priority if there is a conflict. Thus | |
232 | # the order of processing matters, and this program handles the | |
233 | # conflict possibility by processing the overriding input files | |
234 | # last, so that if necessary they replace earlier values. | |
235 | # After this is all done, the program creates the property mappings not | |
236 | # furnished by Unicode, but derivable from what it does give. | |
237 | # The tables of code points that match each property value in each | |
238 | # property that is accessible by regular expressions are created. | |
239 | # The Perl-defined properties are created and populated. Many of these | |
240 | # require data determined from the earlier steps | |
241 | # Any Perl-defined synonyms are created, and name clashes between Perl | |
678f13d5 | 242 | # and Unicode are reconciled and warned about. |
99870f4d KW |
243 | # All the properties are written to files |
244 | # Any other files are written, and final warnings issued. | |
678f13d5 | 245 | # |
99870f4d KW |
246 | # For clarity, a number of operators have been overloaded to work on tables: |
247 | # ~ means invert (take all characters not in the set). The more | |
248 | # conventional '!' is not used because of the possibility of confusing | |
249 | # it with the actual boolean operation. | |
250 | # + means union | |
251 | # - means subtraction | |
252 | # & means intersection | |
253 | # The precedence of these is the order listed. Parentheses should be | |
254 | # copiously used. These are not a general scheme. The operations aren't | |
255 | # defined for a number of things, deliberately, to avoid getting into trouble. | |
256 | # Operations are done on references and affect the underlying structures, so | |
257 | # that the copy constructors for them have been overloaded to not return a new | |
258 | # clone, but the input object itself. | |
678f13d5 | 259 | # |
99870f4d KW |
260 | # The bool operator is deliberately not overloaded to avoid confusion with |
261 | # "should it mean if the object merely exists, or also is non-empty?". | |
99870f4d KW |
262 | # |
263 | # WHY CERTAIN DESIGN DECISIONS WERE MADE | |
678f13d5 KW |
264 | # |
265 | # This program needs to be able to run under miniperl. Therefore, it uses a | |
266 | # minimum of other modules, and hence implements some things itself that could | |
267 | # be gotten from CPAN | |
268 | # | |
269 | # This program uses inputs published by the Unicode Consortium. These can | |
270 | # change incompatibly between releases without the Perl maintainers realizing | |
271 | # it. Therefore this program is now designed to try to flag these. It looks | |
272 | # at the directories where the inputs are, and flags any unrecognized files. | |
273 | # It keeps track of all the properties in the files it handles, and flags any | |
274 | # that it doesn't know how to handle. It also flags any input lines that | |
275 | # don't match the expected syntax, among other checks. | |
276 | # | |
277 | # It is also designed so if a new input file matches one of the known | |
278 | # templates, one hopefully just needs to add it to a list to have it | |
279 | # processed. | |
280 | # | |
281 | # As mentioned earlier, some properties are given in more than one file. In | |
282 | # particular, the files in the extracted directory are supposedly just | |
283 | # reformattings of the others. But they contain information not easily | |
284 | # derivable from the other files, including results for Unihan, which this | |
285 | # program doesn't ordinarily look at, and for unassigned code points. They | |
286 | # also have historically had errors or been incomplete. In an attempt to | |
287 | # create the best possible data, this program thus processes them first to | |
288 | # glean information missing from the other files; then processes those other | |
289 | # files to override any errors in the extracted ones. Much of the design was | |
290 | # driven by this need to store things and then possibly override them. | |
291 | # | |
292 | # It tries to keep fatal errors to a minimum, to generate something usable for | |
293 | # testing purposes. It always looks for files that could be inputs, and will | |
294 | # warn about any that it doesn't know how to handle (the -q option suppresses | |
295 | # the warning). | |
99870f4d | 296 | # |
678f13d5 KW |
297 | # Why is there more than one type of range? |
298 | # This simplified things. There are some very specialized code points that | |
299 | # have to be handled specially for output, such as Hangul syllable names. | |
300 | # By creating a range type (done late in the development process), it | |
301 | # allowed this to be stored with the range, and overridden by other input. | |
302 | # Originally these were stored in another data structure, and it became a | |
303 | # mess trying to decide if a second file that was for the same property was | |
304 | # overriding the earlier one or not. | |
305 | # | |
306 | # Why are there two kinds of tables, match and map? | |
307 | # (And there is a base class shared by the two as well.) As stated above, | |
308 | # they actually are for different things. Development proceeded much more | |
309 | # smoothly when I (khw) realized the distinction. Map tables are used to | |
310 | # give the property value for every code point (actually every code point | |
311 | # that doesn't map to a default value). Match tables are used for regular | |
312 | # expression matches, and are essentially the inverse mapping. Separating | |
313 | # the two allows more specialized methods, and error checks so that one | |
314 | # can't just take the intersection of two map tables, for example, as that | |
315 | # is nonsensical. | |
99870f4d | 316 | # |
23e33b60 KW |
317 | # DEBUGGING |
318 | # | |
678f13d5 KW |
319 | # This program is written so it will run under miniperl. Occasionally changes |
320 | # will cause an error where the backtrace doesn't work well under miniperl. | |
321 | # To diagnose the problem, you can instead run it under regular perl, if you | |
322 | # have one compiled. | |
323 | # | |
324 | # There is a good trace facility. To enable it, first sub DEBUG must be set | |
325 | # to return true. Then a line like | |
326 | # | |
327 | # local $to_trace = 1 if main::DEBUG; | |
328 | # | |
329 | # can be added to enable tracing in its lexical scope or until you insert | |
330 | # another line: | |
331 | # | |
332 | # local $to_trace = 0 if main::DEBUG; | |
333 | # | |
334 | # then use a line like "trace $a, @b, %c, ...; | |
335 | # | |
336 | # Some of the more complex subroutines already have trace statements in them. | |
337 | # Permanent trace statements should be like: | |
338 | # | |
339 | # trace ... if main::DEBUG && $to_trace; | |
340 | # | |
341 | # If there is just one or a few files that you're debugging, you can easily | |
342 | # cause most everything else to be skipped. Change the line | |
343 | # | |
344 | # my $debug_skip = 0; | |
345 | # | |
346 | # to 1, and every file whose object is in @input_file_objects and doesn't have | |
347 | # a, 'non_skip => 1,' in its constructor will be skipped. | |
348 | # | |
b4a0206c | 349 | # To compare the output tables, it may be useful to specify the -annotate |
c4019d52 KW |
350 | # flag. This causes the tables to expand so there is one entry for each |
351 | # non-algorithmically named code point giving, currently its name, and its | |
352 | # graphic representation if printable (and you have a font that knows about | |
353 | # it). This makes it easier to see what the particular code points are in | |
354 | # each output table. The tables are usable, but because they don't have | |
355 | # ranges (for the most part), a Perl using them will run slower. Non-named | |
356 | # code points are annotated with a description of their status, and contiguous | |
357 | # ones with the same description will be output as a range rather than | |
358 | # individually. Algorithmically named characters are also output as ranges, | |
359 | # except when there are just a few contiguous ones. | |
360 | # | |
99870f4d KW |
361 | # FUTURE ISSUES |
362 | # | |
363 | # The program would break if Unicode were to change its names so that | |
364 | # interior white space, underscores, or dashes differences were significant | |
365 | # within property and property value names. | |
366 | # | |
367 | # It might be easier to use the xml versions of the UCD if this program ever | |
368 | # would need heavy revision, and the ability to handle old versions was not | |
369 | # required. | |
370 | # | |
371 | # There is the potential for name collisions, in that Perl has chosen names | |
372 | # that Unicode could decide it also likes. There have been such collisions in | |
373 | # the past, with mostly Perl deciding to adopt the Unicode definition of the | |
374 | # name. However in the 5.2 Unicode beta testing, there were a number of such | |
375 | # collisions, which were withdrawn before the final release, because of Perl's | |
376 | # and other's protests. These all involved new properties which began with | |
377 | # 'Is'. Based on the protests, Unicode is unlikely to try that again. Also, | |
378 | # many of the Perl-defined synonyms, like Any, Word, etc, are listed in a | |
379 | # Unicode document, so they are unlikely to be used by Unicode for another | |
380 | # purpose. However, they might try something beginning with 'In', or use any | |
381 | # of the other Perl-defined properties. This program will warn you of name | |
382 | # collisions, and refuse to generate tables with them, but manual intervention | |
383 | # will be required in this event. One scheme that could be implemented, if | |
384 | # necessary, would be to have this program generate another file, or add a | |
385 | # field to mktables.lst that gives the date of first definition of a property. | |
386 | # Each new release of Unicode would use that file as a basis for the next | |
387 | # iteration. And the Perl synonym addition code could sort based on the age | |
388 | # of the property, so older properties get priority, and newer ones that clash | |
389 | # would be refused; hence existing code would not be impacted, and some other | |
390 | # synonym would have to be used for the new property. This is ugly, and | |
391 | # manual intervention would certainly be easier to do in the short run; lets | |
392 | # hope it never comes to this. | |
678f13d5 | 393 | # |
99870f4d KW |
394 | # A NOTE ON UNIHAN |
395 | # | |
396 | # This program can generate tables from the Unihan database. But it doesn't | |
397 | # by default, letting the CPAN module Unicode::Unihan handle them. Prior to | |
398 | # version 5.2, this database was in a single file, Unihan.txt. In 5.2 the | |
399 | # database was split into 8 different files, all beginning with the letters | |
400 | # 'Unihan'. This program will read those file(s) if present, but it needs to | |
401 | # know which of the many properties in the file(s) should have tables created | |
402 | # for them. It will create tables for any properties listed in | |
403 | # PropertyAliases.txt and PropValueAliases.txt, plus any listed in the | |
404 | # @cjk_properties array and the @cjk_property_values array. Thus, if a | |
405 | # property you want is not in those files of the release you are building | |
406 | # against, you must add it to those two arrays. Starting in 4.0, the | |
407 | # Unicode_Radical_Stroke was listed in those files, so if the Unihan database | |
408 | # is present in the directory, a table will be generated for that property. | |
409 | # In 5.2, several more properties were added. For your convenience, the two | |
5f7264c7 | 410 | # arrays are initialized with all the 6.0 listed properties that are also in |
99870f4d KW |
411 | # earlier releases. But these are commented out. You can just uncomment the |
412 | # ones you want, or use them as a template for adding entries for other | |
413 | # properties. | |
414 | # | |
415 | # You may need to adjust the entries to suit your purposes. setup_unihan(), | |
416 | # and filter_unihan_line() are the functions where this is done. This program | |
417 | # already does some adjusting to make the lines look more like the rest of the | |
418 | # Unicode DB; You can see what that is in filter_unihan_line() | |
419 | # | |
420 | # There is a bug in the 3.2 data file in which some values for the | |
421 | # kPrimaryNumeric property have commas and an unexpected comment. A filter | |
422 | # could be added for these; or for a particular installation, the Unihan.txt | |
423 | # file could be edited to fix them. | |
99870f4d | 424 | # |
678f13d5 KW |
425 | # HOW TO ADD A FILE TO BE PROCESSED |
426 | # | |
427 | # A new file from Unicode needs to have an object constructed for it in | |
428 | # @input_file_objects, probably at the end or at the end of the extracted | |
429 | # ones. The program should warn you if its name will clash with others on | |
430 | # restrictive file systems, like DOS. If so, figure out a better name, and | |
431 | # add lines to the README.perl file giving that. If the file is a character | |
432 | # property, it should be in the format that Unicode has by default | |
433 | # standardized for such files for the more recently introduced ones. | |
434 | # If so, the Input_file constructor for @input_file_objects can just be the | |
435 | # file name and release it first appeared in. If not, then it should be | |
436 | # possible to construct an each_line_handler() to massage the line into the | |
437 | # standardized form. | |
438 | # | |
439 | # For non-character properties, more code will be needed. You can look at | |
440 | # the existing entries for clues. | |
441 | # | |
442 | # UNICODE VERSIONS NOTES | |
443 | # | |
444 | # The Unicode UCD has had a number of errors in it over the versions. And | |
445 | # these remain, by policy, in the standard for that version. Therefore it is | |
446 | # risky to correct them, because code may be expecting the error. So this | |
447 | # program doesn't generally make changes, unless the error breaks the Perl | |
448 | # core. As an example, some versions of 2.1.x Jamo.txt have the wrong value | |
449 | # for U+1105, which causes real problems for the algorithms for Jamo | |
450 | # calculations, so it is changed here. | |
451 | # | |
452 | # But it isn't so clear cut as to what to do about concepts that are | |
453 | # introduced in a later release; should they extend back to earlier releases | |
454 | # where the concept just didn't exist? It was easier to do this than to not, | |
455 | # so that's what was done. For example, the default value for code points not | |
456 | # in the files for various properties was probably undefined until changed by | |
457 | # some version. No_Block for blocks is such an example. This program will | |
458 | # assign No_Block even in Unicode versions that didn't have it. This has the | |
459 | # benefit that code being written doesn't have to special case earlier | |
460 | # versions; and the detriment that it doesn't match the Standard precisely for | |
461 | # the affected versions. | |
462 | # | |
463 | # Here are some observations about some of the issues in early versions: | |
464 | # | |
6426c51b | 465 | # The number of code points in \p{alpha} halved in 2.1.9. It turns out that |
678f13d5 KW |
466 | # the reason is that the CJK block starting at 4E00 was removed from PropList, |
467 | # and was not put back in until 3.1.0 | |
468 | # | |
469 | # Unicode introduced the synonym Space for White_Space in 4.1. Perl has | |
470 | # always had a \p{Space}. In release 3.2 only, they are not synonymous. The | |
471 | # reason is that 3.2 introduced U+205F=medium math space, which was not | |
472 | # classed as white space, but Perl figured out that it should have been. 4.0 | |
473 | # reclassified it correctly. | |
474 | # | |
475 | # Another change between 3.2 and 4.0 is the CCC property value ATBL. In 3.2 | |
476 | # this was erroneously a synonym for 202. In 4.0, ATB became 202, and ATBL | |
477 | # was left with no code points, as all the ones that mapped to 202 stayed | |
478 | # mapped to 202. Thus if your program used the numeric name for the class, | |
479 | # it would not have been affected, but if it used the mnemonic, it would have | |
480 | # been. | |
481 | # | |
482 | # \p{Script=Hrkt} (Katakana_Or_Hiragana) came in 4.0.1. Before that code | |
483 | # points which eventually came to have this script property value, instead | |
484 | # mapped to "Unknown". But in the next release all these code points were | |
485 | # moved to \p{sc=common} instead. | |
99870f4d KW |
486 | # |
487 | # The default for missing code points for BidiClass is complicated. Starting | |
488 | # in 3.1.1, the derived file DBidiClass.txt handles this, but this program | |
489 | # tries to do the best it can for earlier releases. It is done in | |
490 | # process_PropertyAliases() | |
491 | # | |
492 | ############################################################################## | |
493 | ||
494 | my $UNDEF = ':UNDEF:'; # String to print out for undefined values in tracing | |
495 | # and errors | |
496 | my $MAX_LINE_WIDTH = 78; | |
497 | ||
498 | # Debugging aid to skip most files so as to not be distracted by them when | |
499 | # concentrating on the ones being debugged. Add | |
500 | # non_skip => 1, | |
501 | # to the constructor for those files you want processed when you set this. | |
502 | # Files with a first version number of 0 are special: they are always | |
c12f2655 KW |
503 | # processed regardless of the state of this flag. Generally, Jamo.txt and |
504 | # UnicodeData.txt must not be skipped if you want this program to not die | |
505 | # before normal completion. | |
99870f4d KW |
506 | my $debug_skip = 0; |
507 | ||
508 | # Set to 1 to enable tracing. | |
509 | our $to_trace = 0; | |
510 | ||
511 | { # Closure for trace: debugging aid | |
512 | my $print_caller = 1; # ? Include calling subroutine name | |
513 | my $main_with_colon = 'main::'; | |
514 | my $main_colon_length = length($main_with_colon); | |
515 | ||
516 | sub trace { | |
517 | return unless $to_trace; # Do nothing if global flag not set | |
518 | ||
519 | my @input = @_; | |
520 | ||
521 | local $DB::trace = 0; | |
522 | $DB::trace = 0; # Quiet 'used only once' message | |
523 | ||
524 | my $line_number; | |
525 | ||
526 | # Loop looking up the stack to get the first non-trace caller | |
527 | my $caller_line; | |
528 | my $caller_name; | |
529 | my $i = 0; | |
530 | do { | |
531 | $line_number = $caller_line; | |
532 | (my $pkg, my $file, $caller_line, my $caller) = caller $i++; | |
533 | $caller = $main_with_colon unless defined $caller; | |
534 | ||
535 | $caller_name = $caller; | |
536 | ||
537 | # get rid of pkg | |
538 | $caller_name =~ s/.*:://; | |
539 | if (substr($caller_name, 0, $main_colon_length) | |
540 | eq $main_with_colon) | |
541 | { | |
542 | $caller_name = substr($caller_name, $main_colon_length); | |
543 | } | |
544 | ||
545 | } until ($caller_name ne 'trace'); | |
546 | ||
547 | # If the stack was empty, we were called from the top level | |
548 | $caller_name = 'main' if ($caller_name eq "" | |
549 | || $caller_name eq 'trace'); | |
550 | ||
551 | my $output = ""; | |
552 | foreach my $string (@input) { | |
553 | #print STDERR __LINE__, ": ", join ", ", @input, "\n"; | |
554 | if (ref $string eq 'ARRAY' || ref $string eq 'HASH') { | |
555 | $output .= simple_dumper($string); | |
556 | } | |
557 | else { | |
558 | $string = "$string" if ref $string; | |
559 | $string = $UNDEF unless defined $string; | |
560 | chomp $string; | |
561 | $string = '""' if $string eq ""; | |
562 | $output .= " " if $output ne "" | |
563 | && $string ne "" | |
564 | && substr($output, -1, 1) ne " " | |
565 | && substr($string, 0, 1) ne " "; | |
566 | $output .= $string; | |
567 | } | |
568 | } | |
569 | ||
99f78760 KW |
570 | print STDERR sprintf "%4d: ", $line_number if defined $line_number; |
571 | print STDERR "$caller_name: " if $print_caller; | |
99870f4d KW |
572 | print STDERR $output, "\n"; |
573 | return; | |
574 | } | |
575 | } | |
576 | ||
577 | # This is for a rarely used development feature that allows you to compare two | |
578 | # versions of the Unicode standard without having to deal with changes caused | |
c12f2655 KW |
579 | # by the code points introduced in the later version. Change the 0 to a |
580 | # string containing a SINGLE dotted Unicode release number (e.g. "2.1"). Only | |
581 | # code points introduced in that release and earlier will be used; later ones | |
582 | # are thrown away. You use the version number of the earliest one you want to | |
583 | # compare; then run this program on directory structures containing each | |
584 | # release, and compare the outputs. These outputs will therefore include only | |
585 | # the code points common to both releases, and you can see the changes caused | |
586 | # just by the underlying release semantic changes. For versions earlier than | |
587 | # 3.2, you must copy a version of DAge.txt into the directory. | |
588 | my $string_compare_versions = DEBUG && 0; # e.g., "2.1"; | |
99870f4d KW |
589 | my $compare_versions = DEBUG |
590 | && $string_compare_versions | |
591 | && pack "C*", split /\./, $string_compare_versions; | |
592 | ||
593 | sub uniques { | |
594 | # Returns non-duplicated input values. From "Perl Best Practices: | |
595 | # Encapsulated Cleverness". p. 455 in first edition. | |
596 | ||
597 | my %seen; | |
0e407844 NC |
598 | # Arguably this breaks encapsulation, if the goal is to permit multiple |
599 | # distinct objects to stringify to the same value, and be interchangeable. | |
600 | # However, for this program, no two objects stringify identically, and all | |
601 | # lists passed to this function are either objects or strings. So this | |
602 | # doesn't affect correctness, but it does give a couple of percent speedup. | |
603 | no overloading; | |
99870f4d KW |
604 | return grep { ! $seen{$_}++ } @_; |
605 | } | |
606 | ||
607 | $0 = File::Spec->canonpath($0); | |
608 | ||
609 | my $make_test_script = 0; # ? Should we output a test script | |
610 | my $write_unchanged_files = 0; # ? Should we update the output files even if | |
611 | # we don't think they have changed | |
612 | my $use_directory = ""; # ? Should we chdir somewhere. | |
613 | my $pod_directory; # input directory to store the pod file. | |
614 | my $pod_file = 'perluniprops'; | |
615 | my $t_path; # Path to the .t test file | |
616 | my $file_list = 'mktables.lst'; # File to store input and output file names. | |
617 | # This is used to speed up the build, by not | |
618 | # executing the main body of the program if | |
619 | # nothing on the list has changed since the | |
620 | # previous build | |
621 | my $make_list = 1; # ? Should we write $file_list. Set to always | |
622 | # make a list so that when the pumpking is | |
623 | # preparing a release, s/he won't have to do | |
624 | # special things | |
625 | my $glob_list = 0; # ? Should we try to include unknown .txt files | |
626 | # in the input. | |
bd9ebcfd KW |
627 | my $output_range_counts = $debugging_build; # ? Should we include the number |
628 | # of code points in ranges in | |
629 | # the output | |
558712cf | 630 | my $annotate = 0; # ? Should character names be in the output |
9ef2b94f | 631 | |
99870f4d KW |
632 | # Verbosity levels; 0 is quiet |
633 | my $NORMAL_VERBOSITY = 1; | |
634 | my $PROGRESS = 2; | |
635 | my $VERBOSE = 3; | |
636 | ||
637 | my $verbosity = $NORMAL_VERBOSITY; | |
638 | ||
639 | # Process arguments | |
640 | while (@ARGV) { | |
cf25bb62 JH |
641 | my $arg = shift @ARGV; |
642 | if ($arg eq '-v') { | |
99870f4d KW |
643 | $verbosity = $VERBOSE; |
644 | } | |
645 | elsif ($arg eq '-p') { | |
646 | $verbosity = $PROGRESS; | |
647 | $| = 1; # Flush buffers as we go. | |
648 | } | |
649 | elsif ($arg eq '-q') { | |
650 | $verbosity = 0; | |
651 | } | |
652 | elsif ($arg eq '-w') { | |
653 | $write_unchanged_files = 1; # update the files even if havent changed | |
654 | } | |
655 | elsif ($arg eq '-check') { | |
6ae7e459 YO |
656 | my $this = shift @ARGV; |
657 | my $ok = shift @ARGV; | |
658 | if ($this ne $ok) { | |
659 | print "Skipping as check params are not the same.\n"; | |
660 | exit(0); | |
661 | } | |
00a8df5c | 662 | } |
99870f4d KW |
663 | elsif ($arg eq '-P' && defined ($pod_directory = shift)) { |
664 | -d $pod_directory or croak "Directory '$pod_directory' doesn't exist"; | |
665 | } | |
3df51b85 KW |
666 | elsif ($arg eq '-maketest' || ($arg eq '-T' && defined ($t_path = shift))) |
667 | { | |
99870f4d | 668 | $make_test_script = 1; |
99870f4d KW |
669 | } |
670 | elsif ($arg eq '-makelist') { | |
671 | $make_list = 1; | |
672 | } | |
673 | elsif ($arg eq '-C' && defined ($use_directory = shift)) { | |
674 | -d $use_directory or croak "Unknown directory '$use_directory'"; | |
675 | } | |
676 | elsif ($arg eq '-L') { | |
677 | ||
678 | # Existence not tested until have chdir'd | |
679 | $file_list = shift; | |
680 | } | |
681 | elsif ($arg eq '-globlist') { | |
682 | $glob_list = 1; | |
683 | } | |
684 | elsif ($arg eq '-c') { | |
685 | $output_range_counts = ! $output_range_counts | |
686 | } | |
b4a0206c | 687 | elsif ($arg eq '-annotate') { |
558712cf | 688 | $annotate = 1; |
bd9ebcfd KW |
689 | $debugging_build = 1; |
690 | $output_range_counts = 1; | |
9ef2b94f | 691 | } |
99870f4d KW |
692 | else { |
693 | my $with_c = 'with'; | |
694 | $with_c .= 'out' if $output_range_counts; # Complements the state | |
695 | croak <<END; | |
696 | usage: $0 [-c|-p|-q|-v|-w] [-C dir] [-L filelist] [ -P pod_dir ] | |
697 | [ -T test_file_path ] [-globlist] [-makelist] [-maketest] | |
698 | [-check A B ] | |
699 | -c : Output comments $with_c number of code points in ranges | |
700 | -q : Quiet Mode: Only output serious warnings. | |
701 | -p : Set verbosity level to normal plus show progress. | |
702 | -v : Set Verbosity level high: Show progress and non-serious | |
703 | warnings | |
704 | -w : Write files regardless | |
705 | -C dir : Change to this directory before proceeding. All relative paths | |
706 | except those specified by the -P and -T options will be done | |
707 | with respect to this directory. | |
708 | -P dir : Output $pod_file file to directory 'dir'. | |
3df51b85 | 709 | -T path : Create a test script as 'path'; overrides -maketest |
99870f4d KW |
710 | -L filelist : Use alternate 'filelist' instead of standard one |
711 | -globlist : Take as input all non-Test *.txt files in current and sub | |
712 | directories | |
3df51b85 KW |
713 | -maketest : Make test script 'TestProp.pl' in current (or -C directory), |
714 | overrides -T | |
99870f4d | 715 | -makelist : Rewrite the file list $file_list based on current setup |
b4a0206c | 716 | -annotate : Output an annotation for each character in the table files; |
c4019d52 KW |
717 | useful for debugging mktables, looking at diffs; but is slow, |
718 | memory intensive; resulting tables are usable but slow and | |
719 | very large. | |
99870f4d KW |
720 | -check A B : Executes $0 only if A and B are the same |
721 | END | |
722 | } | |
723 | } | |
724 | ||
725 | # Stores the most-recently changed file. If none have changed, can skip the | |
726 | # build | |
aeab6150 | 727 | my $most_recent = (stat $0)[9]; # Do this before the chdir! |
99870f4d KW |
728 | |
729 | # Change directories now, because need to read 'version' early. | |
730 | if ($use_directory) { | |
3df51b85 | 731 | if ($pod_directory && ! File::Spec->file_name_is_absolute($pod_directory)) { |
99870f4d KW |
732 | $pod_directory = File::Spec->rel2abs($pod_directory); |
733 | } | |
3df51b85 | 734 | if ($t_path && ! File::Spec->file_name_is_absolute($t_path)) { |
99870f4d | 735 | $t_path = File::Spec->rel2abs($t_path); |
00a8df5c | 736 | } |
99870f4d | 737 | chdir $use_directory or croak "Failed to chdir to '$use_directory':$!"; |
3df51b85 | 738 | if ($pod_directory && File::Spec->file_name_is_absolute($pod_directory)) { |
99870f4d | 739 | $pod_directory = File::Spec->abs2rel($pod_directory); |
02b1aeec | 740 | } |
3df51b85 | 741 | if ($t_path && File::Spec->file_name_is_absolute($t_path)) { |
99870f4d | 742 | $t_path = File::Spec->abs2rel($t_path); |
02b1aeec | 743 | } |
00a8df5c YO |
744 | } |
745 | ||
99870f4d KW |
746 | # Get Unicode version into regular and v-string. This is done now because |
747 | # various tables below get populated based on it. These tables are populated | |
748 | # here to be near the top of the file, and so easily seeable by those needing | |
749 | # to modify things. | |
750 | open my $VERSION, "<", "version" | |
751 | or croak "$0: can't open required file 'version': $!\n"; | |
752 | my $string_version = <$VERSION>; | |
753 | close $VERSION; | |
754 | chomp $string_version; | |
755 | my $v_version = pack "C*", split /\./, $string_version; # v string | |
756 | ||
757 | # The following are the complete names of properties with property values that | |
758 | # are known to not match any code points in some versions of Unicode, but that | |
759 | # may change in the future so they should be matchable, hence an empty file is | |
760 | # generated for them. | |
761 | my @tables_that_may_be_empty = ( | |
762 | 'Joining_Type=Left_Joining', | |
763 | ); | |
764 | push @tables_that_may_be_empty, 'Script=Common' if $v_version le v4.0.1; | |
765 | push @tables_that_may_be_empty, 'Title' if $v_version lt v2.0.0; | |
766 | push @tables_that_may_be_empty, 'Script=Katakana_Or_Hiragana' | |
767 | if $v_version ge v4.1.0; | |
82aed44a KW |
768 | push @tables_that_may_be_empty, 'Script_Extensions=Katakana_Or_Hiragana' |
769 | if $v_version ge v6.0.0; | |
99870f4d KW |
770 | |
771 | # The lists below are hashes, so the key is the item in the list, and the | |
772 | # value is the reason why it is in the list. This makes generation of | |
773 | # documentation easier. | |
774 | ||
775 | my %why_suppressed; # No file generated for these. | |
776 | ||
777 | # Files aren't generated for empty extraneous properties. This is arguable. | |
778 | # Extraneous properties generally come about because a property is no longer | |
779 | # used in a newer version of Unicode. If we generated a file without code | |
780 | # points, programs that used to work on that property will still execute | |
781 | # without errors. It just won't ever match (or will always match, with \P{}). | |
782 | # This means that the logic is now likely wrong. I (khw) think its better to | |
783 | # find this out by getting an error message. Just move them to the table | |
784 | # above to change this behavior | |
785 | my %why_suppress_if_empty_warn_if_not = ( | |
786 | ||
787 | # It is the only property that has ever officially been removed from the | |
788 | # Standard. The database never contained any code points for it. | |
789 | 'Special_Case_Condition' => 'Obsolete', | |
790 | ||
791 | # Apparently never official, but there were code points in some versions of | |
792 | # old-style PropList.txt | |
793 | 'Non_Break' => 'Obsolete', | |
794 | ); | |
795 | ||
796 | # These would normally go in the warn table just above, but they were changed | |
797 | # a long time before this program was written, so warnings about them are | |
798 | # moot. | |
799 | if ($v_version gt v3.2.0) { | |
800 | push @tables_that_may_be_empty, | |
801 | 'Canonical_Combining_Class=Attached_Below_Left' | |
802 | } | |
803 | ||
5f7264c7 | 804 | # These are listed in the Property aliases file in 6.0, but Unihan is ignored |
99870f4d KW |
805 | # unless explicitly added. |
806 | if ($v_version ge v5.2.0) { | |
807 | my $unihan = 'Unihan; remove from list if using Unihan'; | |
ea25a9b2 | 808 | foreach my $table (qw ( |
99870f4d KW |
809 | kAccountingNumeric |
810 | kOtherNumeric | |
811 | kPrimaryNumeric | |
812 | kCompatibilityVariant | |
813 | kIICore | |
814 | kIRG_GSource | |
815 | kIRG_HSource | |
816 | kIRG_JSource | |
817 | kIRG_KPSource | |
818 | kIRG_MSource | |
819 | kIRG_KSource | |
820 | kIRG_TSource | |
821 | kIRG_USource | |
822 | kIRG_VSource | |
823 | kRSUnicode | |
ea25a9b2 | 824 | )) |
99870f4d KW |
825 | { |
826 | $why_suppress_if_empty_warn_if_not{$table} = $unihan; | |
827 | } | |
ca12659b NC |
828 | } |
829 | ||
272501f6 KW |
830 | # Enum values for to_output_map() method in the Map_Table package. |
831 | my $EXTERNAL_MAP = 1; | |
832 | my $INTERNAL_MAP = 2; | |
833 | ||
fcf1973c KW |
834 | # To override computed values for writing the map tables for these properties. |
835 | # The default for enum map tables is to write them out, so that the Unicode | |
836 | # .txt files can be removed, but all the data to compute any property value | |
837 | # for any code point is available in a more compact form. | |
838 | my %global_to_output_map = ( | |
839 | # Needed by UCD.pm, but don't want to publicize that it exists, so won't | |
c12f2655 KW |
840 | # get stuck supporting it if things change. Since it is a STRING |
841 | # property, it normally would be listed in the pod, but INTERNAL_MAP | |
842 | # suppresses that. | |
fcf1973c KW |
843 | Unicode_1_Name => $INTERNAL_MAP, |
844 | ||
845 | Present_In => 0, # Suppress, as easily computed from Age | |
fcf1973c | 846 | Block => 0, # Suppress, as Blocks.txt is retained. |
53d34b6c KW |
847 | |
848 | # Suppress, as mapping can be found instead from the | |
849 | # Perl_Decomposition_Mapping file | |
850 | Decomposition_Type => 0, | |
fcf1973c KW |
851 | ); |
852 | ||
99870f4d | 853 | # Properties that this program ignores. |
230e0c16 KW |
854 | my @unimplemented_properties; |
855 | ||
856 | # With this release, it is automatically handled if the Unihan db is | |
857 | # downloaded | |
858 | push @unimplemented_properties, 'Unicode_Radical_Stroke' if $v_version le v5.2.0; | |
d73e5302 | 859 | |
99870f4d KW |
860 | # There are several types of obsolete properties defined by Unicode. These |
861 | # must be hand-edited for every new Unicode release. | |
862 | my %why_deprecated; # Generates a deprecated warning message if used. | |
863 | my %why_stabilized; # Documentation only | |
864 | my %why_obsolete; # Documentation only | |
865 | ||
866 | { # Closure | |
867 | my $simple = 'Perl uses the more complete version of this property'; | |
868 | my $unihan = 'Unihan properties are by default not enabled in the Perl core. Instead use CPAN: Unicode::Unihan'; | |
869 | ||
870 | my $other_properties = 'other properties'; | |
871 | my $contributory = "Used by Unicode internally for generating $other_properties and not intended to be used stand-alone"; | |
5d294d41 | 872 | my $why_no_expand = "Deprecated by Unicode. These are characters that expand to more than one character in the specified normalization form, but whether they actually take up more bytes or not depends on the encoding being used. For example, a UTF-8 encoded character may expand to a different number of bytes than a UTF-32 encoded character."; |
99870f4d KW |
873 | |
874 | %why_deprecated = ( | |
5f7264c7 | 875 | 'Grapheme_Link' => 'Deprecated by Unicode: Duplicates ccc=vr (Canonical_Combining_Class=Virama)', |
99870f4d KW |
876 | 'Jamo_Short_Name' => $contributory, |
877 | 'Line_Break=Surrogate' => 'Deprecated by Unicode because surrogates should never appear in well-formed text, and therefore shouldn\'t be the basis for line breaking', | |
878 | 'Other_Alphabetic' => $contributory, | |
879 | 'Other_Default_Ignorable_Code_Point' => $contributory, | |
880 | 'Other_Grapheme_Extend' => $contributory, | |
881 | 'Other_ID_Continue' => $contributory, | |
882 | 'Other_ID_Start' => $contributory, | |
883 | 'Other_Lowercase' => $contributory, | |
884 | 'Other_Math' => $contributory, | |
885 | 'Other_Uppercase' => $contributory, | |
e22aaf5c KW |
886 | 'Expands_On_NFC' => $why_no_expand, |
887 | 'Expands_On_NFD' => $why_no_expand, | |
888 | 'Expands_On_NFKC' => $why_no_expand, | |
889 | 'Expands_On_NFKD' => $why_no_expand, | |
99870f4d KW |
890 | ); |
891 | ||
892 | %why_suppressed = ( | |
5f7264c7 | 893 | # There is a lib/unicore/Decomposition.pl (used by Normalize.pm) which |
99870f4d KW |
894 | # contains the same information, but without the algorithmically |
895 | # determinable Hangul syllables'. This file is not published, so it's | |
896 | # existence is not noted in the comment. | |
897 | 'Decomposition_Mapping' => 'Accessible via Unicode::Normalize', | |
898 | ||
899 | 'ISO_Comment' => 'Apparently no demand for it, but can access it through Unicode::UCD::charinfo. Obsoleted, and code points for it removed in Unicode 5.2', | |
99870f4d KW |
900 | |
901 | 'Simple_Case_Folding' => "$simple. Can access this through Unicode::UCD::casefold", | |
902 | 'Simple_Lowercase_Mapping' => "$simple. Can access this through Unicode::UCD::charinfo", | |
903 | 'Simple_Titlecase_Mapping' => "$simple. Can access this through Unicode::UCD::charinfo", | |
904 | 'Simple_Uppercase_Mapping' => "$simple. Can access this through Unicode::UCD::charinfo", | |
905 | ||
906 | 'Name' => "Accessible via 'use charnames;'", | |
907 | 'Name_Alias' => "Accessible via 'use charnames;'", | |
908 | ||
5f7264c7 | 909 | FC_NFKC_Closure => 'Supplanted in usage by NFKC_Casefold; otherwise not useful', |
99870f4d KW |
910 | ); |
911 | ||
912 | # The following are suppressed because they were made contributory or | |
913 | # deprecated by Unicode before Perl ever thought about supporting them. | |
e22aaf5c KW |
914 | foreach my $property ('Jamo_Short_Name', |
915 | 'Grapheme_Link', | |
916 | 'Expands_On_NFC', | |
917 | 'Expands_On_NFD', | |
918 | 'Expands_On_NFKC', | |
919 | 'Expands_On_NFKD' | |
920 | ) { | |
99870f4d KW |
921 | $why_suppressed{$property} = $why_deprecated{$property}; |
922 | } | |
cf25bb62 | 923 | |
99870f4d KW |
924 | # Customize the message for all the 'Other_' properties |
925 | foreach my $property (keys %why_deprecated) { | |
926 | next if (my $main_property = $property) !~ s/^Other_//; | |
927 | $why_deprecated{$property} =~ s/$other_properties/the $main_property property (which should be used instead)/; | |
928 | } | |
929 | } | |
930 | ||
931 | if ($v_version ge 4.0.0) { | |
932 | $why_stabilized{'Hyphen'} = 'Use the Line_Break property instead; see www.unicode.org/reports/tr14'; | |
5f7264c7 KW |
933 | if ($v_version ge 6.0.0) { |
934 | $why_deprecated{'Hyphen'} = 'Supplanted by Line_Break property values; see www.unicode.org/reports/tr14'; | |
935 | } | |
99870f4d | 936 | } |
5f7264c7 | 937 | if ($v_version ge 5.2.0 && $v_version lt 6.0.0) { |
99870f4d | 938 | $why_obsolete{'ISO_Comment'} = 'Code points for it have been removed'; |
5f7264c7 | 939 | if ($v_version ge 6.0.0) { |
63f74647 | 940 | $why_deprecated{'ISO_Comment'} = 'No longer needed for Unicode\'s internal chart generation; otherwise not useful, and code points for it have been removed'; |
5f7264c7 | 941 | } |
99870f4d KW |
942 | } |
943 | ||
944 | # Probably obsolete forever | |
945 | if ($v_version ge v4.1.0) { | |
82aed44a KW |
946 | $why_suppressed{'Script=Katakana_Or_Hiragana'} = 'Obsolete. All code points previously matched by this have been moved to "Script=Common".'; |
947 | } | |
948 | if ($v_version ge v6.0.0) { | |
2b352efd KW |
949 | $why_suppressed{'Script=Katakana_Or_Hiragana'} .= ' Consider instead using "Script_Extensions=Katakana" or "Script_Extensions=Hiragana (or both)"'; |
950 | $why_suppressed{'Script_Extensions=Katakana_Or_Hiragana'} = 'All code points that would be matched by this are matched by either "Script_Extensions=Katakana" or "Script_Extensions=Hiragana"'; | |
99870f4d KW |
951 | } |
952 | ||
953 | # This program can create files for enumerated-like properties, such as | |
954 | # 'Numeric_Type'. This file would be the same format as for a string | |
955 | # property, with a mapping from code point to its value, so you could look up, | |
956 | # for example, the script a code point is in. But no one so far wants this | |
957 | # mapping, or they have found another way to get it since this is a new | |
958 | # feature. So no file is generated except if it is in this list. | |
959 | my @output_mapped_properties = split "\n", <<END; | |
960 | END | |
961 | ||
c12f2655 KW |
962 | # If you are using the Unihan database in a Unicode version before 5.2, you |
963 | # need to add the properties that you want to extract from it to this table. | |
964 | # For your convenience, the properties in the 6.0 PropertyAliases.txt file are | |
965 | # listed, commented out | |
99870f4d KW |
966 | my @cjk_properties = split "\n", <<'END'; |
967 | #cjkAccountingNumeric; kAccountingNumeric | |
968 | #cjkOtherNumeric; kOtherNumeric | |
969 | #cjkPrimaryNumeric; kPrimaryNumeric | |
970 | #cjkCompatibilityVariant; kCompatibilityVariant | |
971 | #cjkIICore ; kIICore | |
972 | #cjkIRG_GSource; kIRG_GSource | |
973 | #cjkIRG_HSource; kIRG_HSource | |
974 | #cjkIRG_JSource; kIRG_JSource | |
975 | #cjkIRG_KPSource; kIRG_KPSource | |
976 | #cjkIRG_KSource; kIRG_KSource | |
977 | #cjkIRG_TSource; kIRG_TSource | |
978 | #cjkIRG_USource; kIRG_USource | |
979 | #cjkIRG_VSource; kIRG_VSource | |
980 | #cjkRSUnicode; kRSUnicode ; Unicode_Radical_Stroke; URS | |
981 | END | |
982 | ||
983 | # Similarly for the property values. For your convenience, the lines in the | |
5f7264c7 | 984 | # 6.0 PropertyAliases.txt file are listed. Just remove the first BUT NOT both |
c12f2655 | 985 | # '#' marks (for Unicode versions before 5.2) |
99870f4d KW |
986 | my @cjk_property_values = split "\n", <<'END'; |
987 | ## @missing: 0000..10FFFF; cjkAccountingNumeric; NaN | |
988 | ## @missing: 0000..10FFFF; cjkCompatibilityVariant; <code point> | |
989 | ## @missing: 0000..10FFFF; cjkIICore; <none> | |
990 | ## @missing: 0000..10FFFF; cjkIRG_GSource; <none> | |
991 | ## @missing: 0000..10FFFF; cjkIRG_HSource; <none> | |
992 | ## @missing: 0000..10FFFF; cjkIRG_JSource; <none> | |
993 | ## @missing: 0000..10FFFF; cjkIRG_KPSource; <none> | |
994 | ## @missing: 0000..10FFFF; cjkIRG_KSource; <none> | |
995 | ## @missing: 0000..10FFFF; cjkIRG_TSource; <none> | |
996 | ## @missing: 0000..10FFFF; cjkIRG_USource; <none> | |
997 | ## @missing: 0000..10FFFF; cjkIRG_VSource; <none> | |
998 | ## @missing: 0000..10FFFF; cjkOtherNumeric; NaN | |
999 | ## @missing: 0000..10FFFF; cjkPrimaryNumeric; NaN | |
1000 | ## @missing: 0000..10FFFF; cjkRSUnicode; <none> | |
1001 | END | |
1002 | ||
1003 | # The input files don't list every code point. Those not listed are to be | |
1004 | # defaulted to some value. Below are hard-coded what those values are for | |
1005 | # non-binary properties as of 5.1. Starting in 5.0, there are | |
1006 | # machine-parsable comment lines in the files the give the defaults; so this | |
1007 | # list shouldn't have to be extended. The claim is that all missing entries | |
1008 | # for binary properties will default to 'N'. Unicode tried to change that in | |
1009 | # 5.2, but the beta period produced enough protest that they backed off. | |
1010 | # | |
1011 | # The defaults for the fields that appear in UnicodeData.txt in this hash must | |
1012 | # be in the form that it expects. The others may be synonyms. | |
1013 | my $CODE_POINT = '<code point>'; | |
1014 | my %default_mapping = ( | |
1015 | Age => "Unassigned", | |
1016 | # Bidi_Class => Complicated; set in code | |
1017 | Bidi_Mirroring_Glyph => "", | |
1018 | Block => 'No_Block', | |
1019 | Canonical_Combining_Class => 0, | |
1020 | Case_Folding => $CODE_POINT, | |
1021 | Decomposition_Mapping => $CODE_POINT, | |
1022 | Decomposition_Type => 'None', | |
1023 | East_Asian_Width => "Neutral", | |
1024 | FC_NFKC_Closure => $CODE_POINT, | |
1025 | General_Category => 'Cn', | |
1026 | Grapheme_Cluster_Break => 'Other', | |
1027 | Hangul_Syllable_Type => 'NA', | |
1028 | ISO_Comment => "", | |
1029 | Jamo_Short_Name => "", | |
1030 | Joining_Group => "No_Joining_Group", | |
1031 | # Joining_Type => Complicated; set in code | |
1032 | kIICore => 'N', # Is converted to binary | |
1033 | #Line_Break => Complicated; set in code | |
1034 | Lowercase_Mapping => $CODE_POINT, | |
1035 | Name => "", | |
1036 | Name_Alias => "", | |
1037 | NFC_QC => 'Yes', | |
1038 | NFD_QC => 'Yes', | |
1039 | NFKC_QC => 'Yes', | |
1040 | NFKD_QC => 'Yes', | |
1041 | Numeric_Type => 'None', | |
1042 | Numeric_Value => 'NaN', | |
1043 | Script => ($v_version le 4.1.0) ? 'Common' : 'Unknown', | |
1044 | Sentence_Break => 'Other', | |
1045 | Simple_Case_Folding => $CODE_POINT, | |
1046 | Simple_Lowercase_Mapping => $CODE_POINT, | |
1047 | Simple_Titlecase_Mapping => $CODE_POINT, | |
1048 | Simple_Uppercase_Mapping => $CODE_POINT, | |
1049 | Titlecase_Mapping => $CODE_POINT, | |
1050 | Unicode_1_Name => "", | |
1051 | Unicode_Radical_Stroke => "", | |
1052 | Uppercase_Mapping => $CODE_POINT, | |
1053 | Word_Break => 'Other', | |
1054 | ); | |
1055 | ||
1056 | # Below are files that Unicode furnishes, but this program ignores, and why | |
1057 | my %ignored_files = ( | |
73ba1144 KW |
1058 | 'CJKRadicals.txt' => 'Maps the kRSUnicode property values to corresponding code points', |
1059 | 'Index.txt' => 'Alphabetical index of Unicode characters', | |
1060 | 'NamedSqProv.txt' => 'Named sequences proposed for inclusion in a later version of the Unicode Standard; if you need them now, you can append this file to F<NamedSequences.txt> and recompile perl', | |
1061 | 'NamesList.txt' => 'Annotated list of characters', | |
1062 | 'NormalizationCorrections.txt' => 'Documentation of corrections already incorporated into the Unicode data base', | |
1063 | 'Props.txt' => 'Only in very early releases; is a subset of F<PropList.txt> (which is used instead)', | |
1064 | 'ReadMe.txt' => 'Documentation', | |
1065 | 'StandardizedVariants.txt' => 'Certain glyph variations for character display are standardized. This lists the non-Unihan ones; the Unihan ones are also not used by Perl, and are in a separate Unicode data base L<http://www.unicode.org/ivd>', | |
1066 | 'EmojiSources.txt' => 'Maps certain Unicode code points to their legacy Japanese cell-phone values', | |
1067 | 'IndicMatraCategory.txt' => 'Provisional; for the analysis and processing of Indic scripts', | |
1068 | 'IndicSyllabicCategory.txt' => 'Provisional; for the analysis and processing of Indic scripts', | |
1069 | 'auxiliary/WordBreakTest.html' => 'Documentation of validation tests', | |
1070 | 'auxiliary/SentenceBreakTest.html' => 'Documentation of validation tests', | |
1071 | 'auxiliary/GraphemeBreakTest.html' => 'Documentation of validation tests', | |
1072 | 'auxiliary/LineBreakTest.html' => 'Documentation of validation tests', | |
99870f4d KW |
1073 | ); |
1074 | ||
678f13d5 | 1075 | ### End of externally interesting definitions, except for @input_file_objects |
99870f4d KW |
1076 | |
1077 | my $HEADER=<<"EOF"; | |
1078 | # !!!!!!! DO NOT EDIT THIS FILE !!!!!!! | |
3df51b85 KW |
1079 | # This file is machine-generated by $0 from the Unicode |
1080 | # database, Version $string_version. Any changes made here will be lost! | |
cf25bb62 JH |
1081 | EOF |
1082 | ||
126c3d4e | 1083 | my $INTERNAL_ONLY_HEADER = <<"EOF"; |
99870f4d KW |
1084 | |
1085 | # !!!!!!! INTERNAL PERL USE ONLY !!!!!!! | |
fac53429 KW |
1086 | # This file is for internal use by core Perl only. The format and even the |
1087 | # name or existence of this file are subject to change without notice. Don't | |
1088 | # use it directly. | |
99870f4d KW |
1089 | EOF |
1090 | ||
1091 | my $DEVELOPMENT_ONLY=<<"EOF"; | |
1092 | # !!!!!!! DEVELOPMENT USE ONLY !!!!!!! | |
1093 | # This file contains information artificially constrained to code points | |
1094 | # present in Unicode release $string_compare_versions. | |
1095 | # IT CANNOT BE RELIED ON. It is for use during development only and should | |
23e33b60 | 1096 | # not be used for production. |
b6922eda KW |
1097 | |
1098 | EOF | |
1099 | ||
6189eadc KW |
1100 | my $MAX_UNICODE_CODEPOINT_STRING = "10FFFF"; |
1101 | my $MAX_UNICODE_CODEPOINT = hex $MAX_UNICODE_CODEPOINT_STRING; | |
1102 | my $MAX_UNICODE_CODEPOINTS = $MAX_UNICODE_CODEPOINT + 1; | |
99870f4d KW |
1103 | |
1104 | # Matches legal code point. 4-6 hex numbers, If there are 6, the first | |
1105 | # two must be 10; if there are 5, the first must not be a 0. Written this way | |
92199589 KW |
1106 | # to decrease backtracking. The first regex allows the code point to be at |
1107 | # the end of a word, but to work properly, the word shouldn't end with a valid | |
1108 | # hex character. The second one won't match a code point at the end of a | |
1109 | # word, and doesn't have the run-on issue | |
8c32d378 KW |
1110 | my $run_on_code_point_re = |
1111 | qr/ (?: 10[0-9A-F]{4} | [1-9A-F][0-9A-F]{4} | [0-9A-F]{4} ) \b/x; | |
1112 | my $code_point_re = qr/\b$run_on_code_point_re/; | |
99870f4d KW |
1113 | |
1114 | # This matches the beginning of the line in the Unicode db files that give the | |
1115 | # defaults for code points not listed (i.e., missing) in the file. The code | |
1116 | # depends on this ending with a semi-colon, so it can assume it is a valid | |
1117 | # field when the line is split() by semi-colons | |
1118 | my $missing_defaults_prefix = | |
6189eadc | 1119 | qr/^#\s+\@missing:\s+0000\.\.$MAX_UNICODE_CODEPOINT_STRING\s*;/; |
99870f4d KW |
1120 | |
1121 | # Property types. Unicode has more types, but these are sufficient for our | |
1122 | # purposes. | |
1123 | my $UNKNOWN = -1; # initialized to illegal value | |
1124 | my $NON_STRING = 1; # Either binary or enum | |
1125 | my $BINARY = 2; | |
06f26c45 KW |
1126 | my $FORCED_BINARY = 3; # Not a binary property, but, besides its normal |
1127 | # tables, additional true and false tables are | |
1128 | # generated so that false is anything matching the | |
1129 | # default value, and true is everything else. | |
1130 | my $ENUM = 4; # Include catalog | |
1131 | my $STRING = 5; # Anything else: string or misc | |
99870f4d KW |
1132 | |
1133 | # Some input files have lines that give default values for code points not | |
1134 | # contained in the file. Sometimes these should be ignored. | |
1135 | my $NO_DEFAULTS = 0; # Must evaluate to false | |
1136 | my $NOT_IGNORED = 1; | |
1137 | my $IGNORED = 2; | |
1138 | ||
1139 | # Range types. Each range has a type. Most ranges are type 0, for normal, | |
1140 | # and will appear in the main body of the tables in the output files, but | |
1141 | # there are other types of ranges as well, listed below, that are specially | |
1142 | # handled. There are pseudo-types as well that will never be stored as a | |
1143 | # type, but will affect the calculation of the type. | |
1144 | ||
1145 | # 0 is for normal, non-specials | |
1146 | my $MULTI_CP = 1; # Sequence of more than code point | |
1147 | my $HANGUL_SYLLABLE = 2; | |
1148 | my $CP_IN_NAME = 3; # The NAME contains the code point appended to it. | |
1149 | my $NULL = 4; # The map is to the null string; utf8.c can't | |
1150 | # handle these, nor is there an accepted syntax | |
1151 | # for them in \p{} constructs | |
f86864ac | 1152 | my $COMPUTE_NO_MULTI_CP = 5; # Pseudo-type; means that ranges that would |
99870f4d KW |
1153 | # otherwise be $MULTI_CP type are instead type 0 |
1154 | ||
1155 | # process_generic_property_file() can accept certain overrides in its input. | |
1156 | # Each of these must begin AND end with $CMD_DELIM. | |
1157 | my $CMD_DELIM = "\a"; | |
1158 | my $REPLACE_CMD = 'replace'; # Override the Replace | |
1159 | my $MAP_TYPE_CMD = 'map_type'; # Override the Type | |
1160 | ||
1161 | my $NO = 0; | |
1162 | my $YES = 1; | |
1163 | ||
1164 | # Values for the Replace argument to add_range. | |
1165 | # $NO # Don't replace; add only the code points not | |
1166 | # already present. | |
1167 | my $IF_NOT_EQUIVALENT = 1; # Replace only under certain conditions; details in | |
1168 | # the comments at the subroutine definition. | |
1169 | my $UNCONDITIONALLY = 2; # Replace without conditions. | |
1170 | my $MULTIPLE = 4; # Don't replace, but add a duplicate record if | |
1171 | # already there | |
56343c78 | 1172 | my $CROAK = 5; # Die with an error if is already there |
99870f4d KW |
1173 | |
1174 | # Flags to give property statuses. The phrases are to remind maintainers that | |
1175 | # if the flag is changed, the indefinite article referring to it in the | |
1176 | # documentation may need to be as well. | |
1177 | my $NORMAL = ""; | |
1178 | my $SUPPRESSED = 'z'; # The character should never actually be seen, since | |
1179 | # it is suppressed | |
c12f2655 KW |
1180 | my $PLACEHOLDER = 'P'; # A property that is defined as a placeholder in a |
1181 | # Unicode version that doesn't have it, but we need it | |
1182 | # to be defined, if empty, to have things work. | |
1183 | # Implies no pod entry generated | |
99870f4d KW |
1184 | my $DEPRECATED = 'D'; |
1185 | my $a_bold_deprecated = "a 'B<$DEPRECATED>'"; | |
1186 | my $A_bold_deprecated = "A 'B<$DEPRECATED>'"; | |
1187 | my $DISCOURAGED = 'X'; | |
1188 | my $a_bold_discouraged = "an 'B<$DISCOURAGED>'"; | |
1189 | my $A_bold_discouraged = "An 'B<$DISCOURAGED>'"; | |
1190 | my $STRICTER = 'T'; | |
1191 | my $a_bold_stricter = "a 'B<$STRICTER>'"; | |
1192 | my $A_bold_stricter = "A 'B<$STRICTER>'"; | |
1193 | my $STABILIZED = 'S'; | |
1194 | my $a_bold_stabilized = "an 'B<$STABILIZED>'"; | |
1195 | my $A_bold_stabilized = "An 'B<$STABILIZED>'"; | |
1196 | my $OBSOLETE = 'O'; | |
1197 | my $a_bold_obsolete = "an 'B<$OBSOLETE>'"; | |
1198 | my $A_bold_obsolete = "An 'B<$OBSOLETE>'"; | |
1199 | ||
1200 | my %status_past_participles = ( | |
1201 | $DISCOURAGED => 'discouraged', | |
1202 | $SUPPRESSED => 'should never be generated', | |
1203 | $STABILIZED => 'stabilized', | |
1204 | $OBSOLETE => 'obsolete', | |
37e2e78e | 1205 | $DEPRECATED => 'deprecated', |
99870f4d KW |
1206 | ); |
1207 | ||
f5817e0a KW |
1208 | # The format of the values of the tables: |
1209 | my $EMPTY_FORMAT = ""; | |
99870f4d KW |
1210 | my $BINARY_FORMAT = 'b'; |
1211 | my $DECIMAL_FORMAT = 'd'; | |
1212 | my $FLOAT_FORMAT = 'f'; | |
1213 | my $INTEGER_FORMAT = 'i'; | |
1214 | my $HEX_FORMAT = 'x'; | |
1215 | my $RATIONAL_FORMAT = 'r'; | |
1216 | my $STRING_FORMAT = 's'; | |
a14f3cb1 | 1217 | my $DECOMP_STRING_FORMAT = 'c'; |
c3ff2976 | 1218 | my $STRING_WHITE_SPACE_LIST = 'sw'; |
99870f4d KW |
1219 | |
1220 | my %map_table_formats = ( | |
1221 | $BINARY_FORMAT => 'binary', | |
1222 | $DECIMAL_FORMAT => 'single decimal digit', | |
1223 | $FLOAT_FORMAT => 'floating point number', | |
1224 | $INTEGER_FORMAT => 'integer', | |
add63c13 | 1225 | $HEX_FORMAT => 'non-negative hex whole number; a code point', |
99870f4d | 1226 | $RATIONAL_FORMAT => 'rational: an integer or a fraction', |
1a9d544b | 1227 | $STRING_FORMAT => 'string', |
92f9d56c | 1228 | $DECOMP_STRING_FORMAT => 'Perl\'s internal (Normalize.pm) decomposition mapping', |
c3ff2976 | 1229 | $STRING_WHITE_SPACE_LIST => 'string, but some elements are interpreted as a list; white space occurs only as list item separators' |
99870f4d KW |
1230 | ); |
1231 | ||
1232 | # Unicode didn't put such derived files in a separate directory at first. | |
1233 | my $EXTRACTED_DIR = (-d 'extracted') ? 'extracted' : ""; | |
1234 | my $EXTRACTED = ($EXTRACTED_DIR) ? "$EXTRACTED_DIR/" : ""; | |
1235 | my $AUXILIARY = 'auxiliary'; | |
1236 | ||
1237 | # Hashes that will eventually go into Heavy.pl for the use of utf8_heavy.pl | |
1238 | my %loose_to_file_of; # loosely maps table names to their respective | |
1239 | # files | |
1240 | my %stricter_to_file_of; # same; but for stricter mapping. | |
1241 | my %nv_floating_to_rational; # maps numeric values floating point numbers to | |
1242 | # their rational equivalent | |
c12f2655 KW |
1243 | my %loose_property_name_of; # Loosely maps (non_string) property names to |
1244 | # standard form | |
99870f4d | 1245 | |
d867ccfb KW |
1246 | # Most properties are immune to caseless matching, otherwise you would get |
1247 | # nonsensical results, as properties are a function of a code point, not | |
1248 | # everything that is caselessly equivalent to that code point. For example, | |
1249 | # Changes_When_Case_Folded('s') should be false, whereas caselessly it would | |
1250 | # be true because 's' and 'S' are equivalent caselessly. However, | |
1251 | # traditionally, [:upper:] and [:lower:] are equivalent caselessly, so we | |
1252 | # extend that concept to those very few properties that are like this. Each | |
1253 | # such property will match the full range caselessly. They are hard-coded in | |
1254 | # the program; it's not worth trying to make it general as it's extremely | |
1255 | # unlikely that they will ever change. | |
1256 | my %caseless_equivalent_to; | |
1257 | ||
99870f4d KW |
1258 | # These constants names and values were taken from the Unicode standard, |
1259 | # version 5.1, section 3.12. They are used in conjunction with Hangul | |
6e5a209b KW |
1260 | # syllables. The '_string' versions are so generated tables can retain the |
1261 | # hex format, which is the more familiar value | |
1262 | my $SBase_string = "0xAC00"; | |
1263 | my $SBase = CORE::hex $SBase_string; | |
1264 | my $LBase_string = "0x1100"; | |
1265 | my $LBase = CORE::hex $LBase_string; | |
1266 | my $VBase_string = "0x1161"; | |
1267 | my $VBase = CORE::hex $VBase_string; | |
1268 | my $TBase_string = "0x11A7"; | |
1269 | my $TBase = CORE::hex $TBase_string; | |
99870f4d KW |
1270 | my $SCount = 11172; |
1271 | my $LCount = 19; | |
1272 | my $VCount = 21; | |
1273 | my $TCount = 28; | |
1274 | my $NCount = $VCount * $TCount; | |
1275 | ||
1276 | # For Hangul syllables; These store the numbers from Jamo.txt in conjunction | |
1277 | # with the above published constants. | |
1278 | my %Jamo; | |
1279 | my %Jamo_L; # Leading consonants | |
1280 | my %Jamo_V; # Vowels | |
1281 | my %Jamo_T; # Trailing consonants | |
1282 | ||
bb1dd3da KW |
1283 | # For code points whose name contains its ordinal as a '-ABCD' suffix. |
1284 | # The key is the base name of the code point, and the value is an | |
1285 | # array giving all the ranges that use this base name. Each range | |
1286 | # is actually a hash giving the 'low' and 'high' values of it. | |
1287 | my %names_ending_in_code_point; | |
1288 | my %loose_names_ending_in_code_point; # Same as above, but has blanks, dashes | |
1289 | # removed from the names | |
1290 | # Inverse mapping. The list of ranges that have these kinds of | |
1291 | # names. Each element contains the low, high, and base names in an | |
1292 | # anonymous hash. | |
1293 | my @code_points_ending_in_code_point; | |
1294 | ||
1295 | # Boolean: does this Unicode version have the hangul syllables, and are we | |
1296 | # writing out a table for them? | |
1297 | my $has_hangul_syllables = 0; | |
1298 | ||
1299 | # Does this Unicode version have code points whose names end in their | |
1300 | # respective code points, and are we writing out a table for them? 0 for no; | |
1301 | # otherwise points to first property that a table is needed for them, so that | |
1302 | # if multiple tables are needed, we don't create duplicates | |
1303 | my $needing_code_points_ending_in_code_point = 0; | |
1304 | ||
37e2e78e | 1305 | my @backslash_X_tests; # List of tests read in for testing \X |
99870f4d KW |
1306 | my @unhandled_properties; # Will contain a list of properties found in |
1307 | # the input that we didn't process. | |
f86864ac | 1308 | my @match_properties; # Properties that have match tables, to be |
99870f4d KW |
1309 | # listed in the pod |
1310 | my @map_properties; # Properties that get map files written | |
1311 | my @named_sequences; # NamedSequences.txt contents. | |
1312 | my %potential_files; # Generated list of all .txt files in the directory | |
1313 | # structure so we can warn if something is being | |
1314 | # ignored. | |
1315 | my @files_actually_output; # List of files we generated. | |
1316 | my @more_Names; # Some code point names are compound; this is used | |
1317 | # to store the extra components of them. | |
1318 | my $MIN_FRACTION_LENGTH = 3; # How many digits of a floating point number at | |
1319 | # the minimum before we consider it equivalent to a | |
1320 | # candidate rational | |
1321 | my $MAX_FLOATING_SLOP = 10 ** - $MIN_FRACTION_LENGTH; # And in floating terms | |
1322 | ||
1323 | # These store references to certain commonly used property objects | |
1324 | my $gc; | |
1325 | my $perl; | |
1326 | my $block; | |
3e20195b KW |
1327 | my $perl_charname; |
1328 | my $print; | |
7fc6cb55 | 1329 | my $Any; |
359523e2 | 1330 | my $script; |
99870f4d KW |
1331 | |
1332 | # Are there conflicting names because of beginning with 'In_', or 'Is_' | |
1333 | my $has_In_conflicts = 0; | |
1334 | my $has_Is_conflicts = 0; | |
1335 | ||
1336 | sub internal_file_to_platform ($) { | |
1337 | # Convert our file paths which have '/' separators to those of the | |
1338 | # platform. | |
1339 | ||
1340 | my $file = shift; | |
1341 | return undef unless defined $file; | |
1342 | ||
1343 | return File::Spec->join(split '/', $file); | |
d07a55ed | 1344 | } |
5beb625e | 1345 | |
99870f4d KW |
1346 | sub file_exists ($) { # platform independent '-e'. This program internally |
1347 | # uses slash as a path separator. | |
1348 | my $file = shift; | |
1349 | return 0 if ! defined $file; | |
1350 | return -e internal_file_to_platform($file); | |
1351 | } | |
5beb625e | 1352 | |
99870f4d | 1353 | sub objaddr($) { |
23e33b60 KW |
1354 | # Returns the address of the blessed input object. |
1355 | # It doesn't check for blessedness because that would do a string eval | |
1356 | # every call, and the program is structured so that this is never called | |
1357 | # for a non-blessed object. | |
99870f4d | 1358 | |
23e33b60 | 1359 | no overloading; # If overloaded, numifying below won't work. |
99870f4d KW |
1360 | |
1361 | # Numifying a ref gives its address. | |
051df77b | 1362 | return pack 'J', $_[0]; |
99870f4d KW |
1363 | } |
1364 | ||
558712cf | 1365 | # These are used only if $annotate is true. |
c4019d52 KW |
1366 | # The entire range of Unicode characters is examined to populate these |
1367 | # after all the input has been processed. But most can be skipped, as they | |
1368 | # have the same descriptive phrases, such as being unassigned | |
1369 | my @viacode; # Contains the 1 million character names | |
1370 | my @printable; # boolean: And are those characters printable? | |
1371 | my @annotate_char_type; # Contains a type of those characters, specifically | |
1372 | # for the purposes of annotation. | |
1373 | my $annotate_ranges; # A map of ranges of code points that have the same | |
98dc9551 | 1374 | # name for the purposes of annotation. They map to the |
c4019d52 KW |
1375 | # upper edge of the range, so that the end point can |
1376 | # be immediately found. This is used to skip ahead to | |
1377 | # the end of a range, and avoid processing each | |
1378 | # individual code point in it. | |
1379 | my $unassigned_sans_noncharacters; # A Range_List of the unassigned | |
1380 | # characters, but excluding those which are | |
1381 | # also noncharacter code points | |
1382 | ||
1383 | # The annotation types are an extension of the regular range types, though | |
1384 | # some of the latter are folded into one. Make the new types negative to | |
1385 | # avoid conflicting with the regular types | |
1386 | my $SURROGATE_TYPE = -1; | |
1387 | my $UNASSIGNED_TYPE = -2; | |
1388 | my $PRIVATE_USE_TYPE = -3; | |
1389 | my $NONCHARACTER_TYPE = -4; | |
1390 | my $CONTROL_TYPE = -5; | |
1391 | my $UNKNOWN_TYPE = -6; # Used only if there is a bug in this program | |
1392 | ||
1393 | sub populate_char_info ($) { | |
558712cf | 1394 | # Used only with the $annotate option. Populates the arrays with the |
c4019d52 KW |
1395 | # input code point's info that are needed for outputting more detailed |
1396 | # comments. If calling context wants a return, it is the end point of | |
1397 | # any contiguous range of characters that share essentially the same info | |
1398 | ||
1399 | my $i = shift; | |
1400 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
1401 | ||
1402 | $viacode[$i] = $perl_charname->value_of($i) || ""; | |
1403 | ||
1404 | # A character is generally printable if Unicode says it is, | |
1405 | # but below we make sure that most Unicode general category 'C' types | |
1406 | # aren't. | |
1407 | $printable[$i] = $print->contains($i); | |
1408 | ||
1409 | $annotate_char_type[$i] = $perl_charname->type_of($i) || 0; | |
1410 | ||
1411 | # Only these two regular types are treated specially for annotations | |
1412 | # purposes | |
1413 | $annotate_char_type[$i] = 0 if $annotate_char_type[$i] != $CP_IN_NAME | |
1414 | && $annotate_char_type[$i] != $HANGUL_SYLLABLE; | |
1415 | ||
1416 | # Give a generic name to all code points that don't have a real name. | |
1417 | # We output ranges, if applicable, for these. Also calculate the end | |
1418 | # point of the range. | |
1419 | my $end; | |
1420 | if (! $viacode[$i]) { | |
1421 | if ($gc-> table('Surrogate')->contains($i)) { | |
1422 | $viacode[$i] = 'Surrogate'; | |
1423 | $annotate_char_type[$i] = $SURROGATE_TYPE; | |
1424 | $printable[$i] = 0; | |
1425 | $end = $gc->table('Surrogate')->containing_range($i)->end; | |
1426 | } | |
1427 | elsif ($gc-> table('Private_use')->contains($i)) { | |
1428 | $viacode[$i] = 'Private Use'; | |
1429 | $annotate_char_type[$i] = $PRIVATE_USE_TYPE; | |
1430 | $printable[$i] = 0; | |
1431 | $end = $gc->table('Private_Use')->containing_range($i)->end; | |
1432 | } | |
1433 | elsif (Property::property_ref('Noncharacter_Code_Point')-> table('Y')-> | |
1434 | contains($i)) | |
1435 | { | |
1436 | $viacode[$i] = 'Noncharacter'; | |
1437 | $annotate_char_type[$i] = $NONCHARACTER_TYPE; | |
1438 | $printable[$i] = 0; | |
1439 | $end = property_ref('Noncharacter_Code_Point')->table('Y')-> | |
1440 | containing_range($i)->end; | |
1441 | } | |
1442 | elsif ($gc-> table('Control')->contains($i)) { | |
1443 | $viacode[$i] = 'Control'; | |
1444 | $annotate_char_type[$i] = $CONTROL_TYPE; | |
1445 | $printable[$i] = 0; | |
1446 | $end = 0x81 if $i == 0x80; # Hard-code this one known case | |
1447 | } | |
1448 | elsif ($gc-> table('Unassigned')->contains($i)) { | |
1449 | $viacode[$i] = 'Unassigned, block=' . $block-> value_of($i); | |
1450 | $annotate_char_type[$i] = $UNASSIGNED_TYPE; | |
1451 | $printable[$i] = 0; | |
1452 | ||
1453 | # Because we name the unassigned by the blocks they are in, it | |
1454 | # can't go past the end of that block, and it also can't go past | |
1455 | # the unassigned range it is in. The special table makes sure | |
1456 | # that the non-characters, which are unassigned, are separated | |
1457 | # out. | |
1458 | $end = min($block->containing_range($i)->end, | |
1459 | $unassigned_sans_noncharacters-> containing_range($i)-> | |
1460 | end); | |
13ca76ff KW |
1461 | } |
1462 | else { | |
1463 | Carp::my_carp_bug("Can't figure out how to annotate " | |
1464 | . sprintf("U+%04X", $i) | |
1465 | . ". Proceeding anyway."); | |
c4019d52 KW |
1466 | $viacode[$i] = 'UNKNOWN'; |
1467 | $annotate_char_type[$i] = $UNKNOWN_TYPE; | |
1468 | $printable[$i] = 0; | |
1469 | } | |
1470 | } | |
1471 | ||
1472 | # Here, has a name, but if it's one in which the code point number is | |
1473 | # appended to the name, do that. | |
1474 | elsif ($annotate_char_type[$i] == $CP_IN_NAME) { | |
1475 | $viacode[$i] .= sprintf("-%04X", $i); | |
1476 | $end = $perl_charname->containing_range($i)->end; | |
1477 | } | |
1478 | ||
1479 | # And here, has a name, but if it's a hangul syllable one, replace it with | |
1480 | # the correct name from the Unicode algorithm | |
1481 | elsif ($annotate_char_type[$i] == $HANGUL_SYLLABLE) { | |
1482 | use integer; | |
1483 | my $SIndex = $i - $SBase; | |
1484 | my $L = $LBase + $SIndex / $NCount; | |
1485 | my $V = $VBase + ($SIndex % $NCount) / $TCount; | |
1486 | my $T = $TBase + $SIndex % $TCount; | |
1487 | $viacode[$i] = "HANGUL SYLLABLE $Jamo{$L}$Jamo{$V}"; | |
1488 | $viacode[$i] .= $Jamo{$T} if $T != $TBase; | |
1489 | $end = $perl_charname->containing_range($i)->end; | |
1490 | } | |
1491 | ||
1492 | return if ! defined wantarray; | |
1493 | return $i if ! defined $end; # If not a range, return the input | |
1494 | ||
1495 | # Save this whole range so can find the end point quickly | |
1496 | $annotate_ranges->add_map($i, $end, $end); | |
1497 | ||
1498 | return $end; | |
1499 | } | |
1500 | ||
23e33b60 KW |
1501 | # Commented code below should work on Perl 5.8. |
1502 | ## This 'require' doesn't necessarily work in miniperl, and even if it does, | |
1503 | ## the native perl version of it (which is what would operate under miniperl) | |
1504 | ## is extremely slow, as it does a string eval every call. | |
1505 | #my $has_fast_scalar_util = $\18 !~ /miniperl/ | |
1506 | # && defined eval "require Scalar::Util"; | |
1507 | # | |
1508 | #sub objaddr($) { | |
1509 | # # Returns the address of the blessed input object. Uses the XS version if | |
1510 | # # available. It doesn't check for blessedness because that would do a | |
1511 | # # string eval every call, and the program is structured so that this is | |
1512 | # # never called for a non-blessed object. | |
1513 | # | |
1514 | # return Scalar::Util::refaddr($_[0]) if $has_fast_scalar_util; | |
1515 | # | |
1516 | # # Check at least that is a ref. | |
1517 | # my $pkg = ref($_[0]) or return undef; | |
1518 | # | |
1519 | # # Change to a fake package to defeat any overloaded stringify | |
1520 | # bless $_[0], 'main::Fake'; | |
1521 | # | |
1522 | # # Numifying a ref gives its address. | |
051df77b | 1523 | # my $addr = pack 'J', $_[0]; |
23e33b60 KW |
1524 | # |
1525 | # # Return to original class | |
1526 | # bless $_[0], $pkg; | |
1527 | # return $addr; | |
1528 | #} | |
1529 | ||
99870f4d KW |
1530 | sub max ($$) { |
1531 | my $a = shift; | |
1532 | my $b = shift; | |
1533 | return $a if $a >= $b; | |
1534 | return $b; | |
1535 | } | |
1536 | ||
1537 | sub min ($$) { | |
1538 | my $a = shift; | |
1539 | my $b = shift; | |
1540 | return $a if $a <= $b; | |
1541 | return $b; | |
1542 | } | |
1543 | ||
1544 | sub clarify_number ($) { | |
1545 | # This returns the input number with underscores inserted every 3 digits | |
1546 | # in large (5 digits or more) numbers. Input must be entirely digits, not | |
1547 | # checked. | |
1548 | ||
1549 | my $number = shift; | |
1550 | my $pos = length($number) - 3; | |
1551 | return $number if $pos <= 1; | |
1552 | while ($pos > 0) { | |
1553 | substr($number, $pos, 0) = '_'; | |
1554 | $pos -= 3; | |
5beb625e | 1555 | } |
99870f4d | 1556 | return $number; |
99598c8c JH |
1557 | } |
1558 | ||
12ac2576 | 1559 | |
99870f4d | 1560 | package Carp; |
7ebf06b3 | 1561 | |
99870f4d KW |
1562 | # These routines give a uniform treatment of messages in this program. They |
1563 | # are placed in the Carp package to cause the stack trace to not include them, | |
1564 | # although an alternative would be to use another package and set @CARP_NOT | |
1565 | # for it. | |
12ac2576 | 1566 | |
99870f4d | 1567 | our $Verbose = 1 if main::DEBUG; # Useful info when debugging |
12ac2576 | 1568 | |
99f78760 KW |
1569 | # This is a work-around suggested by Nicholas Clark to fix a problem with Carp |
1570 | # and overload trying to load Scalar:Util under miniperl. See | |
1571 | # http://www.xray.mpe.mpg.de/mailing-lists/perl5-porters/2009-11/msg01057.html | |
1572 | undef $overload::VERSION; | |
1573 | ||
99870f4d KW |
1574 | sub my_carp { |
1575 | my $message = shift || ""; | |
1576 | my $nofold = shift || 0; | |
7ebf06b3 | 1577 | |
99870f4d KW |
1578 | if ($message) { |
1579 | $message = main::join_lines($message); | |
1580 | $message =~ s/^$0: *//; # Remove initial program name | |
1581 | $message =~ s/[.;,]+$//; # Remove certain ending punctuation | |
1582 | $message = "\n$0: $message;"; | |
12ac2576 | 1583 | |
99870f4d KW |
1584 | # Fold the message with program name, semi-colon end punctuation |
1585 | # (which looks good with the message that carp appends to it), and a | |
1586 | # hanging indent for continuation lines. | |
1587 | $message = main::simple_fold($message, "", 4) unless $nofold; | |
1588 | $message =~ s/\n$//; # Remove the trailing nl so what carp | |
1589 | # appends is to the same line | |
1590 | } | |
12ac2576 | 1591 | |
99870f4d | 1592 | return $message if defined wantarray; # If a caller just wants the msg |
12ac2576 | 1593 | |
99870f4d KW |
1594 | carp $message; |
1595 | return; | |
1596 | } | |
7ebf06b3 | 1597 | |
99870f4d KW |
1598 | sub my_carp_bug { |
1599 | # This is called when it is clear that the problem is caused by a bug in | |
1600 | # this program. | |
7ebf06b3 | 1601 | |
99870f4d KW |
1602 | my $message = shift; |
1603 | $message =~ s/^$0: *//; | |
1604 | $message = my_carp("Bug in $0. Please report it by running perlbug or if that is unavailable, by sending email to perbug\@perl.org:\n$message"); | |
1605 | carp $message; | |
1606 | return; | |
1607 | } | |
7ebf06b3 | 1608 | |
99870f4d KW |
1609 | sub carp_too_few_args { |
1610 | if (@_ != 2) { | |
1611 | my_carp_bug("Wrong number of arguments: to 'carp_too_few_arguments'. No action taken."); | |
1612 | return; | |
12ac2576 | 1613 | } |
7ebf06b3 | 1614 | |
99870f4d KW |
1615 | my $args_ref = shift; |
1616 | my $count = shift; | |
7ebf06b3 | 1617 | |
99870f4d KW |
1618 | my_carp_bug("Need at least $count arguments to " |
1619 | . (caller 1)[3] | |
1620 | . ". Instead got: '" | |
1621 | . join ', ', @$args_ref | |
1622 | . "'. No action taken."); | |
1623 | return; | |
12ac2576 JP |
1624 | } |
1625 | ||
99870f4d KW |
1626 | sub carp_extra_args { |
1627 | my $args_ref = shift; | |
1628 | my_carp_bug("Too many arguments to 'carp_extra_args': (" . join(', ', @_) . "); Extras ignored.") if @_; | |
12ac2576 | 1629 | |
99870f4d KW |
1630 | unless (ref $args_ref) { |
1631 | my_carp_bug("Argument to 'carp_extra_args' ($args_ref) must be a ref. Not checking arguments."); | |
1632 | return; | |
1633 | } | |
1634 | my ($package, $file, $line) = caller; | |
1635 | my $subroutine = (caller 1)[3]; | |
cf25bb62 | 1636 | |
99870f4d KW |
1637 | my $list; |
1638 | if (ref $args_ref eq 'HASH') { | |
1639 | foreach my $key (keys %$args_ref) { | |
1640 | $args_ref->{$key} = $UNDEF unless defined $args_ref->{$key}; | |
cf25bb62 | 1641 | } |
99870f4d | 1642 | $list = join ', ', each %{$args_ref}; |
cf25bb62 | 1643 | } |
99870f4d KW |
1644 | elsif (ref $args_ref eq 'ARRAY') { |
1645 | foreach my $arg (@$args_ref) { | |
1646 | $arg = $UNDEF unless defined $arg; | |
1647 | } | |
1648 | $list = join ', ', @$args_ref; | |
1649 | } | |
1650 | else { | |
1651 | my_carp_bug("Can't cope with ref " | |
1652 | . ref($args_ref) | |
1653 | . " . argument to 'carp_extra_args'. Not checking arguments."); | |
1654 | return; | |
1655 | } | |
1656 | ||
1657 | my_carp_bug("Unrecognized parameters in options: '$list' to $subroutine. Skipped."); | |
1658 | return; | |
d73e5302 JH |
1659 | } |
1660 | ||
99870f4d KW |
1661 | package main; |
1662 | ||
1663 | { # Closure | |
1664 | ||
1665 | # This program uses the inside-out method for objects, as recommended in | |
1666 | # "Perl Best Practices". This closure aids in generating those. There | |
1667 | # are two routines. setup_package() is called once per package to set | |
1668 | # things up, and then set_access() is called for each hash representing a | |
1669 | # field in the object. These routines arrange for the object to be | |
1670 | # properly destroyed when no longer used, and for standard accessor | |
1671 | # functions to be generated. If you need more complex accessors, just | |
1672 | # write your own and leave those accesses out of the call to set_access(). | |
1673 | # More details below. | |
1674 | ||
1675 | my %constructor_fields; # fields that are to be used in constructors; see | |
1676 | # below | |
1677 | ||
1678 | # The values of this hash will be the package names as keys to other | |
1679 | # hashes containing the name of each field in the package as keys, and | |
1680 | # references to their respective hashes as values. | |
1681 | my %package_fields; | |
1682 | ||
1683 | sub setup_package { | |
1684 | # Sets up the package, creating standard DESTROY and dump methods | |
1685 | # (unless already defined). The dump method is used in debugging by | |
1686 | # simple_dumper(). | |
1687 | # The optional parameters are: | |
1688 | # a) a reference to a hash, that gets populated by later | |
1689 | # set_access() calls with one of the accesses being | |
1690 | # 'constructor'. The caller can then refer to this, but it is | |
1691 | # not otherwise used by these two routines. | |
1692 | # b) a reference to a callback routine to call during destruction | |
1693 | # of the object, before any fields are actually destroyed | |
1694 | ||
1695 | my %args = @_; | |
1696 | my $constructor_ref = delete $args{'Constructor_Fields'}; | |
1697 | my $destroy_callback = delete $args{'Destroy_Callback'}; | |
1698 | Carp::carp_extra_args(\@_) if main::DEBUG && %args; | |
1699 | ||
1700 | my %fields; | |
1701 | my $package = (caller)[0]; | |
1702 | ||
1703 | $package_fields{$package} = \%fields; | |
1704 | $constructor_fields{$package} = $constructor_ref; | |
1705 | ||
1706 | unless ($package->can('DESTROY')) { | |
1707 | my $destroy_name = "${package}::DESTROY"; | |
1708 | no strict "refs"; | |
1709 | ||
1710 | # Use typeglob to give the anonymous subroutine the name we want | |
1711 | *$destroy_name = sub { | |
1712 | my $self = shift; | |
ffe43484 | 1713 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
1714 | |
1715 | $self->$destroy_callback if $destroy_callback; | |
1716 | foreach my $field (keys %{$package_fields{$package}}) { | |
1717 | #print STDERR __LINE__, ": Destroying ", ref $self, " ", sprintf("%04X", $addr), ": ", $field, "\n"; | |
1718 | delete $package_fields{$package}{$field}{$addr}; | |
1719 | } | |
1720 | return; | |
1721 | } | |
1722 | } | |
1723 | ||
1724 | unless ($package->can('dump')) { | |
1725 | my $dump_name = "${package}::dump"; | |
1726 | no strict "refs"; | |
1727 | *$dump_name = sub { | |
1728 | my $self = shift; | |
1729 | return dump_inside_out($self, $package_fields{$package}, @_); | |
1730 | } | |
1731 | } | |
1732 | return; | |
1733 | } | |
1734 | ||
1735 | sub set_access { | |
1736 | # Arrange for the input field to be garbage collected when no longer | |
1737 | # needed. Also, creates standard accessor functions for the field | |
1738 | # based on the optional parameters-- none if none of these parameters: | |
1739 | # 'addable' creates an 'add_NAME()' accessor function. | |
1740 | # 'readable' or 'readable_array' creates a 'NAME()' accessor | |
1741 | # function. | |
1742 | # 'settable' creates a 'set_NAME()' accessor function. | |
1743 | # 'constructor' doesn't create an accessor function, but adds the | |
1744 | # field to the hash that was previously passed to | |
1745 | # setup_package(); | |
1746 | # Any of the accesses can be abbreviated down, so that 'a', 'ad', | |
1747 | # 'add' etc. all mean 'addable'. | |
1748 | # The read accessor function will work on both array and scalar | |
1749 | # values. If another accessor in the parameter list is 'a', the read | |
1750 | # access assumes an array. You can also force it to be array access | |
1751 | # by specifying 'readable_array' instead of 'readable' | |
1752 | # | |
1753 | # A sort-of 'protected' access can be set-up by preceding the addable, | |
1754 | # readable or settable with some initial portion of 'protected_' (but, | |
1755 | # the underscore is required), like 'p_a', 'pro_set', etc. The | |
1756 | # "protection" is only by convention. All that happens is that the | |
1757 | # accessor functions' names begin with an underscore. So instead of | |
1758 | # calling set_foo, the call is _set_foo. (Real protection could be | |
c1739a4a | 1759 | # accomplished by having a new subroutine, end_package, called at the |
99870f4d KW |
1760 | # end of each package, and then storing the __LINE__ ranges and |
1761 | # checking them on every accessor. But that is way overkill.) | |
1762 | ||
1763 | # We create anonymous subroutines as the accessors and then use | |
1764 | # typeglobs to assign them to the proper package and name | |
1765 | ||
1766 | my $name = shift; # Name of the field | |
1767 | my $field = shift; # Reference to the inside-out hash containing the | |
1768 | # field | |
1769 | ||
1770 | my $package = (caller)[0]; | |
1771 | ||
1772 | if (! exists $package_fields{$package}) { | |
1773 | croak "$0: Must call 'setup_package' before 'set_access'"; | |
1774 | } | |
d73e5302 | 1775 | |
99870f4d KW |
1776 | # Stash the field so DESTROY can get it. |
1777 | $package_fields{$package}{$name} = $field; | |
cf25bb62 | 1778 | |
99870f4d KW |
1779 | # Remaining arguments are the accessors. For each... |
1780 | foreach my $access (@_) { | |
1781 | my $access = lc $access; | |
cf25bb62 | 1782 | |
99870f4d | 1783 | my $protected = ""; |
cf25bb62 | 1784 | |
99870f4d KW |
1785 | # Match the input as far as it goes. |
1786 | if ($access =~ /^(p[^_]*)_/) { | |
1787 | $protected = $1; | |
1788 | if (substr('protected_', 0, length $protected) | |
1789 | eq $protected) | |
1790 | { | |
1791 | ||
1792 | # Add 1 for the underscore not included in $protected | |
1793 | $access = substr($access, length($protected) + 1); | |
1794 | $protected = '_'; | |
1795 | } | |
1796 | else { | |
1797 | $protected = ""; | |
1798 | } | |
1799 | } | |
1800 | ||
1801 | if (substr('addable', 0, length $access) eq $access) { | |
1802 | my $subname = "${package}::${protected}add_$name"; | |
1803 | no strict "refs"; | |
1804 | ||
1805 | # add_ accessor. Don't add if already there, which we | |
1806 | # determine using 'eq' for scalars and '==' otherwise. | |
1807 | *$subname = sub { | |
1808 | use strict "refs"; | |
1809 | return Carp::carp_too_few_args(\@_, 2) if main::DEBUG && @_ < 2; | |
1810 | my $self = shift; | |
1811 | my $value = shift; | |
ffe43484 | 1812 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
1813 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; |
1814 | if (ref $value) { | |
f998e60c | 1815 | return if grep { $value == $_ } @{$field->{$addr}}; |
99870f4d KW |
1816 | } |
1817 | else { | |
f998e60c | 1818 | return if grep { $value eq $_ } @{$field->{$addr}}; |
99870f4d | 1819 | } |
f998e60c | 1820 | push @{$field->{$addr}}, $value; |
99870f4d KW |
1821 | return; |
1822 | } | |
1823 | } | |
1824 | elsif (substr('constructor', 0, length $access) eq $access) { | |
1825 | if ($protected) { | |
1826 | Carp::my_carp_bug("Can't set-up 'protected' constructors") | |
1827 | } | |
1828 | else { | |
1829 | $constructor_fields{$package}{$name} = $field; | |
1830 | } | |
1831 | } | |
1832 | elsif (substr('readable_array', 0, length $access) eq $access) { | |
1833 | ||
1834 | # Here has read access. If one of the other parameters for | |
1835 | # access is array, or this one specifies array (by being more | |
1836 | # than just 'readable_'), then create a subroutine that | |
1837 | # assumes the data is an array. Otherwise just a scalar | |
1838 | my $subname = "${package}::${protected}$name"; | |
1839 | if (grep { /^a/i } @_ | |
1840 | or length($access) > length('readable_')) | |
1841 | { | |
1842 | no strict "refs"; | |
1843 | *$subname = sub { | |
1844 | use strict "refs"; | |
23e33b60 | 1845 | Carp::carp_extra_args(\@_) if main::DEBUG && @_ > 1; |
ffe43484 | 1846 | my $addr = do { no overloading; pack 'J', $_[0]; }; |
99870f4d KW |
1847 | if (ref $field->{$addr} ne 'ARRAY') { |
1848 | my $type = ref $field->{$addr}; | |
1849 | $type = 'scalar' unless $type; | |
1850 | Carp::my_carp_bug("Trying to read $name as an array when it is a $type. Big problems."); | |
1851 | return; | |
1852 | } | |
1853 | return scalar @{$field->{$addr}} unless wantarray; | |
1854 | ||
1855 | # Make a copy; had problems with caller modifying the | |
1856 | # original otherwise | |
1857 | my @return = @{$field->{$addr}}; | |
1858 | return @return; | |
1859 | } | |
1860 | } | |
1861 | else { | |
1862 | ||
1863 | # Here not an array value, a simpler function. | |
1864 | no strict "refs"; | |
1865 | *$subname = sub { | |
1866 | use strict "refs"; | |
23e33b60 | 1867 | Carp::carp_extra_args(\@_) if main::DEBUG && @_ > 1; |
f998e60c | 1868 | no overloading; |
051df77b | 1869 | return $field->{pack 'J', $_[0]}; |
99870f4d KW |
1870 | } |
1871 | } | |
1872 | } | |
1873 | elsif (substr('settable', 0, length $access) eq $access) { | |
1874 | my $subname = "${package}::${protected}set_$name"; | |
1875 | no strict "refs"; | |
1876 | *$subname = sub { | |
1877 | use strict "refs"; | |
23e33b60 KW |
1878 | if (main::DEBUG) { |
1879 | return Carp::carp_too_few_args(\@_, 2) if @_ < 2; | |
1880 | Carp::carp_extra_args(\@_) if @_ > 2; | |
1881 | } | |
1882 | # $self is $_[0]; $value is $_[1] | |
f998e60c | 1883 | no overloading; |
051df77b | 1884 | $field->{pack 'J', $_[0]} = $_[1]; |
99870f4d KW |
1885 | return; |
1886 | } | |
1887 | } | |
1888 | else { | |
1889 | Carp::my_carp_bug("Unknown accessor type $access. No accessor set."); | |
1890 | } | |
cf25bb62 | 1891 | } |
99870f4d | 1892 | return; |
cf25bb62 | 1893 | } |
99870f4d KW |
1894 | } |
1895 | ||
1896 | package Input_file; | |
1897 | ||
1898 | # All input files use this object, which stores various attributes about them, | |
1899 | # and provides for convenient, uniform handling. The run method wraps the | |
1900 | # processing. It handles all the bookkeeping of opening, reading, and closing | |
1901 | # the file, returning only significant input lines. | |
1902 | # | |
1903 | # Each object gets a handler which processes the body of the file, and is | |
1904 | # called by run(). Most should use the generic, default handler, which has | |
1905 | # code scrubbed to handle things you might not expect. A handler should | |
1906 | # basically be a while(next_line()) {...} loop. | |
1907 | # | |
1908 | # You can also set up handlers to | |
1909 | # 1) call before the first line is read for pre processing | |
1910 | # 2) call to adjust each line of the input before the main handler gets them | |
1911 | # 3) call upon EOF before the main handler exits its loop | |
1912 | # 4) call at the end for post processing | |
1913 | # | |
1914 | # $_ is used to store the input line, and is to be filtered by the | |
1915 | # each_line_handler()s. So, if the format of the line is not in the desired | |
1916 | # format for the main handler, these are used to do that adjusting. They can | |
1917 | # be stacked (by enclosing them in an [ anonymous array ] in the constructor, | |
1918 | # so the $_ output of one is used as the input to the next. None of the other | |
1919 | # handlers are stackable, but could easily be changed to be so. | |
1920 | # | |
1921 | # Most of the handlers can call insert_lines() or insert_adjusted_lines() | |
1922 | # which insert the parameters as lines to be processed before the next input | |
1923 | # file line is read. This allows the EOF handler to flush buffers, for | |
1924 | # example. The difference between the two routines is that the lines inserted | |
1925 | # by insert_lines() are subjected to the each_line_handler()s. (So if you | |
1926 | # called it from such a handler, you would get infinite recursion.) Lines | |
1927 | # inserted by insert_adjusted_lines() go directly to the main handler without | |
1928 | # any adjustments. If the post-processing handler calls any of these, there | |
1929 | # will be no effect. Some error checking for these conditions could be added, | |
1930 | # but it hasn't been done. | |
1931 | # | |
1932 | # carp_bad_line() should be called to warn of bad input lines, which clears $_ | |
1933 | # to prevent further processing of the line. This routine will output the | |
1934 | # message as a warning once, and then keep a count of the lines that have the | |
1935 | # same message, and output that count at the end of the file's processing. | |
1936 | # This keeps the number of messages down to a manageable amount. | |
1937 | # | |
1938 | # get_missings() should be called to retrieve any @missing input lines. | |
1939 | # Messages will be raised if this isn't done if the options aren't to ignore | |
1940 | # missings. | |
1941 | ||
1942 | sub trace { return main::trace(@_); } | |
1943 | ||
99870f4d KW |
1944 | { # Closure |
1945 | # Keep track of fields that are to be put into the constructor. | |
1946 | my %constructor_fields; | |
1947 | ||
1948 | main::setup_package(Constructor_Fields => \%constructor_fields); | |
1949 | ||
1950 | my %file; # Input file name, required | |
1951 | main::set_access('file', \%file, qw{ c r }); | |
1952 | ||
1953 | my %first_released; # Unicode version file was first released in, required | |
1954 | main::set_access('first_released', \%first_released, qw{ c r }); | |
1955 | ||
1956 | my %handler; # Subroutine to process the input file, defaults to | |
1957 | # 'process_generic_property_file' | |
1958 | main::set_access('handler', \%handler, qw{ c }); | |
1959 | ||
1960 | my %property; | |
1961 | # name of property this file is for. defaults to none, meaning not | |
1962 | # applicable, or is otherwise determinable, for example, from each line. | |
1963 | main::set_access('property', \%property, qw{ c }); | |
1964 | ||
1965 | my %optional; | |
1966 | # If this is true, the file is optional. If not present, no warning is | |
1967 | # output. If it is present, the string given by this parameter is | |
1968 | # evaluated, and if false the file is not processed. | |
1969 | main::set_access('optional', \%optional, 'c', 'r'); | |
1970 | ||
1971 | my %non_skip; | |
1972 | # This is used for debugging, to skip processing of all but a few input | |
1973 | # files. Add 'non_skip => 1' to the constructor for those files you want | |
1974 | # processed when you set the $debug_skip global. | |
1975 | main::set_access('non_skip', \%non_skip, 'c'); | |
1976 | ||
37e2e78e | 1977 | my %skip; |
09ca89ce KW |
1978 | # This is used to skip processing of this input file semi-permanently, |
1979 | # when it evaluates to true. The value should be the reason the file is | |
1980 | # being skipped. It is used for files that we aren't planning to process | |
1981 | # anytime soon, but want to allow to be in the directory and not raise a | |
1982 | # message that we are not handling. Mostly for test files. This is in | |
1983 | # contrast to the non_skip element, which is supposed to be used very | |
1984 | # temporarily for debugging. Sets 'optional' to 1. Also, files that we | |
1985 | # pretty much will never look at can be placed in the global | |
1986 | # %ignored_files instead. Ones used here will be added to that list. | |
37e2e78e KW |
1987 | main::set_access('skip', \%skip, 'c'); |
1988 | ||
99870f4d KW |
1989 | my %each_line_handler; |
1990 | # list of subroutines to look at and filter each non-comment line in the | |
1991 | # file. defaults to none. The subroutines are called in order, each is | |
1992 | # to adjust $_ for the next one, and the final one adjusts it for | |
1993 | # 'handler' | |
1994 | main::set_access('each_line_handler', \%each_line_handler, 'c'); | |
1995 | ||
1996 | my %has_missings_defaults; | |
1997 | # ? Are there lines in the file giving default values for code points | |
1998 | # missing from it?. Defaults to NO_DEFAULTS. Otherwise NOT_IGNORED is | |
1999 | # the norm, but IGNORED means it has such lines, but the handler doesn't | |
2000 | # use them. Having these three states allows us to catch changes to the | |
2001 | # UCD that this program should track | |
2002 | main::set_access('has_missings_defaults', | |
2003 | \%has_missings_defaults, qw{ c r }); | |
2004 | ||
2005 | my %pre_handler; | |
2006 | # Subroutine to call before doing anything else in the file. If undef, no | |
2007 | # such handler is called. | |
2008 | main::set_access('pre_handler', \%pre_handler, qw{ c }); | |
2009 | ||
2010 | my %eof_handler; | |
2011 | # Subroutine to call upon getting an EOF on the input file, but before | |
2012 | # that is returned to the main handler. This is to allow buffers to be | |
2013 | # flushed. The handler is expected to call insert_lines() or | |
2014 | # insert_adjusted() with the buffered material | |
2015 | main::set_access('eof_handler', \%eof_handler, qw{ c r }); | |
2016 | ||
2017 | my %post_handler; | |
2018 | # Subroutine to call after all the lines of the file are read in and | |
2019 | # processed. If undef, no such handler is called. | |
2020 | main::set_access('post_handler', \%post_handler, qw{ c }); | |
2021 | ||
2022 | my %progress_message; | |
2023 | # Message to print to display progress in lieu of the standard one | |
2024 | main::set_access('progress_message', \%progress_message, qw{ c }); | |
2025 | ||
2026 | my %handle; | |
2027 | # cache open file handle, internal. Is undef if file hasn't been | |
2028 | # processed at all, empty if has; | |
2029 | main::set_access('handle', \%handle); | |
2030 | ||
2031 | my %added_lines; | |
2032 | # cache of lines added virtually to the file, internal | |
2033 | main::set_access('added_lines', \%added_lines); | |
2034 | ||
2035 | my %errors; | |
2036 | # cache of errors found, internal | |
2037 | main::set_access('errors', \%errors); | |
2038 | ||
2039 | my %missings; | |
2040 | # storage of '@missing' defaults lines | |
2041 | main::set_access('missings', \%missings); | |
2042 | ||
2043 | sub new { | |
2044 | my $class = shift; | |
2045 | ||
2046 | my $self = bless \do{ my $anonymous_scalar }, $class; | |
ffe43484 | 2047 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2048 | |
2049 | # Set defaults | |
2050 | $handler{$addr} = \&main::process_generic_property_file; | |
2051 | $non_skip{$addr} = 0; | |
37e2e78e | 2052 | $skip{$addr} = 0; |
99870f4d KW |
2053 | $has_missings_defaults{$addr} = $NO_DEFAULTS; |
2054 | $handle{$addr} = undef; | |
2055 | $added_lines{$addr} = [ ]; | |
2056 | $each_line_handler{$addr} = [ ]; | |
2057 | $errors{$addr} = { }; | |
2058 | $missings{$addr} = [ ]; | |
2059 | ||
2060 | # Two positional parameters. | |
99f78760 | 2061 | return Carp::carp_too_few_args(\@_, 2) if main::DEBUG && @_ < 2; |
99870f4d KW |
2062 | $file{$addr} = main::internal_file_to_platform(shift); |
2063 | $first_released{$addr} = shift; | |
2064 | ||
2065 | # The rest of the arguments are key => value pairs | |
2066 | # %constructor_fields has been set up earlier to list all possible | |
2067 | # ones. Either set or push, depending on how the default has been set | |
2068 | # up just above. | |
2069 | my %args = @_; | |
2070 | foreach my $key (keys %args) { | |
2071 | my $argument = $args{$key}; | |
2072 | ||
2073 | # Note that the fields are the lower case of the constructor keys | |
2074 | my $hash = $constructor_fields{lc $key}; | |
2075 | if (! defined $hash) { | |
2076 | Carp::my_carp_bug("Unrecognized parameters '$key => $argument' to new() for $self. Skipped"); | |
2077 | next; | |
2078 | } | |
2079 | if (ref $hash->{$addr} eq 'ARRAY') { | |
2080 | if (ref $argument eq 'ARRAY') { | |
2081 | foreach my $argument (@{$argument}) { | |
2082 | next if ! defined $argument; | |
2083 | push @{$hash->{$addr}}, $argument; | |
2084 | } | |
2085 | } | |
2086 | else { | |
2087 | push @{$hash->{$addr}}, $argument if defined $argument; | |
2088 | } | |
2089 | } | |
2090 | else { | |
2091 | $hash->{$addr} = $argument; | |
2092 | } | |
2093 | delete $args{$key}; | |
2094 | }; | |
2095 | ||
2096 | # If the file has a property for it, it means that the property is not | |
2097 | # listed in the file's entries. So add a handler to the list of line | |
2098 | # handlers to insert the property name into the lines, to provide a | |
2099 | # uniform interface to the final processing subroutine. | |
2100 | # the final code doesn't have to worry about that. | |
2101 | if ($property{$addr}) { | |
2102 | push @{$each_line_handler{$addr}}, \&_insert_property_into_line; | |
2103 | } | |
2104 | ||
2105 | if ($non_skip{$addr} && ! $debug_skip && $verbosity) { | |
2106 | print "Warning: " . __PACKAGE__ . " constructor for $file{$addr} has useless 'non_skip' in it\n"; | |
a3a8c5f0 | 2107 | } |
99870f4d | 2108 | |
09ca89ce KW |
2109 | # If skipping, set to optional, and add to list of ignored files, |
2110 | # including its reason | |
2111 | if ($skip{$addr}) { | |
2112 | $optional{$addr} = 1; | |
2113 | $ignored_files{$file{$addr}} = $skip{$addr} | |
2114 | } | |
37e2e78e | 2115 | |
99870f4d | 2116 | return $self; |
d73e5302 JH |
2117 | } |
2118 | ||
cf25bb62 | 2119 | |
99870f4d KW |
2120 | use overload |
2121 | fallback => 0, | |
2122 | qw("") => "_operator_stringify", | |
2123 | "." => \&main::_operator_dot, | |
2124 | ; | |
cf25bb62 | 2125 | |
99870f4d KW |
2126 | sub _operator_stringify { |
2127 | my $self = shift; | |
cf25bb62 | 2128 | |
99870f4d | 2129 | return __PACKAGE__ . " object for " . $self->file; |
d73e5302 | 2130 | } |
d73e5302 | 2131 | |
99870f4d KW |
2132 | # flag to make sure extracted files are processed early |
2133 | my $seen_non_extracted_non_age = 0; | |
d73e5302 | 2134 | |
99870f4d KW |
2135 | sub run { |
2136 | # Process the input object $self. This opens and closes the file and | |
2137 | # calls all the handlers for it. Currently, this can only be called | |
2138 | # once per file, as it destroy's the EOF handler | |
d73e5302 | 2139 | |
99870f4d KW |
2140 | my $self = shift; |
2141 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
b6922eda | 2142 | |
ffe43484 | 2143 | my $addr = do { no overloading; pack 'J', $self; }; |
b6922eda | 2144 | |
99870f4d | 2145 | my $file = $file{$addr}; |
d73e5302 | 2146 | |
99870f4d KW |
2147 | # Don't process if not expecting this file (because released later |
2148 | # than this Unicode version), and isn't there. This means if someone | |
2149 | # copies it into an earlier version's directory, we will go ahead and | |
2150 | # process it. | |
2151 | return if $first_released{$addr} gt $v_version && ! -e $file; | |
2152 | ||
2153 | # If in debugging mode and this file doesn't have the non-skip | |
2154 | # flag set, and isn't one of the critical files, skip it. | |
2155 | if ($debug_skip | |
2156 | && $first_released{$addr} ne v0 | |
2157 | && ! $non_skip{$addr}) | |
2158 | { | |
2159 | print "Skipping $file in debugging\n" if $verbosity; | |
2160 | return; | |
2161 | } | |
2162 | ||
2163 | # File could be optional | |
37e2e78e | 2164 | if ($optional{$addr}) { |
99870f4d KW |
2165 | return unless -e $file; |
2166 | my $result = eval $optional{$addr}; | |
2167 | if (! defined $result) { | |
2168 | Carp::my_carp_bug("Got '$@' when tried to eval $optional{$addr}. $file Skipped."); | |
2169 | return; | |
2170 | } | |
2171 | if (! $result) { | |
2172 | if ($verbosity) { | |
2173 | print STDERR "Skipping processing input file '$file' because '$optional{$addr}' is not true\n"; | |
2174 | } | |
2175 | return; | |
2176 | } | |
2177 | } | |
2178 | ||
2179 | if (! defined $file || ! -e $file) { | |
2180 | ||
2181 | # If the file doesn't exist, see if have internal data for it | |
2182 | # (based on first_released being 0). | |
2183 | if ($first_released{$addr} eq v0) { | |
2184 | $handle{$addr} = 'pretend_is_open'; | |
2185 | } | |
2186 | else { | |
2187 | if (! $optional{$addr} # File could be optional | |
2188 | && $v_version ge $first_released{$addr}) | |
2189 | { | |
2190 | print STDERR "Skipping processing input file '$file' because not found\n" if $v_version ge $first_released{$addr}; | |
2191 | } | |
2192 | return; | |
2193 | } | |
2194 | } | |
2195 | else { | |
2196 | ||
37e2e78e KW |
2197 | # Here, the file exists. Some platforms may change the case of |
2198 | # its name | |
99870f4d | 2199 | if ($seen_non_extracted_non_age) { |
517956bf | 2200 | if ($file =~ /$EXTRACTED/i) { |
99870f4d | 2201 | Carp::my_carp_bug(join_lines(<<END |
99f78760 | 2202 | $file should be processed just after the 'Prop...Alias' files, and before |
99870f4d KW |
2203 | anything not in the $EXTRACTED_DIR directory. Proceeding, but the results may |
2204 | have subtle problems | |
2205 | END | |
2206 | )); | |
2207 | } | |
2208 | } | |
2209 | elsif ($EXTRACTED_DIR | |
2210 | && $first_released{$addr} ne v0 | |
517956bf CB |
2211 | && $file !~ /$EXTRACTED/i |
2212 | && lc($file) ne 'dage.txt') | |
99870f4d KW |
2213 | { |
2214 | # We don't set this (by the 'if' above) if we have no | |
2215 | # extracted directory, so if running on an early version, | |
2216 | # this test won't work. Not worth worrying about. | |
2217 | $seen_non_extracted_non_age = 1; | |
2218 | } | |
2219 | ||
2220 | # And mark the file as having being processed, and warn if it | |
2221 | # isn't a file we are expecting. As we process the files, | |
2222 | # they are deleted from the hash, so any that remain at the | |
2223 | # end of the program are files that we didn't process. | |
517956bf CB |
2224 | my $fkey = File::Spec->rel2abs($file); |
2225 | my $expecting = delete $potential_files{$fkey}; | |
2226 | $expecting = delete $potential_files{lc($fkey)} unless defined $expecting; | |
678f13d5 KW |
2227 | Carp::my_carp("Was not expecting '$file'.") if |
2228 | ! $expecting | |
99870f4d KW |
2229 | && ! defined $handle{$addr}; |
2230 | ||
37e2e78e KW |
2231 | # Having deleted from expected files, we can quit if not to do |
2232 | # anything. Don't print progress unless really want verbosity | |
2233 | if ($skip{$addr}) { | |
2234 | print "Skipping $file.\n" if $verbosity >= $VERBOSE; | |
2235 | return; | |
2236 | } | |
2237 | ||
99870f4d KW |
2238 | # Open the file, converting the slashes used in this program |
2239 | # into the proper form for the OS | |
2240 | my $file_handle; | |
2241 | if (not open $file_handle, "<", $file) { | |
2242 | Carp::my_carp("Can't open $file. Skipping: $!"); | |
2243 | return 0; | |
2244 | } | |
2245 | $handle{$addr} = $file_handle; # Cache the open file handle | |
2246 | } | |
2247 | ||
2248 | if ($verbosity >= $PROGRESS) { | |
2249 | if ($progress_message{$addr}) { | |
2250 | print "$progress_message{$addr}\n"; | |
2251 | } | |
2252 | else { | |
2253 | # If using a virtual file, say so. | |
2254 | print "Processing ", (-e $file) | |
2255 | ? $file | |
2256 | : "substitute $file", | |
2257 | "\n"; | |
2258 | } | |
2259 | } | |
2260 | ||
2261 | ||
2262 | # Call any special handler for before the file. | |
2263 | &{$pre_handler{$addr}}($self) if $pre_handler{$addr}; | |
2264 | ||
2265 | # Then the main handler | |
2266 | &{$handler{$addr}}($self); | |
2267 | ||
2268 | # Then any special post-file handler. | |
2269 | &{$post_handler{$addr}}($self) if $post_handler{$addr}; | |
2270 | ||
2271 | # If any errors have been accumulated, output the counts (as the first | |
2272 | # error message in each class was output when it was encountered). | |
2273 | if ($errors{$addr}) { | |
2274 | my $total = 0; | |
2275 | my $types = 0; | |
2276 | foreach my $error (keys %{$errors{$addr}}) { | |
2277 | $total += $errors{$addr}->{$error}; | |
2278 | delete $errors{$addr}->{$error}; | |
2279 | $types++; | |
2280 | } | |
2281 | if ($total > 1) { | |
2282 | my $message | |
2283 | = "A total of $total lines had errors in $file. "; | |
2284 | ||
2285 | $message .= ($types == 1) | |
2286 | ? '(Only the first one was displayed.)' | |
2287 | : '(Only the first of each type was displayed.)'; | |
2288 | Carp::my_carp($message); | |
2289 | } | |
2290 | } | |
2291 | ||
2292 | if (@{$missings{$addr}}) { | |
2293 | Carp::my_carp_bug("Handler for $file didn't look at all the \@missing lines. Generated tables likely are wrong"); | |
2294 | } | |
2295 | ||
2296 | # If a real file handle, close it. | |
2297 | close $handle{$addr} or Carp::my_carp("Can't close $file: $!") if | |
2298 | ref $handle{$addr}; | |
2299 | $handle{$addr} = ""; # Uses empty to indicate that has already seen | |
2300 | # the file, as opposed to undef | |
2301 | return; | |
2302 | } | |
2303 | ||
2304 | sub next_line { | |
2305 | # Sets $_ to be the next logical input line, if any. Returns non-zero | |
2306 | # if such a line exists. 'logical' means that any lines that have | |
2307 | # been added via insert_lines() will be returned in $_ before the file | |
2308 | # is read again. | |
2309 | ||
2310 | my $self = shift; | |
2311 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2312 | ||
ffe43484 | 2313 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2314 | |
2315 | # Here the file is open (or if the handle is not a ref, is an open | |
2316 | # 'virtual' file). Get the next line; any inserted lines get priority | |
2317 | # over the file itself. | |
2318 | my $adjusted; | |
2319 | ||
2320 | LINE: | |
2321 | while (1) { # Loop until find non-comment, non-empty line | |
2322 | #local $to_trace = 1 if main::DEBUG; | |
2323 | my $inserted_ref = shift @{$added_lines{$addr}}; | |
2324 | if (defined $inserted_ref) { | |
2325 | ($adjusted, $_) = @{$inserted_ref}; | |
2326 | trace $adjusted, $_ if main::DEBUG && $to_trace; | |
2327 | return 1 if $adjusted; | |
2328 | } | |
2329 | else { | |
2330 | last if ! ref $handle{$addr}; # Don't read unless is real file | |
2331 | last if ! defined ($_ = readline $handle{$addr}); | |
2332 | } | |
2333 | chomp; | |
2334 | trace $_ if main::DEBUG && $to_trace; | |
2335 | ||
2336 | # See if this line is the comment line that defines what property | |
2337 | # value that code points that are not listed in the file should | |
2338 | # have. The format or existence of these lines is not guaranteed | |
2339 | # by Unicode since they are comments, but the documentation says | |
2340 | # that this was added for machine-readability, so probably won't | |
2341 | # change. This works starting in Unicode Version 5.0. They look | |
2342 | # like: | |
2343 | # | |
2344 | # @missing: 0000..10FFFF; Not_Reordered | |
2345 | # @missing: 0000..10FFFF; Decomposition_Mapping; <code point> | |
2346 | # @missing: 0000..10FFFF; ; NaN | |
2347 | # | |
2348 | # Save the line for a later get_missings() call. | |
2349 | if (/$missing_defaults_prefix/) { | |
2350 | if ($has_missings_defaults{$addr} == $NO_DEFAULTS) { | |
2351 | $self->carp_bad_line("Unexpected \@missing line. Assuming no missing entries"); | |
2352 | } | |
2353 | elsif ($has_missings_defaults{$addr} == $NOT_IGNORED) { | |
2354 | my @defaults = split /\s* ; \s*/x, $_; | |
2355 | ||
2356 | # The first field is the @missing, which ends in a | |
2357 | # semi-colon, so can safely shift. | |
2358 | shift @defaults; | |
2359 | ||
2360 | # Some of these lines may have empty field placeholders | |
2361 | # which get in the way. An example is: | |
2362 | # @missing: 0000..10FFFF; ; NaN | |
2363 | # Remove them. Process starting from the top so the | |
2364 | # splice doesn't affect things still to be looked at. | |
2365 | for (my $i = @defaults - 1; $i >= 0; $i--) { | |
2366 | next if $defaults[$i] ne ""; | |
2367 | splice @defaults, $i, 1; | |
2368 | } | |
2369 | ||
2370 | # What's left should be just the property (maybe) and the | |
2371 | # default. Having only one element means it doesn't have | |
2372 | # the property. | |
2373 | my $default; | |
2374 | my $property; | |
2375 | if (@defaults >= 1) { | |
2376 | if (@defaults == 1) { | |
2377 | $default = $defaults[0]; | |
2378 | } | |
2379 | else { | |
2380 | $property = $defaults[0]; | |
2381 | $default = $defaults[1]; | |
2382 | } | |
2383 | } | |
2384 | ||
2385 | if (@defaults < 1 | |
2386 | || @defaults > 2 | |
2387 | || ($default =~ /^</ | |
2388 | && $default !~ /^<code *point>$/i | |
2389 | && $default !~ /^<none>$/i)) | |
2390 | { | |
2391 | $self->carp_bad_line("Unrecognized \@missing line: $_. Assuming no missing entries"); | |
2392 | } | |
2393 | else { | |
2394 | ||
2395 | # If the property is missing from the line, it should | |
2396 | # be the one for the whole file | |
2397 | $property = $property{$addr} if ! defined $property; | |
2398 | ||
2399 | # Change <none> to the null string, which is what it | |
2400 | # really means. If the default is the code point | |
2401 | # itself, set it to <code point>, which is what | |
2402 | # Unicode uses (but sometimes they've forgotten the | |
2403 | # space) | |
2404 | if ($default =~ /^<none>$/i) { | |
2405 | $default = ""; | |
2406 | } | |
2407 | elsif ($default =~ /^<code *point>$/i) { | |
2408 | $default = $CODE_POINT; | |
2409 | } | |
2410 | ||
2411 | # Store them as a sub-arrays with both components. | |
2412 | push @{$missings{$addr}}, [ $default, $property ]; | |
2413 | } | |
2414 | } | |
2415 | ||
2416 | # There is nothing for the caller to process on this comment | |
2417 | # line. | |
2418 | next; | |
2419 | } | |
2420 | ||
2421 | # Remove comments and trailing space, and skip this line if the | |
2422 | # result is empty | |
2423 | s/#.*//; | |
2424 | s/\s+$//; | |
2425 | next if /^$/; | |
2426 | ||
2427 | # Call any handlers for this line, and skip further processing of | |
2428 | # the line if the handler sets the line to null. | |
2429 | foreach my $sub_ref (@{$each_line_handler{$addr}}) { | |
2430 | &{$sub_ref}($self); | |
2431 | next LINE if /^$/; | |
2432 | } | |
2433 | ||
2434 | # Here the line is ok. return success. | |
2435 | return 1; | |
2436 | } # End of looping through lines. | |
2437 | ||
2438 | # If there is an EOF handler, call it (only once) and if it generates | |
2439 | # more lines to process go back in the loop to handle them. | |
2440 | if ($eof_handler{$addr}) { | |
2441 | &{$eof_handler{$addr}}($self); | |
2442 | $eof_handler{$addr} = ""; # Currently only get one shot at it. | |
2443 | goto LINE if $added_lines{$addr}; | |
2444 | } | |
2445 | ||
2446 | # Return failure -- no more lines. | |
2447 | return 0; | |
2448 | ||
2449 | } | |
2450 | ||
2451 | # Not currently used, not fully tested. | |
2452 | # sub peek { | |
2453 | # # Non-destructive look-ahead one non-adjusted, non-comment, non-blank | |
2454 | # # record. Not callable from an each_line_handler(), nor does it call | |
2455 | # # an each_line_handler() on the line. | |
2456 | # | |
2457 | # my $self = shift; | |
ffe43484 | 2458 | # my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2459 | # |
2460 | # foreach my $inserted_ref (@{$added_lines{$addr}}) { | |
2461 | # my ($adjusted, $line) = @{$inserted_ref}; | |
2462 | # next if $adjusted; | |
2463 | # | |
2464 | # # Remove comments and trailing space, and return a non-empty | |
2465 | # # resulting line | |
2466 | # $line =~ s/#.*//; | |
2467 | # $line =~ s/\s+$//; | |
2468 | # return $line if $line ne ""; | |
2469 | # } | |
2470 | # | |
2471 | # return if ! ref $handle{$addr}; # Don't read unless is real file | |
2472 | # while (1) { # Loop until find non-comment, non-empty line | |
2473 | # local $to_trace = 1 if main::DEBUG; | |
2474 | # trace $_ if main::DEBUG && $to_trace; | |
2475 | # return if ! defined (my $line = readline $handle{$addr}); | |
2476 | # chomp $line; | |
2477 | # push @{$added_lines{$addr}}, [ 0, $line ]; | |
2478 | # | |
2479 | # $line =~ s/#.*//; | |
2480 | # $line =~ s/\s+$//; | |
2481 | # return $line if $line ne ""; | |
2482 | # } | |
2483 | # | |
2484 | # return; | |
2485 | # } | |
2486 | ||
2487 | ||
2488 | sub insert_lines { | |
2489 | # Lines can be inserted so that it looks like they were in the input | |
2490 | # file at the place it was when this routine is called. See also | |
2491 | # insert_adjusted_lines(). Lines inserted via this routine go through | |
2492 | # any each_line_handler() | |
2493 | ||
2494 | my $self = shift; | |
2495 | ||
2496 | # Each inserted line is an array, with the first element being 0 to | |
2497 | # indicate that this line hasn't been adjusted, and needs to be | |
2498 | # processed. | |
f998e60c | 2499 | no overloading; |
051df77b | 2500 | push @{$added_lines{pack 'J', $self}}, map { [ 0, $_ ] } @_; |
99870f4d KW |
2501 | return; |
2502 | } | |
2503 | ||
2504 | sub insert_adjusted_lines { | |
2505 | # Lines can be inserted so that it looks like they were in the input | |
2506 | # file at the place it was when this routine is called. See also | |
2507 | # insert_lines(). Lines inserted via this routine are already fully | |
2508 | # adjusted, ready to be processed; each_line_handler()s handlers will | |
2509 | # not be called. This means this is not a completely general | |
2510 | # facility, as only the last each_line_handler on the stack should | |
2511 | # call this. It could be made more general, by passing to each of the | |
2512 | # line_handlers their position on the stack, which they would pass on | |
2513 | # to this routine, and that would replace the boolean first element in | |
2514 | # the anonymous array pushed here, so that the next_line routine could | |
2515 | # use that to call only those handlers whose index is after it on the | |
2516 | # stack. But this is overkill for what is needed now. | |
2517 | ||
2518 | my $self = shift; | |
2519 | trace $_[0] if main::DEBUG && $to_trace; | |
2520 | ||
2521 | # Each inserted line is an array, with the first element being 1 to | |
2522 | # indicate that this line has been adjusted | |
f998e60c | 2523 | no overloading; |
051df77b | 2524 | push @{$added_lines{pack 'J', $self}}, map { [ 1, $_ ] } @_; |
99870f4d KW |
2525 | return; |
2526 | } | |
2527 | ||
2528 | sub get_missings { | |
2529 | # Returns the stored up @missings lines' values, and clears the list. | |
2530 | # The values are in an array, consisting of the default in the first | |
2531 | # element, and the property in the 2nd. However, since these lines | |
2532 | # can be stacked up, the return is an array of all these arrays. | |
2533 | ||
2534 | my $self = shift; | |
2535 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2536 | ||
ffe43484 | 2537 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2538 | |
2539 | # If not accepting a list return, just return the first one. | |
2540 | return shift @{$missings{$addr}} unless wantarray; | |
2541 | ||
2542 | my @return = @{$missings{$addr}}; | |
2543 | undef @{$missings{$addr}}; | |
2544 | return @return; | |
2545 | } | |
2546 | ||
2547 | sub _insert_property_into_line { | |
2548 | # Add a property field to $_, if this file requires it. | |
2549 | ||
f998e60c | 2550 | my $self = shift; |
ffe43484 | 2551 | my $addr = do { no overloading; pack 'J', $self; }; |
f998e60c | 2552 | my $property = $property{$addr}; |
99870f4d KW |
2553 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; |
2554 | ||
2555 | $_ =~ s/(;|$)/; $property$1/; | |
2556 | return; | |
2557 | } | |
2558 | ||
2559 | sub carp_bad_line { | |
2560 | # Output consistent error messages, using either a generic one, or the | |
2561 | # one given by the optional parameter. To avoid gazillions of the | |
2562 | # same message in case the syntax of a file is way off, this routine | |
2563 | # only outputs the first instance of each message, incrementing a | |
2564 | # count so the totals can be output at the end of the file. | |
2565 | ||
2566 | my $self = shift; | |
2567 | my $message = shift; | |
2568 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2569 | ||
ffe43484 | 2570 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2571 | |
2572 | $message = 'Unexpected line' unless $message; | |
2573 | ||
2574 | # No trailing punctuation so as to fit with our addenda. | |
2575 | $message =~ s/[.:;,]$//; | |
2576 | ||
2577 | # If haven't seen this exact message before, output it now. Otherwise | |
2578 | # increment the count of how many times it has occurred | |
2579 | unless ($errors{$addr}->{$message}) { | |
2580 | Carp::my_carp("$message in '$_' in " | |
f998e60c | 2581 | . $file{$addr} |
99870f4d KW |
2582 | . " at line $.. Skipping this line;"); |
2583 | $errors{$addr}->{$message} = 1; | |
2584 | } | |
2585 | else { | |
2586 | $errors{$addr}->{$message}++; | |
2587 | } | |
2588 | ||
2589 | # Clear the line to prevent any further (meaningful) processing of it. | |
2590 | $_ = ""; | |
2591 | ||
2592 | return; | |
2593 | } | |
2594 | } # End closure | |
2595 | ||
2596 | package Multi_Default; | |
2597 | ||
2598 | # Certain properties in early versions of Unicode had more than one possible | |
2599 | # default for code points missing from the files. In these cases, one | |
2600 | # default applies to everything left over after all the others are applied, | |
2601 | # and for each of the others, there is a description of which class of code | |
2602 | # points applies to it. This object helps implement this by storing the | |
2603 | # defaults, and for all but that final default, an eval string that generates | |
2604 | # the class that it applies to. | |
2605 | ||
2606 | ||
2607 | { # Closure | |
2608 | ||
2609 | main::setup_package(); | |
2610 | ||
2611 | my %class_defaults; | |
2612 | # The defaults structure for the classes | |
2613 | main::set_access('class_defaults', \%class_defaults); | |
2614 | ||
2615 | my %other_default; | |
2616 | # The default that applies to everything left over. | |
2617 | main::set_access('other_default', \%other_default, 'r'); | |
2618 | ||
2619 | ||
2620 | sub new { | |
2621 | # The constructor is called with default => eval pairs, terminated by | |
2622 | # the left-over default. e.g. | |
2623 | # Multi_Default->new( | |
2624 | # 'T' => '$gc->table("Mn") + $gc->table("Cf") - 0x200C | |
2625 | # - 0x200D', | |
2626 | # 'R' => 'some other expression that evaluates to code points', | |
2627 | # . | |
2628 | # . | |
2629 | # . | |
2630 | # 'U')); | |
2631 | ||
2632 | my $class = shift; | |
2633 | ||
2634 | my $self = bless \do{my $anonymous_scalar}, $class; | |
ffe43484 | 2635 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2636 | |
2637 | while (@_ > 1) { | |
2638 | my $default = shift; | |
2639 | my $eval = shift; | |
2640 | $class_defaults{$addr}->{$default} = $eval; | |
2641 | } | |
2642 | ||
2643 | $other_default{$addr} = shift; | |
2644 | ||
2645 | return $self; | |
2646 | } | |
2647 | ||
2648 | sub get_next_defaults { | |
2649 | # Iterates and returns the next class of defaults. | |
2650 | my $self = shift; | |
2651 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2652 | ||
ffe43484 | 2653 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2654 | |
2655 | return each %{$class_defaults{$addr}}; | |
2656 | } | |
2657 | } | |
2658 | ||
2659 | package Alias; | |
2660 | ||
2661 | # An alias is one of the names that a table goes by. This class defines them | |
2662 | # including some attributes. Everything is currently setup in the | |
2663 | # constructor. | |
2664 | ||
2665 | ||
2666 | { # Closure | |
2667 | ||
2668 | main::setup_package(); | |
2669 | ||
2670 | my %name; | |
2671 | main::set_access('name', \%name, 'r'); | |
2672 | ||
2673 | my %loose_match; | |
c12f2655 | 2674 | # Should this name match loosely or not. |
99870f4d KW |
2675 | main::set_access('loose_match', \%loose_match, 'r'); |
2676 | ||
33e96e72 KW |
2677 | my %make_re_pod_entry; |
2678 | # Some aliases should not get their own entries in the re section of the | |
2679 | # pod, because they are covered by a wild-card, and some we want to | |
2680 | # discourage use of. Binary | |
2681 | main::set_access('make_re_pod_entry', \%make_re_pod_entry, 'r'); | |
99870f4d KW |
2682 | |
2683 | my %status; | |
2684 | # Aliases have a status, like deprecated, or even suppressed (which means | |
2685 | # they don't appear in documentation). Enum | |
2686 | main::set_access('status', \%status, 'r'); | |
2687 | ||
2688 | my %externally_ok; | |
2689 | # Similarly, some aliases should not be considered as usable ones for | |
2690 | # external use, such as file names, or we don't want documentation to | |
2691 | # recommend them. Boolean | |
2692 | main::set_access('externally_ok', \%externally_ok, 'r'); | |
2693 | ||
2694 | sub new { | |
2695 | my $class = shift; | |
2696 | ||
2697 | my $self = bless \do { my $anonymous_scalar }, $class; | |
ffe43484 | 2698 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2699 | |
2700 | $name{$addr} = shift; | |
2701 | $loose_match{$addr} = shift; | |
33e96e72 | 2702 | $make_re_pod_entry{$addr} = shift; |
99870f4d KW |
2703 | $externally_ok{$addr} = shift; |
2704 | $status{$addr} = shift; | |
2705 | ||
2706 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2707 | ||
2708 | # Null names are never ok externally | |
2709 | $externally_ok{$addr} = 0 if $name{$addr} eq ""; | |
2710 | ||
2711 | return $self; | |
2712 | } | |
2713 | } | |
2714 | ||
2715 | package Range; | |
2716 | ||
2717 | # A range is the basic unit for storing code points, and is described in the | |
2718 | # comments at the beginning of the program. Each range has a starting code | |
2719 | # point; an ending code point (not less than the starting one); a value | |
2720 | # that applies to every code point in between the two end-points, inclusive; | |
2721 | # and an enum type that applies to the value. The type is for the user's | |
2722 | # convenience, and has no meaning here, except that a non-zero type is | |
2723 | # considered to not obey the normal Unicode rules for having standard forms. | |
2724 | # | |
2725 | # The same structure is used for both map and match tables, even though in the | |
2726 | # latter, the value (and hence type) is irrelevant and could be used as a | |
2727 | # comment. In map tables, the value is what all the code points in the range | |
2728 | # map to. Type 0 values have the standardized version of the value stored as | |
2729 | # well, so as to not have to recalculate it a lot. | |
2730 | ||
2731 | sub trace { return main::trace(@_); } | |
2732 | ||
2733 | { # Closure | |
2734 | ||
2735 | main::setup_package(); | |
2736 | ||
2737 | my %start; | |
2738 | main::set_access('start', \%start, 'r', 's'); | |
2739 | ||
2740 | my %end; | |
2741 | main::set_access('end', \%end, 'r', 's'); | |
2742 | ||
2743 | my %value; | |
2744 | main::set_access('value', \%value, 'r'); | |
2745 | ||
2746 | my %type; | |
2747 | main::set_access('type', \%type, 'r'); | |
2748 | ||
2749 | my %standard_form; | |
2750 | # The value in internal standard form. Defined only if the type is 0. | |
2751 | main::set_access('standard_form', \%standard_form); | |
2752 | ||
2753 | # Note that if these fields change, the dump() method should as well | |
2754 | ||
2755 | sub new { | |
2756 | return Carp::carp_too_few_args(\@_, 3) if main::DEBUG && @_ < 3; | |
2757 | my $class = shift; | |
2758 | ||
2759 | my $self = bless \do { my $anonymous_scalar }, $class; | |
ffe43484 | 2760 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2761 | |
2762 | $start{$addr} = shift; | |
2763 | $end{$addr} = shift; | |
2764 | ||
2765 | my %args = @_; | |
2766 | ||
2767 | my $value = delete $args{'Value'}; # Can be 0 | |
2768 | $value = "" unless defined $value; | |
2769 | $value{$addr} = $value; | |
2770 | ||
2771 | $type{$addr} = delete $args{'Type'} || 0; | |
2772 | ||
2773 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
2774 | ||
2775 | if (! $type{$addr}) { | |
2776 | $standard_form{$addr} = main::standardize($value); | |
2777 | } | |
2778 | ||
2779 | return $self; | |
2780 | } | |
2781 | ||
2782 | use overload | |
2783 | fallback => 0, | |
2784 | qw("") => "_operator_stringify", | |
2785 | "." => \&main::_operator_dot, | |
2786 | ; | |
2787 | ||
2788 | sub _operator_stringify { | |
2789 | my $self = shift; | |
ffe43484 | 2790 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2791 | |
2792 | # Output it like '0041..0065 (value)' | |
2793 | my $return = sprintf("%04X", $start{$addr}) | |
2794 | . '..' | |
2795 | . sprintf("%04X", $end{$addr}); | |
2796 | my $value = $value{$addr}; | |
2797 | my $type = $type{$addr}; | |
2798 | $return .= ' ('; | |
2799 | $return .= "$value"; | |
2800 | $return .= ", Type=$type" if $type != 0; | |
2801 | $return .= ')'; | |
2802 | ||
2803 | return $return; | |
2804 | } | |
2805 | ||
2806 | sub standard_form { | |
2807 | # The standard form is the value itself if the standard form is | |
2808 | # undefined (that is if the value is special) | |
2809 | ||
2810 | my $self = shift; | |
2811 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2812 | ||
ffe43484 | 2813 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2814 | |
2815 | return $standard_form{$addr} if defined $standard_form{$addr}; | |
2816 | return $value{$addr}; | |
2817 | } | |
2818 | ||
2819 | sub dump { | |
2820 | # Human, not machine readable. For machine readable, comment out this | |
2821 | # entire routine and let the standard one take effect. | |
2822 | my $self = shift; | |
2823 | my $indent = shift; | |
2824 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
2825 | ||
ffe43484 | 2826 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2827 | |
2828 | my $return = $indent | |
2829 | . sprintf("%04X", $start{$addr}) | |
2830 | . '..' | |
2831 | . sprintf("%04X", $end{$addr}) | |
2832 | . " '$value{$addr}';"; | |
2833 | if (! defined $standard_form{$addr}) { | |
2834 | $return .= "(type=$type{$addr})"; | |
2835 | } | |
2836 | elsif ($standard_form{$addr} ne $value{$addr}) { | |
2837 | $return .= "(standard '$standard_form{$addr}')"; | |
2838 | } | |
2839 | return $return; | |
2840 | } | |
2841 | } # End closure | |
2842 | ||
2843 | package _Range_List_Base; | |
2844 | ||
2845 | # Base class for range lists. A range list is simply an ordered list of | |
2846 | # ranges, so that the ranges with the lowest starting numbers are first in it. | |
2847 | # | |
2848 | # When a new range is added that is adjacent to an existing range that has the | |
2849 | # same value and type, it merges with it to form a larger range. | |
2850 | # | |
2851 | # Ranges generally do not overlap, except that there can be multiple entries | |
2852 | # of single code point ranges. This is because of NameAliases.txt. | |
2853 | # | |
2854 | # In this program, there is a standard value such that if two different | |
2855 | # values, have the same standard value, they are considered equivalent. This | |
2856 | # value was chosen so that it gives correct results on Unicode data | |
2857 | ||
2858 | # There are a number of methods to manipulate range lists, and some operators | |
2859 | # are overloaded to handle them. | |
2860 | ||
99870f4d KW |
2861 | sub trace { return main::trace(@_); } |
2862 | ||
2863 | { # Closure | |
2864 | ||
2865 | our $addr; | |
2866 | ||
2867 | main::setup_package(); | |
2868 | ||
2869 | my %ranges; | |
2870 | # The list of ranges | |
2871 | main::set_access('ranges', \%ranges, 'readable_array'); | |
2872 | ||
2873 | my %max; | |
2874 | # The highest code point in the list. This was originally a method, but | |
2875 | # actual measurements said it was used a lot. | |
2876 | main::set_access('max', \%max, 'r'); | |
2877 | ||
2878 | my %each_range_iterator; | |
2879 | # Iterator position for each_range() | |
2880 | main::set_access('each_range_iterator', \%each_range_iterator); | |
2881 | ||
2882 | my %owner_name_of; | |
2883 | # Name of parent this is attached to, if any. Solely for better error | |
2884 | # messages. | |
2885 | main::set_access('owner_name_of', \%owner_name_of, 'p_r'); | |
2886 | ||
2887 | my %_search_ranges_cache; | |
2888 | # A cache of the previous result from _search_ranges(), for better | |
2889 | # performance | |
2890 | main::set_access('_search_ranges_cache', \%_search_ranges_cache); | |
2891 | ||
2892 | sub new { | |
2893 | my $class = shift; | |
2894 | my %args = @_; | |
2895 | ||
2896 | # Optional initialization data for the range list. | |
2897 | my $initialize = delete $args{'Initialize'}; | |
2898 | ||
2899 | my $self; | |
2900 | ||
2901 | # Use _union() to initialize. _union() returns an object of this | |
2902 | # class, which means that it will call this constructor recursively. | |
2903 | # But it won't have this $initialize parameter so that it won't | |
2904 | # infinitely loop on this. | |
2905 | return _union($class, $initialize, %args) if defined $initialize; | |
2906 | ||
2907 | $self = bless \do { my $anonymous_scalar }, $class; | |
ffe43484 | 2908 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2909 | |
2910 | # Optional parent object, only for debug info. | |
2911 | $owner_name_of{$addr} = delete $args{'Owner'}; | |
2912 | $owner_name_of{$addr} = "" if ! defined $owner_name_of{$addr}; | |
2913 | ||
2914 | # Stringify, in case it is an object. | |
2915 | $owner_name_of{$addr} = "$owner_name_of{$addr}"; | |
2916 | ||
2917 | # This is used only for error messages, and so a colon is added | |
2918 | $owner_name_of{$addr} .= ": " if $owner_name_of{$addr} ne ""; | |
2919 | ||
2920 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
2921 | ||
2922 | # Max is initialized to a negative value that isn't adjacent to 0, | |
2923 | # for simpler tests | |
2924 | $max{$addr} = -2; | |
2925 | ||
2926 | $_search_ranges_cache{$addr} = 0; | |
2927 | $ranges{$addr} = []; | |
2928 | ||
2929 | return $self; | |
2930 | } | |
2931 | ||
2932 | use overload | |
2933 | fallback => 0, | |
2934 | qw("") => "_operator_stringify", | |
2935 | "." => \&main::_operator_dot, | |
2936 | ; | |
2937 | ||
2938 | sub _operator_stringify { | |
2939 | my $self = shift; | |
ffe43484 | 2940 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
2941 | |
2942 | return "Range_List attached to '$owner_name_of{$addr}'" | |
2943 | if $owner_name_of{$addr}; | |
2944 | return "anonymous Range_List " . \$self; | |
2945 | } | |
2946 | ||
2947 | sub _union { | |
2948 | # Returns the union of the input code points. It can be called as | |
2949 | # either a constructor or a method. If called as a method, the result | |
2950 | # will be a new() instance of the calling object, containing the union | |
2951 | # of that object with the other parameter's code points; if called as | |
2952 | # a constructor, the first parameter gives the class the new object | |
2953 | # should be, and the second parameter gives the code points to go into | |
2954 | # it. | |
2955 | # In either case, there are two parameters looked at by this routine; | |
2956 | # any additional parameters are passed to the new() constructor. | |
2957 | # | |
2958 | # The code points can come in the form of some object that contains | |
2959 | # ranges, and has a conventionally named method to access them; or | |
2960 | # they can be an array of individual code points (as integers); or | |
2961 | # just a single code point. | |
2962 | # | |
2963 | # If they are ranges, this routine doesn't make any effort to preserve | |
2964 | # the range values of one input over the other. Therefore this base | |
2965 | # class should not allow _union to be called from other than | |
2966 | # initialization code, so as to prevent two tables from being added | |
2967 | # together where the range values matter. The general form of this | |
2968 | # routine therefore belongs in a derived class, but it was moved here | |
2969 | # to avoid duplication of code. The failure to overload this in this | |
2970 | # class keeps it safe. | |
2971 | # | |
2972 | ||
2973 | my $self; | |
2974 | my @args; # Arguments to pass to the constructor | |
2975 | ||
2976 | my $class = shift; | |
2977 | ||
2978 | # If a method call, will start the union with the object itself, and | |
2979 | # the class of the new object will be the same as self. | |
2980 | if (ref $class) { | |
2981 | $self = $class; | |
2982 | $class = ref $self; | |
2983 | push @args, $self; | |
2984 | } | |
2985 | ||
2986 | # Add the other required parameter. | |
2987 | push @args, shift; | |
2988 | # Rest of parameters are passed on to the constructor | |
2989 | ||
2990 | # Accumulate all records from both lists. | |
2991 | my @records; | |
2992 | for my $arg (@args) { | |
2993 | #local $to_trace = 0 if main::DEBUG; | |
2994 | trace "argument = $arg" if main::DEBUG && $to_trace; | |
2995 | if (! defined $arg) { | |
2996 | my $message = ""; | |
2997 | if (defined $self) { | |
f998e60c | 2998 | no overloading; |
051df77b | 2999 | $message .= $owner_name_of{pack 'J', $self}; |
99870f4d KW |
3000 | } |
3001 | Carp::my_carp_bug($message .= "Undefined argument to _union. No union done."); | |
3002 | return; | |
3003 | } | |
3004 | $arg = [ $arg ] if ! ref $arg; | |
3005 | my $type = ref $arg; | |
3006 | if ($type eq 'ARRAY') { | |
3007 | foreach my $element (@$arg) { | |
3008 | push @records, Range->new($element, $element); | |
3009 | } | |
3010 | } | |
3011 | elsif ($arg->isa('Range')) { | |
3012 | push @records, $arg; | |
3013 | } | |
3014 | elsif ($arg->can('ranges')) { | |
3015 | push @records, $arg->ranges; | |
3016 | } | |
3017 | else { | |
3018 | my $message = ""; | |
3019 | if (defined $self) { | |
f998e60c | 3020 | no overloading; |
051df77b | 3021 | $message .= $owner_name_of{pack 'J', $self}; |
99870f4d KW |
3022 | } |
3023 | Carp::my_carp_bug($message . "Cannot take the union of a $type. No union done."); | |
3024 | return; | |
3025 | } | |
3026 | } | |
3027 | ||
3028 | # Sort with the range containing the lowest ordinal first, but if | |
3029 | # two ranges start at the same code point, sort with the bigger range | |
3030 | # of the two first, because it takes fewer cycles. | |
3031 | @records = sort { ($a->start <=> $b->start) | |
3032 | or | |
3033 | # if b is shorter than a, b->end will be | |
3034 | # less than a->end, and we want to select | |
3035 | # a, so want to return -1 | |
3036 | ($b->end <=> $a->end) | |
3037 | } @records; | |
3038 | ||
3039 | my $new = $class->new(@_); | |
3040 | ||
3041 | # Fold in records so long as they add new information. | |
3042 | for my $set (@records) { | |
3043 | my $start = $set->start; | |
3044 | my $end = $set->end; | |
3045 | my $value = $set->value; | |
3046 | if ($start > $new->max) { | |
3047 | $new->_add_delete('+', $start, $end, $value); | |
3048 | } | |
3049 | elsif ($end > $new->max) { | |
3050 | $new->_add_delete('+', $new->max +1, $end, $value); | |
3051 | } | |
3052 | } | |
3053 | ||
3054 | return $new; | |
3055 | } | |
3056 | ||
3057 | sub range_count { # Return the number of ranges in the range list | |
3058 | my $self = shift; | |
3059 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3060 | ||
f998e60c | 3061 | no overloading; |
051df77b | 3062 | return scalar @{$ranges{pack 'J', $self}}; |
99870f4d KW |
3063 | } |
3064 | ||
3065 | sub min { | |
3066 | # Returns the minimum code point currently in the range list, or if | |
3067 | # the range list is empty, 2 beyond the max possible. This is a | |
3068 | # method because used so rarely, that not worth saving between calls, | |
3069 | # and having to worry about changing it as ranges are added and | |
3070 | # deleted. | |
3071 | ||
3072 | my $self = shift; | |
3073 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3074 | ||
ffe43484 | 3075 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
3076 | |
3077 | # If the range list is empty, return a large value that isn't adjacent | |
3078 | # to any that could be in the range list, for simpler tests | |
6189eadc | 3079 | return $MAX_UNICODE_CODEPOINT + 2 unless scalar @{$ranges{$addr}}; |
99870f4d KW |
3080 | return $ranges{$addr}->[0]->start; |
3081 | } | |
3082 | ||
3083 | sub contains { | |
3084 | # Boolean: Is argument in the range list? If so returns $i such that: | |
3085 | # range[$i]->end < $codepoint <= range[$i+1]->end | |
3086 | # which is one beyond what you want; this is so that the 0th range | |
3087 | # doesn't return false | |
3088 | my $self = shift; | |
3089 | my $codepoint = shift; | |
3090 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3091 | ||
99870f4d KW |
3092 | my $i = $self->_search_ranges($codepoint); |
3093 | return 0 unless defined $i; | |
3094 | ||
3095 | # The search returns $i, such that | |
3096 | # range[$i-1]->end < $codepoint <= range[$i]->end | |
3097 | # So is in the table if and only iff it is at least the start position | |
3098 | # of range $i. | |
f998e60c | 3099 | no overloading; |
051df77b | 3100 | return 0 if $ranges{pack 'J', $self}->[$i]->start > $codepoint; |
99870f4d KW |
3101 | return $i + 1; |
3102 | } | |
3103 | ||
2f7a8815 KW |
3104 | sub containing_range { |
3105 | # Returns the range object that contains the code point, undef if none | |
3106 | ||
3107 | my $self = shift; | |
3108 | my $codepoint = shift; | |
3109 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3110 | ||
3111 | my $i = $self->contains($codepoint); | |
3112 | return unless $i; | |
3113 | ||
3114 | # contains() returns 1 beyond where we should look | |
3115 | no overloading; | |
3116 | return $ranges{pack 'J', $self}->[$i-1]; | |
3117 | } | |
3118 | ||
99870f4d KW |
3119 | sub value_of { |
3120 | # Returns the value associated with the code point, undef if none | |
3121 | ||
3122 | my $self = shift; | |
3123 | my $codepoint = shift; | |
3124 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3125 | ||
d69c231b KW |
3126 | my $range = $self->containing_range($codepoint); |
3127 | return unless defined $range; | |
99870f4d | 3128 | |
d69c231b | 3129 | return $range->value; |
99870f4d KW |
3130 | } |
3131 | ||
0a9dbafc KW |
3132 | sub type_of { |
3133 | # Returns the type of the range containing the code point, undef if | |
3134 | # the code point is not in the table | |
3135 | ||
3136 | my $self = shift; | |
3137 | my $codepoint = shift; | |
3138 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3139 | ||
3140 | my $range = $self->containing_range($codepoint); | |
3141 | return unless defined $range; | |
3142 | ||
3143 | return $range->type; | |
3144 | } | |
3145 | ||
99870f4d KW |
3146 | sub _search_ranges { |
3147 | # Find the range in the list which contains a code point, or where it | |
3148 | # should go if were to add it. That is, it returns $i, such that: | |
3149 | # range[$i-1]->end < $codepoint <= range[$i]->end | |
3150 | # Returns undef if no such $i is possible (e.g. at end of table), or | |
3151 | # if there is an error. | |
3152 | ||
3153 | my $self = shift; | |
3154 | my $code_point = shift; | |
3155 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3156 | ||
ffe43484 | 3157 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
3158 | |
3159 | return if $code_point > $max{$addr}; | |
3160 | my $r = $ranges{$addr}; # The current list of ranges | |
3161 | my $range_list_size = scalar @$r; | |
3162 | my $i; | |
3163 | ||
3164 | use integer; # want integer division | |
3165 | ||
3166 | # Use the cached result as the starting guess for this one, because, | |
3167 | # an experiment on 5.1 showed that 90% of the time the cache was the | |
3168 | # same as the result on the next call (and 7% it was one less). | |
3169 | $i = $_search_ranges_cache{$addr}; | |
3170 | $i = 0 if $i >= $range_list_size; # Reset if no longer valid (prob. | |
3171 | # from an intervening deletion | |
3172 | #local $to_trace = 1 if main::DEBUG; | |
3173 | trace "previous \$i is still valid: $i" if main::DEBUG && $to_trace && $code_point <= $r->[$i]->end && ($i == 0 || $r->[$i-1]->end < $code_point); | |
3174 | return $i if $code_point <= $r->[$i]->end | |
3175 | && ($i == 0 || $r->[$i-1]->end < $code_point); | |
3176 | ||
3177 | # Here the cache doesn't yield the correct $i. Try adding 1. | |
3178 | if ($i < $range_list_size - 1 | |
3179 | && $r->[$i]->end < $code_point && | |
3180 | $code_point <= $r->[$i+1]->end) | |
3181 | { | |
3182 | $i++; | |
3183 | trace "next \$i is correct: $i" if main::DEBUG && $to_trace; | |
3184 | $_search_ranges_cache{$addr} = $i; | |
3185 | return $i; | |
3186 | } | |
3187 | ||
3188 | # Here, adding 1 also didn't work. We do a binary search to | |
3189 | # find the correct position, starting with current $i | |
3190 | my $lower = 0; | |
3191 | my $upper = $range_list_size - 1; | |
3192 | while (1) { | |
3193 | trace "top of loop i=$i:", sprintf("%04X", $r->[$lower]->start), "[$lower] .. ", sprintf("%04X", $r->[$i]->start), "[$i] .. ", sprintf("%04X", $r->[$upper]->start), "[$upper]" if main::DEBUG && $to_trace; | |
3194 | ||
3195 | if ($code_point <= $r->[$i]->end) { | |
3196 | ||
3197 | # Here we have met the upper constraint. We can quit if we | |
3198 | # also meet the lower one. | |
3199 | last if $i == 0 || $r->[$i-1]->end < $code_point; | |
3200 | ||
3201 | $upper = $i; # Still too high. | |
3202 | ||
3203 | } | |
3204 | else { | |
3205 | ||
3206 | # Here, $r[$i]->end < $code_point, so look higher up. | |
3207 | $lower = $i; | |
3208 | } | |
3209 | ||
3210 | # Split search domain in half to try again. | |
3211 | my $temp = ($upper + $lower) / 2; | |
3212 | ||
3213 | # No point in continuing unless $i changes for next time | |
3214 | # in the loop. | |
3215 | if ($temp == $i) { | |
3216 | ||
3217 | # We can't reach the highest element because of the averaging. | |
3218 | # So if one below the upper edge, force it there and try one | |
3219 | # more time. | |
3220 | if ($i == $range_list_size - 2) { | |
3221 | ||
3222 | trace "Forcing to upper edge" if main::DEBUG && $to_trace; | |
3223 | $i = $range_list_size - 1; | |
3224 | ||
3225 | # Change $lower as well so if fails next time through, | |
3226 | # taking the average will yield the same $i, and we will | |
3227 | # quit with the error message just below. | |
3228 | $lower = $i; | |
3229 | next; | |
3230 | } | |
3231 | Carp::my_carp_bug("$owner_name_of{$addr}Can't find where the range ought to go. No action taken."); | |
3232 | return; | |
3233 | } | |
3234 | $i = $temp; | |
3235 | } # End of while loop | |
3236 | ||
3237 | if (main::DEBUG && $to_trace) { | |
3238 | trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i; | |
3239 | trace "i= [ $i ]", $r->[$i]; | |
3240 | trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < $range_list_size - 1; | |
3241 | } | |
3242 | ||
3243 | # Here we have found the offset. Cache it as a starting point for the | |
3244 | # next call. | |
3245 | $_search_ranges_cache{$addr} = $i; | |
3246 | return $i; | |
3247 | } | |
3248 | ||
3249 | sub _add_delete { | |
3250 | # Add, replace or delete ranges to or from a list. The $type | |
3251 | # parameter gives which: | |
3252 | # '+' => insert or replace a range, returning a list of any changed | |
3253 | # ranges. | |
3254 | # '-' => delete a range, returning a list of any deleted ranges. | |
3255 | # | |
3256 | # The next three parameters give respectively the start, end, and | |
3257 | # value associated with the range. 'value' should be null unless the | |
3258 | # operation is '+'; | |
3259 | # | |
3260 | # The range list is kept sorted so that the range with the lowest | |
3261 | # starting position is first in the list, and generally, adjacent | |
c1739a4a | 3262 | # ranges with the same values are merged into a single larger one (see |
99870f4d KW |
3263 | # exceptions below). |
3264 | # | |
c1739a4a | 3265 | # There are more parameters; all are key => value pairs: |
99870f4d KW |
3266 | # Type gives the type of the value. It is only valid for '+'. |
3267 | # All ranges have types; if this parameter is omitted, 0 is | |
3268 | # assumed. Ranges with type 0 are assumed to obey the | |
3269 | # Unicode rules for casing, etc; ranges with other types are | |
3270 | # not. Otherwise, the type is arbitrary, for the caller's | |
3271 | # convenience, and looked at only by this routine to keep | |
3272 | # adjacent ranges of different types from being merged into | |
3273 | # a single larger range, and when Replace => | |
3274 | # $IF_NOT_EQUIVALENT is specified (see just below). | |
3275 | # Replace determines what to do if the range list already contains | |
3276 | # ranges which coincide with all or portions of the input | |
3277 | # range. It is only valid for '+': | |
3278 | # => $NO means that the new value is not to replace | |
3279 | # any existing ones, but any empty gaps of the | |
3280 | # range list coinciding with the input range | |
3281 | # will be filled in with the new value. | |
3282 | # => $UNCONDITIONALLY means to replace the existing values with | |
3283 | # this one unconditionally. However, if the | |
3284 | # new and old values are identical, the | |
3285 | # replacement is skipped to save cycles | |
3286 | # => $IF_NOT_EQUIVALENT means to replace the existing values | |
3287 | # with this one if they are not equivalent. | |
3288 | # Ranges are equivalent if their types are the | |
c1739a4a | 3289 | # same, and they are the same string; or if |
99870f4d KW |
3290 | # both are type 0 ranges, if their Unicode |
3291 | # standard forms are identical. In this last | |
3292 | # case, the routine chooses the more "modern" | |
3293 | # one to use. This is because some of the | |
3294 | # older files are formatted with values that | |
3295 | # are, for example, ALL CAPs, whereas the | |
3296 | # derived files have a more modern style, | |
3297 | # which looks better. By looking for this | |
3298 | # style when the pre-existing and replacement | |
3299 | # standard forms are the same, we can move to | |
3300 | # the modern style | |
3301 | # => $MULTIPLE means that if this range duplicates an | |
3302 | # existing one, but has a different value, | |
3303 | # don't replace the existing one, but insert | |
3304 | # this, one so that the same range can occur | |
53d84487 KW |
3305 | # multiple times. They are stored LIFO, so |
3306 | # that the final one inserted is the first one | |
3307 | # returned in an ordered search of the table. | |
99870f4d KW |
3308 | # => anything else is the same as => $IF_NOT_EQUIVALENT |
3309 | # | |
c1739a4a KW |
3310 | # "same value" means identical for non-type-0 ranges, and it means |
3311 | # having the same standard forms for type-0 ranges. | |
99870f4d KW |
3312 | |
3313 | return Carp::carp_too_few_args(\@_, 5) if main::DEBUG && @_ < 5; | |
3314 | ||
3315 | my $self = shift; | |
3316 | my $operation = shift; # '+' for add/replace; '-' for delete; | |
3317 | my $start = shift; | |
3318 | my $end = shift; | |
3319 | my $value = shift; | |
3320 | ||
3321 | my %args = @_; | |
3322 | ||
3323 | $value = "" if not defined $value; # warning: $value can be "0" | |
3324 | ||
3325 | my $replace = delete $args{'Replace'}; | |
3326 | $replace = $IF_NOT_EQUIVALENT unless defined $replace; | |
3327 | ||
3328 | my $type = delete $args{'Type'}; | |
3329 | $type = 0 unless defined $type; | |
3330 | ||
3331 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
3332 | ||
ffe43484 | 3333 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
3334 | |
3335 | if ($operation ne '+' && $operation ne '-') { | |
3336 | Carp::my_carp_bug("$owner_name_of{$addr}First parameter to _add_delete must be '+' or '-'. No action taken."); | |
3337 | return; | |
3338 | } | |
3339 | unless (defined $start && defined $end) { | |
3340 | Carp::my_carp_bug("$owner_name_of{$addr}Undefined start and/or end to _add_delete. No action taken."); | |
3341 | return; | |
3342 | } | |
3343 | unless ($end >= $start) { | |
3344 | Carp::my_carp_bug("$owner_name_of{$addr}End of range (" . sprintf("%04X", $end) . ") must not be before start (" . sprintf("%04X", $start) . "). No action taken."); | |
3345 | return; | |
3346 | } | |
3347 | #local $to_trace = 1 if main::DEBUG; | |
3348 | ||
3349 | if ($operation eq '-') { | |
3350 | if ($replace != $IF_NOT_EQUIVALENT) { | |
3351 | Carp::my_carp_bug("$owner_name_of{$addr}Replace => \$IF_NOT_EQUIVALENT is required when deleting a range from a range list. Assuming Replace => \$IF_NOT_EQUIVALENT."); | |
3352 | $replace = $IF_NOT_EQUIVALENT; | |
3353 | } | |
3354 | if ($type) { | |
3355 | Carp::my_carp_bug("$owner_name_of{$addr}Type => 0 is required when deleting a range from a range list. Assuming Type => 0."); | |
3356 | $type = 0; | |
3357 | } | |
3358 | if ($value ne "") { | |
3359 | Carp::my_carp_bug("$owner_name_of{$addr}Value => \"\" is required when deleting a range from a range list. Assuming Value => \"\"."); | |
3360 | $value = ""; | |
3361 | } | |
3362 | } | |
3363 | ||
3364 | my $r = $ranges{$addr}; # The current list of ranges | |
3365 | my $range_list_size = scalar @$r; # And its size | |
3366 | my $max = $max{$addr}; # The current high code point in | |
3367 | # the list of ranges | |
3368 | ||
3369 | # Do a special case requiring fewer machine cycles when the new range | |
3370 | # starts after the current highest point. The Unicode input data is | |
3371 | # structured so this is common. | |
3372 | if ($start > $max) { | |
3373 | ||
3374 | trace "$owner_name_of{$addr} $operation", sprintf("%04X", $start) . '..' . sprintf("%04X", $end) . " ($value) type=$type" if main::DEBUG && $to_trace; | |
3375 | return if $operation eq '-'; # Deleting a non-existing range is a | |
3376 | # no-op | |
3377 | ||
3378 | # If the new range doesn't logically extend the current final one | |
3379 | # in the range list, create a new range at the end of the range | |
3380 | # list. (max cleverly is initialized to a negative number not | |
3381 | # adjacent to 0 if the range list is empty, so even adding a range | |
3382 | # to an empty range list starting at 0 will have this 'if' | |
3383 | # succeed.) | |
3384 | if ($start > $max + 1 # non-adjacent means can't extend. | |
3385 | || @{$r}[-1]->value ne $value # values differ, can't extend. | |
3386 | || @{$r}[-1]->type != $type # types differ, can't extend. | |
3387 | ) { | |
3388 | push @$r, Range->new($start, $end, | |
3389 | Value => $value, | |
3390 | Type => $type); | |
3391 | } | |
3392 | else { | |
3393 | ||
3394 | # Here, the new range starts just after the current highest in | |
3395 | # the range list, and they have the same type and value. | |
3396 | # Extend the current range to incorporate the new one. | |
3397 | @{$r}[-1]->set_end($end); | |
3398 | } | |
3399 | ||
3400 | # This becomes the new maximum. | |
3401 | $max{$addr} = $end; | |
3402 | ||
3403 | return; | |
3404 | } | |
3405 | #local $to_trace = 0 if main::DEBUG; | |
3406 | ||
3407 | trace "$owner_name_of{$addr} $operation", sprintf("%04X", $start) . '..' . sprintf("%04X", $end) . " ($value) replace=$replace" if main::DEBUG && $to_trace; | |
3408 | ||
3409 | # Here, the input range isn't after the whole rest of the range list. | |
3410 | # Most likely 'splice' will be needed. The rest of the routine finds | |
3411 | # the needed splice parameters, and if necessary, does the splice. | |
3412 | # First, find the offset parameter needed by the splice function for | |
3413 | # the input range. Note that the input range may span multiple | |
3414 | # existing ones, but we'll worry about that later. For now, just find | |
3415 | # the beginning. If the input range is to be inserted starting in a | |
3416 | # position not currently in the range list, it must (obviously) come | |
3417 | # just after the range below it, and just before the range above it. | |
3418 | # Slightly less obviously, it will occupy the position currently | |
3419 | # occupied by the range that is to come after it. More formally, we | |
3420 | # are looking for the position, $i, in the array of ranges, such that: | |
3421 | # | |
3422 | # r[$i-1]->start <= r[$i-1]->end < $start < r[$i]->start <= r[$i]->end | |
3423 | # | |
3424 | # (The ordered relationships within existing ranges are also shown in | |
3425 | # the equation above). However, if the start of the input range is | |
3426 | # within an existing range, the splice offset should point to that | |
3427 | # existing range's position in the list; that is $i satisfies a | |
3428 | # somewhat different equation, namely: | |
3429 | # | |
3430 | #r[$i-1]->start <= r[$i-1]->end < r[$i]->start <= $start <= r[$i]->end | |
3431 | # | |
3432 | # More briefly, $start can come before or after r[$i]->start, and at | |
3433 | # this point, we don't know which it will be. However, these | |
3434 | # two equations share these constraints: | |
3435 | # | |
3436 | # r[$i-1]->end < $start <= r[$i]->end | |
3437 | # | |
3438 | # And that is good enough to find $i. | |
3439 | ||
3440 | my $i = $self->_search_ranges($start); | |
3441 | if (! defined $i) { | |
3442 | Carp::my_carp_bug("Searching $self for range beginning with $start unexpectedly returned undefined. Operation '$operation' not performed"); | |
3443 | return; | |
3444 | } | |
3445 | ||
3446 | # The search function returns $i such that: | |
3447 | # | |
3448 | # r[$i-1]->end < $start <= r[$i]->end | |
3449 | # | |
3450 | # That means that $i points to the first range in the range list | |
3451 | # that could possibly be affected by this operation. We still don't | |
3452 | # know if the start of the input range is within r[$i], or if it | |
3453 | # points to empty space between r[$i-1] and r[$i]. | |
3454 | trace "[$i] is the beginning splice point. Existing range there is ", $r->[$i] if main::DEBUG && $to_trace; | |
3455 | ||
3456 | # Special case the insertion of data that is not to replace any | |
3457 | # existing data. | |
3458 | if ($replace == $NO) { # If $NO, has to be operation '+' | |
3459 | #local $to_trace = 1 if main::DEBUG; | |
3460 | trace "Doesn't replace" if main::DEBUG && $to_trace; | |
3461 | ||
3462 | # Here, the new range is to take effect only on those code points | |
3463 | # that aren't already in an existing range. This can be done by | |
3464 | # looking through the existing range list and finding the gaps in | |
3465 | # the ranges that this new range affects, and then calling this | |
3466 | # function recursively on each of those gaps, leaving untouched | |
3467 | # anything already in the list. Gather up a list of the changed | |
3468 | # gaps first so that changes to the internal state as new ranges | |
3469 | # are added won't be a problem. | |
3470 | my @gap_list; | |
3471 | ||
3472 | # First, if the starting point of the input range is outside an | |
3473 | # existing one, there is a gap from there to the beginning of the | |
3474 | # existing range -- add a span to fill the part that this new | |
3475 | # range occupies | |
3476 | if ($start < $r->[$i]->start) { | |
3477 | push @gap_list, Range->new($start, | |
3478 | main::min($end, | |
3479 | $r->[$i]->start - 1), | |
3480 | Type => $type); | |
3481 | trace "gap before $r->[$i] [$i], will add", $gap_list[-1] if main::DEBUG && $to_trace; | |
3482 | } | |
3483 | ||
3484 | # Then look through the range list for other gaps until we reach | |
3485 | # the highest range affected by the input one. | |
3486 | my $j; | |
3487 | for ($j = $i+1; $j < $range_list_size; $j++) { | |
3488 | trace "j=[$j]", $r->[$j] if main::DEBUG && $to_trace; | |
3489 | last if $end < $r->[$j]->start; | |
3490 | ||
3491 | # If there is a gap between when this range starts and the | |
3492 | # previous one ends, add a span to fill it. Note that just | |
3493 | # because there are two ranges doesn't mean there is a | |
3494 | # non-zero gap between them. It could be that they have | |
3495 | # different values or types | |
3496 | if ($r->[$j-1]->end + 1 != $r->[$j]->start) { | |
3497 | push @gap_list, | |
3498 | Range->new($r->[$j-1]->end + 1, | |
3499 | $r->[$j]->start - 1, | |
3500 | Type => $type); | |
3501 | trace "gap between $r->[$j-1] and $r->[$j] [$j], will add: $gap_list[-1]" if main::DEBUG && $to_trace; | |
3502 | } | |
3503 | } | |
3504 | ||
3505 | # Here, we have either found an existing range in the range list, | |
3506 | # beyond the area affected by the input one, or we fell off the | |
3507 | # end of the loop because the input range affects the whole rest | |
3508 | # of the range list. In either case, $j is 1 higher than the | |
3509 | # highest affected range. If $j == $i, it means that there are no | |
3510 | # affected ranges, that the entire insertion is in the gap between | |
3511 | # r[$i-1], and r[$i], which we already have taken care of before | |
3512 | # the loop. | |
3513 | # On the other hand, if there are affected ranges, it might be | |
3514 | # that there is a gap that needs filling after the final such | |
3515 | # range to the end of the input range | |
3516 | if ($r->[$j-1]->end < $end) { | |
3517 | push @gap_list, Range->new(main::max($start, | |
3518 | $r->[$j-1]->end + 1), | |
3519 | $end, | |
3520 | Type => $type); | |
3521 | trace "gap after $r->[$j-1], will add $gap_list[-1]" if main::DEBUG && $to_trace; | |
3522 | } | |
3523 | ||
3524 | # Call recursively to fill in all the gaps. | |
3525 | foreach my $gap (@gap_list) { | |
3526 | $self->_add_delete($operation, | |
3527 | $gap->start, | |
3528 | $gap->end, | |
3529 | $value, | |
3530 | Type => $type); | |
3531 | } | |
3532 | ||
3533 | return; | |
3534 | } | |
3535 | ||
53d84487 KW |
3536 | # Here, we have taken care of the case where $replace is $NO. |
3537 | # Remember that here, r[$i-1]->end < $start <= r[$i]->end | |
3538 | # If inserting a multiple record, this is where it goes, before the | |
3539 | # first (if any) existing one. This implies an insertion, and no | |
3540 | # change to any existing ranges. Note that $i can be -1 if this new | |
3541 | # range doesn't actually duplicate any existing, and comes at the | |
3542 | # beginning of the list. | |
3543 | if ($replace == $MULTIPLE) { | |
3544 | ||
3545 | if ($start != $end) { | |
3546 | Carp::my_carp_bug("$owner_name_of{$addr}Can't cope with adding a multiple record when the range ($start..$end) contains more than one code point. No action taken."); | |
3547 | return; | |
3548 | } | |
3549 | ||
3550 | # Don't add an exact duplicate, as it isn't really a multiple | |
3551 | if ($end >= $r->[$i]->start) { | |
1f6798c4 KW |
3552 | my $existing_value = $r->[$i]->value; |
3553 | my $existing_type = $r->[$i]->type; | |
3554 | return if $value eq $existing_value && $type eq $existing_type; | |
3555 | ||
3556 | # If the multiple value is part of an existing range, we want | |
3557 | # to split up that range, so that only the single code point | |
3558 | # is affected. To do this, we first call ourselves | |
3559 | # recursively to delete that code point from the table, having | |
3560 | # preserved its current data above. Then we call ourselves | |
3561 | # recursively again to add the new multiple, which we know by | |
3562 | # the test just above is different than the current code | |
3563 | # point's value, so it will become a range containing a single | |
3564 | # code point: just itself. Finally, we add back in the | |
3565 | # pre-existing code point, which will again be a single code | |
3566 | # point range. Because 'i' likely will have changed as a | |
3567 | # result of these operations, we can't just continue on, but | |
3568 | # do this operation recursively as well. | |
53d84487 | 3569 | if ($r->[$i]->start != $r->[$i]->end) { |
1f6798c4 KW |
3570 | $self->_add_delete('-', $start, $end, ""); |
3571 | $self->_add_delete('+', $start, $end, $value, Type => $type); | |
3572 | return $self->_add_delete('+', $start, $end, $existing_value, Type => $existing_type, Replace => $MULTIPLE); | |
53d84487 | 3573 | } |
53d84487 KW |
3574 | } |
3575 | ||
3576 | trace "Adding multiple record at $i with $start..$end, $value" if main::DEBUG && $to_trace; | |
3577 | my @return = splice @$r, | |
3578 | $i, | |
3579 | 0, | |
3580 | Range->new($start, | |
3581 | $end, | |
3582 | Value => $value, | |
3583 | Type => $type); | |
3584 | if (main::DEBUG && $to_trace) { | |
3585 | trace "After splice:"; | |
3586 | trace 'i-2=[', $i-2, ']', $r->[$i-2] if $i >= 2; | |
3587 | trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i >= 1; | |
3588 | trace "i =[", $i, "]", $r->[$i] if $i >= 0; | |
3589 | trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < @$r - 1; | |
3590 | trace 'i+2=[', $i+2, ']', $r->[$i+2] if $i < @$r - 2; | |
3591 | trace 'i+3=[', $i+3, ']', $r->[$i+3] if $i < @$r - 3; | |
3592 | } | |
3593 | return @return; | |
3594 | } | |
3595 | ||
3596 | # Here, we have taken care of $NO and $MULTIPLE replaces. This leaves | |
3597 | # delete, insert, and replace either unconditionally or if not | |
3598 | # equivalent. $i still points to the first potential affected range. | |
3599 | # Now find the highest range affected, which will determine the length | |
3600 | # parameter to splice. (The input range can span multiple existing | |
3601 | # ones.) If this isn't a deletion, while we are looking through the | |
3602 | # range list, see also if this is a replacement rather than a clean | |
3603 | # insertion; that is if it will change the values of at least one | |
3604 | # existing range. Start off assuming it is an insert, until find it | |
3605 | # isn't. | |
3606 | my $clean_insert = $operation eq '+'; | |
99870f4d KW |
3607 | my $j; # This will point to the highest affected range |
3608 | ||
3609 | # For non-zero types, the standard form is the value itself; | |
3610 | my $standard_form = ($type) ? $value : main::standardize($value); | |
3611 | ||
3612 | for ($j = $i; $j < $range_list_size; $j++) { | |
3613 | trace "Looking for highest affected range; the one at $j is ", $r->[$j] if main::DEBUG && $to_trace; | |
3614 | ||
3615 | # If find a range that it doesn't overlap into, we can stop | |
3616 | # searching | |
3617 | last if $end < $r->[$j]->start; | |
3618 | ||
969a34cc KW |
3619 | # Here, overlaps the range at $j. If the values don't match, |
3620 | # and so far we think this is a clean insertion, it becomes a | |
3621 | # non-clean insertion, i.e., a 'change' or 'replace' instead. | |
3622 | if ($clean_insert) { | |
99870f4d | 3623 | if ($r->[$j]->standard_form ne $standard_form) { |
969a34cc | 3624 | $clean_insert = 0; |
56343c78 KW |
3625 | if ($replace == $CROAK) { |
3626 | main::croak("The range to add " | |
3627 | . sprintf("%04X", $start) | |
3628 | . '-' | |
3629 | . sprintf("%04X", $end) | |
3630 | . " with value '$value' overlaps an existing range $r->[$j]"); | |
3631 | } | |
99870f4d KW |
3632 | } |
3633 | else { | |
3634 | ||
3635 | # Here, the two values are essentially the same. If the | |
3636 | # two are actually identical, replacing wouldn't change | |
3637 | # anything so skip it. | |
3638 | my $pre_existing = $r->[$j]->value; | |
3639 | if ($pre_existing ne $value) { | |
3640 | ||
3641 | # Here the new and old standardized values are the | |
3642 | # same, but the non-standardized values aren't. If | |
3643 | # replacing unconditionally, then replace | |
3644 | if( $replace == $UNCONDITIONALLY) { | |
969a34cc | 3645 | $clean_insert = 0; |
99870f4d KW |
3646 | } |
3647 | else { | |
3648 | ||
3649 | # Here, are replacing conditionally. Decide to | |
3650 | # replace or not based on which appears to look | |
3651 | # the "nicest". If one is mixed case and the | |
3652 | # other isn't, choose the mixed case one. | |
3653 | my $new_mixed = $value =~ /[A-Z]/ | |
3654 | && $value =~ /[a-z]/; | |
3655 | my $old_mixed = $pre_existing =~ /[A-Z]/ | |
3656 | && $pre_existing =~ /[a-z]/; | |
3657 | ||
3658 | if ($old_mixed != $new_mixed) { | |
969a34cc | 3659 | $clean_insert = 0 if $new_mixed; |
99870f4d | 3660 | if (main::DEBUG && $to_trace) { |
969a34cc KW |
3661 | if ($clean_insert) { |
3662 | trace "Retaining $pre_existing over $value"; | |
99870f4d KW |
3663 | } |
3664 | else { | |
969a34cc | 3665 | trace "Replacing $pre_existing with $value"; |
99870f4d KW |
3666 | } |
3667 | } | |
3668 | } | |
3669 | else { | |
3670 | ||
3671 | # Here casing wasn't different between the two. | |
3672 | # If one has hyphens or underscores and the | |
3673 | # other doesn't, choose the one with the | |
3674 | # punctuation. | |
3675 | my $new_punct = $value =~ /[-_]/; | |
3676 | my $old_punct = $pre_existing =~ /[-_]/; | |
3677 | ||
3678 | if ($old_punct != $new_punct) { | |
969a34cc | 3679 | $clean_insert = 0 if $new_punct; |
99870f4d | 3680 | if (main::DEBUG && $to_trace) { |
969a34cc KW |
3681 | if ($clean_insert) { |
3682 | trace "Retaining $pre_existing over $value"; | |
99870f4d KW |
3683 | } |
3684 | else { | |
969a34cc | 3685 | trace "Replacing $pre_existing with $value"; |
99870f4d KW |
3686 | } |
3687 | } | |
3688 | } # else existing one is just as "good"; | |
3689 | # retain it to save cycles. | |
3690 | } | |
3691 | } | |
3692 | } | |
3693 | } | |
3694 | } | |
3695 | } # End of loop looking for highest affected range. | |
3696 | ||
3697 | # Here, $j points to one beyond the highest range that this insertion | |
3698 | # affects (hence to beyond the range list if that range is the final | |
3699 | # one in the range list). | |
3700 | ||
3701 | # The splice length is all the affected ranges. Get it before | |
3702 | # subtracting, for efficiency, so we don't have to later add 1. | |
3703 | my $length = $j - $i; | |
3704 | ||
3705 | $j--; # $j now points to the highest affected range. | |
3706 | trace "Final affected range is $j: $r->[$j]" if main::DEBUG && $to_trace; | |
3707 | ||
99870f4d KW |
3708 | # Here, have taken care of $NO and $MULTIPLE replaces. |
3709 | # $j points to the highest affected range. But it can be < $i or even | |
3710 | # -1. These happen only if the insertion is entirely in the gap | |
3711 | # between r[$i-1] and r[$i]. Here's why: j < i means that the j loop | |
3712 | # above exited first time through with $end < $r->[$i]->start. (And | |
3713 | # then we subtracted one from j) This implies also that $start < | |
3714 | # $r->[$i]->start, but we know from above that $r->[$i-1]->end < | |
3715 | # $start, so the entire input range is in the gap. | |
3716 | if ($j < $i) { | |
3717 | ||
3718 | # Here the entire input range is in the gap before $i. | |
3719 | ||
3720 | if (main::DEBUG && $to_trace) { | |
3721 | if ($i) { | |
3722 | trace "Entire range is between $r->[$i-1] and $r->[$i]"; | |
3723 | } | |
3724 | else { | |
3725 | trace "Entire range is before $r->[$i]"; | |
3726 | } | |
3727 | } | |
3728 | return if $operation ne '+'; # Deletion of a non-existent range is | |
3729 | # a no-op | |
3730 | } | |
3731 | else { | |
3732 | ||
969a34cc KW |
3733 | # Here part of the input range is not in the gap before $i. Thus, |
3734 | # there is at least one affected one, and $j points to the highest | |
3735 | # such one. | |
99870f4d KW |
3736 | |
3737 | # At this point, here is the situation: | |
3738 | # This is not an insertion of a multiple, nor of tentative ($NO) | |
3739 | # data. | |
3740 | # $i points to the first element in the current range list that | |
3741 | # may be affected by this operation. In fact, we know | |
3742 | # that the range at $i is affected because we are in | |
3743 | # the else branch of this 'if' | |
3744 | # $j points to the highest affected range. | |
3745 | # In other words, | |
3746 | # r[$i-1]->end < $start <= r[$i]->end | |
3747 | # And: | |
3748 | # r[$i-1]->end < $start <= $end <= r[$j]->end | |
3749 | # | |
3750 | # Also: | |
969a34cc KW |
3751 | # $clean_insert is a boolean which is set true if and only if |
3752 | # this is a "clean insertion", i.e., not a change nor a | |
3753 | # deletion (multiple was handled above). | |
99870f4d KW |
3754 | |
3755 | # We now have enough information to decide if this call is a no-op | |
969a34cc KW |
3756 | # or not. It is a no-op if this is an insertion of already |
3757 | # existing data. | |
99870f4d | 3758 | |
969a34cc | 3759 | if (main::DEBUG && $to_trace && $clean_insert |
99870f4d KW |
3760 | && $i == $j |
3761 | && $start >= $r->[$i]->start) | |
3762 | { | |
3763 | trace "no-op"; | |
3764 | } | |
969a34cc | 3765 | return if $clean_insert |
99870f4d KW |
3766 | && $i == $j # more than one affected range => not no-op |
3767 | ||
3768 | # Here, r[$i-1]->end < $start <= $end <= r[$i]->end | |
3769 | # Further, $start and/or $end is >= r[$i]->start | |
3770 | # The test below hence guarantees that | |
3771 | # r[$i]->start < $start <= $end <= r[$i]->end | |
3772 | # This means the input range is contained entirely in | |
3773 | # the one at $i, so is a no-op | |
3774 | && $start >= $r->[$i]->start; | |
3775 | } | |
3776 | ||
3777 | # Here, we know that some action will have to be taken. We have | |
3778 | # calculated the offset and length (though adjustments may be needed) | |
3779 | # for the splice. Now start constructing the replacement list. | |
3780 | my @replacement; | |
3781 | my $splice_start = $i; | |
3782 | ||
3783 | my $extends_below; | |
3784 | my $extends_above; | |
3785 | ||
3786 | # See if should extend any adjacent ranges. | |
3787 | if ($operation eq '-') { # Don't extend deletions | |
3788 | $extends_below = $extends_above = 0; | |
3789 | } | |
3790 | else { # Here, should extend any adjacent ranges. See if there are | |
3791 | # any. | |
3792 | $extends_below = ($i > 0 | |
3793 | # can't extend unless adjacent | |
3794 | && $r->[$i-1]->end == $start -1 | |
3795 | # can't extend unless are same standard value | |
3796 | && $r->[$i-1]->standard_form eq $standard_form | |
3797 | # can't extend unless share type | |
3798 | && $r->[$i-1]->type == $type); | |
3799 | $extends_above = ($j+1 < $range_list_size | |
3800 | && $r->[$j+1]->start == $end +1 | |
3801 | && $r->[$j+1]->standard_form eq $standard_form | |
23822bda | 3802 | && $r->[$j+1]->type == $type); |
99870f4d KW |
3803 | } |
3804 | if ($extends_below && $extends_above) { # Adds to both | |
3805 | $splice_start--; # start replace at element below | |
3806 | $length += 2; # will replace on both sides | |
3807 | trace "Extends both below and above ranges" if main::DEBUG && $to_trace; | |
3808 | ||
3809 | # The result will fill in any gap, replacing both sides, and | |
3810 | # create one large range. | |
3811 | @replacement = Range->new($r->[$i-1]->start, | |
3812 | $r->[$j+1]->end, | |
3813 | Value => $value, | |
3814 | Type => $type); | |
3815 | } | |
3816 | else { | |
3817 | ||
3818 | # Here we know that the result won't just be the conglomeration of | |
3819 | # a new range with both its adjacent neighbors. But it could | |
3820 | # extend one of them. | |
3821 | ||
3822 | if ($extends_below) { | |
3823 | ||
3824 | # Here the new element adds to the one below, but not to the | |
3825 | # one above. If inserting, and only to that one range, can | |
3826 | # just change its ending to include the new one. | |
969a34cc | 3827 | if ($length == 0 && $clean_insert) { |
99870f4d KW |
3828 | $r->[$i-1]->set_end($end); |
3829 | trace "inserted range extends range to below so it is now $r->[$i-1]" if main::DEBUG && $to_trace; | |
3830 | return; | |
3831 | } | |
3832 | else { | |
3833 | trace "Changing inserted range to start at ", sprintf("%04X", $r->[$i-1]->start), " instead of ", sprintf("%04X", $start) if main::DEBUG && $to_trace; | |
3834 | $splice_start--; # start replace at element below | |
3835 | $length++; # will replace the element below | |
3836 | $start = $r->[$i-1]->start; | |
3837 | } | |
3838 | } | |
3839 | elsif ($extends_above) { | |
3840 | ||
3841 | # Here the new element adds to the one above, but not below. | |
3842 | # Mirror the code above | |
969a34cc | 3843 | if ($length == 0 && $clean_insert) { |
99870f4d KW |
3844 | $r->[$j+1]->set_start($start); |
3845 | trace "inserted range extends range to above so it is now $r->[$j+1]" if main::DEBUG && $to_trace; | |
3846 | return; | |
3847 | } | |
3848 | else { | |
3849 | trace "Changing inserted range to end at ", sprintf("%04X", $r->[$j+1]->end), " instead of ", sprintf("%04X", $end) if main::DEBUG && $to_trace; | |
3850 | $length++; # will replace the element above | |
3851 | $end = $r->[$j+1]->end; | |
3852 | } | |
3853 | } | |
3854 | ||
3855 | trace "Range at $i is $r->[$i]" if main::DEBUG && $to_trace; | |
3856 | ||
3857 | # Finally, here we know there will have to be a splice. | |
3858 | # If the change or delete affects only the highest portion of the | |
3859 | # first affected range, the range will have to be split. The | |
3860 | # splice will remove the whole range, but will replace it by a new | |
3861 | # range containing just the unaffected part. So, in this case, | |
3862 | # add to the replacement list just this unaffected portion. | |
3863 | if (! $extends_below | |
3864 | && $start > $r->[$i]->start && $start <= $r->[$i]->end) | |
3865 | { | |
3866 | push @replacement, | |
3867 | Range->new($r->[$i]->start, | |
3868 | $start - 1, | |
3869 | Value => $r->[$i]->value, | |
3870 | Type => $r->[$i]->type); | |
3871 | } | |
3872 | ||
3873 | # In the case of an insert or change, but not a delete, we have to | |
3874 | # put in the new stuff; this comes next. | |
3875 | if ($operation eq '+') { | |
3876 | push @replacement, Range->new($start, | |
3877 | $end, | |
3878 | Value => $value, | |
3879 | Type => $type); | |
3880 | } | |
3881 | ||
3882 | trace "Range at $j is $r->[$j]" if main::DEBUG && $to_trace && $j != $i; | |
3883 | #trace "$end >=", $r->[$j]->start, " && $end <", $r->[$j]->end if main::DEBUG && $to_trace; | |
3884 | ||
3885 | # And finally, if we're changing or deleting only a portion of the | |
3886 | # highest affected range, it must be split, as the lowest one was. | |
3887 | if (! $extends_above | |
3888 | && $j >= 0 # Remember that j can be -1 if before first | |
3889 | # current element | |
3890 | && $end >= $r->[$j]->start | |
3891 | && $end < $r->[$j]->end) | |
3892 | { | |
3893 | push @replacement, | |
3894 | Range->new($end + 1, | |
3895 | $r->[$j]->end, | |
3896 | Value => $r->[$j]->value, | |
3897 | Type => $r->[$j]->type); | |
3898 | } | |
3899 | } | |
3900 | ||
3901 | # And do the splice, as calculated above | |
3902 | if (main::DEBUG && $to_trace) { | |
3903 | trace "replacing $length element(s) at $i with "; | |
3904 | foreach my $replacement (@replacement) { | |
3905 | trace " $replacement"; | |
3906 | } | |
3907 | trace "Before splice:"; | |
3908 | trace 'i-2=[', $i-2, ']', $r->[$i-2] if $i >= 2; | |
3909 | trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i >= 1; | |
3910 | trace "i =[", $i, "]", $r->[$i]; | |
3911 | trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < @$r - 1; | |
3912 | trace 'i+2=[', $i+2, ']', $r->[$i+2] if $i < @$r - 2; | |
3913 | } | |
3914 | ||
3915 | my @return = splice @$r, $splice_start, $length, @replacement; | |
3916 | ||
3917 | if (main::DEBUG && $to_trace) { | |
3918 | trace "After splice:"; | |
3919 | trace 'i-2=[', $i-2, ']', $r->[$i-2] if $i >= 2; | |
3920 | trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i >= 1; | |
3921 | trace "i =[", $i, "]", $r->[$i]; | |
3922 | trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < @$r - 1; | |
3923 | trace 'i+2=[', $i+2, ']', $r->[$i+2] if $i < @$r - 2; | |
e6451557 | 3924 | trace "removed ", @return if @return; |
99870f4d KW |
3925 | } |
3926 | ||
3927 | # An actual deletion could have changed the maximum in the list. | |
3928 | # There was no deletion if the splice didn't return something, but | |
3929 | # otherwise recalculate it. This is done too rarely to worry about | |
3930 | # performance. | |
3931 | if ($operation eq '-' && @return) { | |
3932 | $max{$addr} = $r->[-1]->end; | |
3933 | } | |
3934 | return @return; | |
3935 | } | |
3936 | ||
3937 | sub reset_each_range { # reset the iterator for each_range(); | |
3938 | my $self = shift; | |
3939 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3940 | ||
f998e60c | 3941 | no overloading; |
051df77b | 3942 | undef $each_range_iterator{pack 'J', $self}; |
99870f4d KW |
3943 | return; |
3944 | } | |
3945 | ||
3946 | sub each_range { | |
3947 | # Iterate over each range in a range list. Results are undefined if | |
3948 | # the range list is changed during the iteration. | |
3949 | ||
3950 | my $self = shift; | |
3951 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3952 | ||
ffe43484 | 3953 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
3954 | |
3955 | return if $self->is_empty; | |
3956 | ||
3957 | $each_range_iterator{$addr} = -1 | |
3958 | if ! defined $each_range_iterator{$addr}; | |
3959 | $each_range_iterator{$addr}++; | |
3960 | return $ranges{$addr}->[$each_range_iterator{$addr}] | |
3961 | if $each_range_iterator{$addr} < @{$ranges{$addr}}; | |
3962 | undef $each_range_iterator{$addr}; | |
3963 | return; | |
3964 | } | |
3965 | ||
3966 | sub count { # Returns count of code points in range list | |
3967 | my $self = shift; | |
3968 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3969 | ||
ffe43484 | 3970 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
3971 | |
3972 | my $count = 0; | |
3973 | foreach my $range (@{$ranges{$addr}}) { | |
3974 | $count += $range->end - $range->start + 1; | |
3975 | } | |
3976 | return $count; | |
3977 | } | |
3978 | ||
3979 | sub delete_range { # Delete a range | |
3980 | my $self = shift; | |
3981 | my $start = shift; | |
3982 | my $end = shift; | |
3983 | ||
3984 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3985 | ||
3986 | return $self->_add_delete('-', $start, $end, ""); | |
3987 | } | |
3988 | ||
3989 | sub is_empty { # Returns boolean as to if a range list is empty | |
3990 | my $self = shift; | |
3991 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
3992 | ||
f998e60c | 3993 | no overloading; |
051df77b | 3994 | return scalar @{$ranges{pack 'J', $self}} == 0; |
99870f4d KW |
3995 | } |
3996 | ||
3997 | sub hash { | |
3998 | # Quickly returns a scalar suitable for separating tables into | |
3999 | # buckets, i.e. it is a hash function of the contents of a table, so | |
4000 | # there are relatively few conflicts. | |
4001 | ||
4002 | my $self = shift; | |
4003 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4004 | ||
ffe43484 | 4005 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
4006 | |
4007 | # These are quickly computable. Return looks like 'min..max;count' | |
4008 | return $self->min . "..$max{$addr};" . scalar @{$ranges{$addr}}; | |
4009 | } | |
4010 | } # End closure for _Range_List_Base | |
4011 | ||
4012 | package Range_List; | |
4013 | use base '_Range_List_Base'; | |
4014 | ||
4015 | # A Range_List is a range list for match tables; i.e. the range values are | |
4016 | # not significant. Thus a number of operations can be safely added to it, | |
4017 | # such as inversion, intersection. Note that union is also an unsafe | |
4018 | # operation when range values are cared about, and that method is in the base | |
4019 | # class, not here. But things are set up so that that method is callable only | |
4020 | # during initialization. Only in this derived class, is there an operation | |
4021 | # that combines two tables. A Range_Map can thus be used to initialize a | |
4022 | # Range_List, and its mappings will be in the list, but are not significant to | |
4023 | # this class. | |
4024 | ||
4025 | sub trace { return main::trace(@_); } | |
4026 | ||
4027 | { # Closure | |
4028 | ||
4029 | use overload | |
4030 | fallback => 0, | |
4031 | '+' => sub { my $self = shift; | |
4032 | my $other = shift; | |
4033 | ||
4034 | return $self->_union($other) | |
4035 | }, | |
4036 | '&' => sub { my $self = shift; | |
4037 | my $other = shift; | |
4038 | ||
4039 | return $self->_intersect($other, 0); | |
4040 | }, | |
4041 | '~' => "_invert", | |
4042 | '-' => "_subtract", | |
4043 | ; | |
4044 | ||
4045 | sub _invert { | |
4046 | # Returns a new Range_List that gives all code points not in $self. | |
4047 | ||
4048 | my $self = shift; | |
4049 | ||
4050 | my $new = Range_List->new; | |
4051 | ||
4052 | # Go through each range in the table, finding the gaps between them | |
4053 | my $max = -1; # Set so no gap before range beginning at 0 | |
4054 | for my $range ($self->ranges) { | |
4055 | my $start = $range->start; | |
4056 | my $end = $range->end; | |
4057 | ||
4058 | # If there is a gap before this range, the inverse will contain | |
4059 | # that gap. | |
4060 | if ($start > $max + 1) { | |
4061 | $new->add_range($max + 1, $start - 1); | |
4062 | } | |
4063 | $max = $end; | |
4064 | } | |
4065 | ||
4066 | # And finally, add the gap from the end of the table to the max | |
4067 | # possible code point | |
6189eadc KW |
4068 | if ($max < $MAX_UNICODE_CODEPOINT) { |
4069 | $new->add_range($max + 1, $MAX_UNICODE_CODEPOINT); | |
99870f4d KW |
4070 | } |
4071 | return $new; | |
4072 | } | |
4073 | ||
4074 | sub _subtract { | |
4075 | # Returns a new Range_List with the argument deleted from it. The | |
4076 | # argument can be a single code point, a range, or something that has | |
4077 | # a range, with the _range_list() method on it returning them | |
4078 | ||
4079 | my $self = shift; | |
4080 | my $other = shift; | |
4081 | my $reversed = shift; | |
4082 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4083 | ||
4084 | if ($reversed) { | |
4085 | Carp::my_carp_bug("Can't cope with a " | |
4086 | . __PACKAGE__ | |
4087 | . " being the second parameter in a '-'. Subtraction ignored."); | |
4088 | return $self; | |
4089 | } | |
4090 | ||
4091 | my $new = Range_List->new(Initialize => $self); | |
4092 | ||
4093 | if (! ref $other) { # Single code point | |
4094 | $new->delete_range($other, $other); | |
4095 | } | |
4096 | elsif ($other->isa('Range')) { | |
4097 | $new->delete_range($other->start, $other->end); | |
4098 | } | |
4099 | elsif ($other->can('_range_list')) { | |
4100 | foreach my $range ($other->_range_list->ranges) { | |
4101 | $new->delete_range($range->start, $range->end); | |
4102 | } | |
4103 | } | |
4104 | else { | |
4105 | Carp::my_carp_bug("Can't cope with a " | |
4106 | . ref($other) | |
4107 | . " argument to '-'. Subtraction ignored." | |
4108 | ); | |
4109 | return $self; | |
4110 | } | |
4111 | ||
4112 | return $new; | |
4113 | } | |
4114 | ||
4115 | sub _intersect { | |
4116 | # Returns either a boolean giving whether the two inputs' range lists | |
4117 | # intersect (overlap), or a new Range_List containing the intersection | |
4118 | # of the two lists. The optional final parameter being true indicates | |
4119 | # to do the check instead of the intersection. | |
4120 | ||
4121 | my $a_object = shift; | |
4122 | my $b_object = shift; | |
4123 | my $check_if_overlapping = shift; | |
4124 | $check_if_overlapping = 0 unless defined $check_if_overlapping; | |
4125 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4126 | ||
4127 | if (! defined $b_object) { | |
4128 | my $message = ""; | |
4129 | $message .= $a_object->_owner_name_of if defined $a_object; | |
4130 | Carp::my_carp_bug($message .= "Called with undefined value. Intersection not done."); | |
4131 | return; | |
4132 | } | |
4133 | ||
4134 | # a & b = !(!a | !b), or in our terminology = ~ ( ~a + -b ) | |
4135 | # Thus the intersection could be much more simply be written: | |
4136 | # return ~(~$a_object + ~$b_object); | |
4137 | # But, this is slower, and when taking the inverse of a large | |
4138 | # range_size_1 table, back when such tables were always stored that | |
4139 | # way, it became prohibitively slow, hence the code was changed to the | |
4140 | # below | |
4141 | ||
4142 | if ($b_object->isa('Range')) { | |
4143 | $b_object = Range_List->new(Initialize => $b_object, | |
4144 | Owner => $a_object->_owner_name_of); | |
4145 | } | |
4146 | $b_object = $b_object->_range_list if $b_object->can('_range_list'); | |
4147 | ||
4148 | my @a_ranges = $a_object->ranges; | |
4149 | my @b_ranges = $b_object->ranges; | |
4150 | ||
4151 | #local $to_trace = 1 if main::DEBUG; | |
4152 | trace "intersecting $a_object with ", scalar @a_ranges, "ranges and $b_object with", scalar @b_ranges, " ranges" if main::DEBUG && $to_trace; | |
4153 | ||
4154 | # Start with the first range in each list | |
4155 | my $a_i = 0; | |
4156 | my $range_a = $a_ranges[$a_i]; | |
4157 | my $b_i = 0; | |
4158 | my $range_b = $b_ranges[$b_i]; | |
4159 | ||
4160 | my $new = __PACKAGE__->new(Owner => $a_object->_owner_name_of) | |
4161 | if ! $check_if_overlapping; | |
4162 | ||
4163 | # If either list is empty, there is no intersection and no overlap | |
4164 | if (! defined $range_a || ! defined $range_b) { | |
4165 | return $check_if_overlapping ? 0 : $new; | |
4166 | } | |
4167 | trace "range_a[$a_i]=$range_a; range_b[$b_i]=$range_b" if main::DEBUG && $to_trace; | |
4168 | ||
4169 | # Otherwise, must calculate the intersection/overlap. Start with the | |
4170 | # very first code point in each list | |
4171 | my $a = $range_a->start; | |
4172 | my $b = $range_b->start; | |
4173 | ||
4174 | # Loop through all the ranges of each list; in each iteration, $a and | |
4175 | # $b are the current code points in their respective lists | |
4176 | while (1) { | |
4177 | ||
4178 | # If $a and $b are the same code point, ... | |
4179 | if ($a == $b) { | |
4180 | ||
4181 | # it means the lists overlap. If just checking for overlap | |
4182 | # know the answer now, | |
4183 | return 1 if $check_if_overlapping; | |
4184 | ||
4185 | # The intersection includes this code point plus anything else | |
4186 | # common to both current ranges. | |
4187 | my $start = $a; | |
4188 | my $end = main::min($range_a->end, $range_b->end); | |
4189 | if (! $check_if_overlapping) { | |
4190 | trace "adding intersection range ", sprintf("%04X", $start) . ".." . sprintf("%04X", $end) if main::DEBUG && $to_trace; | |
4191 | $new->add_range($start, $end); | |
4192 | } | |
4193 | ||
4194 | # Skip ahead to the end of the current intersect | |
4195 | $a = $b = $end; | |
4196 | ||
4197 | # If the current intersect ends at the end of either range (as | |
4198 | # it must for at least one of them), the next possible one | |
4199 | # will be the beginning code point in it's list's next range. | |
4200 | if ($a == $range_a->end) { | |
4201 | $range_a = $a_ranges[++$a_i]; | |
4202 | last unless defined $range_a; | |
4203 | $a = $range_a->start; | |
4204 | } | |
4205 | if ($b == $range_b->end) { | |
4206 | $range_b = $b_ranges[++$b_i]; | |
4207 | last unless defined $range_b; | |
4208 | $b = $range_b->start; | |
4209 | } | |
4210 | ||
4211 | trace "range_a[$a_i]=$range_a; range_b[$b_i]=$range_b" if main::DEBUG && $to_trace; | |
4212 | } | |
4213 | elsif ($a < $b) { | |
4214 | ||
4215 | # Not equal, but if the range containing $a encompasses $b, | |
4216 | # change $a to be the middle of the range where it does equal | |
4217 | # $b, so the next iteration will get the intersection | |
4218 | if ($range_a->end >= $b) { | |
4219 | $a = $b; | |
4220 | } | |
4221 | else { | |
4222 | ||
4223 | # Here, the current range containing $a is entirely below | |
4224 | # $b. Go try to find a range that could contain $b. | |
4225 | $a_i = $a_object->_search_ranges($b); | |
4226 | ||
4227 | # If no range found, quit. | |
4228 | last unless defined $a_i; | |
4229 | ||
4230 | # The search returns $a_i, such that | |
4231 | # range_a[$a_i-1]->end < $b <= range_a[$a_i]->end | |
4232 | # Set $a to the beginning of this new range, and repeat. | |
4233 | $range_a = $a_ranges[$a_i]; | |
4234 | $a = $range_a->start; | |
4235 | } | |
4236 | } | |
4237 | else { # Here, $b < $a. | |
4238 | ||
4239 | # Mirror image code to the leg just above | |
4240 | if ($range_b->end >= $a) { | |
4241 | $b = $a; | |
4242 | } | |
4243 | else { | |
4244 | $b_i = $b_object->_search_ranges($a); | |
4245 | last unless defined $b_i; | |
4246 | $range_b = $b_ranges[$b_i]; | |
4247 | $b = $range_b->start; | |
4248 | } | |
4249 | } | |
4250 | } # End of looping through ranges. | |
4251 | ||
4252 | # Intersection fully computed, or now know that there is no overlap | |
4253 | return $check_if_overlapping ? 0 : $new; | |
4254 | } | |
4255 | ||
4256 | sub overlaps { | |
4257 | # Returns boolean giving whether the two arguments overlap somewhere | |
4258 | ||
4259 | my $self = shift; | |
4260 | my $other = shift; | |
4261 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4262 | ||
4263 | return $self->_intersect($other, 1); | |
4264 | } | |
4265 | ||
4266 | sub add_range { | |
4267 | # Add a range to the list. | |
4268 | ||
4269 | my $self = shift; | |
4270 | my $start = shift; | |
4271 | my $end = shift; | |
4272 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4273 | ||
4274 | return $self->_add_delete('+', $start, $end, ""); | |
4275 | } | |
4276 | ||
09aba7e4 KW |
4277 | sub matches_identically_to { |
4278 | # Return a boolean as to whether or not two Range_Lists match identical | |
4279 | # sets of code points. | |
4280 | ||
4281 | my $self = shift; | |
4282 | my $other = shift; | |
4283 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4284 | ||
4285 | # These are ordered in increasing real time to figure out (at least | |
4286 | # until a patch changes that and doesn't change this) | |
4287 | return 0 if $self->max != $other->max; | |
4288 | return 0 if $self->min != $other->min; | |
4289 | return 0 if $self->range_count != $other->range_count; | |
4290 | return 0 if $self->count != $other->count; | |
4291 | ||
4292 | # Here they could be identical because all the tests above passed. | |
4293 | # The loop below is somewhat simpler since we know they have the same | |
4294 | # number of elements. Compare range by range, until reach the end or | |
4295 | # find something that differs. | |
4296 | my @a_ranges = $self->ranges; | |
4297 | my @b_ranges = $other->ranges; | |
4298 | for my $i (0 .. @a_ranges - 1) { | |
4299 | my $a = $a_ranges[$i]; | |
4300 | my $b = $b_ranges[$i]; | |
4301 | trace "self $a; other $b" if main::DEBUG && $to_trace; | |
c1c2d9e8 KW |
4302 | return 0 if ! defined $b |
4303 | || $a->start != $b->start | |
4304 | || $a->end != $b->end; | |
09aba7e4 KW |
4305 | } |
4306 | return 1; | |
4307 | } | |
4308 | ||
99870f4d KW |
4309 | sub is_code_point_usable { |
4310 | # This used only for making the test script. See if the input | |
4311 | # proposed trial code point is one that Perl will handle. If second | |
4312 | # parameter is 0, it won't select some code points for various | |
4313 | # reasons, noted below. | |
4314 | ||
4315 | my $code = shift; | |
4316 | my $try_hard = shift; | |
4317 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4318 | ||
4319 | return 0 if $code < 0; # Never use a negative | |
4320 | ||
99870f4d KW |
4321 | # shun null. I'm (khw) not sure why this was done, but NULL would be |
4322 | # the character very frequently used. | |
4323 | return $try_hard if $code == 0x0000; | |
4324 | ||
99870f4d KW |
4325 | # shun non-character code points. |
4326 | return $try_hard if $code >= 0xFDD0 && $code <= 0xFDEF; | |
4327 | return $try_hard if ($code & 0xFFFE) == 0xFFFE; # includes FFFF | |
4328 | ||
6189eadc | 4329 | return $try_hard if $code > $MAX_UNICODE_CODEPOINT; # keep in range |
99870f4d KW |
4330 | return $try_hard if $code >= 0xD800 && $code <= 0xDFFF; # no surrogate |
4331 | ||
4332 | return 1; | |
4333 | } | |
4334 | ||
4335 | sub get_valid_code_point { | |
4336 | # Return a code point that's part of the range list. Returns nothing | |
4337 | # if the table is empty or we can't find a suitable code point. This | |
4338 | # used only for making the test script. | |
4339 | ||
4340 | my $self = shift; | |
4341 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4342 | ||
ffe43484 | 4343 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
4344 | |
4345 | # On first pass, don't choose less desirable code points; if no good | |
4346 | # one is found, repeat, allowing a less desirable one to be selected. | |
4347 | for my $try_hard (0, 1) { | |
4348 | ||
4349 | # Look through all the ranges for a usable code point. | |
4350 | for my $set ($self->ranges) { | |
4351 | ||
4352 | # Try the edge cases first, starting with the end point of the | |
4353 | # range. | |
4354 | my $end = $set->end; | |
4355 | return $end if is_code_point_usable($end, $try_hard); | |
4356 | ||
4357 | # End point didn't, work. Start at the beginning and try | |
4358 | # every one until find one that does work. | |
4359 | for my $trial ($set->start .. $end - 1) { | |
4360 | return $trial if is_code_point_usable($trial, $try_hard); | |
4361 | } | |
4362 | } | |
4363 | } | |
4364 | return (); # If none found, give up. | |
4365 | } | |
4366 | ||
4367 | sub get_invalid_code_point { | |
4368 | # Return a code point that's not part of the table. Returns nothing | |
4369 | # if the table covers all code points or a suitable code point can't | |
4370 | # be found. This used only for making the test script. | |
4371 | ||
4372 | my $self = shift; | |
4373 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4374 | ||
4375 | # Just find a valid code point of the inverse, if any. | |
4376 | return Range_List->new(Initialize => ~ $self)->get_valid_code_point; | |
4377 | } | |
4378 | } # end closure for Range_List | |
4379 | ||
4380 | package Range_Map; | |
4381 | use base '_Range_List_Base'; | |
4382 | ||
4383 | # A Range_Map is a range list in which the range values (called maps) are | |
4384 | # significant, and hence shouldn't be manipulated by our other code, which | |
4385 | # could be ambiguous or lose things. For example, in taking the union of two | |
4386 | # lists, which share code points, but which have differing values, which one | |
4387 | # has precedence in the union? | |
4388 | # It turns out that these operations aren't really necessary for map tables, | |
4389 | # and so this class was created to make sure they aren't accidentally | |
4390 | # applied to them. | |
4391 | ||
4392 | { # Closure | |
4393 | ||
4394 | sub add_map { | |
4395 | # Add a range containing a mapping value to the list | |
4396 | ||
4397 | my $self = shift; | |
4398 | # Rest of parameters passed on | |
4399 | ||
4400 | return $self->_add_delete('+', @_); | |
4401 | } | |
4402 | ||
4403 | sub add_duplicate { | |
4404 | # Adds entry to a range list which can duplicate an existing entry | |
4405 | ||
4406 | my $self = shift; | |
4407 | my $code_point = shift; | |
4408 | my $value = shift; | |
4409 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4410 | ||
4411 | return $self->add_map($code_point, $code_point, | |
4412 | $value, Replace => $MULTIPLE); | |
4413 | } | |
4414 | } # End of closure for package Range_Map | |
4415 | ||
4416 | package _Base_Table; | |
4417 | ||
4418 | # A table is the basic data structure that gets written out into a file for | |
4419 | # use by the Perl core. This is the abstract base class implementing the | |
4420 | # common elements from the derived ones. A list of the methods to be | |
4421 | # furnished by an implementing class is just after the constructor. | |
4422 | ||
4423 | sub standardize { return main::standardize($_[0]); } | |
4424 | sub trace { return main::trace(@_); } | |
4425 | ||
4426 | { # Closure | |
4427 | ||
4428 | main::setup_package(); | |
4429 | ||
4430 | my %range_list; | |
4431 | # Object containing the ranges of the table. | |
4432 | main::set_access('range_list', \%range_list, 'p_r', 'p_s'); | |
4433 | ||
4434 | my %full_name; | |
4435 | # The full table name. | |
4436 | main::set_access('full_name', \%full_name, 'r'); | |
4437 | ||
4438 | my %name; | |
4439 | # The table name, almost always shorter | |
4440 | main::set_access('name', \%name, 'r'); | |
4441 | ||
4442 | my %short_name; | |
4443 | # The shortest of all the aliases for this table, with underscores removed | |
4444 | main::set_access('short_name', \%short_name); | |
4445 | ||
4446 | my %nominal_short_name_length; | |
4447 | # The length of short_name before removing underscores | |
4448 | main::set_access('nominal_short_name_length', | |
4449 | \%nominal_short_name_length); | |
4450 | ||
23e33b60 KW |
4451 | my %complete_name; |
4452 | # The complete name, including property. | |
4453 | main::set_access('complete_name', \%complete_name, 'r'); | |
4454 | ||
99870f4d KW |
4455 | my %property; |
4456 | # Parent property this table is attached to. | |
4457 | main::set_access('property', \%property, 'r'); | |
4458 | ||
4459 | my %aliases; | |
c12f2655 KW |
4460 | # Ordered list of alias objects of the table's name. The first ones in |
4461 | # the list are output first in comments | |
99870f4d KW |
4462 | main::set_access('aliases', \%aliases, 'readable_array'); |
4463 | ||
4464 | my %comment; | |
4465 | # A comment associated with the table for human readers of the files | |
4466 | main::set_access('comment', \%comment, 's'); | |
4467 | ||
4468 | my %description; | |
4469 | # A comment giving a short description of the table's meaning for human | |
4470 | # readers of the files. | |
4471 | main::set_access('description', \%description, 'readable_array'); | |
4472 | ||
4473 | my %note; | |
4474 | # A comment giving a short note about the table for human readers of the | |
4475 | # files. | |
4476 | main::set_access('note', \%note, 'readable_array'); | |
4477 | ||
4478 | my %internal_only; | |
c12f2655 | 4479 | # Boolean; if set this table is for internal core Perl only use. |
88c22f80 | 4480 | main::set_access('internal_only', \%internal_only, 'r'); |
99870f4d KW |
4481 | |
4482 | my %find_table_from_alias; | |
4483 | # The parent property passes this pointer to a hash which this class adds | |
4484 | # all its aliases to, so that the parent can quickly take an alias and | |
4485 | # find this table. | |
4486 | main::set_access('find_table_from_alias', \%find_table_from_alias, 'p_r'); | |
4487 | ||
4488 | my %locked; | |
4489 | # After this table is made equivalent to another one; we shouldn't go | |
4490 | # changing the contents because that could mean it's no longer equivalent | |
4491 | main::set_access('locked', \%locked, 'r'); | |
4492 | ||
4493 | my %file_path; | |
4494 | # This gives the final path to the file containing the table. Each | |
4495 | # directory in the path is an element in the array | |
4496 | main::set_access('file_path', \%file_path, 'readable_array'); | |
4497 | ||
4498 | my %status; | |
4499 | # What is the table's status, normal, $OBSOLETE, etc. Enum | |
4500 | main::set_access('status', \%status, 'r'); | |
4501 | ||
4502 | my %status_info; | |
4503 | # A comment about its being obsolete, or whatever non normal status it has | |
4504 | main::set_access('status_info', \%status_info, 'r'); | |
4505 | ||
d867ccfb KW |
4506 | my %caseless_equivalent; |
4507 | # The table this is equivalent to under /i matching, if any. | |
4508 | main::set_access('caseless_equivalent', \%caseless_equivalent, 'r', 's'); | |
4509 | ||
99870f4d KW |
4510 | my %range_size_1; |
4511 | # Is the table to be output with each range only a single code point? | |
4512 | # This is done to avoid breaking existing code that may have come to rely | |
4513 | # on this behavior in previous versions of this program.) | |
4514 | main::set_access('range_size_1', \%range_size_1, 'r', 's'); | |
4515 | ||
4516 | my %perl_extension; | |
4517 | # A boolean set iff this table is a Perl extension to the Unicode | |
4518 | # standard. | |
4519 | main::set_access('perl_extension', \%perl_extension, 'r'); | |
4520 | ||
0c07e538 KW |
4521 | my %output_range_counts; |
4522 | # A boolean set iff this table is to have comments written in the | |
4523 | # output file that contain the number of code points in the range. | |
4524 | # The constructor can override the global flag of the same name. | |
4525 | main::set_access('output_range_counts', \%output_range_counts, 'r'); | |
4526 | ||
f5817e0a KW |
4527 | my %format; |
4528 | # The format of the entries of the table. This is calculated from the | |
4529 | # data in the table (or passed in the constructor). This is an enum e.g., | |
4530 | # $STRING_FORMAT | |
4531 | main::set_access('format', \%format, 'r', 'p_s'); | |
4532 | ||
99870f4d KW |
4533 | sub new { |
4534 | # All arguments are key => value pairs, which you can see below, most | |
33e96e72 | 4535 | # of which match fields documented above. Otherwise: Re_Pod_Entry, |
99870f4d KW |
4536 | # Externally_Ok, and Fuzzy apply to the names of the table, and are |
4537 | # documented in the Alias package | |
4538 | ||
4539 | return Carp::carp_too_few_args(\@_, 2) if main::DEBUG && @_ < 2; | |
4540 | ||
4541 | my $class = shift; | |
4542 | ||
4543 | my $self = bless \do { my $anonymous_scalar }, $class; | |
ffe43484 | 4544 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
4545 | |
4546 | my %args = @_; | |
4547 | ||
4548 | $name{$addr} = delete $args{'Name'}; | |
4549 | $find_table_from_alias{$addr} = delete $args{'_Alias_Hash'}; | |
4550 | $full_name{$addr} = delete $args{'Full_Name'}; | |
23e33b60 KW |
4551 | my $complete_name = $complete_name{$addr} |
4552 | = delete $args{'Complete_Name'}; | |
f5817e0a | 4553 | $format{$addr} = delete $args{'Format'}; |
a3db1318 | 4554 | $internal_only{$addr} = delete $args{'Internal_Only'} || 0; |
0c07e538 | 4555 | $output_range_counts{$addr} = delete $args{'Output_Range_Counts'}; |
99870f4d KW |
4556 | $property{$addr} = delete $args{'_Property'}; |
4557 | $range_list{$addr} = delete $args{'_Range_List'}; | |
4558 | $status{$addr} = delete $args{'Status'} || $NORMAL; | |
4559 | $status_info{$addr} = delete $args{'_Status_Info'} || ""; | |
4560 | $range_size_1{$addr} = delete $args{'Range_Size_1'} || 0; | |
d867ccfb | 4561 | $caseless_equivalent{$addr} = delete $args{'Caseless_Equivalent'} || 0; |
99870f4d KW |
4562 | |
4563 | my $description = delete $args{'Description'}; | |
4564 | my $externally_ok = delete $args{'Externally_Ok'}; | |
4565 | my $loose_match = delete $args{'Fuzzy'}; | |
4566 | my $note = delete $args{'Note'}; | |
33e96e72 | 4567 | my $make_re_pod_entry = delete $args{'Re_Pod_Entry'}; |
37e2e78e | 4568 | my $perl_extension = delete $args{'Perl_Extension'}; |
99870f4d KW |
4569 | |
4570 | # Shouldn't have any left over | |
4571 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
4572 | ||
4573 | # Can't use || above because conceivably the name could be 0, and | |
4574 | # can't use // operator in case this program gets used in Perl 5.8 | |
4575 | $full_name{$addr} = $name{$addr} if ! defined $full_name{$addr}; | |
0c07e538 KW |
4576 | $output_range_counts{$addr} = $output_range_counts if |
4577 | ! defined $output_range_counts{$addr}; | |
99870f4d KW |
4578 | |
4579 | $aliases{$addr} = [ ]; | |
4580 | $comment{$addr} = [ ]; | |
4581 | $description{$addr} = [ ]; | |
4582 | $note{$addr} = [ ]; | |
4583 | $file_path{$addr} = [ ]; | |
4584 | $locked{$addr} = ""; | |
4585 | ||
4586 | push @{$description{$addr}}, $description if $description; | |
4587 | push @{$note{$addr}}, $note if $note; | |
4588 | ||
37e2e78e KW |
4589 | if ($status{$addr} eq $PLACEHOLDER) { |
4590 | ||
4591 | # A placeholder table doesn't get documented, is a perl extension, | |
4592 | # and quite likely will be empty | |
33e96e72 | 4593 | $make_re_pod_entry = 0 if ! defined $make_re_pod_entry; |
37e2e78e KW |
4594 | $perl_extension = 1 if ! defined $perl_extension; |
4595 | push @tables_that_may_be_empty, $complete_name{$addr}; | |
4596 | } | |
4597 | elsif (! $status{$addr}) { | |
4598 | ||
4599 | # If hasn't set its status already, see if it is on one of the | |
4600 | # lists of properties or tables that have particular statuses; if | |
4601 | # not, is normal. The lists are prioritized so the most serious | |
4602 | # ones are checked first | |
ec11e5f4 | 4603 | if (exists $why_suppressed{$complete_name} |
98dc9551 | 4604 | # Don't suppress if overridden |
ec11e5f4 KW |
4605 | && ! grep { $_ eq $complete_name{$addr} } |
4606 | @output_mapped_properties) | |
4607 | { | |
99870f4d KW |
4608 | $status{$addr} = $SUPPRESSED; |
4609 | } | |
4610 | elsif (exists $why_deprecated{$complete_name}) { | |
4611 | $status{$addr} = $DEPRECATED; | |
4612 | } | |
4613 | elsif (exists $why_stabilized{$complete_name}) { | |
4614 | $status{$addr} = $STABILIZED; | |
4615 | } | |
4616 | elsif (exists $why_obsolete{$complete_name}) { | |
4617 | $status{$addr} = $OBSOLETE; | |
4618 | } | |
4619 | ||
4620 | # Existence above doesn't necessarily mean there is a message | |
4621 | # associated with it. Use the most serious message. | |
4622 | if ($status{$addr}) { | |
4623 | if ($why_suppressed{$complete_name}) { | |
4624 | $status_info{$addr} | |
4625 | = $why_suppressed{$complete_name}; | |
4626 | } | |
4627 | elsif ($why_deprecated{$complete_name}) { | |
4628 | $status_info{$addr} | |
4629 | = $why_deprecated{$complete_name}; | |
4630 | } | |
4631 | elsif ($why_stabilized{$complete_name}) { | |
4632 | $status_info{$addr} | |
4633 | = $why_stabilized{$complete_name}; | |
4634 | } | |
4635 | elsif ($why_obsolete{$complete_name}) { | |
4636 | $status_info{$addr} | |
4637 | = $why_obsolete{$complete_name}; | |
4638 | } | |
4639 | } | |
4640 | } | |
4641 | ||
37e2e78e KW |
4642 | $perl_extension{$addr} = $perl_extension || 0; |
4643 | ||
8050d00f | 4644 | # Don't list a property by default that is internal only |
33e96e72 | 4645 | $make_re_pod_entry = 0 if ! defined $make_re_pod_entry |
bf0b2993 | 4646 | && $internal_only{$addr}; |
8050d00f | 4647 | |
99870f4d KW |
4648 | # By convention what typically gets printed only or first is what's |
4649 | # first in the list, so put the full name there for good output | |
4650 | # clarity. Other routines rely on the full name being first on the | |
4651 | # list | |
4652 | $self->add_alias($full_name{$addr}, | |
4653 | Externally_Ok => $externally_ok, | |
4654 | Fuzzy => $loose_match, | |
33e96e72 | 4655 | Re_Pod_Entry => $make_re_pod_entry, |
99870f4d KW |
4656 | Status => $status{$addr}, |
4657 | ); | |
4658 | ||
4659 | # Then comes the other name, if meaningfully different. | |
4660 | if (standardize($full_name{$addr}) ne standardize($name{$addr})) { | |
4661 | $self->add_alias($name{$addr}, | |
4662 | Externally_Ok => $externally_ok, | |
4663 | Fuzzy => $loose_match, | |
33e96e72 | 4664 | Re_Pod_Entry => $make_re_pod_entry, |
99870f4d KW |
4665 | Status => $status{$addr}, |
4666 | ); | |
4667 | } | |
4668 | ||
4669 | return $self; | |
4670 | } | |
4671 | ||
4672 | # Here are the methods that are required to be defined by any derived | |
4673 | # class | |
ea25a9b2 | 4674 | for my $sub (qw( |
668b3bfc | 4675 | handle_special_range |
99870f4d | 4676 | append_to_body |
99870f4d | 4677 | pre_body |
ea25a9b2 | 4678 | )) |
668b3bfc KW |
4679 | # write() knows how to write out normal ranges, but it calls |
4680 | # handle_special_range() when it encounters a non-normal one. | |
4681 | # append_to_body() is called by it after it has handled all | |
4682 | # ranges to add anything after the main portion of the table. | |
4683 | # And finally, pre_body() is called after all this to build up | |
4684 | # anything that should appear before the main portion of the | |
4685 | # table. Doing it this way allows things in the middle to | |
4686 | # affect what should appear before the main portion of the | |
99870f4d | 4687 | # table. |
99870f4d KW |
4688 | { |
4689 | no strict "refs"; | |
4690 | *$sub = sub { | |
4691 | Carp::my_carp_bug( __LINE__ | |
4692 | . ": Must create method '$sub()' for " | |
4693 | . ref shift); | |
4694 | return; | |
4695 | } | |
4696 | } | |
4697 | ||
4698 | use overload | |
4699 | fallback => 0, | |
4700 | "." => \&main::_operator_dot, | |
4701 | '!=' => \&main::_operator_not_equal, | |
4702 | '==' => \&main::_operator_equal, | |
4703 | ; | |
4704 | ||
4705 | sub ranges { | |
4706 | # Returns the array of ranges associated with this table. | |
4707 | ||
f998e60c | 4708 | no overloading; |
051df77b | 4709 | return $range_list{pack 'J', shift}->ranges; |
99870f4d KW |
4710 | } |
4711 | ||
4712 | sub add_alias { | |
4713 | # Add a synonym for this table. | |
4714 | ||
4715 | return Carp::carp_too_few_args(\@_, 3) if main::DEBUG && @_ < 3; | |
4716 | ||
4717 | my $self = shift; | |
4718 | my $name = shift; # The name to add. | |
4719 | my $pointer = shift; # What the alias hash should point to. For | |
4720 | # map tables, this is the parent property; | |
4721 | # for match tables, it is the table itself. | |
4722 | ||
4723 | my %args = @_; | |
4724 | my $loose_match = delete $args{'Fuzzy'}; | |
4725 | ||
33e96e72 KW |
4726 | my $make_re_pod_entry = delete $args{'Re_Pod_Entry'}; |
4727 | $make_re_pod_entry = $YES unless defined $make_re_pod_entry; | |
99870f4d KW |
4728 | |
4729 | my $externally_ok = delete $args{'Externally_Ok'}; | |
4730 | $externally_ok = 1 unless defined $externally_ok; | |
4731 | ||
4732 | my $status = delete $args{'Status'}; | |
4733 | $status = $NORMAL unless defined $status; | |
4734 | ||
4735 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
4736 | ||
4737 | # Capitalize the first letter of the alias unless it is one of the CJK | |
4738 | # ones which specifically begins with a lower 'k'. Do this because | |
4739 | # Unicode has varied whether they capitalize first letters or not, and | |
4740 | # have later changed their minds and capitalized them, but not the | |
4741 | # other way around. So do it always and avoid changes from release to | |
4742 | # release | |
4743 | $name = ucfirst($name) unless $name =~ /^k[A-Z]/; | |
4744 | ||
ffe43484 | 4745 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
4746 | |
4747 | # Figure out if should be loosely matched if not already specified. | |
4748 | if (! defined $loose_match) { | |
4749 | ||
4750 | # Is a loose_match if isn't null, and doesn't begin with an | |
4751 | # underscore and isn't just a number | |
4752 | if ($name ne "" | |
4753 | && substr($name, 0, 1) ne '_' | |
4754 | && $name !~ qr{^[0-9_.+-/]+$}) | |
4755 | { | |
4756 | $loose_match = 1; | |
4757 | } | |
4758 | else { | |
4759 | $loose_match = 0; | |
4760 | } | |
4761 | } | |
4762 | ||
4763 | # If this alias has already been defined, do nothing. | |
4764 | return if defined $find_table_from_alias{$addr}->{$name}; | |
4765 | ||
4766 | # That includes if it is standardly equivalent to an existing alias, | |
4767 | # in which case, add this name to the list, so won't have to search | |
4768 | # for it again. | |
4769 | my $standard_name = main::standardize($name); | |
4770 | if (defined $find_table_from_alias{$addr}->{$standard_name}) { | |
4771 | $find_table_from_alias{$addr}->{$name} | |
4772 | = $find_table_from_alias{$addr}->{$standard_name}; | |
4773 | return; | |
4774 | } | |
4775 | ||
4776 | # Set the index hash for this alias for future quick reference. | |
4777 | $find_table_from_alias{$addr}->{$name} = $pointer; | |
4778 | $find_table_from_alias{$addr}->{$standard_name} = $pointer; | |
4779 | local $to_trace = 0 if main::DEBUG; | |
4780 | trace "adding alias $name to $pointer" if main::DEBUG && $to_trace; | |
4781 | trace "adding alias $standard_name to $pointer" if main::DEBUG && $to_trace; | |
4782 | ||
4783 | ||
4784 | # Put the new alias at the end of the list of aliases unless the final | |
4785 | # element begins with an underscore (meaning it is for internal perl | |
4786 | # use) or is all numeric, in which case, put the new one before that | |
4787 | # one. This floats any all-numeric or underscore-beginning aliases to | |
4788 | # the end. This is done so that they are listed last in output lists, | |
4789 | # to encourage the user to use a better name (either more descriptive | |
4790 | # or not an internal-only one) instead. This ordering is relied on | |
4791 | # implicitly elsewhere in this program, like in short_name() | |
4792 | my $list = $aliases{$addr}; | |
4793 | my $insert_position = (@$list == 0 | |
4794 | || (substr($list->[-1]->name, 0, 1) ne '_' | |
4795 | && $list->[-1]->name =~ /\D/)) | |
4796 | ? @$list | |
4797 | : @$list - 1; | |
4798 | splice @$list, | |
4799 | $insert_position, | |
4800 | 0, | |
33e96e72 | 4801 | Alias->new($name, $loose_match, $make_re_pod_entry, |
99870f4d KW |
4802 | $externally_ok, $status); |
4803 | ||
4804 | # This name may be shorter than any existing ones, so clear the cache | |
4805 | # of the shortest, so will have to be recalculated. | |
f998e60c | 4806 | no overloading; |
051df77b | 4807 | undef $short_name{pack 'J', $self}; |
99870f4d KW |
4808 | return; |
4809 | } | |
4810 | ||
4811 | sub short_name { | |
4812 | # Returns a name suitable for use as the base part of a file name. | |
4813 | # That is, shorter wins. It can return undef if there is no suitable | |
4814 | # name. The name has all non-essential underscores removed. | |
4815 | ||
4816 | # The optional second parameter is a reference to a scalar in which | |
4817 | # this routine will store the length the returned name had before the | |
4818 | # underscores were removed, or undef if the return is undef. | |
4819 | ||
4820 | # The shortest name can change if new aliases are added. So using | |
4821 | # this should be deferred until after all these are added. The code | |
4822 | # that does that should clear this one's cache. | |
4823 | # Any name with alphabetics is preferred over an all numeric one, even | |
4824 | # if longer. | |
4825 | ||
4826 | my $self = shift; | |
4827 | my $nominal_length_ptr = shift; | |
4828 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4829 | ||
ffe43484 | 4830 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
4831 | |
4832 | # For efficiency, don't recalculate, but this means that adding new | |
4833 | # aliases could change what the shortest is, so the code that does | |
4834 | # that needs to undef this. | |
4835 | if (defined $short_name{$addr}) { | |
4836 | if ($nominal_length_ptr) { | |
4837 | $$nominal_length_ptr = $nominal_short_name_length{$addr}; | |
4838 | } | |
4839 | return $short_name{$addr}; | |
4840 | } | |
4841 | ||
4842 | # Look at each alias | |
4843 | foreach my $alias ($self->aliases()) { | |
4844 | ||
4845 | # Don't use an alias that isn't ok to use for an external name. | |
4846 | next if ! $alias->externally_ok; | |
4847 | ||
4848 | my $name = main::Standardize($alias->name); | |
4849 | trace $self, $name if main::DEBUG && $to_trace; | |
4850 | ||
4851 | # Take the first one, or a shorter one that isn't numeric. This | |
4852 | # relies on numeric aliases always being last in the array | |
4853 | # returned by aliases(). Any alpha one will have precedence. | |
4854 | if (! defined $short_name{$addr} | |
4855 | || ($name =~ /\D/ | |
4856 | && length($name) < length($short_name{$addr}))) | |
4857 | { | |
4858 | # Remove interior underscores. | |
4859 | ($short_name{$addr} = $name) =~ s/ (?<= . ) _ (?= . ) //xg; | |
4860 | ||
4861 | $nominal_short_name_length{$addr} = length $name; | |
4862 | } | |
4863 | } | |
4864 | ||
ff485b9e KW |
4865 | # If the short name isn't a nice one, perhaps an equivalent table has |
4866 | # a better one. | |
4867 | if (! defined $short_name{$addr} | |
4868 | || $short_name{$addr} eq "" | |
4869 | || $short_name{$addr} eq "_") | |
4870 | { | |
4871 | my $return; | |
4872 | foreach my $follower ($self->children) { # All equivalents | |
4873 | my $follower_name = $follower->short_name; | |
4874 | next unless defined $follower_name; | |
4875 | ||
4876 | # Anything (except undefined) is better than underscore or | |
4877 | # empty | |
4878 | if (! defined $return || $return eq "_") { | |
4879 | $return = $follower_name; | |
4880 | next; | |
4881 | } | |
4882 | ||
4883 | # If the new follower name isn't "_" and is shorter than the | |
4884 | # current best one, prefer the new one. | |
4885 | next if $follower_name eq "_"; | |
4886 | next if length $follower_name > length $return; | |
4887 | $return = $follower_name; | |
4888 | } | |
4889 | $short_name{$addr} = $return if defined $return; | |
4890 | } | |
4891 | ||
99870f4d KW |
4892 | # If no suitable external name return undef |
4893 | if (! defined $short_name{$addr}) { | |
4894 | $$nominal_length_ptr = undef if $nominal_length_ptr; | |
4895 | return; | |
4896 | } | |
4897 | ||
c12f2655 | 4898 | # Don't allow a null short name. |
99870f4d KW |
4899 | if ($short_name{$addr} eq "") { |
4900 | $short_name{$addr} = '_'; | |
4901 | $nominal_short_name_length{$addr} = 1; | |
4902 | } | |
4903 | ||
4904 | trace $self, $short_name{$addr} if main::DEBUG && $to_trace; | |
4905 | ||
4906 | if ($nominal_length_ptr) { | |
4907 | $$nominal_length_ptr = $nominal_short_name_length{$addr}; | |
4908 | } | |
4909 | return $short_name{$addr}; | |
4910 | } | |
4911 | ||
4912 | sub external_name { | |
4913 | # Returns the external name that this table should be known by. This | |
c12f2655 KW |
4914 | # is usually the short_name, but not if the short_name is undefined, |
4915 | # in which case the external_name is arbitrarily set to the | |
4916 | # underscore. | |
99870f4d KW |
4917 | |
4918 | my $self = shift; | |
4919 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4920 | ||
4921 | my $short = $self->short_name; | |
4922 | return $short if defined $short; | |
4923 | ||
4924 | return '_'; | |
4925 | } | |
4926 | ||
4927 | sub add_description { # Adds the parameter as a short description. | |
4928 | ||
4929 | my $self = shift; | |
4930 | my $description = shift; | |
4931 | chomp $description; | |
4932 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4933 | ||
f998e60c | 4934 | no overloading; |
051df77b | 4935 | push @{$description{pack 'J', $self}}, $description; |
99870f4d KW |
4936 | |
4937 | return; | |
4938 | } | |
4939 | ||
4940 | sub add_note { # Adds the parameter as a short note. | |
4941 | ||
4942 | my $self = shift; | |
4943 | my $note = shift; | |
4944 | chomp $note; | |
4945 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4946 | ||
f998e60c | 4947 | no overloading; |
051df77b | 4948 | push @{$note{pack 'J', $self}}, $note; |
99870f4d KW |
4949 | |
4950 | return; | |
4951 | } | |
4952 | ||
4953 | sub add_comment { # Adds the parameter as a comment. | |
4954 | ||
bd9ebcfd KW |
4955 | return unless $debugging_build; |
4956 | ||
99870f4d KW |
4957 | my $self = shift; |
4958 | my $comment = shift; | |
4959 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4960 | ||
4961 | chomp $comment; | |
f998e60c KW |
4962 | |
4963 | no overloading; | |
051df77b | 4964 | push @{$comment{pack 'J', $self}}, $comment; |
99870f4d KW |
4965 | |
4966 | return; | |
4967 | } | |
4968 | ||
4969 | sub comment { | |
4970 | # Return the current comment for this table. If called in list | |
4971 | # context, returns the array of comments. In scalar, returns a string | |
4972 | # of each element joined together with a period ending each. | |
4973 | ||
4974 | my $self = shift; | |
4975 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4976 | ||
ffe43484 | 4977 | my $addr = do { no overloading; pack 'J', $self; }; |
f998e60c | 4978 | my @list = @{$comment{$addr}}; |
99870f4d KW |
4979 | return @list if wantarray; |
4980 | my $return = ""; | |
4981 | foreach my $sentence (@list) { | |
4982 | $return .= '. ' if $return; | |
4983 | $return .= $sentence; | |
4984 | $return =~ s/\.$//; | |
4985 | } | |
4986 | $return .= '.' if $return; | |
4987 | return $return; | |
4988 | } | |
4989 | ||
4990 | sub initialize { | |
4991 | # Initialize the table with the argument which is any valid | |
4992 | # initialization for range lists. | |
4993 | ||
4994 | my $self = shift; | |
ffe43484 | 4995 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
4996 | my $initialization = shift; |
4997 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
4998 | ||
4999 | # Replace the current range list with a new one of the same exact | |
5000 | # type. | |
f998e60c KW |
5001 | my $class = ref $range_list{$addr}; |
5002 | $range_list{$addr} = $class->new(Owner => $self, | |
99870f4d KW |
5003 | Initialize => $initialization); |
5004 | return; | |
5005 | ||
5006 | } | |
5007 | ||
5008 | sub header { | |
5009 | # The header that is output for the table in the file it is written | |
5010 | # in. | |
5011 | ||
5012 | my $self = shift; | |
5013 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5014 | ||
5015 | my $return = ""; | |
5016 | $return .= $DEVELOPMENT_ONLY if $compare_versions; | |
5017 | $return .= $HEADER; | |
99870f4d KW |
5018 | return $return; |
5019 | } | |
5020 | ||
5021 | sub write { | |
668b3bfc KW |
5022 | # Write a representation of the table to its file. It calls several |
5023 | # functions furnished by sub-classes of this abstract base class to | |
5024 | # handle non-normal ranges, to add stuff before the table, and at its | |
5025 | # end. | |
99870f4d KW |
5026 | |
5027 | my $self = shift; | |
5028 | my $tab_stops = shift; # The number of tab stops over to put any | |
5029 | # comment. | |
5030 | my $suppress_value = shift; # Optional, if the value associated with | |
5031 | # a range equals this one, don't write | |
5032 | # the range | |
5033 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5034 | ||
ffe43484 | 5035 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5036 | |
5037 | # Start with the header | |
668b3bfc | 5038 | my @HEADER = $self->header; |
99870f4d KW |
5039 | |
5040 | # Then the comments | |
668b3bfc | 5041 | push @HEADER, "\n", main::simple_fold($comment{$addr}, '# '), "\n" |
99870f4d KW |
5042 | if $comment{$addr}; |
5043 | ||
668b3bfc KW |
5044 | # Things discovered processing the main body of the document may |
5045 | # affect what gets output before it, therefore pre_body() isn't called | |
5046 | # until after all other processing of the table is done. | |
99870f4d | 5047 | |
c4019d52 KW |
5048 | # The main body looks like a 'here' document. If annotating, get rid |
5049 | # of the comments before passing to the caller, as some callers, such | |
5050 | # as charnames.pm, can't cope with them. (Outputting range counts | |
5051 | # also introduces comments, but these don't show up in the tables that | |
5052 | # can't cope with comments, and there aren't that many of them that | |
5053 | # it's worth the extra real time to get rid of them). | |
668b3bfc | 5054 | my @OUT; |
558712cf | 5055 | if ($annotate) { |
c4019d52 KW |
5056 | # Use the line below in Perls that don't have /r |
5057 | #push @OUT, 'return join "\n", map { s/\s*#.*//mg; $_ } split "\n", <<\'END\';' . "\n"; | |
5058 | push @OUT, "return <<'END' =~ s/\\s*#.*//mgr;\n"; | |
5059 | } else { | |
5060 | push @OUT, "return <<'END';\n"; | |
5061 | } | |
99870f4d KW |
5062 | |
5063 | if ($range_list{$addr}->is_empty) { | |
5064 | ||
5065 | # This is a kludge for empty tables to silence a warning in | |
5066 | # utf8.c, which can't really deal with empty tables, but it can | |
5067 | # deal with a table that matches nothing, as the inverse of 'Any' | |
5068 | # does. | |
67a53d68 | 5069 | push @OUT, "!utf8::Any\n"; |
99870f4d | 5070 | } |
c69a9c68 KW |
5071 | elsif ($self->name eq 'N' |
5072 | ||
5073 | # To save disk space and table cache space, avoid putting out | |
5074 | # binary N tables, but instead create a file which just inverts | |
5075 | # the Y table. Since the file will still exist and occupy a | |
5076 | # certain number of blocks, might as well output the whole | |
5077 | # thing if it all will fit in one block. The number of | |
5078 | # ranges below is an approximate number for that. | |
06f26c45 KW |
5079 | && ($self->property->type == $BINARY |
5080 | || $self->property->type == $FORCED_BINARY) | |
c69a9c68 KW |
5081 | # && $self->property->tables == 2 Can't do this because the |
5082 | # non-binary properties, like NFDQC aren't specifiable | |
5083 | # by the notation | |
5084 | && $range_list{$addr}->ranges > 15 | |
5085 | && ! $annotate) # Under --annotate, want to see everything | |
5086 | { | |
5087 | push @OUT, "!utf8::" . $self->property->name . "\n"; | |
5088 | } | |
99870f4d KW |
5089 | else { |
5090 | my $range_size_1 = $range_size_1{$addr}; | |
558712cf KW |
5091 | my $format; # Used only in $annotate option |
5092 | my $include_name; # Used only in $annotate option | |
c4019d52 | 5093 | |
558712cf | 5094 | if ($annotate) { |
c4019d52 KW |
5095 | |
5096 | # if annotating each code point, must print 1 per line. | |
5097 | # The variable could point to a subroutine, and we don't want | |
5098 | # to lose that fact, so only set if not set already | |
5099 | $range_size_1 = 1 if ! $range_size_1; | |
5100 | ||
5101 | $format = $self->format; | |
5102 | ||
5103 | # The name of the character is output only for tables that | |
5104 | # don't already include the name in the output. | |
5105 | my $property = $self->property; | |
5106 | $include_name = | |
5107 | ! ($property == $perl_charname | |
5108 | || $property == main::property_ref('Unicode_1_Name') | |
5109 | || $property == main::property_ref('Name') | |
5110 | || $property == main::property_ref('Name_Alias') | |
5111 | ); | |
5112 | } | |
99870f4d KW |
5113 | |
5114 | # Output each range as part of the here document. | |
5a2b5ddb | 5115 | RANGE: |
99870f4d | 5116 | for my $set ($range_list{$addr}->ranges) { |
5a2b5ddb KW |
5117 | if ($set->type != 0) { |
5118 | $self->handle_special_range($set); | |
5119 | next RANGE; | |
5120 | } | |
99870f4d KW |
5121 | my $start = $set->start; |
5122 | my $end = $set->end; | |
5123 | my $value = $set->value; | |
5124 | ||
5125 | # Don't output ranges whose value is the one to suppress | |
c4019d52 KW |
5126 | next RANGE if defined $suppress_value |
5127 | && $value eq $suppress_value; | |
99870f4d | 5128 | |
c4019d52 KW |
5129 | # If there is a range and doesn't need a single point range |
5130 | # output | |
5131 | if ($start != $end && ! $range_size_1) { | |
bd9ebcfd KW |
5132 | push @OUT, sprintf "%04X\t%04X", $start, $end; |
5133 | $OUT[-1] .= "\t$value" if $value ne ""; | |
99870f4d KW |
5134 | |
5135 | # Add a comment with the size of the range, if requested. | |
5136 | # Expand Tabs to make sure they all start in the same | |
5137 | # column, and then unexpand to use mostly tabs. | |
0c07e538 | 5138 | if (! $output_range_counts{$addr}) { |
99870f4d KW |
5139 | $OUT[-1] .= "\n"; |
5140 | } | |
5141 | else { | |
5142 | $OUT[-1] = Text::Tabs::expand($OUT[-1]); | |
5143 | my $count = main::clarify_number($end - $start + 1); | |
5144 | use integer; | |
5145 | ||
5146 | my $width = $tab_stops * 8 - 1; | |
5147 | $OUT[-1] = sprintf("%-*s # [%s]\n", | |
5148 | $width, | |
5149 | $OUT[-1], | |
5150 | $count); | |
5151 | $OUT[-1] = Text::Tabs::unexpand($OUT[-1]); | |
5152 | } | |
c4019d52 KW |
5153 | next RANGE; |
5154 | } | |
5155 | ||
5156 | # Here to output a single code point per line | |
5157 | ||
5158 | # If not to annotate, use the simple formats | |
558712cf | 5159 | if (! $annotate) { |
c4019d52 KW |
5160 | |
5161 | # Use any passed in subroutine to output. | |
5162 | if (ref $range_size_1 eq 'CODE') { | |
5163 | for my $i ($start .. $end) { | |
5164 | push @OUT, &{$range_size_1}($i, $value); | |
5165 | } | |
5166 | } | |
5167 | else { | |
5168 | ||
5169 | # Here, caller is ok with default output. | |
5170 | for (my $i = $start; $i <= $end; $i++) { | |
5171 | push @OUT, sprintf "%04X\t\t%s\n", $i, $value; | |
5172 | } | |
5173 | } | |
5174 | next RANGE; | |
5175 | } | |
5176 | ||
5177 | # Here, wants annotation. | |
5178 | for (my $i = $start; $i <= $end; $i++) { | |
5179 | ||
5180 | # Get character information if don't have it already | |
5181 | main::populate_char_info($i) | |
5182 | if ! defined $viacode[$i]; | |
5183 | my $type = $annotate_char_type[$i]; | |
5184 | ||
5185 | # Figure out if should output the next code points as part | |
5186 | # of a range or not. If this is not in an annotation | |
5187 | # range, then won't output as a range, so returns $i. | |
5188 | # Otherwise use the end of the annotation range, but no | |
5189 | # further than the maximum possible end point of the loop. | |
5190 | my $range_end = main::min($annotate_ranges->value_of($i) | |
5191 | || $i, | |
5192 | $end); | |
5193 | ||
5194 | # Use a range if it is a range, and either is one of the | |
5195 | # special annotation ranges, or the range is at most 3 | |
5196 | # long. This last case causes the algorithmically named | |
5197 | # code points to be output individually in spans of at | |
5198 | # most 3, as they are the ones whose $type is > 0. | |
5199 | if ($range_end != $i | |
5200 | && ( $type < 0 || $range_end - $i > 2)) | |
5201 | { | |
5202 | # Here is to output a range. We don't allow a | |
5203 | # caller-specified output format--just use the | |
5204 | # standard one. | |
5205 | push @OUT, sprintf "%04X\t%04X\t%s\t#", $i, | |
5206 | $range_end, | |
5207 | $value; | |
5208 | my $range_name = $viacode[$i]; | |
5209 | ||
5210 | # For the code points which end in their hex value, we | |
5211 | # eliminate that from the output annotation, and | |
5212 | # capitalize only the first letter of each word. | |
5213 | if ($type == $CP_IN_NAME) { | |
5214 | my $hex = sprintf "%04X", $i; | |
5215 | $range_name =~ s/-$hex$//; | |
5216 | my @words = split " ", $range_name; | |
5217 | for my $word (@words) { | |
5218 | $word = ucfirst(lc($word)) if $word ne 'CJK'; | |
5219 | } | |
5220 | $range_name = join " ", @words; | |
5221 | } | |
5222 | elsif ($type == $HANGUL_SYLLABLE) { | |
5223 | $range_name = "Hangul Syllable"; | |
5224 | } | |
5225 | ||
5226 | $OUT[-1] .= " $range_name" if $range_name; | |
5227 | ||
5228 | # Include the number of code points in the range | |
5229 | my $count = main::clarify_number($range_end - $i + 1); | |
5230 | $OUT[-1] .= " [$count]\n"; | |
5231 | ||
5232 | # Skip to the end of the range | |
5233 | $i = $range_end; | |
5234 | } | |
5235 | else { # Not in a range. | |
5236 | my $comment = ""; | |
5237 | ||
5238 | # When outputting the names of each character, use | |
5239 | # the character itself if printable | |
5240 | $comment .= "'" . chr($i) . "' " if $printable[$i]; | |
5241 | ||
5242 | # To make it more readable, use a minimum indentation | |
5243 | my $comment_indent; | |
5244 | ||
5245 | # Determine the annotation | |
5246 | if ($format eq $DECOMP_STRING_FORMAT) { | |
5247 | ||
5248 | # This is very specialized, with the type of | |
5249 | # decomposition beginning the line enclosed in | |
5250 | # <...>, and the code points that the code point | |
5251 | # decomposes to separated by blanks. Create two | |
5252 | # strings, one of the printable characters, and | |
5253 | # one of their official names. | |
5254 | (my $map = $value) =~ s/ \ * < .*? > \ +//x; | |
5255 | my $tostr = ""; | |
5256 | my $to_name = ""; | |
5257 | my $to_chr = ""; | |
5258 | foreach my $to (split " ", $map) { | |
5259 | $to = CORE::hex $to; | |
5260 | $to_name .= " + " if $to_name; | |
5261 | $to_chr .= chr($to); | |
5262 | main::populate_char_info($to) | |
5263 | if ! defined $viacode[$to]; | |
5264 | $to_name .= $viacode[$to]; | |
5265 | } | |
5266 | ||
5267 | $comment .= | |
5268 | "=> '$to_chr'; $viacode[$i] => $to_name"; | |
5269 | $comment_indent = 25; # Determined by experiment | |
5270 | } | |
5271 | else { | |
5272 | ||
5273 | # Assume that any table that has hex format is a | |
5274 | # mapping of one code point to another. | |
5275 | if ($format eq $HEX_FORMAT) { | |
5276 | my $decimal_value = CORE::hex $value; | |
5277 | main::populate_char_info($decimal_value) | |
5278 | if ! defined $viacode[$decimal_value]; | |
5279 | $comment .= "=> '" | |
5280 | . chr($decimal_value) | |
5281 | . "'; " if $printable[$decimal_value]; | |
5282 | } | |
5283 | $comment .= $viacode[$i] if $include_name | |
5284 | && $viacode[$i]; | |
5285 | if ($format eq $HEX_FORMAT) { | |
5286 | my $decimal_value = CORE::hex $value; | |
5287 | $comment .= " => $viacode[$decimal_value]" | |
5288 | if $viacode[$decimal_value]; | |
5289 | } | |
5290 | ||
5291 | # If including the name, no need to indent, as the | |
5292 | # name will already be way across the line. | |
5293 | $comment_indent = ($include_name) ? 0 : 60; | |
5294 | } | |
5295 | ||
5296 | # Use any passed in routine to output the base part of | |
5297 | # the line. | |
5298 | if (ref $range_size_1 eq 'CODE') { | |
5299 | my $base_part = &{$range_size_1}($i, $value); | |
5300 | chomp $base_part; | |
5301 | push @OUT, $base_part; | |
5302 | } | |
5303 | else { | |
5304 | push @OUT, sprintf "%04X\t\t%s", $i, $value; | |
5305 | } | |
5306 | ||
5307 | # And add the annotation. | |
5308 | $OUT[-1] = sprintf "%-*s\t# %s", $comment_indent, | |
5309 | $OUT[-1], | |
5310 | $comment if $comment; | |
5311 | $OUT[-1] .= "\n"; | |
5312 | } | |
99870f4d KW |
5313 | } |
5314 | } # End of loop through all the table's ranges | |
5315 | } | |
5316 | ||
5317 | # Add anything that goes after the main body, but within the here | |
5318 | # document, | |
5319 | my $append_to_body = $self->append_to_body; | |
5320 | push @OUT, $append_to_body if $append_to_body; | |
5321 | ||
5322 | # And finish the here document. | |
5323 | push @OUT, "END\n"; | |
5324 | ||
668b3bfc KW |
5325 | # Done with the main portion of the body. Can now figure out what |
5326 | # should appear before it in the file. | |
5327 | my $pre_body = $self->pre_body; | |
5328 | push @HEADER, $pre_body, "\n" if $pre_body; | |
668b3bfc | 5329 | |
6b0079b5 KW |
5330 | # All these files should have a .pl suffix added to them. |
5331 | my @file_with_pl = @{$file_path{$addr}}; | |
5332 | $file_with_pl[-1] .= '.pl'; | |
99870f4d | 5333 | |
6b0079b5 | 5334 | main::write(\@file_with_pl, |
558712cf | 5335 | $annotate, # utf8 iff annotating |
9218f1cf KW |
5336 | \@HEADER, |
5337 | \@OUT); | |
99870f4d KW |
5338 | return; |
5339 | } | |
5340 | ||
5341 | sub set_status { # Set the table's status | |
5342 | my $self = shift; | |
5343 | my $status = shift; # The status enum value | |
5344 | my $info = shift; # Any message associated with it. | |
5345 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5346 | ||
ffe43484 | 5347 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5348 | |
5349 | $status{$addr} = $status; | |
5350 | $status_info{$addr} = $info; | |
5351 | return; | |
5352 | } | |
5353 | ||
5354 | sub lock { | |
5355 | # Don't allow changes to the table from now on. This stores a stack | |
5356 | # trace of where it was called, so that later attempts to modify it | |
5357 | # can immediately show where it got locked. | |
5358 | ||
5359 | my $self = shift; | |
5360 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5361 | ||
ffe43484 | 5362 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5363 | |
5364 | $locked{$addr} = ""; | |
5365 | ||
5366 | my $line = (caller(0))[2]; | |
5367 | my $i = 1; | |
5368 | ||
5369 | # Accumulate the stack trace | |
5370 | while (1) { | |
5371 | my ($pkg, $file, $caller_line, $caller) = caller $i++; | |
5372 | ||
5373 | last unless defined $caller; | |
5374 | ||
5375 | $locked{$addr} .= " called from $caller() at line $line\n"; | |
5376 | $line = $caller_line; | |
5377 | } | |
5378 | $locked{$addr} .= " called from main at line $line\n"; | |
5379 | ||
5380 | return; | |
5381 | } | |
5382 | ||
5383 | sub carp_if_locked { | |
5384 | # Return whether a table is locked or not, and, by the way, complain | |
5385 | # if is locked | |
5386 | ||
5387 | my $self = shift; | |
5388 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5389 | ||
ffe43484 | 5390 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5391 | |
5392 | return 0 if ! $locked{$addr}; | |
5393 | Carp::my_carp_bug("Can't modify a locked table. Stack trace of locking:\n$locked{$addr}\n\n"); | |
5394 | return 1; | |
5395 | } | |
5396 | ||
5397 | sub set_file_path { # Set the final directory path for this table | |
5398 | my $self = shift; | |
5399 | # Rest of parameters passed on | |
5400 | ||
f998e60c | 5401 | no overloading; |
051df77b | 5402 | @{$file_path{pack 'J', $self}} = @_; |
99870f4d KW |
5403 | return |
5404 | } | |
5405 | ||
5406 | # Accessors for the range list stored in this table. First for | |
5407 | # unconditional | |
ea25a9b2 | 5408 | for my $sub (qw( |
2f7a8815 | 5409 | containing_range |
99870f4d KW |
5410 | contains |
5411 | count | |
5412 | each_range | |
5413 | hash | |
5414 | is_empty | |
09aba7e4 | 5415 | matches_identically_to |
99870f4d KW |
5416 | max |
5417 | min | |
5418 | range_count | |
5419 | reset_each_range | |
0a9dbafc | 5420 | type_of |
99870f4d | 5421 | value_of |
ea25a9b2 | 5422 | )) |
99870f4d KW |
5423 | { |
5424 | no strict "refs"; | |
5425 | *$sub = sub { | |
5426 | use strict "refs"; | |
5427 | my $self = shift; | |
ec40ee88 | 5428 | return $self->_range_list->$sub(@_); |
99870f4d KW |
5429 | } |
5430 | } | |
5431 | ||
5432 | # Then for ones that should fail if locked | |
ea25a9b2 | 5433 | for my $sub (qw( |
99870f4d | 5434 | delete_range |
ea25a9b2 | 5435 | )) |
99870f4d KW |
5436 | { |
5437 | no strict "refs"; | |
5438 | *$sub = sub { | |
5439 | use strict "refs"; | |
5440 | my $self = shift; | |
5441 | ||
5442 | return if $self->carp_if_locked; | |
f998e60c | 5443 | no overloading; |
ec40ee88 | 5444 | return $self->_range_list->$sub(@_); |
99870f4d KW |
5445 | } |
5446 | } | |
5447 | ||
5448 | } # End closure | |
5449 | ||
5450 | package Map_Table; | |
5451 | use base '_Base_Table'; | |
5452 | ||
5453 | # A Map Table is a table that contains the mappings from code points to | |
5454 | # values. There are two weird cases: | |
5455 | # 1) Anomalous entries are ones that aren't maps of ranges of code points, but | |
5456 | # are written in the table's file at the end of the table nonetheless. It | |
5457 | # requires specially constructed code to handle these; utf8.c can not read | |
5458 | # these in, so they should not go in $map_directory. As of this writing, | |
5459 | # the only case that these happen is for named sequences used in | |
5460 | # charnames.pm. But this code doesn't enforce any syntax on these, so | |
5461 | # something else could come along that uses it. | |
5462 | # 2) Specials are anything that doesn't fit syntactically into the body of the | |
5463 | # table. The ranges for these have a map type of non-zero. The code below | |
5464 | # knows about and handles each possible type. In most cases, these are | |
5465 | # written as part of the header. | |
5466 | # | |
5467 | # A map table deliberately can't be manipulated at will unlike match tables. | |
5468 | # This is because of the ambiguities having to do with what to do with | |
5469 | # overlapping code points. And there just isn't a need for those things; | |
5470 | # what one wants to do is just query, add, replace, or delete mappings, plus | |
5471 | # write the final result. | |
5472 | # However, there is a method to get the list of possible ranges that aren't in | |
5473 | # this table to use for defaulting missing code point mappings. And, | |
5474 | # map_add_or_replace_non_nulls() does allow one to add another table to this | |
5475 | # one, but it is clearly very specialized, and defined that the other's | |
5476 | # non-null values replace this one's if there is any overlap. | |
5477 | ||
5478 | sub trace { return main::trace(@_); } | |
5479 | ||
5480 | { # Closure | |
5481 | ||
5482 | main::setup_package(); | |
5483 | ||
5484 | my %default_map; | |
5485 | # Many input files omit some entries; this gives what the mapping for the | |
5486 | # missing entries should be | |
5487 | main::set_access('default_map', \%default_map, 'r'); | |
5488 | ||
5489 | my %anomalous_entries; | |
5490 | # Things that go in the body of the table which don't fit the normal | |
5491 | # scheme of things, like having a range. Not much can be done with these | |
5492 | # once there except to output them. This was created to handle named | |
5493 | # sequences. | |
5494 | main::set_access('anomalous_entry', \%anomalous_entries, 'a'); | |
5495 | main::set_access('anomalous_entries', # Append singular, read plural | |
5496 | \%anomalous_entries, | |
5497 | 'readable_array'); | |
5498 | ||
99870f4d KW |
5499 | my %core_access; |
5500 | # This is a string, solely for documentation, indicating how one can get | |
5501 | # access to this property via the Perl core. | |
5502 | main::set_access('core_access', \%core_access, 'r', 's'); | |
5503 | ||
99870f4d | 5504 | my %to_output_map; |
8572ace0 | 5505 | # Enum as to whether or not to write out this map table: |
c12f2655 | 5506 | # 0 don't output |
8572ace0 KW |
5507 | # $EXTERNAL_MAP means its existence is noted in the documentation, and |
5508 | # it should not be removed nor its format changed. This | |
5509 | # is done for those files that have traditionally been | |
5510 | # output. | |
5511 | # $INTERNAL_MAP means Perl reserves the right to do anything it wants | |
5512 | # with this file | |
99870f4d KW |
5513 | main::set_access('to_output_map', \%to_output_map, 's'); |
5514 | ||
5515 | ||
5516 | sub new { | |
5517 | my $class = shift; | |
5518 | my $name = shift; | |
5519 | ||
5520 | my %args = @_; | |
5521 | ||
5522 | # Optional initialization data for the table. | |
5523 | my $initialize = delete $args{'Initialize'}; | |
5524 | ||
5525 | my $core_access = delete $args{'Core_Access'}; | |
5526 | my $default_map = delete $args{'Default_Map'}; | |
99870f4d | 5527 | my $property = delete $args{'_Property'}; |
23e33b60 | 5528 | my $full_name = delete $args{'Full_Name'}; |
20863809 | 5529 | |
99870f4d KW |
5530 | # Rest of parameters passed on |
5531 | ||
5532 | my $range_list = Range_Map->new(Owner => $property); | |
5533 | ||
5534 | my $self = $class->SUPER::new( | |
5535 | Name => $name, | |
23e33b60 KW |
5536 | Complete_Name => $full_name, |
5537 | Full_Name => $full_name, | |
99870f4d KW |
5538 | _Property => $property, |
5539 | _Range_List => $range_list, | |
5540 | %args); | |
5541 | ||
ffe43484 | 5542 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5543 | |
5544 | $anomalous_entries{$addr} = []; | |
5545 | $core_access{$addr} = $core_access; | |
5546 | $default_map{$addr} = $default_map; | |
99870f4d KW |
5547 | |
5548 | $self->initialize($initialize) if defined $initialize; | |
5549 | ||
5550 | return $self; | |
5551 | } | |
5552 | ||
5553 | use overload | |
5554 | fallback => 0, | |
5555 | qw("") => "_operator_stringify", | |
5556 | ; | |
5557 | ||
5558 | sub _operator_stringify { | |
5559 | my $self = shift; | |
5560 | ||
5561 | my $name = $self->property->full_name; | |
5562 | $name = '""' if $name eq ""; | |
5563 | return "Map table for Property '$name'"; | |
5564 | } | |
5565 | ||
99870f4d KW |
5566 | sub add_alias { |
5567 | # Add a synonym for this table (which means the property itself) | |
5568 | my $self = shift; | |
5569 | my $name = shift; | |
5570 | # Rest of parameters passed on. | |
5571 | ||
5572 | $self->SUPER::add_alias($name, $self->property, @_); | |
5573 | return; | |
5574 | } | |
5575 | ||
5576 | sub add_map { | |
5577 | # Add a range of code points to the list of specially-handled code | |
5578 | # points. $MULTI_CP is assumed if the type of special is not passed | |
5579 | # in. | |
5580 | ||
5581 | my $self = shift; | |
5582 | my $lower = shift; | |
5583 | my $upper = shift; | |
5584 | my $string = shift; | |
5585 | my %args = @_; | |
5586 | ||
5587 | my $type = delete $args{'Type'} || 0; | |
5588 | # Rest of parameters passed on | |
5589 | ||
5590 | # Can't change the table if locked. | |
5591 | return if $self->carp_if_locked; | |
5592 | ||
ffe43484 | 5593 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d | 5594 | |
99870f4d KW |
5595 | $self->_range_list->add_map($lower, $upper, |
5596 | $string, | |
5597 | @_, | |
5598 | Type => $type); | |
5599 | return; | |
5600 | } | |
5601 | ||
5602 | sub append_to_body { | |
5603 | # Adds to the written HERE document of the table's body any anomalous | |
5604 | # entries in the table.. | |
5605 | ||
5606 | my $self = shift; | |
5607 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5608 | ||
ffe43484 | 5609 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5610 | |
5611 | return "" unless @{$anomalous_entries{$addr}}; | |
5612 | return join("\n", @{$anomalous_entries{$addr}}) . "\n"; | |
5613 | } | |
5614 | ||
5615 | sub map_add_or_replace_non_nulls { | |
5616 | # This adds the mappings in the table $other to $self. Non-null | |
5617 | # mappings from $other override those in $self. It essentially merges | |
5618 | # the two tables, with the second having priority except for null | |
5619 | # mappings. | |
5620 | ||
5621 | my $self = shift; | |
5622 | my $other = shift; | |
5623 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5624 | ||
5625 | return if $self->carp_if_locked; | |
5626 | ||
5627 | if (! $other->isa(__PACKAGE__)) { | |
5628 | Carp::my_carp_bug("$other should be a " | |
5629 | . __PACKAGE__ | |
5630 | . ". Not a '" | |
5631 | . ref($other) | |
5632 | . "'. Not added;"); | |
5633 | return; | |
5634 | } | |
5635 | ||
ffe43484 NC |
5636 | my $addr = do { no overloading; pack 'J', $self; }; |
5637 | my $other_addr = do { no overloading; pack 'J', $other; }; | |
99870f4d KW |
5638 | |
5639 | local $to_trace = 0 if main::DEBUG; | |
5640 | ||
5641 | my $self_range_list = $self->_range_list; | |
5642 | my $other_range_list = $other->_range_list; | |
5643 | foreach my $range ($other_range_list->ranges) { | |
5644 | my $value = $range->value; | |
5645 | next if $value eq ""; | |
5646 | $self_range_list->_add_delete('+', | |
5647 | $range->start, | |
5648 | $range->end, | |
5649 | $value, | |
5650 | Type => $range->type, | |
5651 | Replace => $UNCONDITIONALLY); | |
5652 | } | |
5653 | ||
99870f4d KW |
5654 | return; |
5655 | } | |
5656 | ||
5657 | sub set_default_map { | |
5658 | # Define what code points that are missing from the input files should | |
5659 | # map to | |
5660 | ||
5661 | my $self = shift; | |
5662 | my $map = shift; | |
5663 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5664 | ||
ffe43484 | 5665 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5666 | |
5667 | # Convert the input to the standard equivalent, if any (won't have any | |
5668 | # for $STRING properties) | |
5669 | my $standard = $self->_find_table_from_alias->{$map}; | |
5670 | $map = $standard->name if defined $standard; | |
5671 | ||
5672 | # Warn if there already is a non-equivalent default map for this | |
5673 | # property. Note that a default map can be a ref, which means that | |
5674 | # what it actually means is delayed until later in the program, and it | |
5675 | # IS permissible to override it here without a message. | |
5676 | my $default_map = $default_map{$addr}; | |
5677 | if (defined $default_map | |
5678 | && ! ref($default_map) | |
5679 | && $default_map ne $map | |
5680 | && main::Standardize($map) ne $default_map) | |
5681 | { | |
5682 | my $property = $self->property; | |
5683 | my $map_table = $property->table($map); | |
5684 | my $default_table = $property->table($default_map); | |
5685 | if (defined $map_table | |
5686 | && defined $default_table | |
5687 | && $map_table != $default_table) | |
5688 | { | |
5689 | Carp::my_carp("Changing the default mapping for " | |
5690 | . $property | |
5691 | . " from $default_map to $map'"); | |
5692 | } | |
5693 | } | |
5694 | ||
5695 | $default_map{$addr} = $map; | |
5696 | ||
5697 | # Don't also create any missing table for this map at this point, | |
5698 | # because if we did, it could get done before the main table add is | |
5699 | # done for PropValueAliases.txt; instead the caller will have to make | |
5700 | # sure it exists, if desired. | |
5701 | return; | |
5702 | } | |
5703 | ||
5704 | sub to_output_map { | |
5705 | # Returns boolean: should we write this map table? | |
5706 | ||
5707 | my $self = shift; | |
5708 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5709 | ||
ffe43484 | 5710 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5711 | |
5712 | # If overridden, use that | |
5713 | return $to_output_map{$addr} if defined $to_output_map{$addr}; | |
5714 | ||
5715 | my $full_name = $self->full_name; | |
fcf1973c KW |
5716 | return $global_to_output_map{$full_name} |
5717 | if defined $global_to_output_map{$full_name}; | |
99870f4d | 5718 | |
20863809 | 5719 | # If table says to output, do so; if says to suppress it, do so. |
88c22f80 | 5720 | return $INTERNAL_MAP if $self->internal_only; |
8572ace0 | 5721 | return $EXTERNAL_MAP if grep { $_ eq $full_name } @output_mapped_properties; |
99870f4d KW |
5722 | return 0 if $self->status eq $SUPPRESSED; |
5723 | ||
5724 | my $type = $self->property->type; | |
5725 | ||
5726 | # Don't want to output binary map tables even for debugging. | |
5727 | return 0 if $type == $BINARY; | |
5728 | ||
5729 | # But do want to output string ones. | |
8572ace0 | 5730 | return $EXTERNAL_MAP if $type == $STRING; |
99870f4d | 5731 | |
8572ace0 KW |
5732 | # Otherwise is an $ENUM, do output it, for Perl's purposes |
5733 | return $INTERNAL_MAP; | |
99870f4d KW |
5734 | } |
5735 | ||
5736 | sub inverse_list { | |
5737 | # Returns a Range_List that is gaps of the current table. That is, | |
5738 | # the inversion | |
5739 | ||
5740 | my $self = shift; | |
5741 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5742 | ||
5743 | my $current = Range_List->new(Initialize => $self->_range_list, | |
5744 | Owner => $self->property); | |
5745 | return ~ $current; | |
5746 | } | |
5747 | ||
8572ace0 KW |
5748 | sub header { |
5749 | my $self = shift; | |
5750 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5751 | ||
5752 | my $return = $self->SUPER::header(); | |
5753 | ||
126c3d4e | 5754 | $return .= $INTERNAL_ONLY_HEADER if $self->to_output_map == $INTERNAL_MAP; |
8572ace0 KW |
5755 | return $return; |
5756 | } | |
5757 | ||
99870f4d KW |
5758 | sub set_final_comment { |
5759 | # Just before output, create the comment that heads the file | |
5760 | # containing this table. | |
5761 | ||
bd9ebcfd KW |
5762 | return unless $debugging_build; |
5763 | ||
99870f4d KW |
5764 | my $self = shift; |
5765 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5766 | ||
5767 | # No sense generating a comment if aren't going to write it out. | |
5768 | return if ! $self->to_output_map; | |
5769 | ||
ffe43484 | 5770 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
5771 | |
5772 | my $property = $self->property; | |
5773 | ||
5774 | # Get all the possible names for this property. Don't use any that | |
5775 | # aren't ok for use in a file name, etc. This is perhaps causing that | |
5776 | # flag to do double duty, and may have to be changed in the future to | |
5777 | # have our own flag for just this purpose; but it works now to exclude | |
5778 | # Perl generated synonyms from the lists for properties, where the | |
5779 | # name is always the proper Unicode one. | |
5780 | my @property_aliases = grep { $_->externally_ok } $self->aliases; | |
5781 | ||
5782 | my $count = $self->count; | |
5783 | my $default_map = $default_map{$addr}; | |
5784 | ||
5785 | # The ranges that map to the default aren't output, so subtract that | |
5786 | # to get those actually output. A property with matching tables | |
5787 | # already has the information calculated. | |
5788 | if ($property->type != $STRING) { | |
5789 | $count -= $property->table($default_map)->count; | |
5790 | } | |
5791 | elsif (defined $default_map) { | |
5792 | ||
5793 | # But for $STRING properties, must calculate now. Subtract the | |
5794 | # count from each range that maps to the default. | |
5795 | foreach my $range ($self->_range_list->ranges) { | |
99870f4d KW |
5796 | if ($range->value eq $default_map) { |
5797 | $count -= $range->end +1 - $range->start; | |
5798 | } | |
5799 | } | |
5800 | ||
5801 | } | |
5802 | ||
5803 | # Get a string version of $count with underscores in large numbers, | |
5804 | # for clarity. | |
5805 | my $string_count = main::clarify_number($count); | |
5806 | ||
5807 | my $code_points = ($count == 1) | |
5808 | ? 'single code point' | |
5809 | : "$string_count code points"; | |
5810 | ||
5811 | my $mapping; | |
5812 | my $these_mappings; | |
5813 | my $are; | |
5814 | if (@property_aliases <= 1) { | |
5815 | $mapping = 'mapping'; | |
5816 | $these_mappings = 'this mapping'; | |
5817 | $are = 'is' | |
5818 | } | |
5819 | else { | |
5820 | $mapping = 'synonymous mappings'; | |
5821 | $these_mappings = 'these mappings'; | |
5822 | $are = 'are' | |
5823 | } | |
5824 | my $cp; | |
5825 | if ($count >= $MAX_UNICODE_CODEPOINTS) { | |
5826 | $cp = "any code point in Unicode Version $string_version"; | |
5827 | } | |
5828 | else { | |
5829 | my $map_to; | |
5830 | if ($default_map eq "") { | |
5831 | $map_to = 'the null string'; | |
5832 | } | |
5833 | elsif ($default_map eq $CODE_POINT) { | |
5834 | $map_to = "itself"; | |
5835 | } | |
5836 | else { | |
5837 | $map_to = "'$default_map'"; | |
5838 | } | |
5839 | if ($count == 1) { | |
5840 | $cp = "the single code point"; | |
5841 | } | |
5842 | else { | |
5843 | $cp = "one of the $code_points"; | |
5844 | } | |
5845 | $cp .= " in Unicode Version $string_version for which the mapping is not to $map_to"; | |
5846 | } | |
5847 | ||
5848 | my $comment = ""; | |
5849 | ||
5850 | my $status = $self->status; | |
5851 | if ($status) { | |
5852 | my $warn = uc $status_past_participles{$status}; | |
5853 | $comment .= <<END; | |
5854 | ||
5855 | !!!!!!! $warn !!!!!!!!!!!!!!!!!!! | |
5856 | All property or property=value combinations contained in this file are $warn. | |
5857 | See $unicode_reference_url for what this means. | |
5858 | ||
5859 | END | |
5860 | } | |
5861 | $comment .= "This file returns the $mapping:\n"; | |
5862 | ||
5863 | for my $i (0 .. @property_aliases - 1) { | |
5864 | $comment .= sprintf("%-8s%s\n", | |
5865 | " ", | |
5866 | $property_aliases[$i]->name . '(cp)' | |
5867 | ); | |
5868 | } | |
5869 | $comment .= | |
5870 | "\nwhere 'cp' is $cp. Note that $these_mappings $are "; | |
5871 | ||
5872 | my $access = $core_access{$addr}; | |
5873 | if ($access) { | |
5874 | $comment .= "accessible through the Perl core via $access."; | |
5875 | } | |
5876 | else { | |
5877 | $comment .= "not accessible through the Perl core directly."; | |
5878 | } | |
5879 | ||
5880 | # And append any commentary already set from the actual property. | |
5881 | $comment .= "\n\n" . $self->comment if $self->comment; | |
5882 | if ($self->description) { | |
5883 | $comment .= "\n\n" . join " ", $self->description; | |
5884 | } | |
5885 | if ($self->note) { | |
5886 | $comment .= "\n\n" . join " ", $self->note; | |
5887 | } | |
5888 | $comment .= "\n"; | |
5889 | ||
5890 | if (! $self->perl_extension) { | |
5891 | $comment .= <<END; | |
5892 | ||
5893 | For information about what this property really means, see: | |
5894 | $unicode_reference_url | |
5895 | END | |
5896 | } | |
5897 | ||
5898 | if ($count) { # Format differs for empty table | |
5899 | $comment.= "\nThe format of the "; | |
5900 | if ($self->range_size_1) { | |
5901 | $comment.= <<END; | |
5902 | main body of lines of this file is: CODE_POINT\\t\\tMAPPING where CODE_POINT | |
5903 | is in hex; MAPPING is what CODE_POINT maps to. | |
5904 | END | |
5905 | } | |
5906 | else { | |
5907 | ||
5908 | # There are tables which end up only having one element per | |
5909 | # range, but it is not worth keeping track of for making just | |
5910 | # this comment a little better. | |
5911 | $comment.= <<END; | |
5912 | non-comment portions of the main body of lines of this file is: | |
5913 | START\\tSTOP\\tMAPPING where START is the starting code point of the | |
5914 | range, in hex; STOP is the ending point, or if omitted, the range has just one | |
5915 | code point; MAPPING is what each code point between START and STOP maps to. | |
5916 | END | |
0c07e538 | 5917 | if ($self->output_range_counts) { |
99870f4d KW |
5918 | $comment .= <<END; |
5919 | Numbers in comments in [brackets] indicate how many code points are in the | |
5920 | range (omitted when the range is a single code point or if the mapping is to | |
5921 | the null string). | |
5922 | END | |
5923 | } | |
5924 | } | |
5925 | } | |
5926 | $self->set_comment(main::join_lines($comment)); | |
5927 | return; | |
5928 | } | |
5929 | ||
5930 | my %swash_keys; # Makes sure don't duplicate swash names. | |
5931 | ||
668b3bfc KW |
5932 | # The remaining variables are temporaries used while writing each table, |
5933 | # to output special ranges. | |
668b3bfc KW |
5934 | my @multi_code_point_maps; # Map is to more than one code point. |
5935 | ||
668b3bfc KW |
5936 | sub handle_special_range { |
5937 | # Called in the middle of write when it finds a range it doesn't know | |
5938 | # how to handle. | |
5939 | ||
5940 | my $self = shift; | |
5941 | my $range = shift; | |
5942 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
5943 | ||
5944 | my $addr = do { no overloading; pack 'J', $self; }; | |
5945 | ||
5946 | my $type = $range->type; | |
5947 | ||
5948 | my $low = $range->start; | |
5949 | my $high = $range->end; | |
5950 | my $map = $range->value; | |
5951 | ||
5952 | # No need to output the range if it maps to the default. | |
5953 | return if $map eq $default_map{$addr}; | |
5954 | ||
bb1dd3da KW |
5955 | my $property = $self->property; |
5956 | ||
668b3bfc KW |
5957 | # Switch based on the map type... |
5958 | if ($type == $HANGUL_SYLLABLE) { | |
5959 | ||
5960 | # These are entirely algorithmically determinable based on | |
5961 | # some constants furnished by Unicode; for now, just set a | |
5962 | # flag to indicate that have them. After everything is figured | |
bb1dd3da KW |
5963 | # out, we will output the code that does the algorithm. (Don't |
5964 | # output them if not needed because we are suppressing this | |
5965 | # property.) | |
5966 | $has_hangul_syllables = 1 if $property->to_output_map; | |
668b3bfc KW |
5967 | } |
5968 | elsif ($type == $CP_IN_NAME) { | |
5969 | ||
bb1dd3da | 5970 | # Code points whose name ends in their code point are also |
668b3bfc KW |
5971 | # algorithmically determinable, but need information about the map |
5972 | # to do so. Both the map and its inverse are stored in data | |
bb1dd3da KW |
5973 | # structures output in the file. They are stored in the mean time |
5974 | # in global lists The lists will be written out later into Name.pm, | |
5975 | # which is created only if needed. In order to prevent duplicates | |
5976 | # in the list, only add to them for one property, should multiple | |
5977 | # ones need them. | |
5978 | if ($needing_code_points_ending_in_code_point == 0) { | |
5979 | $needing_code_points_ending_in_code_point = $property; | |
5980 | } | |
5981 | if ($property == $needing_code_points_ending_in_code_point) { | |
6c1bafed KW |
5982 | push @{$names_ending_in_code_point{$map}->{'low'}}, $low; |
5983 | push @{$names_ending_in_code_point{$map}->{'high'}}, $high; | |
5984 | ||
5985 | my $squeezed = $map =~ s/[-\s]+//gr; | |
5986 | push @{$loose_names_ending_in_code_point{$squeezed}->{'low'}}, | |
5987 | $low; | |
5988 | push @{$loose_names_ending_in_code_point{$squeezed}->{'high'}}, | |
5989 | $high; | |
5990 | ||
5991 | push @code_points_ending_in_code_point, { low => $low, | |
5992 | high => $high, | |
5993 | name => $map | |
5994 | }; | |
bb1dd3da | 5995 | } |
668b3bfc KW |
5996 | } |
5997 | elsif ($range->type == $MULTI_CP || $range->type == $NULL) { | |
5998 | ||
5999 | # Multi-code point maps and null string maps have an entry | |
6000 | # for each code point in the range. They use the same | |
6001 | # output format. | |
6002 | for my $code_point ($low .. $high) { | |
6003 | ||
c12f2655 KW |
6004 | # The pack() below can't cope with surrogates. XXX This may |
6005 | # no longer be true | |
668b3bfc | 6006 | if ($code_point >= 0xD800 && $code_point <= 0xDFFF) { |
98dc9551 | 6007 | Carp::my_carp("Surrogate code point '$code_point' in mapping to '$map' in $self. No map created"); |
668b3bfc KW |
6008 | next; |
6009 | } | |
6010 | ||
6011 | # Generate the hash entries for these in the form that | |
6012 | # utf8.c understands. | |
6013 | my $tostr = ""; | |
6014 | my $to_name = ""; | |
6015 | my $to_chr = ""; | |
6016 | foreach my $to (split " ", $map) { | |
6017 | if ($to !~ /^$code_point_re$/) { | |
6018 | Carp::my_carp("Illegal code point '$to' in mapping '$map' from $code_point in $self. No map created"); | |
6019 | next; | |
6020 | } | |
6021 | $tostr .= sprintf "\\x{%s}", $to; | |
6022 | $to = CORE::hex $to; | |
558712cf | 6023 | if ($annotate) { |
c4019d52 KW |
6024 | $to_name .= " + " if $to_name; |
6025 | $to_chr .= chr($to); | |
6026 | main::populate_char_info($to) | |
6027 | if ! defined $viacode[$to]; | |
6028 | $to_name .= $viacode[$to]; | |
6029 | } | |
668b3bfc KW |
6030 | } |
6031 | ||
6032 | # I (khw) have never waded through this line to | |
6033 | # understand it well enough to comment it. | |
6034 | my $utf8 = sprintf(qq["%s" => "$tostr",], | |
6035 | join("", map { sprintf "\\x%02X", $_ } | |
6036 | unpack("U0C*", pack("U", $code_point)))); | |
6037 | ||
6038 | # Add a comment so that a human reader can more easily | |
6039 | # see what's going on. | |
6040 | push @multi_code_point_maps, | |
6041 | sprintf("%-45s # U+%04X", $utf8, $code_point); | |
558712cf | 6042 | if (! $annotate) { |
c4019d52 KW |
6043 | $multi_code_point_maps[-1] .= " => $map"; |
6044 | } | |
6045 | else { | |
6046 | main::populate_char_info($code_point) | |
6047 | if ! defined $viacode[$code_point]; | |
6048 | $multi_code_point_maps[-1] .= " '" | |
6049 | . chr($code_point) | |
6050 | . "' => '$to_chr'; $viacode[$code_point] => $to_name"; | |
6051 | } | |
668b3bfc KW |
6052 | } |
6053 | } | |
6054 | else { | |
6055 | Carp::my_carp("Unrecognized map type '$range->type' in '$range' in $self. Not written"); | |
6056 | } | |
6057 | ||
6058 | return; | |
6059 | } | |
6060 | ||
99870f4d KW |
6061 | sub pre_body { |
6062 | # Returns the string that should be output in the file before the main | |
668b3bfc KW |
6063 | # body of this table. It isn't called until the main body is |
6064 | # calculated, saving a pass. The string includes some hash entries | |
6065 | # identifying the format of the body, and what the single value should | |
6066 | # be for all ranges missing from it. It also includes any code points | |
6067 | # which have map_types that don't go in the main table. | |
99870f4d KW |
6068 | |
6069 | my $self = shift; | |
6070 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6071 | ||
ffe43484 | 6072 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
6073 | |
6074 | my $name = $self->property->swash_name; | |
6075 | ||
6076 | if (defined $swash_keys{$name}) { | |
6077 | Carp::my_carp(join_lines(<<END | |
6078 | Already created a swash name '$name' for $swash_keys{$name}. This means that | |
6079 | the same name desired for $self shouldn't be used. Bad News. This must be | |
6080 | fixed before production use, but proceeding anyway | |
6081 | END | |
6082 | )); | |
6083 | } | |
6084 | $swash_keys{$name} = "$self"; | |
6085 | ||
99870f4d | 6086 | my $pre_body = ""; |
99870f4d | 6087 | |
668b3bfc KW |
6088 | # Here we assume we were called after have gone through the whole |
6089 | # file. If we actually generated anything for each map type, add its | |
6090 | # respective header and trailer | |
ec2f0128 | 6091 | my $specials_name = ""; |
668b3bfc | 6092 | if (@multi_code_point_maps) { |
ec2f0128 | 6093 | $specials_name = "utf8::ToSpec$name"; |
668b3bfc | 6094 | $pre_body .= <<END; |
99870f4d KW |
6095 | |
6096 | # Some code points require special handling because their mappings are each to | |
6097 | # multiple code points. These do not appear in the main body, but are defined | |
6098 | # in the hash below. | |
6099 | ||
76591e2b KW |
6100 | # Each key is the string of N bytes that together make up the UTF-8 encoding |
6101 | # for the code point. (i.e. the same as looking at the code point's UTF-8 | |
6102 | # under "use bytes"). Each value is the UTF-8 of the translation, for speed. | |
ec2f0128 | 6103 | \%$specials_name = ( |
99870f4d | 6104 | END |
668b3bfc KW |
6105 | $pre_body .= join("\n", @multi_code_point_maps) . "\n);\n"; |
6106 | } | |
99870f4d | 6107 | |
668b3bfc KW |
6108 | my $format = $self->format; |
6109 | ||
6110 | my $return = <<END; | |
6111 | # The name this swash is to be known by, with the format of the mappings in | |
6112 | # the main body of the table, and what all code points missing from this file | |
6113 | # map to. | |
6114 | \$utf8::SwashInfo{'To$name'}{'format'} = '$format'; # $map_table_formats{$format} | |
6115 | END | |
ec2f0128 KW |
6116 | if ($specials_name) { |
6117 | $return .= <<END; | |
6118 | \$utf8::SwashInfo{'To$name'}{'specials_name'} = '$specials_name'; # Name of hash of special mappings | |
6119 | END | |
6120 | } | |
668b3bfc KW |
6121 | my $default_map = $default_map{$addr}; |
6122 | $return .= "\$utf8::SwashInfo{'To$name'}{'missing'} = '$default_map';"; | |
6123 | ||
6124 | if ($default_map eq $CODE_POINT) { | |
6125 | $return .= ' # code point maps to itself'; | |
6126 | } | |
6127 | elsif ($default_map eq "") { | |
6128 | $return .= ' # code point maps to the null string'; | |
6129 | } | |
6130 | $return .= "\n"; | |
6131 | ||
6132 | $return .= $pre_body; | |
6133 | ||
6134 | return $return; | |
6135 | } | |
6136 | ||
6137 | sub write { | |
6138 | # Write the table to the file. | |
6139 | ||
6140 | my $self = shift; | |
6141 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6142 | ||
6143 | my $addr = do { no overloading; pack 'J', $self; }; | |
6144 | ||
6145 | # Clear the temporaries | |
668b3bfc | 6146 | undef @multi_code_point_maps; |
99870f4d KW |
6147 | |
6148 | # Calculate the format of the table if not already done. | |
f5817e0a | 6149 | my $format = $self->format; |
668b3bfc KW |
6150 | my $type = $self->property->type; |
6151 | my $default_map = $self->default_map; | |
99870f4d KW |
6152 | if (! defined $format) { |
6153 | if ($type == $BINARY) { | |
6154 | ||
6155 | # Don't bother checking the values, because we elsewhere | |
6156 | # verify that a binary table has only 2 values. | |
6157 | $format = $BINARY_FORMAT; | |
6158 | } | |
6159 | else { | |
6160 | my @ranges = $self->_range_list->ranges; | |
6161 | ||
6162 | # default an empty table based on its type and default map | |
6163 | if (! @ranges) { | |
6164 | ||
6165 | # But it turns out that the only one we can say is a | |
6166 | # non-string (besides binary, handled above) is when the | |
6167 | # table is a string and the default map is to a code point | |
6168 | if ($type == $STRING && $default_map eq $CODE_POINT) { | |
6169 | $format = $HEX_FORMAT; | |
6170 | } | |
6171 | else { | |
6172 | $format = $STRING_FORMAT; | |
6173 | } | |
6174 | } | |
6175 | else { | |
6176 | ||
6177 | # Start with the most restrictive format, and as we find | |
6178 | # something that doesn't fit with that, change to the next | |
6179 | # most restrictive, and so on. | |
6180 | $format = $DECIMAL_FORMAT; | |
6181 | foreach my $range (@ranges) { | |
668b3bfc KW |
6182 | next if $range->type != 0; # Non-normal ranges don't |
6183 | # affect the main body | |
99870f4d KW |
6184 | my $map = $range->value; |
6185 | if ($map ne $default_map) { | |
6186 | last if $format eq $STRING_FORMAT; # already at | |
6187 | # least | |
6188 | # restrictive | |
6189 | $format = $INTEGER_FORMAT | |
6190 | if $format eq $DECIMAL_FORMAT | |
6191 | && $map !~ / ^ [0-9] $ /x; | |
6192 | $format = $FLOAT_FORMAT | |
6193 | if $format eq $INTEGER_FORMAT | |
6194 | && $map !~ / ^ -? [0-9]+ $ /x; | |
6195 | $format = $RATIONAL_FORMAT | |
6196 | if $format eq $FLOAT_FORMAT | |
6197 | && $map !~ / ^ -? [0-9]+ \. [0-9]* $ /x; | |
6198 | $format = $HEX_FORMAT | |
6199 | if $format eq $RATIONAL_FORMAT | |
6200 | && $map !~ / ^ -? [0-9]+ ( \/ [0-9]+ )? $ /x; | |
6201 | $format = $STRING_FORMAT if $format eq $HEX_FORMAT | |
6202 | && $map =~ /[^0-9A-F]/; | |
6203 | } | |
6204 | } | |
6205 | } | |
6206 | } | |
6207 | } # end of calculating format | |
6208 | ||
668b3bfc | 6209 | if ($default_map eq $CODE_POINT |
99870f4d | 6210 | && $format ne $HEX_FORMAT |
668b3bfc KW |
6211 | && ! defined $self->format) # manual settings are always |
6212 | # considered ok | |
99870f4d KW |
6213 | { |
6214 | Carp::my_carp_bug("Expecting hex format for mapping table for $self, instead got '$format'") | |
6215 | } | |
99870f4d | 6216 | |
668b3bfc | 6217 | $self->_set_format($format); |
99870f4d | 6218 | |
0911a63d KW |
6219 | # Core Perl has a different definition of mapping ranges than we do, |
6220 | # that is applicable mainly to mapping code points, so for tables | |
6221 | # where it is possible that core Perl could be used to read it, | |
6222 | # make it range size 1 to prevent possible confusion | |
6223 | $self->set_range_size_1(1) if $format eq $HEX_FORMAT; | |
6224 | ||
99870f4d KW |
6225 | return $self->SUPER::write( |
6226 | ($self->property == $block) | |
6227 | ? 7 # block file needs more tab stops | |
6228 | : 3, | |
668b3bfc | 6229 | $default_map); # don't write defaulteds |
99870f4d KW |
6230 | } |
6231 | ||
6232 | # Accessors for the underlying list that should fail if locked. | |
ea25a9b2 | 6233 | for my $sub (qw( |
99870f4d | 6234 | add_duplicate |
ea25a9b2 | 6235 | )) |
99870f4d KW |
6236 | { |
6237 | no strict "refs"; | |
6238 | *$sub = sub { | |
6239 | use strict "refs"; | |
6240 | my $self = shift; | |
6241 | ||
6242 | return if $self->carp_if_locked; | |
6243 | return $self->_range_list->$sub(@_); | |
6244 | } | |
6245 | } | |
6246 | } # End closure for Map_Table | |
6247 | ||
6248 | package Match_Table; | |
6249 | use base '_Base_Table'; | |
6250 | ||
6251 | # A Match table is one which is a list of all the code points that have | |
6252 | # the same property and property value, for use in \p{property=value} | |
6253 | # constructs in regular expressions. It adds very little data to the base | |
6254 | # structure, but many methods, as these lists can be combined in many ways to | |
6255 | # form new ones. | |
6256 | # There are only a few concepts added: | |
6257 | # 1) Equivalents and Relatedness. | |
6258 | # Two tables can match the identical code points, but have different names. | |
6259 | # This always happens when there is a perl single form extension | |
6260 | # \p{IsProperty} for the Unicode compound form \P{Property=True}. The two | |
6261 | # tables are set to be related, with the Perl extension being a child, and | |
6262 | # the Unicode property being the parent. | |
6263 | # | |
6264 | # It may be that two tables match the identical code points and we don't | |
6265 | # know if they are related or not. This happens most frequently when the | |
6266 | # Block and Script properties have the exact range. But note that a | |
6267 | # revision to Unicode could add new code points to the script, which would | |
6268 | # now have to be in a different block (as the block was filled, or there | |
6269 | # would have been 'Unknown' script code points in it and they wouldn't have | |
6270 | # been identical). So we can't rely on any two properties from Unicode | |
6271 | # always matching the same code points from release to release, and thus | |
6272 | # these tables are considered coincidentally equivalent--not related. When | |
6273 | # two tables are unrelated but equivalent, one is arbitrarily chosen as the | |
6274 | # 'leader', and the others are 'equivalents'. This concept is useful | |
6275 | # to minimize the number of tables written out. Only one file is used for | |
6276 | # any identical set of code points, with entries in Heavy.pl mapping all | |
6277 | # the involved tables to it. | |
6278 | # | |
6279 | # Related tables will always be identical; we set them up to be so. Thus | |
6280 | # if the Unicode one is deprecated, the Perl one will be too. Not so for | |
6281 | # unrelated tables. Relatedness makes generating the documentation easier. | |
6282 | # | |
c12f2655 KW |
6283 | # 2) Complement. |
6284 | # Like equivalents, two tables may be the inverses of each other, the | |
6285 | # intersection between them is null, and the union is every Unicode code | |
6286 | # point. The two tables that occupy a binary property are necessarily like | |
6287 | # this. By specifying one table as the complement of another, we can avoid | |
6288 | # storing it on disk (using the other table and performing a fast | |
6289 | # transform), and some memory and calculations. | |
6290 | # | |
6291 | # 3) Conflicting. It may be that there will eventually be name clashes, with | |
99870f4d KW |
6292 | # the same name meaning different things. For a while, there actually were |
6293 | # conflicts, but they have so far been resolved by changing Perl's or | |
6294 | # Unicode's definitions to match the other, but when this code was written, | |
6295 | # it wasn't clear that that was what was going to happen. (Unicode changed | |
6296 | # because of protests during their beta period.) Name clashes are warned | |
6297 | # about during compilation, and the documentation. The generated tables | |
6298 | # are sane, free of name clashes, because the code suppresses the Perl | |
6299 | # version. But manual intervention to decide what the actual behavior | |
6300 | # should be may be required should this happen. The introductory comments | |
6301 | # have more to say about this. | |
6302 | ||
6303 | sub standardize { return main::standardize($_[0]); } | |
6304 | sub trace { return main::trace(@_); } | |
6305 | ||
6306 | ||
6307 | { # Closure | |
6308 | ||
6309 | main::setup_package(); | |
6310 | ||
6311 | my %leader; | |
6312 | # The leader table of this one; initially $self. | |
6313 | main::set_access('leader', \%leader, 'r'); | |
6314 | ||
6315 | my %equivalents; | |
6316 | # An array of any tables that have this one as their leader | |
6317 | main::set_access('equivalents', \%equivalents, 'readable_array'); | |
6318 | ||
6319 | my %parent; | |
6320 | # The parent table to this one, initially $self. This allows us to | |
c12f2655 KW |
6321 | # distinguish between equivalent tables that are related (for which this |
6322 | # is set to), and those which may not be, but share the same output file | |
6323 | # because they match the exact same set of code points in the current | |
6324 | # Unicode release. | |
99870f4d KW |
6325 | main::set_access('parent', \%parent, 'r'); |
6326 | ||
6327 | my %children; | |
6328 | # An array of any tables that have this one as their parent | |
6329 | main::set_access('children', \%children, 'readable_array'); | |
6330 | ||
6331 | my %conflicting; | |
6332 | # Array of any tables that would have the same name as this one with | |
6333 | # a different meaning. This is used for the generated documentation. | |
6334 | main::set_access('conflicting', \%conflicting, 'readable_array'); | |
6335 | ||
6336 | my %matches_all; | |
6337 | # Set in the constructor for tables that are expected to match all code | |
6338 | # points. | |
6339 | main::set_access('matches_all', \%matches_all, 'r'); | |
6340 | ||
a92d5c2e KW |
6341 | my %complement; |
6342 | # Points to the complement that this table is expressed in terms of; 0 if | |
6343 | # none. | |
8ae00c8a | 6344 | main::set_access('complement', \%complement, 'r'); |
a92d5c2e | 6345 | |
99870f4d KW |
6346 | sub new { |
6347 | my $class = shift; | |
6348 | ||
6349 | my %args = @_; | |
6350 | ||
6351 | # The property for which this table is a listing of property values. | |
6352 | my $property = delete $args{'_Property'}; | |
6353 | ||
23e33b60 KW |
6354 | my $name = delete $args{'Name'}; |
6355 | my $full_name = delete $args{'Full_Name'}; | |
6356 | $full_name = $name if ! defined $full_name; | |
6357 | ||
99870f4d KW |
6358 | # Optional |
6359 | my $initialize = delete $args{'Initialize'}; | |
6360 | my $matches_all = delete $args{'Matches_All'} || 0; | |
f5817e0a | 6361 | my $format = delete $args{'Format'}; |
99870f4d KW |
6362 | # Rest of parameters passed on. |
6363 | ||
6364 | my $range_list = Range_List->new(Initialize => $initialize, | |
6365 | Owner => $property); | |
6366 | ||
23e33b60 KW |
6367 | my $complete = $full_name; |
6368 | $complete = '""' if $complete eq ""; # A null name shouldn't happen, | |
6369 | # but this helps debug if it | |
6370 | # does | |
6371 | # The complete name for a match table includes it's property in a | |
6372 | # compound form 'property=table', except if the property is the | |
6373 | # pseudo-property, perl, in which case it is just the single form, | |
6374 | # 'table' (If you change the '=' must also change the ':' in lots of | |
6375 | # places in this program that assume an equal sign) | |
6376 | $complete = $property->full_name . "=$complete" if $property != $perl; | |
678f13d5 | 6377 | |
99870f4d | 6378 | my $self = $class->SUPER::new(%args, |
23e33b60 KW |
6379 | Name => $name, |
6380 | Complete_Name => $complete, | |
6381 | Full_Name => $full_name, | |
99870f4d KW |
6382 | _Property => $property, |
6383 | _Range_List => $range_list, | |
f5817e0a | 6384 | Format => $EMPTY_FORMAT, |
99870f4d | 6385 | ); |
ffe43484 | 6386 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
6387 | |
6388 | $conflicting{$addr} = [ ]; | |
6389 | $equivalents{$addr} = [ ]; | |
6390 | $children{$addr} = [ ]; | |
6391 | $matches_all{$addr} = $matches_all; | |
6392 | $leader{$addr} = $self; | |
6393 | $parent{$addr} = $self; | |
a92d5c2e | 6394 | $complement{$addr} = 0; |
99870f4d | 6395 | |
f5817e0a KW |
6396 | if (defined $format && $format ne $EMPTY_FORMAT) { |
6397 | Carp::my_carp_bug("'Format' must be '$EMPTY_FORMAT' in a match table instead of '$format'. Using '$EMPTY_FORMAT'"); | |
6398 | } | |
6399 | ||
99870f4d KW |
6400 | return $self; |
6401 | } | |
6402 | ||
6403 | # See this program's beginning comment block about overloading these. | |
6404 | use overload | |
6405 | fallback => 0, | |
6406 | qw("") => "_operator_stringify", | |
6407 | '=' => sub { | |
6408 | my $self = shift; | |
6409 | ||
6410 | return if $self->carp_if_locked; | |
6411 | return $self; | |
6412 | }, | |
6413 | ||
6414 | '+' => sub { | |
6415 | my $self = shift; | |
6416 | my $other = shift; | |
6417 | ||
6418 | return $self->_range_list + $other; | |
6419 | }, | |
6420 | '&' => sub { | |
6421 | my $self = shift; | |
6422 | my $other = shift; | |
6423 | ||
6424 | return $self->_range_list & $other; | |
6425 | }, | |
6426 | '+=' => sub { | |
6427 | my $self = shift; | |
6428 | my $other = shift; | |
6429 | ||
6430 | return if $self->carp_if_locked; | |
6431 | ||
ffe43484 | 6432 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
6433 | |
6434 | if (ref $other) { | |
6435 | ||
6436 | # Change the range list of this table to be the | |
6437 | # union of the two. | |
6438 | $self->_set_range_list($self->_range_list | |
6439 | + $other); | |
6440 | } | |
6441 | else { # $other is just a simple value | |
6442 | $self->add_range($other, $other); | |
6443 | } | |
6444 | return $self; | |
6445 | }, | |
6446 | '-' => sub { my $self = shift; | |
6447 | my $other = shift; | |
6448 | my $reversed = shift; | |
6449 | ||
6450 | if ($reversed) { | |
6451 | Carp::my_carp_bug("Can't cope with a " | |
6452 | . __PACKAGE__ | |
6453 | . " being the first parameter in a '-'. Subtraction ignored."); | |
6454 | return; | |
6455 | } | |
6456 | ||
6457 | return $self->_range_list - $other; | |
6458 | }, | |
6459 | '~' => sub { my $self = shift; | |
6460 | return ~ $self->_range_list; | |
6461 | }, | |
6462 | ; | |
6463 | ||
6464 | sub _operator_stringify { | |
6465 | my $self = shift; | |
6466 | ||
23e33b60 | 6467 | my $name = $self->complete_name; |
99870f4d KW |
6468 | return "Table '$name'"; |
6469 | } | |
6470 | ||
ec40ee88 KW |
6471 | sub _range_list { |
6472 | # Returns the range list associated with this table, which will be the | |
6473 | # complement's if it has one. | |
6474 | ||
6475 | my $self = shift; | |
6476 | my $complement; | |
6477 | if (($complement = $self->complement) != 0) { | |
6478 | return ~ $complement->_range_list; | |
6479 | } | |
6480 | else { | |
6481 | return $self->SUPER::_range_list; | |
6482 | } | |
6483 | } | |
6484 | ||
99870f4d KW |
6485 | sub add_alias { |
6486 | # Add a synonym for this table. See the comments in the base class | |
6487 | ||
6488 | my $self = shift; | |
6489 | my $name = shift; | |
6490 | # Rest of parameters passed on. | |
6491 | ||
6492 | $self->SUPER::add_alias($name, $self, @_); | |
6493 | return; | |
6494 | } | |
6495 | ||
6496 | sub add_conflicting { | |
6497 | # Add the name of some other object to the list of ones that name | |
6498 | # clash with this match table. | |
6499 | ||
6500 | my $self = shift; | |
6501 | my $conflicting_name = shift; # The name of the conflicting object | |
6502 | my $p = shift || 'p'; # Optional, is this a \p{} or \P{} ? | |
6503 | my $conflicting_object = shift; # Optional, the conflicting object | |
6504 | # itself. This is used to | |
6505 | # disambiguate the text if the input | |
6506 | # name is identical to any of the | |
6507 | # aliases $self is known by. | |
6508 | # Sometimes the conflicting object is | |
6509 | # merely hypothetical, so this has to | |
6510 | # be an optional parameter. | |
6511 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6512 | ||
ffe43484 | 6513 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
6514 | |
6515 | # Check if the conflicting name is exactly the same as any existing | |
6516 | # alias in this table (as long as there is a real object there to | |
6517 | # disambiguate with). | |
6518 | if (defined $conflicting_object) { | |
6519 | foreach my $alias ($self->aliases) { | |
6520 | if ($alias->name eq $conflicting_name) { | |
6521 | ||
6522 | # Here, there is an exact match. This results in | |
6523 | # ambiguous comments, so disambiguate by changing the | |
6524 | # conflicting name to its object's complete equivalent. | |
6525 | $conflicting_name = $conflicting_object->complete_name; | |
6526 | last; | |
6527 | } | |
6528 | } | |
6529 | } | |
6530 | ||
6531 | # Convert to the \p{...} final name | |
6532 | $conflicting_name = "\\$p" . "{$conflicting_name}"; | |
6533 | ||
6534 | # Only add once | |
6535 | return if grep { $conflicting_name eq $_ } @{$conflicting{$addr}}; | |
6536 | ||
6537 | push @{$conflicting{$addr}}, $conflicting_name; | |
6538 | ||
6539 | return; | |
6540 | } | |
6541 | ||
6505c6e2 | 6542 | sub is_set_equivalent_to { |
99870f4d KW |
6543 | # Return boolean of whether or not the other object is a table of this |
6544 | # type and has been marked equivalent to this one. | |
6545 | ||
6546 | my $self = shift; | |
6547 | my $other = shift; | |
6548 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6549 | ||
6550 | return 0 if ! defined $other; # Can happen for incomplete early | |
6551 | # releases | |
6552 | unless ($other->isa(__PACKAGE__)) { | |
6553 | my $ref_other = ref $other; | |
6554 | my $ref_self = ref $self; | |
6505c6e2 | 6555 | Carp::my_carp_bug("Argument to 'is_set_equivalent_to' must be another $ref_self, not a '$ref_other'. $other not set equivalent to $self."); |
99870f4d KW |
6556 | return 0; |
6557 | } | |
6558 | ||
6559 | # Two tables are equivalent if they have the same leader. | |
f998e60c | 6560 | no overloading; |
051df77b | 6561 | return $leader{pack 'J', $self} == $leader{pack 'J', $other}; |
99870f4d KW |
6562 | return; |
6563 | } | |
6564 | ||
99870f4d KW |
6565 | sub set_equivalent_to { |
6566 | # Set $self equivalent to the parameter table. | |
6567 | # The required Related => 'x' parameter is a boolean indicating | |
6568 | # whether these tables are related or not. If related, $other becomes | |
6569 | # the 'parent' of $self; if unrelated it becomes the 'leader' | |
6570 | # | |
6571 | # Related tables share all characteristics except names; equivalents | |
6572 | # not quite so many. | |
6573 | # If they are related, one must be a perl extension. This is because | |
6574 | # we can't guarantee that Unicode won't change one or the other in a | |
98dc9551 | 6575 | # later release even if they are identical now. |
99870f4d KW |
6576 | |
6577 | my $self = shift; | |
6578 | my $other = shift; | |
6579 | ||
6580 | my %args = @_; | |
6581 | my $related = delete $args{'Related'}; | |
6582 | ||
6583 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
6584 | ||
6585 | return if ! defined $other; # Keep on going; happens in some early | |
6586 | # Unicode releases. | |
6587 | ||
6588 | if (! defined $related) { | |
6589 | Carp::my_carp_bug("set_equivalent_to must have 'Related => [01] parameter. Assuming $self is not related to $other"); | |
6590 | $related = 0; | |
6591 | } | |
6592 | ||
6593 | # If already are equivalent, no need to re-do it; if subroutine | |
6594 | # returns null, it found an error, also do nothing | |
6505c6e2 | 6595 | my $are_equivalent = $self->is_set_equivalent_to($other); |
99870f4d KW |
6596 | return if ! defined $are_equivalent || $are_equivalent; |
6597 | ||
ffe43484 | 6598 | my $addr = do { no overloading; pack 'J', $self; }; |
f998e60c | 6599 | my $current_leader = ($related) ? $parent{$addr} : $leader{$addr}; |
99870f4d | 6600 | |
45e32b91 KW |
6601 | if ($related) { |
6602 | if ($current_leader->perl_extension) { | |
6603 | if ($other->perl_extension) { | |
6604 | Carp::my_carp_bug("Use add_alias() to set two Perl tables '$self' and '$other', equivalent."); | |
6605 | return; | |
6606 | } | |
7610e9e2 KW |
6607 | } elsif ($self->property != $other->property # Depending on |
6608 | # situation, might | |
6609 | # be better to use | |
6610 | # add_alias() | |
6611 | # instead for same | |
6612 | # property | |
6613 | && ! $other->perl_extension) | |
6614 | { | |
45e32b91 KW |
6615 | Carp::my_carp_bug("set_equivalent_to should have 'Related => 0 for equivalencing two Unicode properties. Assuming $self is not related to $other"); |
6616 | $related = 0; | |
6617 | } | |
6618 | } | |
6619 | ||
6620 | if (! $self->is_empty && ! $self->matches_identically_to($other)) { | |
6621 | Carp::my_carp_bug("$self should be empty or match identically to $other. Not setting equivalent"); | |
6622 | return; | |
99870f4d KW |
6623 | } |
6624 | ||
ffe43484 NC |
6625 | my $leader = do { no overloading; pack 'J', $current_leader; }; |
6626 | my $other_addr = do { no overloading; pack 'J', $other; }; | |
99870f4d KW |
6627 | |
6628 | # Any tables that are equivalent to or children of this table must now | |
6629 | # instead be equivalent to or (children) to the new leader (parent), | |
6630 | # still equivalent. The equivalency includes their matches_all info, | |
6631 | # and for related tables, their status | |
6632 | # All related tables are of necessity equivalent, but the converse | |
6633 | # isn't necessarily true | |
6634 | my $status = $other->status; | |
6635 | my $status_info = $other->status_info; | |
6636 | my $matches_all = $matches_all{other_addr}; | |
d867ccfb | 6637 | my $caseless_equivalent = $other->caseless_equivalent; |
99870f4d KW |
6638 | foreach my $table ($current_leader, @{$equivalents{$leader}}) { |
6639 | next if $table == $other; | |
6640 | trace "setting $other to be the leader of $table, status=$status" if main::DEBUG && $to_trace; | |
6641 | ||
ffe43484 | 6642 | my $table_addr = do { no overloading; pack 'J', $table; }; |
99870f4d KW |
6643 | $leader{$table_addr} = $other; |
6644 | $matches_all{$table_addr} = $matches_all; | |
6645 | $self->_set_range_list($other->_range_list); | |
6646 | push @{$equivalents{$other_addr}}, $table; | |
6647 | if ($related) { | |
6648 | $parent{$table_addr} = $other; | |
6649 | push @{$children{$other_addr}}, $table; | |
6650 | $table->set_status($status, $status_info); | |
d867ccfb | 6651 | $self->set_caseless_equivalent($caseless_equivalent); |
99870f4d KW |
6652 | } |
6653 | } | |
6654 | ||
6655 | # Now that we've declared these to be equivalent, any changes to one | |
6656 | # of the tables would invalidate that equivalency. | |
6657 | $self->lock; | |
6658 | $other->lock; | |
6659 | return; | |
6660 | } | |
6661 | ||
8ae00c8a KW |
6662 | sub set_complement { |
6663 | # Set $self to be the complement of the parameter table. $self is | |
6664 | # locked, as what it contains should all come from the other table. | |
6665 | ||
6666 | my $self = shift; | |
6667 | my $other = shift; | |
6668 | ||
6669 | my %args = @_; | |
6670 | Carp::carp_extra_args(\%args) if main::DEBUG && %args; | |
6671 | ||
6672 | if ($other->complement != 0) { | |
6673 | Carp::my_carp_bug("Can't set $self to be the complement of $other, which itself is the complement of " . $other->complement); | |
6674 | return; | |
6675 | } | |
6676 | my $addr = do { no overloading; pack 'J', $self; }; | |
6677 | $complement{$addr} = $other; | |
6678 | $self->lock; | |
6679 | return; | |
6680 | } | |
6681 | ||
99870f4d KW |
6682 | sub add_range { # Add a range to the list for this table. |
6683 | my $self = shift; | |
6684 | # Rest of parameters passed on | |
6685 | ||
6686 | return if $self->carp_if_locked; | |
6687 | return $self->_range_list->add_range(@_); | |
6688 | } | |
6689 | ||
88c22f80 KW |
6690 | sub header { |
6691 | my $self = shift; | |
6692 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6693 | ||
6694 | # All match tables are to be used only by the Perl core. | |
126c3d4e | 6695 | return $self->SUPER::header() . $INTERNAL_ONLY_HEADER; |
88c22f80 KW |
6696 | } |
6697 | ||
99870f4d KW |
6698 | sub pre_body { # Does nothing for match tables. |
6699 | return | |
6700 | } | |
6701 | ||
6702 | sub append_to_body { # Does nothing for match tables. | |
6703 | return | |
6704 | } | |
6705 | ||
6706 | sub write { | |
6707 | my $self = shift; | |
6708 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6709 | ||
6710 | return $self->SUPER::write(2); # 2 tab stops | |
6711 | } | |
6712 | ||
6713 | sub set_final_comment { | |
6714 | # This creates a comment for the file that is to hold the match table | |
6715 | # $self. It is somewhat convoluted to make the English read nicely, | |
6716 | # but, heh, it's just a comment. | |
6717 | # This should be called only with the leader match table of all the | |
6718 | # ones that share the same file. It lists all such tables, ordered so | |
6719 | # that related ones are together. | |
6720 | ||
bd9ebcfd KW |
6721 | return unless $debugging_build; |
6722 | ||
99870f4d KW |
6723 | my $leader = shift; # Should only be called on the leader table of |
6724 | # an equivalent group | |
6725 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
6726 | ||
ffe43484 | 6727 | my $addr = do { no overloading; pack 'J', $leader; }; |
99870f4d KW |
6728 | |
6729 | if ($leader{$addr} != $leader) { | |
6730 | Carp::my_carp_bug(<<END | |
6731 | set_final_comment() must be called on a leader table, which $leader is not. | |
6732 | It is equivalent to $leader{$addr}. No comment created | |
6733 | END | |
6734 | ); | |
6735 | return; | |
6736 | } | |
6737 | ||
6738 | # Get the number of code points matched by each of the tables in this | |
6739 | # file, and add underscores for clarity. | |
6740 | my $count = $leader->count; | |
6741 | my $string_count = main::clarify_number($count); | |
6742 | ||
6743 | my $loose_count = 0; # how many aliases loosely matched | |
6744 | my $compound_name = ""; # ? Are any names compound?, and if so, an | |
6745 | # example | |
6746 | my $properties_with_compound_names = 0; # count of these | |
6747 | ||
6748 | ||
6749 | my %flags; # The status flags used in the file | |
6750 | my $total_entries = 0; # number of entries written in the comment | |
6751 | my $matches_comment = ""; # The portion of the comment about the | |
6752 | # \p{}'s | |
6753 | my @global_comments; # List of all the tables' comments that are | |
6754 | # there before this routine was called. | |
6755 | ||
6756 | # Get list of all the parent tables that are equivalent to this one | |
6757 | # (including itself). | |
6758 | my @parents = grep { $parent{main::objaddr $_} == $_ } | |
6759 | main::uniques($leader, @{$equivalents{$addr}}); | |
6760 | my $has_unrelated = (@parents >= 2); # boolean, ? are there unrelated | |
6761 | # tables | |
6762 | ||
6763 | for my $parent (@parents) { | |
6764 | ||
6765 | my $property = $parent->property; | |
6766 | ||
6767 | # Special case 'N' tables in properties with two match tables when | |
6768 | # the other is a 'Y' one. These are likely to be binary tables, | |
6769 | # but not necessarily. In either case, \P{} will match the | |
6770 | # complement of \p{}, and so if something is a synonym of \p, the | |
6771 | # complement of that something will be the synonym of \P. This | |
6772 | # would be true of any property with just two match tables, not | |
6773 | # just those whose values are Y and N; but that would require a | |
6774 | # little extra work, and there are none such so far in Unicode. | |
6775 | my $perl_p = 'p'; # which is it? \p{} or \P{} | |
6776 | my @yes_perl_synonyms; # list of any synonyms for the 'Y' table | |
6777 | ||
6778 | if (scalar $property->tables == 2 | |
6779 | && $parent == $property->table('N') | |
6780 | && defined (my $yes = $property->table('Y'))) | |
6781 | { | |
ffe43484 | 6782 | my $yes_addr = do { no overloading; pack 'J', $yes; }; |
99870f4d KW |
6783 | @yes_perl_synonyms |
6784 | = grep { $_->property == $perl } | |
6785 | main::uniques($yes, | |
6786 | $parent{$yes_addr}, | |
6787 | $parent{$yes_addr}->children); | |
6788 | ||
6789 | # But these synonyms are \P{} ,not \p{} | |
6790 | $perl_p = 'P'; | |
6791 | } | |
6792 | ||
6793 | my @description; # Will hold the table description | |
6794 | my @note; # Will hold the table notes. | |
6795 | my @conflicting; # Will hold the table conflicts. | |
6796 | ||
6797 | # Look at the parent, any yes synonyms, and all the children | |
ffe43484 | 6798 | my $parent_addr = do { no overloading; pack 'J', $parent; }; |
99870f4d KW |
6799 | for my $table ($parent, |
6800 | @yes_perl_synonyms, | |
f998e60c | 6801 | @{$children{$parent_addr}}) |
99870f4d | 6802 | { |
ffe43484 | 6803 | my $table_addr = do { no overloading; pack 'J', $table; }; |
99870f4d KW |
6804 | my $table_property = $table->property; |
6805 | ||
6806 | # Tables are separated by a blank line to create a grouping. | |
6807 | $matches_comment .= "\n" if $matches_comment; | |
6808 | ||
6809 | # The table is named based on the property and value | |
6810 | # combination it is for, like script=greek. But there may be | |
6811 | # a number of synonyms for each side, like 'sc' for 'script', | |
6812 | # and 'grek' for 'greek'. Any combination of these is a valid | |
6813 | # name for this table. In this case, there are three more, | |
6814 | # 'sc=grek', 'sc=greek', and 'script='grek'. Rather than | |
6815 | # listing all possible combinations in the comment, we make | |
6816 | # sure that each synonym occurs at least once, and add | |
6817 | # commentary that the other combinations are possible. | |
da912e1e KW |
6818 | # Because regular expressions don't recognize things like |
6819 | # \p{jsn=}, only look at non-null right-hand-sides | |
99870f4d | 6820 | my @property_aliases = $table_property->aliases; |
da912e1e | 6821 | my @table_aliases = grep { $_->name ne "" } $table->aliases; |
99870f4d KW |
6822 | |
6823 | # The alias lists above are already ordered in the order we | |
6824 | # want to output them. To ensure that each synonym is listed, | |
da912e1e KW |
6825 | # we must use the max of the two numbers. But if there are no |
6826 | # legal synonyms (nothing in @table_aliases), then we don't | |
6827 | # list anything. | |
6828 | my $listed_combos = (@table_aliases) | |
6829 | ? main::max(scalar @table_aliases, | |
6830 | scalar @property_aliases) | |
6831 | : 0; | |
99870f4d KW |
6832 | trace "$listed_combos, tables=", scalar @table_aliases, "; names=", scalar @property_aliases if main::DEBUG; |
6833 | ||
da912e1e | 6834 | |
99870f4d KW |
6835 | my $property_had_compound_name = 0; |
6836 | ||
6837 | for my $i (0 .. $listed_combos - 1) { | |
6838 | $total_entries++; | |
6839 | ||
6840 | # The current alias for the property is the next one on | |
6841 | # the list, or if beyond the end, start over. Similarly | |
6842 | # for the table (\p{prop=table}) | |
6843 | my $property_alias = $property_aliases | |
6844 | [$i % @property_aliases]->name; | |
6845 | my $table_alias_object = $table_aliases | |
6846 | [$i % @table_aliases]; | |
6847 | my $table_alias = $table_alias_object->name; | |
6848 | my $loose_match = $table_alias_object->loose_match; | |
6849 | ||
6850 | if ($table_alias !~ /\D/) { # Clarify large numbers. | |
6851 | $table_alias = main::clarify_number($table_alias) | |
6852 | } | |
6853 | ||
6854 | # Add a comment for this alias combination | |
6855 | my $current_match_comment; | |
6856 | if ($table_property == $perl) { | |
6857 | $current_match_comment = "\\$perl_p" | |
6858 | . "{$table_alias}"; | |
6859 | } | |
6860 | else { | |
6861 | $current_match_comment | |
6862 | = "\\p{$property_alias=$table_alias}"; | |
6863 | $property_had_compound_name = 1; | |
6864 | } | |
6865 | ||
6866 | # Flag any abnormal status for this table. | |
6867 | my $flag = $property->status | |
6868 | || $table->status | |
6869 | || $table_alias_object->status; | |
37e2e78e KW |
6870 | if ($flag) { |
6871 | if ($flag ne $PLACEHOLDER) { | |
6872 | $flags{$flag} = $status_past_participles{$flag}; | |
6873 | } else { | |
6874 | $flags{$flag} = <<END; | |
6875 | a placeholder because it is not in Version $string_version of Unicode, but is | |
6876 | needed by the Perl core to work gracefully. Because it is not in this version | |
6877 | of Unicode, it will not be listed in $pod_file.pod | |
6878 | END | |
6879 | } | |
6880 | } | |
99870f4d KW |
6881 | |
6882 | $loose_count++; | |
6883 | ||
6884 | # Pretty up the comment. Note the \b; it says don't make | |
6885 | # this line a continuation. | |
6886 | $matches_comment .= sprintf("\b%-1s%-s%s\n", | |
6887 | $flag, | |
6888 | " " x 7, | |
6889 | $current_match_comment); | |
6890 | } # End of generating the entries for this table. | |
6891 | ||
6892 | # Save these for output after this group of related tables. | |
6893 | push @description, $table->description; | |
6894 | push @note, $table->note; | |
6895 | push @conflicting, $table->conflicting; | |
6896 | ||
37e2e78e KW |
6897 | # And this for output after all the tables. |
6898 | push @global_comments, $table->comment; | |
6899 | ||
99870f4d KW |
6900 | # Compute an alternate compound name using the final property |
6901 | # synonym and the first table synonym with a colon instead of | |
6902 | # the equal sign used elsewhere. | |
6903 | if ($property_had_compound_name) { | |
6904 | $properties_with_compound_names ++; | |
6905 | if (! $compound_name || @property_aliases > 1) { | |
6906 | $compound_name = $property_aliases[-1]->name | |
6907 | . ': ' | |
6908 | . $table_aliases[0]->name; | |
6909 | } | |
6910 | } | |
6911 | } # End of looping through all children of this table | |
6912 | ||
6913 | # Here have assembled in $matches_comment all the related tables | |
6914 | # to the current parent (preceded by the same info for all the | |
6915 | # previous parents). Put out information that applies to all of | |
6916 | # the current family. | |
6917 | if (@conflicting) { | |
6918 | ||
6919 | # But output the conflicting information now, as it applies to | |
6920 | # just this table. | |
6921 | my $conflicting = join ", ", @conflicting; | |
6922 | if ($conflicting) { | |
6923 | $matches_comment .= <<END; | |
6924 | ||
6925 | Note that contrary to what you might expect, the above is NOT the same as | |
6926 | END | |
6927 | $matches_comment .= "any of: " if @conflicting > 1; | |
6928 | $matches_comment .= "$conflicting\n"; | |
6929 | } | |
6930 | } | |
6931 | if (@description) { | |
6932 | $matches_comment .= "\n Meaning: " | |
6933 | . join('; ', @description) | |
6934 | . "\n"; | |
6935 | } | |
6936 | if (@note) { | |
6937 | $matches_comment .= "\n Note: " | |
6938 | . join("\n ", @note) | |
6939 | . "\n"; | |
6940 | } | |
6941 | } # End of looping through all tables | |
6942 | ||
6943 | ||
6944 | my $code_points; | |
6945 | my $match; | |
6946 | my $any_of_these; | |
6947 | if ($count == 1) { | |
6948 | $match = 'matches'; | |
6949 | $code_points = 'single code point'; | |
6950 | } | |
6951 | else { | |
6952 | $match = 'match'; | |
6953 | $code_points = "$string_count code points"; | |
6954 | } | |
6955 | ||
6956 | my $synonyms; | |
6957 | my $entries; | |
da912e1e | 6958 | if ($total_entries == 1) { |
99870f4d KW |
6959 | $synonyms = ""; |
6960 | $entries = 'entry'; | |
6961 | $any_of_these = 'this' | |
6962 | } | |
6963 | else { | |
6964 | $synonyms = " any of the following regular expression constructs"; | |
6965 | $entries = 'entries'; | |
6966 | $any_of_these = 'any of these' | |
6967 | } | |
6968 | ||
6969 | my $comment = ""; | |
6970 | if ($has_unrelated) { | |
6971 | $comment .= <<END; | |
6972 | This file is for tables that are not necessarily related: To conserve | |
6973 | resources, every table that matches the identical set of code points in this | |
6974 | version of Unicode uses this file. Each one is listed in a separate group | |
6975 | below. It could be that the tables will match the same set of code points in | |
6976 | other Unicode releases, or it could be purely coincidence that they happen to | |
6977 | be the same in Unicode $string_version, and hence may not in other versions. | |
6978 | ||
6979 | END | |
6980 | } | |
6981 | ||
6982 | if (%flags) { | |
6983 | foreach my $flag (sort keys %flags) { | |
6984 | $comment .= <<END; | |
37e2e78e | 6985 | '$flag' below means that this form is $flags{$flag}. |
99870f4d | 6986 | END |
37e2e78e KW |
6987 | next if $flag eq $PLACEHOLDER; |
6988 | $comment .= "Consult $pod_file.pod\n"; | |
99870f4d KW |
6989 | } |
6990 | $comment .= "\n"; | |
6991 | } | |
6992 | ||
da912e1e KW |
6993 | if ($total_entries == 0) { |
6994 | Carp::my_carp("No regular expression construct can match $leader, as all names for it are the null string. Creating file anyway."); | |
6995 | $comment .= <<END; | |
6996 | This file returns the $code_points in Unicode Version $string_version for | |
6997 | $leader, but it is inaccessible through Perl regular expressions, as | |
6998 | "\\p{prop=}" is not recognized. | |
6999 | END | |
7000 | ||
7001 | } else { | |
7002 | $comment .= <<END; | |
99870f4d KW |
7003 | This file returns the $code_points in Unicode Version $string_version that |
7004 | $match$synonyms: | |
7005 | ||
7006 | $matches_comment | |
37e2e78e | 7007 | $pod_file.pod should be consulted for the syntax rules for $any_of_these, |
99870f4d KW |
7008 | including if adding or subtracting white space, underscore, and hyphen |
7009 | characters matters or doesn't matter, and other permissible syntactic | |
7010 | variants. Upper/lower case distinctions never matter. | |
7011 | END | |
7012 | ||
da912e1e | 7013 | } |
99870f4d KW |
7014 | if ($compound_name) { |
7015 | $comment .= <<END; | |
7016 | ||
7017 | A colon can be substituted for the equals sign, and | |
7018 | END | |
7019 | if ($properties_with_compound_names > 1) { | |
7020 | $comment .= <<END; | |
7021 | within each group above, | |
7022 | END | |
7023 | } | |
7024 | $compound_name = sprintf("%-8s\\p{%s}", " ", $compound_name); | |
7025 | ||
7026 | # Note the \b below, it says don't make that line a continuation. | |
7027 | $comment .= <<END; | |
7028 | anything to the left of the equals (or colon) can be combined with anything to | |
7029 | the right. Thus, for example, | |
7030 | $compound_name | |
7031 | \bis also valid. | |
7032 | END | |
7033 | } | |
7034 | ||
7035 | # And append any comment(s) from the actual tables. They are all | |
7036 | # gathered here, so may not read all that well. | |
37e2e78e KW |
7037 | if (@global_comments) { |
7038 | $comment .= "\n" . join("\n\n", @global_comments) . "\n"; | |
7039 | } | |
99870f4d KW |
7040 | |
7041 | if ($count) { # The format differs if no code points, and needs no | |
7042 | # explanation in that case | |
7043 | $comment.= <<END; | |
7044 | ||
7045 | The format of the lines of this file is: | |
7046 | END | |
7047 | $comment.= <<END; | |
7048 | START\\tSTOP\\twhere START is the starting code point of the range, in hex; | |
7049 | STOP is the ending point, or if omitted, the range has just one code point. | |
7050 | END | |
0c07e538 | 7051 | if ($leader->output_range_counts) { |
99870f4d KW |
7052 | $comment .= <<END; |
7053 | Numbers in comments in [brackets] indicate how many code points are in the | |
7054 | range. | |
7055 | END | |
7056 | } | |
7057 | } | |
7058 | ||
7059 | $leader->set_comment(main::join_lines($comment)); | |
7060 | return; | |
7061 | } | |
7062 | ||
7063 | # Accessors for the underlying list | |
ea25a9b2 | 7064 | for my $sub (qw( |
99870f4d KW |
7065 | get_valid_code_point |
7066 | get_invalid_code_point | |
ea25a9b2 | 7067 | )) |
99870f4d KW |
7068 | { |
7069 | no strict "refs"; | |
7070 | *$sub = sub { | |
7071 | use strict "refs"; | |
7072 | my $self = shift; | |
7073 | ||
7074 | return $self->_range_list->$sub(@_); | |
7075 | } | |
7076 | } | |
7077 | } # End closure for Match_Table | |
7078 | ||
7079 | package Property; | |
7080 | ||
7081 | # The Property class represents a Unicode property, or the $perl | |
7082 | # pseudo-property. It contains a map table initialized empty at construction | |
7083 | # time, and for properties accessible through regular expressions, various | |
7084 | # match tables, created through the add_match_table() method, and referenced | |
7085 | # by the table('NAME') or tables() methods, the latter returning a list of all | |
7086 | # of the match tables. Otherwise table operations implicitly are for the map | |
7087 | # table. | |
7088 | # | |
7089 | # Most of the data in the property is actually about its map table, so it | |
7090 | # mostly just uses that table's accessors for most methods. The two could | |
7091 | # have been combined into one object, but for clarity because of their | |
7092 | # differing semantics, they have been kept separate. It could be argued that | |
7093 | # the 'file' and 'directory' fields should be kept with the map table. | |
7094 | # | |
7095 | # Each property has a type. This can be set in the constructor, or in the | |
7096 | # set_type accessor, but mostly it is figured out by the data. Every property | |
7097 | # starts with unknown type, overridden by a parameter to the constructor, or | |
7098 | # as match tables are added, or ranges added to the map table, the data is | |
7099 | # inspected, and the type changed. After the table is mostly or entirely | |
7100 | # filled, compute_type() should be called to finalize they analysis. | |
7101 | # | |
7102 | # There are very few operations defined. One can safely remove a range from | |
7103 | # the map table, and property_add_or_replace_non_nulls() adds the maps from another | |
7104 | # table to this one, replacing any in the intersection of the two. | |
7105 | ||
7106 | sub standardize { return main::standardize($_[0]); } | |
7107 | sub trace { return main::trace(@_) if main::DEBUG && $to_trace } | |
7108 | ||
7109 | { # Closure | |
7110 | ||
7111 | # This hash will contain as keys, all the aliases of all properties, and | |
7112 | # as values, pointers to their respective property objects. This allows | |
7113 | # quick look-up of a property from any of its names. | |
7114 | my %alias_to_property_of; | |
7115 | ||
7116 | sub dump_alias_to_property_of { | |
7117 | # For debugging | |
7118 | ||
7119 | print "\n", main::simple_dumper (\%alias_to_property_of), "\n"; | |
7120 | return; | |
7121 | } | |
7122 | ||
7123 | sub property_ref { | |
7124 | # This is a package subroutine, not called as a method. | |
7125 | # If the single parameter is a literal '*' it returns a list of all | |
7126 | # defined properties. | |
7127 | # Otherwise, the single parameter is a name, and it returns a pointer | |
7128 | # to the corresponding property object, or undef if none. | |
7129 | # | |
7130 | # Properties can have several different names. The 'standard' form of | |
7131 | # each of them is stored in %alias_to_property_of as they are defined. | |
7132 | # But it's possible that this subroutine will be called with some | |
7133 | # variant, so if the initial lookup fails, it is repeated with the | |
98dc9551 | 7134 | # standardized form of the input name. If found, besides returning the |
99870f4d KW |
7135 | # result, the input name is added to the list so future calls won't |
7136 | # have to do the conversion again. | |
7137 | ||
7138 | my $name = shift; | |
7139 | ||
7140 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7141 | ||
7142 | if (! defined $name) { | |
7143 | Carp::my_carp_bug("Undefined input property. No action taken."); | |
7144 | return; | |
7145 | } | |
7146 | ||
7147 | return main::uniques(values %alias_to_property_of) if $name eq '*'; | |
7148 | ||
7149 | # Return cached result if have it. | |
7150 | my $result = $alias_to_property_of{$name}; | |
7151 | return $result if defined $result; | |
7152 | ||
7153 | # Convert the input to standard form. | |
7154 | my $standard_name = standardize($name); | |
7155 | ||
7156 | $result = $alias_to_property_of{$standard_name}; | |
7157 | return unless defined $result; # Don't cache undefs | |
7158 | ||
7159 | # Cache the result before returning it. | |
7160 | $alias_to_property_of{$name} = $result; | |
7161 | return $result; | |
7162 | } | |
7163 | ||
7164 | ||
7165 | main::setup_package(); | |
7166 | ||
7167 | my %map; | |
7168 | # A pointer to the map table object for this property | |
7169 | main::set_access('map', \%map); | |
7170 | ||
7171 | my %full_name; | |
7172 | # The property's full name. This is a duplicate of the copy kept in the | |
7173 | # map table, but is needed because stringify needs it during | |
7174 | # construction of the map table, and then would have a chicken before egg | |
7175 | # problem. | |
7176 | main::set_access('full_name', \%full_name, 'r'); | |
7177 | ||
7178 | my %table_ref; | |
7179 | # This hash will contain as keys, all the aliases of any match tables | |
7180 | # attached to this property, and as values, the pointers to their | |
7181 | # respective tables. This allows quick look-up of a table from any of its | |
7182 | # names. | |
7183 | main::set_access('table_ref', \%table_ref); | |
7184 | ||
7185 | my %type; | |
7186 | # The type of the property, $ENUM, $BINARY, etc | |
7187 | main::set_access('type', \%type, 'r'); | |
7188 | ||
7189 | my %file; | |
7190 | # The filename where the map table will go (if actually written). | |
7191 | # Normally defaulted, but can be overridden. | |
7192 | main::set_access('file', \%file, 'r', 's'); | |
7193 | ||
7194 | my %directory; | |
7195 | # The directory where the map table will go (if actually written). | |
7196 | # Normally defaulted, but can be overridden. | |
7197 | main::set_access('directory', \%directory, 's'); | |
7198 | ||
7199 | my %pseudo_map_type; | |
7200 | # This is used to affect the calculation of the map types for all the | |
7201 | # ranges in the table. It should be set to one of the values that signify | |
7202 | # to alter the calculation. | |
7203 | main::set_access('pseudo_map_type', \%pseudo_map_type, 'r'); | |
7204 | ||
7205 | my %has_only_code_point_maps; | |
7206 | # A boolean used to help in computing the type of data in the map table. | |
7207 | main::set_access('has_only_code_point_maps', \%has_only_code_point_maps); | |
7208 | ||
7209 | my %unique_maps; | |
7210 | # A list of the first few distinct mappings this property has. This is | |
7211 | # used to disambiguate between binary and enum property types, so don't | |
7212 | # have to keep more than three. | |
7213 | main::set_access('unique_maps', \%unique_maps); | |
7214 | ||
56557540 KW |
7215 | my %pre_declared_maps; |
7216 | # A boolean that gives whether the input data should declare all the | |
7217 | # tables used, or not. If the former, unknown ones raise a warning. | |
7218 | main::set_access('pre_declared_maps', | |
047274f2 | 7219 | \%pre_declared_maps, 'r', 's'); |
56557540 | 7220 | |
99870f4d KW |
7221 | sub new { |
7222 | # The only required parameter is the positionally first, name. All | |
7223 | # other parameters are key => value pairs. See the documentation just | |
7224 | # above for the meanings of the ones not passed directly on to the map | |
7225 | # table constructor. | |
7226 | ||
7227 | my $class = shift; | |
7228 | my $name = shift || ""; | |
7229 | ||
7230 | my $self = property_ref($name); | |
7231 | if (defined $self) { | |
7232 | my $options_string = join ", ", @_; | |
7233 | $options_string = ". Ignoring options $options_string" if $options_string; | |
7234 | Carp::my_carp("$self is already in use. Using existing one$options_string;"); | |
7235 | return $self; | |
7236 | } | |
7237 | ||
7238 | my %args = @_; | |
7239 | ||
7240 | $self = bless \do { my $anonymous_scalar }, $class; | |
ffe43484 | 7241 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7242 | |
7243 | $directory{$addr} = delete $args{'Directory'}; | |
7244 | $file{$addr} = delete $args{'File'}; | |
7245 | $full_name{$addr} = delete $args{'Full_Name'} || $name; | |
7246 | $type{$addr} = delete $args{'Type'} || $UNKNOWN; | |
7247 | $pseudo_map_type{$addr} = delete $args{'Map_Type'}; | |
56557540 KW |
7248 | $pre_declared_maps{$addr} = delete $args{'Pre_Declared_Maps'} |
7249 | # Starting in this release, property | |
7250 | # values should be defined for all | |
7251 | # properties, except those overriding this | |
7252 | // $v_version ge v5.1.0; | |
c12f2655 | 7253 | |
99870f4d KW |
7254 | # Rest of parameters passed on. |
7255 | ||
7256 | $has_only_code_point_maps{$addr} = 1; | |
7257 | $table_ref{$addr} = { }; | |
7258 | $unique_maps{$addr} = { }; | |
7259 | ||
7260 | $map{$addr} = Map_Table->new($name, | |
7261 | Full_Name => $full_name{$addr}, | |
7262 | _Alias_Hash => \%alias_to_property_of, | |
7263 | _Property => $self, | |
7264 | %args); | |
7265 | return $self; | |
7266 | } | |
7267 | ||
7268 | # See this program's beginning comment block about overloading the copy | |
7269 | # constructor. Few operations are defined on properties, but a couple are | |
7270 | # useful. It is safe to take the inverse of a property, and to remove a | |
7271 | # single code point from it. | |
7272 | use overload | |
7273 | fallback => 0, | |
7274 | qw("") => "_operator_stringify", | |
7275 | "." => \&main::_operator_dot, | |
7276 | '==' => \&main::_operator_equal, | |
7277 | '!=' => \&main::_operator_not_equal, | |
7278 | '=' => sub { return shift }, | |
7279 | '-=' => "_minus_and_equal", | |
7280 | ; | |
7281 | ||
7282 | sub _operator_stringify { | |
7283 | return "Property '" . shift->full_name . "'"; | |
7284 | } | |
7285 | ||
7286 | sub _minus_and_equal { | |
7287 | # Remove a single code point from the map table of a property. | |
7288 | ||
7289 | my $self = shift; | |
7290 | my $other = shift; | |
7291 | my $reversed = shift; | |
7292 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7293 | ||
7294 | if (ref $other) { | |
7295 | Carp::my_carp_bug("Can't cope with a " | |
7296 | . ref($other) | |
7297 | . " argument to '-='. Subtraction ignored."); | |
7298 | return $self; | |
7299 | } | |
98dc9551 | 7300 | elsif ($reversed) { # Shouldn't happen in a -=, but just in case |
99870f4d KW |
7301 | Carp::my_carp_bug("Can't cope with a " |
7302 | . __PACKAGE__ | |
7303 | . " being the first parameter in a '-='. Subtraction ignored."); | |
7304 | return $self; | |
7305 | } | |
7306 | else { | |
f998e60c | 7307 | no overloading; |
051df77b | 7308 | $map{pack 'J', $self}->delete_range($other, $other); |
99870f4d KW |
7309 | } |
7310 | return $self; | |
7311 | } | |
7312 | ||
7313 | sub add_match_table { | |
7314 | # Add a new match table for this property, with name given by the | |
7315 | # parameter. It returns a pointer to the table. | |
7316 | ||
7317 | my $self = shift; | |
7318 | my $name = shift; | |
7319 | my %args = @_; | |
7320 | ||
ffe43484 | 7321 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7322 | |
7323 | my $table = $table_ref{$addr}{$name}; | |
7324 | my $standard_name = main::standardize($name); | |
7325 | if (defined $table | |
7326 | || (defined ($table = $table_ref{$addr}{$standard_name}))) | |
7327 | { | |
7328 | Carp::my_carp("Table '$name' in $self is already in use. Using existing one"); | |
7329 | $table_ref{$addr}{$name} = $table; | |
7330 | return $table; | |
7331 | } | |
7332 | else { | |
7333 | ||
7334 | # See if this is a perl extension, if not passed in. | |
7335 | my $perl_extension = delete $args{'Perl_Extension'}; | |
7336 | $perl_extension | |
7337 | = $self->perl_extension if ! defined $perl_extension; | |
7338 | ||
7339 | $table = Match_Table->new( | |
7340 | Name => $name, | |
7341 | Perl_Extension => $perl_extension, | |
7342 | _Alias_Hash => $table_ref{$addr}, | |
7343 | _Property => $self, | |
7344 | ||
7345 | # gets property's status by default | |
7346 | Status => $self->status, | |
7347 | _Status_Info => $self->status_info, | |
88c22f80 | 7348 | %args); |
99870f4d KW |
7349 | return unless defined $table; |
7350 | } | |
7351 | ||
7352 | # Save the names for quick look up | |
7353 | $table_ref{$addr}{$standard_name} = $table; | |
7354 | $table_ref{$addr}{$name} = $table; | |
7355 | ||
7356 | # Perhaps we can figure out the type of this property based on the | |
7357 | # fact of adding this match table. First, string properties don't | |
7358 | # have match tables; second, a binary property can't have 3 match | |
7359 | # tables | |
7360 | if ($type{$addr} == $UNKNOWN) { | |
7361 | $type{$addr} = $NON_STRING; | |
7362 | } | |
7363 | elsif ($type{$addr} == $STRING) { | |
7364 | Carp::my_carp("$self Added a match table '$name' to a string property '$self'. Changed it to a non-string property. Bad News."); | |
7365 | $type{$addr} = $NON_STRING; | |
7366 | } | |
06f26c45 | 7367 | elsif ($type{$addr} != $ENUM && $type{$addr} != $FORCED_BINARY) { |
99870f4d KW |
7368 | if (scalar main::uniques(values %{$table_ref{$addr}}) > 2 |
7369 | && $type{$addr} == $BINARY) | |
7370 | { | |
7371 | Carp::my_carp("$self now has more than 2 tables (with the addition of '$name'), and so is no longer binary. Changing its type to 'enum'. Bad News."); | |
7372 | $type{$addr} = $ENUM; | |
7373 | } | |
7374 | } | |
7375 | ||
7376 | return $table; | |
7377 | } | |
7378 | ||
4b9b0bc5 KW |
7379 | sub delete_match_table { |
7380 | # Delete the table referred to by $2 from the property $1. | |
7381 | ||
7382 | my $self = shift; | |
7383 | my $table_to_remove = shift; | |
7384 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7385 | ||
7386 | my $addr = do { no overloading; pack 'J', $self; }; | |
7387 | ||
7388 | # Remove all names that refer to it. | |
7389 | foreach my $key (keys %{$table_ref{$addr}}) { | |
7390 | delete $table_ref{$addr}{$key} | |
7391 | if $table_ref{$addr}{$key} == $table_to_remove; | |
7392 | } | |
7393 | ||
7394 | $table_to_remove->DESTROY; | |
7395 | return; | |
7396 | } | |
7397 | ||
99870f4d KW |
7398 | sub table { |
7399 | # Return a pointer to the match table (with name given by the | |
7400 | # parameter) associated with this property; undef if none. | |
7401 | ||
7402 | my $self = shift; | |
7403 | my $name = shift; | |
7404 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7405 | ||
ffe43484 | 7406 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7407 | |
7408 | return $table_ref{$addr}{$name} if defined $table_ref{$addr}{$name}; | |
7409 | ||
7410 | # If quick look-up failed, try again using the standard form of the | |
7411 | # input name. If that succeeds, cache the result before returning so | |
7412 | # won't have to standardize this input name again. | |
7413 | my $standard_name = main::standardize($name); | |
7414 | return unless defined $table_ref{$addr}{$standard_name}; | |
7415 | ||
7416 | $table_ref{$addr}{$name} = $table_ref{$addr}{$standard_name}; | |
7417 | return $table_ref{$addr}{$name}; | |
7418 | } | |
7419 | ||
7420 | sub tables { | |
7421 | # Return a list of pointers to all the match tables attached to this | |
7422 | # property | |
7423 | ||
f998e60c | 7424 | no overloading; |
051df77b | 7425 | return main::uniques(values %{$table_ref{pack 'J', shift}}); |
99870f4d KW |
7426 | } |
7427 | ||
7428 | sub directory { | |
7429 | # Returns the directory the map table for this property should be | |
7430 | # output in. If a specific directory has been specified, that has | |
7431 | # priority; 'undef' is returned if the type isn't defined; | |
7432 | # or $map_directory for everything else. | |
7433 | ||
ffe43484 | 7434 | my $addr = do { no overloading; pack 'J', shift; }; |
99870f4d KW |
7435 | |
7436 | return $directory{$addr} if defined $directory{$addr}; | |
7437 | return undef if $type{$addr} == $UNKNOWN; | |
7438 | return $map_directory; | |
7439 | } | |
7440 | ||
7441 | sub swash_name { | |
7442 | # Return the name that is used to both: | |
7443 | # 1) Name the file that the map table is written to. | |
7444 | # 2) The name of swash related stuff inside that file. | |
7445 | # The reason for this is that the Perl core historically has used | |
7446 | # certain names that aren't the same as the Unicode property names. | |
7447 | # To continue using these, $file is hard-coded in this file for those, | |
7448 | # but otherwise the standard name is used. This is different from the | |
7449 | # external_name, so that the rest of the files, like in lib can use | |
7450 | # the standard name always, without regard to historical precedent. | |
7451 | ||
7452 | my $self = shift; | |
7453 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7454 | ||
ffe43484 | 7455 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7456 | |
7457 | return $file{$addr} if defined $file{$addr}; | |
7458 | return $map{$addr}->external_name; | |
7459 | } | |
7460 | ||
7461 | sub to_create_match_tables { | |
7462 | # Returns a boolean as to whether or not match tables should be | |
7463 | # created for this property. | |
7464 | ||
7465 | my $self = shift; | |
7466 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7467 | ||
7468 | # The whole point of this pseudo property is match tables. | |
7469 | return 1 if $self == $perl; | |
7470 | ||
ffe43484 | 7471 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7472 | |
7473 | # Don't generate tables of code points that match the property values | |
7474 | # of a string property. Such a list would most likely have many | |
7475 | # property values, each with just one or very few code points mapping | |
7476 | # to it. | |
7477 | return 0 if $type{$addr} == $STRING; | |
7478 | ||
7479 | # Don't generate anything for unimplemented properties. | |
7480 | return 0 if grep { $self->complete_name eq $_ } | |
7481 | @unimplemented_properties; | |
7482 | # Otherwise, do. | |
7483 | return 1; | |
7484 | } | |
7485 | ||
7486 | sub property_add_or_replace_non_nulls { | |
7487 | # This adds the mappings in the property $other to $self. Non-null | |
7488 | # mappings from $other override those in $self. It essentially merges | |
7489 | # the two properties, with the second having priority except for null | |
7490 | # mappings. | |
7491 | ||
7492 | my $self = shift; | |
7493 | my $other = shift; | |
7494 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7495 | ||
7496 | if (! $other->isa(__PACKAGE__)) { | |
7497 | Carp::my_carp_bug("$other should be a " | |
7498 | . __PACKAGE__ | |
7499 | . ". Not a '" | |
7500 | . ref($other) | |
7501 | . "'. Not added;"); | |
7502 | return; | |
7503 | } | |
7504 | ||
f998e60c | 7505 | no overloading; |
051df77b | 7506 | return $map{pack 'J', $self}->map_add_or_replace_non_nulls($map{pack 'J', $other}); |
99870f4d KW |
7507 | } |
7508 | ||
7509 | sub set_type { | |
7510 | # Set the type of the property. Mostly this is figured out by the | |
7511 | # data in the table. But this is used to set it explicitly. The | |
7512 | # reason it is not a standard accessor is that when setting a binary | |
7513 | # property, we need to make sure that all the true/false aliases are | |
7514 | # present, as they were omitted in early Unicode releases. | |
7515 | ||
7516 | my $self = shift; | |
7517 | my $type = shift; | |
7518 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7519 | ||
06f26c45 KW |
7520 | if ($type != $ENUM |
7521 | && $type != $BINARY | |
7522 | && $type != $FORCED_BINARY | |
7523 | && $type != $STRING) | |
7524 | { | |
99870f4d KW |
7525 | Carp::my_carp("Unrecognized type '$type'. Type not set"); |
7526 | return; | |
7527 | } | |
7528 | ||
051df77b | 7529 | { no overloading; $type{pack 'J', $self} = $type; } |
06f26c45 | 7530 | return if $type != $BINARY && $type != $FORCED_BINARY; |
99870f4d KW |
7531 | |
7532 | my $yes = $self->table('Y'); | |
7533 | $yes = $self->table('Yes') if ! defined $yes; | |
01adf4be KW |
7534 | $yes = $self->add_match_table('Y', Full_Name => 'Yes') |
7535 | if ! defined $yes; | |
7536 | ||
3c6bf941 KW |
7537 | # Add aliases in order wanted, duplicates will be ignored. We use a |
7538 | # binary property present in all releases for its ordered lists of | |
7539 | # true/false aliases. Note, that could run into problems in | |
7540 | # outputting things in that we don't distinguish between the name and | |
7541 | # full name of these. Hopefully, if the table was already created | |
7542 | # before this code is executed, it was done with these set properly. | |
7543 | my $bm = property_ref("Bidi_Mirrored"); | |
7544 | foreach my $alias ($bm->table("Y")->aliases) { | |
7545 | $yes->add_alias($alias->name); | |
7546 | } | |
99870f4d KW |
7547 | my $no = $self->table('N'); |
7548 | $no = $self->table('No') if ! defined $no; | |
01adf4be | 7549 | $no = $self->add_match_table('N', Full_Name => 'No') if ! defined $no; |
3c6bf941 KW |
7550 | foreach my $alias ($bm->table("N")->aliases) { |
7551 | $no->add_alias($alias->name); | |
7552 | } | |
c12f2655 | 7553 | |
99870f4d KW |
7554 | return; |
7555 | } | |
7556 | ||
7557 | sub add_map { | |
7558 | # Add a map to the property's map table. This also keeps | |
7559 | # track of the maps so that the property type can be determined from | |
7560 | # its data. | |
7561 | ||
7562 | my $self = shift; | |
7563 | my $start = shift; # First code point in range | |
7564 | my $end = shift; # Final code point in range | |
7565 | my $map = shift; # What the range maps to. | |
7566 | # Rest of parameters passed on. | |
7567 | ||
ffe43484 | 7568 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7569 | |
7570 | # If haven't the type of the property, gather information to figure it | |
7571 | # out. | |
7572 | if ($type{$addr} == $UNKNOWN) { | |
7573 | ||
7574 | # If the map contains an interior blank or dash, or most other | |
7575 | # nonword characters, it will be a string property. This | |
7576 | # heuristic may actually miss some string properties. If so, they | |
7577 | # may need to have explicit set_types called for them. This | |
7578 | # happens in the Unihan properties. | |
7579 | if ($map =~ / (?<= . ) [ -] (?= . ) /x | |
7580 | || $map =~ / [^\w.\/\ -] /x) | |
7581 | { | |
7582 | $self->set_type($STRING); | |
7583 | ||
7584 | # $unique_maps is used for disambiguating between ENUM and | |
7585 | # BINARY later; since we know the property is not going to be | |
7586 | # one of those, no point in keeping the data around | |
7587 | undef $unique_maps{$addr}; | |
7588 | } | |
7589 | else { | |
7590 | ||
7591 | # Not necessarily a string. The final decision has to be | |
7592 | # deferred until all the data are in. We keep track of if all | |
7593 | # the values are code points for that eventual decision. | |
7594 | $has_only_code_point_maps{$addr} &= | |
7595 | $map =~ / ^ $code_point_re $/x; | |
7596 | ||
7597 | # For the purposes of disambiguating between binary and other | |
7598 | # enumerations at the end, we keep track of the first three | |
7599 | # distinct property values. Once we get to three, we know | |
7600 | # it's not going to be binary, so no need to track more. | |
7601 | if (scalar keys %{$unique_maps{$addr}} < 3) { | |
7602 | $unique_maps{$addr}{main::standardize($map)} = 1; | |
7603 | } | |
7604 | } | |
7605 | } | |
7606 | ||
7607 | # Add the mapping by calling our map table's method | |
7608 | return $map{$addr}->add_map($start, $end, $map, @_); | |
7609 | } | |
7610 | ||
7611 | sub compute_type { | |
7612 | # Compute the type of the property: $ENUM, $STRING, or $BINARY. This | |
7613 | # should be called after the property is mostly filled with its maps. | |
7614 | # We have been keeping track of what the property values have been, | |
7615 | # and now have the necessary information to figure out the type. | |
7616 | ||
7617 | my $self = shift; | |
7618 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
7619 | ||
ffe43484 | 7620 | my $addr = do { no overloading; pack 'J', $self; }; |
99870f4d KW |
7621 | |
7622 | my $type = $type{$addr}; | |
7623 | ||
7624 | # If already have figured these out, no need to do so again, but we do | |
7625 | # a double check on ENUMS to make sure that a string property hasn't | |
7626 | # improperly been classified as an ENUM, so continue on with those. | |
06f26c45 KW |
7627 | return if $type == $STRING |
7628 | || $type == $BINARY | |
7629 | || $type == $FORCED_BINARY; | |
99870f4d KW |
7630 | |
7631 | # If every map is to a code point, is a string property. | |
7632 | if ($type == $UNKNOWN | |
7633 | && ($has_only_code_point_maps{$addr} | |
7634 | || (defined $map{$addr}->default_map | |
7635 | && $map{$addr}->default_map eq ""))) | |
7636 | { | |
7637 | $self->set_type($STRING); | |
7638 | } | |
7639 | else { | |
7640 | ||
7641 | # Otherwise, it is to some sort of enumeration. (The case where | |
7642 | # it is a Unicode miscellaneous property, and treated like a | |
7643 | # string in this program is handled in add_map()). Distinguish | |
7644 | # between binary and some other enumeration type. Of course, if | |
7645 | # there are more than two values, it's not binary. But more | |
7646 | # subtle is the test that the default mapping is defined means it | |
7647 | # isn't binary. This in fact may change in the future if Unicode | |
7648 | # changes the way its data is structured. But so far, no binary | |
7649 | # properties ever have @missing lines for them, so the default map | |
7650 | # isn't defined for them. The few properties that are two-valued | |
7651 | # and aren't considered binary have the default map defined | |
7652 | # starting in Unicode 5.0, when the @missing lines appeared; and | |
7653 | # this program has special code to put in a default map for them | |
7654 | # for earlier than 5.0 releases. | |
7655 | if ($type == $ENUM | |
7656 | || scalar keys %{$unique_maps{$addr}} > 2 | |
7657 | || defined $self->default_map) | |
7658 | { | |
7659 | my $tables = $self->tables; | |
7660 | my $count = $self->count; | |
7661 | if ($verbosity && $count > 500 && $tables/$count > .1) { | |
7662 | Carp::my_carp_bug("It appears that $self should be a \$STRING property, not an \$ENUM because it has too many match tables: $count\n"); | |
7663 | } | |
7664 | $self->set_type($ENUM); | |
7665 | } | |
7666 | else { | |
7667 | $self->set_type($BINARY); | |
7668 | } | |
7669 | } | |
7670 | undef $unique_maps{$addr}; # Garbage collect | |
7671 | return; | |
7672 | } | |
7673 | ||
7674 | # Most of the accessors for a property actually apply to its map table. | |
7675 | # Setup up accessor functions for those, referring to %map | |
ea25a9b2 | 7676 | for my $sub (qw( |
99870f4d KW |
7677 | add_alias |
7678 | add_anomalous_entry | |
7679 | add_comment | |
7680 | add_conflicting | |
7681 | add_description | |
7682 | add_duplicate | |
7683 | add_note | |
7684 | aliases | |
7685 | comment | |
7686 | complete_name | |
2f7a8815 | 7687 | containing_range |
99870f4d KW |
7688 | core_access |
7689 | count | |
7690 | default_map | |
7691 | delete_range | |
7692 | description | |
7693 | each_range | |
7694 | external_name | |
7695 | file_path | |
7696 | format | |
7697 | initialize | |
7698 | inverse_list | |
7699 | is_empty | |
7700 | name | |
7701 | note | |
7702 | perl_extension | |
7703 | property | |
7704 | range_count | |
7705 | ranges | |
7706 | range_size_1 | |
7707 | reset_each_range | |
7708 | set_comment | |
7709 | set_core_access | |
7710 | set_default_map | |
7711 | set_file_path | |
7712 | set_final_comment | |
7713 | set_range_size_1 | |
7714 | set_status | |
7715 | set_to_output_map | |
7716 | short_name | |
7717 | status | |
7718 | status_info | |
7719 | to_output_map | |
0a9dbafc | 7720 | type_of |
99870f4d KW |
7721 | value_of |
7722 | write | |
ea25a9b2 | 7723 | )) |
99870f4d KW |
7724 | # 'property' above is for symmetry, so that one can take |
7725 | # the property of a property and get itself, and so don't | |
7726 | # have to distinguish between properties and tables in | |
7727 | # calling code | |
7728 | { | |
7729 | no strict "refs"; | |
7730 | *$sub = sub { | |
7731 | use strict "refs"; | |
7732 | my $self = shift; | |
f998e60c | 7733 | no overloading; |
051df77b | 7734 | return $map{pack 'J', $self}->$sub(@_); |
99870f4d KW |
7735 | } |
7736 | } | |
7737 | ||
7738 | ||
7739 | } # End closure | |
7740 | ||
7741 | package main; | |
7742 | ||
7743 | sub join_lines($) { | |
7744 | # Returns lines of the input joined together, so that they can be folded | |
7745 | # properly. | |
7746 | # This causes continuation lines to be joined together into one long line | |
7747 | # for folding. A continuation line is any line that doesn't begin with a | |
7748 | # space or "\b" (the latter is stripped from the output). This is so | |
7749 | # lines can be be in a HERE document so as to fit nicely in the terminal | |
7750 | # width, but be joined together in one long line, and then folded with | |
7751 | # indents, '#' prefixes, etc, properly handled. | |
7752 | # A blank separates the joined lines except if there is a break; an extra | |
7753 | # blank is inserted after a period ending a line. | |
7754 | ||
98dc9551 | 7755 | # Initialize the return with the first line. |
99870f4d KW |
7756 | my ($return, @lines) = split "\n", shift; |
7757 | ||
7758 | # If the first line is null, it was an empty line, add the \n back in | |
7759 | $return = "\n" if $return eq ""; | |
7760 | ||
7761 | # Now join the remainder of the physical lines. | |
7762 | for my $line (@lines) { | |
7763 | ||
7764 | # An empty line means wanted a blank line, so add two \n's to get that | |
7765 | # effect, and go to the next line. | |
7766 | if (length $line == 0) { | |
7767 | $return .= "\n\n"; | |
7768 | next; | |
7769 | } | |
7770 | ||
7771 | # Look at the last character of what we have so far. | |
7772 | my $previous_char = substr($return, -1, 1); | |
7773 | ||
7774 | # And at the next char to be output. | |
7775 | my $next_char = substr($line, 0, 1); | |
7776 | ||
7777 | if ($previous_char ne "\n") { | |
7778 | ||
7779 | # Here didn't end wth a nl. If the next char a blank or \b, it | |
7780 | # means that here there is a break anyway. So add a nl to the | |
7781 | # output. | |
7782 | if ($next_char eq " " || $next_char eq "\b") { | |
7783 | $previous_char = "\n"; | |
7784 | $return .= $previous_char; | |
7785 | } | |
7786 | ||
7787 | # Add an extra space after periods. | |
7788 | $return .= " " if $previous_char eq '.'; | |
7789 | } | |
7790 | ||
7791 | # Here $previous_char is still the latest character to be output. If | |
7792 | # it isn't a nl, it means that the next line is to be a continuation | |
7793 | # line, with a blank inserted between them. | |
7794 | $return .= " " if $previous_char ne "\n"; | |
7795 | ||
7796 | # Get rid of any \b | |
7797 | substr($line, 0, 1) = "" if $next_char eq "\b"; | |
7798 | ||
7799 | # And append this next line. | |
7800 | $return .= $line; | |
7801 | } | |
7802 | ||
7803 | return $return; | |
7804 | } | |
7805 | ||
7806 | sub simple_fold($;$$$) { | |
7807 | # Returns a string of the input (string or an array of strings) folded | |
7808 | # into multiple-lines each of no more than $MAX_LINE_WIDTH characters plus | |
7809 | # a \n | |
7810 | # This is tailored for the kind of text written by this program, | |
7811 | # especially the pod file, which can have very long names with | |
7812 | # underscores in the middle, or words like AbcDefgHij.... We allow | |
7813 | # breaking in the middle of such constructs if the line won't fit | |
7814 | # otherwise. The break in such cases will come either just after an | |
7815 | # underscore, or just before one of the Capital letters. | |
7816 | ||
7817 | local $to_trace = 0 if main::DEBUG; | |
7818 | ||
7819 | my $line = shift; | |
7820 | my $prefix = shift; # Optional string to prepend to each output | |
7821 | # line | |
7822 | $prefix = "" unless defined $prefix; | |
7823 | ||
7824 | my $hanging_indent = shift; # Optional number of spaces to indent | |
7825 | # continuation lines | |
7826 | $hanging_indent = 0 unless $hanging_indent; | |
7827 | ||
7828 | my $right_margin = shift; # Optional number of spaces to narrow the | |
7829 | # total width by. | |
7830 | $right_margin = 0 unless defined $right_margin; | |
7831 | ||
7832 | # Call carp with the 'nofold' option to avoid it from trying to call us | |
7833 | # recursively | |
7834 | Carp::carp_extra_args(\@_, 'nofold') if main::DEBUG && @_; | |
7835 | ||
7836 | # The space available doesn't include what's automatically prepended | |
7837 | # to each line, or what's reserved on the right. | |
7838 | my $max = $MAX_LINE_WIDTH - length($prefix) - $right_margin; | |
7839 | # XXX Instead of using the 'nofold' perhaps better to look up the stack | |
7840 | ||
7841 | if (DEBUG && $hanging_indent >= $max) { | |
7842 | Carp::my_carp("Too large a hanging indent ($hanging_indent); must be < $max. Using 0", 'nofold'); | |
7843 | $hanging_indent = 0; | |
7844 | } | |
7845 | ||
7846 | # First, split into the current physical lines. | |
7847 | my @line; | |
7848 | if (ref $line) { # Better be an array, because not bothering to | |
7849 | # test | |
7850 | foreach my $line (@{$line}) { | |
7851 | push @line, split /\n/, $line; | |
7852 | } | |
7853 | } | |
7854 | else { | |
7855 | @line = split /\n/, $line; | |
7856 | } | |
7857 | ||
7858 | #local $to_trace = 1 if main::DEBUG; | |
7859 | trace "", join(" ", @line), "\n" if main::DEBUG && $to_trace; | |
7860 | ||
7861 | # Look at each current physical line. | |
7862 | for (my $i = 0; $i < @line; $i++) { | |
7863 | Carp::my_carp("Tabs don't work well.", 'nofold') if $line[$i] =~ /\t/; | |
7864 | #local $to_trace = 1 if main::DEBUG; | |
7865 | trace "i=$i: $line[$i]\n" if main::DEBUG && $to_trace; | |
7866 | ||
7867 | # Remove prefix, because will be added back anyway, don't want | |
7868 | # doubled prefix | |
7869 | $line[$i] =~ s/^$prefix//; | |
7870 | ||
7871 | # Remove trailing space | |
7872 | $line[$i] =~ s/\s+\Z//; | |
7873 | ||
7874 | # If the line is too long, fold it. | |
7875 | if (length $line[$i] > $max) { | |
7876 | my $remainder; | |
7877 | ||
7878 | # Here needs to fold. Save the leading space in the line for | |
7879 | # later. | |
7880 | $line[$i] =~ /^ ( \s* )/x; | |
7881 | my $leading_space = $1; | |
7882 | trace "line length", length $line[$i], "; lead length", length($leading_space) if main::DEBUG && $to_trace; | |
7883 | ||
7884 | # If character at final permissible position is white space, | |
7885 | # fold there, which will delete that white space | |
7886 | if (substr($line[$i], $max - 1, 1) =~ /\s/) { | |
7887 | $remainder = substr($line[$i], $max); | |
7888 | $line[$i] = substr($line[$i], 0, $max - 1); | |
7889 | } | |
7890 | else { | |
7891 | ||
7892 | # Otherwise fold at an acceptable break char closest to | |
7893 | # the max length. Look at just the maximal initial | |
7894 | # segment of the line | |
7895 | my $segment = substr($line[$i], 0, $max - 1); | |
7896 | if ($segment =~ | |
7897 | /^ ( .{$hanging_indent} # Don't look before the | |
7898 | # indent. | |
7899 | \ * # Don't look in leading | |
7900 | # blanks past the indent | |
7901 | [^ ] .* # Find the right-most | |
7902 | (?: # acceptable break: | |
7903 | [ \s = ] # space or equal | |
7904 | | - (?! [.0-9] ) # or non-unary minus. | |
7905 | ) # $1 includes the character | |
7906 | )/x) | |
7907 | { | |
7908 | # Split into the initial part that fits, and remaining | |
7909 | # part of the input | |
7910 | $remainder = substr($line[$i], length $1); | |
7911 | $line[$i] = $1; | |
7912 | trace $line[$i] if DEBUG && $to_trace; | |
7913 | trace $remainder if DEBUG && $to_trace; | |
7914 | } | |
7915 | ||
7916 | # If didn't find a good breaking spot, see if there is a | |
7917 | # not-so-good breaking spot. These are just after | |
7918 | # underscores or where the case changes from lower to | |
7919 | # upper. Use \a as a soft hyphen, but give up | |
7920 | # and don't break the line if there is actually a \a | |
7921 | # already in the input. We use an ascii character for the | |
7922 | # soft-hyphen to avoid any attempt by miniperl to try to | |
7923 | # access the files that this program is creating. | |
7924 | elsif ($segment !~ /\a/ | |
7925 | && ($segment =~ s/_/_\a/g | |
7926 | || $segment =~ s/ ( [a-z] ) (?= [A-Z] )/$1\a/xg)) | |
7927 | { | |
7928 | # Here were able to find at least one place to insert | |
7929 | # our substitute soft hyphen. Find the right-most one | |
7930 | # and replace it by a real hyphen. | |
7931 | trace $segment if DEBUG && $to_trace; | |
7932 | substr($segment, | |
7933 | rindex($segment, "\a"), | |
7934 | 1) = '-'; | |
7935 | ||
7936 | # Then remove the soft hyphen substitutes. | |
7937 | $segment =~ s/\a//g; | |
7938 | trace $segment if DEBUG && $to_trace; | |
7939 | ||
7940 | # And split into the initial part that fits, and | |
7941 | # remainder of the line | |
7942 | my $pos = rindex($segment, '-'); | |
7943 | $remainder = substr($line[$i], $pos); | |
7944 | trace $remainder if DEBUG && $to_trace; | |
7945 | $line[$i] = substr($segment, 0, $pos + 1); | |
7946 | } | |
7947 | } | |
7948 | ||
7949 | # Here we know if we can fold or not. If we can, $remainder | |
7950 | # is what remains to be processed in the next iteration. | |
7951 | if (defined $remainder) { | |
7952 | trace "folded='$line[$i]'" if main::DEBUG && $to_trace; | |
7953 | ||
7954 | # Insert the folded remainder of the line as a new element | |
7955 | # of the array. (It may still be too long, but we will | |
7956 | # deal with that next time through the loop.) Omit any | |
7957 | # leading space in the remainder. | |
7958 | $remainder =~ s/^\s+//; | |
7959 | trace "remainder='$remainder'" if main::DEBUG && $to_trace; | |
7960 | ||
7961 | # But then indent by whichever is larger of: | |
7962 | # 1) the leading space on the input line; | |
7963 | # 2) the hanging indent. | |
7964 | # This preserves indentation in the original line. | |
7965 | my $lead = ($leading_space) | |
7966 | ? length $leading_space | |
7967 | : $hanging_indent; | |
7968 | $lead = max($lead, $hanging_indent); | |
7969 | splice @line, $i+1, 0, (" " x $lead) . $remainder; | |
7970 | } | |
7971 | } | |
7972 | ||
7973 | # Ready to output the line. Get rid of any trailing space | |
7974 | # And prefix by the required $prefix passed in. | |
7975 | $line[$i] =~ s/\s+$//; | |
7976 | $line[$i] = "$prefix$line[$i]\n"; | |
7977 | } # End of looping through all the lines. | |
7978 | ||
7979 | return join "", @line; | |
7980 | } | |
7981 | ||
7982 | sub property_ref { # Returns a reference to a property object. | |
7983 | return Property::property_ref(@_); | |
7984 | } | |
7985 | ||
7986 | sub force_unlink ($) { | |
7987 | my $filename = shift; | |
7988 | return unless file_exists($filename); | |
7989 | return if CORE::unlink($filename); | |
7990 | ||
7991 | # We might need write permission | |
7992 | chmod 0777, $filename; | |
7993 | CORE::unlink($filename) or Carp::my_carp("Couldn't unlink $filename. Proceeding anyway: $!"); | |
7994 | return; | |
7995 | } | |
7996 | ||
9218f1cf | 7997 | sub write ($$@) { |
9abe8df8 KW |
7998 | # Given a filename and references to arrays of lines, write the lines of |
7999 | # each array to the file | |
99870f4d KW |
8000 | # Filename can be given as an arrayref of directory names |
8001 | ||
9218f1cf | 8002 | return Carp::carp_too_few_args(\@_, 3) if main::DEBUG && @_ < 3; |
99870f4d | 8003 | |
9abe8df8 | 8004 | my $file = shift; |
9218f1cf | 8005 | my $use_utf8 = shift; |
99870f4d KW |
8006 | |
8007 | # Get into a single string if an array, and get rid of, in Unix terms, any | |
8008 | # leading '.' | |
8009 | $file= File::Spec->join(@$file) if ref $file eq 'ARRAY'; | |
8010 | $file = File::Spec->canonpath($file); | |
8011 | ||
8012 | # If has directories, make sure that they all exist | |
8013 | (undef, my $directories, undef) = File::Spec->splitpath($file); | |
8014 | File::Path::mkpath($directories) if $directories && ! -d $directories; | |
8015 | ||
8016 | push @files_actually_output, $file; | |
8017 | ||
99870f4d KW |
8018 | force_unlink ($file); |
8019 | ||
8020 | my $OUT; | |
8021 | if (not open $OUT, ">", $file) { | |
8022 | Carp::my_carp("can't open $file for output. Skipping this file: $!"); | |
8023 | return; | |
8024 | } | |
430ada4c | 8025 | |
9218f1cf KW |
8026 | binmode $OUT, ":utf8" if $use_utf8; |
8027 | ||
9abe8df8 KW |
8028 | while (defined (my $lines_ref = shift)) { |
8029 | unless (@$lines_ref) { | |
8030 | Carp::my_carp("An array of lines for writing to file '$file' is empty; writing it anyway;"); | |
8031 | } | |
8032 | ||
8033 | print $OUT @$lines_ref or die Carp::my_carp("write to '$file' failed: $!"); | |
8034 | } | |
430ada4c NC |
8035 | close $OUT or die Carp::my_carp("close '$file' failed: $!"); |
8036 | ||
99870f4d KW |
8037 | print "$file written.\n" if $verbosity >= $VERBOSE; |
8038 | ||
99870f4d KW |
8039 | return; |
8040 | } | |
8041 | ||
8042 | ||
8043 | sub Standardize($) { | |
8044 | # This converts the input name string into a standardized equivalent to | |
8045 | # use internally. | |
8046 | ||
8047 | my $name = shift; | |
8048 | unless (defined $name) { | |
8049 | Carp::my_carp_bug("Standardize() called with undef. Returning undef."); | |
8050 | return; | |
8051 | } | |
8052 | ||
8053 | # Remove any leading or trailing white space | |
8054 | $name =~ s/^\s+//g; | |
8055 | $name =~ s/\s+$//g; | |
8056 | ||
98dc9551 | 8057 | # Convert interior white space and hyphens into underscores. |
99870f4d KW |
8058 | $name =~ s/ (?<= .) [ -]+ (.) /_$1/xg; |
8059 | ||
8060 | # Capitalize the letter following an underscore, and convert a sequence of | |
8061 | # multiple underscores to a single one | |
8062 | $name =~ s/ (?<= .) _+ (.) /_\u$1/xg; | |
8063 | ||
8064 | # And capitalize the first letter, but not for the special cjk ones. | |
8065 | $name = ucfirst($name) unless $name =~ /^k[A-Z]/; | |
8066 | return $name; | |
8067 | } | |
8068 | ||
8069 | sub standardize ($) { | |
8070 | # Returns a lower-cased standardized name, without underscores. This form | |
8071 | # is chosen so that it can distinguish between any real versus superficial | |
8072 | # Unicode name differences. It relies on the fact that Unicode doesn't | |
8073 | # have interior underscores, white space, nor dashes in any | |
8074 | # stricter-matched name. It should not be used on Unicode code point | |
8075 | # names (the Name property), as they mostly, but not always follow these | |
8076 | # rules. | |
8077 | ||
8078 | my $name = Standardize(shift); | |
8079 | return if !defined $name; | |
8080 | ||
8081 | $name =~ s/ (?<= .) _ (?= . ) //xg; | |
8082 | return lc $name; | |
8083 | } | |
8084 | ||
c85f591a KW |
8085 | sub utf8_heavy_name ($$) { |
8086 | # Returns the name that utf8_heavy.pl will use to find a table. XXX | |
8087 | # perhaps this function should be placed somewhere, like Heavy.pl so that | |
8088 | # utf8_heavy can use it directly without duplicating code that can get | |
8089 | # out-of sync. | |
8090 | ||
8091 | my $table = shift; | |
8092 | my $alias = shift; | |
8093 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8094 | ||
8095 | my $property = $table->property; | |
8096 | $property = ($property == $perl) | |
8097 | ? "" # 'perl' is never explicitly stated | |
8098 | : standardize($property->name) . '='; | |
8099 | if ($alias->loose_match) { | |
8100 | return $property . standardize($alias->name); | |
8101 | } | |
8102 | else { | |
8103 | return lc ($property . $alias->name); | |
8104 | } | |
8105 | ||
8106 | return; | |
8107 | } | |
8108 | ||
99870f4d KW |
8109 | { # Closure |
8110 | ||
8111 | my $indent_increment = " " x 2; | |
8112 | my %already_output; | |
8113 | ||
8114 | $main::simple_dumper_nesting = 0; | |
8115 | ||
8116 | sub simple_dumper { | |
8117 | # Like Simple Data::Dumper. Good enough for our needs. We can't use | |
8118 | # the real thing as we have to run under miniperl. | |
8119 | ||
8120 | # It is designed so that on input it is at the beginning of a line, | |
8121 | # and the final thing output in any call is a trailing ",\n". | |
8122 | ||
8123 | my $item = shift; | |
8124 | my $indent = shift; | |
8125 | $indent = "" if ! defined $indent; | |
8126 | ||
8127 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8128 | ||
8129 | # nesting level is localized, so that as the call stack pops, it goes | |
8130 | # back to the prior value. | |
8131 | local $main::simple_dumper_nesting = $main::simple_dumper_nesting; | |
8132 | undef %already_output if $main::simple_dumper_nesting == 0; | |
8133 | $main::simple_dumper_nesting++; | |
8134 | #print STDERR __LINE__, ": $main::simple_dumper_nesting: $indent$item\n"; | |
8135 | ||
8136 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8137 | ||
8138 | # Determine the indent for recursive calls. | |
8139 | my $next_indent = $indent . $indent_increment; | |
8140 | ||
8141 | my $output; | |
8142 | if (! ref $item) { | |
8143 | ||
8144 | # Dump of scalar: just output it in quotes if not a number. To do | |
8145 | # so we must escape certain characters, and therefore need to | |
8146 | # operate on a copy to avoid changing the original | |
8147 | my $copy = $item; | |
8148 | $copy = $UNDEF unless defined $copy; | |
8149 | ||
02cc6656 KW |
8150 | # Quote non-integers (integers also have optional leading '-') |
8151 | if ($copy eq "" || $copy !~ /^ -? \d+ $/x) { | |
99870f4d KW |
8152 | |
8153 | # Escape apostrophe and backslash | |
8154 | $copy =~ s/ ( ['\\] ) /\\$1/xg; | |
8155 | $copy = "'$copy'"; | |
8156 | } | |
8157 | $output = "$indent$copy,\n"; | |
8158 | } | |
8159 | else { | |
8160 | ||
8161 | # Keep track of cycles in the input, and refuse to infinitely loop | |
ffe43484 | 8162 | my $addr = do { no overloading; pack 'J', $item; }; |
f998e60c | 8163 | if (defined $already_output{$addr}) { |
99870f4d KW |
8164 | return "${indent}ALREADY OUTPUT: $item\n"; |
8165 | } | |
f998e60c | 8166 | $already_output{$addr} = $item; |
99870f4d KW |
8167 | |
8168 | if (ref $item eq 'ARRAY') { | |
8169 | my $using_brackets; | |
8170 | $output = $indent; | |
8171 | if ($main::simple_dumper_nesting > 1) { | |
8172 | $output .= '['; | |
8173 | $using_brackets = 1; | |
8174 | } | |
8175 | else { | |
8176 | $using_brackets = 0; | |
8177 | } | |
8178 | ||
8179 | # If the array is empty, put the closing bracket on the same | |
8180 | # line. Otherwise, recursively add each array element | |
8181 | if (@$item == 0) { | |
8182 | $output .= " "; | |
8183 | } | |
8184 | else { | |
8185 | $output .= "\n"; | |
8186 | for (my $i = 0; $i < @$item; $i++) { | |
8187 | ||
8188 | # Indent array elements one level | |
8189 | $output .= &simple_dumper($item->[$i], $next_indent); | |
c12f2655 KW |
8190 | $output =~ s/\n$//; # Remove any trailing nl so |
8191 | $output .= " # [$i]\n"; # as to add a comment giving | |
8192 | # the array index | |
99870f4d KW |
8193 | } |
8194 | $output .= $indent; # Indent closing ']' to orig level | |
8195 | } | |
8196 | $output .= ']' if $using_brackets; | |
8197 | $output .= ",\n"; | |
8198 | } | |
8199 | elsif (ref $item eq 'HASH') { | |
8200 | my $is_first_line; | |
8201 | my $using_braces; | |
8202 | my $body_indent; | |
8203 | ||
8204 | # No surrounding braces at top level | |
8205 | $output .= $indent; | |
8206 | if ($main::simple_dumper_nesting > 1) { | |
8207 | $output .= "{\n"; | |
8208 | $is_first_line = 0; | |
8209 | $body_indent = $next_indent; | |
8210 | $next_indent .= $indent_increment; | |
8211 | $using_braces = 1; | |
8212 | } | |
8213 | else { | |
8214 | $is_first_line = 1; | |
8215 | $body_indent = $indent; | |
8216 | $using_braces = 0; | |
8217 | } | |
8218 | ||
8219 | # Output hashes sorted alphabetically instead of apparently | |
8220 | # random. Use caseless alphabetic sort | |
8221 | foreach my $key (sort { lc $a cmp lc $b } keys %$item) | |
8222 | { | |
8223 | if ($is_first_line) { | |
8224 | $is_first_line = 0; | |
8225 | } | |
8226 | else { | |
8227 | $output .= "$body_indent"; | |
8228 | } | |
8229 | ||
8230 | # The key must be a scalar, but this recursive call quotes | |
8231 | # it | |
8232 | $output .= &simple_dumper($key); | |
8233 | ||
8234 | # And change the trailing comma and nl to the hash fat | |
8235 | # comma for clarity, and so the value can be on the same | |
8236 | # line | |
8237 | $output =~ s/,\n$/ => /; | |
8238 | ||
8239 | # Recursively call to get the value's dump. | |
8240 | my $next = &simple_dumper($item->{$key}, $next_indent); | |
8241 | ||
8242 | # If the value is all on one line, remove its indent, so | |
8243 | # will follow the => immediately. If it takes more than | |
8244 | # one line, start it on a new line. | |
8245 | if ($next !~ /\n.*\n/) { | |
8246 | $next =~ s/^ *//; | |
8247 | } | |
8248 | else { | |
8249 | $output .= "\n"; | |
8250 | } | |
8251 | $output .= $next; | |
8252 | } | |
8253 | ||
8254 | $output .= "$indent},\n" if $using_braces; | |
8255 | } | |
8256 | elsif (ref $item eq 'CODE' || ref $item eq 'GLOB') { | |
8257 | $output = $indent . ref($item) . "\n"; | |
8258 | # XXX see if blessed | |
8259 | } | |
8260 | elsif ($item->can('dump')) { | |
8261 | ||
8262 | # By convention in this program, objects furnish a 'dump' | |
8263 | # method. Since not doing any output at this level, just pass | |
8264 | # on the input indent | |
8265 | $output = $item->dump($indent); | |
8266 | } | |
8267 | else { | |
8268 | Carp::my_carp("Can't cope with dumping a " . ref($item) . ". Skipping."); | |
8269 | } | |
8270 | } | |
8271 | return $output; | |
8272 | } | |
8273 | } | |
8274 | ||
8275 | sub dump_inside_out { | |
8276 | # Dump inside-out hashes in an object's state by converting them to a | |
8277 | # regular hash and then calling simple_dumper on that. | |
8278 | ||
8279 | my $object = shift; | |
8280 | my $fields_ref = shift; | |
8281 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8282 | ||
ffe43484 | 8283 | my $addr = do { no overloading; pack 'J', $object; }; |
99870f4d KW |
8284 | |
8285 | my %hash; | |
8286 | foreach my $key (keys %$fields_ref) { | |
8287 | $hash{$key} = $fields_ref->{$key}{$addr}; | |
8288 | } | |
8289 | ||
8290 | return simple_dumper(\%hash, @_); | |
8291 | } | |
8292 | ||
8293 | sub _operator_dot { | |
8294 | # Overloaded '.' method that is common to all packages. It uses the | |
8295 | # package's stringify method. | |
8296 | ||
8297 | my $self = shift; | |
8298 | my $other = shift; | |
8299 | my $reversed = shift; | |
8300 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8301 | ||
8302 | $other = "" unless defined $other; | |
8303 | ||
8304 | foreach my $which (\$self, \$other) { | |
8305 | next unless ref $$which; | |
8306 | if ($$which->can('_operator_stringify')) { | |
8307 | $$which = $$which->_operator_stringify; | |
8308 | } | |
8309 | else { | |
8310 | my $ref = ref $$which; | |
ffe43484 | 8311 | my $addr = do { no overloading; pack 'J', $$which; }; |
99870f4d KW |
8312 | $$which = "$ref ($addr)"; |
8313 | } | |
8314 | } | |
8315 | return ($reversed) | |
8316 | ? "$other$self" | |
8317 | : "$self$other"; | |
8318 | } | |
8319 | ||
8320 | sub _operator_equal { | |
8321 | # Generic overloaded '==' routine. To be equal, they must be the exact | |
8322 | # same object | |
8323 | ||
8324 | my $self = shift; | |
8325 | my $other = shift; | |
8326 | ||
8327 | return 0 unless defined $other; | |
8328 | return 0 unless ref $other; | |
f998e60c | 8329 | no overloading; |
2100aa98 | 8330 | return $self == $other; |
99870f4d KW |
8331 | } |
8332 | ||
8333 | sub _operator_not_equal { | |
8334 | my $self = shift; | |
8335 | my $other = shift; | |
8336 | ||
8337 | return ! _operator_equal($self, $other); | |
8338 | } | |
8339 | ||
8340 | sub process_PropertyAliases($) { | |
8341 | # This reads in the PropertyAliases.txt file, which contains almost all | |
8342 | # the character properties in Unicode and their equivalent aliases: | |
8343 | # scf ; Simple_Case_Folding ; sfc | |
8344 | # | |
8345 | # Field 0 is the preferred short name for the property. | |
8346 | # Field 1 is the full name. | |
8347 | # Any succeeding ones are other accepted names. | |
8348 | ||
8349 | my $file= shift; | |
8350 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8351 | ||
8352 | # This whole file was non-existent in early releases, so use our own | |
8353 | # internal one. | |
8354 | $file->insert_lines(get_old_property_aliases()) | |
8355 | if ! -e 'PropertyAliases.txt'; | |
8356 | ||
8357 | # Add any cjk properties that may have been defined. | |
8358 | $file->insert_lines(@cjk_properties); | |
8359 | ||
8360 | while ($file->next_line) { | |
8361 | ||
8362 | my @data = split /\s*;\s*/; | |
8363 | ||
8364 | my $full = $data[1]; | |
8365 | ||
8366 | my $this = Property->new($data[0], Full_Name => $full); | |
8367 | ||
8368 | # Start looking for more aliases after these two. | |
8369 | for my $i (2 .. @data - 1) { | |
8370 | $this->add_alias($data[$i]); | |
8371 | } | |
8372 | ||
8373 | } | |
8374 | return; | |
8375 | } | |
8376 | ||
8377 | sub finish_property_setup { | |
8378 | # Finishes setting up after PropertyAliases. | |
8379 | ||
8380 | my $file = shift; | |
8381 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8382 | ||
8383 | # This entry was missing from this file in earlier Unicode versions | |
8384 | if (-e 'Jamo.txt') { | |
8385 | my $jsn = property_ref('JSN'); | |
8386 | if (! defined $jsn) { | |
8387 | $jsn = Property->new('JSN', Full_Name => 'Jamo_Short_Name'); | |
8388 | } | |
8389 | } | |
8390 | ||
5f7264c7 | 8391 | # This entry is still missing as of 6.0, perhaps because no short name for |
99870f4d KW |
8392 | # it. |
8393 | if (-e 'NameAliases.txt') { | |
8394 | my $aliases = property_ref('Name_Alias'); | |
8395 | if (! defined $aliases) { | |
8396 | $aliases = Property->new('Name_Alias'); | |
8397 | } | |
8398 | } | |
8399 | ||
8400 | # These are used so much, that we set globals for them. | |
8401 | $gc = property_ref('General_Category'); | |
8402 | $block = property_ref('Block'); | |
359523e2 | 8403 | $script = property_ref('Script'); |
99870f4d KW |
8404 | |
8405 | # Perl adds this alias. | |
8406 | $gc->add_alias('Category'); | |
8407 | ||
8408 | # For backwards compatibility, these property files have particular names. | |
8409 | my $upper = property_ref('Uppercase_Mapping'); | |
8410 | $upper->set_core_access('uc()'); | |
8411 | $upper->set_file('Upper'); # This is what utf8.c calls it | |
8412 | ||
8413 | my $lower = property_ref('Lowercase_Mapping'); | |
8414 | $lower->set_core_access('lc()'); | |
8415 | $lower->set_file('Lower'); | |
8416 | ||
8417 | my $title = property_ref('Titlecase_Mapping'); | |
8418 | $title->set_core_access('ucfirst()'); | |
8419 | $title->set_file('Title'); | |
8420 | ||
8421 | my $fold = property_ref('Case_Folding'); | |
8422 | $fold->set_file('Fold') if defined $fold; | |
8423 | ||
d3cbe105 KW |
8424 | # Unicode::Normalize expects this file with this name and directory. |
8425 | my $ccc = property_ref('Canonical_Combining_Class'); | |
8426 | if (defined $ccc) { | |
8427 | $ccc->set_file('CombiningClass'); | |
8428 | $ccc->set_directory(File::Spec->curdir()); | |
8429 | } | |
8430 | ||
2cd56239 KW |
8431 | # utf8.c has a different meaning for non range-size-1 for map properties |
8432 | # that this program doesn't currently handle; and even if it were changed | |
8433 | # to do so, some other code may be using them expecting range size 1. | |
99870f4d KW |
8434 | foreach my $property (qw { |
8435 | Case_Folding | |
8436 | Lowercase_Mapping | |
8437 | Titlecase_Mapping | |
8438 | Uppercase_Mapping | |
8439 | }) | |
8440 | { | |
8441 | property_ref($property)->set_range_size_1(1); | |
8442 | } | |
8443 | ||
8444 | # These two properties aren't actually used in the core, but unfortunately | |
8445 | # the names just above that are in the core interfere with these, so | |
8446 | # choose different names. These aren't a problem unless the map tables | |
8447 | # for these files get written out. | |
8448 | my $lowercase = property_ref('Lowercase'); | |
8449 | $lowercase->set_file('IsLower') if defined $lowercase; | |
8450 | my $uppercase = property_ref('Uppercase'); | |
8451 | $uppercase->set_file('IsUpper') if defined $uppercase; | |
8452 | ||
8453 | # Set up the hard-coded default mappings, but only on properties defined | |
8454 | # for this release | |
8455 | foreach my $property (keys %default_mapping) { | |
8456 | my $property_object = property_ref($property); | |
8457 | next if ! defined $property_object; | |
8458 | my $default_map = $default_mapping{$property}; | |
8459 | $property_object->set_default_map($default_map); | |
8460 | ||
8461 | # A map of <code point> implies the property is string. | |
8462 | if ($property_object->type == $UNKNOWN | |
8463 | && $default_map eq $CODE_POINT) | |
8464 | { | |
8465 | $property_object->set_type($STRING); | |
8466 | } | |
8467 | } | |
8468 | ||
8469 | # The following use the Multi_Default class to create objects for | |
8470 | # defaults. | |
8471 | ||
8472 | # Bidi class has a complicated default, but the derived file takes care of | |
8473 | # the complications, leaving just 'L'. | |
8474 | if (file_exists("${EXTRACTED}DBidiClass.txt")) { | |
8475 | property_ref('Bidi_Class')->set_default_map('L'); | |
8476 | } | |
8477 | else { | |
8478 | my $default; | |
8479 | ||
8480 | # The derived file was introduced in 3.1.1. The values below are | |
8481 | # taken from table 3-8, TUS 3.0 | |
8482 | my $default_R = | |
8483 | 'my $default = Range_List->new; | |
8484 | $default->add_range(0x0590, 0x05FF); | |
8485 | $default->add_range(0xFB1D, 0xFB4F);' | |
8486 | ; | |
8487 | ||
8488 | # The defaults apply only to unassigned characters | |
a67f160a | 8489 | $default_R .= '$gc->table("Unassigned") & $default;'; |
99870f4d KW |
8490 | |
8491 | if ($v_version lt v3.0.0) { | |
8492 | $default = Multi_Default->new(R => $default_R, 'L'); | |
8493 | } | |
8494 | else { | |
8495 | ||
8496 | # AL apparently not introduced until 3.0: TUS 2.x references are | |
8497 | # not on-line to check it out | |
8498 | my $default_AL = | |
8499 | 'my $default = Range_List->new; | |
8500 | $default->add_range(0x0600, 0x07BF); | |
8501 | $default->add_range(0xFB50, 0xFDFF); | |
8502 | $default->add_range(0xFE70, 0xFEFF);' | |
8503 | ; | |
8504 | ||
8505 | # Non-character code points introduced in this release; aren't AL | |
8506 | if ($v_version ge 3.1.0) { | |
8507 | $default_AL .= '$default->delete_range(0xFDD0, 0xFDEF);'; | |
8508 | } | |
a67f160a | 8509 | $default_AL .= '$gc->table("Unassigned") & $default'; |
99870f4d KW |
8510 | $default = Multi_Default->new(AL => $default_AL, |
8511 | R => $default_R, | |
8512 | 'L'); | |
8513 | } | |
8514 | property_ref('Bidi_Class')->set_default_map($default); | |
8515 | } | |
8516 | ||
8517 | # Joining type has a complicated default, but the derived file takes care | |
8518 | # of the complications, leaving just 'U' (or Non_Joining), except the file | |
8519 | # is bad in 3.1.0 | |
8520 | if (file_exists("${EXTRACTED}DJoinType.txt") || -e 'ArabicShaping.txt') { | |
8521 | if (file_exists("${EXTRACTED}DJoinType.txt") && $v_version ne 3.1.0) { | |
8522 | property_ref('Joining_Type')->set_default_map('Non_Joining'); | |
8523 | } | |
8524 | else { | |
8525 | ||
8526 | # Otherwise, there are not one, but two possibilities for the | |
8527 | # missing defaults: T and U. | |
8528 | # The missing defaults that evaluate to T are given by: | |
8529 | # T = Mn + Cf - ZWNJ - ZWJ | |
8530 | # where Mn and Cf are the general category values. In other words, | |
8531 | # any non-spacing mark or any format control character, except | |
8532 | # U+200C ZERO WIDTH NON-JOINER (joining type U) and U+200D ZERO | |
8533 | # WIDTH JOINER (joining type C). | |
8534 | my $default = Multi_Default->new( | |
8535 | 'T' => '$gc->table("Mn") + $gc->table("Cf") - 0x200C - 0x200D', | |
8536 | 'Non_Joining'); | |
8537 | property_ref('Joining_Type')->set_default_map($default); | |
8538 | } | |
8539 | } | |
8540 | ||
8541 | # Line break has a complicated default in early releases. It is 'Unknown' | |
8542 | # for non-assigned code points; 'AL' for assigned. | |
8543 | if (file_exists("${EXTRACTED}DLineBreak.txt") || -e 'LineBreak.txt') { | |
8544 | my $lb = property_ref('Line_Break'); | |
8545 | if ($v_version gt 3.2.0) { | |
8546 | $lb->set_default_map('Unknown'); | |
8547 | } | |
8548 | else { | |
8549 | my $default = Multi_Default->new( 'Unknown' => '$gc->table("Cn")', | |
8550 | 'AL'); | |
8551 | $lb->set_default_map($default); | |
8552 | } | |
8553 | ||
8554 | # If has the URS property, make sure that the standard aliases are in | |
8555 | # it, since not in the input tables in some versions. | |
8556 | my $urs = property_ref('Unicode_Radical_Stroke'); | |
8557 | if (defined $urs) { | |
8558 | $urs->add_alias('cjkRSUnicode'); | |
8559 | $urs->add_alias('kRSUnicode'); | |
8560 | } | |
8561 | } | |
8562 | return; | |
8563 | } | |
8564 | ||
8565 | sub get_old_property_aliases() { | |
8566 | # Returns what would be in PropertyAliases.txt if it existed in very old | |
8567 | # versions of Unicode. It was derived from the one in 3.2, and pared | |
8568 | # down based on the data that was actually in the older releases. | |
8569 | # An attempt was made to use the existence of files to mean inclusion or | |
8570 | # not of various aliases, but if this was not sufficient, using version | |
8571 | # numbers was resorted to. | |
8572 | ||
8573 | my @return; | |
8574 | ||
8575 | # These are to be used in all versions (though some are constructed by | |
8576 | # this program if missing) | |
8577 | push @return, split /\n/, <<'END'; | |
8578 | bc ; Bidi_Class | |
8579 | Bidi_M ; Bidi_Mirrored | |
8580 | cf ; Case_Folding | |
8581 | ccc ; Canonical_Combining_Class | |
8582 | dm ; Decomposition_Mapping | |
8583 | dt ; Decomposition_Type | |
8584 | gc ; General_Category | |
8585 | isc ; ISO_Comment | |
8586 | lc ; Lowercase_Mapping | |
8587 | na ; Name | |
8588 | na1 ; Unicode_1_Name | |
8589 | nt ; Numeric_Type | |
8590 | nv ; Numeric_Value | |
8591 | sfc ; Simple_Case_Folding | |
8592 | slc ; Simple_Lowercase_Mapping | |
8593 | stc ; Simple_Titlecase_Mapping | |
8594 | suc ; Simple_Uppercase_Mapping | |
8595 | tc ; Titlecase_Mapping | |
8596 | uc ; Uppercase_Mapping | |
8597 | END | |
8598 | ||
8599 | if (-e 'Blocks.txt') { | |
8600 | push @return, "blk ; Block\n"; | |
8601 | } | |
8602 | if (-e 'ArabicShaping.txt') { | |
8603 | push @return, split /\n/, <<'END'; | |
8604 | jg ; Joining_Group | |
8605 | jt ; Joining_Type | |
8606 | END | |
8607 | } | |
8608 | if (-e 'PropList.txt') { | |
8609 | ||
8610 | # This first set is in the original old-style proplist. | |
8611 | push @return, split /\n/, <<'END'; | |
8612 | Alpha ; Alphabetic | |
8613 | Bidi_C ; Bidi_Control | |
8614 | Dash ; Dash | |
8615 | Dia ; Diacritic | |
8616 | Ext ; Extender | |
8617 | Hex ; Hex_Digit | |
8618 | Hyphen ; Hyphen | |
8619 | IDC ; ID_Continue | |
8620 | Ideo ; Ideographic | |
8621 | Join_C ; Join_Control | |
8622 | Math ; Math | |
8623 | QMark ; Quotation_Mark | |
8624 | Term ; Terminal_Punctuation | |
8625 | WSpace ; White_Space | |
8626 | END | |
8627 | # The next sets were added later | |
8628 | if ($v_version ge v3.0.0) { | |
8629 | push @return, split /\n/, <<'END'; | |
8630 | Upper ; Uppercase | |
8631 | Lower ; Lowercase | |
8632 | END | |
8633 | } | |
8634 | if ($v_version ge v3.0.1) { | |
8635 | push @return, split /\n/, <<'END'; | |
8636 | NChar ; Noncharacter_Code_Point | |
8637 | END | |
8638 | } | |
8639 | # The next sets were added in the new-style | |
8640 | if ($v_version ge v3.1.0) { | |
8641 | push @return, split /\n/, <<'END'; | |
8642 | OAlpha ; Other_Alphabetic | |
8643 | OLower ; Other_Lowercase | |
8644 | OMath ; Other_Math | |
8645 | OUpper ; Other_Uppercase | |
8646 | END | |
8647 | } | |
8648 | if ($v_version ge v3.1.1) { | |
8649 | push @return, "AHex ; ASCII_Hex_Digit\n"; | |
8650 | } | |
8651 | } | |
8652 | if (-e 'EastAsianWidth.txt') { | |
8653 | push @return, "ea ; East_Asian_Width\n"; | |
8654 | } | |
8655 | if (-e 'CompositionExclusions.txt') { | |
8656 | push @return, "CE ; Composition_Exclusion\n"; | |
8657 | } | |
8658 | if (-e 'LineBreak.txt') { | |
8659 | push @return, "lb ; Line_Break\n"; | |
8660 | } | |
8661 | if (-e 'BidiMirroring.txt') { | |
8662 | push @return, "bmg ; Bidi_Mirroring_Glyph\n"; | |
8663 | } | |
8664 | if (-e 'Scripts.txt') { | |
8665 | push @return, "sc ; Script\n"; | |
8666 | } | |
8667 | if (-e 'DNormalizationProps.txt') { | |
8668 | push @return, split /\n/, <<'END'; | |
8669 | Comp_Ex ; Full_Composition_Exclusion | |
8670 | FC_NFKC ; FC_NFKC_Closure | |
8671 | NFC_QC ; NFC_Quick_Check | |
8672 | NFD_QC ; NFD_Quick_Check | |
8673 | NFKC_QC ; NFKC_Quick_Check | |
8674 | NFKD_QC ; NFKD_Quick_Check | |
8675 | XO_NFC ; Expands_On_NFC | |
8676 | XO_NFD ; Expands_On_NFD | |
8677 | XO_NFKC ; Expands_On_NFKC | |
8678 | XO_NFKD ; Expands_On_NFKD | |
8679 | END | |
8680 | } | |
8681 | if (-e 'DCoreProperties.txt') { | |
8682 | push @return, split /\n/, <<'END'; | |
8683 | IDS ; ID_Start | |
8684 | XIDC ; XID_Continue | |
8685 | XIDS ; XID_Start | |
8686 | END | |
8687 | # These can also appear in some versions of PropList.txt | |
8688 | push @return, "Lower ; Lowercase\n" | |
8689 | unless grep { $_ =~ /^Lower\b/} @return; | |
8690 | push @return, "Upper ; Uppercase\n" | |
8691 | unless grep { $_ =~ /^Upper\b/} @return; | |
8692 | } | |
8693 | ||
8694 | # This flag requires the DAge.txt file to be copied into the directory. | |
8695 | if (DEBUG && $compare_versions) { | |
8696 | push @return, 'age ; Age'; | |
8697 | } | |
8698 | ||
8699 | return @return; | |
8700 | } | |
8701 | ||
8702 | sub process_PropValueAliases { | |
8703 | # This file contains values that properties look like: | |
8704 | # bc ; AL ; Arabic_Letter | |
8705 | # blk; n/a ; Greek_And_Coptic ; Greek | |
8706 | # | |
8707 | # Field 0 is the property. | |
8708 | # Field 1 is the short name of a property value or 'n/a' if no | |
8709 | # short name exists; | |
8710 | # Field 2 is the full property value name; | |
8711 | # Any other fields are more synonyms for the property value. | |
8712 | # Purely numeric property values are omitted from the file; as are some | |
8713 | # others, fewer and fewer in later releases | |
8714 | ||
8715 | # Entries for the ccc property have an extra field before the | |
8716 | # abbreviation: | |
8717 | # ccc; 0; NR ; Not_Reordered | |
8718 | # It is the numeric value that the names are synonyms for. | |
8719 | ||
8720 | # There are comment entries for values missing from this file: | |
8721 | # # @missing: 0000..10FFFF; ISO_Comment; <none> | |
8722 | # # @missing: 0000..10FFFF; Lowercase_Mapping; <code point> | |
8723 | ||
8724 | my $file= shift; | |
8725 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
8726 | ||
8727 | # This whole file was non-existent in early releases, so use our own | |
8728 | # internal one if necessary. | |
8729 | if (! -e 'PropValueAliases.txt') { | |
8730 | $file->insert_lines(get_old_property_value_aliases()); | |
8731 | } | |
8732 | ||
8733 | # Add any explicit cjk values | |
8734 | $file->insert_lines(@cjk_property_values); | |
8735 | ||
8736 | # This line is used only for testing the code that checks for name | |
8737 | # conflicts. There is a script Inherited, and when this line is executed | |
8738 | # it causes there to be a name conflict with the 'Inherited' that this | |
8739 | # program generates for this block property value | |
8740 | #$file->insert_lines('blk; n/a; Herited'); | |
8741 | ||
8742 | ||
8743 | # Process each line of the file ... | |
8744 | while ($file->next_line) { | |
8745 | ||
8746 | my ($property, @data) = split /\s*;\s*/; | |
8747 | ||
66b4eb0a KW |
8748 | # The ccc property has an extra field at the beginning, which is the |
8749 | # numeric value. Move it to be after the other two, mnemonic, fields, | |
8750 | # so that those will be used as the property value's names, and the | |
8751 | # number will be an extra alias. (Rightmost splice removes field 1-2, | |
8752 | # returning them in a slice; left splice inserts that before anything, | |
8753 | # thus shifting the former field 0 to after them.) | |
8754 | splice (@data, 0, 0, splice(@data, 1, 2)) if $property eq 'ccc'; | |
8755 | ||
8756 | # Field 0 is a short name unless "n/a"; field 1 is the full name. If | |
8757 | # there is no short name, use the full one in element 1 | |
027866c1 KW |
8758 | if ($data[0] eq "n/a") { |
8759 | $data[0] = $data[1]; | |
8760 | } | |
8761 | elsif ($data[0] ne $data[1] | |
8762 | && standardize($data[0]) eq standardize($data[1]) | |
8763 | && $data[1] !~ /[[:upper:]]/) | |
8764 | { | |
8765 | # Also, there is a bug in the file in which "n/a" is omitted, and | |
8766 | # the two fields are identical except for case, and the full name | |
8767 | # is all lower case. Copy the "short" name unto the full one to | |
8768 | # give it some upper case. | |
8769 | ||
8770 | $data[1] = $data[0]; | |
8771 | } | |
99870f4d KW |
8772 | |
8773 | # Earlier releases had the pseudo property 'qc' that should expand to | |
8774 | # the ones that replace it below. | |
8775 | if ($property eq 'qc') { | |
8776 | if (lc $data[0] eq 'y') { | |
8777 | $file->insert_lines('NFC_QC; Y ; Yes', | |
8778 | 'NFD_QC; Y ; Yes', | |
8779 | 'NFKC_QC; Y ; Yes', | |
8780 | 'NFKD_QC; Y ; Yes', | |
8781 | ); | |
8782 | } | |
8783 | elsif (lc $data[0] eq 'n') { | |
8784 | $file->insert_lines('NFC_QC; N ; No', | |
8785 | 'NFD_QC; N ; No', | |
8786 | 'NFKC_QC; N ; No', | |
8787 | 'NFKD_QC; N ; No', | |
8788 | ); | |
8789 | } | |
8790 | elsif (lc $data[0] eq 'm') { | |
8791 | $file->insert_lines('NFC_QC; M ; Maybe', | |
8792 | 'NFKC_QC; M ; Maybe', | |
8793 | ); | |
8794 | } | |
8795 | else { | |
8796 | $file->carp_bad_line("qc followed by unexpected '$data[0]"); | |
8797 | } | |
8798 | next; | |
8799 | } | |
8800 | ||
8801 | # The first field is the short name, 2nd is the full one. | |
8802 | my $property_object = property_ref($property); | |
8803 | my $table = $property_object->add_match_table($data[0], | |
8804 | Full_Name => $data[1]); | |
8805 | ||
8806 | # Start looking for more aliases after these two. | |
8807 | for my $i (2 .. @data - 1) { | |
8808 | $table->add_alias($data[$i]); | |
8809 | } | |
8810 | } # End of looping through the file | |
8811 | ||
8812 | # As noted in the comments early in the program, it generates tables for | |
8813 | # the default values for all releases, even those for which the concept | |
8814 | # didn't exist at the time. Here we add those if missing. | |
8815 | my $age = property_ref('age'); | |
8816 | if (defined $age && ! defined $age->table('Unassigned')) { | |
8817 | $age->add_match_table('Unassigned'); | |
8818 | } | |
8819 | $block->add_match_table('No_Block') if -e 'Blocks.txt' | |
8820 | && ! defined $block->table('No_Block'); | |
8821 | ||
8822 | ||
8823 | # Now set the default mappings of the properties from the file. This is | |
8824 | # done after the loop because a number of properties have only @missings | |
8825 | # entries in the file, and may not show up until the end. | |
8826 | my @defaults = $file->get_missings; | |
8827 | foreach my $default_ref (@defaults) { | |
8828 | my $default = $default_ref->[0]; | |
8829 | my $property = property_ref($default_ref->[1]); | |
8830 | $property->set_default_map($default); | |
8831 | } | |
8832 | return; | |
8833 | } | |
8834 | ||
8835 | sub get_old_property_value_aliases () { | |
8836 | # Returns what would be in PropValueAliases.txt if it existed in very old | |
8837 | # versions of Unicode. It was derived from the one in 3.2, and pared | |
8838 | # down. An attempt was made to use the existence of files to mean | |
8839 | # inclusion or not of various aliases, but if this was not sufficient, | |
8840 | # using version numbers was resorted to. | |
8841 | ||
8842 | my @return = split /\n/, <<'END'; | |
8843 | bc ; AN ; Arabic_Number | |
8844 | bc ; B ; Paragraph_Separator | |
8845 | bc ; CS ; Common_Separator | |
8846 | bc ; EN ; European_Number | |
8847 | bc ; ES ; European_Separator | |
8848 | bc ; ET ; European_Terminator | |
8849 | bc ; L ; Left_To_Right | |
8850 | bc ; ON ; Other_Neutral | |
8851 | bc ; R ; Right_To_Left | |
8852 | bc ; WS ; White_Space | |
8853 | ||
8854 | # The standard combining classes are very much different in v1, so only use | |
8855 | # ones that look right (not checked thoroughly) | |
8856 | ccc; 0; NR ; Not_Reordered | |
8857 | ccc; 1; OV ; Overlay | |
8858 | ccc; 7; NK ; Nukta | |
8859 | ccc; 8; KV ; Kana_Voicing | |
8860 | ccc; 9; VR ; Virama | |
8861 | ccc; 202; ATBL ; Attached_Below_Left | |
8862 | ccc; 216; ATAR ; Attached_Above_Right | |
8863 | ccc; 218; BL ; Below_Left | |
8864 | ccc; 220; B ; Below | |
8865 | ccc; 222; BR ; Below_Right | |
8866 | ccc; 224; L ; Left | |
8867 | ccc; 228; AL ; Above_Left | |
8868 | ccc; 230; A ; Above | |
8869 | ccc; 232; AR ; Above_Right | |
8870 | ccc; 234; DA ; Double_Above | |
8871 | ||
8872 | dt ; can ; canonical | |
8873 | dt ; enc ; circle | |
8874 | dt ; fin ; final | |
8875 | dt ; font ; font | |
8876 | dt ; fra ; fraction | |
8877 | dt ; init ; initial | |
8878 | dt ; iso ; isolated | |
8879 | dt ; med ; medial | |
8880 | dt ; n/a ; none | |
8881 | dt ; nb ; noBreak | |
8882 | dt ; sqr ; square | |
8883 | dt ; sub ; sub | |
8884 | dt ; sup ; super | |
8885 | ||
8886 | gc ; C ; Other # Cc | Cf | Cn | Co | Cs | |
8887 | gc ; Cc ; Control | |
8888 | gc ; Cn ; Unassigned | |
8889 | gc ; Co ; Private_Use | |
8890 | gc ; L ; Letter # Ll | Lm | Lo | Lt | Lu | |
8891 | gc ; LC ; Cased_Letter # Ll | Lt | Lu | |
8892 | gc ; Ll ; Lowercase_Letter | |
8893 | gc ; Lm ; Modifier_Letter | |
8894 | gc ; Lo ; Other_Letter | |
8895 | gc ; Lu ; Uppercase_Letter | |
8896 | gc ; M ; Mark # Mc | Me | Mn | |
8897 | gc ; Mc ; Spacing_Mark | |
8898 | gc ; Mn ; Nonspacing_Mark | |
8899 | gc ; N ; Number # Nd | Nl | No | |
8900 | gc ; Nd ; Decimal_Number | |
8901 | gc ; No ; Other_Number | |
8902 | gc ; P ; Punctuation # Pc | Pd | Pe | Pf | Pi | Po | Ps | |
8903 | gc ; Pd ; Dash_Punctuation | |
8904 | gc ; Pe ; Close_Punctuation | |
8905 | gc ; Po ; Other_Punctuation | |
8906 | gc ; Ps ; Open_Punctuation | |
8907 | gc ; S ; Symbol # Sc | Sk | Sm | So | |
8908 | gc ; Sc ; Currency_Symbol | |
8909 | gc ; Sm ; Math_Symbol | |
8910 | gc ; So ; Other_Symbol | |
8911 | gc ; Z ; Separator # Zl | Zp | Zs | |
8912 | gc ; Zl ; Line_Separator | |
8913 | gc ; Zp ; Paragraph_Separator | |
8914 | gc ; Zs ; Space_Separator | |
8915 | ||
8916 | nt ; de ; Decimal | |
8917 | nt ; di ; Digit | |
8918 | nt ; n/a ; None | |
8919 | nt ; nu ; Numeric | |
8920 | END | |
8921 | ||
8922 | if (-e 'ArabicShaping.txt') { | |
8923 | push @return, split /\n/, <<'END'; | |
8924 | jg ; n/a ; AIN | |
8925 | jg ; n/a ; ALEF | |
8926 | jg ; n/a ; DAL | |
8927 | jg ; n/a ; GAF | |
8928 | jg ; n/a ; LAM | |
8929 | jg ; n/a ; MEEM | |
8930 | jg ; n/a ; NO_JOINING_GROUP | |
8931 | jg ; n/a ; NOON | |
8932 | jg ; n/a ; QAF | |
8933 | jg ; n/a ; SAD | |
8934 | jg ; n/a ; SEEN | |
8935 | jg ; n/a ; TAH | |
8936 | jg ; n/a ; WAW | |
8937 | ||
8938 | jt ; C ; Join_Causing | |
8939 | jt ; D ; Dual_Joining | |
8940 | jt ; L ; Left_Joining | |
8941 | jt ; R ; Right_Joining | |
8942 | jt ; U ; Non_Joining | |
8943 | jt ; T ; Transparent | |
8944 | END | |
8945 | if ($v_version ge v3.0.0) { | |
8946 | push @return, split /\n/, <<'END'; | |
8947 | jg ; n/a ; ALAPH | |
8948 | jg ; n/a ; BEH | |
8949 | jg ; n/a ; BETH | |
8950 | jg ; n/a ; DALATH_RISH | |
8951 | jg ; n/a ; E | |
8952 | jg ; n/a ; FEH | |
8953 | jg ; n/a ; FINAL_SEMKATH | |
8954 | jg ; n/a ; GAMAL | |
8955 | jg ; n/a ; HAH | |
8956 | jg ; n/a ; HAMZA_ON_HEH_GOAL | |
8957 | jg ; n/a ; HE | |
8958 | jg ; n/a ; HEH | |
8959 | jg ; n/a ; HEH_GOAL | |
8960 | jg ; n/a ; HETH | |
8961 | jg ; n/a ; KAF | |
8962 | jg ; n/a ; KAPH | |
8963 | jg ; n/a ; KNOTTED_HEH | |
8964 | jg ; n/a ; LAMADH | |
8965 | jg ; n/a ; MIM | |
8966 | jg ; n/a ; NUN | |
8967 | jg ; n/a ; PE | |
8968 | jg ; n/a ; QAPH | |
8969 | jg ; n/a ; REH | |
8970 | jg ; n/a ; REVERSED_PE | |
8971 | jg ; n/a ; SADHE | |
8972 | jg ; n/a ; SEMKATH | |
8973 | jg ; n/a ; SHIN | |
8974 | jg ; n/a ; SWASH_KAF | |
8975 | jg ; n/a ; TAW | |
8976 | jg ; n/a ; TEH_MARBUTA | |
8977 | jg ; n/a ; TETH | |
8978 | jg ; n/a ; YEH | |
8979 | jg ; n/a ; YEH_BARREE | |
8980 | jg ; n/a ; YEH_WITH_TAIL | |
8981 | jg ; n/a ; YUDH | |
8982 | jg ; n/a ; YUDH_HE | |
8983 | jg ; n/a ; ZAIN | |
8984 | END | |
8985 | } | |
8986 | } | |
8987 | ||
8988 | ||
8989 | if (-e 'EastAsianWidth.txt') { | |
8990 | push @return, split /\n/, <<'END'; | |
8991 | ea ; A ; Ambiguous | |
8992 | ea ; F ; Fullwidth | |
8993 | ea ; H ; Halfwidth | |
8994 | ea ; N ; Neutral | |
8995 | ea ; Na ; Narrow | |
8996 | ea ; W ; Wide | |
8997 | END | |
8998 | } | |
8999 | ||
9000 | if (-e 'LineBreak.txt') { | |
9001 | push @return, split /\n/, <<'END'; | |
9002 | lb ; AI ; Ambiguous | |
9003 | lb ; AL ; Alphabetic | |
9004 | lb ; B2 ; Break_Both | |
9005 | lb ; BA ; Break_After | |
9006 | lb ; BB ; Break_Before | |
9007 | lb ; BK ; Mandatory_Break | |
9008 | lb ; CB ; Contingent_Break | |
9009 | lb ; CL ; Close_Punctuation | |
9010 | lb ; CM ; Combining_Mark | |
9011 | lb ; CR ; Carriage_Return | |
9012 | lb ; EX ; Exclamation | |
9013 | lb ; GL ; Glue | |
9014 | lb ; HY ; Hyphen | |
9015 | lb ; ID ; Ideographic | |
9016 | lb ; IN ; Inseperable | |
9017 | lb ; IS ; Infix_Numeric | |
9018 | lb ; LF ; Line_Feed | |
9019 | lb ; NS ; Nonstarter | |
9020 | lb ; NU ; Numeric | |
9021 | lb ; OP ; Open_Punctuation | |
9022 | lb ; PO ; Postfix_Numeric | |
9023 | lb ; PR ; Prefix_Numeric | |
9024 | lb ; QU ; Quotation | |
9025 | lb ; SA ; Complex_Context | |
9026 | lb ; SG ; Surrogate | |
9027 | lb ; SP ; Space | |
9028 | lb ; SY ; Break_Symbols | |
9029 | lb ; XX ; Unknown | |
9030 | lb ; ZW ; ZWSpace | |
9031 | END | |
9032 | } | |
9033 | ||
9034 | if (-e 'DNormalizationProps.txt') { | |
9035 | push @return, split /\n/, <<'END'; | |
9036 | qc ; M ; Maybe | |
9037 | qc ; N ; No | |
9038 | qc ; Y ; Yes | |
9039 | END | |
9040 | } | |
9041 | ||
9042 | if (-e 'Scripts.txt') { | |
9043 | push @return, split /\n/, <<'END'; | |
9044 | sc ; Arab ; Arabic | |
9045 | sc ; Armn ; Armenian | |
9046 | sc ; Beng ; Bengali | |
9047 | sc ; Bopo ; Bopomofo | |
9048 | sc ; Cans ; Canadian_Aboriginal | |
9049 | sc ; Cher ; Cherokee | |
9050 | sc ; Cyrl ; Cyrillic | |
9051 | sc ; Deva ; Devanagari | |
9052 | sc ; Dsrt ; Deseret | |
9053 | sc ; Ethi ; Ethiopic | |
9054 | sc ; Geor ; Georgian | |
9055 | sc ; Goth ; Gothic | |
9056 | sc ; Grek ; Greek | |
9057 | sc ; Gujr ; Gujarati | |
9058 | sc ; Guru ; Gurmukhi | |
9059 | sc ; Hang ; Hangul | |
9060 | sc ; Hani ; Han | |
9061 | sc ; Hebr ; Hebrew | |
9062 | sc ; Hira ; Hiragana | |
9063 | sc ; Ital ; Old_Italic | |
9064 | sc ; Kana ; Katakana | |
9065 | sc ; Khmr ; Khmer | |
9066 | sc ; Knda ; Kannada | |
9067 | sc ; Laoo ; Lao | |
9068 | sc ; Latn ; Latin | |
9069 | sc ; Mlym ; Malayalam | |
9070 | sc ; Mong ; Mongolian | |
9071 | sc ; Mymr ; Myanmar | |
9072 | sc ; Ogam ; Ogham | |
9073 | sc ; Orya ; Oriya | |
9074 | sc ; Qaai ; Inherited | |
9075 | sc ; Runr ; Runic | |
9076 | sc ; Sinh ; Sinhala | |
9077 | sc ; Syrc ; Syriac | |
9078 | sc ; Taml ; Tamil | |
9079 | sc ; Telu ; Telugu | |
9080 | sc ; Thaa ; Thaana | |
9081 | sc ; Thai ; Thai | |
9082 | sc ; Tibt ; Tibetan | |
9083 | sc ; Yiii ; Yi | |
9084 | sc ; Zyyy ; Common | |
9085 | END | |
9086 | } | |
9087 | ||
9088 | if ($v_version ge v2.0.0) { | |
9089 | push @return, split /\n/, <<'END'; | |
9090 | dt ; com ; compat | |
9091 | dt ; nar ; narrow | |
9092 | dt ; sml ; small | |
9093 | dt ; vert ; vertical | |
9094 | dt ; wide ; wide | |
9095 | ||
9096 | gc ; Cf ; Format | |
9097 | gc ; Cs ; Surrogate | |
9098 | gc ; Lt ; Titlecase_Letter | |
9099 | gc ; Me ; Enclosing_Mark | |
9100 | gc ; Nl ; Letter_Number | |
9101 | gc ; Pc ; Connector_Punctuation | |
9102 | gc ; Sk ; Modifier_Symbol | |
9103 | END | |
9104 | } | |
9105 | if ($v_version ge v2.1.2) { | |
9106 | push @return, "bc ; S ; Segment_Separator\n"; | |
9107 | } | |
9108 | if ($v_version ge v2.1.5) { | |
9109 | push @return, split /\n/, <<'END'; | |
9110 | gc ; Pf ; Final_Punctuation | |
9111 | gc ; Pi ; Initial_Punctuation | |
9112 | END | |
9113 | } | |
9114 | if ($v_version ge v2.1.8) { | |
9115 | push @return, "ccc; 240; IS ; Iota_Subscript\n"; | |
9116 | } | |
9117 | ||
9118 | if ($v_version ge v3.0.0) { | |
9119 | push @return, split /\n/, <<'END'; | |
9120 | bc ; AL ; Arabic_Letter | |
9121 | bc ; BN ; Boundary_Neutral | |
9122 | bc ; LRE ; Left_To_Right_Embedding | |
9123 | bc ; LRO ; Left_To_Right_Override | |
9124 | bc ; NSM ; Nonspacing_Mark | |
9125 | bc ; PDF ; Pop_Directional_Format | |
9126 | bc ; RLE ; Right_To_Left_Embedding | |
9127 | bc ; RLO ; Right_To_Left_Override | |
9128 | ||
9129 | ccc; 233; DB ; Double_Below | |
9130 | END | |
9131 | } | |
9132 | ||
9133 | if ($v_version ge v3.1.0) { | |
9134 | push @return, "ccc; 226; R ; Right\n"; | |
9135 | } | |
9136 | ||
9137 | return @return; | |
9138 | } | |
9139 | ||
b1c167a3 KW |
9140 | sub output_perl_charnames_line ($$) { |
9141 | ||
9142 | # Output the entries in Perl_charnames specially, using 5 digits instead | |
9143 | # of four. This makes the entries a constant length, and simplifies | |
9144 | # charnames.pm which this table is for. Unicode can have 6 digit | |
9145 | # ordinals, but they are all private use or noncharacters which do not | |
9146 | # have names, so won't be in this table. | |
9147 | ||
73d9566f | 9148 | return sprintf "%05X\t%s\n", $_[0], $_[1]; |
b1c167a3 KW |
9149 | } |
9150 | ||
99870f4d KW |
9151 | { # Closure |
9152 | # This is used to store the range list of all the code points usable when | |
9153 | # the little used $compare_versions feature is enabled. | |
9154 | my $compare_versions_range_list; | |
9155 | ||
96cfc54a KW |
9156 | # These are constants to the $property_info hash in this subroutine, to |
9157 | # avoid using a quoted-string which might have a typo. | |
9158 | my $TYPE = 'type'; | |
9159 | my $DEFAULT_MAP = 'default_map'; | |
9160 | my $DEFAULT_TABLE = 'default_table'; | |
9161 | my $PSEUDO_MAP_TYPE = 'pseudo_map_type'; | |
9162 | my $MISSINGS = 'missings'; | |
9163 | ||
99870f4d KW |
9164 | sub process_generic_property_file { |
9165 | # This processes a file containing property mappings and puts them | |
9166 | # into internal map tables. It should be used to handle any property | |
9167 | # files that have mappings from a code point or range thereof to | |
9168 | # something else. This means almost all the UCD .txt files. | |
9169 | # each_line_handlers() should be set to adjust the lines of these | |
9170 | # files, if necessary, to what this routine understands: | |
9171 | # | |
9172 | # 0374 ; NFD_QC; N | |
9173 | # 003C..003E ; Math | |
9174 | # | |
92f9d56c | 9175 | # the fields are: "codepoint-range ; property; map" |
99870f4d KW |
9176 | # |
9177 | # meaning the codepoints in the range all have the value 'map' under | |
9178 | # 'property'. | |
98dc9551 | 9179 | # Beginning and trailing white space in each field are not significant. |
99870f4d KW |
9180 | # Note there is not a trailing semi-colon in the above. A trailing |
9181 | # semi-colon means the map is a null-string. An omitted map, as | |
9182 | # opposed to a null-string, is assumed to be 'Y', based on Unicode | |
9183 | # table syntax. (This could have been hidden from this routine by | |
9184 | # doing it in the $file object, but that would require parsing of the | |
9185 | # line there, so would have to parse it twice, or change the interface | |
9186 | # to pass this an array. So not done.) | |
9187 | # | |
9188 | # The map field may begin with a sequence of commands that apply to | |
9189 | # this range. Each such command begins and ends with $CMD_DELIM. | |
9190 | # These are used to indicate, for example, that the mapping for a | |
9191 | # range has a non-default type. | |
9192 | # | |
9193 | # This loops through the file, calling it's next_line() method, and | |
9194 | # then taking the map and adding it to the property's table. | |
9195 | # Complications arise because any number of properties can be in the | |
9196 | # file, in any order, interspersed in any way. The first time a | |
9197 | # property is seen, it gets information about that property and | |
f86864ac | 9198 | # caches it for quick retrieval later. It also normalizes the maps |
5d7f7709 KW |
9199 | # so that only one of many synonyms is stored. The Unicode input |
9200 | # files do use some multiple synonyms. | |
99870f4d KW |
9201 | |
9202 | my $file = shift; | |
9203 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
9204 | ||
9205 | my %property_info; # To keep track of what properties | |
9206 | # have already had entries in the | |
9207 | # current file, and info about each, | |
9208 | # so don't have to recompute. | |
9209 | my $property_name; # property currently being worked on | |
9210 | my $property_type; # and its type | |
9211 | my $previous_property_name = ""; # name from last time through loop | |
9212 | my $property_object; # pointer to the current property's | |
9213 | # object | |
9214 | my $property_addr; # the address of that object | |
9215 | my $default_map; # the string that code points missing | |
9216 | # from the file map to | |
9217 | my $default_table; # For non-string properties, a | |
9218 | # reference to the match table that | |
9219 | # will contain the list of code | |
9220 | # points that map to $default_map. | |
9221 | ||
9222 | # Get the next real non-comment line | |
9223 | LINE: | |
9224 | while ($file->next_line) { | |
9225 | ||
9226 | # Default replacement type; means that if parts of the range have | |
9227 | # already been stored in our tables, the new map overrides them if | |
9228 | # they differ more than cosmetically | |
9229 | my $replace = $IF_NOT_EQUIVALENT; | |
9230 | my $map_type; # Default type for the map of this range | |
9231 | ||
9232 | #local $to_trace = 1 if main::DEBUG; | |
9233 | trace $_ if main::DEBUG && $to_trace; | |
9234 | ||
9235 | # Split the line into components | |
9236 | my ($range, $property_name, $map, @remainder) | |
9237 | = split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields | |
9238 | ||
9239 | # If more or less on the line than we are expecting, warn and skip | |
9240 | # the line | |
9241 | if (@remainder) { | |
9242 | $file->carp_bad_line('Extra fields'); | |
9243 | next LINE; | |
9244 | } | |
9245 | elsif ( ! defined $property_name) { | |
9246 | $file->carp_bad_line('Missing property'); | |
9247 | next LINE; | |
9248 | } | |
9249 | ||
9250 | # Examine the range. | |
9251 | if ($range !~ /^ ($code_point_re) (?:\.\. ($code_point_re) )? $/x) | |
9252 | { | |
9253 | $file->carp_bad_line("Range '$range' not of the form 'CP1' or 'CP1..CP2' (where CP1,2 are code points in hex)"); | |
9254 | next LINE; | |
9255 | } | |
9256 | my $low = hex $1; | |
9257 | my $high = (defined $2) ? hex $2 : $low; | |
9258 | ||
9259 | # For the very specialized case of comparing two Unicode | |
9260 | # versions... | |
9261 | if (DEBUG && $compare_versions) { | |
9262 | if ($property_name eq 'Age') { | |
9263 | ||
9264 | # Only allow code points at least as old as the version | |
9265 | # specified. | |
9266 | my $age = pack "C*", split(/\./, $map); # v string | |
9267 | next LINE if $age gt $compare_versions; | |
9268 | } | |
9269 | else { | |
9270 | ||
9271 | # Again, we throw out code points younger than those of | |
9272 | # the specified version. By now, the Age property is | |
9273 | # populated. We use the intersection of each input range | |
9274 | # with this property to find what code points in it are | |
9275 | # valid. To do the intersection, we have to convert the | |
9276 | # Age property map to a Range_list. We only have to do | |
9277 | # this once. | |
9278 | if (! defined $compare_versions_range_list) { | |
9279 | my $age = property_ref('Age'); | |
9280 | if (! -e 'DAge.txt') { | |
9281 | croak "Need to have 'DAge.txt' file to do version comparison"; | |
9282 | } | |
9283 | elsif ($age->count == 0) { | |
9284 | croak "The 'Age' table is empty, but its file exists"; | |
9285 | } | |
9286 | $compare_versions_range_list | |
9287 | = Range_List->new(Initialize => $age); | |
9288 | } | |
9289 | ||
9290 | # An undefined map is always 'Y' | |
9291 | $map = 'Y' if ! defined $map; | |
9292 | ||
9293 | # Calculate the intersection of the input range with the | |
9294 | # code points that are known in the specified version | |
9295 | my @ranges = ($compare_versions_range_list | |
9296 | & Range->new($low, $high))->ranges; | |
9297 | ||
9298 | # If the intersection is empty, throw away this range | |
9299 | next LINE unless @ranges; | |
9300 | ||
9301 | # Only examine the first range this time through the loop. | |
9302 | my $this_range = shift @ranges; | |
9303 | ||
9304 | # Put any remaining ranges in the queue to be processed | |
9305 | # later. Note that there is unnecessary work here, as we | |
9306 | # will do the intersection again for each of these ranges | |
9307 | # during some future iteration of the LINE loop, but this | |
9308 | # code is not used in production. The later intersections | |
9309 | # are guaranteed to not splinter, so this will not become | |
9310 | # an infinite loop. | |
9311 | my $line = join ';', $property_name, $map; | |
9312 | foreach my $range (@ranges) { | |
9313 | $file->insert_adjusted_lines(sprintf("%04X..%04X; %s", | |
9314 | $range->start, | |
9315 | $range->end, | |
9316 | $line)); | |
9317 | } | |
9318 | ||
9319 | # And process the first range, like any other. | |
9320 | $low = $this_range->start; | |
9321 | $high = $this_range->end; | |
9322 | } | |
9323 | } # End of $compare_versions | |
9324 | ||
9325 | # If changing to a new property, get the things constant per | |
9326 | # property | |
9327 | if ($previous_property_name ne $property_name) { | |
9328 | ||
9329 | $property_object = property_ref($property_name); | |
9330 | if (! defined $property_object) { | |
9331 | $file->carp_bad_line("Unexpected property '$property_name'. Skipped"); | |
9332 | next LINE; | |
9333 | } | |
051df77b | 9334 | { no overloading; $property_addr = pack 'J', $property_object; } |
99870f4d KW |
9335 | |
9336 | # Defer changing names until have a line that is acceptable | |
9337 | # (the 'next' statement above means is unacceptable) | |
9338 | $previous_property_name = $property_name; | |
9339 | ||
9340 | # If not the first time for this property, retrieve info about | |
9341 | # it from the cache | |
96cfc54a KW |
9342 | if (defined ($property_info{$property_addr}{$TYPE})) { |
9343 | $property_type = $property_info{$property_addr}{$TYPE}; | |
9344 | $default_map = $property_info{$property_addr}{$DEFAULT_MAP}; | |
99870f4d | 9345 | $map_type |
96cfc54a | 9346 | = $property_info{$property_addr}{$PSEUDO_MAP_TYPE}; |
99870f4d | 9347 | $default_table |
96cfc54a | 9348 | = $property_info{$property_addr}{$DEFAULT_TABLE}; |
99870f4d KW |
9349 | } |
9350 | else { | |
9351 | ||
9352 | # Here, is the first time for this property. Set up the | |
9353 | # cache. | |
96cfc54a | 9354 | $property_type = $property_info{$property_addr}{$TYPE} |
99870f4d KW |
9355 | = $property_object->type; |
9356 | $map_type | |
96cfc54a | 9357 | = $property_info{$property_addr}{$PSEUDO_MAP_TYPE} |
99870f4d KW |
9358 | = $property_object->pseudo_map_type; |
9359 | ||
9360 | # The Unicode files are set up so that if the map is not | |
9361 | # defined, it is a binary property | |
9362 | if (! defined $map && $property_type != $BINARY) { | |
9363 | if ($property_type != $UNKNOWN | |
9364 | && $property_type != $NON_STRING) | |
9365 | { | |
9366 | $file->carp_bad_line("No mapping defined on a non-binary property. Using 'Y' for the map"); | |
9367 | } | |
9368 | else { | |
9369 | $property_object->set_type($BINARY); | |
9370 | $property_type | |
96cfc54a | 9371 | = $property_info{$property_addr}{$TYPE} |
99870f4d KW |
9372 | = $BINARY; |
9373 | } | |
9374 | } | |
9375 | ||
9376 | # Get any @missings default for this property. This | |
9377 | # should precede the first entry for the property in the | |
9378 | # input file, and is located in a comment that has been | |
9379 | # stored by the Input_file class until we access it here. | |
9380 | # It's possible that there is more than one such line | |
9381 | # waiting for us; collect them all, and parse | |
9382 | my @missings_list = $file->get_missings | |
9383 | if $file->has_missings_defaults; | |
9384 | foreach my $default_ref (@missings_list) { | |
9385 | my $default = $default_ref->[0]; | |
ffe43484 | 9386 | my $addr = do { no overloading; pack 'J', property_ref($default_ref->[1]); }; |
99870f4d KW |
9387 | |
9388 | # For string properties, the default is just what the | |
9389 | # file says, but non-string properties should already | |
9390 | # have set up a table for the default property value; | |
9391 | # use the table for these, so can resolve synonyms | |
9392 | # later to a single standard one. | |
9393 | if ($property_type == $STRING | |
9394 | || $property_type == $UNKNOWN) | |
9395 | { | |
96cfc54a | 9396 | $property_info{$addr}{$MISSINGS} = $default; |
99870f4d KW |
9397 | } |
9398 | else { | |
96cfc54a | 9399 | $property_info{$addr}{$MISSINGS} |
99870f4d KW |
9400 | = $property_object->table($default); |
9401 | } | |
9402 | } | |
9403 | ||
9404 | # Finished storing all the @missings defaults in the input | |
9405 | # file so far. Get the one for the current property. | |
96cfc54a | 9406 | my $missings = $property_info{$property_addr}{$MISSINGS}; |
99870f4d KW |
9407 | |
9408 | # But we likely have separately stored what the default | |
9409 | # should be. (This is to accommodate versions of the | |
9410 | # standard where the @missings lines are absent or | |
9411 | # incomplete.) Hopefully the two will match. But check | |
9412 | # it out. | |
9413 | $default_map = $property_object->default_map; | |
9414 | ||
9415 | # If the map is a ref, it means that the default won't be | |
9416 | # processed until later, so undef it, so next few lines | |
9417 | # will redefine it to something that nothing will match | |
9418 | undef $default_map if ref $default_map; | |
9419 | ||
9420 | # Create a $default_map if don't have one; maybe a dummy | |
9421 | # that won't match anything. | |
9422 | if (! defined $default_map) { | |
9423 | ||
9424 | # Use any @missings line in the file. | |
9425 | if (defined $missings) { | |
9426 | if (ref $missings) { | |
9427 | $default_map = $missings->full_name; | |
9428 | $default_table = $missings; | |
9429 | } | |
9430 | else { | |
9431 | $default_map = $missings; | |
9432 | } | |
678f13d5 | 9433 | |
99870f4d KW |
9434 | # And store it with the property for outside use. |
9435 | $property_object->set_default_map($default_map); | |
9436 | } | |
9437 | else { | |
9438 | ||
9439 | # Neither an @missings nor a default map. Create | |
9440 | # a dummy one, so won't have to test definedness | |
9441 | # in the main loop. | |
9442 | $default_map = '_Perl This will never be in a file | |
9443 | from Unicode'; | |
9444 | } | |
9445 | } | |
9446 | ||
9447 | # Here, we have $default_map defined, possibly in terms of | |
9448 | # $missings, but maybe not, and possibly is a dummy one. | |
9449 | if (defined $missings) { | |
9450 | ||
9451 | # Make sure there is no conflict between the two. | |
9452 | # $missings has priority. | |
9453 | if (ref $missings) { | |
23e33b60 KW |
9454 | $default_table |
9455 | = $property_object->table($default_map); | |
99870f4d KW |
9456 | if (! defined $default_table |
9457 | || $default_table != $missings) | |
9458 | { | |
9459 | if (! defined $default_table) { | |
9460 | $default_table = $UNDEF; | |
9461 | } | |
9462 | $file->carp_bad_line(<<END | |
9463 | The \@missings line for $property_name in $file says that missings default to | |
9464 | $missings, but we expect it to be $default_table. $missings used. | |
9465 | END | |
9466 | ); | |
9467 | $default_table = $missings; | |
9468 | $default_map = $missings->full_name; | |
9469 | } | |
96cfc54a | 9470 | $property_info{$property_addr}{$DEFAULT_TABLE} |
99870f4d KW |
9471 | = $default_table; |
9472 | } | |
9473 | elsif ($default_map ne $missings) { | |
9474 | $file->carp_bad_line(<<END | |
9475 | The \@missings line for $property_name in $file says that missings default to | |
9476 | $missings, but we expect it to be $default_map. $missings used. | |
9477 | END | |
9478 | ); | |
9479 | $default_map = $missings; | |
9480 | } | |
9481 | } | |
9482 | ||
96cfc54a | 9483 | $property_info{$property_addr}{$DEFAULT_MAP} |
99870f4d KW |
9484 | = $default_map; |
9485 | ||
9486 | # If haven't done so already, find the table corresponding | |
9487 | # to this map for non-string properties. | |
9488 | if (! defined $default_table | |
9489 | && $property_type != $STRING | |
9490 | && $property_type != $UNKNOWN) | |
9491 | { | |
9492 | $default_table = $property_info{$property_addr} | |
96cfc54a | 9493 | {$DEFAULT_TABLE} |
99870f4d KW |
9494 | = $property_object->table($default_map); |
9495 | } | |
9496 | } # End of is first time for this property | |
9497 | } # End of switching properties. | |
9498 | ||
9499 | # Ready to process the line. | |
9500 | # The Unicode files are set up so that if the map is not defined, | |
9501 | # it is a binary property with value 'Y' | |
9502 | if (! defined $map) { | |
9503 | $map = 'Y'; | |
9504 | } | |
9505 | else { | |
9506 | ||
9507 | # If the map begins with a special command to us (enclosed in | |
9508 | # delimiters), extract the command(s). | |
a35d7f90 KW |
9509 | while ($map =~ s/ ^ $CMD_DELIM (.*?) $CMD_DELIM //x) { |
9510 | my $command = $1; | |
9511 | if ($command =~ / ^ $REPLACE_CMD= (.*) /x) { | |
9512 | $replace = $1; | |
99870f4d | 9513 | } |
a35d7f90 KW |
9514 | elsif ($command =~ / ^ $MAP_TYPE_CMD= (.*) /x) { |
9515 | $map_type = $1; | |
9516 | } | |
9517 | else { | |
9518 | $file->carp_bad_line("Unknown command line: '$1'"); | |
9519 | next LINE; | |
9520 | } | |
9521 | } | |
99870f4d KW |
9522 | } |
9523 | ||
9524 | if ($default_map eq $CODE_POINT && $map =~ / ^ $code_point_re $/x) | |
9525 | { | |
9526 | ||
9527 | # Here, we have a map to a particular code point, and the | |
9528 | # default map is to a code point itself. If the range | |
9529 | # includes the particular code point, change that portion of | |
9530 | # the range to the default. This makes sure that in the final | |
9531 | # table only the non-defaults are listed. | |
9532 | my $decimal_map = hex $map; | |
9533 | if ($low <= $decimal_map && $decimal_map <= $high) { | |
9534 | ||
9535 | # If the range includes stuff before or after the map | |
9536 | # we're changing, split it and process the split-off parts | |
9537 | # later. | |
9538 | if ($low < $decimal_map) { | |
9539 | $file->insert_adjusted_lines( | |
9540 | sprintf("%04X..%04X; %s; %s", | |
9541 | $low, | |
9542 | $decimal_map - 1, | |
9543 | $property_name, | |
9544 | $map)); | |
9545 | } | |
9546 | if ($high > $decimal_map) { | |
9547 | $file->insert_adjusted_lines( | |
9548 | sprintf("%04X..%04X; %s; %s", | |
9549 | $decimal_map + 1, | |
9550 | $high, | |
9551 | $property_name, | |
9552 | $map)); | |
9553 | } | |
9554 | $low = $high = $decimal_map; | |
9555 | $map = $CODE_POINT; | |
9556 | } | |
9557 | } | |
9558 | ||
9559 | # If we can tell that this is a synonym for the default map, use | |
9560 | # the default one instead. | |
9561 | if ($property_type != $STRING | |
9562 | && $property_type != $UNKNOWN) | |
9563 | { | |
9564 | my $table = $property_object->table($map); | |
9565 | if (defined $table && $table == $default_table) { | |
9566 | $map = $default_map; | |
9567 | } | |
9568 | } | |
9569 | ||
9570 | # And figure out the map type if not known. | |
9571 | if (! defined $map_type || $map_type == $COMPUTE_NO_MULTI_CP) { | |
9572 | if ($map eq "") { # Nulls are always $NULL map type | |
9573 | $map_type = $NULL; | |
9574 | } # Otherwise, non-strings, and those that don't allow | |
9575 | # $MULTI_CP, and those that aren't multiple code points are | |
9576 | # 0 | |
9577 | elsif | |
9578 | (($property_type != $STRING && $property_type != $UNKNOWN) | |
9579 | || (defined $map_type && $map_type == $COMPUTE_NO_MULTI_CP) | |
9580 | || $map !~ /^ $code_point_re ( \ $code_point_re )+ $ /x) | |
9581 | { | |
9582 | $map_type = 0; | |
9583 | } | |
9584 | else { | |
9585 | $map_type = $MULTI_CP; | |
9586 | } | |
9587 | } | |
9588 | ||
9589 | $property_object->add_map($low, $high, | |
9590 | $map, | |
9591 | Type => $map_type, | |
9592 | Replace => $replace); | |
9593 | } # End of loop through file's lines | |
9594 | ||
9595 | return; | |
9596 | } | |
9597 | } | |
9598 | ||
99870f4d KW |
9599 | { # Closure for UnicodeData.txt handling |
9600 | ||
9601 | # This file was the first one in the UCD; its design leads to some | |
9602 | # awkwardness in processing. Here is a sample line: | |
9603 | # 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061; | |
9604 | # The fields in order are: | |
9605 | my $i = 0; # The code point is in field 0, and is shifted off. | |
28093d0e | 9606 | my $CHARNAME = $i++; # character name (e.g. "LATIN CAPITAL LETTER A") |
99870f4d KW |
9607 | my $CATEGORY = $i++; # category (e.g. "Lu") |
9608 | my $CCC = $i++; # Canonical combining class (e.g. "230") | |
9609 | my $BIDI = $i++; # directional class (e.g. "L") | |
9610 | my $PERL_DECOMPOSITION = $i++; # decomposition mapping | |
9611 | my $PERL_DECIMAL_DIGIT = $i++; # decimal digit value | |
9612 | my $NUMERIC_TYPE_OTHER_DIGIT = $i++; # digit value, like a superscript | |
9613 | # Dual-use in this program; see below | |
9614 | my $NUMERIC = $i++; # numeric value | |
9615 | my $MIRRORED = $i++; # ? mirrored | |
9616 | my $UNICODE_1_NAME = $i++; # name in Unicode 1.0 | |
9617 | my $COMMENT = $i++; # iso comment | |
9618 | my $UPPER = $i++; # simple uppercase mapping | |
9619 | my $LOWER = $i++; # simple lowercase mapping | |
9620 | my $TITLE = $i++; # simple titlecase mapping | |
9621 | my $input_field_count = $i; | |
9622 | ||
9623 | # This routine in addition outputs these extra fields: | |
9624 | my $DECOMP_TYPE = $i++; # Decomposition type | |
28093d0e KW |
9625 | |
9626 | # These fields are modifications of ones above, and are usually | |
9627 | # suppressed; they must come last, as for speed, the loop upper bound is | |
9628 | # normally set to ignore them | |
9629 | my $NAME = $i++; # This is the strict name field, not the one that | |
9630 | # charnames uses. | |
9631 | my $DECOMP_MAP = $i++; # Strict decomposition mapping; not the one used | |
9632 | # by Unicode::Normalize | |
99870f4d KW |
9633 | my $last_field = $i - 1; |
9634 | ||
9635 | # All these are read into an array for each line, with the indices defined | |
9636 | # above. The empty fields in the example line above indicate that the | |
9637 | # value is defaulted. The handler called for each line of the input | |
9638 | # changes these to their defaults. | |
9639 | ||
9640 | # Here are the official names of the properties, in a parallel array: | |
9641 | my @field_names; | |
9642 | $field_names[$BIDI] = 'Bidi_Class'; | |
9643 | $field_names[$CATEGORY] = 'General_Category'; | |
9644 | $field_names[$CCC] = 'Canonical_Combining_Class'; | |
28093d0e | 9645 | $field_names[$CHARNAME] = 'Perl_Charnames'; |
99870f4d KW |
9646 | $field_names[$COMMENT] = 'ISO_Comment'; |
9647 | $field_names[$DECOMP_MAP] = 'Decomposition_Mapping'; | |
9648 | $field_names[$DECOMP_TYPE] = 'Decomposition_Type'; | |
959ce5bf | 9649 | $field_names[$LOWER] = 'Lowercase_Mapping'; |
99870f4d KW |
9650 | $field_names[$MIRRORED] = 'Bidi_Mirrored'; |
9651 | $field_names[$NAME] = 'Name'; | |
9652 | $field_names[$NUMERIC] = 'Numeric_Value'; | |
9653 | $field_names[$NUMERIC_TYPE_OTHER_DIGIT] = 'Numeric_Type'; | |
9654 | $field_names[$PERL_DECIMAL_DIGIT] = 'Perl_Decimal_Digit'; | |
9655 | $field_names[$PERL_DECOMPOSITION] = 'Perl_Decomposition_Mapping'; | |
959ce5bf | 9656 | $field_names[$TITLE] = 'Titlecase_Mapping'; |
99870f4d | 9657 | $field_names[$UNICODE_1_NAME] = 'Unicode_1_Name'; |
959ce5bf | 9658 | $field_names[$UPPER] = 'Uppercase_Mapping'; |
99870f4d | 9659 | |
28093d0e KW |
9660 | # Some of these need a little more explanation: |
9661 | # The $PERL_DECIMAL_DIGIT field does not lead to an official Unicode | |
9662 | # property, but is used in calculating the Numeric_Type. Perl however, | |
9663 | # creates a file from this field, so a Perl property is created from it. | |
9664 | # Similarly, the Other_Digit field is used only for calculating the | |
9665 | # Numeric_Type, and so it can be safely re-used as the place to store | |
9666 | # the value for Numeric_Type; hence it is referred to as | |
9667 | # $NUMERIC_TYPE_OTHER_DIGIT. | |
9668 | # The input field named $PERL_DECOMPOSITION is a combination of both the | |
9669 | # decomposition mapping and its type. Perl creates a file containing | |
9670 | # exactly this field, so it is used for that. The two properties are | |
9671 | # separated into two extra output fields, $DECOMP_MAP and $DECOMP_TYPE. | |
9672 | # $DECOMP_MAP is usually suppressed (unless the lists are changed to | |
9673 | # output it), as Perl doesn't use it directly. | |
9674 | # The input field named here $CHARNAME is used to construct the | |
9675 | # Perl_Charnames property, which is a combination of the Name property | |
9676 | # (which the input field contains), and the Unicode_1_Name property, and | |
9677 | # others from other files. Since, the strict Name property is not used | |
9678 | # by Perl, this field is used for the table that Perl does use. The | |
9679 | # strict Name property table is usually suppressed (unless the lists are | |
9680 | # changed to output it), so it is accumulated in a separate field, | |
9681 | # $NAME, which to save time is discarded unless the table is actually to | |
9682 | # be output | |
99870f4d KW |
9683 | |
9684 | # This file is processed like most in this program. Control is passed to | |
9685 | # process_generic_property_file() which calls filter_UnicodeData_line() | |
9686 | # for each input line. This filter converts the input into line(s) that | |
9687 | # process_generic_property_file() understands. There is also a setup | |
9688 | # routine called before any of the file is processed, and a handler for | |
9689 | # EOF processing, all in this closure. | |
9690 | ||
9691 | # A huge speed-up occurred at the cost of some added complexity when these | |
9692 | # routines were altered to buffer the outputs into ranges. Almost all the | |
9693 | # lines of the input file apply to just one code point, and for most | |
9694 | # properties, the map for the next code point up is the same as the | |
9695 | # current one. So instead of creating a line for each property for each | |
9696 | # input line, filter_UnicodeData_line() remembers what the previous map | |
9697 | # of a property was, and doesn't generate a line to pass on until it has | |
9698 | # to, as when the map changes; and that passed-on line encompasses the | |
9699 | # whole contiguous range of code points that have the same map for that | |
9700 | # property. This means a slight amount of extra setup, and having to | |
9701 | # flush these buffers on EOF, testing if the maps have changed, plus | |
9702 | # remembering state information in the closure. But it means a lot less | |
9703 | # real time in not having to change the data base for each property on | |
9704 | # each line. | |
9705 | ||
9706 | # Another complication is that there are already a few ranges designated | |
9707 | # in the input. There are two lines for each, with the same maps except | |
9708 | # the code point and name on each line. This was actually the hardest | |
9709 | # thing to design around. The code points in those ranges may actually | |
9710 | # have real maps not given by these two lines. These maps will either | |
98dc9551 | 9711 | # be algorithmically determinable, or in the extracted files furnished |
99870f4d KW |
9712 | # with the UCD. In the event of conflicts between these extracted files, |
9713 | # and this one, Unicode says that this one prevails. But it shouldn't | |
9714 | # prevail for conflicts that occur in these ranges. The data from the | |
9715 | # extracted files prevails in those cases. So, this program is structured | |
9716 | # so that those files are processed first, storing maps. Then the other | |
9717 | # files are processed, generally overwriting what the extracted files | |
9718 | # stored. But just the range lines in this input file are processed | |
9719 | # without overwriting. This is accomplished by adding a special string to | |
9720 | # the lines output to tell process_generic_property_file() to turn off the | |
9721 | # overwriting for just this one line. | |
9722 | # A similar mechanism is used to tell it that the map is of a non-default | |
9723 | # type. | |
9724 | ||
9725 | sub setup_UnicodeData { # Called before any lines of the input are read | |
9726 | my $file = shift; | |
9727 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
9728 | ||
28093d0e KW |
9729 | # Create a new property specially located that is a combination of the |
9730 | # various Name properties: Name, Unicode_1_Name, Named Sequences, and | |
9731 | # Name_Alias properties. (The final duplicates elements of the | |
9732 | # first.) A comment for it will later be constructed based on the | |
9733 | # actual properties present and used | |
3e20195b | 9734 | $perl_charname = Property->new('Perl_Charnames', |
28093d0e KW |
9735 | Core_Access => '\N{...} and "use charnames"', |
9736 | Default_Map => "", | |
9737 | Directory => File::Spec->curdir(), | |
9738 | File => 'Name', | |
a3db1318 | 9739 | Internal_Only => 1, |
28093d0e | 9740 | Perl_Extension => 1, |
b1c167a3 | 9741 | Range_Size_1 => \&output_perl_charnames_line, |
28093d0e KW |
9742 | Type => $STRING, |
9743 | ); | |
9744 | ||
99870f4d | 9745 | my $Perl_decomp = Property->new('Perl_Decomposition_Mapping', |
517956bf | 9746 | Directory => File::Spec->curdir(), |
99870f4d | 9747 | File => 'Decomposition', |
a14f3cb1 | 9748 | Format => $DECOMP_STRING_FORMAT, |
a3db1318 | 9749 | Internal_Only => 1, |
99870f4d KW |
9750 | Perl_Extension => 1, |
9751 | Default_Map => $CODE_POINT, | |
9752 | ||
0c07e538 KW |
9753 | # normalize.pm can't cope with these |
9754 | Output_Range_Counts => 0, | |
9755 | ||
99870f4d KW |
9756 | # This is a specially formatted table |
9757 | # explicitly for normalize.pm, which | |
9758 | # is expecting a particular format, | |
9759 | # which means that mappings containing | |
9760 | # multiple code points are in the main | |
9761 | # body of the table | |
9762 | Map_Type => $COMPUTE_NO_MULTI_CP, | |
9763 | Type => $STRING, | |
9764 | ); | |
9765 | $Perl_decomp->add_comment(join_lines(<<END | |
9766 | This mapping is a combination of the Unicode 'Decomposition_Type' and | |
9767 | 'Decomposition_Mapping' properties, formatted for use by normalize.pm. It is | |
8d6427a5 | 9768 | identical to the official Unicode 'Decomposition_Mapping' property except for |
99870f4d KW |
9769 | two things: |
9770 | 1) It omits the algorithmically determinable Hangul syllable decompositions, | |
9771 | which normalize.pm handles algorithmically. | |
9772 | 2) It contains the decomposition type as well. Non-canonical decompositions | |
9773 | begin with a word in angle brackets, like <super>, which denotes the | |
9774 | compatible decomposition type. If the map does not begin with the <angle | |
9775 | brackets>, the decomposition is canonical. | |
9776 | END | |
9777 | )); | |
9778 | ||
9779 | my $Decimal_Digit = Property->new("Perl_Decimal_Digit", | |
9780 | Default_Map => "", | |
9781 | Perl_Extension => 1, | |
9782 | File => 'Digit', # Trad. location | |
9783 | Directory => $map_directory, | |
9784 | Type => $STRING, | |
9785 | Range_Size_1 => 1, | |
9786 | ); | |
9787 | $Decimal_Digit->add_comment(join_lines(<<END | |
9788 | This file gives the mapping of all code points which represent a single | |
9789 | decimal digit [0-9] to their respective digits. For example, the code point | |
9790 | U+0031 (an ASCII '1') is mapped to a numeric 1. These code points are those | |
9791 | that have Numeric_Type=Decimal; not special things, like subscripts nor Roman | |
9792 | numerals. | |
9793 | END | |
9794 | )); | |
9795 | ||
28093d0e KW |
9796 | # These properties are not used for generating anything else, and are |
9797 | # usually not output. By making them last in the list, we can just | |
99870f4d | 9798 | # change the high end of the loop downwards to avoid the work of |
28093d0e KW |
9799 | # generating a table(s) that is/are just going to get thrown away. |
9800 | if (! property_ref('Decomposition_Mapping')->to_output_map | |
9801 | && ! property_ref('Name')->to_output_map) | |
9802 | { | |
9803 | $last_field = min($NAME, $DECOMP_MAP) - 1; | |
9804 | } elsif (property_ref('Decomposition_Mapping')->to_output_map) { | |
9805 | $last_field = $DECOMP_MAP; | |
9806 | } elsif (property_ref('Name')->to_output_map) { | |
9807 | $last_field = $NAME; | |
99870f4d KW |
9808 | } |
9809 | return; | |
9810 | } | |
9811 | ||
9812 | my $first_time = 1; # ? Is this the first line of the file | |
9813 | my $in_range = 0; # ? Are we in one of the file's ranges | |
9814 | my $previous_cp; # hex code point of previous line | |
9815 | my $decimal_previous_cp = -1; # And its decimal equivalent | |
9816 | my @start; # For each field, the current starting | |
9817 | # code point in hex for the range | |
9818 | # being accumulated. | |
9819 | my @fields; # The input fields; | |
9820 | my @previous_fields; # And those from the previous call | |
9821 | ||
9822 | sub filter_UnicodeData_line { | |
9823 | # Handle a single input line from UnicodeData.txt; see comments above | |
9824 | # Conceptually this takes a single line from the file containing N | |
9825 | # properties, and converts it into N lines with one property per line, | |
9826 | # which is what the final handler expects. But there are | |
9827 | # complications due to the quirkiness of the input file, and to save | |
9828 | # time, it accumulates ranges where the property values don't change | |
9829 | # and only emits lines when necessary. This is about an order of | |
9830 | # magnitude fewer lines emitted. | |
9831 | ||
9832 | my $file = shift; | |
9833 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
9834 | ||
9835 | # $_ contains the input line. | |
9836 | # -1 in split means retain trailing null fields | |
9837 | (my $cp, @fields) = split /\s*;\s*/, $_, -1; | |
9838 | ||
9839 | #local $to_trace = 1 if main::DEBUG; | |
9840 | trace $cp, @fields , $input_field_count if main::DEBUG && $to_trace; | |
9841 | if (@fields > $input_field_count) { | |
9842 | $file->carp_bad_line('Extra fields'); | |
9843 | $_ = ""; | |
9844 | return; | |
9845 | } | |
9846 | ||
9847 | my $decimal_cp = hex $cp; | |
9848 | ||
9849 | # We have to output all the buffered ranges when the next code point | |
9850 | # is not exactly one after the previous one, which means there is a | |
9851 | # gap in the ranges. | |
9852 | my $force_output = ($decimal_cp != $decimal_previous_cp + 1); | |
9853 | ||
9854 | # The decomposition mapping field requires special handling. It looks | |
9855 | # like either: | |
9856 | # | |
9857 | # <compat> 0032 0020 | |
9858 | # 0041 0300 | |
9859 | # | |
9860 | # The decomposition type is enclosed in <brackets>; if missing, it | |
9861 | # means the type is canonical. There are two decomposition mapping | |
9862 | # tables: the one for use by Perl's normalize.pm has a special format | |
9863 | # which is this field intact; the other, for general use is of | |
9864 | # standard format. In either case we have to find the decomposition | |
9865 | # type. Empty fields have None as their type, and map to the code | |
9866 | # point itself | |
9867 | if ($fields[$PERL_DECOMPOSITION] eq "") { | |
9868 | $fields[$DECOMP_TYPE] = 'None'; | |
9869 | $fields[$DECOMP_MAP] = $fields[$PERL_DECOMPOSITION] = $CODE_POINT; | |
9870 | } | |
9871 | else { | |
9872 | ($fields[$DECOMP_TYPE], my $map) = $fields[$PERL_DECOMPOSITION] | |
9873 | =~ / < ( .+? ) > \s* ( .+ ) /x; | |
9874 | if (! defined $fields[$DECOMP_TYPE]) { | |
9875 | $fields[$DECOMP_TYPE] = 'Canonical'; | |
9876 | $fields[$DECOMP_MAP] = $fields[$PERL_DECOMPOSITION]; | |
9877 | } | |
9878 | else { | |
9879 | $fields[$DECOMP_MAP] = $map; | |
9880 | } | |
9881 | } | |
9882 | ||
9883 | # The 3 numeric fields also require special handling. The 2 digit | |
9884 | # fields must be either empty or match the number field. This means | |
9885 | # that if it is empty, they must be as well, and the numeric type is | |
9886 | # None, and the numeric value is 'Nan'. | |
9887 | # The decimal digit field must be empty or match the other digit | |
9888 | # field. If the decimal digit field is non-empty, the code point is | |
9889 | # a decimal digit, and the other two fields will have the same value. | |
9890 | # If it is empty, but the other digit field is non-empty, the code | |
9891 | # point is an 'other digit', and the number field will have the same | |
9892 | # value as the other digit field. If the other digit field is empty, | |
9893 | # but the number field is non-empty, the code point is a generic | |
9894 | # numeric type. | |
9895 | if ($fields[$NUMERIC] eq "") { | |
9896 | if ($fields[$PERL_DECIMAL_DIGIT] ne "" | |
9897 | || $fields[$NUMERIC_TYPE_OTHER_DIGIT] ne "" | |
9898 | ) { | |
9899 | $file->carp_bad_line("Numeric values inconsistent. Trying to process anyway"); | |
9900 | } | |
9901 | $fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'None'; | |
9902 | $fields[$NUMERIC] = 'NaN'; | |
9903 | } | |
9904 | else { | |
9905 | $file->carp_bad_line("'$fields[$NUMERIC]' should be a whole or rational number. Processing as if it were") if $fields[$NUMERIC] !~ qr{ ^ -? \d+ ( / \d+ )? $ }x; | |
9906 | if ($fields[$PERL_DECIMAL_DIGIT] ne "") { | |
9907 | $file->carp_bad_line("$fields[$PERL_DECIMAL_DIGIT] should equal $fields[$NUMERIC]. Processing anyway") if $fields[$PERL_DECIMAL_DIGIT] != $fields[$NUMERIC]; | |
9908 | $fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'Decimal'; | |
9909 | } | |
9910 | elsif ($fields[$NUMERIC_TYPE_OTHER_DIGIT] ne "") { | |
9911 | $file->carp_bad_line("$fields[$NUMERIC_TYPE_OTHER_DIGIT] should equal $fields[$NUMERIC]. Processing anyway") if $fields[$NUMERIC_TYPE_OTHER_DIGIT] != $fields[$NUMERIC]; | |
9912 | $fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'Digit'; | |
9913 | } | |
9914 | else { | |
9915 | $fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'Numeric'; | |
9916 | ||
9917 | # Rationals require extra effort. | |
9918 | register_fraction($fields[$NUMERIC]) | |
9919 | if $fields[$NUMERIC] =~ qr{/}; | |
9920 | } | |
9921 | } | |
9922 | ||
9923 | # For the properties that have empty fields in the file, and which | |
9924 | # mean something different from empty, change them to that default. | |
9925 | # Certain fields just haven't been empty so far in any Unicode | |
9926 | # version, so don't look at those, namely $MIRRORED, $BIDI, $CCC, | |
9927 | # $CATEGORY. This leaves just the two fields, and so we hard-code in | |
c1739a4a | 9928 | # the defaults; which are very unlikely to ever change. |
99870f4d KW |
9929 | $fields[$UPPER] = $CODE_POINT if $fields[$UPPER] eq ""; |
9930 | $fields[$LOWER] = $CODE_POINT if $fields[$LOWER] eq ""; | |
9931 | ||
9932 | # UAX44 says that if title is empty, it is the same as whatever upper | |
9933 | # is, | |
9934 | $fields[$TITLE] = $fields[$UPPER] if $fields[$TITLE] eq ""; | |
9935 | ||
9936 | # There are a few pairs of lines like: | |
9937 | # AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;; | |
9938 | # D7A3;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;; | |
9939 | # that define ranges. These should be processed after the fields are | |
9940 | # adjusted above, as they may override some of them; but mostly what | |
28093d0e | 9941 | # is left is to possibly adjust the $CHARNAME field. The names of all the |
99870f4d KW |
9942 | # paired lines start with a '<', but this is also true of '<control>, |
9943 | # which isn't one of these special ones. | |
28093d0e | 9944 | if ($fields[$CHARNAME] eq '<control>') { |
99870f4d KW |
9945 | |
9946 | # Some code points in this file have the pseudo-name | |
9947 | # '<control>', but the official name for such ones is the null | |
28093d0e | 9948 | # string. For charnames.pm, we use the Unicode version 1 name |
99870f4d | 9949 | $fields[$NAME] = ""; |
28093d0e | 9950 | $fields[$CHARNAME] = $fields[$UNICODE_1_NAME]; |
99870f4d KW |
9951 | |
9952 | # We had better not be in between range lines. | |
9953 | if ($in_range) { | |
28093d0e | 9954 | $file->carp_bad_line("Expecting a closing range line, not a $fields[$CHARNAME]'. Trying anyway"); |
99870f4d KW |
9955 | $in_range = 0; |
9956 | } | |
9957 | } | |
28093d0e | 9958 | elsif (substr($fields[$CHARNAME], 0, 1) ne '<') { |
99870f4d KW |
9959 | |
9960 | # Here is a non-range line. We had better not be in between range | |
9961 | # lines. | |
9962 | if ($in_range) { | |
28093d0e | 9963 | $file->carp_bad_line("Expecting a closing range line, not a $fields[$CHARNAME]'. Trying anyway"); |
99870f4d KW |
9964 | $in_range = 0; |
9965 | } | |
edb80b88 KW |
9966 | if ($fields[$CHARNAME] =~ s/- $cp $//x) { |
9967 | ||
9968 | # These are code points whose names end in their code points, | |
9969 | # which means the names are algorithmically derivable from the | |
9970 | # code points. To shorten the output Name file, the algorithm | |
9971 | # for deriving these is placed in the file instead of each | |
9972 | # code point, so they have map type $CP_IN_NAME | |
9973 | $fields[$CHARNAME] = $CMD_DELIM | |
9974 | . $MAP_TYPE_CMD | |
9975 | . '=' | |
9976 | . $CP_IN_NAME | |
9977 | . $CMD_DELIM | |
9978 | . $fields[$CHARNAME]; | |
9979 | } | |
28093d0e | 9980 | $fields[$NAME] = $fields[$CHARNAME]; |
99870f4d | 9981 | } |
28093d0e KW |
9982 | elsif ($fields[$CHARNAME] =~ /^<(.+), First>$/) { |
9983 | $fields[$CHARNAME] = $fields[$NAME] = $1; | |
99870f4d KW |
9984 | |
9985 | # Here we are at the beginning of a range pair. | |
9986 | if ($in_range) { | |
28093d0e | 9987 | $file->carp_bad_line("Expecting a closing range line, not a beginning one, $fields[$CHARNAME]'. Trying anyway"); |
99870f4d KW |
9988 | } |
9989 | $in_range = 1; | |
9990 | ||
9991 | # Because the properties in the range do not overwrite any already | |
9992 | # in the db, we must flush the buffers of what's already there, so | |
9993 | # they get handled in the normal scheme. | |
9994 | $force_output = 1; | |
9995 | ||
9996 | } | |
28093d0e KW |
9997 | elsif ($fields[$CHARNAME] !~ s/^<(.+), Last>$/$1/) { |
9998 | $file->carp_bad_line("Unexpected name starting with '<' $fields[$CHARNAME]. Ignoring this line."); | |
99870f4d KW |
9999 | $_ = ""; |
10000 | return; | |
10001 | } | |
10002 | else { # Here, we are at the last line of a range pair. | |
10003 | ||
10004 | if (! $in_range) { | |
28093d0e | 10005 | $file->carp_bad_line("Unexpected end of range $fields[$CHARNAME] when not in one. Ignoring this line."); |
99870f4d KW |
10006 | $_ = ""; |
10007 | return; | |
10008 | } | |
10009 | $in_range = 0; | |
10010 | ||
28093d0e KW |
10011 | $fields[$NAME] = $fields[$CHARNAME]; |
10012 | ||
99870f4d KW |
10013 | # Check that the input is valid: that the closing of the range is |
10014 | # the same as the beginning. | |
10015 | foreach my $i (0 .. $last_field) { | |
10016 | next if $fields[$i] eq $previous_fields[$i]; | |
10017 | $file->carp_bad_line("Expecting '$fields[$i]' to be the same as '$previous_fields[$i]'. Bad News. Trying anyway"); | |
10018 | } | |
10019 | ||
10020 | # The processing differs depending on the type of range, | |
28093d0e KW |
10021 | # determined by its $CHARNAME |
10022 | if ($fields[$CHARNAME] =~ /^Hangul Syllable/) { | |
99870f4d KW |
10023 | |
10024 | # Check that the data looks right. | |
10025 | if ($decimal_previous_cp != $SBase) { | |
10026 | $file->carp_bad_line("Unexpected Hangul syllable start = $previous_cp. Bad News. Results will be wrong"); | |
10027 | } | |
10028 | if ($decimal_cp != $SBase + $SCount - 1) { | |
10029 | $file->carp_bad_line("Unexpected Hangul syllable end = $cp. Bad News. Results will be wrong"); | |
10030 | } | |
10031 | ||
10032 | # The Hangul syllable range has a somewhat complicated name | |
10033 | # generation algorithm. Each code point in it has a canonical | |
10034 | # decomposition also computable by an algorithm. The | |
10035 | # perl decomposition map table built from these is used only | |
10036 | # by normalize.pm, which has the algorithm built in it, so the | |
10037 | # decomposition maps are not needed, and are large, so are | |
10038 | # omitted from it. If the full decomposition map table is to | |
10039 | # be output, the decompositions are generated for it, in the | |
10040 | # EOF handling code for this input file. | |
10041 | ||
10042 | $previous_fields[$DECOMP_TYPE] = 'Canonical'; | |
10043 | ||
10044 | # This range is stored in our internal structure with its | |
10045 | # own map type, different from all others. | |
28093d0e KW |
10046 | $previous_fields[$CHARNAME] = $previous_fields[$NAME] |
10047 | = $CMD_DELIM | |
99870f4d KW |
10048 | . $MAP_TYPE_CMD |
10049 | . '=' | |
10050 | . $HANGUL_SYLLABLE | |
10051 | . $CMD_DELIM | |
28093d0e | 10052 | . $fields[$CHARNAME]; |
99870f4d | 10053 | } |
28093d0e | 10054 | elsif ($fields[$CHARNAME] =~ /^CJK/) { |
99870f4d KW |
10055 | |
10056 | # The name for these contains the code point itself, and all | |
10057 | # are defined to have the same base name, regardless of what | |
10058 | # is in the file. They are stored in our internal structure | |
10059 | # with a map type of $CP_IN_NAME | |
28093d0e KW |
10060 | $previous_fields[$CHARNAME] = $previous_fields[$NAME] |
10061 | = $CMD_DELIM | |
99870f4d KW |
10062 | . $MAP_TYPE_CMD |
10063 | . '=' | |
10064 | . $CP_IN_NAME | |
10065 | . $CMD_DELIM | |
10066 | . 'CJK UNIFIED IDEOGRAPH'; | |
10067 | ||
10068 | } | |
10069 | elsif ($fields[$CATEGORY] eq 'Co' | |
10070 | || $fields[$CATEGORY] eq 'Cs') | |
10071 | { | |
10072 | # The names of all the code points in these ranges are set to | |
10073 | # null, as there are no names for the private use and | |
10074 | # surrogate code points. | |
10075 | ||
28093d0e | 10076 | $previous_fields[$CHARNAME] = $previous_fields[$NAME] = ""; |
99870f4d KW |
10077 | } |
10078 | else { | |
28093d0e | 10079 | $file->carp_bad_line("Unexpected code point range $fields[$CHARNAME] because category is $fields[$CATEGORY]. Attempting to process it."); |
99870f4d KW |
10080 | } |
10081 | ||
10082 | # The first line of the range caused everything else to be output, | |
10083 | # and then its values were stored as the beginning values for the | |
10084 | # next set of ranges, which this one ends. Now, for each value, | |
10085 | # add a command to tell the handler that these values should not | |
10086 | # replace any existing ones in our database. | |
10087 | foreach my $i (0 .. $last_field) { | |
10088 | $previous_fields[$i] = $CMD_DELIM | |
10089 | . $REPLACE_CMD | |
10090 | . '=' | |
10091 | . $NO | |
10092 | . $CMD_DELIM | |
10093 | . $previous_fields[$i]; | |
10094 | } | |
10095 | ||
10096 | # And change things so it looks like the entire range has been | |
10097 | # gone through with this being the final part of it. Adding the | |
10098 | # command above to each field will cause this range to be flushed | |
10099 | # during the next iteration, as it guaranteed that the stored | |
10100 | # field won't match whatever value the next one has. | |
10101 | $previous_cp = $cp; | |
10102 | $decimal_previous_cp = $decimal_cp; | |
10103 | ||
10104 | # We are now set up for the next iteration; so skip the remaining | |
10105 | # code in this subroutine that does the same thing, but doesn't | |
10106 | # know about these ranges. | |
10107 | $_ = ""; | |
c1739a4a | 10108 | |
99870f4d KW |
10109 | return; |
10110 | } | |
10111 | ||
10112 | # On the very first line, we fake it so the code below thinks there is | |
10113 | # nothing to output, and initialize so that when it does get output it | |
10114 | # uses the first line's values for the lowest part of the range. | |
10115 | # (One could avoid this by using peek(), but then one would need to | |
10116 | # know the adjustments done above and do the same ones in the setup | |
10117 | # routine; not worth it) | |
10118 | if ($first_time) { | |
10119 | $first_time = 0; | |
10120 | @previous_fields = @fields; | |
10121 | @start = ($cp) x scalar @fields; | |
10122 | $decimal_previous_cp = $decimal_cp - 1; | |
10123 | } | |
10124 | ||
10125 | # For each field, output the stored up ranges that this code point | |
10126 | # doesn't fit in. Earlier we figured out if all ranges should be | |
10127 | # terminated because of changing the replace or map type styles, or if | |
10128 | # there is a gap between this new code point and the previous one, and | |
10129 | # that is stored in $force_output. But even if those aren't true, we | |
10130 | # need to output the range if this new code point's value for the | |
10131 | # given property doesn't match the stored range's. | |
10132 | #local $to_trace = 1 if main::DEBUG; | |
10133 | foreach my $i (0 .. $last_field) { | |
10134 | my $field = $fields[$i]; | |
10135 | if ($force_output || $field ne $previous_fields[$i]) { | |
10136 | ||
10137 | # Flush the buffer of stored values. | |
10138 | $file->insert_adjusted_lines("$start[$i]..$previous_cp; $field_names[$i]; $previous_fields[$i]"); | |
10139 | ||
10140 | # Start a new range with this code point and its value | |
10141 | $start[$i] = $cp; | |
10142 | $previous_fields[$i] = $field; | |
10143 | } | |
10144 | } | |
10145 | ||
10146 | # Set the values for the next time. | |
10147 | $previous_cp = $cp; | |
10148 | $decimal_previous_cp = $decimal_cp; | |
10149 | ||
10150 | # The input line has generated whatever adjusted lines are needed, and | |
10151 | # should not be looked at further. | |
10152 | $_ = ""; | |
10153 | return; | |
10154 | } | |
10155 | ||
10156 | sub EOF_UnicodeData { | |
10157 | # Called upon EOF to flush the buffers, and create the Hangul | |
10158 | # decomposition mappings if needed. | |
10159 | ||
10160 | my $file = shift; | |
10161 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10162 | ||
10163 | # Flush the buffers. | |
10164 | foreach my $i (1 .. $last_field) { | |
10165 | $file->insert_adjusted_lines("$start[$i]..$previous_cp; $field_names[$i]; $previous_fields[$i]"); | |
10166 | } | |
10167 | ||
10168 | if (-e 'Jamo.txt') { | |
10169 | ||
10170 | # The algorithm is published by Unicode, based on values in | |
10171 | # Jamo.txt, (which should have been processed before this | |
10172 | # subroutine), and the results left in %Jamo | |
10173 | unless (%Jamo) { | |
10174 | Carp::my_carp_bug("Jamo.txt should be processed before Unicode.txt. Hangul syllables not generated."); | |
10175 | return; | |
10176 | } | |
10177 | ||
10178 | # If the full decomposition map table is being output, insert | |
10179 | # into it the Hangul syllable mappings. This is to avoid having | |
10180 | # to publish a subroutine in it to compute them. (which would | |
10181 | # essentially be this code.) This uses the algorithm published by | |
10182 | # Unicode. | |
10183 | if (property_ref('Decomposition_Mapping')->to_output_map) { | |
10184 | for (my $S = $SBase; $S < $SBase + $SCount; $S++) { | |
10185 | use integer; | |
10186 | my $SIndex = $S - $SBase; | |
10187 | my $L = $LBase + $SIndex / $NCount; | |
10188 | my $V = $VBase + ($SIndex % $NCount) / $TCount; | |
10189 | my $T = $TBase + $SIndex % $TCount; | |
10190 | ||
10191 | trace "L=$L, V=$V, T=$T" if main::DEBUG && $to_trace; | |
10192 | my $decomposition = sprintf("%04X %04X", $L, $V); | |
10193 | $decomposition .= sprintf(" %04X", $T) if $T != $TBase; | |
10194 | $file->insert_adjusted_lines( | |
10195 | sprintf("%04X; Decomposition_Mapping; %s", | |
10196 | $S, | |
10197 | $decomposition)); | |
10198 | } | |
10199 | } | |
10200 | } | |
10201 | ||
10202 | return; | |
10203 | } | |
10204 | ||
10205 | sub filter_v1_ucd { | |
10206 | # Fix UCD lines in version 1. This is probably overkill, but this | |
10207 | # fixes some glaring errors in Version 1 UnicodeData.txt. That file: | |
10208 | # 1) had many Hangul (U+3400 - U+4DFF) code points that were later | |
10209 | # removed. This program retains them | |
10210 | # 2) didn't include ranges, which it should have, and which are now | |
10211 | # added in @corrected_lines below. It was hand populated by | |
10212 | # taking the data from Version 2, verified by analyzing | |
10213 | # DAge.txt. | |
10214 | # 3) There is a syntax error in the entry for U+09F8 which could | |
10215 | # cause problems for utf8_heavy, and so is changed. It's | |
10216 | # numeric value was simply a minus sign, without any number. | |
10217 | # (Eventually Unicode changed the code point to non-numeric.) | |
10218 | # 4) The decomposition types often don't match later versions | |
10219 | # exactly, and the whole syntax of that field is different; so | |
10220 | # the syntax is changed as well as the types to their later | |
10221 | # terminology. Otherwise normalize.pm would be very unhappy | |
10222 | # 5) Many ccc classes are different. These are left intact. | |
10223 | # 6) U+FF10 - U+FF19 are missing their numeric values in all three | |
10224 | # fields. These are unchanged because it doesn't really cause | |
10225 | # problems for Perl. | |
10226 | # 7) A number of code points, such as controls, don't have their | |
10227 | # Unicode Version 1 Names in this file. These are unchanged. | |
10228 | ||
10229 | my @corrected_lines = split /\n/, <<'END'; | |
10230 | 4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;; | |
10231 | 9FA5;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;; | |
10232 | E000;<Private Use, First>;Co;0;L;;;;;N;;;;; | |
10233 | F8FF;<Private Use, Last>;Co;0;L;;;;;N;;;;; | |
10234 | F900;<CJK Compatibility Ideograph, First>;Lo;0;L;;;;;N;;;;; | |
10235 | FA2D;<CJK Compatibility Ideograph, Last>;Lo;0;L;;;;;N;;;;; | |
10236 | END | |
10237 | ||
10238 | my $file = shift; | |
10239 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10240 | ||
10241 | #local $to_trace = 1 if main::DEBUG; | |
10242 | trace $_ if main::DEBUG && $to_trace; | |
10243 | ||
10244 | # -1 => retain trailing null fields | |
10245 | my ($code_point, @fields) = split /\s*;\s*/, $_, -1; | |
10246 | ||
10247 | # At the first place that is wrong in the input, insert all the | |
10248 | # corrections, replacing the wrong line. | |
10249 | if ($code_point eq '4E00') { | |
10250 | my @copy = @corrected_lines; | |
10251 | $_ = shift @copy; | |
10252 | ($code_point, @fields) = split /\s*;\s*/, $_, -1; | |
10253 | ||
10254 | $file->insert_lines(@copy); | |
10255 | } | |
10256 | ||
10257 | ||
10258 | if ($fields[$NUMERIC] eq '-') { | |
10259 | $fields[$NUMERIC] = '-1'; # This is what 2.0 made it. | |
10260 | } | |
10261 | ||
10262 | if ($fields[$PERL_DECOMPOSITION] ne "") { | |
10263 | ||
10264 | # Several entries have this change to superscript 2 or 3 in the | |
10265 | # middle. Convert these to the modern version, which is to use | |
10266 | # the actual U+00B2 and U+00B3 (the superscript forms) instead. | |
10267 | # So 'HHHH HHHH <+sup> 0033 <-sup> HHHH' becomes | |
10268 | # 'HHHH HHHH 00B3 HHHH'. | |
10269 | # It turns out that all of these that don't have another | |
10270 | # decomposition defined at the beginning of the line have the | |
10271 | # <square> decomposition in later releases. | |
10272 | if ($code_point ne '00B2' && $code_point ne '00B3') { | |
10273 | if ($fields[$PERL_DECOMPOSITION] | |
10274 | =~ s/<\+sup> 003([23]) <-sup>/00B$1/) | |
10275 | { | |
10276 | if (substr($fields[$PERL_DECOMPOSITION], 0, 1) ne '<') { | |
10277 | $fields[$PERL_DECOMPOSITION] = '<square> ' | |
10278 | . $fields[$PERL_DECOMPOSITION]; | |
10279 | } | |
10280 | } | |
10281 | } | |
10282 | ||
10283 | # If is like '<+circled> 0052 <-circled>', convert to | |
10284 | # '<circled> 0052' | |
10285 | $fields[$PERL_DECOMPOSITION] =~ | |
10286 | s/ < \+ ( .*? ) > \s* (.*?) \s* <-\1> /<$1> $2/x; | |
10287 | ||
10288 | # Convert '<join> HHHH HHHH <join>' to '<medial> HHHH HHHH', etc. | |
10289 | $fields[$PERL_DECOMPOSITION] =~ | |
10290 | s/ <join> \s* (.*?) \s* <no-join> /<final> $1/x | |
10291 | or $fields[$PERL_DECOMPOSITION] =~ | |
10292 | s/ <join> \s* (.*?) \s* <join> /<medial> $1/x | |
10293 | or $fields[$PERL_DECOMPOSITION] =~ | |
10294 | s/ <no-join> \s* (.*?) \s* <join> /<initial> $1/x | |
10295 | or $fields[$PERL_DECOMPOSITION] =~ | |
10296 | s/ <no-join> \s* (.*?) \s* <no-join> /<isolated> $1/x; | |
10297 | ||
10298 | # Convert '<break> HHHH HHHH <break>' to '<break> HHHH', etc. | |
10299 | $fields[$PERL_DECOMPOSITION] =~ | |
10300 | s/ <(break|no-break)> \s* (.*?) \s* <\1> /<$1> $2/x; | |
10301 | ||
10302 | # Change names to modern form. | |
10303 | $fields[$PERL_DECOMPOSITION] =~ s/<font variant>/<font>/g; | |
10304 | $fields[$PERL_DECOMPOSITION] =~ s/<no-break>/<noBreak>/g; | |
10305 | $fields[$PERL_DECOMPOSITION] =~ s/<circled>/<circle>/g; | |
10306 | $fields[$PERL_DECOMPOSITION] =~ s/<break>/<fraction>/g; | |
10307 | ||
10308 | # One entry has weird braces | |
10309 | $fields[$PERL_DECOMPOSITION] =~ s/[{}]//g; | |
10310 | } | |
10311 | ||
10312 | $_ = join ';', $code_point, @fields; | |
10313 | trace $_ if main::DEBUG && $to_trace; | |
10314 | return; | |
10315 | } | |
10316 | ||
10317 | sub filter_v2_1_5_ucd { | |
10318 | # A dozen entries in this 2.1.5 file had the mirrored and numeric | |
10319 | # columns swapped; These all had mirrored be 'N'. So if the numeric | |
10320 | # column appears to be N, swap it back. | |
10321 | ||
10322 | my ($code_point, @fields) = split /\s*;\s*/, $_, -1; | |
10323 | if ($fields[$NUMERIC] eq 'N') { | |
10324 | $fields[$NUMERIC] = $fields[$MIRRORED]; | |
10325 | $fields[$MIRRORED] = 'N'; | |
10326 | $_ = join ';', $code_point, @fields; | |
10327 | } | |
10328 | return; | |
10329 | } | |
3ffed8c2 KW |
10330 | |
10331 | sub filter_v6_ucd { | |
10332 | ||
c12f2655 KW |
10333 | # Unicode 6.0 co-opted the name BELL for U+1F514, but we haven't |
10334 | # accepted that yet to allow for some deprecation cycles. | |
3ffed8c2 | 10335 | |
484741e1 | 10336 | return if $_ !~ /^(?:0007|1F514|070F);/; |
3ffed8c2 KW |
10337 | |
10338 | my ($code_point, @fields) = split /\s*;\s*/, $_, -1; | |
10339 | if ($code_point eq '0007') { | |
dcd72625 | 10340 | $fields[$CHARNAME] = ""; |
3ffed8c2 | 10341 | } |
484741e1 KW |
10342 | elsif ($code_point eq '070F') { # Unicode Corrigendum #8; see |
10343 | # http://www.unicode.org/versions/corrigendum8.html | |
10344 | $fields[$BIDI] = "AL"; | |
10345 | } | |
10914c78 | 10346 | elsif ($^V lt v5.17.0) { # For 5.18 will convert to use Unicode's name |
3ffed8c2 KW |
10347 | $fields[$CHARNAME] = ""; |
10348 | } | |
10349 | ||
10350 | $_ = join ';', $code_point, @fields; | |
10351 | ||
10352 | return; | |
10353 | } | |
99870f4d KW |
10354 | } # End closure for UnicodeData |
10355 | ||
37e2e78e KW |
10356 | sub process_GCB_test { |
10357 | ||
10358 | my $file = shift; | |
10359 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10360 | ||
10361 | while ($file->next_line) { | |
10362 | push @backslash_X_tests, $_; | |
10363 | } | |
678f13d5 | 10364 | |
37e2e78e KW |
10365 | return; |
10366 | } | |
10367 | ||
99870f4d KW |
10368 | sub process_NamedSequences { |
10369 | # NamedSequences.txt entries are just added to an array. Because these | |
10370 | # don't look like the other tables, they have their own handler. | |
10371 | # An example: | |
10372 | # LATIN CAPITAL LETTER A WITH MACRON AND GRAVE;0100 0300 | |
10373 | # | |
10374 | # This just adds the sequence to an array for later handling | |
10375 | ||
99870f4d KW |
10376 | my $file = shift; |
10377 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10378 | ||
10379 | while ($file->next_line) { | |
10380 | my ($name, $sequence, @remainder) = split /\s*;\s*/, $_, -1; | |
10381 | if (@remainder) { | |
10382 | $file->carp_bad_line( | |
10383 | "Doesn't look like 'KHMER VOWEL SIGN OM;17BB 17C6'"); | |
10384 | next; | |
10385 | } | |
fb121860 KW |
10386 | |
10387 | # Note single \t in keeping with special output format of | |
10388 | # Perl_charnames. But it turns out that the code points don't have to | |
10389 | # be 5 digits long, like the rest, based on the internal workings of | |
10390 | # charnames.pm. This could be easily changed for consistency. | |
10391 | push @named_sequences, "$sequence\t$name"; | |
99870f4d KW |
10392 | } |
10393 | return; | |
10394 | } | |
10395 | ||
10396 | { # Closure | |
10397 | ||
10398 | my $first_range; | |
10399 | ||
10400 | sub filter_early_ea_lb { | |
10401 | # Fixes early EastAsianWidth.txt and LineBreak.txt files. These had a | |
10402 | # third field be the name of the code point, which can be ignored in | |
10403 | # most cases. But it can be meaningful if it marks a range: | |
10404 | # 33FE;W;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE | |
10405 | # 3400;W;<CJK Ideograph Extension A, First> | |
10406 | # | |
10407 | # We need to see the First in the example above to know it's a range. | |
10408 | # They did not use the later range syntaxes. This routine changes it | |
10409 | # to use the modern syntax. | |
10410 | # $1 is the Input_file object. | |
10411 | ||
10412 | my @fields = split /\s*;\s*/; | |
10413 | if ($fields[2] =~ /^<.*, First>/) { | |
10414 | $first_range = $fields[0]; | |
10415 | $_ = ""; | |
10416 | } | |
10417 | elsif ($fields[2] =~ /^<.*, Last>/) { | |
10418 | $_ = $_ = "$first_range..$fields[0]; $fields[1]"; | |
10419 | } | |
10420 | else { | |
10421 | undef $first_range; | |
10422 | $_ = "$fields[0]; $fields[1]"; | |
10423 | } | |
10424 | ||
10425 | return; | |
10426 | } | |
10427 | } | |
10428 | ||
10429 | sub filter_old_style_arabic_shaping { | |
10430 | # Early versions used a different term for the later one. | |
10431 | ||
10432 | my @fields = split /\s*;\s*/; | |
10433 | $fields[3] =~ s/<no shaping>/No_Joining_Group/; | |
10434 | $fields[3] =~ s/\s+/_/g; # Change spaces to underscores | |
10435 | $_ = join ';', @fields; | |
10436 | return; | |
10437 | } | |
10438 | ||
10439 | sub filter_arabic_shaping_line { | |
10440 | # ArabicShaping.txt has entries that look like: | |
10441 | # 062A; TEH; D; BEH | |
10442 | # The field containing 'TEH' is not used. The next field is Joining_Type | |
10443 | # and the last is Joining_Group | |
10444 | # This generates two lines to pass on, one for each property on the input | |
10445 | # line. | |
10446 | ||
10447 | my $file = shift; | |
10448 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10449 | ||
10450 | my @fields = split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields | |
10451 | ||
10452 | if (@fields > 4) { | |
10453 | $file->carp_bad_line('Extra fields'); | |
10454 | $_ = ""; | |
10455 | return; | |
10456 | } | |
10457 | ||
10458 | $file->insert_adjusted_lines("$fields[0]; Joining_Group; $fields[3]"); | |
10459 | $_ = "$fields[0]; Joining_Type; $fields[2]"; | |
10460 | ||
10461 | return; | |
10462 | } | |
10463 | ||
d3fed3dd KW |
10464 | { # Closure |
10465 | my $lc; # Table for lowercase mapping | |
10466 | my $tc; | |
10467 | my $uc; | |
10468 | ||
6c0259ad KW |
10469 | sub setup_special_casing { |
10470 | # SpecialCasing.txt contains the non-simple case change mappings. The | |
10471 | # simple ones are in UnicodeData.txt, which should already have been | |
10472 | # read in to the full property data structures, so as to initialize | |
10473 | # these with the simple ones. Then the SpecialCasing.txt entries | |
10474 | # overwrite the ones which have different full mappings. | |
10475 | ||
10476 | # This routine sees if the simple mappings are to be output, and if | |
10477 | # so, copies what has already been put into the full mapping tables, | |
10478 | # while they still contain only the simple mappings. | |
10479 | ||
10480 | # The reason it is done this way is that the simple mappings are | |
10481 | # probably not going to be output, so it saves work to initialize the | |
10482 | # full tables with the simple mappings, and then overwrite those | |
10483 | # relatively few entries in them that have different full mappings, | |
10484 | # and thus skip the simple mapping tables altogether. | |
10485 | ||
c12f2655 KW |
10486 | # New tables with just the simple mappings that are overridden by the |
10487 | # full ones are constructed. These are for Unicode::UCD, which | |
10488 | # requires the simple mappings. The Case_Folding table is a combined | |
10489 | # table of both the simple and full mappings, with the full ones being | |
10490 | # in the hash, and the simple ones, even those overridden by the hash, | |
10491 | # being in the base table. That same mechanism could have been | |
10492 | # employed here, except that the docs have said that the generated | |
10493 | # files are usuable directly by programs, so we dare not change the | |
10494 | # format in any way. | |
10495 | ||
6c0259ad KW |
10496 | my $file= shift; |
10497 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
99870f4d | 10498 | |
6c0259ad KW |
10499 | $lc = property_ref('lc'); |
10500 | $tc = property_ref('tc'); | |
10501 | $uc = property_ref('uc'); | |
10502 | ||
10503 | # For each of the case change mappings... | |
10504 | foreach my $case_table ($lc, $tc, $uc) { | |
10505 | my $case = $case_table->name; | |
10506 | my $full = property_ref($case); | |
10507 | unless (defined $full && ! $full->is_empty) { | |
10508 | Carp::my_carp_bug("Need to process UnicodeData before SpecialCasing. Only special casing will be generated."); | |
10509 | } | |
10510 | ||
10511 | # The simple version's name in each mapping merely has an 's' in | |
10512 | # front of the full one's | |
10513 | my $simple = property_ref('s' . $case); | |
10514 | $simple->initialize($full) if $simple->to_output_map(); | |
10515 | ||
10516 | my $simple_only = Property->new("_s$case", | |
10517 | Type => $STRING, | |
10518 | Default_Map => $CODE_POINT, | |
10519 | Perl_Extension => 1, | |
6ba2d696 | 10520 | Internal_Only => 1, |
6c0259ad KW |
10521 | Description => "The simple mappings for $case for code points that have full mappings as well"); |
10522 | $simple_only->set_to_output_map($INTERNAL_MAP); | |
10523 | $simple_only->add_comment(join_lines( <<END | |
d3fed3dd KW |
10524 | This file is for UCD.pm so that it can construct simple mappings that would |
10525 | otherwise be lost because they are overridden by full mappings. | |
10526 | END | |
6c0259ad KW |
10527 | )); |
10528 | } | |
99870f4d | 10529 | |
6c0259ad KW |
10530 | return; |
10531 | } | |
99870f4d | 10532 | |
6c0259ad KW |
10533 | sub filter_special_casing_line { |
10534 | # Change the format of $_ from SpecialCasing.txt into something that | |
10535 | # the generic handler understands. Each input line contains three | |
10536 | # case mappings. This will generate three lines to pass to the | |
10537 | # generic handler for each of those. | |
99870f4d | 10538 | |
6c0259ad KW |
10539 | # The input syntax (after stripping comments and trailing white space |
10540 | # is like one of the following (with the final two being entries that | |
10541 | # we ignore): | |
10542 | # 00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S | |
10543 | # 03A3; 03C2; 03A3; 03A3; Final_Sigma; | |
10544 | # 0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE | |
10545 | # Note the trailing semi-colon, unlike many of the input files. That | |
10546 | # means that there will be an extra null field generated by the split | |
99870f4d | 10547 | |
6c0259ad KW |
10548 | my $file = shift; |
10549 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
99870f4d | 10550 | |
6c0259ad KW |
10551 | my @fields = split /\s*;\s*/, $_, -1; # -1 => retain trailing null |
10552 | # fields | |
10553 | ||
10554 | # field #4 is when this mapping is conditional. If any of these get | |
10555 | # implemented, it would be by hard-coding in the casing functions in | |
10556 | # the Perl core, not through tables. But if there is a new condition | |
10557 | # we don't know about, output a warning. We know about all the | |
10558 | # conditions through 6.0 | |
10559 | if ($fields[4] ne "") { | |
10560 | my @conditions = split ' ', $fields[4]; | |
10561 | if ($conditions[0] ne 'tr' # We know that these languages have | |
10562 | # conditions, and some are multiple | |
10563 | && $conditions[0] ne 'az' | |
10564 | && $conditions[0] ne 'lt' | |
10565 | ||
10566 | # And, we know about a single condition Final_Sigma, but | |
10567 | # nothing else. | |
10568 | && ($v_version gt v5.2.0 | |
10569 | && (@conditions > 1 || $conditions[0] ne 'Final_Sigma'))) | |
10570 | { | |
10571 | $file->carp_bad_line("Unknown condition '$fields[4]'. You should inspect it and either add code to handle it, or add to list of those that are to ignore"); | |
10572 | } | |
10573 | elsif ($conditions[0] ne 'Final_Sigma') { | |
99870f4d | 10574 | |
6c0259ad KW |
10575 | # Don't print out a message for Final_Sigma, because we |
10576 | # have hard-coded handling for it. (But the standard | |
10577 | # could change what the rule should be, but it wouldn't | |
10578 | # show up here anyway. | |
99870f4d | 10579 | |
6c0259ad | 10580 | print "# SKIPPING Special Casing: $_\n" |
99870f4d | 10581 | if $verbosity >= $VERBOSE; |
6c0259ad KW |
10582 | } |
10583 | $_ = ""; | |
10584 | return; | |
10585 | } | |
10586 | elsif (@fields > 6 || (@fields == 6 && $fields[5] ne "" )) { | |
10587 | $file->carp_bad_line('Extra fields'); | |
10588 | $_ = ""; | |
10589 | return; | |
99870f4d | 10590 | } |
99870f4d | 10591 | |
6c0259ad KW |
10592 | $_ = "$fields[0]; lc; $fields[1]"; |
10593 | $file->insert_adjusted_lines("$fields[0]; tc; $fields[2]"); | |
10594 | $file->insert_adjusted_lines("$fields[0]; uc; $fields[3]"); | |
99870f4d | 10595 | |
6c0259ad KW |
10596 | # Copy any simple case change to the special tables constructed if |
10597 | # being overridden by a multi-character case change. | |
10598 | if ($fields[1] ne $fields[0] | |
10599 | && (my $value = $lc->value_of(hex $fields[0])) ne $CODE_POINT) | |
10600 | { | |
10601 | $file->insert_adjusted_lines("$fields[0]; _slc; $value"); | |
10602 | } | |
10603 | if ($fields[2] ne $fields[0] | |
10604 | && (my $value = $tc->value_of(hex $fields[0])) ne $CODE_POINT) | |
10605 | { | |
10606 | $file->insert_adjusted_lines("$fields[0]; _stc; $value"); | |
10607 | } | |
10608 | if ($fields[3] ne $fields[0] | |
10609 | && (my $value = $uc->value_of(hex $fields[0])) ne $CODE_POINT) | |
10610 | { | |
10611 | $file->insert_adjusted_lines("$fields[0]; _suc; $value"); | |
10612 | } | |
d3fed3dd | 10613 | |
6c0259ad KW |
10614 | return; |
10615 | } | |
d3fed3dd | 10616 | } |
99870f4d KW |
10617 | |
10618 | sub filter_old_style_case_folding { | |
10619 | # This transforms $_ containing the case folding style of 3.0.1, to 3.1 | |
f86864ac | 10620 | # and later style. Different letters were used in the earlier. |
99870f4d KW |
10621 | |
10622 | my $file = shift; | |
10623 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10624 | ||
10625 | my @fields = split /\s*;\s*/; | |
10626 | if ($fields[0] =~ /^ 013 [01] $/x) { # The two turkish fields | |
10627 | $fields[1] = 'I'; | |
10628 | } | |
10629 | elsif ($fields[1] eq 'L') { | |
10630 | $fields[1] = 'C'; # L => C always | |
10631 | } | |
10632 | elsif ($fields[1] eq 'E') { | |
10633 | if ($fields[2] =~ / /) { # E => C if one code point; F otherwise | |
10634 | $fields[1] = 'F' | |
10635 | } | |
10636 | else { | |
10637 | $fields[1] = 'C' | |
10638 | } | |
10639 | } | |
10640 | else { | |
10641 | $file->carp_bad_line("Expecting L or E in second field"); | |
10642 | $_ = ""; | |
10643 | return; | |
10644 | } | |
10645 | $_ = join("; ", @fields) . ';'; | |
10646 | return; | |
10647 | } | |
10648 | ||
10649 | { # Closure for case folding | |
10650 | ||
10651 | # Create the map for simple only if are going to output it, for otherwise | |
10652 | # it takes no part in anything we do. | |
10653 | my $to_output_simple; | |
10654 | ||
99870f4d KW |
10655 | sub setup_case_folding($) { |
10656 | # Read in the case foldings in CaseFolding.txt. This handles both | |
10657 | # simple and full case folding. | |
10658 | ||
10659 | $to_output_simple | |
10660 | = property_ref('Simple_Case_Folding')->to_output_map; | |
10661 | ||
6f2a3287 KW |
10662 | # If we ever wanted to show that these tables were combined, a new |
10663 | # property method could be created, like set_combined_props() | |
10664 | property_ref('Case_Folding')->add_comment(join_lines( <<END | |
10665 | This file includes both the simple and full case folding maps. The simple | |
10666 | ones are in the main body of the table below, and the full ones adding to or | |
10667 | overriding them are in the hash. | |
10668 | END | |
10669 | )); | |
99870f4d KW |
10670 | return; |
10671 | } | |
10672 | ||
10673 | sub filter_case_folding_line { | |
10674 | # Called for each line in CaseFolding.txt | |
10675 | # Input lines look like: | |
10676 | # 0041; C; 0061; # LATIN CAPITAL LETTER A | |
10677 | # 00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S | |
10678 | # 1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S | |
10679 | # | |
10680 | # 'C' means that folding is the same for both simple and full | |
10681 | # 'F' that it is only for full folding | |
10682 | # 'S' that it is only for simple folding | |
10683 | # 'T' is locale-dependent, and ignored | |
10684 | # 'I' is a type of 'F' used in some early releases. | |
10685 | # Note the trailing semi-colon, unlike many of the input files. That | |
10686 | # means that there will be an extra null field generated by the split | |
10687 | # below, which we ignore and hence is not an error. | |
10688 | ||
10689 | my $file = shift; | |
10690 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10691 | ||
10692 | my ($range, $type, $map, @remainder) = split /\s*;\s*/, $_, -1; | |
10693 | if (@remainder > 1 || (@remainder == 1 && $remainder[0] ne "" )) { | |
10694 | $file->carp_bad_line('Extra fields'); | |
10695 | $_ = ""; | |
10696 | return; | |
10697 | } | |
10698 | ||
10699 | if ($type eq 'T') { # Skip Turkic case folding, is locale dependent | |
10700 | $_ = ""; | |
10701 | return; | |
10702 | } | |
10703 | ||
10704 | # C: complete, F: full, or I: dotted uppercase I -> dotless lowercase | |
3c099872 KW |
10705 | # I are all full foldings; S is single-char. For S, there is always |
10706 | # an F entry, so we must allow multiple values for the same code | |
10707 | # point. Fortunately this table doesn't need further manipulation | |
10708 | # which would preclude using multiple-values. The S is now included | |
10709 | # so that _swash_inversion_hash() is able to construct closures | |
10710 | # without having to worry about F mappings. | |
10711 | if ($type eq 'C' || $type eq 'F' || $type eq 'I' || $type eq 'S') { | |
10712 | $_ = "$range; Case_Folding; $CMD_DELIM$REPLACE_CMD=$MULTIPLE$CMD_DELIM$map"; | |
99870f4d KW |
10713 | } |
10714 | else { | |
10715 | $_ = ""; | |
3c099872 | 10716 | $file->carp_bad_line('Expecting C F I S or T in second field'); |
99870f4d KW |
10717 | } |
10718 | ||
10719 | # C and S are simple foldings, but simple case folding is not needed | |
10720 | # unless we explicitly want its map table output. | |
10721 | if ($to_output_simple && $type eq 'C' || $type eq 'S') { | |
10722 | $file->insert_adjusted_lines("$range; Simple_Case_Folding; $map"); | |
10723 | } | |
10724 | ||
99870f4d KW |
10725 | return; |
10726 | } | |
10727 | ||
99870f4d KW |
10728 | } # End case fold closure |
10729 | ||
10730 | sub filter_jamo_line { | |
10731 | # Filter Jamo.txt lines. This routine mainly is used to populate hashes | |
10732 | # from this file that is used in generating the Name property for Jamo | |
10733 | # code points. But, it also is used to convert early versions' syntax | |
10734 | # into the modern form. Here are two examples: | |
10735 | # 1100; G # HANGUL CHOSEONG KIYEOK # Modern syntax | |
10736 | # U+1100; G; HANGUL CHOSEONG KIYEOK # 2.0 syntax | |
10737 | # | |
10738 | # The input is $_, the output is $_ filtered. | |
10739 | ||
10740 | my @fields = split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields | |
10741 | ||
10742 | # Let the caller handle unexpected input. In earlier versions, there was | |
10743 | # a third field which is supposed to be a comment, but did not have a '#' | |
10744 | # before it. | |
10745 | return if @fields > (($v_version gt v3.0.0) ? 2 : 3); | |
10746 | ||
10747 | $fields[0] =~ s/^U\+//; # Also, early versions had this extraneous | |
10748 | # beginning. | |
10749 | ||
10750 | # Some 2.1 versions had this wrong. Causes havoc with the algorithm. | |
10751 | $fields[1] = 'R' if $fields[0] eq '1105'; | |
10752 | ||
10753 | # Add to structure so can generate Names from it. | |
10754 | my $cp = hex $fields[0]; | |
10755 | my $short_name = $fields[1]; | |
10756 | $Jamo{$cp} = $short_name; | |
10757 | if ($cp <= $LBase + $LCount) { | |
10758 | $Jamo_L{$short_name} = $cp - $LBase; | |
10759 | } | |
10760 | elsif ($cp <= $VBase + $VCount) { | |
10761 | $Jamo_V{$short_name} = $cp - $VBase; | |
10762 | } | |
10763 | elsif ($cp <= $TBase + $TCount) { | |
10764 | $Jamo_T{$short_name} = $cp - $TBase; | |
10765 | } | |
10766 | else { | |
10767 | Carp::my_carp_bug("Unexpected Jamo code point in $_"); | |
10768 | } | |
10769 | ||
10770 | ||
10771 | # Reassemble using just the first two fields to look like a typical | |
10772 | # property file line | |
10773 | $_ = "$fields[0]; $fields[1]"; | |
10774 | ||
10775 | return; | |
10776 | } | |
10777 | ||
99870f4d KW |
10778 | sub register_fraction($) { |
10779 | # This registers the input rational number so that it can be passed on to | |
10780 | # utf8_heavy.pl, both in rational and floating forms. | |
10781 | ||
10782 | my $rational = shift; | |
10783 | ||
10784 | my $float = eval $rational; | |
10785 | $nv_floating_to_rational{$float} = $rational; | |
10786 | return; | |
10787 | } | |
10788 | ||
10789 | sub filter_numeric_value_line { | |
10790 | # DNumValues contains lines of a different syntax than the typical | |
10791 | # property file: | |
10792 | # 0F33 ; -0.5 ; ; -1/2 # No TIBETAN DIGIT HALF ZERO | |
10793 | # | |
10794 | # This routine transforms $_ containing the anomalous syntax to the | |
10795 | # typical, by filtering out the extra columns, and convert early version | |
10796 | # decimal numbers to strings that look like rational numbers. | |
10797 | ||
10798 | my $file = shift; | |
10799 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10800 | ||
10801 | # Starting in 5.1, there is a rational field. Just use that, omitting the | |
10802 | # extra columns. Otherwise convert the decimal number in the second field | |
10803 | # to a rational, and omit extraneous columns. | |
10804 | my @fields = split /\s*;\s*/, $_, -1; | |
10805 | my $rational; | |
10806 | ||
10807 | if ($v_version ge v5.1.0) { | |
10808 | if (@fields != 4) { | |
10809 | $file->carp_bad_line('Not 4 semi-colon separated fields'); | |
10810 | $_ = ""; | |
10811 | return; | |
10812 | } | |
10813 | $rational = $fields[3]; | |
10814 | $_ = join '; ', @fields[ 0, 3 ]; | |
10815 | } | |
10816 | else { | |
10817 | ||
10818 | # Here, is an older Unicode file, which has decimal numbers instead of | |
10819 | # rationals in it. Use the fraction to calculate the denominator and | |
10820 | # convert to rational. | |
10821 | ||
10822 | if (@fields != 2 && @fields != 3) { | |
10823 | $file->carp_bad_line('Not 2 or 3 semi-colon separated fields'); | |
10824 | $_ = ""; | |
10825 | return; | |
10826 | } | |
10827 | ||
10828 | my $codepoints = $fields[0]; | |
10829 | my $decimal = $fields[1]; | |
10830 | if ($decimal =~ s/\.0+$//) { | |
10831 | ||
10832 | # Anything ending with a decimal followed by nothing but 0's is an | |
10833 | # integer | |
10834 | $_ = "$codepoints; $decimal"; | |
10835 | $rational = $decimal; | |
10836 | } | |
10837 | else { | |
10838 | ||
10839 | my $denominator; | |
10840 | if ($decimal =~ /\.50*$/) { | |
10841 | $denominator = 2; | |
10842 | } | |
10843 | ||
10844 | # Here have the hardcoded repeating decimals in the fraction, and | |
10845 | # the denominator they imply. There were only a few denominators | |
10846 | # in the older Unicode versions of this file which this code | |
10847 | # handles, so it is easy to convert them. | |
10848 | ||
10849 | # The 4 is because of a round-off error in the Unicode 3.2 files | |
10850 | elsif ($decimal =~ /\.33*[34]$/ || $decimal =~ /\.6+7$/) { | |
10851 | $denominator = 3; | |
10852 | } | |
10853 | elsif ($decimal =~ /\.[27]50*$/) { | |
10854 | $denominator = 4; | |
10855 | } | |
10856 | elsif ($decimal =~ /\.[2468]0*$/) { | |
10857 | $denominator = 5; | |
10858 | } | |
10859 | elsif ($decimal =~ /\.16+7$/ || $decimal =~ /\.83+$/) { | |
10860 | $denominator = 6; | |
10861 | } | |
10862 | elsif ($decimal =~ /\.(12|37|62|87)50*$/) { | |
10863 | $denominator = 8; | |
10864 | } | |
10865 | if ($denominator) { | |
10866 | my $sign = ($decimal < 0) ? "-" : ""; | |
10867 | my $numerator = int((abs($decimal) * $denominator) + .5); | |
10868 | $rational = "$sign$numerator/$denominator"; | |
10869 | $_ = "$codepoints; $rational"; | |
10870 | } | |
10871 | else { | |
10872 | $file->carp_bad_line("Can't cope with number '$decimal'."); | |
10873 | $_ = ""; | |
10874 | return; | |
10875 | } | |
10876 | } | |
10877 | } | |
10878 | ||
10879 | register_fraction($rational) if $rational =~ qr{/}; | |
10880 | return; | |
10881 | } | |
10882 | ||
10883 | { # Closure | |
10884 | my %unihan_properties; | |
99870f4d KW |
10885 | |
10886 | sub setup_unihan { | |
10887 | # Do any special setup for Unihan properties. | |
10888 | ||
10889 | # This property gives the wrong computed type, so override. | |
10890 | my $usource = property_ref('kIRG_USource'); | |
10891 | $usource->set_type($STRING) if defined $usource; | |
10892 | ||
b2abbe5b KW |
10893 | # This property is to be considered binary (it says so in |
10894 | # http://www.unicode.org/reports/tr38/) | |
46b2142f | 10895 | my $iicore = property_ref('kIICore'); |
99870f4d | 10896 | if (defined $iicore) { |
46b2142f KW |
10897 | $iicore->set_type($FORCED_BINARY); |
10898 | $iicore->table("Y")->add_note("Forced to a binary property as per unicode.org UAX #38."); | |
10899 | ||
10900 | # Unicode doesn't include the maps for this property, so don't | |
10901 | # warn that they are missing. | |
10902 | $iicore->set_pre_declared_maps(0); | |
10903 | $iicore->add_comment(join_lines( <<END | |
10904 | This property contains enum values, but Unicode UAX #38 says it should be | |
10905 | interpreted as binary, so Perl creates tables for both 1) its enum values, | |
10906 | plus 2) true/false tables in which it is considered true for all code points | |
10907 | that have a non-null value | |
10908 | END | |
10909 | )); | |
99870f4d KW |
10910 | } |
10911 | ||
10912 | return; | |
10913 | } | |
10914 | ||
10915 | sub filter_unihan_line { | |
10916 | # Change unihan db lines to look like the others in the db. Here is | |
10917 | # an input sample: | |
10918 | # U+341C kCangjie IEKN | |
10919 | ||
10920 | # Tabs are used instead of semi-colons to separate fields; therefore | |
10921 | # they may have semi-colons embedded in them. Change these to periods | |
10922 | # so won't screw up the rest of the code. | |
10923 | s/;/./g; | |
10924 | ||
10925 | # Remove lines that don't look like ones we accept. | |
10926 | if ($_ !~ /^ [^\t]* \t ( [^\t]* ) /x) { | |
10927 | $_ = ""; | |
10928 | return; | |
10929 | } | |
10930 | ||
10931 | # Extract the property, and save a reference to its object. | |
10932 | my $property = $1; | |
10933 | if (! exists $unihan_properties{$property}) { | |
10934 | $unihan_properties{$property} = property_ref($property); | |
10935 | } | |
10936 | ||
10937 | # Don't do anything unless the property is one we're handling, which | |
10938 | # we determine by seeing if there is an object defined for it or not | |
10939 | if (! defined $unihan_properties{$property}) { | |
10940 | $_ = ""; | |
10941 | return; | |
10942 | } | |
10943 | ||
99870f4d KW |
10944 | # Convert the tab separators to our standard semi-colons, and convert |
10945 | # the U+HHHH notation to the rest of the standard's HHHH | |
10946 | s/\t/;/g; | |
10947 | s/\b U \+ (?= $code_point_re )//xg; | |
10948 | ||
10949 | #local $to_trace = 1 if main::DEBUG; | |
10950 | trace $_ if main::DEBUG && $to_trace; | |
10951 | ||
10952 | return; | |
10953 | } | |
10954 | } | |
10955 | ||
10956 | sub filter_blocks_lines { | |
10957 | # In the Blocks.txt file, the names of the blocks don't quite match the | |
10958 | # names given in PropertyValueAliases.txt, so this changes them so they | |
10959 | # do match: Blanks and hyphens are changed into underscores. Also makes | |
10960 | # early release versions look like later ones | |
10961 | # | |
10962 | # $_ is transformed to the correct value. | |
10963 | ||
10964 | my $file = shift; | |
10965 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
10966 | ||
10967 | if ($v_version lt v3.2.0) { | |
10968 | if (/FEFF.*Specials/) { # Bug in old versions: line wrongly inserted | |
10969 | $_ = ""; | |
10970 | return; | |
10971 | } | |
10972 | ||
10973 | # Old versions used a different syntax to mark the range. | |
10974 | $_ =~ s/;\s+/../ if $v_version lt v3.1.0; | |
10975 | } | |
10976 | ||
10977 | my @fields = split /\s*;\s*/, $_, -1; | |
10978 | if (@fields != 2) { | |
10979 | $file->carp_bad_line("Expecting exactly two fields"); | |
10980 | $_ = ""; | |
10981 | return; | |
10982 | } | |
10983 | ||
10984 | # Change hyphens and blanks in the block name field only | |
10985 | $fields[1] =~ s/[ -]/_/g; | |
10986 | $fields[1] =~ s/_ ( [a-z] ) /_\u$1/g; # Capitalize first letter of word | |
10987 | ||
10988 | $_ = join("; ", @fields); | |
10989 | return; | |
10990 | } | |
10991 | ||
10992 | { # Closure | |
10993 | my $current_property; | |
10994 | ||
10995 | sub filter_old_style_proplist { | |
10996 | # PropList.txt has been in Unicode since version 2.0. Until 3.1, it | |
10997 | # was in a completely different syntax. Ken Whistler of Unicode says | |
10998 | # that it was something he used as an aid for his own purposes, but | |
10999 | # was never an official part of the standard. However, comments in | |
11000 | # DAge.txt indicate that non-character code points were available in | |
11001 | # the UCD as of 3.1. It is unclear to me (khw) how they could be | |
11002 | # there except through this file (but on the other hand, they first | |
11003 | # appeared there in 3.0.1), so maybe it was part of the UCD, and maybe | |
11004 | # not. But the claim is that it was published as an aid to others who | |
11005 | # might want some more information than was given in the official UCD | |
11006 | # of the time. Many of the properties in it were incorporated into | |
11007 | # the later PropList.txt, but some were not. This program uses this | |
11008 | # early file to generate property tables that are otherwise not | |
11009 | # accessible in the early UCD's, and most were probably not really | |
11010 | # official at that time, so one could argue that it should be ignored, | |
11011 | # and you can easily modify things to skip this. And there are bugs | |
11012 | # in this file in various versions. (For example, the 2.1.9 version | |
11013 | # removes from Alphabetic the CJK range starting at 4E00, and they | |
11014 | # weren't added back in until 3.1.0.) Many of this file's properties | |
11015 | # were later sanctioned, so this code generates tables for those | |
11016 | # properties that aren't otherwise in the UCD of the time but | |
11017 | # eventually did become official, and throws away the rest. Here is a | |
11018 | # list of all the ones that are thrown away: | |
11019 | # Bidi=* duplicates UnicodeData.txt | |
11020 | # Combining never made into official property; | |
11021 | # is \P{ccc=0} | |
11022 | # Composite never made into official property. | |
11023 | # Currency Symbol duplicates UnicodeData.txt: gc=sc | |
11024 | # Decimal Digit duplicates UnicodeData.txt: gc=nd | |
11025 | # Delimiter never made into official property; | |
11026 | # removed in 3.0.1 | |
11027 | # Format Control never made into official property; | |
11028 | # similar to gc=cf | |
11029 | # High Surrogate duplicates Blocks.txt | |
11030 | # Ignorable Control never made into official property; | |
11031 | # similar to di=y | |
11032 | # ISO Control duplicates UnicodeData.txt: gc=cc | |
11033 | # Left of Pair never made into official property; | |
11034 | # Line Separator duplicates UnicodeData.txt: gc=zl | |
11035 | # Low Surrogate duplicates Blocks.txt | |
11036 | # Non-break was actually listed as a property | |
11037 | # in 3.2, but without any code | |
11038 | # points. Unicode denies that this | |
11039 | # was ever an official property | |
11040 | # Non-spacing duplicate UnicodeData.txt: gc=mn | |
11041 | # Numeric duplicates UnicodeData.txt: gc=cc | |
11042 | # Paired Punctuation never made into official property; | |
11043 | # appears to be gc=ps + gc=pe | |
11044 | # Paragraph Separator duplicates UnicodeData.txt: gc=cc | |
11045 | # Private Use duplicates UnicodeData.txt: gc=co | |
11046 | # Private Use High Surrogate duplicates Blocks.txt | |
11047 | # Punctuation duplicates UnicodeData.txt: gc=p | |
11048 | # Space different definition than eventual | |
11049 | # one. | |
11050 | # Titlecase duplicates UnicodeData.txt: gc=lt | |
11051 | # Unassigned Code Value duplicates UnicodeData.txt: gc=cc | |
98dc9551 | 11052 | # Zero-width never made into official property; |
99870f4d KW |
11053 | # subset of gc=cf |
11054 | # Most of the properties have the same names in this file as in later | |
11055 | # versions, but a couple do not. | |
11056 | # | |
11057 | # This subroutine filters $_, converting it from the old style into | |
11058 | # the new style. Here's a sample of the old-style | |
11059 | # | |
11060 | # ******************************************* | |
11061 | # | |
11062 | # Property dump for: 0x100000A0 (Join Control) | |
11063 | # | |
11064 | # 200C..200D (2 chars) | |
11065 | # | |
11066 | # In the example, the property is "Join Control". It is kept in this | |
11067 | # closure between calls to the subroutine. The numbers beginning with | |
11068 | # 0x were internal to Ken's program that generated this file. | |
11069 | ||
11070 | # If this line contains the property name, extract it. | |
11071 | if (/^Property dump for: [^(]*\((.*)\)/) { | |
11072 | $_ = $1; | |
11073 | ||
11074 | # Convert white space to underscores. | |
11075 | s/ /_/g; | |
11076 | ||
11077 | # Convert the few properties that don't have the same name as | |
11078 | # their modern counterparts | |
11079 | s/Identifier_Part/ID_Continue/ | |
11080 | or s/Not_a_Character/NChar/; | |
11081 | ||
11082 | # If the name matches an existing property, use it. | |
11083 | if (defined property_ref($_)) { | |
11084 | trace "new property=", $_ if main::DEBUG && $to_trace; | |
11085 | $current_property = $_; | |
11086 | } | |
11087 | else { # Otherwise discard it | |
11088 | trace "rejected property=", $_ if main::DEBUG && $to_trace; | |
11089 | undef $current_property; | |
11090 | } | |
11091 | $_ = ""; # The property is saved for the next lines of the | |
11092 | # file, but this defining line is of no further use, | |
11093 | # so clear it so that the caller won't process it | |
11094 | # further. | |
11095 | } | |
11096 | elsif (! defined $current_property || $_ !~ /^$code_point_re/) { | |
11097 | ||
11098 | # Here, the input line isn't a header defining a property for the | |
11099 | # following section, and either we aren't in such a section, or | |
11100 | # the line doesn't look like one that defines the code points in | |
11101 | # such a section. Ignore this line. | |
11102 | $_ = ""; | |
11103 | } | |
11104 | else { | |
11105 | ||
11106 | # Here, we have a line defining the code points for the current | |
11107 | # stashed property. Anything starting with the first blank is | |
11108 | # extraneous. Otherwise, it should look like a normal range to | |
11109 | # the caller. Append the property name so that it looks just like | |
11110 | # a modern PropList entry. | |
11111 | ||
11112 | $_ =~ s/\s.*//; | |
11113 | $_ .= "; $current_property"; | |
11114 | } | |
11115 | trace $_ if main::DEBUG && $to_trace; | |
11116 | return; | |
11117 | } | |
11118 | } # End closure for old style proplist | |
11119 | ||
11120 | sub filter_old_style_normalization_lines { | |
11121 | # For early releases of Unicode, the lines were like: | |
11122 | # 74..2A76 ; NFKD_NO | |
11123 | # For later releases this became: | |
11124 | # 74..2A76 ; NFKD_QC; N | |
11125 | # Filter $_ to look like those in later releases. | |
11126 | # Similarly for MAYBEs | |
11127 | ||
11128 | s/ _NO \b /_QC; N/x || s/ _MAYBE \b /_QC; M/x; | |
11129 | ||
11130 | # Also, the property FC_NFKC was abbreviated to FNC | |
11131 | s/FNC/FC_NFKC/; | |
11132 | return; | |
11133 | } | |
11134 | ||
82aed44a KW |
11135 | sub setup_script_extensions { |
11136 | # The Script_Extensions property starts out with a clone of the Script | |
11137 | # property. | |
11138 | ||
11139 | my $sc = property_ref("Script"); | |
11140 | my $scx = Property->new("scx", Full_Name => "Script_Extensions", | |
11141 | Initialize => $sc, | |
11142 | Default_Map => $sc->default_map, | |
11143 | Pre_Declared_Maps => 0, | |
c3ff2976 | 11144 | Format => $STRING_WHITE_SPACE_LIST, |
82aed44a KW |
11145 | ); |
11146 | $scx->add_comment(join_lines( <<END | |
11147 | The values for code points that appear in one script are just the same as for | |
11148 | the 'Script' property. Likewise the values for those that appear in many | |
11149 | scripts are either 'Common' or 'Inherited', same as with 'Script'. But the | |
11150 | values of code points that appear in a few scripts are a space separated list | |
11151 | of those scripts. | |
11152 | END | |
11153 | )); | |
11154 | ||
11155 | # Make the scx's tables and aliases for them the same as sc's | |
11156 | foreach my $table ($sc->tables) { | |
11157 | my $scx_table = $scx->add_match_table($table->name, | |
11158 | Full_Name => $table->full_name); | |
11159 | foreach my $alias ($table->aliases) { | |
11160 | $scx_table->add_alias($alias->name); | |
11161 | } | |
11162 | } | |
11163 | } | |
11164 | ||
fbe1e607 KW |
11165 | sub filter_script_extensions_line { |
11166 | # The Scripts file comes with the full name for the scripts; the | |
11167 | # ScriptExtensions, with the short name. The final mapping file is a | |
11168 | # combination of these, and without adjustment, would have inconsistent | |
11169 | # entries. This filters the latter file to convert to full names. | |
11170 | # Entries look like this: | |
11171 | # 064B..0655 ; Arab Syrc # Mn [11] ARABIC FATHATAN..ARABIC HAMZA BELOW | |
11172 | ||
11173 | my @fields = split /\s*;\s*/; | |
11174 | my @full_names; | |
11175 | foreach my $short_name (split " ", $fields[1]) { | |
11176 | push @full_names, $script->table($short_name)->full_name; | |
11177 | } | |
11178 | $fields[1] = join " ", @full_names; | |
11179 | $_ = join "; ", @fields; | |
11180 | ||
11181 | return; | |
11182 | } | |
11183 | ||
dcd72625 KW |
11184 | sub setup_v6_name_alias { |
11185 | property_ref('Name_Alias')->add_map(7, 7, "ALERT"); | |
11186 | } | |
11187 | ||
99870f4d KW |
11188 | sub finish_Unicode() { |
11189 | # This routine should be called after all the Unicode files have been read | |
11190 | # in. It: | |
11191 | # 1) Adds the mappings for code points missing from the files which have | |
11192 | # defaults specified for them. | |
11193 | # 2) At this this point all mappings are known, so it computes the type of | |
11194 | # each property whose type hasn't been determined yet. | |
11195 | # 3) Calculates all the regular expression match tables based on the | |
11196 | # mappings. | |
11197 | # 3) Calculates and adds the tables which are defined by Unicode, but | |
11198 | # which aren't derived by them | |
11199 | ||
11200 | # For each property, fill in any missing mappings, and calculate the re | |
11201 | # match tables. If a property has more than one missing mapping, the | |
11202 | # default is a reference to a data structure, and requires data from other | |
11203 | # properties to resolve. The sort is used to cause these to be processed | |
11204 | # last, after all the other properties have been calculated. | |
11205 | # (Fortunately, the missing properties so far don't depend on each other.) | |
11206 | foreach my $property | |
11207 | (sort { (defined $a->default_map && ref $a->default_map) ? 1 : -1 } | |
11208 | property_ref('*')) | |
11209 | { | |
11210 | # $perl has been defined, but isn't one of the Unicode properties that | |
11211 | # need to be finished up. | |
11212 | next if $property == $perl; | |
11213 | ||
11214 | # Handle the properties that have more than one possible default | |
11215 | if (ref $property->default_map) { | |
11216 | my $default_map = $property->default_map; | |
11217 | ||
11218 | # These properties have stored in the default_map: | |
11219 | # One or more of: | |
11220 | # 1) A default map which applies to all code points in a | |
11221 | # certain class | |
11222 | # 2) an expression which will evaluate to the list of code | |
11223 | # points in that class | |
11224 | # And | |
11225 | # 3) the default map which applies to every other missing code | |
11226 | # point. | |
11227 | # | |
11228 | # Go through each list. | |
11229 | while (my ($default, $eval) = $default_map->get_next_defaults) { | |
11230 | ||
11231 | # Get the class list, and intersect it with all the so-far | |
11232 | # unspecified code points yielding all the code points | |
11233 | # in the class that haven't been specified. | |
11234 | my $list = eval $eval; | |
11235 | if ($@) { | |
11236 | Carp::my_carp("Can't set some defaults for missing code points for $property because eval '$eval' failed with '$@'"); | |
11237 | last; | |
11238 | } | |
11239 | ||
11240 | # Narrow down the list to just those code points we don't have | |
11241 | # maps for yet. | |
11242 | $list = $list & $property->inverse_list; | |
11243 | ||
11244 | # Add mappings to the property for each code point in the list | |
11245 | foreach my $range ($list->ranges) { | |
56343c78 KW |
11246 | $property->add_map($range->start, $range->end, $default, |
11247 | Replace => $CROAK); | |
99870f4d KW |
11248 | } |
11249 | } | |
11250 | ||
11251 | # All remaining code points have the other mapping. Set that up | |
11252 | # so the normal single-default mapping code will work on them | |
11253 | $property->set_default_map($default_map->other_default); | |
11254 | ||
11255 | # And fall through to do that | |
11256 | } | |
11257 | ||
11258 | # We should have enough data now to compute the type of the property. | |
11259 | $property->compute_type; | |
11260 | my $property_type = $property->type; | |
11261 | ||
11262 | next if ! $property->to_create_match_tables; | |
11263 | ||
11264 | # Here want to create match tables for this property | |
11265 | ||
11266 | # The Unicode db always (so far, and they claim into the future) have | |
11267 | # the default for missing entries in binary properties be 'N' (unless | |
11268 | # there is a '@missing' line that specifies otherwise) | |
11269 | if ($property_type == $BINARY && ! defined $property->default_map) { | |
11270 | $property->set_default_map('N'); | |
11271 | } | |
11272 | ||
11273 | # Add any remaining code points to the mapping, using the default for | |
5d7f7709 | 11274 | # missing code points. |
d8fb1cc3 | 11275 | my $default_table; |
99870f4d | 11276 | if (defined (my $default_map = $property->default_map)) { |
1520492f | 11277 | |
f4c2a127 | 11278 | # Make sure there is a match table for the default |
f4c2a127 KW |
11279 | if (! defined ($default_table = $property->table($default_map))) { |
11280 | $default_table = $property->add_match_table($default_map); | |
11281 | } | |
11282 | ||
a92d5c2e KW |
11283 | # And, if the property is binary, the default table will just |
11284 | # be the complement of the other table. | |
11285 | if ($property_type == $BINARY) { | |
11286 | my $non_default_table; | |
11287 | ||
11288 | # Find the non-default table. | |
11289 | for my $table ($property->tables) { | |
11290 | next if $table == $default_table; | |
11291 | $non_default_table = $table; | |
11292 | } | |
11293 | $default_table->set_complement($non_default_table); | |
11294 | } | |
862fd107 | 11295 | else { |
a92d5c2e | 11296 | |
3981d009 KW |
11297 | # This fills in any missing values with the default. It's not |
11298 | # necessary to do this with binary properties, as the default | |
11299 | # is defined completely in terms of the Y table. | |
6189eadc | 11300 | $property->add_map(0, $MAX_UNICODE_CODEPOINT, |
3981d009 | 11301 | $default_map, Replace => $NO); |
862fd107 | 11302 | } |
99870f4d KW |
11303 | } |
11304 | ||
11305 | # Have all we need to populate the match tables. | |
11306 | my $property_name = $property->name; | |
56557540 | 11307 | my $maps_should_be_defined = $property->pre_declared_maps; |
99870f4d KW |
11308 | foreach my $range ($property->ranges) { |
11309 | my $map = $range->value; | |
f5e9a6ca | 11310 | my $table = $property->table($map); |
99870f4d KW |
11311 | if (! defined $table) { |
11312 | ||
11313 | # Integral and rational property values are not necessarily | |
56557540 KW |
11314 | # defined in PropValueAliases, but whether all the other ones |
11315 | # should be depends on the property. | |
11316 | if ($maps_should_be_defined | |
99870f4d KW |
11317 | && $map !~ /^ -? \d+ ( \/ \d+ )? $/x) |
11318 | { | |
11319 | Carp::my_carp("Table '$property_name=$map' should have been defined. Defining it now.") | |
11320 | } | |
f5e9a6ca | 11321 | $table = $property->add_match_table($map); |
99870f4d KW |
11322 | } |
11323 | ||
862fd107 | 11324 | next if $table->complement != 0; # Don't need to populate these |
99870f4d KW |
11325 | $table->add_range($range->start, $range->end); |
11326 | } | |
11327 | ||
06f26c45 KW |
11328 | # A forced binary property has additional true/false tables which |
11329 | # should have been set up when it was forced into binary. The false | |
11330 | # table matches exactly the same set as the property's default table. | |
11331 | # The true table matches the complement of that. The false table is | |
11332 | # not the same as an additional set of aliases on top of the default | |
11333 | # table, so use 'set_equivalent_to'. If it were implemented as | |
11334 | # additional aliases, various things would have to be adjusted, but | |
11335 | # especially, if the user wants to get a list of names for the table | |
11336 | # using Unicode::UCD::prop_value_aliases(), s/he should get a | |
11337 | # different set depending on whether they want the default table or | |
11338 | # the false table. | |
11339 | if ($property_type == $FORCED_BINARY) { | |
11340 | $property->table('N')->set_equivalent_to($default_table, | |
11341 | Related => 1); | |
11342 | $property->table('Y')->set_complement($default_table); | |
11343 | } | |
11344 | ||
807807b7 KW |
11345 | # For Perl 5.6 compatibility, all properties matchable in regexes can |
11346 | # have an optional 'Is_' prefix. This is now done in utf8_heavy.pl. | |
11347 | # But warn if this creates a conflict with a (new) Unicode property | |
11348 | # name, although it appears that Unicode has made a decision never to | |
11349 | # begin a property name with 'Is_', so this shouldn't happen. | |
99870f4d KW |
11350 | foreach my $alias ($property->aliases) { |
11351 | my $Is_name = 'Is_' . $alias->name; | |
807807b7 | 11352 | if (defined (my $pre_existing = property_ref($Is_name))) { |
99870f4d | 11353 | Carp::my_carp(<<END |
807807b7 KW |
11354 | There is already an alias named $Is_name (from " . $pre_existing . "), so |
11355 | creating one for $property won't work. This is bad news. If it is not too | |
11356 | late, get Unicode to back off. Otherwise go back to the old scheme (findable | |
11357 | from the git blame log for this area of the code that suppressed individual | |
11358 | aliases that conflict with the new Unicode names. Proceeding anyway. | |
99870f4d KW |
11359 | END |
11360 | ); | |
99870f4d KW |
11361 | } |
11362 | } # End of loop through aliases for this property | |
11363 | } # End of loop through all Unicode properties. | |
11364 | ||
11365 | # Fill in the mappings that Unicode doesn't completely furnish. First the | |
11366 | # single letter major general categories. If Unicode were to start | |
11367 | # delivering the values, this would be redundant, but better that than to | |
11368 | # try to figure out if should skip and not get it right. Ths could happen | |
11369 | # if a new major category were to be introduced, and the hard-coded test | |
11370 | # wouldn't know about it. | |
11371 | # This routine depends on the standard names for the general categories | |
11372 | # being what it thinks they are, like 'Cn'. The major categories are the | |
11373 | # union of all the general category tables which have the same first | |
11374 | # letters. eg. L = Lu + Lt + Ll + Lo + Lm | |
11375 | foreach my $minor_table ($gc->tables) { | |
11376 | my $minor_name = $minor_table->name; | |
11377 | next if length $minor_name == 1; | |
11378 | if (length $minor_name != 2) { | |
11379 | Carp::my_carp_bug("Unexpected general category '$minor_name'. Skipped."); | |
11380 | next; | |
11381 | } | |
11382 | ||
11383 | my $major_name = uc(substr($minor_name, 0, 1)); | |
11384 | my $major_table = $gc->table($major_name); | |
11385 | $major_table += $minor_table; | |
11386 | } | |
11387 | ||
11388 | # LC is Ll, Lu, and Lt. (used to be L& or L_, but PropValueAliases.txt | |
11389 | # defines it as LC) | |
11390 | my $LC = $gc->table('LC'); | |
11391 | $LC->add_alias('L_', Status => $DISCOURAGED); # For backwards... | |
11392 | $LC->add_alias('L&', Status => $DISCOURAGED); # compatibility. | |
11393 | ||
11394 | ||
11395 | if ($LC->is_empty) { # Assume if not empty that Unicode has started to | |
11396 | # deliver the correct values in it | |
11397 | $LC->initialize($gc->table('Ll') + $gc->table('Lu')); | |
11398 | ||
11399 | # Lt not in release 1. | |
a5c376b7 KW |
11400 | if (defined $gc->table('Lt')) { |
11401 | $LC += $gc->table('Lt'); | |
11402 | $gc->table('Lt')->set_caseless_equivalent($LC); | |
11403 | } | |
99870f4d KW |
11404 | } |
11405 | $LC->add_description('[\p{Ll}\p{Lu}\p{Lt}]'); | |
11406 | ||
a5c376b7 KW |
11407 | $gc->table('Ll')->set_caseless_equivalent($LC); |
11408 | $gc->table('Lu')->set_caseless_equivalent($LC); | |
11409 | ||
99870f4d | 11410 | my $Cs = $gc->table('Cs'); |
99870f4d KW |
11411 | |
11412 | ||
11413 | # Folding information was introduced later into Unicode data. To get | |
11414 | # Perl's case ignore (/i) to work at all in releases that don't have | |
11415 | # folding, use the best available alternative, which is lower casing. | |
11416 | my $fold = property_ref('Simple_Case_Folding'); | |
11417 | if ($fold->is_empty) { | |
11418 | $fold->initialize(property_ref('Simple_Lowercase_Mapping')); | |
11419 | $fold->add_note(join_lines(<<END | |
11420 | WARNING: This table uses lower case as a substitute for missing fold | |
11421 | information | |
11422 | END | |
11423 | )); | |
11424 | } | |
11425 | ||
11426 | # Multiple-character mapping was introduced later into Unicode data. If | |
11427 | # missing, use the single-characters maps as best available alternative | |
11428 | foreach my $map (qw { Uppercase_Mapping | |
11429 | Lowercase_Mapping | |
11430 | Titlecase_Mapping | |
11431 | Case_Folding | |
11432 | } ) { | |
11433 | my $full = property_ref($map); | |
11434 | if ($full->is_empty) { | |
11435 | my $simple = property_ref('Simple_' . $map); | |
11436 | $full->initialize($simple); | |
11437 | $full->add_comment($simple->comment) if ($simple->comment); | |
11438 | $full->add_note(join_lines(<<END | |
11439 | WARNING: This table uses simple mapping (single-character only) as a | |
11440 | substitute for missing multiple-character information | |
11441 | END | |
11442 | )); | |
11443 | } | |
11444 | } | |
82aed44a KW |
11445 | |
11446 | # The Script_Extensions property started out as a clone of the Script | |
11447 | # property. But processing its data file caused some elements to be | |
11448 | # replaced with different data. (These elements were for the Common and | |
11449 | # Inherited properties.) This data is a qw() list of all the scripts that | |
11450 | # the code points in the given range are in. An example line is: | |
11451 | # 060C ; Arab Syrc Thaa # Po ARABIC COMMA | |
11452 | # | |
11453 | # The code above has created a new match table named "Arab Syrc Thaa" | |
11454 | # which contains 060C. (The cloned table started out with this code point | |
11455 | # mapping to "Common".) Now we add 060C to each of the Arab, Syrc, and | |
11456 | # Thaa match tables. Then we delete the now spurious "Arab Syrc Thaa" | |
11457 | # match table. This is repeated for all these tables and ranges. The map | |
11458 | # data is retained in the map table for reference, but the spurious match | |
11459 | # tables are deleted. | |
11460 | ||
11461 | my $scx = property_ref("Script_Extensions"); | |
d53a7e7d | 11462 | if (defined $scx) { |
c3a37f64 KW |
11463 | foreach my $table ($scx->tables) { |
11464 | next unless $table->name =~ /\s/; # All the new and only the new | |
11465 | # tables have a space in their | |
11466 | # names | |
11467 | my @scripts = split /\s+/, $table->name; | |
11468 | foreach my $script (@scripts) { | |
11469 | my $script_table = $scx->table($script); | |
11470 | $script_table += $table; | |
11471 | } | |
11472 | $scx->delete_match_table($table); | |
82aed44a | 11473 | } |
d53a7e7d | 11474 | } |
82aed44a KW |
11475 | |
11476 | return; | |
99870f4d KW |
11477 | } |
11478 | ||
11479 | sub compile_perl() { | |
11480 | # Create perl-defined tables. Almost all are part of the pseudo-property | |
11481 | # named 'perl' internally to this program. Many of these are recommended | |
11482 | # in UTS#18 "Unicode Regular Expressions", and their derivations are based | |
11483 | # on those found there. | |
11484 | # Almost all of these are equivalent to some Unicode property. | |
11485 | # A number of these properties have equivalents restricted to the ASCII | |
11486 | # range, with their names prefaced by 'Posix', to signify that these match | |
11487 | # what the Posix standard says they should match. A couple are | |
11488 | # effectively this, but the name doesn't have 'Posix' in it because there | |
cbc24f92 KW |
11489 | # just isn't any Posix equivalent. 'XPosix' are the Posix tables extended |
11490 | # to the full Unicode range, by our guesses as to what is appropriate. | |
99870f4d KW |
11491 | |
11492 | # 'Any' is all code points. As an error check, instead of just setting it | |
11493 | # to be that, construct it to be the union of all the major categories | |
7fc6cb55 | 11494 | $Any = $perl->add_match_table('Any', |
6189eadc | 11495 | Description => "[\\x{0000}-\\x{$MAX_UNICODE_CODEPOINT_STRING}]", |
99870f4d KW |
11496 | Matches_All => 1); |
11497 | ||
11498 | foreach my $major_table ($gc->tables) { | |
11499 | ||
11500 | # Major categories are the ones with single letter names. | |
11501 | next if length($major_table->name) != 1; | |
11502 | ||
11503 | $Any += $major_table; | |
11504 | } | |
11505 | ||
6189eadc | 11506 | if ($Any->max != $MAX_UNICODE_CODEPOINT) { |
99870f4d KW |
11507 | Carp::my_carp_bug("Generated highest code point (" |
11508 | . sprintf("%X", $Any->max) | |
6189eadc | 11509 | . ") doesn't match expected value $MAX_UNICODE_CODEPOINT_STRING.") |
99870f4d KW |
11510 | } |
11511 | if ($Any->range_count != 1 || $Any->min != 0) { | |
11512 | Carp::my_carp_bug("Generated table 'Any' doesn't match all code points.") | |
11513 | } | |
11514 | ||
11515 | $Any->add_alias('All'); | |
11516 | ||
11517 | # Assigned is the opposite of gc=unassigned | |
11518 | my $Assigned = $perl->add_match_table('Assigned', | |
11519 | Description => "All assigned code points", | |
11520 | Initialize => ~ $gc->table('Unassigned'), | |
11521 | ); | |
11522 | ||
11523 | # Our internal-only property should be treated as more than just a | |
8050d00f | 11524 | # synonym; grandfather it in to the pod. |
33e96e72 | 11525 | $perl->add_match_table('_CombAbove', Re_Pod_Entry => 1) |
99870f4d KW |
11526 | ->set_equivalent_to(property_ref('ccc')->table('Above'), |
11527 | Related => 1); | |
11528 | ||
11529 | my $ASCII = $perl->add_match_table('ASCII', Description => '[[:ASCII:]]'); | |
11530 | if (defined $block) { # This is equivalent to the block if have it. | |
11531 | my $Unicode_ASCII = $block->table('Basic_Latin'); | |
11532 | if (defined $Unicode_ASCII && ! $Unicode_ASCII->is_empty) { | |
11533 | $ASCII->set_equivalent_to($Unicode_ASCII, Related => 1); | |
11534 | } | |
11535 | } | |
11536 | ||
11537 | # Very early releases didn't have blocks, so initialize ASCII ourselves if | |
11538 | # necessary | |
11539 | if ($ASCII->is_empty) { | |
11540 | $ASCII->initialize([ 0..127 ]); | |
11541 | } | |
11542 | ||
99870f4d KW |
11543 | # Get the best available case definitions. Early Unicode versions didn't |
11544 | # have Uppercase and Lowercase defined, so use the general category | |
11545 | # instead for them. | |
11546 | my $Lower = $perl->add_match_table('Lower'); | |
11547 | my $Unicode_Lower = property_ref('Lowercase'); | |
11548 | if (defined $Unicode_Lower && ! $Unicode_Lower->is_empty) { | |
11549 | $Lower->set_equivalent_to($Unicode_Lower->table('Y'), Related => 1); | |
a5c376b7 KW |
11550 | $Unicode_Lower->table('Y')->set_caseless_equivalent(property_ref('Cased')->table('Y')); |
11551 | $Unicode_Lower->table('N')->set_caseless_equivalent(property_ref('Cased')->table('N')); | |
11552 | $Lower->set_caseless_equivalent(property_ref('Cased')->table('Y')); | |
11553 | ||
99870f4d KW |
11554 | } |
11555 | else { | |
11556 | $Lower->set_equivalent_to($gc->table('Lowercase_Letter'), | |
11557 | Related => 1); | |
11558 | } | |
cbc24f92 | 11559 | $Lower->add_alias('XPosixLower'); |
a5c376b7 | 11560 | my $Posix_Lower = $perl->add_match_table("PosixLower", |
ad5e8af1 KW |
11561 | Description => "[a-z]", |
11562 | Initialize => $Lower & $ASCII, | |
11563 | ); | |
99870f4d KW |
11564 | |
11565 | my $Upper = $perl->add_match_table('Upper'); | |
11566 | my $Unicode_Upper = property_ref('Uppercase'); | |
11567 | if (defined $Unicode_Upper && ! $Unicode_Upper->is_empty) { | |
11568 | $Upper->set_equivalent_to($Unicode_Upper->table('Y'), Related => 1); | |
a5c376b7 KW |
11569 | $Unicode_Upper->table('Y')->set_caseless_equivalent(property_ref('Cased')->table('Y')); |
11570 | $Unicode_Upper->table('N')->set_caseless_equivalent(property_ref('Cased')->table('N')); | |
11571 | $Upper->set_caseless_equivalent(property_ref('Cased')->table('Y')); | |
99870f4d KW |
11572 | } |
11573 | else { | |
11574 | $Upper->set_equivalent_to($gc->table('Uppercase_Letter'), | |
11575 | Related => 1); | |
11576 | } | |
cbc24f92 | 11577 | $Upper->add_alias('XPosixUpper'); |
a5c376b7 | 11578 | my $Posix_Upper = $perl->add_match_table("PosixUpper", |
ad5e8af1 KW |
11579 | Description => "[A-Z]", |
11580 | Initialize => $Upper & $ASCII, | |
11581 | ); | |
99870f4d KW |
11582 | |
11583 | # Earliest releases didn't have title case. Initialize it to empty if not | |
11584 | # otherwise present | |
4364919a KW |
11585 | my $Title = $perl->add_match_table('Title', Full_Name => 'Titlecase', |
11586 | Description => '(= \p{Gc=Lt})'); | |
99870f4d | 11587 | my $lt = $gc->table('Lt'); |
a5c376b7 KW |
11588 | |
11589 | # Earlier versions of mktables had this related to $lt since they have | |
c12f2655 KW |
11590 | # identical code points, but their caseless equivalents are not the same, |
11591 | # one being 'Cased' and the other being 'LC', and so now must be kept as | |
11592 | # separate entities. | |
a5c376b7 | 11593 | $Title += $lt if defined $lt; |
99870f4d KW |
11594 | |
11595 | # If this Unicode version doesn't have Cased, set up our own. From | |
11596 | # Unicode 5.1: Definition D120: A character C is defined to be cased if | |
11597 | # and only if C has the Lowercase or Uppercase property or has a | |
11598 | # General_Category value of Titlecase_Letter. | |
a5c376b7 KW |
11599 | my $Unicode_Cased = property_ref('Cased'); |
11600 | unless (defined $Unicode_Cased) { | |
99870f4d KW |
11601 | my $cased = $perl->add_match_table('Cased', |
11602 | Initialize => $Lower + $Upper + $Title, | |
11603 | Description => 'Uppercase or Lowercase or Titlecase', | |
11604 | ); | |
a5c376b7 | 11605 | $Unicode_Cased = $cased; |
99870f4d | 11606 | } |
a5c376b7 | 11607 | $Title->set_caseless_equivalent($Unicode_Cased->table('Y')); |
99870f4d KW |
11608 | |
11609 | # Similarly, set up our own Case_Ignorable property if this Unicode | |
11610 | # version doesn't have it. From Unicode 5.1: Definition D121: A character | |
11611 | # C is defined to be case-ignorable if C has the value MidLetter or the | |
11612 | # value MidNumLet for the Word_Break property or its General_Category is | |
11613 | # one of Nonspacing_Mark (Mn), Enclosing_Mark (Me), Format (Cf), | |
11614 | # Modifier_Letter (Lm), or Modifier_Symbol (Sk). | |
11615 | ||
8050d00f KW |
11616 | # Perl has long had an internal-only alias for this property; grandfather |
11617 | # it in to the pod, but discourage its use. | |
11618 | my $perl_case_ignorable = $perl->add_match_table('_Case_Ignorable', | |
33e96e72 | 11619 | Re_Pod_Entry => 1); |
99870f4d KW |
11620 | my $case_ignorable = property_ref('Case_Ignorable'); |
11621 | if (defined $case_ignorable && ! $case_ignorable->is_empty) { | |
11622 | $perl_case_ignorable->set_equivalent_to($case_ignorable->table('Y'), | |
11623 | Related => 1); | |
11624 | } | |
11625 | else { | |
11626 | ||
11627 | $perl_case_ignorable->initialize($gc->table('Mn') + $gc->table('Lm')); | |
11628 | ||
11629 | # The following three properties are not in early releases | |
11630 | $perl_case_ignorable += $gc->table('Me') if defined $gc->table('Me'); | |
11631 | $perl_case_ignorable += $gc->table('Cf') if defined $gc->table('Cf'); | |
11632 | $perl_case_ignorable += $gc->table('Sk') if defined $gc->table('Sk'); | |
11633 | ||
11634 | # For versions 4.1 - 5.0, there is no MidNumLet property, and | |
11635 | # correspondingly the case-ignorable definition lacks that one. For | |
11636 | # 4.0, it appears that it was meant to be the same definition, but was | |
11637 | # inadvertently omitted from the standard's text, so add it if the | |
11638 | # property actually is there | |
11639 | my $wb = property_ref('Word_Break'); | |
11640 | if (defined $wb) { | |
11641 | my $midlet = $wb->table('MidLetter'); | |
11642 | $perl_case_ignorable += $midlet if defined $midlet; | |
11643 | my $midnumlet = $wb->table('MidNumLet'); | |
11644 | $perl_case_ignorable += $midnumlet if defined $midnumlet; | |
11645 | } | |
11646 | else { | |
11647 | ||
11648 | # In earlier versions of the standard, instead of the above two | |
11649 | # properties , just the following characters were used: | |
11650 | $perl_case_ignorable += 0x0027 # APOSTROPHE | |
11651 | + 0x00AD # SOFT HYPHEN (SHY) | |
11652 | + 0x2019; # RIGHT SINGLE QUOTATION MARK | |
11653 | } | |
11654 | } | |
11655 | ||
11656 | # The remaining perl defined tables are mostly based on Unicode TR 18, | |
11657 | # "Annex C: Compatibility Properties". All of these have two versions, | |
11658 | # one whose name generally begins with Posix that is posix-compliant, and | |
11659 | # one that matches Unicode characters beyond the Posix, ASCII range | |
11660 | ||
ad5e8af1 | 11661 | my $Alpha = $perl->add_match_table('Alpha'); |
99870f4d KW |
11662 | |
11663 | # Alphabetic was not present in early releases | |
11664 | my $Alphabetic = property_ref('Alphabetic'); | |
11665 | if (defined $Alphabetic && ! $Alphabetic->is_empty) { | |
11666 | $Alpha->set_equivalent_to($Alphabetic->table('Y'), Related => 1); | |
11667 | } | |
11668 | else { | |
11669 | ||
11670 | # For early releases, we don't get it exactly right. The below | |
11671 | # includes more than it should, which in 5.2 terms is: L + Nl + | |
11672 | # Other_Alphabetic. Other_Alphabetic contains many characters from | |
11673 | # Mn and Mc. It's better to match more than we should, than less than | |
11674 | # we should. | |
11675 | $Alpha->initialize($gc->table('Letter') | |
11676 | + $gc->table('Mn') | |
11677 | + $gc->table('Mc')); | |
11678 | $Alpha += $gc->table('Nl') if defined $gc->table('Nl'); | |
ad5e8af1 | 11679 | $Alpha->add_description('Alphabetic'); |
99870f4d | 11680 | } |
cbc24f92 | 11681 | $Alpha->add_alias('XPosixAlpha'); |
a5c376b7 | 11682 | my $Posix_Alpha = $perl->add_match_table("PosixAlpha", |
ad5e8af1 KW |
11683 | Description => "[A-Za-z]", |
11684 | Initialize => $Alpha & $ASCII, | |
11685 | ); | |
a5c376b7 KW |
11686 | $Posix_Upper->set_caseless_equivalent($Posix_Alpha); |
11687 | $Posix_Lower->set_caseless_equivalent($Posix_Alpha); | |
99870f4d KW |
11688 | |
11689 | my $Alnum = $perl->add_match_table('Alnum', | |
ad5e8af1 | 11690 | Description => 'Alphabetic and (Decimal) Numeric', |
99870f4d KW |
11691 | Initialize => $Alpha + $gc->table('Decimal_Number'), |
11692 | ); | |
cbc24f92 | 11693 | $Alnum->add_alias('XPosixAlnum'); |
ad5e8af1 KW |
11694 | $perl->add_match_table("PosixAlnum", |
11695 | Description => "[A-Za-z0-9]", | |
11696 | Initialize => $Alnum & $ASCII, | |
11697 | ); | |
99870f4d KW |
11698 | |
11699 | my $Word = $perl->add_match_table('Word', | |
d35dd6c6 KW |
11700 | Description => '\w, including beyond ASCII;' |
11701 | . ' = \p{Alnum} + \pM + \p{Pc}', | |
99870f4d KW |
11702 | Initialize => $Alnum + $gc->table('Mark'), |
11703 | ); | |
cbc24f92 | 11704 | $Word->add_alias('XPosixWord'); |
99870f4d KW |
11705 | my $Pc = $gc->table('Connector_Punctuation'); # 'Pc' Not in release 1 |
11706 | $Word += $Pc if defined $Pc; | |
11707 | ||
f38f76ae | 11708 | # This is a Perl extension, so the name doesn't begin with Posix. |
cbc24f92 | 11709 | my $PerlWord = $perl->add_match_table('PerlWord', |
99870f4d KW |
11710 | Description => '\w, restricted to ASCII = [A-Za-z0-9_]', |
11711 | Initialize => $Word & $ASCII, | |
11712 | ); | |
cbc24f92 | 11713 | $PerlWord->add_alias('PosixWord'); |
99870f4d KW |
11714 | |
11715 | my $Blank = $perl->add_match_table('Blank', | |
11716 | Description => '\h, Horizontal white space', | |
11717 | ||
11718 | # 200B is Zero Width Space which is for line | |
11719 | # break control, and was listed as | |
11720 | # Space_Separator in early releases | |
11721 | Initialize => $gc->table('Space_Separator') | |
11722 | + 0x0009 # TAB | |
11723 | - 0x200B, # ZWSP | |
11724 | ); | |
11725 | $Blank->add_alias('HorizSpace'); # Another name for it. | |
cbc24f92 | 11726 | $Blank->add_alias('XPosixBlank'); |
ad5e8af1 KW |
11727 | $perl->add_match_table("PosixBlank", |
11728 | Description => "\\t and ' '", | |
11729 | Initialize => $Blank & $ASCII, | |
11730 | ); | |
99870f4d KW |
11731 | |
11732 | my $VertSpace = $perl->add_match_table('VertSpace', | |
11733 | Description => '\v', | |
11734 | Initialize => $gc->table('Line_Separator') | |
11735 | + $gc->table('Paragraph_Separator') | |
11736 | + 0x000A # LINE FEED | |
11737 | + 0x000B # VERTICAL TAB | |
11738 | + 0x000C # FORM FEED | |
11739 | + 0x000D # CARRIAGE RETURN | |
11740 | + 0x0085, # NEL | |
11741 | ); | |
11742 | # No Posix equivalent for vertical space | |
11743 | ||
11744 | my $Space = $perl->add_match_table('Space', | |
ad5e8af1 KW |
11745 | Description => '\s including beyond ASCII plus vertical tab', |
11746 | Initialize => $Blank + $VertSpace, | |
99870f4d | 11747 | ); |
cbc24f92 | 11748 | $Space->add_alias('XPosixSpace'); |
ad5e8af1 | 11749 | $perl->add_match_table("PosixSpace", |
f38f76ae | 11750 | Description => "\\t, \\n, \\cK, \\f, \\r, and ' '. (\\cK is vertical tab)", |
ad5e8af1 KW |
11751 | Initialize => $Space & $ASCII, |
11752 | ); | |
99870f4d KW |
11753 | |
11754 | # Perl's traditional space doesn't include Vertical Tab | |
cbc24f92 | 11755 | my $XPerlSpace = $perl->add_match_table('XPerlSpace', |
99870f4d KW |
11756 | Description => '\s, including beyond ASCII', |
11757 | Initialize => $Space - 0x000B, | |
11758 | ); | |
cbc24f92 KW |
11759 | $XPerlSpace->add_alias('SpacePerl'); # A pre-existing synonym |
11760 | my $PerlSpace = $perl->add_match_table('PerlSpace', | |
de25ec47 KW |
11761 | Description => '\s, restricted to ASCII = [ \f\n\r\t]', |
11762 | Initialize => $XPerlSpace & $ASCII, | |
99870f4d KW |
11763 | ); |
11764 | ||
cbc24f92 | 11765 | |
99870f4d | 11766 | my $Cntrl = $perl->add_match_table('Cntrl', |
ad5e8af1 | 11767 | Description => 'Control characters'); |
99870f4d | 11768 | $Cntrl->set_equivalent_to($gc->table('Cc'), Related => 1); |
cbc24f92 | 11769 | $Cntrl->add_alias('XPosixCntrl'); |
ad5e8af1 | 11770 | $perl->add_match_table("PosixCntrl", |
f38f76ae | 11771 | Description => "ASCII control characters: NUL, SOH, STX, ETX, EOT, ENQ, ACK, BEL, BS, HT, LF, VT, FF, CR, SO, SI, DLE, DC1, DC2, DC3, DC4, NAK, SYN, ETB, CAN, EOM, SUB, ESC, FS, GS, RS, US, and DEL", |
ad5e8af1 KW |
11772 | Initialize => $Cntrl & $ASCII, |
11773 | ); | |
99870f4d KW |
11774 | |
11775 | # $controls is a temporary used to construct Graph. | |
11776 | my $controls = Range_List->new(Initialize => $gc->table('Unassigned') | |
11777 | + $gc->table('Control')); | |
11778 | # Cs not in release 1 | |
11779 | $controls += $gc->table('Surrogate') if defined $gc->table('Surrogate'); | |
11780 | ||
11781 | # Graph is ~space & ~(Cc|Cs|Cn) = ~(space + $controls) | |
11782 | my $Graph = $perl->add_match_table('Graph', | |
ad5e8af1 | 11783 | Description => 'Characters that are graphical', |
99870f4d KW |
11784 | Initialize => ~ ($Space + $controls), |
11785 | ); | |
cbc24f92 | 11786 | $Graph->add_alias('XPosixGraph'); |
ad5e8af1 | 11787 | $perl->add_match_table("PosixGraph", |
f38f76ae KW |
11788 | Description => |
11789 | '[-!"#$%&\'()*+,./:;<>?@[\\\]^_`{|}~0-9A-Za-z]', | |
ad5e8af1 KW |
11790 | Initialize => $Graph & $ASCII, |
11791 | ); | |
99870f4d | 11792 | |
3e20195b | 11793 | $print = $perl->add_match_table('Print', |
ad5e8af1 | 11794 | Description => 'Characters that are graphical plus space characters (but no controls)', |
ae5b72c8 | 11795 | Initialize => $Blank + $Graph - $gc->table('Control'), |
99870f4d | 11796 | ); |
cbc24f92 | 11797 | $print->add_alias('XPosixPrint'); |
ad5e8af1 | 11798 | $perl->add_match_table("PosixPrint", |
66fd7fd0 | 11799 | Description => |
f38f76ae | 11800 | '[- 0-9A-Za-z!"#$%&\'()*+,./:;<>?@[\\\]^_`{|}~]', |
3e20195b | 11801 | Initialize => $print & $ASCII, |
ad5e8af1 | 11802 | ); |
99870f4d KW |
11803 | |
11804 | my $Punct = $perl->add_match_table('Punct'); | |
11805 | $Punct->set_equivalent_to($gc->table('Punctuation'), Related => 1); | |
11806 | ||
11807 | # \p{punct} doesn't include the symbols, which posix does | |
cbc24f92 KW |
11808 | my $XPosixPunct = $perl->add_match_table('XPosixPunct', |
11809 | Description => '\p{Punct} + ASCII-range \p{Symbol}', | |
11810 | Initialize => $gc->table('Punctuation') | |
11811 | + ($ASCII & $gc->table('Symbol')), | |
bb080638 | 11812 | Perl_Extension => 1 |
cbc24f92 | 11813 | ); |
bb080638 | 11814 | $perl->add_match_table('PosixPunct', Perl_Extension => 1, |
f38f76ae | 11815 | Description => '[-!"#$%&\'()*+,./:;<>?@[\\\]^_`{|}~]', |
cbc24f92 | 11816 | Initialize => $ASCII & $XPosixPunct, |
ad5e8af1 | 11817 | ); |
99870f4d KW |
11818 | |
11819 | my $Digit = $perl->add_match_table('Digit', | |
f3a73f6e | 11820 | Description => '[0-9] + all other decimal digits'); |
99870f4d | 11821 | $Digit->set_equivalent_to($gc->table('Decimal_Number'), Related => 1); |
cbc24f92 | 11822 | $Digit->add_alias('XPosixDigit'); |
ad5e8af1 KW |
11823 | my $PosixDigit = $perl->add_match_table("PosixDigit", |
11824 | Description => '[0-9]', | |
11825 | Initialize => $Digit & $ASCII, | |
11826 | ); | |
99870f4d | 11827 | |
eadadd41 KW |
11828 | # Hex_Digit was not present in first release |
11829 | my $Xdigit = $perl->add_match_table('XDigit'); | |
cbc24f92 | 11830 | $Xdigit->add_alias('XPosixXDigit'); |
eadadd41 KW |
11831 | my $Hex = property_ref('Hex_Digit'); |
11832 | if (defined $Hex && ! $Hex->is_empty) { | |
11833 | $Xdigit->set_equivalent_to($Hex->table('Y'), Related => 1); | |
99870f4d KW |
11834 | } |
11835 | else { | |
eadadd41 KW |
11836 | # (Have to use hex instead of e.g. '0', because could be running on an |
11837 | # non-ASCII machine, and we want the Unicode (ASCII) values) | |
11838 | $Xdigit->initialize([ 0x30..0x39, 0x41..0x46, 0x61..0x66, | |
11839 | 0xFF10..0xFF19, 0xFF21..0xFF26, 0xFF41..0xFF46]); | |
11840 | $Xdigit->add_description('[0-9A-Fa-f] and corresponding fullwidth versions, like U+FF10: FULLWIDTH DIGIT ZERO'); | |
99870f4d | 11841 | } |
4efcc33b KW |
11842 | |
11843 | # AHex was not present in early releases | |
11844 | my $PosixXDigit = $perl->add_match_table('PosixXDigit'); | |
11845 | my $AHex = property_ref('ASCII_Hex_Digit'); | |
11846 | if (defined $AHex && ! $AHex->is_empty) { | |
11847 | $PosixXDigit->set_equivalent_to($AHex->table('Y'), Related => 1); | |
11848 | } | |
11849 | else { | |
11850 | $PosixXDigit->initialize($Xdigit & $ASCII); | |
11851 | } | |
11852 | $PosixXDigit->add_description('[0-9A-Fa-f]'); | |
99870f4d | 11853 | |
99870f4d KW |
11854 | my $dt = property_ref('Decomposition_Type'); |
11855 | $dt->add_match_table('Non_Canon', Full_Name => 'Non_Canonical', | |
11856 | Initialize => ~ ($dt->table('None') + $dt->table('Canonical')), | |
11857 | Perl_Extension => 1, | |
d57ccc9a | 11858 | Note => 'Union of all non-canonical decompositions', |
99870f4d KW |
11859 | ); |
11860 | ||
11861 | # _CanonDCIJ is equivalent to Soft_Dotted, but if on a release earlier | |
11862 | # than SD appeared, construct it ourselves, based on the first release SD | |
8050d00f | 11863 | # was in. A pod entry is grandfathered in for it |
33e96e72 | 11864 | my $CanonDCIJ = $perl->add_match_table('_CanonDCIJ', Re_Pod_Entry => 1, |
6ba2d696 | 11865 | Perl_Extension => 1, Internal_Only => 1); |
99870f4d KW |
11866 | my $soft_dotted = property_ref('Soft_Dotted'); |
11867 | if (defined $soft_dotted && ! $soft_dotted->is_empty) { | |
11868 | $CanonDCIJ->set_equivalent_to($soft_dotted->table('Y'), Related => 1); | |
11869 | } | |
11870 | else { | |
11871 | ||
11872 | # This list came from 3.2 Soft_Dotted. | |
11873 | $CanonDCIJ->initialize([ 0x0069, | |
11874 | 0x006A, | |
11875 | 0x012F, | |
11876 | 0x0268, | |
11877 | 0x0456, | |
11878 | 0x0458, | |
11879 | 0x1E2D, | |
11880 | 0x1ECB, | |
11881 | ]); | |
11882 | $CanonDCIJ = $CanonDCIJ & $Assigned; | |
11883 | } | |
11884 | ||
f86864ac | 11885 | # These are used in Unicode's definition of \X |
6ba2d696 KW |
11886 | my $begin = $perl->add_match_table('_X_Begin', Perl_Extension => 1, |
11887 | Internal_Only => 1); | |
11888 | my $extend = $perl->add_match_table('_X_Extend', Perl_Extension => 1, | |
11889 | Internal_Only => 1); | |
37e2e78e | 11890 | |
ee24a51c KW |
11891 | # For backward compatibility, Perl has its own definition for IDStart |
11892 | # First, we include the underscore, and then the regular XID_Start also | |
11893 | # have to be Words | |
11894 | $perl->add_match_table('_Perl_IDStart', | |
11895 | Perl_Extension => 1, | |
11896 | Internal_Only => 1, | |
11897 | Initialize => | |
11898 | ord('_') | |
11899 | + (property_ref('XID_Start')->table('Y') & $Word) | |
11900 | ); | |
11901 | ||
99870f4d | 11902 | my $gcb = property_ref('Grapheme_Cluster_Break'); |
37e2e78e | 11903 | |
678f13d5 | 11904 | # The 'extended' grapheme cluster came in 5.1. The non-extended |
37e2e78e KW |
11905 | # definition differs too much from the traditional Perl one to use. |
11906 | if (defined $gcb && defined $gcb->table('SpacingMark')) { | |
11907 | ||
11908 | # Note that assumes HST is defined; it came in an earlier release than | |
11909 | # GCB. In the line below, two negatives means: yes hangul | |
11910 | $begin += ~ property_ref('Hangul_Syllable_Type') | |
11911 | ->table('Not_Applicable') | |
11912 | + ~ ($gcb->table('Control') | |
11913 | + $gcb->table('CR') | |
11914 | + $gcb->table('LF')); | |
11915 | $begin->add_comment('For use in \X; matches: Hangul_Syllable | ! Control'); | |
11916 | ||
11917 | $extend += $gcb->table('Extend') + $gcb->table('SpacingMark'); | |
11918 | $extend->add_comment('For use in \X; matches: Extend | SpacingMark'); | |
99870f4d KW |
11919 | } |
11920 | else { # Old definition, used on early releases. | |
f86864ac | 11921 | $extend += $gc->table('Mark') |
37e2e78e KW |
11922 | + 0x200C # ZWNJ |
11923 | + 0x200D; # ZWJ | |
11924 | $begin += ~ $extend; | |
11925 | ||
11926 | # Here we may have a release that has the regular grapheme cluster | |
11927 | # defined, or a release that doesn't have anything defined. | |
11928 | # We set things up so the Perl core degrades gracefully, possibly with | |
11929 | # placeholders that match nothing. | |
11930 | ||
11931 | if (! defined $gcb) { | |
11932 | $gcb = Property->new('GCB', Status => $PLACEHOLDER); | |
11933 | } | |
11934 | my $hst = property_ref('HST'); | |
11935 | if (!defined $hst) { | |
11936 | $hst = Property->new('HST', Status => $PLACEHOLDER); | |
11937 | $hst->add_match_table('Not_Applicable', | |
11938 | Initialize => $Any, | |
11939 | Matches_All => 1); | |
11940 | } | |
11941 | ||
11942 | # On some releases, here we may not have the needed tables for the | |
11943 | # perl core, in some releases we may. | |
11944 | foreach my $name (qw{ L LV LVT T V prepend }) { | |
11945 | my $table = $gcb->table($name); | |
11946 | if (! defined $table) { | |
11947 | $table = $gcb->add_match_table($name); | |
11948 | push @tables_that_may_be_empty, $table->complete_name; | |
11949 | } | |
11950 | ||
11951 | # The HST property predates the GCB one, and has identical tables | |
11952 | # for some of them, so use it if we can. | |
11953 | if ($table->is_empty | |
11954 | && defined $hst | |
11955 | && defined $hst->table($name)) | |
11956 | { | |
11957 | $table += $hst->table($name); | |
11958 | } | |
11959 | } | |
11960 | } | |
11961 | ||
11962 | # More GCB. If we found some hangul syllables, populate a combined | |
11963 | # table. | |
6ba2d696 | 11964 | my $lv_lvt_v = $perl->add_match_table('_X_LV_LVT_V', Perl_Extension => 1, Internal_Only => 1); |
37e2e78e KW |
11965 | my $LV = $gcb->table('LV'); |
11966 | if ($LV->is_empty) { | |
11967 | push @tables_that_may_be_empty, $lv_lvt_v->complete_name; | |
11968 | } else { | |
11969 | $lv_lvt_v += $LV + $gcb->table('LVT') + $gcb->table('V'); | |
11970 | $lv_lvt_v->add_comment('For use in \X; matches: HST=LV | HST=LVT | HST=V'); | |
99870f4d KW |
11971 | } |
11972 | ||
28093d0e | 11973 | # Was previously constructed to contain both Name and Unicode_1_Name |
99870f4d KW |
11974 | my @composition = ('Name', 'Unicode_1_Name'); |
11975 | ||
11976 | if (@named_sequences) { | |
11977 | push @composition, 'Named_Sequence'; | |
11978 | foreach my $sequence (@named_sequences) { | |
11979 | $perl_charname->add_anomalous_entry($sequence); | |
11980 | } | |
11981 | } | |
11982 | ||
11983 | my $alias_sentence = ""; | |
11984 | my $alias = property_ref('Name_Alias'); | |
11985 | if (defined $alias) { | |
11986 | push @composition, 'Name_Alias'; | |
11987 | $alias->reset_each_range; | |
11988 | while (my ($range) = $alias->each_range) { | |
11989 | next if $range->value eq ""; | |
11990 | if ($range->start != $range->end) { | |
11991 | Carp::my_carp("Expecting only one code point in the range $range. Just to keep going, using just the first code point;"); | |
11992 | } | |
11993 | $perl_charname->add_duplicate($range->start, $range->value); | |
11994 | } | |
11995 | $alias_sentence = <<END; | |
11996 | The Name_Alias property adds duplicate code point entries with a corrected | |
11997 | name. The original (less correct, but still valid) name will be physically | |
53d84487 | 11998 | last. |
99870f4d KW |
11999 | END |
12000 | } | |
12001 | my $comment; | |
12002 | if (@composition <= 2) { # Always at least 2 | |
12003 | $comment = join " and ", @composition; | |
12004 | } | |
12005 | else { | |
12006 | $comment = join ", ", @composition[0 .. scalar @composition - 2]; | |
12007 | $comment .= ", and $composition[-1]"; | |
12008 | } | |
12009 | ||
99870f4d KW |
12010 | $perl_charname->add_comment(join_lines( <<END |
12011 | This file is for charnames.pm. It is the union of the $comment properties. | |
12012 | Unicode_1_Name entries are used only for otherwise nameless code | |
12013 | points. | |
12014 | $alias_sentence | |
a03f0b9f KW |
12015 | This file doesn't include the algorithmically determinable names. For those, |
12016 | use 'unicore/Name.pm' | |
12017 | END | |
12018 | )); | |
12019 | property_ref('Name')->add_comment(join_lines( <<END | |
12020 | This file doesn't include the algorithmically determinable names. For those, | |
12021 | use 'unicore/Name.pm' | |
99870f4d KW |
12022 | END |
12023 | )); | |
12024 | ||
99870f4d KW |
12025 | # Construct the Present_In property from the Age property. |
12026 | if (-e 'DAge.txt' && defined (my $age = property_ref('Age'))) { | |
12027 | my $default_map = $age->default_map; | |
12028 | my $in = Property->new('In', | |
12029 | Default_Map => $default_map, | |
12030 | Full_Name => "Present_In", | |
99870f4d KW |
12031 | Perl_Extension => 1, |
12032 | Type => $ENUM, | |
12033 | Initialize => $age, | |
12034 | ); | |
12035 | $in->add_comment(join_lines(<<END | |
c12f2655 | 12036 | THIS FILE SHOULD NOT BE USED FOR ANY PURPOSE. The values in this file are the |
99870f4d KW |
12037 | same as for $age, and not for what $in really means. This is because anything |
12038 | defined in a given release should have multiple values: that release and all | |
12039 | higher ones. But only one value per code point can be represented in a table | |
12040 | like this. | |
12041 | END | |
12042 | )); | |
12043 | ||
12044 | # The Age tables are named like 1.5, 2.0, 2.1, .... Sort so that the | |
12045 | # lowest numbered (earliest) come first, with the non-numeric one | |
12046 | # last. | |
12047 | my ($first_age, @rest_ages) = sort { ($a->name !~ /^[\d.]*$/) | |
12048 | ? 1 | |
12049 | : ($b->name !~ /^[\d.]*$/) | |
12050 | ? -1 | |
12051 | : $a->name <=> $b->name | |
12052 | } $age->tables; | |
12053 | ||
12054 | # The Present_In property is the cumulative age properties. The first | |
12055 | # one hence is identical to the first age one. | |
12056 | my $previous_in = $in->add_match_table($first_age->name); | |
12057 | $previous_in->set_equivalent_to($first_age, Related => 1); | |
12058 | ||
12059 | my $description_start = "Code point's usage introduced in version "; | |
12060 | $first_age->add_description($description_start . $first_age->name); | |
12061 | ||
98dc9551 | 12062 | # To construct the accumulated values, for each of the age tables |
99870f4d KW |
12063 | # starting with the 2nd earliest, merge the earliest with it, to get |
12064 | # all those code points existing in the 2nd earliest. Repeat merging | |
12065 | # the new 2nd earliest with the 3rd earliest to get all those existing | |
12066 | # in the 3rd earliest, and so on. | |
12067 | foreach my $current_age (@rest_ages) { | |
12068 | next if $current_age->name !~ /^[\d.]*$/; # Skip the non-numeric | |
12069 | ||
12070 | my $current_in = $in->add_match_table( | |
12071 | $current_age->name, | |
12072 | Initialize => $current_age + $previous_in, | |
12073 | Description => $description_start | |
12074 | . $current_age->name | |
12075 | . ' or earlier', | |
12076 | ); | |
12077 | $previous_in = $current_in; | |
12078 | ||
12079 | # Add clarifying material for the corresponding age file. This is | |
12080 | # in part because of the confusing and contradictory information | |
12081 | # given in the Standard's documentation itself, as of 5.2. | |
12082 | $current_age->add_description( | |
12083 | "Code point's usage was introduced in version " | |
12084 | . $current_age->name); | |
12085 | $current_age->add_note("See also $in"); | |
12086 | ||
12087 | } | |
12088 | ||
12089 | # And finally the code points whose usages have yet to be decided are | |
12090 | # the same in both properties. Note that permanently unassigned code | |
12091 | # points actually have their usage assigned (as being permanently | |
12092 | # unassigned), so that these tables are not the same as gc=cn. | |
12093 | my $unassigned = $in->add_match_table($default_map); | |
12094 | my $age_default = $age->table($default_map); | |
12095 | $age_default->add_description(<<END | |
12096 | Code point's usage has not been assigned in any Unicode release thus far. | |
12097 | END | |
12098 | ); | |
12099 | $unassigned->set_equivalent_to($age_default, Related => 1); | |
12100 | } | |
12101 | ||
12102 | ||
12103 | # Finished creating all the perl properties. All non-internal non-string | |
12104 | # ones have a synonym of 'Is_' prefixed. (Internal properties begin with | |
12105 | # an underscore.) These do not get a separate entry in the pod file | |
12106 | foreach my $table ($perl->tables) { | |
12107 | foreach my $alias ($table->aliases) { | |
12108 | next if $alias->name =~ /^_/; | |
12109 | $table->add_alias('Is_' . $alias->name, | |
33e96e72 | 12110 | Re_Pod_Entry => 0, |
99870f4d KW |
12111 | Status => $alias->status, |
12112 | Externally_Ok => 0); | |
12113 | } | |
12114 | } | |
12115 | ||
c4019d52 KW |
12116 | # Here done with all the basic stuff. Ready to populate the information |
12117 | # about each character if annotating them. | |
558712cf | 12118 | if ($annotate) { |
c4019d52 KW |
12119 | |
12120 | # See comments at its declaration | |
12121 | $annotate_ranges = Range_Map->new; | |
12122 | ||
12123 | # This separates out the non-characters from the other unassigneds, so | |
12124 | # can give different annotations for each. | |
12125 | $unassigned_sans_noncharacters = Range_List->new( | |
12126 | Initialize => $gc->table('Unassigned') | |
12127 | & property_ref('Noncharacter_Code_Point')->table('N')); | |
12128 | ||
6189eadc | 12129 | for (my $i = 0; $i <= $MAX_UNICODE_CODEPOINT; $i++ ) { |
c4019d52 KW |
12130 | $i = populate_char_info($i); # Note sets $i so may cause skips |
12131 | } | |
12132 | } | |
12133 | ||
99870f4d KW |
12134 | return; |
12135 | } | |
12136 | ||
12137 | sub add_perl_synonyms() { | |
12138 | # A number of Unicode tables have Perl synonyms that are expressed in | |
12139 | # the single-form, \p{name}. These are: | |
12140 | # All the binary property Y tables, so that \p{Name=Y} gets \p{Name} and | |
12141 | # \p{Is_Name} as synonyms | |
12142 | # \p{Script=Value} gets \p{Value}, \p{Is_Value} as synonyms | |
12143 | # \p{General_Category=Value} gets \p{Value}, \p{Is_Value} as synonyms | |
12144 | # \p{Block=Value} gets \p{In_Value} as a synonym, and, if there is no | |
12145 | # conflict, \p{Value} and \p{Is_Value} as well | |
12146 | # | |
12147 | # This routine generates these synonyms, warning of any unexpected | |
12148 | # conflicts. | |
12149 | ||
12150 | # Construct the list of tables to get synonyms for. Start with all the | |
12151 | # binary and the General_Category ones. | |
06f26c45 KW |
12152 | my @tables = grep { $_->type == $BINARY || $_->type == $FORCED_BINARY } |
12153 | property_ref('*'); | |
99870f4d KW |
12154 | push @tables, $gc->tables; |
12155 | ||
12156 | # If the version of Unicode includes the Script property, add its tables | |
359523e2 | 12157 | push @tables, $script->tables if defined $script; |
99870f4d KW |
12158 | |
12159 | # The Block tables are kept separate because they are treated differently. | |
12160 | # And the earliest versions of Unicode didn't include them, so add only if | |
12161 | # there are some. | |
12162 | my @blocks; | |
12163 | push @blocks, $block->tables if defined $block; | |
12164 | ||
12165 | # Here, have the lists of tables constructed. Process blocks last so that | |
12166 | # if there are name collisions with them, blocks have lowest priority. | |
12167 | # Should there ever be other collisions, manual intervention would be | |
12168 | # required. See the comments at the beginning of the program for a | |
12169 | # possible way to handle those semi-automatically. | |
12170 | foreach my $table (@tables, @blocks) { | |
12171 | ||
12172 | # For non-binary properties, the synonym is just the name of the | |
12173 | # table, like Greek, but for binary properties the synonym is the name | |
12174 | # of the property, and means the code points in its 'Y' table. | |
12175 | my $nominal = $table; | |
12176 | my $nominal_property = $nominal->property; | |
12177 | my $actual; | |
12178 | if (! $nominal->isa('Property')) { | |
12179 | $actual = $table; | |
12180 | } | |
12181 | else { | |
12182 | ||
12183 | # Here is a binary property. Use the 'Y' table. Verify that is | |
12184 | # there | |
12185 | my $yes = $nominal->table('Y'); | |
12186 | unless (defined $yes) { # Must be defined, but is permissible to | |
12187 | # be empty. | |
12188 | Carp::my_carp_bug("Undefined $nominal, 'Y'. Skipping."); | |
12189 | next; | |
12190 | } | |
12191 | $actual = $yes; | |
12192 | } | |
12193 | ||
12194 | foreach my $alias ($nominal->aliases) { | |
12195 | ||
12196 | # Attempt to create a table in the perl directory for the | |
12197 | # candidate table, using whatever aliases in it that don't | |
12198 | # conflict. Also add non-conflicting aliases for all these | |
12199 | # prefixed by 'Is_' (and/or 'In_' for Block property tables) | |
12200 | PREFIX: | |
12201 | foreach my $prefix ("", 'Is_', 'In_') { | |
12202 | ||
12203 | # Only Block properties can have added 'In_' aliases. | |
12204 | next if $prefix eq 'In_' and $nominal_property != $block; | |
12205 | ||
12206 | my $proposed_name = $prefix . $alias->name; | |
12207 | ||
12208 | # No Is_Is, In_In, nor combinations thereof | |
12209 | trace "$proposed_name is a no-no" if main::DEBUG && $to_trace && $proposed_name =~ /^ I [ns] _I [ns] _/x; | |
12210 | next if $proposed_name =~ /^ I [ns] _I [ns] _/x; | |
12211 | ||
12212 | trace "Seeing if can add alias or table: 'perl=$proposed_name' based on $nominal" if main::DEBUG && $to_trace; | |
12213 | ||
12214 | # Get a reference to any existing table in the perl | |
12215 | # directory with the desired name. | |
12216 | my $pre_existing = $perl->table($proposed_name); | |
12217 | ||
12218 | if (! defined $pre_existing) { | |
12219 | ||
12220 | # No name collision, so ok to add the perl synonym. | |
12221 | ||
33e96e72 | 12222 | my $make_re_pod_entry; |
99870f4d | 12223 | my $externally_ok; |
4cd1260a | 12224 | my $status = $alias->status; |
99870f4d KW |
12225 | if ($nominal_property == $block) { |
12226 | ||
12227 | # For block properties, the 'In' form is preferred for | |
12228 | # external use; the pod file contains wild cards for | |
12229 | # this and the 'Is' form so no entries for those; and | |
12230 | # we don't want people using the name without the | |
12231 | # 'In', so discourage that. | |
12232 | if ($prefix eq "") { | |
33e96e72 | 12233 | $make_re_pod_entry = 1; |
99870f4d KW |
12234 | $status = $status || $DISCOURAGED; |
12235 | $externally_ok = 0; | |
12236 | } | |
12237 | elsif ($prefix eq 'In_') { | |
33e96e72 | 12238 | $make_re_pod_entry = 0; |
99870f4d KW |
12239 | $status = $status || $NORMAL; |
12240 | $externally_ok = 1; | |
12241 | } | |
12242 | else { | |
33e96e72 | 12243 | $make_re_pod_entry = 0; |
99870f4d KW |
12244 | $status = $status || $DISCOURAGED; |
12245 | $externally_ok = 0; | |
12246 | } | |
12247 | } | |
12248 | elsif ($prefix ne "") { | |
12249 | ||
12250 | # The 'Is' prefix is handled in the pod by a wild | |
12251 | # card, and we won't use it for an external name | |
33e96e72 | 12252 | $make_re_pod_entry = 0; |
99870f4d KW |
12253 | $status = $status || $NORMAL; |
12254 | $externally_ok = 0; | |
12255 | } | |
12256 | else { | |
12257 | ||
12258 | # Here, is an empty prefix, non block. This gets its | |
12259 | # own pod entry and can be used for an external name. | |
33e96e72 | 12260 | $make_re_pod_entry = 1; |
99870f4d KW |
12261 | $status = $status || $NORMAL; |
12262 | $externally_ok = 1; | |
12263 | } | |
12264 | ||
12265 | # Here, there isn't a perl pre-existing table with the | |
12266 | # name. Look through the list of equivalents of this | |
12267 | # table to see if one is a perl table. | |
12268 | foreach my $equivalent ($actual->leader->equivalents) { | |
12269 | next if $equivalent->property != $perl; | |
12270 | ||
12271 | # Here, have found a table for $perl. Add this alias | |
12272 | # to it, and are done with this prefix. | |
12273 | $equivalent->add_alias($proposed_name, | |
33e96e72 | 12274 | Re_Pod_Entry => $make_re_pod_entry, |
99870f4d KW |
12275 | Status => $status, |
12276 | Externally_Ok => $externally_ok); | |
12277 | trace "adding alias perl=$proposed_name to $equivalent" if main::DEBUG && $to_trace; | |
12278 | next PREFIX; | |
12279 | } | |
12280 | ||
12281 | # Here, $perl doesn't already have a table that is a | |
12282 | # synonym for this property, add one. | |
12283 | my $added_table = $perl->add_match_table($proposed_name, | |
33e96e72 | 12284 | Re_Pod_Entry => $make_re_pod_entry, |
99870f4d KW |
12285 | Status => $status, |
12286 | Externally_Ok => $externally_ok); | |
12287 | # And it will be related to the actual table, since it is | |
12288 | # based on it. | |
12289 | $added_table->set_equivalent_to($actual, Related => 1); | |
12290 | trace "added ", $perl->table($proposed_name) if main::DEBUG && $to_trace; | |
12291 | next; | |
12292 | } # End of no pre-existing. | |
12293 | ||
12294 | # Here, there is a pre-existing table that has the proposed | |
12295 | # name. We could be in trouble, but not if this is just a | |
12296 | # synonym for another table that we have already made a child | |
12297 | # of the pre-existing one. | |
6505c6e2 | 12298 | if ($pre_existing->is_set_equivalent_to($actual)) { |
99870f4d KW |
12299 | trace "$pre_existing is already equivalent to $actual; adding alias perl=$proposed_name to it" if main::DEBUG && $to_trace; |
12300 | $pre_existing->add_alias($proposed_name); | |
12301 | next; | |
12302 | } | |
12303 | ||
12304 | # Here, there is a name collision, but it still could be ok if | |
12305 | # the tables match the identical set of code points, in which | |
12306 | # case, we can combine the names. Compare each table's code | |
12307 | # point list to see if they are identical. | |
12308 | trace "Potential name conflict with $pre_existing having ", $pre_existing->count, " code points" if main::DEBUG && $to_trace; | |
12309 | if ($pre_existing->matches_identically_to($actual)) { | |
12310 | ||
12311 | # Here, they do match identically. Not a real conflict. | |
12312 | # Make the perl version a child of the Unicode one, except | |
12313 | # in the non-obvious case of where the perl name is | |
12314 | # already a synonym of another Unicode property. (This is | |
12315 | # excluded by the test for it being its own parent.) The | |
12316 | # reason for this exclusion is that then the two Unicode | |
12317 | # properties become related; and we don't really know if | |
12318 | # they are or not. We generate documentation based on | |
12319 | # relatedness, and this would be misleading. Code | |
12320 | # later executed in the process will cause the tables to | |
12321 | # be represented by a single file anyway, without making | |
12322 | # it look in the pod like they are necessarily related. | |
12323 | if ($pre_existing->parent == $pre_existing | |
12324 | && ($pre_existing->property == $perl | |
12325 | || $actual->property == $perl)) | |
12326 | { | |
12327 | trace "Setting $pre_existing equivalent to $actual since one is \$perl, and match identical sets" if main::DEBUG && $to_trace; | |
12328 | $pre_existing->set_equivalent_to($actual, Related => 1); | |
12329 | } | |
12330 | elsif (main::DEBUG && $to_trace) { | |
12331 | trace "$pre_existing is equivalent to $actual since match identical sets, but not setting them equivalent, to preserve the separateness of the perl aliases"; | |
12332 | trace $pre_existing->parent; | |
12333 | } | |
12334 | next PREFIX; | |
12335 | } | |
12336 | ||
12337 | # Here they didn't match identically, there is a real conflict | |
12338 | # between our new name and a pre-existing property. | |
12339 | $actual->add_conflicting($proposed_name, 'p', $pre_existing); | |
12340 | $pre_existing->add_conflicting($nominal->full_name, | |
12341 | 'p', | |
12342 | $actual); | |
12343 | ||
12344 | # Don't output a warning for aliases for the block | |
12345 | # properties (unless they start with 'In_') as it is | |
12346 | # expected that there will be conflicts and the block | |
12347 | # form loses. | |
12348 | if ($verbosity >= $NORMAL_VERBOSITY | |
12349 | && ($actual->property != $block || $prefix eq 'In_')) | |
12350 | { | |
12351 | print simple_fold(join_lines(<<END | |
12352 | There is already an alias named $proposed_name (from " . $pre_existing . "), | |
12353 | so not creating this alias for " . $actual | |
12354 | END | |
12355 | ), "", 4); | |
12356 | } | |
12357 | ||
12358 | # Keep track for documentation purposes. | |
12359 | $has_In_conflicts++ if $prefix eq 'In_'; | |
12360 | $has_Is_conflicts++ if $prefix eq 'Is_'; | |
12361 | } | |
12362 | } | |
12363 | } | |
12364 | ||
12365 | # There are some properties which have No and Yes (and N and Y) as | |
12366 | # property values, but aren't binary, and could possibly be confused with | |
12367 | # binary ones. So create caveats for them. There are tables that are | |
12368 | # named 'No', and tables that are named 'N', but confusion is not likely | |
12369 | # unless they are the same table. For example, N meaning Number or | |
12370 | # Neutral is not likely to cause confusion, so don't add caveats to things | |
12371 | # like them. | |
06f26c45 KW |
12372 | foreach my $property (grep { $_->type != $BINARY |
12373 | && $_->type != $FORCED_BINARY } | |
12374 | property_ref('*')) | |
12375 | { | |
99870f4d KW |
12376 | my $yes = $property->table('Yes'); |
12377 | if (defined $yes) { | |
12378 | my $y = $property->table('Y'); | |
12379 | if (defined $y && $yes == $y) { | |
12380 | foreach my $alias ($property->aliases) { | |
12381 | $yes->add_conflicting($alias->name); | |
12382 | } | |
12383 | } | |
12384 | } | |
12385 | my $no = $property->table('No'); | |
12386 | if (defined $no) { | |
12387 | my $n = $property->table('N'); | |
12388 | if (defined $n && $no == $n) { | |
12389 | foreach my $alias ($property->aliases) { | |
12390 | $no->add_conflicting($alias->name, 'P'); | |
12391 | } | |
12392 | } | |
12393 | } | |
12394 | } | |
12395 | ||
12396 | return; | |
12397 | } | |
12398 | ||
12399 | sub register_file_for_name($$$) { | |
12400 | # Given info about a table and a datafile that it should be associated | |
98dc9551 | 12401 | # with, register that association |
99870f4d KW |
12402 | |
12403 | my $table = shift; | |
12404 | my $directory_ref = shift; # Array of the directory path for the file | |
e6ebc4c0 | 12405 | my $file = shift; # The file name in the final directory. |
99870f4d KW |
12406 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; |
12407 | ||
12408 | trace "table=$table, file=$file, directory=@$directory_ref" if main::DEBUG && $to_trace; | |
12409 | ||
12410 | if ($table->isa('Property')) { | |
12411 | $table->set_file_path(@$directory_ref, $file); | |
12412 | push @map_properties, $table | |
12413 | if $directory_ref->[0] eq $map_directory; | |
12414 | return; | |
12415 | } | |
12416 | ||
12417 | # Do all of the work for all equivalent tables when called with the leader | |
12418 | # table, so skip if isn't the leader. | |
12419 | return if $table->leader != $table; | |
12420 | ||
a92d5c2e KW |
12421 | # If this is a complement of another file, use that other file instead, |
12422 | # with a ! prepended to it. | |
12423 | my $complement; | |
12424 | if (($complement = $table->complement) != 0) { | |
12425 | my @directories = $complement->file_path; | |
12426 | ||
12427 | # This assumes that the 0th element is something like 'lib', | |
12428 | # the 1th element the property name (in its own directory), like | |
12429 | # 'AHex', and the 2th element the file like 'Y' which will have a .pl | |
12430 | # appended to it later. | |
12431 | $directories[1] =~ s/^/!/; | |
12432 | $file = pop @directories; | |
12433 | $directory_ref =\@directories; | |
12434 | } | |
12435 | ||
99870f4d KW |
12436 | # Join all the file path components together, using slashes. |
12437 | my $full_filename = join('/', @$directory_ref, $file); | |
12438 | ||
12439 | # All go in the same subdirectory of unicore | |
12440 | if ($directory_ref->[0] ne $matches_directory) { | |
12441 | Carp::my_carp("Unexpected directory in " | |
12442 | . join('/', @{$directory_ref}, $file)); | |
12443 | } | |
12444 | ||
12445 | # For this table and all its equivalents ... | |
12446 | foreach my $table ($table, $table->equivalents) { | |
12447 | ||
12448 | # Associate it with its file internally. Don't include the | |
12449 | # $matches_directory first component | |
12450 | $table->set_file_path(@$directory_ref, $file); | |
12451 | my $sub_filename = join('/', $directory_ref->[1, -1], $file); | |
12452 | ||
12453 | my $property = $table->property; | |
12454 | $property = ($property == $perl) | |
12455 | ? "" # 'perl' is never explicitly stated | |
12456 | : standardize($property->name) . '='; | |
12457 | ||
12458 | my $deprecated = ($table->status eq $DEPRECATED) | |
12459 | ? $table->status_info | |
12460 | : ""; | |
d867ccfb | 12461 | my $caseless_equivalent = $table->caseless_equivalent; |
99870f4d KW |
12462 | |
12463 | # And for each of the table's aliases... This inner loop eventually | |
12464 | # goes through all aliases in the UCD that we generate regex match | |
12465 | # files for | |
12466 | foreach my $alias ($table->aliases) { | |
c85f591a | 12467 | my $standard = utf8_heavy_name($table, $alias); |
99870f4d KW |
12468 | |
12469 | # Generate an entry in either the loose or strict hashes, which | |
12470 | # will translate the property and alias names combination into the | |
12471 | # file where the table for them is stored. | |
99870f4d | 12472 | if ($alias->loose_match) { |
99870f4d KW |
12473 | if (exists $loose_to_file_of{$standard}) { |
12474 | Carp::my_carp("Can't change file registered to $loose_to_file_of{$standard} to '$sub_filename'."); | |
12475 | } | |
12476 | else { | |
12477 | $loose_to_file_of{$standard} = $sub_filename; | |
12478 | } | |
12479 | } | |
12480 | else { | |
99870f4d KW |
12481 | if (exists $stricter_to_file_of{$standard}) { |
12482 | Carp::my_carp("Can't change file registered to $stricter_to_file_of{$standard} to '$sub_filename'."); | |
12483 | } | |
12484 | else { | |
12485 | $stricter_to_file_of{$standard} = $sub_filename; | |
12486 | ||
12487 | # Tightly coupled with how utf8_heavy.pl works, for a | |
12488 | # floating point number that is a whole number, get rid of | |
12489 | # the trailing decimal point and 0's, so that utf8_heavy | |
12490 | # will work. Also note that this assumes that such a | |
12491 | # number is matched strictly; so if that were to change, | |
12492 | # this would be wrong. | |
c85f591a | 12493 | if ((my $integer_name = $alias->name) |
99870f4d KW |
12494 | =~ s/^ ( -? \d+ ) \.0+ $ /$1/x) |
12495 | { | |
12496 | $stricter_to_file_of{$property . $integer_name} | |
c12f2655 | 12497 | = $sub_filename; |
99870f4d KW |
12498 | } |
12499 | } | |
12500 | } | |
12501 | ||
12502 | # Keep a list of the deprecated properties and their filenames | |
a92d5c2e | 12503 | if ($deprecated && $complement == 0) { |
99870f4d KW |
12504 | $utf8::why_deprecated{$sub_filename} = $deprecated; |
12505 | } | |
d867ccfb KW |
12506 | |
12507 | # And a substitute table, if any, for case-insensitive matching | |
12508 | if ($caseless_equivalent != 0) { | |
12509 | $caseless_equivalent_to{$standard} = $caseless_equivalent; | |
12510 | } | |
99870f4d KW |
12511 | } |
12512 | } | |
12513 | ||
12514 | return; | |
12515 | } | |
12516 | ||
12517 | { # Closure | |
12518 | my %base_names; # Names already used for avoiding DOS 8.3 filesystem | |
12519 | # conflicts | |
12520 | my %full_dir_name_of; # Full length names of directories used. | |
12521 | ||
12522 | sub construct_filename($$$) { | |
12523 | # Return a file name for a table, based on the table name, but perhaps | |
12524 | # changed to get rid of non-portable characters in it, and to make | |
12525 | # sure that it is unique on a file system that allows the names before | |
12526 | # any period to be at most 8 characters (DOS). While we're at it | |
12527 | # check and complain if there are any directory conflicts. | |
12528 | ||
12529 | my $name = shift; # The name to start with | |
12530 | my $mutable = shift; # Boolean: can it be changed? If no, but | |
12531 | # yet it must be to work properly, a warning | |
12532 | # is given | |
12533 | my $directories_ref = shift; # A reference to an array containing the | |
12534 | # path to the file, with each element one path | |
12535 | # component. This is used because the same | |
12536 | # name can be used in different directories. | |
12537 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
12538 | ||
12539 | my $warn = ! defined wantarray; # If true, then if the name is | |
12540 | # changed, a warning is issued as well. | |
12541 | ||
12542 | if (! defined $name) { | |
12543 | Carp::my_carp("Undefined name in directory " | |
12544 | . File::Spec->join(@$directories_ref) | |
12545 | . ". '_' used"); | |
12546 | return '_'; | |
12547 | } | |
12548 | ||
12549 | # Make sure that no directory names conflict with each other. Look at | |
12550 | # each directory in the input file's path. If it is already in use, | |
12551 | # assume it is correct, and is merely being re-used, but if we | |
12552 | # truncate it to 8 characters, and find that there are two directories | |
12553 | # that are the same for the first 8 characters, but differ after that, | |
12554 | # then that is a problem. | |
12555 | foreach my $directory (@$directories_ref) { | |
12556 | my $short_dir = substr($directory, 0, 8); | |
12557 | if (defined $full_dir_name_of{$short_dir}) { | |
12558 | next if $full_dir_name_of{$short_dir} eq $directory; | |
12559 | Carp::my_carp("$directory conflicts with $full_dir_name_of{$short_dir}. Bad News. Continuing anyway"); | |
12560 | } | |
12561 | else { | |
12562 | $full_dir_name_of{$short_dir} = $directory; | |
12563 | } | |
12564 | } | |
12565 | ||
12566 | my $path = join '/', @$directories_ref; | |
12567 | $path .= '/' if $path; | |
12568 | ||
12569 | # Remove interior underscores. | |
12570 | (my $filename = $name) =~ s/ (?<=.) _ (?=.) //xg; | |
12571 | ||
12572 | # Change any non-word character into an underscore, and truncate to 8. | |
12573 | $filename =~ s/\W+/_/g; # eg., "L&" -> "L_" | |
12574 | substr($filename, 8) = "" if length($filename) > 8; | |
12575 | ||
12576 | # Make sure the basename doesn't conflict with something we | |
12577 | # might have already written. If we have, say, | |
12578 | # InGreekExtended1 | |
12579 | # InGreekExtended2 | |
12580 | # they become | |
12581 | # InGreekE | |
12582 | # InGreek2 | |
12583 | my $warned = 0; | |
12584 | while (my $num = $base_names{$path}{lc $filename}++) { | |
12585 | $num++; # so basenames with numbers start with '2', which | |
12586 | # just looks more natural. | |
12587 | ||
12588 | # Want to append $num, but if it'll make the basename longer | |
12589 | # than 8 characters, pre-truncate $filename so that the result | |
12590 | # is acceptable. | |
12591 | my $delta = length($filename) + length($num) - 8; | |
12592 | if ($delta > 0) { | |
12593 | substr($filename, -$delta) = $num; | |
12594 | } | |
12595 | else { | |
12596 | $filename .= $num; | |
12597 | } | |
12598 | if ($warn && ! $warned) { | |
12599 | $warned = 1; | |
12600 | Carp::my_carp("'$path$name' conflicts with another name on a filesystem with 8 significant characters (like DOS). Proceeding anyway."); | |
12601 | } | |
12602 | } | |
12603 | ||
12604 | return $filename if $mutable; | |
12605 | ||
12606 | # If not changeable, must return the input name, but warn if needed to | |
12607 | # change it beyond shortening it. | |
12608 | if ($name ne $filename | |
12609 | && substr($name, 0, length($filename)) ne $filename) { | |
12610 | Carp::my_carp("'$path$name' had to be changed into '$filename'. Bad News. Proceeding anyway."); | |
12611 | } | |
12612 | return $name; | |
12613 | } | |
12614 | } | |
12615 | ||
12616 | # The pod file contains a very large table. Many of the lines in that table | |
12617 | # would exceed a typical output window's size, and so need to be wrapped with | |
12618 | # a hanging indent to make them look good. The pod language is really | |
12619 | # insufficient here. There is no general construct to do that in pod, so it | |
12620 | # is done here by beginning each such line with a space to cause the result to | |
12621 | # be output without formatting, and doing all the formatting here. This leads | |
12622 | # to the result that if the eventual display window is too narrow it won't | |
12623 | # look good, and if the window is too wide, no advantage is taken of that | |
12624 | # extra width. A further complication is that the output may be indented by | |
12625 | # the formatter so that there is less space than expected. What I (khw) have | |
12626 | # done is to assume that that indent is a particular number of spaces based on | |
12627 | # what it is in my Linux system; people can always resize their windows if | |
12628 | # necessary, but this is obviously less than desirable, but the best that can | |
12629 | # be expected. | |
12630 | my $automatic_pod_indent = 8; | |
12631 | ||
12632 | # Try to format so that uses fewest lines, but few long left column entries | |
12633 | # slide into the right column. An experiment on 5.1 data yielded the | |
12634 | # following percentages that didn't cut into the other side along with the | |
12635 | # associated first-column widths | |
12636 | # 69% = 24 | |
12637 | # 80% not too bad except for a few blocks | |
12638 | # 90% = 33; # , cuts 353/3053 lines from 37 = 12% | |
12639 | # 95% = 37; | |
12640 | my $indent_info_column = 27; # 75% of lines didn't have overlap | |
12641 | ||
12642 | my $FILLER = 3; # Length of initial boiler-plate columns in a pod line | |
12643 | # The 3 is because of: | |
12644 | # 1 for the leading space to tell the pod formatter to | |
12645 | # output as-is | |
12646 | # 1 for the flag | |
12647 | # 1 for the space between the flag and the main data | |
12648 | ||
12649 | sub format_pod_line ($$$;$$) { | |
12650 | # Take a pod line and return it, formatted properly | |
12651 | ||
12652 | my $first_column_width = shift; | |
12653 | my $entry = shift; # Contents of left column | |
12654 | my $info = shift; # Contents of right column | |
12655 | ||
12656 | my $status = shift || ""; # Any flag | |
12657 | ||
12658 | my $loose_match = shift; # Boolean. | |
12659 | $loose_match = 1 unless defined $loose_match; | |
12660 | ||
12661 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
12662 | ||
12663 | my $flags = ""; | |
12664 | $flags .= $STRICTER if ! $loose_match; | |
12665 | ||
12666 | $flags .= $status if $status; | |
12667 | ||
12668 | # There is a blank in the left column to cause the pod formatter to | |
12669 | # output the line as-is. | |
12670 | return sprintf " %-*s%-*s %s\n", | |
12671 | # The first * in the format is replaced by this, the -1 is | |
12672 | # to account for the leading blank. There isn't a | |
12673 | # hard-coded blank after this to separate the flags from | |
12674 | # the rest of the line, so that in the unlikely event that | |
12675 | # multiple flags are shown on the same line, they both | |
12676 | # will get displayed at the expense of that separation, | |
12677 | # but since they are left justified, a blank will be | |
12678 | # inserted in the normal case. | |
12679 | $FILLER - 1, | |
12680 | $flags, | |
12681 | ||
12682 | # The other * in the format is replaced by this number to | |
12683 | # cause the first main column to right fill with blanks. | |
12684 | # The -1 is for the guaranteed blank following it. | |
12685 | $first_column_width - $FILLER - 1, | |
12686 | $entry, | |
12687 | $info; | |
12688 | } | |
12689 | ||
12690 | my @zero_match_tables; # List of tables that have no matches in this release | |
12691 | ||
12692 | sub make_table_pod_entries($) { | |
12693 | # This generates the entries for the pod file for a given table. | |
12694 | # Also done at this time are any children tables. The output looks like: | |
12695 | # \p{Common} \p{Script=Common} (Short: \p{Zyyy}) (5178) | |
12696 | ||
12697 | my $input_table = shift; # Table the entry is for | |
12698 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
12699 | ||
12700 | # Generate parent and all its children at the same time. | |
12701 | return if $input_table->parent != $input_table; | |
12702 | ||
12703 | my $property = $input_table->property; | |
12704 | my $type = $property->type; | |
12705 | my $full_name = $property->full_name; | |
12706 | ||
12707 | my $count = $input_table->count; | |
12708 | my $string_count = clarify_number($count); | |
12709 | my $status = $input_table->status; | |
12710 | my $status_info = $input_table->status_info; | |
56ca34ca | 12711 | my $caseless_equivalent = $input_table->caseless_equivalent; |
99870f4d KW |
12712 | |
12713 | my $entry_for_first_table; # The entry for the first table output. | |
12714 | # Almost certainly, it is the parent. | |
12715 | ||
12716 | # For each related table (including itself), we will generate a pod entry | |
12717 | # for each name each table goes by | |
12718 | foreach my $table ($input_table, $input_table->children) { | |
12719 | ||
d4da3f74 KW |
12720 | # utf8_heavy.pl cannot deal with null string property values, so skip |
12721 | # any tables that have no non-null names. | |
12722 | next if ! grep { $_->name ne "" } $table->aliases; | |
99870f4d KW |
12723 | |
12724 | # First, gather all the info that applies to this table as a whole. | |
12725 | ||
12726 | push @zero_match_tables, $table if $count == 0; | |
12727 | ||
12728 | my $table_property = $table->property; | |
12729 | ||
12730 | # The short name has all the underscores removed, while the full name | |
12731 | # retains them. Later, we decide whether to output a short synonym | |
12732 | # for the full one, we need to compare apples to apples, so we use the | |
12733 | # short name's length including underscores. | |
12734 | my $table_property_short_name_length; | |
12735 | my $table_property_short_name | |
12736 | = $table_property->short_name(\$table_property_short_name_length); | |
12737 | my $table_property_full_name = $table_property->full_name; | |
12738 | ||
12739 | # Get how much savings there is in the short name over the full one | |
12740 | # (delta will always be <= 0) | |
12741 | my $table_property_short_delta = $table_property_short_name_length | |
12742 | - length($table_property_full_name); | |
12743 | my @table_description = $table->description; | |
12744 | my @table_note = $table->note; | |
12745 | ||
12746 | # Generate an entry for each alias in this table. | |
12747 | my $entry_for_first_alias; # saves the first one encountered. | |
12748 | foreach my $alias ($table->aliases) { | |
12749 | ||
12750 | # Skip if not to go in pod. | |
33e96e72 | 12751 | next unless $alias->make_re_pod_entry; |
99870f4d KW |
12752 | |
12753 | # Start gathering all the components for the entry | |
12754 | my $name = $alias->name; | |
12755 | ||
d4da3f74 KW |
12756 | # Skip if name is empty, as can't be accessed by regexes. |
12757 | next if $name eq ""; | |
12758 | ||
99870f4d KW |
12759 | my $entry; # Holds the left column, may include extras |
12760 | my $entry_ref; # To refer to the left column's contents from | |
12761 | # another entry; has no extras | |
12762 | ||
12763 | # First the left column of the pod entry. Tables for the $perl | |
12764 | # property always use the single form. | |
12765 | if ($table_property == $perl) { | |
12766 | $entry = "\\p{$name}"; | |
12767 | $entry_ref = "\\p{$name}"; | |
12768 | } | |
12769 | else { # Compound form. | |
12770 | ||
12771 | # Only generate one entry for all the aliases that mean true | |
12772 | # or false in binary properties. Append a '*' to indicate | |
12773 | # some are missing. (The heading comment notes this.) | |
60e471b3 | 12774 | my $rhs; |
99870f4d KW |
12775 | if ($type == $BINARY) { |
12776 | next if $name ne 'N' && $name ne 'Y'; | |
60e471b3 | 12777 | $rhs = "$name*"; |
99870f4d | 12778 | } |
06f26c45 | 12779 | elsif ($type != $FORCED_BINARY) { |
60e471b3 | 12780 | $rhs = $name; |
99870f4d | 12781 | } |
06f26c45 KW |
12782 | else { |
12783 | ||
12784 | # Forced binary properties require special handling. It | |
12785 | # has two sets of tables, one set is true/false; and the | |
12786 | # other set is everything else. Entries are generated for | |
12787 | # each set. Use the Bidi_Mirrored property (which appears | |
12788 | # in all Unicode versions) to get a list of the aliases | |
12789 | # for the true/false tables. Of these, only output the N | |
12790 | # and Y ones, the same as, a regular binary property. And | |
12791 | # output all the rest, same as a non-binary property. | |
12792 | my $bm = property_ref("Bidi_Mirrored"); | |
12793 | if ($name eq 'N' || $name eq 'Y') { | |
12794 | $rhs = "$name*"; | |
12795 | } elsif (grep { $name eq $_->name } $bm->table("Y")->aliases, | |
12796 | $bm->table("N")->aliases) | |
12797 | { | |
12798 | next; | |
12799 | } | |
12800 | else { | |
12801 | $rhs = $name; | |
12802 | } | |
12803 | } | |
99870f4d KW |
12804 | |
12805 | # Colon-space is used to give a little more space to be easier | |
12806 | # to read; | |
12807 | $entry = "\\p{" | |
12808 | . $table_property_full_name | |
60e471b3 | 12809 | . ": $rhs}"; |
99870f4d KW |
12810 | |
12811 | # But for the reference to this entry, which will go in the | |
12812 | # right column, where space is at a premium, use equals | |
12813 | # without a space | |
12814 | $entry_ref = "\\p{" . $table_property_full_name . "=$name}"; | |
12815 | } | |
12816 | ||
12817 | # Then the right (info) column. This is stored as components of | |
12818 | # an array for the moment, then joined into a string later. For | |
12819 | # non-internal only properties, begin the info with the entry for | |
12820 | # the first table we encountered (if any), as things are ordered | |
12821 | # so that that one is the most descriptive. This leads to the | |
12822 | # info column of an entry being a more descriptive version of the | |
12823 | # name column | |
12824 | my @info; | |
12825 | if ($name =~ /^_/) { | |
12826 | push @info, | |
12827 | '(For internal use by Perl, not necessarily stable)'; | |
12828 | } | |
12829 | elsif ($entry_for_first_alias) { | |
12830 | push @info, $entry_for_first_alias; | |
12831 | } | |
12832 | ||
12833 | # If this entry is equivalent to another, add that to the info, | |
12834 | # using the first such table we encountered | |
12835 | if ($entry_for_first_table) { | |
12836 | if (@info) { | |
12837 | push @info, "(= $entry_for_first_table)"; | |
12838 | } | |
12839 | else { | |
12840 | push @info, $entry_for_first_table; | |
12841 | } | |
12842 | } | |
12843 | ||
12844 | # If the name is a large integer, add an equivalent with an | |
12845 | # exponent for better readability | |
12846 | if ($name =~ /^[+-]?[\d]+$/ && $name >= 10_000) { | |
12847 | push @info, sprintf "(= %.1e)", $name | |
12848 | } | |
12849 | ||
12850 | my $parenthesized = ""; | |
12851 | if (! $entry_for_first_alias) { | |
12852 | ||
12853 | # This is the first alias for the current table. The alias | |
12854 | # array is ordered so that this is the fullest, most | |
12855 | # descriptive alias, so it gets the fullest info. The other | |
12856 | # aliases are mostly merely pointers to this one, using the | |
12857 | # information already added above. | |
12858 | ||
12859 | # Display any status message, but only on the parent table | |
12860 | if ($status && ! $entry_for_first_table) { | |
12861 | push @info, $status_info; | |
12862 | } | |
12863 | ||
12864 | # Put out any descriptive info | |
12865 | if (@table_description || @table_note) { | |
12866 | push @info, join "; ", @table_description, @table_note; | |
12867 | } | |
12868 | ||
12869 | # Look to see if there is a shorter name we can point people | |
12870 | # at | |
12871 | my $standard_name = standardize($name); | |
12872 | my $short_name; | |
12873 | my $proposed_short = $table->short_name; | |
12874 | if (defined $proposed_short) { | |
12875 | my $standard_short = standardize($proposed_short); | |
12876 | ||
12877 | # If the short name is shorter than the standard one, or | |
12878 | # even it it's not, but the combination of it and its | |
12879 | # short property name (as in \p{prop=short} ($perl doesn't | |
12880 | # have this form)) saves at least two characters, then, | |
12881 | # cause it to be listed as a shorter synonym. | |
12882 | if (length $standard_short < length $standard_name | |
12883 | || ($table_property != $perl | |
12884 | && (length($standard_short) | |
12885 | - length($standard_name) | |
12886 | + $table_property_short_delta) # (<= 0) | |
12887 | < -2)) | |
12888 | { | |
12889 | $short_name = $proposed_short; | |
12890 | if ($table_property != $perl) { | |
12891 | $short_name = $table_property_short_name | |
12892 | . "=$short_name"; | |
12893 | } | |
12894 | $short_name = "\\p{$short_name}"; | |
12895 | } | |
12896 | } | |
12897 | ||
12898 | # And if this is a compound form name, see if there is a | |
12899 | # single form equivalent | |
12900 | my $single_form; | |
12901 | if ($table_property != $perl) { | |
12902 | ||
12903 | # Special case the binary N tables, so that will print | |
12904 | # \P{single}, but use the Y table values to populate | |
c12f2655 | 12905 | # 'single', as we haven't likewise populated the N table. |
06f26c45 KW |
12906 | # For forced binary tables, we can't just look at the N |
12907 | # table, but must see if this table is equivalent to the N | |
12908 | # one, as there are two equivalent beasts in these | |
12909 | # properties. | |
99870f4d KW |
12910 | my $test_table; |
12911 | my $p; | |
06f26c45 KW |
12912 | if ( ($type == $BINARY |
12913 | && $input_table == $property->table('No')) | |
12914 | || ($type == $FORCED_BINARY | |
12915 | && $property->table('No')-> | |
12916 | is_set_equivalent_to($input_table))) | |
99870f4d KW |
12917 | { |
12918 | $test_table = $property->table('Yes'); | |
12919 | $p = 'P'; | |
12920 | } | |
12921 | else { | |
12922 | $test_table = $input_table; | |
12923 | $p = 'p'; | |
12924 | } | |
12925 | ||
12926 | # Look for a single form amongst all the children. | |
12927 | foreach my $table ($test_table->children) { | |
12928 | next if $table->property != $perl; | |
12929 | my $proposed_name = $table->short_name; | |
12930 | next if ! defined $proposed_name; | |
12931 | ||
12932 | # Don't mention internal-only properties as a possible | |
12933 | # single form synonym | |
12934 | next if substr($proposed_name, 0, 1) eq '_'; | |
12935 | ||
12936 | $proposed_name = "\\$p\{$proposed_name}"; | |
12937 | if (! defined $single_form | |
12938 | || length($proposed_name) < length $single_form) | |
12939 | { | |
12940 | $single_form = $proposed_name; | |
12941 | ||
12942 | # The goal here is to find a single form; not the | |
12943 | # shortest possible one. We've already found a | |
12944 | # short name. So, stop at the first single form | |
12945 | # found, which is likely to be closer to the | |
12946 | # original. | |
12947 | last; | |
12948 | } | |
12949 | } | |
12950 | } | |
12951 | ||
12952 | # Ouput both short and single in the same parenthesized | |
12953 | # expression, but with only one of 'Single', 'Short' if there | |
12954 | # are both items. | |
12955 | if ($short_name || $single_form || $table->conflicting) { | |
99870f4d KW |
12956 | $parenthesized .= "Short: $short_name" if $short_name; |
12957 | if ($short_name && $single_form) { | |
12958 | $parenthesized .= ', '; | |
12959 | } | |
12960 | elsif ($single_form) { | |
12961 | $parenthesized .= 'Single: '; | |
12962 | } | |
12963 | $parenthesized .= $single_form if $single_form; | |
12964 | } | |
12965 | } | |
12966 | ||
56ca34ca KW |
12967 | if ($caseless_equivalent != 0) { |
12968 | $parenthesized .= '; ' if $parenthesized ne ""; | |
12969 | $parenthesized .= "/i= " . $caseless_equivalent->complete_name; | |
12970 | } | |
12971 | ||
99870f4d KW |
12972 | |
12973 | # Warn if this property isn't the same as one that a | |
12974 | # semi-casual user might expect. The other components of this | |
12975 | # parenthesized structure are calculated only for the first entry | |
12976 | # for this table, but the conflicting is deemed important enough | |
12977 | # to go on every entry. | |
12978 | my $conflicting = join " NOR ", $table->conflicting; | |
12979 | if ($conflicting) { | |
e5228720 | 12980 | $parenthesized .= '; ' if $parenthesized ne ""; |
99870f4d KW |
12981 | $parenthesized .= "NOT $conflicting"; |
12982 | } | |
99870f4d | 12983 | |
e5228720 | 12984 | push @info, "($parenthesized)" if $parenthesized; |
d57ccc9a | 12985 | |
0f88d393 KW |
12986 | if ($name =~ /_$/ && $alias->loose_match) { |
12987 | push @info, "Note the trailing '_' matters in spite of loose matching rules."; | |
12988 | } | |
12989 | ||
d57ccc9a KW |
12990 | if ($table_property != $perl && $table->perl_extension) { |
12991 | push @info, '(Perl extension)'; | |
12992 | } | |
2cf724d4 | 12993 | push @info, "($string_count)"; |
99870f4d KW |
12994 | |
12995 | # Now, we have both the entry and info so add them to the | |
12996 | # list of all the properties. | |
12997 | push @match_properties, | |
12998 | format_pod_line($indent_info_column, | |
12999 | $entry, | |
13000 | join( " ", @info), | |
13001 | $alias->status, | |
13002 | $alias->loose_match); | |
13003 | ||
13004 | $entry_for_first_alias = $entry_ref unless $entry_for_first_alias; | |
13005 | } # End of looping through the aliases for this table. | |
13006 | ||
13007 | if (! $entry_for_first_table) { | |
13008 | $entry_for_first_table = $entry_for_first_alias; | |
13009 | } | |
13010 | } # End of looping through all the related tables | |
13011 | return; | |
13012 | } | |
13013 | ||
13014 | sub pod_alphanumeric_sort { | |
13015 | # Sort pod entries alphanumerically. | |
13016 | ||
99f78760 KW |
13017 | # The first few character columns are filler, plus the '\p{'; and get rid |
13018 | # of all the trailing stuff, starting with the trailing '}', so as to sort | |
13019 | # on just 'Name=Value' | |
13020 | (my $a = lc $a) =~ s/^ .*? { //x; | |
99870f4d | 13021 | $a =~ s/}.*//; |
99f78760 | 13022 | (my $b = lc $b) =~ s/^ .*? { //x; |
99870f4d KW |
13023 | $b =~ s/}.*//; |
13024 | ||
99f78760 KW |
13025 | # Determine if the two operands are both internal only or both not. |
13026 | # Character 0 should be a '\'; 1 should be a p; 2 should be '{', so 3 | |
13027 | # should be the underscore that begins internal only | |
13028 | my $a_is_internal = (substr($a, 0, 1) eq '_'); | |
13029 | my $b_is_internal = (substr($b, 0, 1) eq '_'); | |
13030 | ||
13031 | # Sort so the internals come last in the table instead of first (which the | |
13032 | # leading underscore would otherwise indicate). | |
13033 | if ($a_is_internal != $b_is_internal) { | |
13034 | return 1 if $a_is_internal; | |
13035 | return -1 | |
13036 | } | |
13037 | ||
99870f4d | 13038 | # Determine if the two operands are numeric property values or not. |
99f78760 | 13039 | # A numeric property will look like xyz: 3. But the number |
99870f4d | 13040 | # can begin with an optional minus sign, and may have a |
99f78760 | 13041 | # fraction or rational component, like xyz: 3/2. If either |
99870f4d KW |
13042 | # isn't numeric, use alphabetic sort. |
13043 | my ($a_initial, $a_number) = | |
99f78760 | 13044 | ($a =~ /^ ( [^:=]+ [:=] \s* ) (-? \d+ (?: [.\/] \d+)? )/ix); |
99870f4d KW |
13045 | return $a cmp $b unless defined $a_number; |
13046 | my ($b_initial, $b_number) = | |
99f78760 | 13047 | ($b =~ /^ ( [^:=]+ [:=] \s* ) (-? \d+ (?: [.\/] \d+)? )/ix); |
99870f4d KW |
13048 | return $a cmp $b unless defined $b_number; |
13049 | ||
13050 | # Here they are both numeric, but use alphabetic sort if the | |
13051 | # initial parts don't match | |
13052 | return $a cmp $b if $a_initial ne $b_initial; | |
13053 | ||
13054 | # Convert rationals to floating for the comparison. | |
13055 | $a_number = eval $a_number if $a_number =~ qr{/}; | |
13056 | $b_number = eval $b_number if $b_number =~ qr{/}; | |
13057 | ||
13058 | return $a_number <=> $b_number; | |
13059 | } | |
13060 | ||
13061 | sub make_pod () { | |
13062 | # Create the .pod file. This generates the various subsections and then | |
13063 | # combines them in one big HERE document. | |
13064 | ||
07c070a8 KW |
13065 | my $Is_flags_text = "If an entry has flag(s) at its beginning, like \"$DEPRECATED\", the \"Is_\" form has the same flag(s)"; |
13066 | ||
99870f4d KW |
13067 | return unless defined $pod_directory; |
13068 | print "Making pod file\n" if $verbosity >= $PROGRESS; | |
13069 | ||
13070 | my $exception_message = | |
13071 | '(Any exceptions are individually noted beginning with the word NOT.)'; | |
13072 | my @block_warning; | |
13073 | if (-e 'Blocks.txt') { | |
13074 | ||
13075 | # Add the line: '\p{In_*} \p{Block: *}', with the warning message | |
13076 | # if the global $has_In_conflicts indicates we have them. | |
13077 | push @match_properties, format_pod_line($indent_info_column, | |
13078 | '\p{In_*}', | |
13079 | '\p{Block: *}' | |
13080 | . (($has_In_conflicts) | |
13081 | ? " $exception_message" | |
13082 | : "")); | |
13083 | @block_warning = << "END"; | |
13084 | ||
77173124 KW |
13085 | Matches in the Block property have shortcuts that begin with "In_". For |
13086 | example, C<\\p{Block=Latin1}> can be written as C<\\p{In_Latin1}>. For | |
13087 | backward compatibility, if there is no conflict with another shortcut, these | |
13088 | may also be written as C<\\p{Latin1}> or C<\\p{Is_Latin1}>. But, N.B., there | |
13089 | are numerous such conflicting shortcuts. Use of these forms for Block is | |
13090 | discouraged, and are flagged as such, not only because of the potential | |
13091 | confusion as to what is meant, but also because a later release of Unicode may | |
13092 | preempt the shortcut, and your program would no longer be correct. Use the | |
13093 | "In_" form instead to avoid this, or even more clearly, use the compound form, | |
13094 | e.g., C<\\p{blk:latin1}>. See L<perlunicode/"Blocks"> for more information | |
13095 | about this. | |
99870f4d KW |
13096 | END |
13097 | } | |
07c070a8 | 13098 | my $text = $Is_flags_text; |
99870f4d KW |
13099 | $text = "$exception_message $text" if $has_Is_conflicts; |
13100 | ||
13101 | # And the 'Is_ line'; | |
13102 | push @match_properties, format_pod_line($indent_info_column, | |
13103 | '\p{Is_*}', | |
13104 | "\\p{*} $text"); | |
13105 | ||
13106 | # Sort the properties array for output. It is sorted alphabetically | |
13107 | # except numerically for numeric properties, and only output unique lines. | |
13108 | @match_properties = sort pod_alphanumeric_sort uniques @match_properties; | |
13109 | ||
13110 | my $formatted_properties = simple_fold(\@match_properties, | |
13111 | "", | |
13112 | # indent succeeding lines by two extra | |
13113 | # which looks better | |
13114 | $indent_info_column + 2, | |
13115 | ||
13116 | # shorten the line length by how much | |
13117 | # the formatter indents, so the folded | |
13118 | # line will fit in the space | |
13119 | # presumably available | |
13120 | $automatic_pod_indent); | |
13121 | # Add column headings, indented to be a little more centered, but not | |
13122 | # exactly | |
13123 | $formatted_properties = format_pod_line($indent_info_column, | |
13124 | ' NAME', | |
13125 | ' INFO') | |
13126 | . "\n" | |
13127 | . $formatted_properties; | |
13128 | ||
13129 | # Generate pod documentation lines for the tables that match nothing | |
0090c5d1 | 13130 | my $zero_matches = ""; |
99870f4d KW |
13131 | if (@zero_match_tables) { |
13132 | @zero_match_tables = uniques(@zero_match_tables); | |
13133 | $zero_matches = join "\n\n", | |
13134 | map { $_ = '=item \p{' . $_->complete_name . "}" } | |
13135 | sort { $a->complete_name cmp $b->complete_name } | |
c0de960f | 13136 | @zero_match_tables; |
99870f4d KW |
13137 | |
13138 | $zero_matches = <<END; | |
13139 | ||
77173124 | 13140 | =head2 Legal C<\\p{}> and C<\\P{}> constructs that match no characters |
99870f4d KW |
13141 | |
13142 | Unicode has some property-value pairs that currently don't match anything. | |
c12f2655 KW |
13143 | This happens generally either because they are obsolete, or they exist for |
13144 | symmetry with other forms, but no language has yet been encoded that uses | |
13145 | them. In this version of Unicode, the following match zero code points: | |
99870f4d KW |
13146 | |
13147 | =over 4 | |
13148 | ||
13149 | $zero_matches | |
13150 | ||
13151 | =back | |
13152 | ||
13153 | END | |
13154 | } | |
13155 | ||
13156 | # Generate list of properties that we don't accept, grouped by the reasons | |
13157 | # why. This is so only put out the 'why' once, and then list all the | |
13158 | # properties that have that reason under it. | |
13159 | ||
13160 | my %why_list; # The keys are the reasons; the values are lists of | |
13161 | # properties that have the key as their reason | |
13162 | ||
13163 | # For each property, add it to the list that are suppressed for its reason | |
13164 | # The sort will cause the alphabetically first properties to be added to | |
13165 | # each list first, so each list will be sorted. | |
13166 | foreach my $property (sort keys %why_suppressed) { | |
13167 | push @{$why_list{$why_suppressed{$property}}}, $property; | |
13168 | } | |
13169 | ||
13170 | # For each reason (sorted by the first property that has that reason)... | |
13171 | my @bad_re_properties; | |
13172 | foreach my $why (sort { $why_list{$a}->[0] cmp $why_list{$b}->[0] } | |
13173 | keys %why_list) | |
13174 | { | |
54ce19c9 | 13175 | # Add to the output, all the properties that have that reason. |
99870f4d KW |
13176 | my $has_item = 0; # Flag if actually output anything. |
13177 | foreach my $name (@{$why_list{$why}}) { | |
13178 | ||
13179 | # Split compound names into $property and $table components | |
13180 | my $property = $name; | |
13181 | my $table; | |
13182 | if ($property =~ / (.*) = (.*) /x) { | |
13183 | $property = $1; | |
13184 | $table = $2; | |
13185 | } | |
13186 | ||
13187 | # This release of Unicode may not have a property that is | |
13188 | # suppressed, so don't reference a non-existent one. | |
13189 | $property = property_ref($property); | |
13190 | next if ! defined $property; | |
13191 | ||
13192 | # And since this list is only for match tables, don't list the | |
13193 | # ones that don't have match tables. | |
13194 | next if ! $property->to_create_match_tables; | |
13195 | ||
13196 | # Find any abbreviation, and turn it into a compound name if this | |
13197 | # is a property=value pair. | |
13198 | my $short_name = $property->name; | |
13199 | $short_name .= '=' . $property->table($table)->name if $table; | |
13200 | ||
54ce19c9 KW |
13201 | # Start with an empty line. |
13202 | push @bad_re_properties, "\n\n" unless $has_item; | |
13203 | ||
99870f4d KW |
13204 | # And add the property as an item for the reason. |
13205 | push @bad_re_properties, "\n=item I<$name> ($short_name)\n"; | |
13206 | $has_item = 1; | |
13207 | } | |
13208 | ||
13209 | # And add the reason under the list of properties, if such a list | |
13210 | # actually got generated. Note that the header got added | |
13211 | # unconditionally before. But pod ignores extra blank lines, so no | |
13212 | # harm. | |
13213 | push @bad_re_properties, "\n$why\n" if $has_item; | |
13214 | ||
13215 | } # End of looping through each reason. | |
13216 | ||
54ce19c9 KW |
13217 | if (! @bad_re_properties) { |
13218 | push @bad_re_properties, | |
13219 | "*** This installation accepts ALL non-Unihan properties ***"; | |
13220 | } | |
13221 | else { | |
13222 | # Add =over only if non-empty to avoid an empty =over/=back section, | |
13223 | # which is considered bad form. | |
13224 | unshift @bad_re_properties, "\n=over 4\n"; | |
13225 | push @bad_re_properties, "\n=back\n"; | |
13226 | } | |
13227 | ||
8d099389 KW |
13228 | # Similiarly, generate a list of files that we don't use, grouped by the |
13229 | # reasons why. First, create a hash whose keys are the reasons, and whose | |
13230 | # values are anonymous arrays of all the files that share that reason. | |
13231 | my %grouped_by_reason; | |
13232 | foreach my $file (keys %ignored_files) { | |
13233 | push @{$grouped_by_reason{$ignored_files{$file}}}, $file; | |
13234 | } | |
13235 | ||
13236 | # Then, sort each group. | |
13237 | foreach my $group (keys %grouped_by_reason) { | |
13238 | @{$grouped_by_reason{$group}} = sort { lc $a cmp lc $b } | |
13239 | @{$grouped_by_reason{$group}} ; | |
13240 | } | |
13241 | ||
13242 | # Finally, create the output text. For each reason (sorted by the | |
13243 | # alphabetically first file that has that reason)... | |
13244 | my @unused_files; | |
13245 | foreach my $reason (sort { lc $grouped_by_reason{$a}->[0] | |
13246 | cmp lc $grouped_by_reason{$b}->[0] | |
13247 | } | |
13248 | keys %grouped_by_reason) | |
13249 | { | |
13250 | # Add all the files that have that reason to the output. Start | |
13251 | # with an empty line. | |
13252 | push @unused_files, "\n\n"; | |
13253 | push @unused_files, map { "\n=item F<$_> \n" } | |
13254 | @{$grouped_by_reason{$reason}}; | |
13255 | # And add the reason under the list of files | |
13256 | push @unused_files, "\n$reason\n"; | |
13257 | } | |
13258 | ||
99870f4d KW |
13259 | # Generate a list of the properties whose map table we output, from the |
13260 | # global @map_properties. | |
13261 | my @map_tables_actually_output; | |
13262 | my $info_indent = 20; # Left column is narrower than \p{} table. | |
13263 | foreach my $property (@map_properties) { | |
13264 | ||
13265 | # Get the path to the file; don't output any not in the standard | |
13266 | # directory. | |
13267 | my @path = $property->file_path; | |
13268 | next if $path[0] ne $map_directory; | |
8572ace0 KW |
13269 | |
13270 | # Don't mention map tables that are for internal-use only | |
13271 | next if $property->to_output_map == $INTERNAL_MAP; | |
13272 | ||
99870f4d KW |
13273 | shift @path; # Remove the standard name |
13274 | ||
13275 | my $file = join '/', @path; # In case is in sub directory | |
13276 | my $info = $property->full_name; | |
13277 | my $short_name = $property->name; | |
13278 | if ($info ne $short_name) { | |
13279 | $info .= " ($short_name)"; | |
13280 | } | |
13281 | foreach my $more_info ($property->description, | |
13282 | $property->note, | |
13283 | $property->status_info) | |
13284 | { | |
13285 | next unless $more_info; | |
13286 | $info =~ s/\.\Z//; | |
13287 | $info .= ". $more_info"; | |
13288 | } | |
13289 | push @map_tables_actually_output, format_pod_line($info_indent, | |
13290 | $file, | |
13291 | $info, | |
13292 | $property->status); | |
13293 | } | |
13294 | ||
13295 | # Sort alphabetically, and fold for output | |
13296 | @map_tables_actually_output = sort | |
13297 | pod_alphanumeric_sort @map_tables_actually_output; | |
13298 | @map_tables_actually_output | |
13299 | = simple_fold(\@map_tables_actually_output, | |
13300 | ' ', | |
13301 | $info_indent, | |
13302 | $automatic_pod_indent); | |
13303 | ||
13304 | # Generate a list of the formats that can appear in the map tables. | |
13305 | my @map_table_formats; | |
13306 | foreach my $format (sort keys %map_table_formats) { | |
78632ea9 KW |
13307 | push @map_table_formats, |
13308 | Text::Tabs::expand("$format\t$map_table_formats{$format}\n"); | |
99870f4d | 13309 | } |
78632ea9 KW |
13310 | @map_table_formats = simple_fold(\@map_table_formats, |
13311 | ' ', | |
13312 | 8, | |
13313 | $automatic_pod_indent); | |
12916dad MS |
13314 | local $" = ""; |
13315 | ||
99870f4d KW |
13316 | # Everything is ready to assemble. |
13317 | my @OUT = << "END"; | |
13318 | =begin comment | |
13319 | ||
13320 | $HEADER | |
13321 | ||
13322 | To change this file, edit $0 instead. | |
13323 | ||
13324 | =end comment | |
13325 | ||
13326 | =head1 NAME | |
13327 | ||
8d099389 | 13328 | $pod_file - Index of Unicode Version $string_version character properties in Perl |
99870f4d KW |
13329 | |
13330 | =head1 DESCRIPTION | |
13331 | ||
8d099389 KW |
13332 | This document provides information about the portion of the Unicode database |
13333 | that deals with character properties, that is the portion that is defined on | |
13334 | single code points. (L</Other information in the Unicode data base> | |
13335 | below briefly mentions other data that Unicode provides.) | |
99870f4d | 13336 | |
8d099389 KW |
13337 | Perl can provide access to all non-provisional Unicode character properties, |
13338 | though not all are enabled by default. The omitted ones are the Unihan | |
13339 | properties (accessible via the CPAN module L<Unicode::Unihan>) and certain | |
13340 | deprecated or Unicode-internal properties. (An installation may choose to | |
13341 | recompile Perl's tables to change this. See L<Unicode regular expression | |
13342 | properties that are NOT accepted by Perl>.) | |
13343 | ||
13344 | Perl also provides some additional extensions and short-cut synonyms | |
13345 | for Unicode properties. | |
99870f4d KW |
13346 | |
13347 | This document merely lists all available properties and does not attempt to | |
13348 | explain what each property really means. There is a brief description of each | |
043f3b3f KW |
13349 | Perl extension; see L<perlunicode/Other Properties> for more information on |
13350 | these. There is some detail about Blocks, Scripts, General_Category, | |
99870f4d | 13351 | and Bidi_Class in L<perlunicode>, but to find out about the intricacies of the |
043f3b3f KW |
13352 | official Unicode properties, refer to the Unicode standard. A good starting |
13353 | place is L<$unicode_reference_url>. | |
99870f4d KW |
13354 | |
13355 | Note that you can define your own properties; see | |
13356 | L<perlunicode/"User-Defined Character Properties">. | |
13357 | ||
77173124 | 13358 | =head1 Properties accessible through C<\\p{}> and C<\\P{}> |
99870f4d | 13359 | |
77173124 KW |
13360 | The Perl regular expression C<\\p{}> and C<\\P{}> constructs give access to |
13361 | most of the Unicode character properties. The table below shows all these | |
13362 | constructs, both single and compound forms. | |
99870f4d KW |
13363 | |
13364 | B<Compound forms> consist of two components, separated by an equals sign or a | |
13365 | colon. The first component is the property name, and the second component is | |
13366 | the particular value of the property to match against, for example, | |
77173124 | 13367 | C<\\p{Script: Greek}> and C<\\p{Script=Greek}> both mean to match characters |
99870f4d KW |
13368 | whose Script property is Greek. |
13369 | ||
77173124 | 13370 | B<Single forms>, like C<\\p{Greek}>, are mostly Perl-defined shortcuts for |
99870f4d | 13371 | their equivalent compound forms. The table shows these equivalences. (In our |
77173124 | 13372 | example, C<\\p{Greek}> is a just a shortcut for C<\\p{Script=Greek}>.) |
99870f4d | 13373 | There are also a few Perl-defined single forms that are not shortcuts for a |
77173124 | 13374 | compound form. One such is C<\\p{Word}>. These are also listed in the table. |
99870f4d KW |
13375 | |
13376 | In parsing these constructs, Perl always ignores Upper/lower case differences | |
77173124 KW |
13377 | everywhere within the {braces}. Thus C<\\p{Greek}> means the same thing as |
13378 | C<\\p{greek}>. But note that changing the case of the C<"p"> or C<"P"> before | |
13379 | the left brace completely changes the meaning of the construct, from "match" | |
13380 | (for C<\\p{}>) to "doesn't match" (for C<\\P{}>). Casing in this document is | |
13381 | for improved legibility. | |
99870f4d KW |
13382 | |
13383 | Also, white space, hyphens, and underscores are also normally ignored | |
13384 | everywhere between the {braces}, and hence can be freely added or removed | |
13385 | even if the C</x> modifier hasn't been specified on the regular expression. | |
13386 | But $a_bold_stricter at the beginning of an entry in the table below | |
13387 | means that tighter (stricter) rules are used for that entry: | |
13388 | ||
13389 | =over 4 | |
13390 | ||
77173124 | 13391 | =item Single form (C<\\p{name}>) tighter rules: |
99870f4d KW |
13392 | |
13393 | White space, hyphens, and underscores ARE significant | |
13394 | except for: | |
13395 | ||
13396 | =over 4 | |
13397 | ||
13398 | =item * white space adjacent to a non-word character | |
13399 | ||
13400 | =item * underscores separating digits in numbers | |
13401 | ||
13402 | =back | |
13403 | ||
13404 | That means, for example, that you can freely add or remove white space | |
13405 | adjacent to (but within) the braces without affecting the meaning. | |
13406 | ||
77173124 | 13407 | =item Compound form (C<\\p{name=value}> or C<\\p{name:value}>) tighter rules: |
99870f4d KW |
13408 | |
13409 | The tighter rules given above for the single form apply to everything to the | |
13410 | right of the colon or equals; the looser rules still apply to everything to | |
13411 | the left. | |
13412 | ||
13413 | That means, for example, that you can freely add or remove white space | |
13414 | adjacent to (but within) the braces and the colon or equal sign. | |
13415 | ||
13416 | =back | |
13417 | ||
78bb419c KW |
13418 | Some properties are considered obsolete by Unicode, but still available. |
13419 | There are several varieties of obsolescence: | |
99870f4d KW |
13420 | |
13421 | =over 4 | |
13422 | ||
99870f4d KW |
13423 | =item Stabilized |
13424 | ||
f8c38b14 | 13425 | A property may be stabilized. Such a determination does not indicate |
5f7264c7 KW |
13426 | that the property should or should not be used; instead it is a declaration |
13427 | that the property will not be maintained nor extended for newly encoded | |
13428 | characters. Such properties are marked with $a_bold_stabilized in the | |
13429 | table. | |
99870f4d KW |
13430 | |
13431 | =item Deprecated | |
13432 | ||
f8c38b14 | 13433 | A property may be deprecated, perhaps because its original intent |
78bb419c KW |
13434 | has been replaced by another property, or because its specification was |
13435 | somehow defective. This means that its use is strongly | |
99870f4d KW |
13436 | discouraged, so much so that a warning will be issued if used, unless the |
13437 | regular expression is in the scope of a C<S<no warnings 'deprecated'>> | |
13438 | statement. $A_bold_deprecated flags each such entry in the table, and | |
13439 | the entry there for the longest, most descriptive version of the property will | |
13440 | give the reason it is deprecated, and perhaps advice. Perl may issue such a | |
13441 | warning, even for properties that aren't officially deprecated by Unicode, | |
13442 | when there used to be characters or code points that were matched by them, but | |
13443 | no longer. This is to warn you that your program may not work like it did on | |
13444 | earlier Unicode releases. | |
13445 | ||
13446 | A deprecated property may be made unavailable in a future Perl version, so it | |
13447 | is best to move away from them. | |
13448 | ||
c12f2655 KW |
13449 | A deprecated property may also be stabilized, but this fact is not shown. |
13450 | ||
13451 | =item Obsolete | |
13452 | ||
13453 | Properties marked with $a_bold_obsolete in the table are considered (plain) | |
13454 | obsolete. Generally this designation is given to properties that Unicode once | |
13455 | used for internal purposes (but not any longer). | |
13456 | ||
99870f4d KW |
13457 | =back |
13458 | ||
13459 | Some Perl extensions are present for backwards compatibility and are | |
c12f2655 KW |
13460 | discouraged from being used, but are not obsolete. $A_bold_discouraged |
13461 | flags each such entry in the table. Future Unicode versions may force | |
13462 | some of these extensions to be removed without warning, replaced by another | |
13463 | property with the same name that means something different. Use the | |
13464 | equivalent shown instead. | |
99870f4d KW |
13465 | |
13466 | @block_warning | |
13467 | ||
77173124 | 13468 | The table below has two columns. The left column contains the C<\\p{}> |
98dc9551 | 13469 | constructs to look up, possibly preceded by the flags mentioned above; and |
99870f4d KW |
13470 | the right column contains information about them, like a description, or |
13471 | synonyms. It shows both the single and compound forms for each property that | |
13472 | has them. If the left column is a short name for a property, the right column | |
13473 | will give its longer, more descriptive name; and if the left column is the | |
13474 | longest name, the right column will show any equivalent shortest name, in both | |
13475 | single and compound forms if applicable. | |
13476 | ||
13477 | The right column will also caution you if a property means something different | |
13478 | than what might normally be expected. | |
13479 | ||
d57ccc9a KW |
13480 | All single forms are Perl extensions; a few compound forms are as well, and |
13481 | are noted as such. | |
13482 | ||
99870f4d KW |
13483 | Numbers in (parentheses) indicate the total number of code points matched by |
13484 | the property. For emphasis, those properties that match no code points at all | |
13485 | are listed as well in a separate section following the table. | |
13486 | ||
56ca34ca KW |
13487 | Most properties match the same code points regardless of whether C<"/i"> |
13488 | case-insensitive matching is specified or not. But a few properties are | |
13489 | affected. These are shown with the notation | |
13490 | ||
13491 | (/i= other_property) | |
13492 | ||
13493 | in the second column. Under case-insensitive matching they match the | |
13494 | same code pode points as the property "other_property". | |
13495 | ||
99870f4d | 13496 | There is no description given for most non-Perl defined properties (See |
77173124 | 13497 | L<$unicode_reference_url> for that). |
d73e5302 | 13498 | |
99870f4d KW |
13499 | For compactness, 'B<*>' is used as a wildcard instead of showing all possible |
13500 | combinations. For example, entries like: | |
d73e5302 | 13501 | |
99870f4d | 13502 | \\p{Gc: *} \\p{General_Category: *} |
5beb625e | 13503 | |
99870f4d KW |
13504 | mean that 'Gc' is a synonym for 'General_Category', and anything that is valid |
13505 | for the latter is also valid for the former. Similarly, | |
5beb625e | 13506 | |
99870f4d | 13507 | \\p{Is_*} \\p{*} |
5beb625e | 13508 | |
77173124 KW |
13509 | means that if and only if, for example, C<\\p{Foo}> exists, then |
13510 | C<\\p{Is_Foo}> and C<\\p{IsFoo}> are also valid and all mean the same thing. | |
13511 | And similarly, C<\\p{Foo=Bar}> means the same as C<\\p{Is_Foo=Bar}> and | |
13512 | C<\\p{IsFoo=Bar}>. "*" here is restricted to something not beginning with an | |
13513 | underscore. | |
5beb625e | 13514 | |
99870f4d KW |
13515 | Also, in binary properties, 'Yes', 'T', and 'True' are all synonyms for 'Y'. |
13516 | And 'No', 'F', and 'False' are all synonyms for 'N'. The table shows 'Y*' and | |
13517 | 'N*' to indicate this, and doesn't have separate entries for the other | |
13518 | possibilities. Note that not all properties which have values 'Yes' and 'No' | |
13519 | are binary, and they have all their values spelled out without using this wild | |
13520 | card, and a C<NOT> clause in their description that highlights their not being | |
13521 | binary. These also require the compound form to match them, whereas true | |
13522 | binary properties have both single and compound forms available. | |
5beb625e | 13523 | |
99870f4d KW |
13524 | Note that all non-essential underscores are removed in the display of the |
13525 | short names below. | |
5beb625e | 13526 | |
c12f2655 | 13527 | B<Legend summary:> |
5beb625e | 13528 | |
99870f4d | 13529 | =over 4 |
cf25bb62 | 13530 | |
21405004 | 13531 | =item Z<>B<*> is a wild-card |
cf25bb62 | 13532 | |
99870f4d KW |
13533 | =item B<(\\d+)> in the info column gives the number of code points matched by |
13534 | this property. | |
cf25bb62 | 13535 | |
99870f4d | 13536 | =item B<$DEPRECATED> means this is deprecated. |
cf25bb62 | 13537 | |
99870f4d | 13538 | =item B<$OBSOLETE> means this is obsolete. |
cf25bb62 | 13539 | |
99870f4d | 13540 | =item B<$STABILIZED> means this is stabilized. |
cf25bb62 | 13541 | |
99870f4d | 13542 | =item B<$STRICTER> means tighter (stricter) name matching applies. |
d73e5302 | 13543 | |
c12f2655 KW |
13544 | =item B<$DISCOURAGED> means use of this form is discouraged, and may not be |
13545 | stable. | |
5beb625e | 13546 | |
99870f4d | 13547 | =back |
da7fcca4 | 13548 | |
99870f4d | 13549 | $formatted_properties |
cf25bb62 | 13550 | |
99870f4d | 13551 | $zero_matches |
cf25bb62 | 13552 | |
99870f4d | 13553 | =head1 Properties not accessible through \\p{} and \\P{} |
cf25bb62 | 13554 | |
99870f4d KW |
13555 | A few properties are accessible in Perl via various function calls only. |
13556 | These are: | |
78bb419c | 13557 | |
99870f4d KW |
13558 | Lowercase_Mapping lc() and lcfirst() |
13559 | Titlecase_Mapping ucfirst() | |
13560 | Uppercase_Mapping uc() | |
12ac2576 | 13561 | |
043f3b3f KW |
13562 | Also, Case_Folding is accessible through the C</i> modifier in regular |
13563 | expressions. | |
cf25bb62 | 13564 | |
043f3b3f KW |
13565 | And, the Name and Name_Aliases properties are accessible through the C<\\N{}> |
13566 | interpolation in double-quoted strings and regular expressions, but both | |
13567 | usages require a L<use charnames;|charnames> to be specified, which also | |
13568 | contains related functions viacode(), vianame(), and string_vianame(). | |
cf25bb62 | 13569 | |
99870f4d | 13570 | =head1 Unicode regular expression properties that are NOT accepted by Perl |
d2d499f5 | 13571 | |
99870f4d KW |
13572 | Perl will generate an error for a few character properties in Unicode when |
13573 | used in a regular expression. The non-Unihan ones are listed below, with the | |
13574 | reasons they are not accepted, perhaps with work-arounds. The short names for | |
13575 | the properties are listed enclosed in (parentheses). | |
c12f2655 KW |
13576 | As described after the list, an installation can change the defaults and choose |
13577 | to accept any of these. The list is machine generated based on the | |
13578 | choices made for the installation that generated this document. | |
ae6979a8 | 13579 | |
99870f4d | 13580 | @bad_re_properties |
a3a8c5f0 | 13581 | |
b7986f4f KW |
13582 | An installation can choose to allow any of these to be matched by downloading |
13583 | the Unicode database from L<http://www.unicode.org/Public/> to | |
f3514a2f KW |
13584 | C<\$Config{privlib}>/F<unicore/> in the Perl source tree, changing the |
13585 | controlling lists contained in the program | |
13586 | C<\$Config{privlib}>/F<unicore/mktables> and then re-compiling and installing. | |
13587 | (C<\%Config> is available from the Config module). | |
d73e5302 | 13588 | |
99870f4d | 13589 | =head1 Files in the I<To> directory (for serious hackers only) |
12ac2576 | 13590 | |
99870f4d KW |
13591 | All Unicode properties are really mappings (in the mathematical sense) from |
13592 | code points to their respective values. As part of its build process, | |
13593 | Perl constructs tables containing these mappings for all properties that it | |
50b27e73 | 13594 | deals with. Some, but not all, of these are written out into files. |
99870f4d | 13595 | Those written out are in the directory C<\$Config{privlib}>/F<unicore/To/> |
77173124 | 13596 | (C<%Config> is available from the C<Config> module). |
7ebf06b3 | 13597 | |
50b27e73 KW |
13598 | Perl reserves the right to change the format and even the existence of any of |
13599 | those files without notice, except the ones that were in existence prior to | |
c6d31e50 | 13600 | release 5.14. If those change, a deprecation cycle will be done first. These |
50b27e73 | 13601 | are: |
12ac2576 | 13602 | |
99870f4d | 13603 | @map_tables_actually_output |
12ac2576 | 13604 | |
ec2f0128 KW |
13605 | Each of the files in this directory defines several hash entries to help |
13606 | reading programs decipher it. One of them looks like this: | |
12ac2576 | 13607 | |
99870f4d | 13608 | \$utf8::SwashInfo{'ToNAME'}{'format'} = 's'; |
d73e5302 | 13609 | |
77173124 KW |
13610 | where "NAME" is a name to indicate the property. For backwards compatibility, |
13611 | this is not necessarily the property's official Unicode name. (The "To" is | |
99870f4d KW |
13612 | also for backwards compatibility.) The hash entry gives the format of the |
13613 | mapping fields of the table, currently one of the following: | |
d73e5302 | 13614 | |
12916dad | 13615 | @map_table_formats |
d73e5302 | 13616 | |
99870f4d KW |
13617 | This format applies only to the entries in the main body of the table. |
13618 | Entries defined in hashes or ones that are missing from the list can have a | |
13619 | different format. | |
d73e5302 | 13620 | |
ec2f0128 | 13621 | The value that the missing entries have is given by another SwashInfo hash |
99870f4d | 13622 | entry line; it looks like this: |
d73e5302 | 13623 | |
99870f4d | 13624 | \$utf8::SwashInfo{'ToNAME'}{'missing'} = 'NaN'; |
d73e5302 | 13625 | |
99870f4d | 13626 | This example line says that any Unicode code points not explicitly listed in |
77173124 | 13627 | the file have the value "NaN" under the property indicated by NAME. If the |
99870f4d KW |
13628 | value is the special string C<< <code point> >>, it means that the value for |
13629 | any missing code point is the code point itself. This happens, for example, | |
13630 | in the file for Uppercase_Mapping (To/Upper.pl), in which code points like the | |
77173124 | 13631 | character "A", are missing because the uppercase of "A" is itself. |
d73e5302 | 13632 | |
ec2f0128 KW |
13633 | Finally, if the file contains a hash for special case entries, its name is |
13634 | specified by an entry that looks like this: | |
13635 | ||
13636 | \$utf8::SwashInfo{'ToNAME'}{'specials_name'} = 'utf8::ToSpecNAME'; | |
13637 | ||
8d099389 KW |
13638 | |
13639 | =head1 Other information in the Unicode data base | |
13640 | ||
13641 | The Unicode data base is delivered in two different formats. The XML version | |
13642 | is valid for more modern Unicode releases. The other version is a collection | |
13643 | of files. The two are intended to give equivalent information. Perl uses the | |
13644 | older form; this allows you to recompile Perl to use early Unicode releases. | |
13645 | ||
13646 | The only non-character property that Perl currently supports is Named | |
13647 | Sequences, in which a sequence of code points | |
13648 | is given a name and generally treated as a single entity. (Perl supports | |
13649 | these via the C<\\N{...}> double-quotish construct, | |
13650 | L<charnames/charnames::string_vianame(name)>, and L<Unicode::UCD/namedseq()>. | |
13651 | ||
13652 | Below is a list of the files in the Unicode data base that Perl doesn't | |
13653 | currently use, along with very brief descriptions of their purposes. | |
13654 | Some of the names of the files have been shortened from those that Unicode | |
13655 | uses, in order to allow them to be distinguishable from similarly named files | |
13656 | on file systems for which only the first 8 characters of a name are | |
13657 | significant. | |
13658 | ||
13659 | =over 4 | |
13660 | ||
13661 | @unused_files | |
13662 | ||
13663 | =back | |
13664 | ||
99870f4d | 13665 | =head1 SEE ALSO |
d73e5302 | 13666 | |
99870f4d | 13667 | L<$unicode_reference_url> |
12ac2576 | 13668 | |
99870f4d | 13669 | L<perlrecharclass> |
12ac2576 | 13670 | |
99870f4d | 13671 | L<perlunicode> |
d73e5302 | 13672 | |
99870f4d | 13673 | END |
d73e5302 | 13674 | |
9218f1cf KW |
13675 | # And write it. The 0 means no utf8. |
13676 | main::write([ $pod_directory, "$pod_file.pod" ], 0, \@OUT); | |
99870f4d KW |
13677 | return; |
13678 | } | |
d73e5302 | 13679 | |
99870f4d KW |
13680 | sub make_Heavy () { |
13681 | # Create and write Heavy.pl, which passes info about the tables to | |
13682 | # utf8_heavy.pl | |
12ac2576 | 13683 | |
99870f4d KW |
13684 | my @heavy = <<END; |
13685 | $HEADER | |
126c3d4e | 13686 | $INTERNAL_ONLY_HEADER |
d73e5302 | 13687 | |
99870f4d | 13688 | # This file is for the use of utf8_heavy.pl |
12ac2576 | 13689 | |
c12f2655 KW |
13690 | # Maps Unicode (not Perl single-form extensions) property names in loose |
13691 | # standard form to their corresponding standard names | |
99870f4d KW |
13692 | \%utf8::loose_property_name_of = ( |
13693 | END | |
cf25bb62 | 13694 | |
99870f4d KW |
13695 | push @heavy, simple_dumper (\%loose_property_name_of, ' ' x 4); |
13696 | push @heavy, <<END; | |
13697 | ); | |
12ac2576 | 13698 | |
99870f4d KW |
13699 | # Maps property, table to file for those using stricter matching |
13700 | \%utf8::stricter_to_file_of = ( | |
13701 | END | |
13702 | push @heavy, simple_dumper (\%stricter_to_file_of, ' ' x 4); | |
13703 | push @heavy, <<END; | |
13704 | ); | |
12ac2576 | 13705 | |
99870f4d KW |
13706 | # Maps property, table to file for those using loose matching |
13707 | \%utf8::loose_to_file_of = ( | |
13708 | END | |
13709 | push @heavy, simple_dumper (\%loose_to_file_of, ' ' x 4); | |
13710 | push @heavy, <<END; | |
13711 | ); | |
12ac2576 | 13712 | |
99870f4d KW |
13713 | # Maps floating point to fractional form |
13714 | \%utf8::nv_floating_to_rational = ( | |
13715 | END | |
13716 | push @heavy, simple_dumper (\%nv_floating_to_rational, ' ' x 4); | |
13717 | push @heavy, <<END; | |
13718 | ); | |
12ac2576 | 13719 | |
99870f4d KW |
13720 | # If a floating point number doesn't have enough digits in it to get this |
13721 | # close to a fraction, it isn't considered to be that fraction even if all the | |
13722 | # digits it does have match. | |
13723 | \$utf8::max_floating_slop = $MAX_FLOATING_SLOP; | |
12ac2576 | 13724 | |
99870f4d KW |
13725 | # Deprecated tables to generate a warning for. The key is the file containing |
13726 | # the table, so as to avoid duplication, as many property names can map to the | |
13727 | # file, but we only need one entry for all of them. | |
13728 | \%utf8::why_deprecated = ( | |
13729 | END | |
12ac2576 | 13730 | |
99870f4d KW |
13731 | push @heavy, simple_dumper (\%utf8::why_deprecated, ' ' x 4); |
13732 | push @heavy, <<END; | |
13733 | ); | |
12ac2576 | 13734 | |
d867ccfb KW |
13735 | # A few properties have different behavior under /i matching. This maps the |
13736 | # those to substitute files to use under /i. | |
13737 | \%utf8::caseless_equivalent = ( | |
13738 | END | |
13739 | ||
d867ccfb KW |
13740 | # We set the key to the file when we associated files with tables, but we |
13741 | # couldn't do the same for the value then, as we might not have the file | |
13742 | # for the alternate table figured out at that time. | |
13743 | foreach my $cased (keys %caseless_equivalent_to) { | |
13744 | my @path = $caseless_equivalent_to{$cased}->file_path; | |
13745 | my $path = join '/', @path[1, -1]; | |
d867ccfb KW |
13746 | $utf8::caseless_equivalent_to{$cased} = $path; |
13747 | } | |
13748 | push @heavy, simple_dumper (\%utf8::caseless_equivalent_to, ' ' x 4); | |
13749 | push @heavy, <<END; | |
13750 | ); | |
13751 | ||
99870f4d KW |
13752 | 1; |
13753 | END | |
12ac2576 | 13754 | |
9218f1cf | 13755 | main::write("Heavy.pl", 0, \@heavy); # The 0 means no utf8. |
99870f4d | 13756 | return; |
12ac2576 JP |
13757 | } |
13758 | ||
52dc8b5d | 13759 | sub make_Name_pm () { |
6f424f62 | 13760 | # Create and write Name.pm, which contains subroutines and data to use in |
52dc8b5d KW |
13761 | # conjunction with Name.pl |
13762 | ||
bb1dd3da KW |
13763 | # Maybe there's nothing to do. |
13764 | return unless $has_hangul_syllables || @code_points_ending_in_code_point; | |
13765 | ||
52dc8b5d KW |
13766 | my @name = <<END; |
13767 | $HEADER | |
126c3d4e | 13768 | $INTERNAL_ONLY_HEADER |
52dc8b5d | 13769 | END |
0f6f7bc2 | 13770 | |
fb848dce KW |
13771 | # Convert these structures to output format. |
13772 | my $code_points_ending_in_code_point = | |
13773 | main::simple_dumper(\@code_points_ending_in_code_point, | |
13774 | ' ' x 8); | |
13775 | my $names = main::simple_dumper(\%names_ending_in_code_point, | |
13776 | ' ' x 8); | |
13777 | my $loose_names = main::simple_dumper(\%loose_names_ending_in_code_point, | |
0f6f7bc2 | 13778 | ' ' x 8); |
0f6f7bc2 | 13779 | |
fb848dce KW |
13780 | # Do the same with the Hangul names, |
13781 | my $jamo; | |
13782 | my $jamo_l; | |
13783 | my $jamo_v; | |
13784 | my $jamo_t; | |
13785 | my $jamo_re; | |
13786 | if ($has_hangul_syllables) { | |
0f6f7bc2 | 13787 | |
fb848dce KW |
13788 | # Construct a regular expression of all the possible |
13789 | # combinations of the Hangul syllables. | |
13790 | my @L_re; # Leading consonants | |
13791 | for my $i ($LBase .. $LBase + $LCount - 1) { | |
13792 | push @L_re, $Jamo{$i} | |
13793 | } | |
13794 | my @V_re; # Middle vowels | |
13795 | for my $i ($VBase .. $VBase + $VCount - 1) { | |
13796 | push @V_re, $Jamo{$i} | |
13797 | } | |
13798 | my @T_re; # Trailing consonants | |
13799 | for my $i ($TBase + 1 .. $TBase + $TCount - 1) { | |
13800 | push @T_re, $Jamo{$i} | |
13801 | } | |
0f6f7bc2 | 13802 | |
fb848dce KW |
13803 | # The whole re is made up of the L V T combination. |
13804 | $jamo_re = '(' | |
13805 | . join ('|', sort @L_re) | |
13806 | . ')(' | |
13807 | . join ('|', sort @V_re) | |
13808 | . ')(' | |
13809 | . join ('|', sort @T_re) | |
13810 | . ')?'; | |
0f6f7bc2 | 13811 | |
fb848dce KW |
13812 | # These hashes needed by the algorithm were generated |
13813 | # during reading of the Jamo.txt file | |
13814 | $jamo = main::simple_dumper(\%Jamo, ' ' x 8); | |
13815 | $jamo_l = main::simple_dumper(\%Jamo_L, ' ' x 8); | |
13816 | $jamo_v = main::simple_dumper(\%Jamo_V, ' ' x 8); | |
13817 | $jamo_t = main::simple_dumper(\%Jamo_T, ' ' x 8); | |
13818 | } | |
0f6f7bc2 | 13819 | |
6f424f62 | 13820 | push @name, <<END; |
0f6f7bc2 | 13821 | |
6f424f62 KW |
13822 | # This module contains machine-generated tables and code for the |
13823 | # algorithmically-determinable Unicode character names. The following | |
13824 | # routines can be used to translate between name and code point and vice versa | |
0f6f7bc2 KW |
13825 | |
13826 | { # Closure | |
13827 | ||
92199589 KW |
13828 | # Matches legal code point. 4-6 hex numbers, If there are 6, the first |
13829 | # two must be 10; if there are 5, the first must not be a 0. Written this | |
13830 | # way to decrease backtracking. The first regex allows the code point to | |
13831 | # be at the end of a word, but to work properly, the word shouldn't end | |
13832 | # with a valid hex character. The second one won't match a code point at | |
13833 | # the end of a word, and doesn't have the run-on issue | |
0f6f7bc2 KW |
13834 | my \$run_on_code_point_re = qr/$run_on_code_point_re/; |
13835 | my \$code_point_re = qr/$code_point_re/; | |
13836 | ||
13837 | # In the following hash, the keys are the bases of names which includes | |
13838 | # the code point in the name, like CJK UNIFIED IDEOGRAPH-4E01. The values | |
13839 | # of each key is another hash which is used to get the low and high ends | |
13840 | # for each range of code points that apply to the name. | |
13841 | my %names_ending_in_code_point = ( | |
13842 | $names | |
13843 | ); | |
13844 | ||
13845 | # The following hash is a copy of the previous one, except is for loose | |
13846 | # matching, so each name has blanks and dashes squeezed out | |
13847 | my %loose_names_ending_in_code_point = ( | |
13848 | $loose_names | |
13849 | ); | |
13850 | ||
13851 | # And the following array gives the inverse mapping from code points to | |
13852 | # names. Lowest code points are first | |
13853 | my \@code_points_ending_in_code_point = ( | |
13854 | $code_points_ending_in_code_point | |
13855 | ); | |
13856 | END | |
fb848dce KW |
13857 | # Earlier releases didn't have Jamos. No sense outputting |
13858 | # them unless will be used. | |
13859 | if ($has_hangul_syllables) { | |
6f424f62 | 13860 | push @name, <<END; |
0f6f7bc2 KW |
13861 | |
13862 | # Convert from code point to Jamo short name for use in composing Hangul | |
13863 | # syllable names | |
13864 | my %Jamo = ( | |
13865 | $jamo | |
13866 | ); | |
13867 | ||
13868 | # Leading consonant (can be null) | |
13869 | my %Jamo_L = ( | |
13870 | $jamo_l | |
13871 | ); | |
13872 | ||
13873 | # Vowel | |
13874 | my %Jamo_V = ( | |
13875 | $jamo_v | |
13876 | ); | |
13877 | ||
13878 | # Optional trailing consonant | |
13879 | my %Jamo_T = ( | |
13880 | $jamo_t | |
13881 | ); | |
13882 | ||
13883 | # Computed re that splits up a Hangul name into LVT or LV syllables | |
13884 | my \$syllable_re = qr/$jamo_re/; | |
13885 | ||
13886 | my \$HANGUL_SYLLABLE = "HANGUL SYLLABLE "; | |
13887 | my \$loose_HANGUL_SYLLABLE = "HANGULSYLLABLE"; | |
13888 | ||
13889 | # These constants names and values were taken from the Unicode standard, | |
13890 | # version 5.1, section 3.12. They are used in conjunction with Hangul | |
13891 | # syllables | |
13892 | my \$SBase = $SBase_string; | |
13893 | my \$LBase = $LBase_string; | |
13894 | my \$VBase = $VBase_string; | |
13895 | my \$TBase = $TBase_string; | |
13896 | my \$SCount = $SCount; | |
13897 | my \$LCount = $LCount; | |
13898 | my \$VCount = $VCount; | |
13899 | my \$TCount = $TCount; | |
13900 | my \$NCount = \$VCount * \$TCount; | |
13901 | END | |
fb848dce | 13902 | } # End of has Jamos |
0f6f7bc2 | 13903 | |
6f424f62 | 13904 | push @name, << 'END'; |
0f6f7bc2 KW |
13905 | |
13906 | sub name_to_code_point_special { | |
13907 | my ($name, $loose) = @_; | |
13908 | ||
13909 | # Returns undef if not one of the specially handled names; otherwise | |
13910 | # returns the code point equivalent to the input name | |
13911 | # $loose is non-zero if to use loose matching, 'name' in that case | |
13912 | # must be input as upper case with all blanks and dashes squeezed out. | |
13913 | END | |
fb848dce | 13914 | if ($has_hangul_syllables) { |
6f424f62 | 13915 | push @name, << 'END'; |
0f6f7bc2 KW |
13916 | |
13917 | if ((! $loose && $name =~ s/$HANGUL_SYLLABLE//) | |
13918 | || ($loose && $name =~ s/$loose_HANGUL_SYLLABLE//)) | |
13919 | { | |
13920 | return if $name !~ qr/^$syllable_re$/; | |
13921 | my $L = $Jamo_L{$1}; | |
13922 | my $V = $Jamo_V{$2}; | |
13923 | my $T = (defined $3) ? $Jamo_T{$3} : 0; | |
13924 | return ($L * $VCount + $V) * $TCount + $T + $SBase; | |
13925 | } | |
13926 | END | |
fb848dce | 13927 | } |
6f424f62 | 13928 | push @name, << 'END'; |
0f6f7bc2 KW |
13929 | |
13930 | # Name must end in 'code_point' for this to handle. | |
13931 | return if (($loose && $name !~ /^ (.*?) ($run_on_code_point_re) $/x) | |
13932 | || (! $loose && $name !~ /^ (.*) ($code_point_re) $/x)); | |
13933 | ||
13934 | my $base = $1; | |
13935 | my $code_point = CORE::hex $2; | |
13936 | my $names_ref; | |
13937 | ||
13938 | if ($loose) { | |
13939 | $names_ref = \%loose_names_ending_in_code_point; | |
13940 | } | |
13941 | else { | |
13942 | return if $base !~ s/-$//; | |
13943 | $names_ref = \%names_ending_in_code_point; | |
13944 | } | |
13945 | ||
13946 | # Name must be one of the ones which has the code point in it. | |
13947 | return if ! $names_ref->{$base}; | |
13948 | ||
13949 | # Look through the list of ranges that apply to this name to see if | |
13950 | # the code point is in one of them. | |
13951 | for (my $i = 0; $i < scalar @{$names_ref->{$base}{'low'}}; $i++) { | |
13952 | return if $names_ref->{$base}{'low'}->[$i] > $code_point; | |
13953 | next if $names_ref->{$base}{'high'}->[$i] < $code_point; | |
13954 | ||
13955 | # Here, the code point is in the range. | |
13956 | return $code_point; | |
13957 | } | |
13958 | ||
13959 | # Here, looked like the name had a code point number in it, but | |
13960 | # did not match one of the valid ones. | |
13961 | return; | |
13962 | } | |
13963 | ||
13964 | sub code_point_to_name_special { | |
13965 | my $code_point = shift; | |
13966 | ||
13967 | # Returns the name of a code point if algorithmically determinable; | |
13968 | # undef if not | |
13969 | END | |
fb848dce | 13970 | if ($has_hangul_syllables) { |
6f424f62 | 13971 | push @name, << 'END'; |
0f6f7bc2 KW |
13972 | |
13973 | # If in the Hangul range, calculate the name based on Unicode's | |
13974 | # algorithm | |
13975 | if ($code_point >= $SBase && $code_point <= $SBase + $SCount -1) { | |
13976 | use integer; | |
13977 | my $SIndex = $code_point - $SBase; | |
13978 | my $L = $LBase + $SIndex / $NCount; | |
13979 | my $V = $VBase + ($SIndex % $NCount) / $TCount; | |
13980 | my $T = $TBase + $SIndex % $TCount; | |
13981 | $name = "$HANGUL_SYLLABLE$Jamo{$L}$Jamo{$V}"; | |
13982 | $name .= $Jamo{$T} if $T != $TBase; | |
13983 | return $name; | |
13984 | } | |
13985 | END | |
fb848dce | 13986 | } |
6f424f62 | 13987 | push @name, << 'END'; |
0f6f7bc2 KW |
13988 | |
13989 | # Look through list of these code points for one in range. | |
13990 | foreach my $hash (@code_points_ending_in_code_point) { | |
13991 | return if $code_point < $hash->{'low'}; | |
13992 | if ($code_point <= $hash->{'high'}) { | |
13993 | return sprintf("%s-%04X", $hash->{'name'}, $code_point); | |
13994 | } | |
13995 | } | |
13996 | return; # None found | |
13997 | } | |
13998 | } # End closure | |
13999 | ||
6f424f62 | 14000 | 1; |
0f6f7bc2 | 14001 | END |
52dc8b5d KW |
14002 | |
14003 | main::write("Name.pm", 0, \@name); # The 0 means no utf8. | |
14004 | return; | |
14005 | } | |
14006 | ||
14007 | ||
99870f4d KW |
14008 | sub write_all_tables() { |
14009 | # Write out all the tables generated by this program to files, as well as | |
14010 | # the supporting data structures, pod file, and .t file. | |
14011 | ||
14012 | my @writables; # List of tables that actually get written | |
14013 | my %match_tables_to_write; # Used to collapse identical match tables | |
14014 | # into one file. Each key is a hash function | |
14015 | # result to partition tables into buckets. | |
14016 | # Each value is an array of the tables that | |
14017 | # fit in the bucket. | |
14018 | ||
14019 | # For each property ... | |
14020 | # (sort so that if there is an immutable file name, it has precedence, so | |
14021 | # some other property can't come in and take over its file name. If b's | |
14022 | # file name is defined, will return 1, meaning to take it first; don't | |
7fc6cb55 KW |
14023 | # care if both defined, as they had better be different anyway. And the |
14024 | # property named 'Perl' needs to be first (it doesn't have any immutable | |
14025 | # file name) because empty properties are defined in terms of it's table | |
14026 | # named 'Any'.) | |
99870f4d | 14027 | PROPERTY: |
7fc6cb55 KW |
14028 | foreach my $property (sort { return -1 if $a == $perl; |
14029 | return 1 if $b == $perl; | |
14030 | return defined $b->file | |
14031 | } property_ref('*')) | |
14032 | { | |
99870f4d KW |
14033 | my $type = $property->type; |
14034 | ||
14035 | # And for each table for that property, starting with the mapping | |
14036 | # table for it ... | |
14037 | TABLE: | |
14038 | foreach my $table($property, | |
14039 | ||
14040 | # and all the match tables for it (if any), sorted so | |
14041 | # the ones with the shortest associated file name come | |
14042 | # first. The length sorting prevents problems of a | |
14043 | # longer file taking a name that might have to be used | |
14044 | # by a shorter one. The alphabetic sorting prevents | |
14045 | # differences between releases | |
14046 | sort { my $ext_a = $a->external_name; | |
14047 | return 1 if ! defined $ext_a; | |
14048 | my $ext_b = $b->external_name; | |
14049 | return -1 if ! defined $ext_b; | |
a92d5c2e KW |
14050 | |
14051 | # But return the non-complement table before | |
14052 | # the complement one, as the latter is defined | |
14053 | # in terms of the former, and needs to have | |
14054 | # the information for the former available. | |
14055 | return 1 if $a->complement != 0; | |
14056 | return -1 if $b->complement != 0; | |
14057 | ||
0a695432 KW |
14058 | # Similarly, return a subservient table after |
14059 | # a leader | |
14060 | return 1 if $a->leader != $a; | |
14061 | return -1 if $b->leader != $b; | |
14062 | ||
99870f4d KW |
14063 | my $cmp = length $ext_a <=> length $ext_b; |
14064 | ||
14065 | # Return result if lengths not equal | |
14066 | return $cmp if $cmp; | |
14067 | ||
14068 | # Alphabetic if lengths equal | |
14069 | return $ext_a cmp $ext_b | |
14070 | } $property->tables | |
14071 | ) | |
14072 | { | |
12ac2576 | 14073 | |
99870f4d KW |
14074 | # Here we have a table associated with a property. It could be |
14075 | # the map table (done first for each property), or one of the | |
14076 | # other tables. Determine which type. | |
14077 | my $is_property = $table->isa('Property'); | |
14078 | ||
14079 | my $name = $table->name; | |
14080 | my $complete_name = $table->complete_name; | |
14081 | ||
14082 | # See if should suppress the table if is empty, but warn if it | |
14083 | # contains something. | |
14084 | my $suppress_if_empty_warn_if_not = grep { $complete_name eq $_ } | |
14085 | keys %why_suppress_if_empty_warn_if_not; | |
14086 | ||
14087 | # Calculate if this table should have any code points associated | |
14088 | # with it or not. | |
14089 | my $expected_empty = | |
14090 | ||
14091 | # $perl should be empty, as well as properties that we just | |
14092 | # don't do anything with | |
14093 | ($is_property | |
14094 | && ($table == $perl | |
14095 | || grep { $complete_name eq $_ } | |
14096 | @unimplemented_properties | |
14097 | ) | |
14098 | ) | |
14099 | ||
14100 | # Match tables in properties we skipped populating should be | |
14101 | # empty | |
14102 | || (! $is_property && ! $property->to_create_match_tables) | |
14103 | ||
14104 | # Tables and properties that are expected to have no code | |
14105 | # points should be empty | |
14106 | || $suppress_if_empty_warn_if_not | |
14107 | ; | |
14108 | ||
14109 | # Set a boolean if this table is the complement of an empty binary | |
14110 | # table | |
14111 | my $is_complement_of_empty_binary = | |
14112 | $type == $BINARY && | |
14113 | (($table == $property->table('Y') | |
14114 | && $property->table('N')->is_empty) | |
14115 | || ($table == $property->table('N') | |
14116 | && $property->table('Y')->is_empty)); | |
14117 | ||
14118 | ||
14119 | # Some tables should match everything | |
14120 | my $expected_full = | |
14121 | ($is_property) | |
14122 | ? # All these types of map tables will be full because | |
14123 | # they will have been populated with defaults | |
06f26c45 | 14124 | ($type == $ENUM || $type == $FORCED_BINARY) |
99870f4d KW |
14125 | |
14126 | : # A match table should match everything if its method | |
14127 | # shows it should | |
14128 | ($table->matches_all | |
14129 | ||
14130 | # The complement of an empty binary table will match | |
14131 | # everything | |
14132 | || $is_complement_of_empty_binary | |
14133 | ) | |
14134 | ; | |
14135 | ||
14136 | if ($table->is_empty) { | |
14137 | ||
99870f4d KW |
14138 | if ($suppress_if_empty_warn_if_not) { |
14139 | $table->set_status($SUPPRESSED, | |
14140 | $why_suppress_if_empty_warn_if_not{$complete_name}); | |
14141 | } | |
12ac2576 | 14142 | |
c12f2655 | 14143 | # Suppress (by skipping them) expected empty tables. |
99870f4d KW |
14144 | next TABLE if $expected_empty; |
14145 | ||
14146 | # And setup to later output a warning for those that aren't | |
14147 | # known to be allowed to be empty. Don't do the warning if | |
14148 | # this table is a child of another one to avoid duplicating | |
14149 | # the warning that should come from the parent one. | |
14150 | if (($table == $property || $table->parent == $table) | |
14151 | && $table->status ne $SUPPRESSED | |
14152 | && ! grep { $complete_name =~ /^$_$/ } | |
14153 | @tables_that_may_be_empty) | |
14154 | { | |
14155 | push @unhandled_properties, "$table"; | |
14156 | } | |
7fc6cb55 KW |
14157 | |
14158 | # An empty table is just the complement of everything. | |
14159 | $table->set_complement($Any) if $table != $property; | |
99870f4d KW |
14160 | } |
14161 | elsif ($expected_empty) { | |
14162 | my $because = ""; | |
14163 | if ($suppress_if_empty_warn_if_not) { | |
14164 | $because = " because $why_suppress_if_empty_warn_if_not{$complete_name}"; | |
14165 | } | |
12ac2576 | 14166 | |
99870f4d KW |
14167 | Carp::my_carp("Not expecting property $table$because. Generating file for it anyway."); |
14168 | } | |
12ac2576 | 14169 | |
99870f4d KW |
14170 | my $count = $table->count; |
14171 | if ($expected_full) { | |
14172 | if ($count != $MAX_UNICODE_CODEPOINTS) { | |
14173 | Carp::my_carp("$table matches only " | |
14174 | . clarify_number($count) | |
14175 | . " Unicode code points but should match " | |
14176 | . clarify_number($MAX_UNICODE_CODEPOINTS) | |
14177 | . " (off by " | |
14178 | . clarify_number(abs($MAX_UNICODE_CODEPOINTS - $count)) | |
14179 | . "). Proceeding anyway."); | |
14180 | } | |
12ac2576 | 14181 | |
99870f4d KW |
14182 | # Here is expected to be full. If it is because it is the |
14183 | # complement of an (empty) binary table that is to be | |
14184 | # suppressed, then suppress this one as well. | |
14185 | if ($is_complement_of_empty_binary) { | |
14186 | my $opposing_name = ($name eq 'Y') ? 'N' : 'Y'; | |
14187 | my $opposing = $property->table($opposing_name); | |
14188 | my $opposing_status = $opposing->status; | |
14189 | if ($opposing_status) { | |
14190 | $table->set_status($opposing_status, | |
14191 | $opposing->status_info); | |
14192 | } | |
14193 | } | |
14194 | } | |
14195 | elsif ($count == $MAX_UNICODE_CODEPOINTS) { | |
14196 | if ($table == $property || $table->leader == $table) { | |
14197 | Carp::my_carp("$table unexpectedly matches all Unicode code points. Proceeding anyway."); | |
14198 | } | |
14199 | } | |
d73e5302 | 14200 | |
99870f4d KW |
14201 | if ($table->status eq $SUPPRESSED) { |
14202 | if (! $is_property) { | |
14203 | my @children = $table->children; | |
14204 | foreach my $child (@children) { | |
14205 | if ($child->status ne $SUPPRESSED) { | |
14206 | Carp::my_carp_bug("'$table' is suppressed and has a child '$child' which isn't"); | |
14207 | } | |
14208 | } | |
14209 | } | |
14210 | next TABLE; | |
d73e5302 | 14211 | |
99870f4d KW |
14212 | } |
14213 | if (! $is_property) { | |
14214 | ||
14215 | # Several things need to be done just once for each related | |
14216 | # group of match tables. Do them on the parent. | |
14217 | if ($table->parent == $table) { | |
14218 | ||
14219 | # Add an entry in the pod file for the table; it also does | |
14220 | # the children. | |
23e33b60 | 14221 | make_table_pod_entries($table) if defined $pod_directory; |
99870f4d KW |
14222 | |
14223 | # See if the the table matches identical code points with | |
14224 | # something that has already been output. In that case, | |
14225 | # no need to have two files with the same code points in | |
14226 | # them. We use the table's hash() method to store these | |
14227 | # in buckets, so that it is quite likely that if two | |
14228 | # tables are in the same bucket they will be identical, so | |
14229 | # don't have to compare tables frequently. The tables | |
14230 | # have to have the same status to share a file, so add | |
14231 | # this to the bucket hash. (The reason for this latter is | |
14232 | # that Heavy.pl associates a status with a file.) | |
06671cbc KW |
14233 | # We don't check tables that are inverses of others, as it |
14234 | # would lead to some coding complications, and checking | |
14235 | # all the regular ones should find everything. | |
14236 | if ($table->complement == 0) { | |
21be712a | 14237 | my $hash = $table->hash . ';' . $table->status; |
99870f4d | 14238 | |
21be712a KW |
14239 | # Look at each table that is in the same bucket as |
14240 | # this one would be. | |
14241 | foreach my $comparison | |
14242 | (@{$match_tables_to_write{$hash}}) | |
14243 | { | |
14244 | if ($table->matches_identically_to($comparison)) { | |
14245 | $table->set_equivalent_to($comparison, | |
99870f4d | 14246 | Related => 0); |
21be712a KW |
14247 | next TABLE; |
14248 | } | |
99870f4d | 14249 | } |
d73e5302 | 14250 | |
21be712a KW |
14251 | # Here, not equivalent, add this table to the bucket. |
14252 | push @{$match_tables_to_write{$hash}}, $table; | |
06671cbc | 14253 | } |
99870f4d KW |
14254 | } |
14255 | } | |
14256 | else { | |
14257 | ||
14258 | # Here is the property itself. | |
14259 | # Don't write out or make references to the $perl property | |
14260 | next if $table == $perl; | |
14261 | ||
14262 | if ($type != $STRING) { | |
14263 | ||
14264 | # There is a mapping stored of the various synonyms to the | |
14265 | # standardized name of the property for utf8_heavy.pl. | |
14266 | # Also, the pod file contains entries of the form: | |
14267 | # \p{alias: *} \p{full: *} | |
14268 | # rather than show every possible combination of things. | |
14269 | ||
14270 | my @property_aliases = $property->aliases; | |
14271 | ||
14272 | # The full name of this property is stored by convention | |
14273 | # first in the alias array | |
14274 | my $full_property_name = | |
14275 | '\p{' . $property_aliases[0]->name . ': *}'; | |
14276 | my $standard_property_name = standardize($table->name); | |
14277 | ||
14278 | # For each synonym ... | |
14279 | for my $i (0 .. @property_aliases - 1) { | |
14280 | my $alias = $property_aliases[$i]; | |
14281 | my $alias_name = $alias->name; | |
14282 | my $alias_standard = standardize($alias_name); | |
14283 | ||
c12f2655 | 14284 | # For utf8_heavy, set the mapping of the alias to the |
99870f4d KW |
14285 | # property |
14286 | if (exists ($loose_property_name_of{$alias_standard})) | |
14287 | { | |
14288 | Carp::my_carp("There already is a property with the same standard name as $alias_name: $loose_property_name_of{$alias_standard}. Old name is retained"); | |
14289 | } | |
14290 | else { | |
14291 | $loose_property_name_of{$alias_standard} | |
14292 | = $standard_property_name; | |
14293 | } | |
14294 | ||
23e33b60 KW |
14295 | # Now for the pod entry for this alias. Skip if not |
14296 | # outputting a pod; skip the first one, which is the | |
14297 | # full name so won't have an entry like: '\p{full: *} | |
14298 | # \p{full: *}', and skip if don't want an entry for | |
14299 | # this one. | |
14300 | next if $i == 0 | |
14301 | || ! defined $pod_directory | |
33e96e72 | 14302 | || ! $alias->make_re_pod_entry; |
99870f4d | 14303 | |
d57ccc9a KW |
14304 | my $rhs = $full_property_name; |
14305 | if ($property != $perl && $table->perl_extension) { | |
14306 | $rhs .= ' (Perl extension)'; | |
14307 | } | |
99870f4d KW |
14308 | push @match_properties, |
14309 | format_pod_line($indent_info_column, | |
14310 | '\p{' . $alias->name . ': *}', | |
d57ccc9a | 14311 | $rhs, |
99870f4d KW |
14312 | $alias->status); |
14313 | } | |
14314 | } # End of non-string-like property code | |
d73e5302 | 14315 | |
d73e5302 | 14316 | |
c12f2655 | 14317 | # Don't write out a mapping file if not desired. |
99870f4d KW |
14318 | next if ! $property->to_output_map; |
14319 | } | |
d73e5302 | 14320 | |
99870f4d KW |
14321 | # Here, we know we want to write out the table, but don't do it |
14322 | # yet because there may be other tables that come along and will | |
14323 | # want to share the file, and the file's comments will change to | |
14324 | # mention them. So save for later. | |
14325 | push @writables, $table; | |
14326 | ||
14327 | } # End of looping through the property and all its tables. | |
14328 | } # End of looping through all properties. | |
14329 | ||
14330 | # Now have all the tables that will have files written for them. Do it. | |
14331 | foreach my $table (@writables) { | |
14332 | my @directory; | |
14333 | my $filename; | |
14334 | my $property = $table->property; | |
14335 | my $is_property = ($table == $property); | |
14336 | if (! $is_property) { | |
14337 | ||
14338 | # Match tables for the property go in lib/$subdirectory, which is | |
14339 | # the property's name. Don't use the standard file name for this, | |
14340 | # as may get an unfamiliar alias | |
14341 | @directory = ($matches_directory, $property->external_name); | |
14342 | } | |
14343 | else { | |
d73e5302 | 14344 | |
99870f4d KW |
14345 | @directory = $table->directory; |
14346 | $filename = $table->file; | |
14347 | } | |
d73e5302 | 14348 | |
98dc9551 | 14349 | # Use specified filename if available, or default to property's |
99870f4d KW |
14350 | # shortest name. We need an 8.3 safe filename (which means "an 8 |
14351 | # safe" filename, since after the dot is only 'pl', which is < 3) | |
14352 | # The 2nd parameter is if the filename shouldn't be changed, and | |
14353 | # it shouldn't iff there is a hard-coded name for this table. | |
14354 | $filename = construct_filename( | |
14355 | $filename || $table->external_name, | |
14356 | ! $filename, # mutable if no filename | |
14357 | \@directory); | |
d73e5302 | 14358 | |
99870f4d | 14359 | register_file_for_name($table, \@directory, $filename); |
d73e5302 | 14360 | |
99870f4d KW |
14361 | # Only need to write one file when shared by more than one |
14362 | # property | |
a92d5c2e KW |
14363 | next if ! $is_property |
14364 | && ($table->leader != $table || $table->complement != 0); | |
d73e5302 | 14365 | |
99870f4d KW |
14366 | # Construct a nice comment to add to the file |
14367 | $table->set_final_comment; | |
14368 | ||
14369 | $table->write; | |
cf25bb62 | 14370 | } |
d73e5302 | 14371 | |
d73e5302 | 14372 | |
99870f4d KW |
14373 | # Write out the pod file |
14374 | make_pod; | |
14375 | ||
52dc8b5d | 14376 | # And Heavy.pl, Name.pm |
99870f4d | 14377 | make_Heavy; |
52dc8b5d | 14378 | make_Name_pm; |
d73e5302 | 14379 | |
99870f4d KW |
14380 | make_property_test_script() if $make_test_script; |
14381 | return; | |
cf25bb62 | 14382 | } |
d73e5302 | 14383 | |
99870f4d KW |
14384 | my @white_space_separators = ( # This used only for making the test script. |
14385 | "", | |
14386 | ' ', | |
14387 | "\t", | |
14388 | ' ' | |
14389 | ); | |
d73e5302 | 14390 | |
99870f4d KW |
14391 | sub generate_separator($) { |
14392 | # This used only for making the test script. It generates the colon or | |
14393 | # equal separator between the property and property value, with random | |
14394 | # white space surrounding the separator | |
d73e5302 | 14395 | |
99870f4d | 14396 | my $lhs = shift; |
d73e5302 | 14397 | |
99870f4d | 14398 | return "" if $lhs eq ""; # No separator if there's only one (the r) side |
d73e5302 | 14399 | |
99870f4d KW |
14400 | # Choose space before and after randomly |
14401 | my $spaces_before =$white_space_separators[rand(@white_space_separators)]; | |
14402 | my $spaces_after = $white_space_separators[rand(@white_space_separators)]; | |
76ccdbe2 | 14403 | |
99870f4d KW |
14404 | # And return the whole complex, half the time using a colon, half the |
14405 | # equals | |
14406 | return $spaces_before | |
14407 | . (rand() < 0.5) ? '=' : ':' | |
14408 | . $spaces_after; | |
14409 | } | |
76ccdbe2 | 14410 | |
430ada4c | 14411 | sub generate_tests($$$$$) { |
99870f4d KW |
14412 | # This used only for making the test script. It generates test cases that |
14413 | # are expected to compile successfully in perl. Note that the lhs and | |
14414 | # rhs are assumed to already be as randomized as the caller wants. | |
14415 | ||
99870f4d KW |
14416 | my $lhs = shift; # The property: what's to the left of the colon |
14417 | # or equals separator | |
14418 | my $rhs = shift; # The property value; what's to the right | |
14419 | my $valid_code = shift; # A code point that's known to be in the | |
14420 | # table given by lhs=rhs; undef if table is | |
14421 | # empty | |
14422 | my $invalid_code = shift; # A code point known to not be in the table; | |
14423 | # undef if the table is all code points | |
14424 | my $warning = shift; | |
14425 | ||
14426 | # Get the colon or equal | |
14427 | my $separator = generate_separator($lhs); | |
14428 | ||
14429 | # The whole 'property=value' | |
14430 | my $name = "$lhs$separator$rhs"; | |
14431 | ||
430ada4c | 14432 | my @output; |
99870f4d KW |
14433 | # Create a complete set of tests, with complements. |
14434 | if (defined $valid_code) { | |
430ada4c NC |
14435 | push @output, <<"EOC" |
14436 | Expect(1, $valid_code, '\\p{$name}', $warning); | |
14437 | Expect(0, $valid_code, '\\p{^$name}', $warning); | |
14438 | Expect(0, $valid_code, '\\P{$name}', $warning); | |
14439 | Expect(1, $valid_code, '\\P{^$name}', $warning); | |
14440 | EOC | |
99870f4d KW |
14441 | } |
14442 | if (defined $invalid_code) { | |
430ada4c NC |
14443 | push @output, <<"EOC" |
14444 | Expect(0, $invalid_code, '\\p{$name}', $warning); | |
14445 | Expect(1, $invalid_code, '\\p{^$name}', $warning); | |
14446 | Expect(1, $invalid_code, '\\P{$name}', $warning); | |
14447 | Expect(0, $invalid_code, '\\P{^$name}', $warning); | |
14448 | EOC | |
14449 | } | |
14450 | return @output; | |
99870f4d | 14451 | } |
cf25bb62 | 14452 | |
430ada4c | 14453 | sub generate_error($$$) { |
99870f4d KW |
14454 | # This used only for making the test script. It generates test cases that |
14455 | # are expected to not only not match, but to be syntax or similar errors | |
14456 | ||
99870f4d KW |
14457 | my $lhs = shift; # The property: what's to the left of the |
14458 | # colon or equals separator | |
14459 | my $rhs = shift; # The property value; what's to the right | |
14460 | my $already_in_error = shift; # Boolean; if true it's known that the | |
14461 | # unmodified lhs and rhs will cause an error. | |
14462 | # This routine should not force another one | |
14463 | # Get the colon or equal | |
14464 | my $separator = generate_separator($lhs); | |
14465 | ||
14466 | # Since this is an error only, don't bother to randomly decide whether to | |
14467 | # put the error on the left or right side; and assume that the rhs is | |
14468 | # loosely matched, again for convenience rather than rigor. | |
14469 | $rhs = randomize_loose_name($rhs, 'ERROR') unless $already_in_error; | |
14470 | ||
14471 | my $property = $lhs . $separator . $rhs; | |
14472 | ||
430ada4c NC |
14473 | return <<"EOC"; |
14474 | Error('\\p{$property}'); | |
14475 | Error('\\P{$property}'); | |
14476 | EOC | |
d73e5302 JH |
14477 | } |
14478 | ||
99870f4d KW |
14479 | # These are used only for making the test script |
14480 | # XXX Maybe should also have a bad strict seps, which includes underscore. | |
14481 | ||
14482 | my @good_loose_seps = ( | |
14483 | " ", | |
14484 | "-", | |
14485 | "\t", | |
14486 | "", | |
14487 | "_", | |
14488 | ); | |
14489 | my @bad_loose_seps = ( | |
14490 | "/a/", | |
14491 | ':=', | |
14492 | ); | |
14493 | ||
14494 | sub randomize_stricter_name { | |
14495 | # This used only for making the test script. Take the input name and | |
14496 | # return a randomized, but valid version of it under the stricter matching | |
14497 | # rules. | |
14498 | ||
14499 | my $name = shift; | |
14500 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
14501 | ||
14502 | # If the name looks like a number (integer, floating, or rational), do | |
14503 | # some extra work | |
14504 | if ($name =~ qr{ ^ ( -? ) (\d+ ( ( [./] ) \d+ )? ) $ }x) { | |
14505 | my $sign = $1; | |
14506 | my $number = $2; | |
14507 | my $separator = $3; | |
14508 | ||
14509 | # If there isn't a sign, part of the time add a plus | |
14510 | # Note: Not testing having any denominator having a minus sign | |
14511 | if (! $sign) { | |
14512 | $sign = '+' if rand() <= .3; | |
14513 | } | |
14514 | ||
14515 | # And add 0 or more leading zeros. | |
14516 | $name = $sign . ('0' x int rand(10)) . $number; | |
14517 | ||
14518 | if (defined $separator) { | |
14519 | my $extra_zeros = '0' x int rand(10); | |
cf25bb62 | 14520 | |
99870f4d KW |
14521 | if ($separator eq '.') { |
14522 | ||
14523 | # Similarly, add 0 or more trailing zeros after a decimal | |
14524 | # point | |
14525 | $name .= $extra_zeros; | |
14526 | } | |
14527 | else { | |
14528 | ||
14529 | # Or, leading zeros before the denominator | |
14530 | $name =~ s,/,/$extra_zeros,; | |
14531 | } | |
14532 | } | |
cf25bb62 | 14533 | } |
d73e5302 | 14534 | |
99870f4d KW |
14535 | # For legibility of the test, only change the case of whole sections at a |
14536 | # time. To do this, first split into sections. The split returns the | |
14537 | # delimiters | |
14538 | my @sections; | |
14539 | for my $section (split / ( [ - + \s _ . ]+ ) /x, $name) { | |
14540 | trace $section if main::DEBUG && $to_trace; | |
14541 | ||
14542 | if (length $section > 1 && $section !~ /\D/) { | |
14543 | ||
14544 | # If the section is a sequence of digits, about half the time | |
14545 | # randomly add underscores between some of them. | |
14546 | if (rand() > .5) { | |
14547 | ||
14548 | # Figure out how many underscores to add. max is 1 less than | |
14549 | # the number of digits. (But add 1 at the end to make sure | |
14550 | # result isn't 0, and compensate earlier by subtracting 2 | |
14551 | # instead of 1) | |
14552 | my $num_underscores = int rand(length($section) - 2) + 1; | |
14553 | ||
14554 | # And add them evenly throughout, for convenience, not rigor | |
14555 | use integer; | |
14556 | my $spacing = (length($section) - 1)/ $num_underscores; | |
14557 | my $temp = $section; | |
14558 | $section = ""; | |
14559 | for my $i (1 .. $num_underscores) { | |
14560 | $section .= substr($temp, 0, $spacing, "") . '_'; | |
14561 | } | |
14562 | $section .= $temp; | |
14563 | } | |
14564 | push @sections, $section; | |
14565 | } | |
14566 | else { | |
d73e5302 | 14567 | |
99870f4d KW |
14568 | # Here not a sequence of digits. Change the case of the section |
14569 | # randomly | |
14570 | my $switch = int rand(4); | |
14571 | if ($switch == 0) { | |
14572 | push @sections, uc $section; | |
14573 | } | |
14574 | elsif ($switch == 1) { | |
14575 | push @sections, lc $section; | |
14576 | } | |
14577 | elsif ($switch == 2) { | |
14578 | push @sections, ucfirst $section; | |
14579 | } | |
14580 | else { | |
14581 | push @sections, $section; | |
14582 | } | |
14583 | } | |
cf25bb62 | 14584 | } |
99870f4d KW |
14585 | trace "returning", join "", @sections if main::DEBUG && $to_trace; |
14586 | return join "", @sections; | |
14587 | } | |
71d929cb | 14588 | |
99870f4d KW |
14589 | sub randomize_loose_name($;$) { |
14590 | # This used only for making the test script | |
71d929cb | 14591 | |
99870f4d KW |
14592 | my $name = shift; |
14593 | my $want_error = shift; # if true, make an error | |
14594 | Carp::carp_extra_args(\@_) if main::DEBUG && @_; | |
14595 | ||
14596 | $name = randomize_stricter_name($name); | |
5beb625e JH |
14597 | |
14598 | my @parts; | |
99870f4d | 14599 | push @parts, $good_loose_seps[rand(@good_loose_seps)]; |
45376db6 KW |
14600 | |
14601 | # Preserve trailing ones for the sake of not stripping the underscore from | |
14602 | # 'L_' | |
14603 | for my $part (split /[-\s_]+ (?= . )/, $name) { | |
5beb625e | 14604 | if (@parts) { |
99870f4d KW |
14605 | if ($want_error and rand() < 0.3) { |
14606 | push @parts, $bad_loose_seps[rand(@bad_loose_seps)]; | |
14607 | $want_error = 0; | |
14608 | } | |
14609 | else { | |
14610 | push @parts, $good_loose_seps[rand(@good_loose_seps)]; | |
5beb625e JH |
14611 | } |
14612 | } | |
99870f4d | 14613 | push @parts, $part; |
5beb625e | 14614 | } |
99870f4d KW |
14615 | my $new = join("", @parts); |
14616 | trace "$name => $new" if main::DEBUG && $to_trace; | |
5beb625e | 14617 | |
99870f4d | 14618 | if ($want_error) { |
5beb625e | 14619 | if (rand() >= 0.5) { |
99870f4d KW |
14620 | $new .= $bad_loose_seps[rand(@bad_loose_seps)]; |
14621 | } | |
14622 | else { | |
14623 | $new = $bad_loose_seps[rand(@bad_loose_seps)] . $new; | |
5beb625e JH |
14624 | } |
14625 | } | |
14626 | return $new; | |
14627 | } | |
14628 | ||
99870f4d KW |
14629 | # Used to make sure don't generate duplicate test cases. |
14630 | my %test_generated; | |
5beb625e | 14631 | |
99870f4d KW |
14632 | sub make_property_test_script() { |
14633 | # This used only for making the test script | |
14634 | # this written directly -- it's huge. | |
5beb625e | 14635 | |
99870f4d | 14636 | print "Making test script\n" if $verbosity >= $PROGRESS; |
5beb625e | 14637 | |
99870f4d KW |
14638 | # This uses randomness to test different possibilities without testing all |
14639 | # possibilities. To ensure repeatability, set the seed to 0. But if | |
14640 | # tests are added, it will perturb all later ones in the .t file | |
14641 | srand 0; | |
5beb625e | 14642 | |
3df51b85 KW |
14643 | $t_path = 'TestProp.pl' unless defined $t_path; # the traditional name |
14644 | ||
99870f4d KW |
14645 | # Keep going down an order of magnitude |
14646 | # until find that adding this quantity to | |
14647 | # 1 remains 1; but put an upper limit on | |
14648 | # this so in case this algorithm doesn't | |
14649 | # work properly on some platform, that we | |
14650 | # won't loop forever. | |
14651 | my $digits = 0; | |
14652 | my $min_floating_slop = 1; | |
14653 | while (1+ $min_floating_slop != 1 | |
14654 | && $digits++ < 50) | |
5beb625e | 14655 | { |
99870f4d KW |
14656 | my $next = $min_floating_slop / 10; |
14657 | last if $next == 0; # If underflows, | |
14658 | # use previous one | |
14659 | $min_floating_slop = $next; | |
5beb625e | 14660 | } |
430ada4c NC |
14661 | |
14662 | # It doesn't matter whether the elements of this array contain single lines | |
14663 | # or multiple lines. main::write doesn't count the lines. | |
14664 | my @output; | |
99870f4d KW |
14665 | |
14666 | foreach my $property (property_ref('*')) { | |
14667 | foreach my $table ($property->tables) { | |
14668 | ||
14669 | # Find code points that match, and don't match this table. | |
14670 | my $valid = $table->get_valid_code_point; | |
14671 | my $invalid = $table->get_invalid_code_point; | |
14672 | my $warning = ($table->status eq $DEPRECATED) | |
14673 | ? "'deprecated'" | |
14674 | : '""'; | |
14675 | ||
14676 | # Test each possible combination of the property's aliases with | |
14677 | # the table's. If this gets to be too many, could do what is done | |
14678 | # in the set_final_comment() for Tables | |
14679 | my @table_aliases = $table->aliases; | |
14680 | my @property_aliases = $table->property->aliases; | |
807807b7 KW |
14681 | |
14682 | # Every property can be optionally be prefixed by 'Is_', so test | |
14683 | # that those work, by creating such a new alias for each | |
14684 | # pre-existing one. | |
14685 | push @property_aliases, map { Alias->new("Is_" . $_->name, | |
14686 | $_->loose_match, | |
33e96e72 | 14687 | $_->make_re_pod_entry, |
807807b7 KW |
14688 | $_->externally_ok, |
14689 | $_->status) | |
14690 | } @property_aliases; | |
99870f4d KW |
14691 | my $max = max(scalar @table_aliases, scalar @property_aliases); |
14692 | for my $j (0 .. $max - 1) { | |
14693 | ||
14694 | # The current alias for property is the next one on the list, | |
14695 | # or if beyond the end, start over. Similarly for table | |
14696 | my $property_name | |
14697 | = $property_aliases[$j % @property_aliases]->name; | |
14698 | ||
14699 | $property_name = "" if $table->property == $perl; | |
14700 | my $table_alias = $table_aliases[$j % @table_aliases]; | |
14701 | my $table_name = $table_alias->name; | |
14702 | my $loose_match = $table_alias->loose_match; | |
14703 | ||
14704 | # If the table doesn't have a file, any test for it is | |
14705 | # already guaranteed to be in error | |
14706 | my $already_error = ! $table->file_path; | |
14707 | ||
14708 | # Generate error cases for this alias. | |
430ada4c NC |
14709 | push @output, generate_error($property_name, |
14710 | $table_name, | |
14711 | $already_error); | |
99870f4d KW |
14712 | |
14713 | # If the table is guaranteed to always generate an error, | |
14714 | # quit now without generating success cases. | |
14715 | next if $already_error; | |
14716 | ||
14717 | # Now for the success cases. | |
14718 | my $random; | |
14719 | if ($loose_match) { | |
14720 | ||
14721 | # For loose matching, create an extra test case for the | |
14722 | # standard name. | |
14723 | my $standard = standardize($table_name); | |
14724 | ||
14725 | # $test_name should be a unique combination for each test | |
14726 | # case; used just to avoid duplicate tests | |
14727 | my $test_name = "$property_name=$standard"; | |
14728 | ||
14729 | # Don't output duplicate test cases. | |
14730 | if (! exists $test_generated{$test_name}) { | |
14731 | $test_generated{$test_name} = 1; | |
430ada4c NC |
14732 | push @output, generate_tests($property_name, |
14733 | $standard, | |
14734 | $valid, | |
14735 | $invalid, | |
14736 | $warning, | |
14737 | ); | |
5beb625e | 14738 | } |
99870f4d KW |
14739 | $random = randomize_loose_name($table_name) |
14740 | } | |
14741 | else { # Stricter match | |
14742 | $random = randomize_stricter_name($table_name); | |
99598c8c | 14743 | } |
99598c8c | 14744 | |
99870f4d KW |
14745 | # Now for the main test case for this alias. |
14746 | my $test_name = "$property_name=$random"; | |
14747 | if (! exists $test_generated{$test_name}) { | |
14748 | $test_generated{$test_name} = 1; | |
430ada4c NC |
14749 | push @output, generate_tests($property_name, |
14750 | $random, | |
14751 | $valid, | |
14752 | $invalid, | |
14753 | $warning, | |
14754 | ); | |
99870f4d KW |
14755 | |
14756 | # If the name is a rational number, add tests for the | |
14757 | # floating point equivalent. | |
14758 | if ($table_name =~ qr{/}) { | |
14759 | ||
14760 | # Calculate the float, and find just the fraction. | |
14761 | my $float = eval $table_name; | |
14762 | my ($whole, $fraction) | |
14763 | = $float =~ / (.*) \. (.*) /x; | |
14764 | ||
14765 | # Starting with one digit after the decimal point, | |
14766 | # create a test for each possible precision (number of | |
14767 | # digits past the decimal point) until well beyond the | |
14768 | # native number found on this machine. (If we started | |
14769 | # with 0 digits, it would be an integer, which could | |
14770 | # well match an unrelated table) | |
14771 | PLACE: | |
14772 | for my $i (1 .. $min_floating_slop + 3) { | |
14773 | my $table_name = sprintf("%.*f", $i, $float); | |
14774 | if ($i < $MIN_FRACTION_LENGTH) { | |
14775 | ||
14776 | # If the test case has fewer digits than the | |
14777 | # minimum acceptable precision, it shouldn't | |
14778 | # succeed, so we expect an error for it. | |
14779 | # E.g., 2/3 = .7 at one decimal point, and we | |
14780 | # shouldn't say it matches .7. We should make | |
14781 | # it be .667 at least before agreeing that the | |
14782 | # intent was to match 2/3. But at the | |
14783 | # less-than- acceptable level of precision, it | |
14784 | # might actually match an unrelated number. | |
14785 | # So don't generate a test case if this | |
14786 | # conflating is possible. In our example, we | |
14787 | # don't want 2/3 matching 7/10, if there is | |
14788 | # a 7/10 code point. | |
14789 | for my $existing | |
14790 | (keys %nv_floating_to_rational) | |
14791 | { | |
14792 | next PLACE | |
14793 | if abs($table_name - $existing) | |
14794 | < $MAX_FLOATING_SLOP; | |
14795 | } | |
430ada4c NC |
14796 | push @output, generate_error($property_name, |
14797 | $table_name, | |
14798 | 1 # 1 => already an error | |
14799 | ); | |
99870f4d KW |
14800 | } |
14801 | else { | |
14802 | ||
14803 | # Here the number of digits exceeds the | |
14804 | # minimum we think is needed. So generate a | |
14805 | # success test case for it. | |
430ada4c NC |
14806 | push @output, generate_tests($property_name, |
14807 | $table_name, | |
14808 | $valid, | |
14809 | $invalid, | |
14810 | $warning, | |
14811 | ); | |
99870f4d KW |
14812 | } |
14813 | } | |
99598c8c JH |
14814 | } |
14815 | } | |
99870f4d KW |
14816 | } |
14817 | } | |
14818 | } | |
37e2e78e | 14819 | |
9218f1cf KW |
14820 | &write($t_path, |
14821 | 0, # Not utf8; | |
14822 | [<DATA>, | |
14823 | @output, | |
14824 | (map {"Test_X('$_');\n"} @backslash_X_tests), | |
14825 | "Finished();\n"]); | |
99870f4d KW |
14826 | return; |
14827 | } | |
99598c8c | 14828 | |
99870f4d KW |
14829 | # This is a list of the input files and how to handle them. The files are |
14830 | # processed in their order in this list. Some reordering is possible if | |
14831 | # desired, but the v0 files should be first, and the extracted before the | |
14832 | # others except DAge.txt (as data in an extracted file can be over-ridden by | |
14833 | # the non-extracted. Some other files depend on data derived from an earlier | |
14834 | # file, like UnicodeData requires data from Jamo, and the case changing and | |
14835 | # folding requires data from Unicode. Mostly, it safest to order by first | |
14836 | # version releases in (except the Jamo). DAge.txt is read before the | |
14837 | # extracted ones because of the rarely used feature $compare_versions. In the | |
14838 | # unlikely event that there were ever an extracted file that contained the Age | |
14839 | # property information, it would have to go in front of DAge. | |
14840 | # | |
14841 | # The version strings allow the program to know whether to expect a file or | |
14842 | # not, but if a file exists in the directory, it will be processed, even if it | |
14843 | # is in a version earlier than expected, so you can copy files from a later | |
14844 | # release into an earlier release's directory. | |
14845 | my @input_file_objects = ( | |
14846 | Input_file->new('PropertyAliases.txt', v0, | |
14847 | Handler => \&process_PropertyAliases, | |
14848 | ), | |
14849 | Input_file->new(undef, v0, # No file associated with this | |
3df51b85 | 14850 | Progress_Message => 'Finishing property setup', |
99870f4d KW |
14851 | Handler => \&finish_property_setup, |
14852 | ), | |
14853 | Input_file->new('PropValueAliases.txt', v0, | |
14854 | Handler => \&process_PropValueAliases, | |
14855 | Has_Missings_Defaults => $NOT_IGNORED, | |
14856 | ), | |
14857 | Input_file->new('DAge.txt', v3.2.0, | |
14858 | Has_Missings_Defaults => $NOT_IGNORED, | |
14859 | Property => 'Age' | |
14860 | ), | |
14861 | Input_file->new("${EXTRACTED}DGeneralCategory.txt", v3.1.0, | |
14862 | Property => 'General_Category', | |
14863 | ), | |
14864 | Input_file->new("${EXTRACTED}DCombiningClass.txt", v3.1.0, | |
14865 | Property => 'Canonical_Combining_Class', | |
14866 | Has_Missings_Defaults => $NOT_IGNORED, | |
14867 | ), | |
14868 | Input_file->new("${EXTRACTED}DNumType.txt", v3.1.0, | |
14869 | Property => 'Numeric_Type', | |
14870 | Has_Missings_Defaults => $NOT_IGNORED, | |
14871 | ), | |
14872 | Input_file->new("${EXTRACTED}DEastAsianWidth.txt", v3.1.0, | |
14873 | Property => 'East_Asian_Width', | |
14874 | Has_Missings_Defaults => $NOT_IGNORED, | |
14875 | ), | |
14876 | Input_file->new("${EXTRACTED}DLineBreak.txt", v3.1.0, | |
14877 | Property => 'Line_Break', | |
14878 | Has_Missings_Defaults => $NOT_IGNORED, | |
14879 | ), | |
14880 | Input_file->new("${EXTRACTED}DBidiClass.txt", v3.1.1, | |
14881 | Property => 'Bidi_Class', | |
14882 | Has_Missings_Defaults => $NOT_IGNORED, | |
14883 | ), | |
14884 | Input_file->new("${EXTRACTED}DDecompositionType.txt", v3.1.0, | |
14885 | Property => 'Decomposition_Type', | |
14886 | Has_Missings_Defaults => $NOT_IGNORED, | |
14887 | ), | |
14888 | Input_file->new("${EXTRACTED}DBinaryProperties.txt", v3.1.0), | |
14889 | Input_file->new("${EXTRACTED}DNumValues.txt", v3.1.0, | |
14890 | Property => 'Numeric_Value', | |
14891 | Each_Line_Handler => \&filter_numeric_value_line, | |
14892 | Has_Missings_Defaults => $NOT_IGNORED, | |
14893 | ), | |
14894 | Input_file->new("${EXTRACTED}DJoinGroup.txt", v3.1.0, | |
14895 | Property => 'Joining_Group', | |
14896 | Has_Missings_Defaults => $NOT_IGNORED, | |
14897 | ), | |
14898 | ||
14899 | Input_file->new("${EXTRACTED}DJoinType.txt", v3.1.0, | |
14900 | Property => 'Joining_Type', | |
14901 | Has_Missings_Defaults => $NOT_IGNORED, | |
14902 | ), | |
14903 | Input_file->new('Jamo.txt', v2.0.0, | |
14904 | Property => 'Jamo_Short_Name', | |
14905 | Each_Line_Handler => \&filter_jamo_line, | |
14906 | ), | |
14907 | Input_file->new('UnicodeData.txt', v1.1.5, | |
14908 | Pre_Handler => \&setup_UnicodeData, | |
14909 | ||
14910 | # We clean up this file for some early versions. | |
14911 | Each_Line_Handler => [ (($v_version lt v2.0.0 ) | |
14912 | ? \&filter_v1_ucd | |
14913 | : ($v_version eq v2.1.5) | |
14914 | ? \&filter_v2_1_5_ucd | |
3ffed8c2 KW |
14915 | |
14916 | # And for 5.14 Perls with 6.0, | |
14917 | # have to also make changes | |
14918 | : ($v_version ge v6.0.0) | |
14919 | ? \&filter_v6_ucd | |
14920 | : undef), | |
99870f4d KW |
14921 | |
14922 | # And the main filter | |
14923 | \&filter_UnicodeData_line, | |
14924 | ], | |
14925 | EOF_Handler => \&EOF_UnicodeData, | |
14926 | ), | |
14927 | Input_file->new('ArabicShaping.txt', v2.0.0, | |
14928 | Each_Line_Handler => | |
14929 | [ ($v_version lt 4.1.0) | |
14930 | ? \&filter_old_style_arabic_shaping | |
14931 | : undef, | |
14932 | \&filter_arabic_shaping_line, | |
14933 | ], | |
14934 | Has_Missings_Defaults => $NOT_IGNORED, | |
14935 | ), | |
14936 | Input_file->new('Blocks.txt', v2.0.0, | |
14937 | Property => 'Block', | |
14938 | Has_Missings_Defaults => $NOT_IGNORED, | |
14939 | Each_Line_Handler => \&filter_blocks_lines | |
14940 | ), | |
14941 | Input_file->new('PropList.txt', v2.0.0, | |
14942 | Each_Line_Handler => (($v_version lt v3.1.0) | |
14943 | ? \&filter_old_style_proplist | |
14944 | : undef), | |
14945 | ), | |
14946 | Input_file->new('Unihan.txt', v2.0.0, | |
14947 | Pre_Handler => \&setup_unihan, | |
14948 | Optional => 1, | |
14949 | Each_Line_Handler => \&filter_unihan_line, | |
14950 | ), | |
14951 | Input_file->new('SpecialCasing.txt', v2.1.8, | |
14952 | Each_Line_Handler => \&filter_special_casing_line, | |
14953 | Pre_Handler => \&setup_special_casing, | |
14954 | ), | |
14955 | Input_file->new( | |
14956 | 'LineBreak.txt', v3.0.0, | |
14957 | Has_Missings_Defaults => $NOT_IGNORED, | |
14958 | Property => 'Line_Break', | |
14959 | # Early versions had problematic syntax | |
14960 | Each_Line_Handler => (($v_version lt v3.1.0) | |
14961 | ? \&filter_early_ea_lb | |
14962 | : undef), | |
14963 | ), | |
14964 | Input_file->new('EastAsianWidth.txt', v3.0.0, | |
14965 | Property => 'East_Asian_Width', | |
14966 | Has_Missings_Defaults => $NOT_IGNORED, | |
14967 | # Early versions had problematic syntax | |
14968 | Each_Line_Handler => (($v_version lt v3.1.0) | |
14969 | ? \&filter_early_ea_lb | |
14970 | : undef), | |
14971 | ), | |
14972 | Input_file->new('CompositionExclusions.txt', v3.0.0, | |
14973 | Property => 'Composition_Exclusion', | |
14974 | ), | |
14975 | Input_file->new('BidiMirroring.txt', v3.0.1, | |
14976 | Property => 'Bidi_Mirroring_Glyph', | |
14977 | ), | |
37e2e78e | 14978 | Input_file->new("NormalizationTest.txt", v3.0.1, |
09ca89ce | 14979 | Skip => 'Validation Tests', |
37e2e78e | 14980 | ), |
99870f4d KW |
14981 | Input_file->new('CaseFolding.txt', v3.0.1, |
14982 | Pre_Handler => \&setup_case_folding, | |
14983 | Each_Line_Handler => | |
14984 | [ ($v_version lt v3.1.0) | |
14985 | ? \&filter_old_style_case_folding | |
14986 | : undef, | |
14987 | \&filter_case_folding_line | |
14988 | ], | |
99870f4d KW |
14989 | ), |
14990 | Input_file->new('DCoreProperties.txt', v3.1.0, | |
14991 | # 5.2 changed this file | |
14992 | Has_Missings_Defaults => (($v_version ge v5.2.0) | |
14993 | ? $NOT_IGNORED | |
14994 | : $NO_DEFAULTS), | |
14995 | ), | |
14996 | Input_file->new('Scripts.txt', v3.1.0, | |
14997 | Property => 'Script', | |
14998 | Has_Missings_Defaults => $NOT_IGNORED, | |
14999 | ), | |
15000 | Input_file->new('DNormalizationProps.txt', v3.1.0, | |
15001 | Has_Missings_Defaults => $NOT_IGNORED, | |
15002 | Each_Line_Handler => (($v_version lt v4.0.1) | |
15003 | ? \&filter_old_style_normalization_lines | |
15004 | : undef), | |
15005 | ), | |
15006 | Input_file->new('HangulSyllableType.txt', v4.0.0, | |
15007 | Has_Missings_Defaults => $NOT_IGNORED, | |
15008 | Property => 'Hangul_Syllable_Type'), | |
15009 | Input_file->new("$AUXILIARY/WordBreakProperty.txt", v4.1.0, | |
15010 | Property => 'Word_Break', | |
15011 | Has_Missings_Defaults => $NOT_IGNORED, | |
15012 | ), | |
15013 | Input_file->new("$AUXILIARY/GraphemeBreakProperty.txt", v4.1.0, | |
15014 | Property => 'Grapheme_Cluster_Break', | |
15015 | Has_Missings_Defaults => $NOT_IGNORED, | |
15016 | ), | |
37e2e78e KW |
15017 | Input_file->new("$AUXILIARY/GCBTest.txt", v4.1.0, |
15018 | Handler => \&process_GCB_test, | |
15019 | ), | |
15020 | Input_file->new("$AUXILIARY/LBTest.txt", v4.1.0, | |
09ca89ce | 15021 | Skip => 'Validation Tests', |
37e2e78e KW |
15022 | ), |
15023 | Input_file->new("$AUXILIARY/SBTest.txt", v4.1.0, | |
09ca89ce | 15024 | Skip => 'Validation Tests', |
37e2e78e KW |
15025 | ), |
15026 | Input_file->new("$AUXILIARY/WBTest.txt", v4.1.0, | |
09ca89ce | 15027 | Skip => 'Validation Tests', |
37e2e78e | 15028 | ), |
99870f4d KW |
15029 | Input_file->new("$AUXILIARY/SentenceBreakProperty.txt", v4.1.0, |
15030 | Property => 'Sentence_Break', | |
15031 | Has_Missings_Defaults => $NOT_IGNORED, | |
15032 | ), | |
15033 | Input_file->new('NamedSequences.txt', v4.1.0, | |
15034 | Handler => \&process_NamedSequences | |
15035 | ), | |
15036 | Input_file->new('NameAliases.txt', v5.0.0, | |
15037 | Property => 'Name_Alias', | |
dcd72625 KW |
15038 | Pre_Handler => ($v_version ge v6.0.0) |
15039 | ? \&setup_v6_name_alias | |
15040 | : undef, | |
99870f4d | 15041 | ), |
37e2e78e | 15042 | Input_file->new("BidiTest.txt", v5.2.0, |
09ca89ce | 15043 | Skip => 'Validation Tests', |
37e2e78e | 15044 | ), |
99870f4d KW |
15045 | Input_file->new('UnihanIndicesDictionary.txt', v5.2.0, |
15046 | Optional => 1, | |
15047 | Each_Line_Handler => \&filter_unihan_line, | |
15048 | ), | |
15049 | Input_file->new('UnihanDataDictionaryLike.txt', v5.2.0, | |
15050 | Optional => 1, | |
15051 | Each_Line_Handler => \&filter_unihan_line, | |
15052 | ), | |
15053 | Input_file->new('UnihanIRGSources.txt', v5.2.0, | |
15054 | Optional => 1, | |
15055 | Pre_Handler => \&setup_unihan, | |
15056 | Each_Line_Handler => \&filter_unihan_line, | |
15057 | ), | |
15058 | Input_file->new('UnihanNumericValues.txt', v5.2.0, | |
15059 | Optional => 1, | |
15060 | Each_Line_Handler => \&filter_unihan_line, | |
15061 | ), | |
15062 | Input_file->new('UnihanOtherMappings.txt', v5.2.0, | |
15063 | Optional => 1, | |
15064 | Each_Line_Handler => \&filter_unihan_line, | |
15065 | ), | |
15066 | Input_file->new('UnihanRadicalStrokeCounts.txt', v5.2.0, | |
15067 | Optional => 1, | |
15068 | Each_Line_Handler => \&filter_unihan_line, | |
15069 | ), | |
15070 | Input_file->new('UnihanReadings.txt', v5.2.0, | |
15071 | Optional => 1, | |
15072 | Each_Line_Handler => \&filter_unihan_line, | |
15073 | ), | |
15074 | Input_file->new('UnihanVariants.txt', v5.2.0, | |
15075 | Optional => 1, | |
15076 | Each_Line_Handler => \&filter_unihan_line, | |
15077 | ), | |
82aed44a KW |
15078 | Input_file->new('ScriptExtensions.txt', v6.0.0, |
15079 | Property => 'Script_Extensions', | |
15080 | Pre_Handler => \&setup_script_extensions, | |
fbe1e607 | 15081 | Each_Line_Handler => \&filter_script_extensions_line, |
82aed44a | 15082 | ), |
99870f4d | 15083 | ); |
99598c8c | 15084 | |
99870f4d KW |
15085 | # End of all the preliminaries. |
15086 | # Do it... | |
99598c8c | 15087 | |
99870f4d KW |
15088 | if ($compare_versions) { |
15089 | Carp::my_carp(<<END | |
15090 | Warning. \$compare_versions is set. Output is not suitable for production | |
15091 | END | |
15092 | ); | |
15093 | } | |
99598c8c | 15094 | |
99870f4d KW |
15095 | # Put into %potential_files a list of all the files in the directory structure |
15096 | # that could be inputs to this program, excluding those that we should ignore. | |
37e2e78e | 15097 | # Use absolute file names because it makes it easier across machine types. |
99870f4d KW |
15098 | my @ignored_files_full_names = map { File::Spec->rel2abs( |
15099 | internal_file_to_platform($_)) | |
15100 | } keys %ignored_files; | |
15101 | File::Find::find({ | |
15102 | wanted=>sub { | |
37e2e78e | 15103 | return unless /\.txt$/i; # Some platforms change the name's case |
517956bf | 15104 | my $full = lc(File::Spec->rel2abs($_)); |
99870f4d | 15105 | $potential_files{$full} = 1 |
37e2e78e | 15106 | if ! grep { $full eq lc($_) } @ignored_files_full_names; |
99870f4d KW |
15107 | return; |
15108 | } | |
15109 | }, File::Spec->curdir()); | |
99598c8c | 15110 | |
99870f4d | 15111 | my @mktables_list_output_files; |
cdcef19a | 15112 | my $old_start_time = 0; |
cf25bb62 | 15113 | |
3644ba60 KW |
15114 | if (! -e $file_list) { |
15115 | print "'$file_list' doesn't exist, so forcing rebuild.\n" if $verbosity >= $VERBOSE; | |
15116 | $write_unchanged_files = 1; | |
15117 | } elsif ($write_unchanged_files) { | |
99870f4d KW |
15118 | print "Not checking file list '$file_list'.\n" if $verbosity >= $VERBOSE; |
15119 | } | |
15120 | else { | |
15121 | print "Reading file list '$file_list'\n" if $verbosity >= $VERBOSE; | |
15122 | my $file_handle; | |
23e33b60 | 15123 | if (! open $file_handle, "<", $file_list) { |
3644ba60 | 15124 | Carp::my_carp("Failed to open '$file_list'; turning on -globlist option instead: $!"); |
99870f4d KW |
15125 | $glob_list = 1; |
15126 | } | |
15127 | else { | |
15128 | my @input; | |
15129 | ||
15130 | # Read and parse mktables.lst, placing the results from the first part | |
15131 | # into @input, and the second part into @mktables_list_output_files | |
15132 | for my $list ( \@input, \@mktables_list_output_files ) { | |
15133 | while (<$file_handle>) { | |
15134 | s/^ \s+ | \s+ $//xg; | |
cdcef19a KW |
15135 | if (/^ \s* \# .* Autogenerated\ starting\ on\ (\d+)/x) { |
15136 | $old_start_time = $1; | |
15137 | } | |
99870f4d KW |
15138 | next if /^ \s* (?: \# .* )? $/x; |
15139 | last if /^ =+ $/x; | |
15140 | my ( $file ) = split /\t/; | |
15141 | push @$list, $file; | |
cf25bb62 | 15142 | } |
99870f4d KW |
15143 | @$list = uniques(@$list); |
15144 | next; | |
cf25bb62 JH |
15145 | } |
15146 | ||
99870f4d KW |
15147 | # Look through all the input files |
15148 | foreach my $input (@input) { | |
15149 | next if $input eq 'version'; # Already have checked this. | |
cf25bb62 | 15150 | |
99870f4d KW |
15151 | # Ignore if doesn't exist. The checking about whether we care or |
15152 | # not is done via the Input_file object. | |
15153 | next if ! file_exists($input); | |
5beb625e | 15154 | |
99870f4d KW |
15155 | # The paths are stored with relative names, and with '/' as the |
15156 | # delimiter; convert to absolute on this machine | |
517956bf | 15157 | my $full = lc(File::Spec->rel2abs(internal_file_to_platform($input))); |
99870f4d | 15158 | $potential_files{$full} = 1 |
517956bf | 15159 | if ! grep { lc($full) eq lc($_) } @ignored_files_full_names; |
99870f4d | 15160 | } |
5beb625e | 15161 | } |
cf25bb62 | 15162 | |
99870f4d KW |
15163 | close $file_handle; |
15164 | } | |
15165 | ||
15166 | if ($glob_list) { | |
15167 | ||
15168 | # Here wants to process all .txt files in the directory structure. | |
15169 | # Convert them to full path names. They are stored in the platform's | |
15170 | # relative style | |
f86864ac KW |
15171 | my @known_files; |
15172 | foreach my $object (@input_file_objects) { | |
15173 | my $file = $object->file; | |
15174 | next unless defined $file; | |
15175 | push @known_files, File::Spec->rel2abs($file); | |
15176 | } | |
99870f4d KW |
15177 | |
15178 | my @unknown_input_files; | |
15179 | foreach my $file (keys %potential_files) { | |
517956bf | 15180 | next if grep { lc($file) eq lc($_) } @known_files; |
99870f4d KW |
15181 | |
15182 | # Here, the file is unknown to us. Get relative path name | |
15183 | $file = File::Spec->abs2rel($file); | |
15184 | push @unknown_input_files, $file; | |
15185 | ||
15186 | # What will happen is we create a data structure for it, and add it to | |
15187 | # the list of input files to process. First get the subdirectories | |
15188 | # into an array | |
15189 | my (undef, $directories, undef) = File::Spec->splitpath($file); | |
15190 | $directories =~ s;/$;;; # Can have extraneous trailing '/' | |
15191 | my @directories = File::Spec->splitdir($directories); | |
15192 | ||
15193 | # If the file isn't extracted (meaning none of the directories is the | |
15194 | # extracted one), just add it to the end of the list of inputs. | |
15195 | if (! grep { $EXTRACTED_DIR eq $_ } @directories) { | |
99f78760 | 15196 | push @input_file_objects, Input_file->new($file, v0); |
99870f4d KW |
15197 | } |
15198 | else { | |
15199 | ||
15200 | # Here, the file is extracted. It needs to go ahead of most other | |
15201 | # processing. Search for the first input file that isn't a | |
15202 | # special required property (that is, find one whose first_release | |
15203 | # is non-0), and isn't extracted. Also, the Age property file is | |
15204 | # processed before the extracted ones, just in case | |
15205 | # $compare_versions is set. | |
15206 | for (my $i = 0; $i < @input_file_objects; $i++) { | |
15207 | if ($input_file_objects[$i]->first_released ne v0 | |
517956bf CB |
15208 | && lc($input_file_objects[$i]->file) ne 'dage.txt' |
15209 | && $input_file_objects[$i]->file !~ /$EXTRACTED_DIR/i) | |
99870f4d | 15210 | { |
99f78760 | 15211 | splice @input_file_objects, $i, 0, |
37e2e78e | 15212 | Input_file->new($file, v0); |
99870f4d KW |
15213 | last; |
15214 | } | |
cf25bb62 | 15215 | } |
99870f4d | 15216 | |
cf25bb62 | 15217 | } |
d2d499f5 | 15218 | } |
99870f4d | 15219 | if (@unknown_input_files) { |
23e33b60 | 15220 | print STDERR simple_fold(join_lines(<<END |
99870f4d KW |
15221 | |
15222 | The following files are unknown as to how to handle. Assuming they are | |
15223 | typical property files. You'll know by later error messages if it worked or | |
15224 | not: | |
15225 | END | |
99f78760 | 15226 | ) . " " . join(", ", @unknown_input_files) . "\n\n"); |
99870f4d KW |
15227 | } |
15228 | } # End of looking through directory structure for more .txt files. | |
5beb625e | 15229 | |
99870f4d KW |
15230 | # Create the list of input files from the objects we have defined, plus |
15231 | # version | |
15232 | my @input_files = 'version'; | |
15233 | foreach my $object (@input_file_objects) { | |
15234 | my $file = $object->file; | |
15235 | next if ! defined $file; # Not all objects have files | |
15236 | next if $object->optional && ! -e $file; | |
15237 | push @input_files, $file; | |
15238 | } | |
5beb625e | 15239 | |
99870f4d KW |
15240 | if ( $verbosity >= $VERBOSE ) { |
15241 | print "Expecting ".scalar( @input_files )." input files. ", | |
15242 | "Checking ".scalar( @mktables_list_output_files )." output files.\n"; | |
15243 | } | |
cf25bb62 | 15244 | |
aeab6150 KW |
15245 | # We set $most_recent to be the most recently changed input file, including |
15246 | # this program itself (done much earlier in this file) | |
99870f4d | 15247 | foreach my $in (@input_files) { |
cdcef19a KW |
15248 | next unless -e $in; # Keep going even if missing a file |
15249 | my $mod_time = (stat $in)[9]; | |
aeab6150 | 15250 | $most_recent = $mod_time if $mod_time > $most_recent; |
99870f4d KW |
15251 | |
15252 | # See that the input files have distinct names, to warn someone if they | |
15253 | # are adding a new one | |
15254 | if ($make_list) { | |
15255 | my ($volume, $directories, $file ) = File::Spec->splitpath($in); | |
15256 | $directories =~ s;/$;;; # Can have extraneous trailing '/' | |
15257 | my @directories = File::Spec->splitdir($directories); | |
15258 | my $base = $file =~ s/\.txt$//; | |
15259 | construct_filename($file, 'mutable', \@directories); | |
cf25bb62 | 15260 | } |
99870f4d | 15261 | } |
cf25bb62 | 15262 | |
dff6c046 | 15263 | my $rebuild = $write_unchanged_files # Rebuild: if unconditional rebuild |
cdcef19a | 15264 | || ! scalar @mktables_list_output_files # or if no outputs known |
aeab6150 | 15265 | || $old_start_time < $most_recent; # or out-of-date |
cf25bb62 | 15266 | |
99870f4d KW |
15267 | # Now we check to see if any output files are older than youngest, if |
15268 | # they are, we need to continue on, otherwise we can presumably bail. | |
d1d1cd7a | 15269 | if (! $rebuild) { |
99870f4d KW |
15270 | foreach my $out (@mktables_list_output_files) { |
15271 | if ( ! file_exists($out)) { | |
15272 | print "'$out' is missing.\n" if $verbosity >= $VERBOSE; | |
d1d1cd7a | 15273 | $rebuild = 1; |
99870f4d KW |
15274 | last; |
15275 | } | |
15276 | #local $to_trace = 1 if main::DEBUG; | |
aeab6150 KW |
15277 | trace $most_recent, (stat $out)[9] if main::DEBUG && $to_trace; |
15278 | if ( (stat $out)[9] <= $most_recent ) { | |
15279 | #trace "$out: most recent mod time: ", (stat $out)[9], ", youngest: $most_recent\n" if main::DEBUG && $to_trace; | |
99870f4d | 15280 | print "'$out' is too old.\n" if $verbosity >= $VERBOSE; |
d1d1cd7a | 15281 | $rebuild = 1; |
99870f4d | 15282 | last; |
cf25bb62 | 15283 | } |
cf25bb62 | 15284 | } |
99870f4d | 15285 | } |
d1d1cd7a | 15286 | if (! $rebuild) { |
1265e11f | 15287 | print "Files seem to be ok, not bothering to rebuild. Add '-w' option to force build\n"; |
99870f4d KW |
15288 | exit(0); |
15289 | } | |
15290 | print "Must rebuild tables.\n" if $verbosity >= $VERBOSE; | |
cf25bb62 | 15291 | |
99870f4d KW |
15292 | # Ready to do the major processing. First create the perl pseudo-property. |
15293 | $perl = Property->new('perl', Type => $NON_STRING, Perl_Extension => 1); | |
cf25bb62 | 15294 | |
99870f4d KW |
15295 | # Process each input file |
15296 | foreach my $file (@input_file_objects) { | |
15297 | $file->run; | |
d2d499f5 JH |
15298 | } |
15299 | ||
99870f4d | 15300 | # Finish the table generation. |
c4051cc5 | 15301 | |
99870f4d KW |
15302 | print "Finishing processing Unicode properties\n" if $verbosity >= $PROGRESS; |
15303 | finish_Unicode(); | |
c4051cc5 | 15304 | |
99870f4d KW |
15305 | print "Compiling Perl properties\n" if $verbosity >= $PROGRESS; |
15306 | compile_perl(); | |
c4051cc5 | 15307 | |
99870f4d KW |
15308 | print "Creating Perl synonyms\n" if $verbosity >= $PROGRESS; |
15309 | add_perl_synonyms(); | |
c4051cc5 | 15310 | |
99870f4d KW |
15311 | print "Writing tables\n" if $verbosity >= $PROGRESS; |
15312 | write_all_tables(); | |
c4051cc5 | 15313 | |
99870f4d KW |
15314 | # Write mktables.lst |
15315 | if ( $file_list and $make_list ) { | |
c4051cc5 | 15316 | |
99870f4d KW |
15317 | print "Updating '$file_list'\n" if $verbosity >= $PROGRESS; |
15318 | foreach my $file (@input_files, @files_actually_output) { | |
15319 | my (undef, $directories, $file) = File::Spec->splitpath($file); | |
15320 | my @directories = File::Spec->splitdir($directories); | |
15321 | $file = join '/', @directories, $file; | |
15322 | } | |
15323 | ||
15324 | my $ofh; | |
15325 | if (! open $ofh,">",$file_list) { | |
15326 | Carp::my_carp("Can't write to '$file_list'. Skipping: $!"); | |
15327 | return | |
15328 | } | |
15329 | else { | |
cdcef19a | 15330 | my $localtime = localtime $start_time; |
99870f4d KW |
15331 | print $ofh <<"END"; |
15332 | # | |
15333 | # $file_list -- File list for $0. | |
97050450 | 15334 | # |
cdcef19a | 15335 | # Autogenerated starting on $start_time ($localtime) |
97050450 YO |
15336 | # |
15337 | # - First section is input files | |
99870f4d | 15338 | # ($0 itself is not listed but is automatically considered an input) |
98dc9551 | 15339 | # - Section separator is /^=+\$/ |
97050450 YO |
15340 | # - Second section is a list of output files. |
15341 | # - Lines matching /^\\s*#/ are treated as comments | |
15342 | # which along with blank lines are ignored. | |
15343 | # | |
15344 | ||
15345 | # Input files: | |
15346 | ||
99870f4d KW |
15347 | END |
15348 | print $ofh "$_\n" for sort(@input_files); | |
15349 | print $ofh "\n=================================\n# Output files:\n\n"; | |
15350 | print $ofh "$_\n" for sort @files_actually_output; | |
15351 | print $ofh "\n# ",scalar(@input_files)," input files\n", | |
15352 | "# ",scalar(@files_actually_output)+1," output files\n\n", | |
15353 | "# End list\n"; | |
15354 | close $ofh | |
15355 | or Carp::my_carp("Failed to close $ofh: $!"); | |
15356 | ||
15357 | print "Filelist has ",scalar(@input_files)," input files and ", | |
15358 | scalar(@files_actually_output)+1," output files\n" | |
15359 | if $verbosity >= $VERBOSE; | |
15360 | } | |
15361 | } | |
15362 | ||
15363 | # Output these warnings unless -q explicitly specified. | |
c83dffeb | 15364 | if ($verbosity >= $NORMAL_VERBOSITY && ! $debug_skip) { |
99870f4d KW |
15365 | if (@unhandled_properties) { |
15366 | print "\nProperties and tables that unexpectedly have no code points\n"; | |
15367 | foreach my $property (sort @unhandled_properties) { | |
15368 | print $property, "\n"; | |
15369 | } | |
15370 | } | |
15371 | ||
15372 | if (%potential_files) { | |
15373 | print "\nInput files that are not considered:\n"; | |
15374 | foreach my $file (sort keys %potential_files) { | |
15375 | print File::Spec->abs2rel($file), "\n"; | |
15376 | } | |
15377 | } | |
15378 | print "\nAll done\n" if $verbosity >= $VERBOSE; | |
15379 | } | |
5beb625e | 15380 | exit(0); |
cf25bb62 | 15381 | |
99870f4d | 15382 | # TRAILING CODE IS USED BY make_property_test_script() |
5beb625e | 15383 | __DATA__ |
99870f4d | 15384 | |
5beb625e JH |
15385 | use strict; |
15386 | use warnings; | |
15387 | ||
66fd7fd0 KW |
15388 | # If run outside the normal test suite on an ASCII platform, you can |
15389 | # just create a latin1_to_native() function that just returns its | |
15390 | # inputs, because that's the only function used from test.pl | |
15391 | require "test.pl"; | |
15392 | ||
37e2e78e KW |
15393 | # Test qr/\X/ and the \p{} regular expression constructs. This file is |
15394 | # constructed by mktables from the tables it generates, so if mktables is | |
15395 | # buggy, this won't necessarily catch those bugs. Tests are generated for all | |
15396 | # feasible properties; a few aren't currently feasible; see | |
15397 | # is_code_point_usable() in mktables for details. | |
99870f4d KW |
15398 | |
15399 | # Standard test packages are not used because this manipulates SIG_WARN. It | |
15400 | # exits 0 if every non-skipped test succeeded; -1 if any failed. | |
15401 | ||
5beb625e JH |
15402 | my $Tests = 0; |
15403 | my $Fails = 0; | |
99870f4d | 15404 | |
99870f4d KW |
15405 | sub Expect($$$$) { |
15406 | my $expected = shift; | |
15407 | my $ord = shift; | |
15408 | my $regex = shift; | |
15409 | my $warning_type = shift; # Type of warning message, like 'deprecated' | |
15410 | # or empty if none | |
15411 | my $line = (caller)[2]; | |
66fd7fd0 | 15412 | $ord = ord(latin1_to_native(chr($ord))); |
37e2e78e | 15413 | |
99870f4d | 15414 | # Convert the code point to hex form |
23e33b60 | 15415 | my $string = sprintf "\"\\x{%04X}\"", $ord; |
99870f4d | 15416 | |
99870f4d | 15417 | my @tests = ""; |
5beb625e | 15418 | |
37e2e78e KW |
15419 | # The first time through, use all warnings. If the input should generate |
15420 | # a warning, add another time through with them turned off | |
99870f4d KW |
15421 | push @tests, "no warnings '$warning_type';" if $warning_type; |
15422 | ||
15423 | foreach my $no_warnings (@tests) { | |
15424 | ||
15425 | # Store any warning messages instead of outputting them | |
15426 | local $SIG{__WARN__} = $SIG{__WARN__}; | |
15427 | my $warning_message; | |
15428 | $SIG{__WARN__} = sub { $warning_message = $_[0] }; | |
15429 | ||
15430 | $Tests++; | |
15431 | ||
15432 | # A string eval is needed because of the 'no warnings'. | |
15433 | # Assumes no parens in the regular expression | |
15434 | my $result = eval "$no_warnings | |
15435 | my \$RegObj = qr($regex); | |
15436 | $string =~ \$RegObj ? 1 : 0"; | |
15437 | if (not defined $result) { | |
15438 | print "not ok $Tests - couldn't compile /$regex/; line $line: $@\n"; | |
15439 | $Fails++; | |
15440 | } | |
15441 | elsif ($result ^ $expected) { | |
15442 | print "not ok $Tests - expected $expected but got $result for $string =~ qr/$regex/; line $line\n"; | |
15443 | $Fails++; | |
15444 | } | |
15445 | elsif ($warning_message) { | |
15446 | if (! $warning_type || ($warning_type && $no_warnings)) { | |
15447 | print "not ok $Tests - for qr/$regex/ did not expect warning message '$warning_message'; line $line\n"; | |
15448 | $Fails++; | |
15449 | } | |
15450 | else { | |
15451 | print "ok $Tests - expected and got a warning message for qr/$regex/; line $line\n"; | |
15452 | } | |
15453 | } | |
15454 | elsif ($warning_type && ! $no_warnings) { | |
15455 | print "not ok $Tests - for qr/$regex/ expected a $warning_type warning message, but got none; line $line\n"; | |
15456 | $Fails++; | |
15457 | } | |
15458 | else { | |
15459 | print "ok $Tests - got $result for $string =~ qr/$regex/; line $line\n"; | |
15460 | } | |
5beb625e | 15461 | } |
99870f4d | 15462 | return; |
5beb625e | 15463 | } |
d73e5302 | 15464 | |
99870f4d KW |
15465 | sub Error($) { |
15466 | my $regex = shift; | |
5beb625e | 15467 | $Tests++; |
99870f4d | 15468 | if (eval { 'x' =~ qr/$regex/; 1 }) { |
5beb625e | 15469 | $Fails++; |
99870f4d KW |
15470 | my $line = (caller)[2]; |
15471 | print "not ok $Tests - re compiled ok, but expected error for qr/$regex/; line $line: $@\n"; | |
5beb625e | 15472 | } |
99870f4d KW |
15473 | else { |
15474 | my $line = (caller)[2]; | |
15475 | print "ok $Tests - got and expected error for qr/$regex/; line $line\n"; | |
15476 | } | |
15477 | return; | |
5beb625e JH |
15478 | } |
15479 | ||
37e2e78e KW |
15480 | # GCBTest.txt character that separates grapheme clusters |
15481 | my $breakable_utf8 = my $breakable = chr(0xF7); | |
15482 | utf8::upgrade($breakable_utf8); | |
15483 | ||
15484 | # GCBTest.txt character that indicates that the adjoining code points are part | |
15485 | # of the same grapheme cluster | |
15486 | my $nobreak_utf8 = my $nobreak = chr(0xD7); | |
15487 | utf8::upgrade($nobreak_utf8); | |
15488 | ||
15489 | sub Test_X($) { | |
15490 | # Test qr/\X/ matches. The input is a line from auxiliary/GCBTest.txt | |
15491 | # Each such line is a sequence of code points given by their hex numbers, | |
15492 | # separated by the two characters defined just before this subroutine that | |
15493 | # indicate that either there can or cannot be a break between the adjacent | |
15494 | # code points. If there isn't a break, that means the sequence forms an | |
15495 | # extended grapheme cluster, which means that \X should match the whole | |
15496 | # thing. If there is a break, \X should stop there. This is all | |
15497 | # converted by this routine into a match: | |
15498 | # $string =~ /(\X)/, | |
15499 | # Each \X should match the next cluster; and that is what is checked. | |
15500 | ||
15501 | my $template = shift; | |
15502 | ||
15503 | my $line = (caller)[2]; | |
15504 | ||
15505 | # The line contains characters above the ASCII range, but in Latin1. It | |
15506 | # may or may not be in utf8, and if it is, it may or may not know it. So, | |
15507 | # convert these characters to 8 bits. If knows is in utf8, simply | |
15508 | # downgrade. | |
15509 | if (utf8::is_utf8($template)) { | |
15510 | utf8::downgrade($template); | |
15511 | } else { | |
15512 | ||
15513 | # Otherwise, if it is in utf8, but doesn't know it, the next lines | |
15514 | # convert the two problematic characters to their 8-bit equivalents. | |
15515 | # If it isn't in utf8, they don't harm anything. | |
15516 | use bytes; | |
15517 | $template =~ s/$nobreak_utf8/$nobreak/g; | |
15518 | $template =~ s/$breakable_utf8/$breakable/g; | |
15519 | } | |
15520 | ||
15521 | # Get rid of the leading and trailing breakables | |
15522 | $template =~ s/^ \s* $breakable \s* //x; | |
15523 | $template =~ s/ \s* $breakable \s* $ //x; | |
15524 | ||
15525 | # And no-breaks become just a space. | |
15526 | $template =~ s/ \s* $nobreak \s* / /xg; | |
15527 | ||
15528 | # Split the input into segments that are breakable between them. | |
15529 | my @segments = split /\s*$breakable\s*/, $template; | |
15530 | ||
15531 | my $string = ""; | |
15532 | my $display_string = ""; | |
15533 | my @should_match; | |
15534 | my @should_display; | |
15535 | ||
15536 | # Convert the code point sequence in each segment into a Perl string of | |
15537 | # characters | |
15538 | foreach my $segment (@segments) { | |
15539 | my @code_points = split /\s+/, $segment; | |
15540 | my $this_string = ""; | |
15541 | my $this_display = ""; | |
15542 | foreach my $code_point (@code_points) { | |
66fd7fd0 | 15543 | $this_string .= latin1_to_native(chr(hex $code_point)); |
37e2e78e KW |
15544 | $this_display .= "\\x{$code_point}"; |
15545 | } | |
15546 | ||
15547 | # The next cluster should match the string in this segment. | |
15548 | push @should_match, $this_string; | |
15549 | push @should_display, $this_display; | |
15550 | $string .= $this_string; | |
15551 | $display_string .= $this_display; | |
15552 | } | |
15553 | ||
15554 | # If a string can be represented in both non-ut8 and utf8, test both cases | |
15555 | UPGRADE: | |
15556 | for my $to_upgrade (0 .. 1) { | |
678f13d5 | 15557 | |
37e2e78e KW |
15558 | if ($to_upgrade) { |
15559 | ||
15560 | # If already in utf8, would just be a repeat | |
15561 | next UPGRADE if utf8::is_utf8($string); | |
15562 | ||
15563 | utf8::upgrade($string); | |
15564 | } | |
15565 | ||
15566 | # Finally, do the \X match. | |
15567 | my @matches = $string =~ /(\X)/g; | |
15568 | ||
15569 | # Look through each matched cluster to verify that it matches what we | |
15570 | # expect. | |
15571 | my $min = (@matches < @should_match) ? @matches : @should_match; | |
15572 | for my $i (0 .. $min - 1) { | |
15573 | $Tests++; | |
15574 | if ($matches[$i] eq $should_match[$i]) { | |
15575 | print "ok $Tests - "; | |
15576 | if ($i == 0) { | |
15577 | print "In \"$display_string\" =~ /(\\X)/g, \\X #1"; | |
15578 | } else { | |
15579 | print "And \\X #", $i + 1, | |
15580 | } | |
15581 | print " correctly matched $should_display[$i]; line $line\n"; | |
15582 | } else { | |
15583 | $matches[$i] = join("", map { sprintf "\\x{%04X}", $_ } | |
15584 | unpack("U*", $matches[$i])); | |
15585 | print "not ok $Tests - In \"$display_string\" =~ /(\\X)/g, \\X #", | |
15586 | $i + 1, | |
15587 | " should have matched $should_display[$i]", | |
15588 | " but instead matched $matches[$i]", | |
15589 | ". Abandoning rest of line $line\n"; | |
15590 | next UPGRADE; | |
15591 | } | |
15592 | } | |
15593 | ||
15594 | # And the number of matches should equal the number of expected matches. | |
15595 | $Tests++; | |
15596 | if (@matches == @should_match) { | |
15597 | print "ok $Tests - Nothing was left over; line $line\n"; | |
15598 | } else { | |
15599 | print "not ok $Tests - There were ", scalar @should_match, " \\X matches expected, but got ", scalar @matches, " instead; line $line\n"; | |
15600 | } | |
15601 | } | |
15602 | ||
15603 | return; | |
15604 | } | |
15605 | ||
99870f4d | 15606 | sub Finished() { |
f86864ac | 15607 | print "1..$Tests\n"; |
99870f4d | 15608 | exit($Fails ? -1 : 0); |
5beb625e | 15609 | } |
99870f4d KW |
15610 | |
15611 | Error('\p{Script=InGreek}'); # Bug #69018 | |
37e2e78e | 15612 | Test_X("1100 $nobreak 1161"); # Bug #70940 |
ae5b72c8 KW |
15613 | Expect(0, 0x2028, '\p{Print}', ""); # Bug # 71722 |
15614 | Expect(0, 0x2029, '\p{Print}', ""); # Bug # 71722 | |
eadadd41 | 15615 | Expect(1, 0xFF10, '\p{XDigit}', ""); # Bug # 71726 |