This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
utf8_heavy.pl: white-space only
[perl5.git] / lib / utf8_heavy.pl
CommitLineData
a0ed51b3 1package utf8;
cf25bb62
JH
2use strict;
3use warnings;
a0ed51b3 4
15732964 5sub DEBUG () { 0 }
a0ed51b3
LW
6
7sub DESTROY {}
8
5beb625e
JH
9my %Cache;
10
86916d66
JH
11sub croak { require Carp; Carp::croak(@_) }
12
c4ab2516 13sub _loose_name ($) {
45376db6
KW
14 # Given a lowercase property or property-value name, return its
15 # standardized version that is expected for look-up in the 'loose' hashes
16 # in Heavy.pl (hence, this depends on what mktables does). This squeezes
17 # out blanks, underscores and dashes. The complication stems from the
18 # grandfathered-in 'L_', which retains a single trailing underscore.
19
20 my $loose = $_[0] =~ s/[-\s_]//rg;
21
22 return $loose if $loose !~ / ^ (?: is )? l $/x;
23 return 'l_' if $_[0] =~ / l .* _ /x; # If original had a trailing '_'
24 return $loose;
25}
26
5beb625e 27##
3a2263fe
RGS
28## "SWASH" == "SWATCH HASH". A "swatch" is a swatch of the Unicode landscape.
29## It's a data structure that encodes a set of Unicode characters.
5beb625e
JH
30##
31
99870f4d
KW
32{
33 # If a floating point number is within this distance from the value of a
34 # fraction, it is considered to be that fraction, even if many more digits
35 # are specified that don't exactly match.
36 my $min_floating_slop;
37
3dd10fe8
KW
38 # To guard against this program calling something that in turn ends up
39 # calling this program with the same inputs, and hence infinitely
40 # recursing, we keep a stack of the properties that are currently in
41 # progress, pushed upon entry, popped upon return.
42 my @recursed;
43
99870f4d
KW
44 sub SWASHNEW {
45 my ($class, $type, $list, $minbits, $none) = @_;
0fada5e1 46 my $user_defined = 0;
99870f4d
KW
47 local $^D = 0 if $^D;
48
3dd10fe8 49 $class = "" unless defined $class;
5bcf08c4
KW
50 print STDERR __LINE__, ": class=$class, type=$type, list=",
51 (defined $list) ? $list : ':undef:',
52 ", minbits=$minbits, none=$none\n" if DEBUG;
99870f4d
KW
53
54 ##
55 ## Get the list of codepoints for the type.
56 ## Called from swash_init (see utf8.c) or SWASHNEW itself.
57 ##
58 ## Callers of swash_init:
59 ## op.c:pmtrans -- for tr/// and y///
60 ## regexec.c:regclass_swash -- for /[]/, \p, and \P
61 ## utf8.c:is_utf8_common -- for common Unicode properties
62 ## utf8.c:to_utf8_case -- for lc, uc, ucfirst, etc. and //i
681d705c 63 ## Unicode::UCD::prop_invlist
99870f4d
KW
64 ##
65 ## Given a $type, our goal is to fill $list with the set of codepoint
66 ## ranges. If $type is false, $list passed is used.
67 ##
68 ## $minbits:
69 ## For binary properties, $minbits must be 1.
70 ## For character mappings (case and transliteration), $minbits must
71 ## be a number except 1.
72 ##
73 ## $list (or that filled according to $type):
74 ## Refer to perlunicode.pod, "User-Defined Character Properties."
75 ##
76 ## For binary properties, only characters with the property value
77 ## of True should be listed. The 3rd column, if any, will be ignored
78 ##
79 ## $none is undocumented, so I'm (khw) trying to do some documentation
80 ## of it now. It appears to be if there is a mapping in an input file
81 ## that maps to 'XXXX', then that is replaced by $none+1, expressed in
83fe8199 82 ## hexadecimal. It is used somehow in tr///.
99870f4d
KW
83 ##
84 ## To make the parsing of $type clear, this code takes the a rather
85 ## unorthodox approach of last'ing out of the block once we have the
86 ## info we need. Were this to be a subroutine, the 'last' would just
87 ## be a 'return'.
88 ##
fbe6f3cf
KW
89 # If a problem is found $type is returned;
90 # Upon success, a new (or cached) blessed object is returned with
91 # keys TYPE, BITS, EXTRAS, LIST, and NONE with values having the
92 # same meanings as the input parameters.
a6517ebf
KW
93 # SPECIALS contains a reference to any special-treatment hash in the
94 # INVERT_IT is non-zero if the result should be inverted before use
0fada5e1
KW
95 # USER_DEFINED is non-zero if the result came from a user-defined
96 # property.
99870f4d
KW
97 my $file; ## file to load data from, and also part of the %Cache key.
98 my $ListSorted = 0;
99
100 # Change this to get a different set of Unicode tables
101 my $unicore_dir = 'unicore';
a6517ebf 102 my $invert_it = 0;
99870f4d
KW
103
104 if ($type)
cf25bb62 105 {
3dd10fe8
KW
106
107 # Verify that this isn't a recursive call for this property.
108 # Can't use croak, as it may try to recurse here itself.
109 my $class_type = $class . "::$type";
110 if (grep { $_ eq $class_type } @recursed) {
111 CORE::die "panic: Infinite recursion in SWASHNEW for '$type'\n";
112 }
113 push @recursed, $class_type;
114
99870f4d
KW
115 $type =~ s/^\s+//;
116 $type =~ s/\s+$//;
117
4ff124d1
KW
118 # regcomp.c surrounds the property name with '__" and '_i' if this
119 # is to be caseless matching.
120 my $caseless = $type =~ s/^__(.*)_i$/$1/;
121
122 print STDERR __LINE__, ": type=$type, caseless=$caseless\n" if DEBUG;
99870f4d
KW
123
124 GETFILE:
125 {
126 ##
a44f4b74
KW
127 ## It could be a user-defined property. Look in current
128 ## package if no package given
99870f4d
KW
129 ##
130
131 my $caller1 = $type =~ s/(.+)::// ? $1 : caller(1);
132
d658a8a8 133 if (defined $caller1 && $type =~ /^I[ns]\w+$/) {
99870f4d
KW
134 my $prop = "${caller1}::$type";
135 if (exists &{$prop}) {
0e9be77f
DM
136 # stolen from Scalar::Util::PP::tainted()
137 my $tainted;
138 {
139 local($@, $SIG{__DIE__}, $SIG{__WARN__});
140 local $^W = 0;
141 no warnings;
142 eval { kill 0 * $prop };
143 $tainted = 1 if $@ =~ /^Insecure/;
144 }
145 die "Insecure user-defined property \\p{$prop}\n"
146 if $tainted;
99870f4d 147 no strict 'refs';
4ff124d1 148 $list = &{$prop}($caseless);
0fada5e1 149 $user_defined = 1;
99870f4d
KW
150 last GETFILE;
151 }
152 }
153
a44f4b74
KW
154 # During Perl's compilation, this routine may be called before
155 # the tables are constructed. If so, we have a chicken/egg
156 # problem. If we die, the tables never get constructed, so
157 # keep going, but return an empty table so only what the code
158 # has compiled in internally (currently ASCII/Latin1 range
159 # matching) will work.
1363cbd0
FC
160 BEGIN {
161 # Poor man's constant, to avoid a run-time check.
162 $utf8::{miniperl}
163 = \! defined &DynaLoader::boot_DynaLoader;
164 }
165 if (miniperl) {
a44f4b74
KW
166 eval "require '$unicore_dir/Heavy.pl'";
167 last GETFILE if $@;
168 }
169 else {
170 require "$unicore_dir/Heavy.pl";
171 }
1363cbd0 172 BEGIN { delete $utf8::{miniperl} }
99870f4d 173
4ff124d1 174 # All property names are matched caselessly
49801ec2 175 my $property_and_table = CORE::lc $type;
99870f4d
KW
176 print STDERR __LINE__, ": $property_and_table\n" if DEBUG;
177
178 # See if is of the compound form 'property=value', where the
179 # value indicates the table we should use.
180 my ($property, $table, @remainder) =
181 split /\s*[:=]\s*/, $property_and_table, -1;
3dd10fe8
KW
182 if (@remainder) {
183 pop @recursed if @recursed;
184 return $type;
185 }
99870f4d
KW
186
187 my $prefix;
188 if (! defined $table) {
189
190 # Here, is the single form. The property becomes empty, and
191 # the whole value is the table.
192 $table = $property;
193 $prefix = $property = "";
194 } else {
195 print STDERR __LINE__, ": $property\n" if DEBUG;
196
197 # Here it is the compound property=table form. The property
807807b7
KW
198 # name is always loosely matched, and always can have an
199 # optional 'is' prefix (which isn't true in the single
200 # form).
c4ab2516 201 $property = _loose_name($property) =~ s/^is//r;
99870f4d
KW
202
203 # And convert to canonical form. Quit if not valid.
204 $property = $utf8::loose_property_name_of{$property};
3dd10fe8
KW
205 if (! defined $property) {
206 pop @recursed if @recursed;
207 return $type;
208 }
99870f4d
KW
209
210 $prefix = "$property=";
211
212 # If the rhs looks like it is a number...
213 print STDERR __LINE__, ": table=$table\n" if DEBUG;
214 if ($table =~ qr{ ^ [ \s 0-9 _ + / . -]+ $ }x) {
215 print STDERR __LINE__, ": table=$table\n" if DEBUG;
216
217 # Don't allow leading nor trailing slashes
3dd10fe8
KW
218 if ($table =~ / ^ \/ | \/ $ /x) {
219 pop @recursed if @recursed;
220 return $type;
221 }
99870f4d
KW
222
223 # Split on slash, in case it is a rational, like \p{1/5}
224 my @parts = split qr{ \s* / \s* }x, $table, -1;
225 print __LINE__, ": $type\n" if @parts > 2 && DEBUG;
226
227 # Can have maximum of one slash
3dd10fe8
KW
228 if (@parts > 2) {
229 pop @recursed if @recursed;
230 return $type;
231 }
99870f4d
KW
232
233 foreach my $part (@parts) {
234 print __LINE__, ": part=$part\n" if DEBUG;
235
236 $part =~ s/^\+\s*//; # Remove leading plus
237 $part =~ s/^-\s*/-/; # Remove blanks after unary
238 # minus
239
240 # Remove underscores between digits.
241 $part =~ s/( ?<= [0-9] ) _ (?= [0-9] ) //xg;
242
243 # No leading zeros (but don't make a single '0'
244 # into a null string)
245 $part =~ s/ ^ ( -? ) 0+ /$1/x;
246 $part .= '0' if $part eq '-' || $part eq "";
247
248 # No trailing zeros after a decimal point
249 $part =~ s/ ( \. .*? ) 0+ $ /$1/x;
250
251 # Begin with a 0 if a leading decimal point
252 $part =~ s/ ^ ( -? ) \. /${1}0./x;
253
254 # Ensure not a trailing decimal point: turn into an
255 # integer
256 $part =~ s/ \. $ //x;
257
258 print STDERR __LINE__, ": part=$part\n" if DEBUG;
259 #return $type if $part eq "";
260
261 # Result better look like a number. (This test is
262 # needed because, for example could have a plus in
263 # the middle.)
3dd10fe8
KW
264 if ($part !~ / ^ -? [0-9]+ ( \. [0-9]+)? $ /x) {
265 pop @recursed if @recursed;
266 return $type;
267 }
99870f4d
KW
268 }
269
270 # If a rational...
271 if (@parts == 2) {
272
273 # If denominator is negative, get rid of it, and ...
274 if ($parts[1] =~ s/^-//) {
275
276 # If numerator is also negative, convert the
277 # whole thing to positive, or move the minus to
278 # the numerator
279 if ($parts[0] !~ s/^-//) {
280 $parts[0] = '-' . $parts[0];
281 }
282 }
283 $table = join '/', @parts;
284 }
285 elsif ($property ne 'nv' || $parts[0] !~ /\./) {
286
287 # Here is not numeric value, or doesn't have a
288 # decimal point. No further manipulation is
289 # necessary. (Note the hard-coded property name.
290 # This could fail if other properties eventually
291 # had fractions as well; perhaps the cjk ones
292 # could evolve to do that. This hard-coding could
293 # be fixed by mktables generating a list of
294 # properties that could have fractions.)
295 $table = $parts[0];
296 } else {
297
298 # Here is a floating point numeric_value. Try to
299 # convert to rational. First see if is in the list
300 # of known ones.
301 if (exists $utf8::nv_floating_to_rational{$parts[0]}) {
302 $table = $utf8::nv_floating_to_rational{$parts[0]};
303 } else {
304
305 # Here not in the list. See if is close
306 # enough to something in the list. First
307 # determine what 'close enough' means. It has
308 # to be as tight as what mktables says is the
309 # maximum slop, and as tight as how many
310 # digits we were passed. That is, if the user
311 # said .667, .6667, .66667, etc. we match as
312 # many digits as they passed until get to
313 # where it doesn't matter any more due to the
314 # machine's precision. If they said .6666668,
315 # we fail.
316 (my $fraction = $parts[0]) =~ s/^.*\.//;
317 my $epsilon = 10 ** - (length($fraction));
318 if ($epsilon > $utf8::max_floating_slop) {
319 $epsilon = $utf8::max_floating_slop;
320 }
321
322 # But it can't be tighter than the minimum
323 # precision for this machine. If haven't
324 # already calculated that minimum, do so now.
325 if (! defined $min_floating_slop) {
326
327 # Keep going down an order of magnitude
328 # until find that adding this quantity to
329 # 1 remains 1; but put an upper limit on
330 # this so in case this algorithm doesn't
331 # work properly on some platform, that we
332 # won't loop forever.
333 my $count = 0;
334 $min_floating_slop = 1;
335 while (1+ $min_floating_slop != 1
336 && $count++ < 50)
337 {
338 my $next = $min_floating_slop / 10;
339 last if $next == 0; # If underflows,
340 # use previous one
341 $min_floating_slop = $next;
342 print STDERR __LINE__, ": min_float_slop=$min_floating_slop\n" if DEBUG;
343 }
344
345 # Back off a couple orders of magnitude,
346 # just to be safe.
347 $min_floating_slop *= 100;
348 }
349
350 if ($epsilon < $min_floating_slop) {
351 $epsilon = $min_floating_slop;
352 }
353 print STDERR __LINE__, ": fraction=.$fraction; epsilon=$epsilon\n" if DEBUG;
354
355 undef $table;
356
357 # And for each possible rational in the table,
358 # see if it is within epsilon of the input.
359 foreach my $official
360 (keys %utf8::nv_floating_to_rational)
361 {
362 print STDERR __LINE__, ": epsilon=$epsilon, official=$official, diff=", abs($parts[0] - $official), "\n" if DEBUG;
363 if (abs($parts[0] - $official) < $epsilon) {
364 $table =
365 $utf8::nv_floating_to_rational{$official};
366 last;
367 }
368 }
369
370 # Quit if didn't find one.
3dd10fe8
KW
371 if (! defined $table) {
372 pop @recursed if @recursed;
373 return $type;
374 }
99870f4d
KW
375 }
376 }
377 print STDERR __LINE__, ": $property=$table\n" if DEBUG;
378 }
379 }
380
381 # Combine lhs (if any) and rhs to get something that matches
382 # the syntax of the lookups.
383 $property_and_table = "$prefix$table";
384 print STDERR __LINE__, ": $property_and_table\n" if DEBUG;
385
386 # First try stricter matching.
387 $file = $utf8::stricter_to_file_of{$property_and_table};
388
389 # If didn't find it, try again with looser matching by editing
390 # out the applicable characters on the rhs and looking up
391 # again.
392 if (! defined $file) {
c4ab2516 393 $table = _loose_name($table);
99870f4d
KW
394 $property_and_table = "$prefix$table";
395 print STDERR __LINE__, ": $property_and_table\n" if DEBUG;
396 $file = $utf8::loose_to_file_of{$property_and_table};
397 }
398
399 # Add the constant and go fetch it in.
400 if (defined $file) {
a6517ebf
KW
401
402 # A beginning ! means to invert
403 $invert_it = $file =~ s/^!//;
404
99870f4d
KW
405 if ($utf8::why_deprecated{$file}) {
406 warnings::warnif('deprecated', "Use of '$type' in \\p{} or \\P{} is deprecated because: $utf8::why_deprecated{$file};");
407 }
4ff124d1
KW
408
409 if ($caseless
410 && exists $utf8::caseless_equivalent{$property_and_table})
411 {
412 $file = $utf8::caseless_equivalent{$property_and_table};
413 }
99870f4d
KW
414 $file= "$unicore_dir/lib/$file.pl";
415 last GETFILE;
416 }
417 print STDERR __LINE__, ": didn't find $property_and_table\n" if DEBUG;
418
419 ##
99870f4d
KW
420 ## Last attempt -- see if it's a standard "To" name
421 ## (e.g. "ToLower") ToTitle is used by ucfirst().
422 ## The user-level way to access ToDigit() and ToFold()
423 ## is to use Unicode::UCD.
424 ##
08432a98 425
814687ad 426 if ($minbits != 1) { # Only check if caller wants non-binary
010c8e19
KW
427 if ($type =~ /^To(Digit|Fold|Lower|Title|Upper)$/) {
428 $file = "$unicore_dir/To/$1.pl";
429 ## would like to test to see if $file actually exists....
430 last GETFILE;
431 }
814687ad 432 }
99870f4d
KW
433
434 ##
435 ## If we reach this line, it's because we couldn't figure
436 ## out what to do with $type. Ouch.
437 ##
438
3dd10fe8 439 pop @recursed if @recursed;
99870f4d 440 return $type;
cf25bb62
JH
441 }
442
99870f4d
KW
443 if (defined $file) {
444 print STDERR __LINE__, ": found it (file='$file')\n" if DEBUG;
445
446 ##
447 ## If we reach here, it was due to a 'last GETFILE' above
448 ## (exception: user-defined properties and mappings), so we
449 ## have a filename, so now we load it if we haven't already.
450 ## If we have, return the cached results. The cache key is the
451 ## class and file to load.
452 ##
453 my $found = $Cache{$class, $file};
454 if ($found and ref($found) eq $class) {
a6517ebf 455 print STDERR __LINE__, ": Returning cached '$file' for \\p{$type}; invert_it=$invert_it\n" if DEBUG;
3dd10fe8 456 pop @recursed if @recursed;
a6517ebf 457 $found->{'INVERT_IT'} = $invert_it;
99870f4d
KW
458 return $found;
459 }
460
461 local $@;
462 local $!;
463 $list = do $file; die $@ if $@;
0a0ffbce 464 }
cf25bb62 465
99870f4d 466 $ListSorted = 1; ## we know that these lists are sorted
cf25bb62
JH
467 }
468
99870f4d
KW
469 my $extras;
470 my $bits = $minbits;
471
99870f4d 472 if ($list) {
0e9be77f 473 my $taint = substr($list,0,0); # maintain taint
99870f4d
KW
474 my @tmp = split(/^/m, $list);
475 my %seen;
476 no warnings;
0e9be77f
DM
477 $extras = join '', $taint, grep /^[^0-9a-fA-F]/, @tmp;
478 $list = join '', $taint,
99870f4d
KW
479 map { $_->[1] }
480 sort { $a->[0] <=> $b->[0] }
481 map { /^([0-9a-fA-F]+)/; [ CORE::hex($1), $_ ] }
482 grep { /^([0-9a-fA-F]+)/ and not $seen{$1}++ } @tmp; # XXX doesn't do ranges right
483 }
a0ed51b3 484
99870f4d
KW
485 if ($none) {
486 my $hextra = sprintf "%04x", $none + 1;
487 $list =~ s/\tXXXX$/\t$hextra/mg;
488 }
a0ed51b3 489
99870f4d
KW
490 if ($minbits != 1 && $minbits < 32) { # not binary property
491 my $top = 0;
492 while ($list =~ /^([0-9a-fA-F]+)(?:[\t]([0-9a-fA-F]+)?)(?:[ \t]([0-9a-fA-F]+))?/mg) {
493 my $min = CORE::hex $1;
494 my $max = defined $2 ? CORE::hex $2 : $min;
495 my $val = defined $3 ? CORE::hex $3 : 0;
496 $val += $max - $min if defined $3;
497 $top = $val if $val > $top;
498 }
499 my $topbits =
500 $top > 0xffff ? 32 :
501 $top > 0xff ? 16 : 8;
502 $bits = $topbits if $bits < $topbits;
503 }
a0ed51b3 504
99870f4d
KW
505 my @extras;
506 if ($extras) {
507 for my $x ($extras) {
0e9be77f 508 my $taint = substr($x,0,0); # maintain taint
99870f4d
KW
509 pos $x = 0;
510 while ($x =~ /^([^0-9a-fA-F\n])(.*)/mg) {
0e9be77f
DM
511 my $char = "$1$taint";
512 my $name = "$2$taint";
513 print STDERR __LINE__, ": char [$char] => name [$name]\n"
514 if DEBUG;
99870f4d
KW
515 if ($char =~ /[-+!&]/) {
516 my ($c,$t) = split(/::/, $name, 2); # bogus use of ::, really
517 my $subobj;
518 if ($c eq 'utf8') {
519 $subobj = utf8->SWASHNEW($t, "", $minbits, 0);
520 }
521 elsif (exists &$name) {
522 $subobj = utf8->SWASHNEW($name, "", $minbits, 0);
523 }
524 elsif ($c =~ /^([0-9a-fA-F]+)/) {
525 $subobj = utf8->SWASHNEW("", $c, $minbits, 0);
526 }
3dd10fe8
KW
527 if (! ref $subobj) {
528 pop @recursed if @recursed && $type;
529 return $subobj;
530 }
99870f4d
KW
531 push @extras, $name => $subobj;
532 $bits = $subobj->{BITS} if $bits < $subobj->{BITS};
533 }
534 }
535 }
536 }
a0ed51b3 537
99870f4d 538 if (DEBUG) {
0fada5e1 539 print STDERR __LINE__, ": CLASS = $class, TYPE => $type, BITS => $bits, NONE => $none, INVERT_IT => $invert_it, USER_DEFINED => $user_defined";
99870f4d
KW
540 print STDERR "\nLIST =>\n$list" if defined $list;
541 print STDERR "\nEXTRAS =>\n$extras" if defined $extras;
542 print STDERR "\n";
543 }
a0ed51b3 544
99870f4d
KW
545 my $SWASH = bless {
546 TYPE => $type,
547 BITS => $bits,
548 EXTRAS => $extras,
549 LIST => $list,
550 NONE => $none,
0fada5e1 551 USER_DEFINED => $user_defined,
99870f4d
KW
552 @extras,
553 } => $class;
554
555 if ($file) {
556 $Cache{$class, $file} = $SWASH;
64870e05
KW
557 if ($type
558 && exists $utf8::SwashInfo{$type}
559 && exists $utf8::SwashInfo{$type}{'specials_name'})
560 {
561 my $specials_name = $utf8::SwashInfo{$type}{'specials_name'};
562 no strict "refs";
69b36ee0 563 print STDERR "\nspecials_name => $specials_name\n" if DEBUG;
64870e05
KW
564 $SWASH->{'SPECIALS'} = \%$specials_name;
565 }
a6517ebf 566 $SWASH->{'INVERT_IT'} = $invert_it;
99870f4d 567 }
5beb625e 568
3dd10fe8
KW
569 pop @recursed if @recursed && $type;
570
99870f4d 571 return $SWASH;
5beb625e 572 }
a0ed51b3
LW
573}
574
979f2922 575# Now SWASHGET is recasted into a C function S_swash_get (see utf8.c).
a0ed51b3
LW
576
5771;