This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
utf8_heavy.pl: Turn on $| if debugging
[perl5.git] / lib / utf8_heavy.pl
CommitLineData
a0ed51b3 1package utf8;
cf25bb62
JH
2use strict;
3use warnings;
a0ed51b3 4
15732964 5sub DEBUG () { 0 }
4b17cb47 6$|=1 if DEBUG;
a0ed51b3
LW
7
8sub DESTROY {}
9
5beb625e
JH
10my %Cache;
11
86916d66
JH
12sub croak { require Carp; Carp::croak(@_) }
13
c4ab2516 14sub _loose_name ($) {
45376db6
KW
15 # Given a lowercase property or property-value name, return its
16 # standardized version that is expected for look-up in the 'loose' hashes
17 # in Heavy.pl (hence, this depends on what mktables does). This squeezes
18 # out blanks, underscores and dashes. The complication stems from the
19 # grandfathered-in 'L_', which retains a single trailing underscore.
20
21 my $loose = $_[0] =~ s/[-\s_]//rg;
22
cf2cd801 23 return $loose if $loose !~ / ^ (?: is | to )? l $/x;
45376db6
KW
24 return 'l_' if $_[0] =~ / l .* _ /x; # If original had a trailing '_'
25 return $loose;
26}
27
5beb625e 28##
3a2263fe
RGS
29## "SWASH" == "SWATCH HASH". A "swatch" is a swatch of the Unicode landscape.
30## It's a data structure that encodes a set of Unicode characters.
5beb625e
JH
31##
32
99870f4d
KW
33{
34 # If a floating point number is within this distance from the value of a
35 # fraction, it is considered to be that fraction, even if many more digits
36 # are specified that don't exactly match.
37 my $min_floating_slop;
38
3dd10fe8
KW
39 # To guard against this program calling something that in turn ends up
40 # calling this program with the same inputs, and hence infinitely
41 # recursing, we keep a stack of the properties that are currently in
42 # progress, pushed upon entry, popped upon return.
43 my @recursed;
44
99870f4d
KW
45 sub SWASHNEW {
46 my ($class, $type, $list, $minbits, $none) = @_;
0fada5e1 47 my $user_defined = 0;
99870f4d
KW
48 local $^D = 0 if $^D;
49
3dd10fe8 50 $class = "" unless defined $class;
5bcf08c4
KW
51 print STDERR __LINE__, ": class=$class, type=$type, list=",
52 (defined $list) ? $list : ':undef:',
53 ", minbits=$minbits, none=$none\n" if DEBUG;
99870f4d
KW
54
55 ##
56 ## Get the list of codepoints for the type.
57 ## Called from swash_init (see utf8.c) or SWASHNEW itself.
58 ##
59 ## Callers of swash_init:
60 ## op.c:pmtrans -- for tr/// and y///
61 ## regexec.c:regclass_swash -- for /[]/, \p, and \P
62 ## utf8.c:is_utf8_common -- for common Unicode properties
63 ## utf8.c:to_utf8_case -- for lc, uc, ucfirst, etc. and //i
681d705c 64 ## Unicode::UCD::prop_invlist
99870f4d
KW
65 ##
66 ## Given a $type, our goal is to fill $list with the set of codepoint
67 ## ranges. If $type is false, $list passed is used.
68 ##
69 ## $minbits:
70 ## For binary properties, $minbits must be 1.
71 ## For character mappings (case and transliteration), $minbits must
72 ## be a number except 1.
73 ##
74 ## $list (or that filled according to $type):
75 ## Refer to perlunicode.pod, "User-Defined Character Properties."
76 ##
77 ## For binary properties, only characters with the property value
78 ## of True should be listed. The 3rd column, if any, will be ignored
79 ##
80 ## $none is undocumented, so I'm (khw) trying to do some documentation
81 ## of it now. It appears to be if there is a mapping in an input file
82 ## that maps to 'XXXX', then that is replaced by $none+1, expressed in
83fe8199 83 ## hexadecimal. It is used somehow in tr///.
99870f4d
KW
84 ##
85 ## To make the parsing of $type clear, this code takes the a rather
86 ## unorthodox approach of last'ing out of the block once we have the
87 ## info we need. Were this to be a subroutine, the 'last' would just
88 ## be a 'return'.
89 ##
fbe6f3cf
KW
90 # If a problem is found $type is returned;
91 # Upon success, a new (or cached) blessed object is returned with
92 # keys TYPE, BITS, EXTRAS, LIST, and NONE with values having the
93 # same meanings as the input parameters.
a6517ebf
KW
94 # SPECIALS contains a reference to any special-treatment hash in the
95 # INVERT_IT is non-zero if the result should be inverted before use
0fada5e1
KW
96 # USER_DEFINED is non-zero if the result came from a user-defined
97 # property.
99870f4d
KW
98 my $file; ## file to load data from, and also part of the %Cache key.
99 my $ListSorted = 0;
100
101 # Change this to get a different set of Unicode tables
102 my $unicore_dir = 'unicore';
a6517ebf 103 my $invert_it = 0;
99870f4d
KW
104
105 if ($type)
cf25bb62 106 {
3dd10fe8
KW
107
108 # Verify that this isn't a recursive call for this property.
109 # Can't use croak, as it may try to recurse here itself.
110 my $class_type = $class . "::$type";
111 if (grep { $_ eq $class_type } @recursed) {
112 CORE::die "panic: Infinite recursion in SWASHNEW for '$type'\n";
113 }
114 push @recursed, $class_type;
115
99870f4d
KW
116 $type =~ s/^\s+//;
117 $type =~ s/\s+$//;
118
4ff124d1
KW
119 # regcomp.c surrounds the property name with '__" and '_i' if this
120 # is to be caseless matching.
121 my $caseless = $type =~ s/^__(.*)_i$/$1/;
122
123 print STDERR __LINE__, ": type=$type, caseless=$caseless\n" if DEBUG;
99870f4d
KW
124
125 GETFILE:
126 {
127 ##
a44f4b74
KW
128 ## It could be a user-defined property. Look in current
129 ## package if no package given
99870f4d
KW
130 ##
131
132 my $caller1 = $type =~ s/(.+)::// ? $1 : caller(1);
133
d658a8a8 134 if (defined $caller1 && $type =~ /^I[ns]\w+$/) {
99870f4d
KW
135 my $prop = "${caller1}::$type";
136 if (exists &{$prop}) {
0e9be77f
DM
137 # stolen from Scalar::Util::PP::tainted()
138 my $tainted;
139 {
140 local($@, $SIG{__DIE__}, $SIG{__WARN__});
141 local $^W = 0;
142 no warnings;
143 eval { kill 0 * $prop };
144 $tainted = 1 if $@ =~ /^Insecure/;
145 }
146 die "Insecure user-defined property \\p{$prop}\n"
147 if $tainted;
99870f4d 148 no strict 'refs';
4ff124d1 149 $list = &{$prop}($caseless);
0fada5e1 150 $user_defined = 1;
99870f4d
KW
151 last GETFILE;
152 }
153 }
154
a44f4b74
KW
155 # During Perl's compilation, this routine may be called before
156 # the tables are constructed. If so, we have a chicken/egg
157 # problem. If we die, the tables never get constructed, so
158 # keep going, but return an empty table so only what the code
159 # has compiled in internally (currently ASCII/Latin1 range
160 # matching) will work.
1363cbd0
FC
161 BEGIN {
162 # Poor man's constant, to avoid a run-time check.
163 $utf8::{miniperl}
164 = \! defined &DynaLoader::boot_DynaLoader;
165 }
166 if (miniperl) {
a44f4b74
KW
167 eval "require '$unicore_dir/Heavy.pl'";
168 last GETFILE if $@;
169 }
170 else {
171 require "$unicore_dir/Heavy.pl";
172 }
1363cbd0 173 BEGIN { delete $utf8::{miniperl} }
99870f4d 174
4ff124d1 175 # All property names are matched caselessly
49801ec2 176 my $property_and_table = CORE::lc $type;
99870f4d
KW
177 print STDERR __LINE__, ": $property_and_table\n" if DEBUG;
178
179 # See if is of the compound form 'property=value', where the
180 # value indicates the table we should use.
181 my ($property, $table, @remainder) =
182 split /\s*[:=]\s*/, $property_and_table, -1;
3dd10fe8
KW
183 if (@remainder) {
184 pop @recursed if @recursed;
185 return $type;
186 }
99870f4d
KW
187
188 my $prefix;
189 if (! defined $table) {
190
191 # Here, is the single form. The property becomes empty, and
192 # the whole value is the table.
193 $table = $property;
194 $prefix = $property = "";
195 } else {
196 print STDERR __LINE__, ": $property\n" if DEBUG;
197
198 # Here it is the compound property=table form. The property
807807b7
KW
199 # name is always loosely matched, and always can have an
200 # optional 'is' prefix (which isn't true in the single
201 # form).
c4ab2516 202 $property = _loose_name($property) =~ s/^is//r;
99870f4d
KW
203
204 # And convert to canonical form. Quit if not valid.
205 $property = $utf8::loose_property_name_of{$property};
3dd10fe8
KW
206 if (! defined $property) {
207 pop @recursed if @recursed;
208 return $type;
209 }
99870f4d
KW
210
211 $prefix = "$property=";
212
213 # If the rhs looks like it is a number...
214 print STDERR __LINE__, ": table=$table\n" if DEBUG;
215 if ($table =~ qr{ ^ [ \s 0-9 _ + / . -]+ $ }x) {
216 print STDERR __LINE__, ": table=$table\n" if DEBUG;
217
218 # Don't allow leading nor trailing slashes
3dd10fe8
KW
219 if ($table =~ / ^ \/ | \/ $ /x) {
220 pop @recursed if @recursed;
221 return $type;
222 }
99870f4d
KW
223
224 # Split on slash, in case it is a rational, like \p{1/5}
225 my @parts = split qr{ \s* / \s* }x, $table, -1;
226 print __LINE__, ": $type\n" if @parts > 2 && DEBUG;
227
228 # Can have maximum of one slash
3dd10fe8
KW
229 if (@parts > 2) {
230 pop @recursed if @recursed;
231 return $type;
232 }
99870f4d
KW
233
234 foreach my $part (@parts) {
235 print __LINE__, ": part=$part\n" if DEBUG;
236
237 $part =~ s/^\+\s*//; # Remove leading plus
238 $part =~ s/^-\s*/-/; # Remove blanks after unary
239 # minus
240
241 # Remove underscores between digits.
242 $part =~ s/( ?<= [0-9] ) _ (?= [0-9] ) //xg;
243
244 # No leading zeros (but don't make a single '0'
245 # into a null string)
246 $part =~ s/ ^ ( -? ) 0+ /$1/x;
247 $part .= '0' if $part eq '-' || $part eq "";
248
249 # No trailing zeros after a decimal point
250 $part =~ s/ ( \. .*? ) 0+ $ /$1/x;
251
252 # Begin with a 0 if a leading decimal point
253 $part =~ s/ ^ ( -? ) \. /${1}0./x;
254
255 # Ensure not a trailing decimal point: turn into an
256 # integer
257 $part =~ s/ \. $ //x;
258
259 print STDERR __LINE__, ": part=$part\n" if DEBUG;
260 #return $type if $part eq "";
261
262 # Result better look like a number. (This test is
263 # needed because, for example could have a plus in
264 # the middle.)
3dd10fe8
KW
265 if ($part !~ / ^ -? [0-9]+ ( \. [0-9]+)? $ /x) {
266 pop @recursed if @recursed;
267 return $type;
268 }
99870f4d
KW
269 }
270
271 # If a rational...
272 if (@parts == 2) {
273
274 # If denominator is negative, get rid of it, and ...
275 if ($parts[1] =~ s/^-//) {
276
277 # If numerator is also negative, convert the
278 # whole thing to positive, or move the minus to
279 # the numerator
280 if ($parts[0] !~ s/^-//) {
281 $parts[0] = '-' . $parts[0];
282 }
283 }
284 $table = join '/', @parts;
285 }
286 elsif ($property ne 'nv' || $parts[0] !~ /\./) {
287
288 # Here is not numeric value, or doesn't have a
289 # decimal point. No further manipulation is
290 # necessary. (Note the hard-coded property name.
291 # This could fail if other properties eventually
292 # had fractions as well; perhaps the cjk ones
293 # could evolve to do that. This hard-coding could
294 # be fixed by mktables generating a list of
295 # properties that could have fractions.)
296 $table = $parts[0];
297 } else {
298
299 # Here is a floating point numeric_value. Try to
300 # convert to rational. First see if is in the list
301 # of known ones.
302 if (exists $utf8::nv_floating_to_rational{$parts[0]}) {
303 $table = $utf8::nv_floating_to_rational{$parts[0]};
304 } else {
305
306 # Here not in the list. See if is close
307 # enough to something in the list. First
308 # determine what 'close enough' means. It has
309 # to be as tight as what mktables says is the
310 # maximum slop, and as tight as how many
311 # digits we were passed. That is, if the user
312 # said .667, .6667, .66667, etc. we match as
313 # many digits as they passed until get to
314 # where it doesn't matter any more due to the
315 # machine's precision. If they said .6666668,
316 # we fail.
317 (my $fraction = $parts[0]) =~ s/^.*\.//;
318 my $epsilon = 10 ** - (length($fraction));
319 if ($epsilon > $utf8::max_floating_slop) {
320 $epsilon = $utf8::max_floating_slop;
321 }
322
323 # But it can't be tighter than the minimum
324 # precision for this machine. If haven't
325 # already calculated that minimum, do so now.
326 if (! defined $min_floating_slop) {
327
328 # Keep going down an order of magnitude
329 # until find that adding this quantity to
330 # 1 remains 1; but put an upper limit on
331 # this so in case this algorithm doesn't
332 # work properly on some platform, that we
333 # won't loop forever.
334 my $count = 0;
335 $min_floating_slop = 1;
336 while (1+ $min_floating_slop != 1
337 && $count++ < 50)
338 {
339 my $next = $min_floating_slop / 10;
340 last if $next == 0; # If underflows,
341 # use previous one
342 $min_floating_slop = $next;
343 print STDERR __LINE__, ": min_float_slop=$min_floating_slop\n" if DEBUG;
344 }
345
346 # Back off a couple orders of magnitude,
347 # just to be safe.
348 $min_floating_slop *= 100;
349 }
350
351 if ($epsilon < $min_floating_slop) {
352 $epsilon = $min_floating_slop;
353 }
354 print STDERR __LINE__, ": fraction=.$fraction; epsilon=$epsilon\n" if DEBUG;
355
356 undef $table;
357
358 # And for each possible rational in the table,
359 # see if it is within epsilon of the input.
360 foreach my $official
361 (keys %utf8::nv_floating_to_rational)
362 {
363 print STDERR __LINE__, ": epsilon=$epsilon, official=$official, diff=", abs($parts[0] - $official), "\n" if DEBUG;
364 if (abs($parts[0] - $official) < $epsilon) {
365 $table =
366 $utf8::nv_floating_to_rational{$official};
367 last;
368 }
369 }
370
371 # Quit if didn't find one.
3dd10fe8
KW
372 if (! defined $table) {
373 pop @recursed if @recursed;
374 return $type;
375 }
99870f4d
KW
376 }
377 }
378 print STDERR __LINE__, ": $property=$table\n" if DEBUG;
379 }
380 }
381
382 # Combine lhs (if any) and rhs to get something that matches
383 # the syntax of the lookups.
384 $property_and_table = "$prefix$table";
385 print STDERR __LINE__, ": $property_and_table\n" if DEBUG;
386
387 # First try stricter matching.
388 $file = $utf8::stricter_to_file_of{$property_and_table};
389
390 # If didn't find it, try again with looser matching by editing
391 # out the applicable characters on the rhs and looking up
392 # again.
393 if (! defined $file) {
c4ab2516 394 $table = _loose_name($table);
99870f4d
KW
395 $property_and_table = "$prefix$table";
396 print STDERR __LINE__, ": $property_and_table\n" if DEBUG;
397 $file = $utf8::loose_to_file_of{$property_and_table};
398 }
399
400 # Add the constant and go fetch it in.
401 if (defined $file) {
a6517ebf
KW
402
403 # A beginning ! means to invert
404 $invert_it = $file =~ s/^!//;
405
99870f4d
KW
406 if ($utf8::why_deprecated{$file}) {
407 warnings::warnif('deprecated', "Use of '$type' in \\p{} or \\P{} is deprecated because: $utf8::why_deprecated{$file};");
408 }
4ff124d1
KW
409
410 if ($caseless
411 && exists $utf8::caseless_equivalent{$property_and_table})
412 {
413 $file = $utf8::caseless_equivalent{$property_and_table};
414 }
99870f4d
KW
415 $file= "$unicore_dir/lib/$file.pl";
416 last GETFILE;
417 }
418 print STDERR __LINE__, ": didn't find $property_and_table\n" if DEBUG;
419
420 ##
99870f4d
KW
421 ## Last attempt -- see if it's a standard "To" name
422 ## (e.g. "ToLower") ToTitle is used by ucfirst().
423 ## The user-level way to access ToDigit() and ToFold()
424 ## is to use Unicode::UCD.
425 ##
cf2cd801
KW
426 # Only check if caller wants non-binary
427 if ($minbits != 1 && $property_and_table =~ s/^to//) {
cf2cd801
KW
428
429 # Look input up in list of properties for which we have
430 # mapping files.
431 if (defined ($file =
432 $utf8::loose_property_to_file_of{$property_and_table}))
433 {
434 $type = $utf8::file_to_swash_name{$file};
435 print STDERR __LINE__, ": type set to $type\n" if DEBUG;
436 $file = "$unicore_dir/$file.pl";
437 last GETFILE;
438 }
814687ad 439 }
99870f4d
KW
440
441 ##
442 ## If we reach this line, it's because we couldn't figure
443 ## out what to do with $type. Ouch.
444 ##
445
3dd10fe8 446 pop @recursed if @recursed;
99870f4d 447 return $type;
c9db9632 448 } # end of GETFILE block
cf25bb62 449
99870f4d
KW
450 if (defined $file) {
451 print STDERR __LINE__, ": found it (file='$file')\n" if DEBUG;
452
453 ##
454 ## If we reach here, it was due to a 'last GETFILE' above
455 ## (exception: user-defined properties and mappings), so we
456 ## have a filename, so now we load it if we haven't already.
457 ## If we have, return the cached results. The cache key is the
458 ## class and file to load.
459 ##
460 my $found = $Cache{$class, $file};
461 if ($found and ref($found) eq $class) {
a6517ebf 462 print STDERR __LINE__, ": Returning cached '$file' for \\p{$type}; invert_it=$invert_it\n" if DEBUG;
3dd10fe8 463 pop @recursed if @recursed;
a6517ebf 464 $found->{'INVERT_IT'} = $invert_it;
99870f4d
KW
465 return $found;
466 }
467
468 local $@;
469 local $!;
470 $list = do $file; die $@ if $@;
0a0ffbce 471 }
cf25bb62 472
99870f4d 473 $ListSorted = 1; ## we know that these lists are sorted
c9db9632
KW
474 } # End of $type is non-null
475
476 # Here, either $type was null, or we found the requested property and
477 # read it into $list
cf25bb62 478
99870f4d 479 my $extras;
c9db9632 480
99870f4d
KW
481 my $bits = $minbits;
482
99870f4d 483 if ($list) {
0e9be77f 484 my $taint = substr($list,0,0); # maintain taint
99870f4d
KW
485 my @tmp = split(/^/m, $list);
486 my %seen;
487 no warnings;
0e9be77f
DM
488 $extras = join '', $taint, grep /^[^0-9a-fA-F]/, @tmp;
489 $list = join '', $taint,
99870f4d
KW
490 map { $_->[1] }
491 sort { $a->[0] <=> $b->[0] }
492 map { /^([0-9a-fA-F]+)/; [ CORE::hex($1), $_ ] }
493 grep { /^([0-9a-fA-F]+)/ and not $seen{$1}++ } @tmp; # XXX doesn't do ranges right
494 }
a0ed51b3 495
99870f4d
KW
496 if ($none) {
497 my $hextra = sprintf "%04x", $none + 1;
498 $list =~ s/\tXXXX$/\t$hextra/mg;
499 }
a0ed51b3 500
99870f4d
KW
501 if ($minbits != 1 && $minbits < 32) { # not binary property
502 my $top = 0;
503 while ($list =~ /^([0-9a-fA-F]+)(?:[\t]([0-9a-fA-F]+)?)(?:[ \t]([0-9a-fA-F]+))?/mg) {
504 my $min = CORE::hex $1;
505 my $max = defined $2 ? CORE::hex $2 : $min;
506 my $val = defined $3 ? CORE::hex $3 : 0;
507 $val += $max - $min if defined $3;
508 $top = $val if $val > $top;
509 }
510 my $topbits =
511 $top > 0xffff ? 32 :
512 $top > 0xff ? 16 : 8;
513 $bits = $topbits if $bits < $topbits;
514 }
a0ed51b3 515
99870f4d
KW
516 my @extras;
517 if ($extras) {
518 for my $x ($extras) {
0e9be77f 519 my $taint = substr($x,0,0); # maintain taint
99870f4d
KW
520 pos $x = 0;
521 while ($x =~ /^([^0-9a-fA-F\n])(.*)/mg) {
0e9be77f
DM
522 my $char = "$1$taint";
523 my $name = "$2$taint";
524 print STDERR __LINE__, ": char [$char] => name [$name]\n"
525 if DEBUG;
99870f4d
KW
526 if ($char =~ /[-+!&]/) {
527 my ($c,$t) = split(/::/, $name, 2); # bogus use of ::, really
528 my $subobj;
529 if ($c eq 'utf8') {
530 $subobj = utf8->SWASHNEW($t, "", $minbits, 0);
531 }
532 elsif (exists &$name) {
533 $subobj = utf8->SWASHNEW($name, "", $minbits, 0);
534 }
535 elsif ($c =~ /^([0-9a-fA-F]+)/) {
536 $subobj = utf8->SWASHNEW("", $c, $minbits, 0);
537 }
3dd10fe8
KW
538 if (! ref $subobj) {
539 pop @recursed if @recursed && $type;
540 return $subobj;
541 }
99870f4d
KW
542 push @extras, $name => $subobj;
543 $bits = $subobj->{BITS} if $bits < $subobj->{BITS};
544 }
545 }
546 }
547 }
a0ed51b3 548
99870f4d 549 if (DEBUG) {
0fada5e1 550 print STDERR __LINE__, ": CLASS = $class, TYPE => $type, BITS => $bits, NONE => $none, INVERT_IT => $invert_it, USER_DEFINED => $user_defined";
99870f4d
KW
551 print STDERR "\nLIST =>\n$list" if defined $list;
552 print STDERR "\nEXTRAS =>\n$extras" if defined $extras;
553 print STDERR "\n";
554 }
a0ed51b3 555
99870f4d
KW
556 my $SWASH = bless {
557 TYPE => $type,
558 BITS => $bits,
559 EXTRAS => $extras,
560 LIST => $list,
561 NONE => $none,
0fada5e1 562 USER_DEFINED => $user_defined,
99870f4d
KW
563 @extras,
564 } => $class;
565
566 if ($file) {
567 $Cache{$class, $file} = $SWASH;
64870e05
KW
568 if ($type
569 && exists $utf8::SwashInfo{$type}
570 && exists $utf8::SwashInfo{$type}{'specials_name'})
571 {
572 my $specials_name = $utf8::SwashInfo{$type}{'specials_name'};
573 no strict "refs";
69b36ee0 574 print STDERR "\nspecials_name => $specials_name\n" if DEBUG;
64870e05
KW
575 $SWASH->{'SPECIALS'} = \%$specials_name;
576 }
a6517ebf 577 $SWASH->{'INVERT_IT'} = $invert_it;
99870f4d 578 }
5beb625e 579
3dd10fe8
KW
580 pop @recursed if @recursed && $type;
581
99870f4d 582 return $SWASH;
5beb625e 583 }
a0ed51b3
LW
584}
585
979f2922 586# Now SWASHGET is recasted into a C function S_swash_get (see utf8.c).
a0ed51b3
LW
587
5881;