This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Unicode::UCD: Remove dead code
[perl5.git] / lib / Unicode / UCD.pm
CommitLineData
55d7b906 1package Unicode::UCD;
561c79ed
JH
2
3use strict;
4use warnings;
36c2430c 5no warnings 'surrogate'; # surrogates can be inputs to this
98ef7649 6use charnames ();
561c79ed 7
ce6e23cf 8our $VERSION = '0.62';
561c79ed
JH
9
10require Exporter;
11
12our @ISA = qw(Exporter);
74f8133e 13
10a6ecd2
JH
14our @EXPORT_OK = qw(charinfo
15 charblock charscript
16 charblocks charscripts
b08cd201 17 charinrange
12fee290 18 charprop
40416981 19 charprops_all
ea508aee 20 general_categories bidi_types
b08cd201 21 compexcl
66aa79e2 22 casefold all_casefolds casespec
7319f91d
KW
23 namedseq
24 num
7ef25837
KW
25 prop_aliases
26 prop_value_aliases
9024667a 27 prop_values
681d705c 28 prop_invlist
62b3b855 29 prop_invmap
1fdd5e53 30 search_invlist
681d705c 31 MAX_CP
7319f91d 32 );
561c79ed
JH
33
34use Carp;
35
a1ae4420
KW
36sub IS_ASCII_PLATFORM { ord("A") == 65 }
37
561c79ed
JH
38=head1 NAME
39
55d7b906 40Unicode::UCD - Unicode character database
561c79ed
JH
41
42=head1 SYNOPSIS
43
55d7b906 44 use Unicode::UCD 'charinfo';
b08cd201 45 my $charinfo = charinfo($codepoint);
561c79ed 46
12fee290
KW
47 use Unicode::UCD 'charprop';
48 my $value = charprop($codepoint, $property);
49
40416981
KW
50 use Unicode::UCD 'charprops_all';
51 my $all_values_hash_ref = charprops_all($codepoint);
52
956cae9a 53 use Unicode::UCD 'casefold';
d25148c1 54 my $casefold = casefold($codepoint);
956cae9a 55
66aa79e2
KW
56 use Unicode::UCD 'all_casefolds';
57 my $all_casefolds_ref = all_casefolds();
58
5d8e6e41 59 use Unicode::UCD 'casespec';
d25148c1 60 my $casespec = casespec($codepoint);
5d8e6e41 61
55d7b906 62 use Unicode::UCD 'charblock';
e882dd67
JH
63 my $charblock = charblock($codepoint);
64
55d7b906 65 use Unicode::UCD 'charscript';
65044554 66 my $charscript = charscript($codepoint);
561c79ed 67
55d7b906 68 use Unicode::UCD 'charblocks';
e145285f
JH
69 my $charblocks = charblocks();
70
55d7b906 71 use Unicode::UCD 'charscripts';
ea508aee 72 my $charscripts = charscripts();
e145285f 73
55d7b906 74 use Unicode::UCD qw(charscript charinrange);
e145285f
JH
75 my $range = charscript($script);
76 print "looks like $script\n" if charinrange($range, $codepoint);
77
ea508aee
JH
78 use Unicode::UCD qw(general_categories bidi_types);
79 my $categories = general_categories();
80 my $types = bidi_types();
81
7ef25837
KW
82 use Unicode::UCD 'prop_aliases';
83 my @space_names = prop_aliases("space");
84
85 use Unicode::UCD 'prop_value_aliases';
86 my @gc_punct_names = prop_value_aliases("Gc", "Punct");
87
9024667a
KW
88 use Unicode::UCD 'prop_values';
89 my @all_EA_short_names = prop_values("East_Asian_Width");
90
681d705c
KW
91 use Unicode::UCD 'prop_invlist';
92 my @puncts = prop_invlist("gc=punctuation");
93
62b3b855
KW
94 use Unicode::UCD 'prop_invmap';
95 my ($list_ref, $map_ref, $format, $missing)
96 = prop_invmap("General Category");
97
1fdd5e53
KW
98 use Unicode::UCD 'search_invlist';
99 my $index = search_invlist(\@invlist, $code_point);
100
55d7b906 101 use Unicode::UCD 'compexcl';
e145285f
JH
102 my $compexcl = compexcl($codepoint);
103
a2bd7410
JH
104 use Unicode::UCD 'namedseq';
105 my $namedseq = namedseq($named_sequence_name);
106
55d7b906 107 my $unicode_version = Unicode::UCD::UnicodeVersion();
e145285f 108
7319f91d 109 my $convert_to_numeric =
62a8c8c2 110 Unicode::UCD::num("\N{RUMI DIGIT ONE}\N{RUMI DIGIT TWO}");
7319f91d 111
561c79ed
JH
112=head1 DESCRIPTION
113
a452d459
KW
114The Unicode::UCD module offers a series of functions that
115provide a simple interface to the Unicode
8b731da2 116Character Database.
561c79ed 117
a452d459
KW
118=head2 code point argument
119
120Some of the functions are called with a I<code point argument>, which is either
a1ae4420 121a decimal or a hexadecimal scalar designating a code point in the platform's
91e78470
KW
122native character set (extended to Unicode), or a string containing C<U+>
123followed by hexadecimals
a1ae4420
KW
124designating a Unicode code point. A leading 0 will force a hexadecimal
125interpretation, as will a hexadecimal digit that isn't a decimal digit.
bc75372e
KW
126
127Examples:
128
a1ae4420
KW
129 223 # Decimal 223 in native character set
130 0223 # Hexadecimal 223, native (= 547 decimal)
131 0xDF # Hexadecimal DF, native (= 223 decimal
91e78470 132 'U+DF' # Hexadecimal DF, in Unicode's character set
a1ae4420 133 (= LATIN SMALL LETTER SHARP S)
bc75372e
KW
134
135Note that the largest code point in Unicode is U+10FFFF.
c3e5bc54 136
561c79ed
JH
137=cut
138
10a6ecd2 139my $BLOCKSFH;
10a6ecd2 140my $VERSIONFH;
b08cd201
JH
141my $CASEFOLDFH;
142my $CASESPECFH;
a2bd7410 143my $NAMEDSEQFH;
e80c2d9d 144my $v_unicode_version; # v-string.
561c79ed
JH
145
146sub openunicode {
147 my ($rfh, @path) = @_;
148 my $f;
149 unless (defined $$rfh) {
150 for my $d (@INC) {
151 use File::Spec;
55d7b906 152 $f = File::Spec->catfile($d, "unicore", @path);
32c16050 153 last if open($$rfh, $f);
e882dd67 154 undef $f;
561c79ed 155 }
e882dd67
JH
156 croak __PACKAGE__, ": failed to find ",
157 File::Spec->catfile(@path), " in @INC"
158 unless defined $f;
561c79ed
JH
159 }
160 return $f;
161}
162
cb3150f5
KW
163sub _dclone ($) { # Use Storable::dclone if available; otherwise emulate it.
164
165 use if defined &DynaLoader::boot_DynaLoader, Storable => qw(dclone);
166
167 return dclone(shift) if defined &dclone;
168
169 my $arg = shift;
170 my $type = ref $arg;
171 return $arg unless $type; # No deep cloning needed for scalars
172
173 if ($type eq 'ARRAY') {
174 my @return;
175 foreach my $element (@$arg) {
176 push @return, &_dclone($element);
177 }
178 return \@return;
179 }
180 elsif ($type eq 'HASH') {
181 my %return;
182 foreach my $key (keys %$arg) {
183 $return{$key} = &_dclone($arg->{$key});
184 }
185 return \%return;
186 }
187 else {
188 croak "_dclone can't handle " . $type;
189 }
190}
191
a452d459 192=head2 B<charinfo()>
561c79ed 193
55d7b906 194 use Unicode::UCD 'charinfo';
561c79ed 195
b08cd201 196 my $charinfo = charinfo(0x41);
561c79ed 197
a452d459
KW
198This returns information about the input L</code point argument>
199as a reference to a hash of fields as defined by the Unicode
200standard. If the L</code point argument> is not assigned in the standard
201(i.e., has the general category C<Cn> meaning C<Unassigned>)
202or is a non-character (meaning it is guaranteed to never be assigned in
203the standard),
a18e976f 204C<undef> is returned.
a452d459
KW
205
206Fields that aren't applicable to the particular code point argument exist in the
207returned hash, and are empty.
208
12fee290
KW
209For results that are less "raw" than this function returns, or to get the values for
210any property, not just the few covered by this function, use the
211L</charprop()> function.
212
a452d459
KW
213The keys in the hash with the meanings of their values are:
214
215=over
216
217=item B<code>
218
a1ae4420
KW
219the input native L</code point argument> expressed in hexadecimal, with
220leading zeros
a452d459
KW
221added if necessary to make it contain at least four hexdigits
222
223=item B<name>
224
225name of I<code>, all IN UPPER CASE.
226Some control-type code points do not have names.
227This field will be empty for C<Surrogate> and C<Private Use> code points,
228and for the others without a name,
229it will contain a description enclosed in angle brackets, like
230C<E<lt>controlE<gt>>.
231
232
233=item B<category>
234
235The short name of the general category of I<code>.
236This will match one of the keys in the hash returned by L</general_categories()>.
237
7ef25837
KW
238The L</prop_value_aliases()> function can be used to get all the synonyms
239of the category name.
240
a452d459
KW
241=item B<combining>
242
243the combining class number for I<code> used in the Canonical Ordering Algorithm.
244For Unicode 5.1, this is described in Section 3.11 C<Canonical Ordering Behavior>
245available at
246L<http://www.unicode.org/versions/Unicode5.1.0/>
247
7ef25837
KW
248The L</prop_value_aliases()> function can be used to get all the synonyms
249of the combining class number.
250
a452d459
KW
251=item B<bidi>
252
253bidirectional type of I<code>.
254This will match one of the keys in the hash returned by L</bidi_types()>.
255
7ef25837
KW
256The L</prop_value_aliases()> function can be used to get all the synonyms
257of the bidi type name.
258
a452d459
KW
259=item B<decomposition>
260
261is empty if I<code> has no decomposition; or is one or more codes
a18e976f 262(separated by spaces) that, taken in order, represent a decomposition for
a452d459 263I<code>. Each has at least four hexdigits.
53cb2385 264The codes may be preceded by a word enclosed in angle brackets, then a space,
a452d459
KW
265like C<E<lt>compatE<gt> >, giving the type of decomposition
266
06bba7d5 267This decomposition may be an intermediate one whose components are also
b08d569e
KW
268decomposable. Use L<Unicode::Normalize> to get the final decomposition in one
269step.
06bba7d5 270
a452d459
KW
271=item B<decimal>
272
53cb2385 273if I<code> represents a decimal digit this is its integer numeric value
a452d459
KW
274
275=item B<digit>
276
89e4a205
KW
277if I<code> represents some other digit-like number, this is its integer
278numeric value
a452d459
KW
279
280=item B<numeric>
281
282if I<code> represents a whole or rational number, this is its numeric value.
283Rational values are expressed as a string like C<1/4>.
284
285=item B<mirrored>
286
287C<Y> or C<N> designating if I<code> is mirrored in bidirectional text
288
289=item B<unicode10>
290
291name of I<code> in the Unicode 1.0 standard if one
292existed for this code point and is different from the current name
293
294=item B<comment>
295
89e4a205 296As of Unicode 6.0, this is always empty.
a452d459
KW
297
298=item B<upper>
299
91e78470
KW
300is, if non-empty, the uppercase mapping for I<code> expressed as at least four
301hexdigits. This indicates that the full uppercase mapping is a single
302character, and is identical to the simple (single-character only) mapping.
303When this field is empty, it means that the simple uppercase mapping is
12fee290 304I<code> itself; you'll need some other means, (like L</charprop()> or
91e78470 305L</casespec()> to get the full mapping.
a452d459
KW
306
307=item B<lower>
308
91e78470
KW
309is, if non-empty, the lowercase mapping for I<code> expressed as at least four
310hexdigits. This indicates that the full lowercase mapping is a single
311character, and is identical to the simple (single-character only) mapping.
312When this field is empty, it means that the simple lowercase mapping is
12fee290 313I<code> itself; you'll need some other means, (like L</charprop()> or
91e78470 314L</casespec()> to get the full mapping.
a452d459
KW
315
316=item B<title>
317
91e78470
KW
318is, if non-empty, the titlecase mapping for I<code> expressed as at least four
319hexdigits. This indicates that the full titlecase mapping is a single
320character, and is identical to the simple (single-character only) mapping.
321When this field is empty, it means that the simple titlecase mapping is
12fee290 322I<code> itself; you'll need some other means, (like L</charprop()> or
91e78470 323L</casespec()> to get the full mapping.
a452d459
KW
324
325=item B<block>
326
a18e976f 327the block I<code> belongs to (used in C<\p{Blk=...}>).
b08d569e
KW
328The L</prop_value_aliases()> function can be used to get all the synonyms
329of the block name.
a452d459 330
b08d569e 331See L</Blocks versus Scripts>.
a452d459
KW
332
333=item B<script>
334
a18e976f 335the script I<code> belongs to.
b08d569e
KW
336The L</prop_value_aliases()> function can be used to get all the synonyms
337of the script name.
338
a452d459
KW
339See L</Blocks versus Scripts>.
340
341=back
32c16050
JH
342
343Note that you cannot do (de)composition and casing based solely on the
b08d569e
KW
344I<decomposition>, I<combining>, I<lower>, I<upper>, and I<title> fields; you
345will need also the L</casespec()> function and the C<Composition_Exclusion>
346property. (Or you could just use the L<lc()|perlfunc/lc>,
347L<uc()|perlfunc/uc>, and L<ucfirst()|perlfunc/ucfirst> functions, and the
348L<Unicode::Normalize> module.)
561c79ed
JH
349
350=cut
351
e10d7780 352# NB: This function is nearly duplicated in charnames.pm
10a6ecd2
JH
353sub _getcode {
354 my $arg = shift;
355
dc0a4417 356 if ($arg =~ /^[1-9]\d*$/) {
10a6ecd2 357 return $arg;
a1ae4420
KW
358 }
359 elsif ($arg =~ /^(?:0[xX])?([[:xdigit:]]+)$/) {
360 return CORE::hex($1);
361 }
362 elsif ($arg =~ /^[Uu]\+([[:xdigit:]]+)$/) { # Is of form U+0000, means
363 # wants the Unicode code
364 # point, not the native one
365 my $decimal = CORE::hex($1);
366 return $decimal if IS_ASCII_PLATFORM;
367 return utf8::unicode_to_native($decimal);
10a6ecd2
JH
368 }
369
370 return;
371}
372
05dbc6f8
KW
373# Populated by _num. Converts real number back to input rational
374my %real_to_rational;
375
376# To store the contents of files found on disk.
377my @BIDIS;
378my @CATEGORIES;
379my @DECOMPOSITIONS;
380my @NUMERIC_TYPES;
5c3b35c9
KW
381my %SIMPLE_LOWER;
382my %SIMPLE_TITLE;
383my %SIMPLE_UPPER;
384my %UNICODE_1_NAMES;
72fcb9f0 385my %ISO_COMMENT;
05dbc6f8 386
05dbc6f8 387sub charinfo {
a6fa416b 388
05dbc6f8
KW
389 # This function has traditionally mimicked what is in UnicodeData.txt,
390 # warts and all. This is a re-write that avoids UnicodeData.txt so that
391 # it can be removed to save disk space. Instead, this assembles
392 # information gotten by other methods that get data from various other
393 # files. It uses charnames to get the character name; and various
394 # mktables tables.
324f9e44 395
05dbc6f8 396 use feature 'unicode_strings';
a6fa416b 397
cb3150f5
KW
398 # Will fail if called under minitest
399 use if defined &DynaLoader::boot_DynaLoader, "Unicode::Normalize" => qw(getCombinClass NFD);
400
10a6ecd2
JH
401 my $arg = shift;
402 my $code = _getcode($arg);
05dbc6f8
KW
403 croak __PACKAGE__, "::charinfo: unknown code '$arg'" unless defined $code;
404
405 # Non-unicode implies undef.
406 return if $code > 0x10FFFF;
407
408 my %prop;
409 my $char = chr($code);
410
35a865d4 411 @CATEGORIES =_read_table("To/Gc.pl") unless @CATEGORIES;
05dbc6f8
KW
412 $prop{'category'} = _search(\@CATEGORIES, 0, $#CATEGORIES, $code)
413 // $utf8::SwashInfo{'ToGc'}{'missing'};
09edd811
KW
414 # Return undef if category value is 'Unassigned' or one of its synonyms
415 return if grep { lc $_ eq 'unassigned' }
416 prop_value_aliases('Gc', $prop{'category'});
05dbc6f8
KW
417
418 $prop{'code'} = sprintf "%04X", $code;
419 $prop{'name'} = ($char =~ /\p{Cntrl}/) ? '<control>'
420 : (charnames::viacode($code) // "");
421
422 $prop{'combining'} = getCombinClass($code);
423
35a865d4 424 @BIDIS =_read_table("To/Bc.pl") unless @BIDIS;
05dbc6f8
KW
425 $prop{'bidi'} = _search(\@BIDIS, 0, $#BIDIS, $code)
426 // $utf8::SwashInfo{'ToBc'}{'missing'};
427
428 # For most code points, we can just read in "unicore/Decomposition.pl", as
429 # its contents are exactly what should be output. But that file doesn't
430 # contain the data for the Hangul syllable decompositions, which can be
94c91ffc
KW
431 # algorithmically computed, and NFD() does that, so we call NFD() for
432 # those. We can't use NFD() for everything, as it does a complete
05dbc6f8 433 # recursive decomposition, and what this function has always done is to
94c91ffc
KW
434 # return what's in UnicodeData.txt which doesn't show that recursiveness.
435 # Fortunately, the NFD() of the Hanguls doesn't have any recursion
436 # issues.
437 # Having no decomposition implies an empty field; otherwise, all but
438 # "Canonical" imply a compatible decomposition, and the type is prefixed
439 # to that, as it is in UnicodeData.txt
9abdc62b
KW
440 UnicodeVersion() unless defined $v_unicode_version;
441 if ($v_unicode_version ge v2.0.0 && $char =~ /\p{Block=Hangul_Syllables}/) {
05dbc6f8
KW
442 # The code points of the decomposition are output in standard Unicode
443 # hex format, separated by blanks.
444 $prop{'decomposition'} = join " ", map { sprintf("%04X", $_)}
94c91ffc 445 unpack "U*", NFD($char);
a6fa416b 446 }
05dbc6f8 447 else {
35a865d4 448 @DECOMPOSITIONS = _read_table("Decomposition.pl")
05dbc6f8
KW
449 unless @DECOMPOSITIONS;
450 $prop{'decomposition'} = _search(\@DECOMPOSITIONS, 0, $#DECOMPOSITIONS,
451 $code) // "";
561c79ed 452 }
05dbc6f8
KW
453
454 # Can use num() to get the numeric values, if any.
455 if (! defined (my $value = num($char))) {
456 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = "";
457 }
458 else {
459 if ($char =~ /\d/) {
460 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = $value;
461 }
462 else {
463
464 # For non-decimal-digits, we have to read in the Numeric type
465 # to distinguish them. It is not just a matter of integer vs.
466 # rational, as some whole number values are not considered digits,
467 # e.g., TAMIL NUMBER TEN.
468 $prop{'decimal'} = "";
469
35a865d4 470 @NUMERIC_TYPES =_read_table("To/Nt.pl") unless @NUMERIC_TYPES;
05dbc6f8
KW
471 if ((_search(\@NUMERIC_TYPES, 0, $#NUMERIC_TYPES, $code) // "")
472 eq 'Digit')
473 {
474 $prop{'digit'} = $prop{'numeric'} = $value;
475 }
476 else {
477 $prop{'digit'} = "";
478 $prop{'numeric'} = $real_to_rational{$value} // $value;
479 }
480 }
481 }
482
483 $prop{'mirrored'} = ($char =~ /\p{Bidi_Mirrored}/) ? 'Y' : 'N';
484
35a865d4 485 %UNICODE_1_NAMES =_read_table("To/Na1.pl", "use_hash") unless %UNICODE_1_NAMES;
5c3b35c9 486 $prop{'unicode10'} = $UNICODE_1_NAMES{$code} // "";
05dbc6f8 487
72fcb9f0
KW
488 UnicodeVersion() unless defined $v_unicode_version;
489 if ($v_unicode_version ge v6.0.0) {
490 $prop{'comment'} = "";
491 }
492 else {
493 %ISO_COMMENT = _read_table("To/Isc.pl", "use_hash") unless %ISO_COMMENT;
494 $prop{'comment'} = (defined $ISO_COMMENT{$code})
495 ? $ISO_COMMENT{$code}
496 : "";
497 }
05dbc6f8 498
35a865d4 499 %SIMPLE_UPPER = _read_table("To/Uc.pl", "use_hash") unless %SIMPLE_UPPER;
bf7fe2df 500 $prop{'upper'} = (defined $SIMPLE_UPPER{$code})
d11155ec 501 ? sprintf("%04X", $SIMPLE_UPPER{$code})
bf7fe2df 502 : "";
75e7c50b 503
35a865d4 504 %SIMPLE_LOWER = _read_table("To/Lc.pl", "use_hash") unless %SIMPLE_LOWER;
bf7fe2df 505 $prop{'lower'} = (defined $SIMPLE_LOWER{$code})
d11155ec 506 ? sprintf("%04X", $SIMPLE_LOWER{$code})
bf7fe2df 507 : "";
75e7c50b 508
35a865d4 509 %SIMPLE_TITLE = _read_table("To/Tc.pl", "use_hash") unless %SIMPLE_TITLE;
bf7fe2df 510 $prop{'title'} = (defined $SIMPLE_TITLE{$code})
d11155ec 511 ? sprintf("%04X", $SIMPLE_TITLE{$code})
bf7fe2df 512 : "";
05dbc6f8
KW
513
514 $prop{block} = charblock($code);
515 $prop{script} = charscript($code);
516 return \%prop;
561c79ed
JH
517}
518
e882dd67
JH
519sub _search { # Binary search in a [[lo,hi,prop],[...],...] table.
520 my ($table, $lo, $hi, $code) = @_;
521
522 return if $lo > $hi;
523
524 my $mid = int(($lo+$hi) / 2);
525
526 if ($table->[$mid]->[0] < $code) {
10a6ecd2 527 if ($table->[$mid]->[1] >= $code) {
e882dd67
JH
528 return $table->[$mid]->[2];
529 } else {
530 _search($table, $mid + 1, $hi, $code);
531 }
532 } elsif ($table->[$mid]->[0] > $code) {
533 _search($table, $lo, $mid - 1, $code);
534 } else {
535 return $table->[$mid]->[2];
536 }
537}
538
cb366075 539sub _read_table ($;$) {
3a12600d
KW
540
541 # Returns the contents of the mktables generated table file located at $1
cb366075
KW
542 # in the form of either an array of arrays or a hash, depending on if the
543 # optional second parameter is true (for hash return) or not. In the case
544 # of a hash return, each key is a code point, and its corresponding value
545 # is what the table gives as the code point's corresponding value. In the
546 # case of an array return, each outer array denotes a range with [0] the
547 # start point of that range; [1] the end point; and [2] the value that
548 # every code point in the range has. The hash return is useful for fast
549 # lookup when the table contains only single code point ranges. The array
550 # return takes much less memory when there are large ranges.
3a12600d 551 #
cb366075 552 # This function has the side effect of setting
3a12600d
KW
553 # $utf8::SwashInfo{$property}{'format'} to be the mktables format of the
554 # table; and
555 # $utf8::SwashInfo{$property}{'missing'} to be the value for all entries
556 # not listed in the table.
557 # where $property is the Unicode property name, preceded by 'To' for map
558 # properties., e.g., 'ToSc'.
559 #
560 # Table entries look like one of:
561 # 0000 0040 Common # [65]
562 # 00AA Latin
563
564 my $table = shift;
cb366075
KW
565 my $return_hash = shift;
566 $return_hash = 0 unless defined $return_hash;
3a12600d 567 my @return;
cb366075 568 my %return;
3a12600d 569 local $_;
d11155ec 570 my $list = do "unicore/$table";
3a12600d 571
d11155ec
KW
572 # Look up if this property requires adjustments, which we do below if it
573 # does.
574 require "unicore/Heavy.pl";
575 my $property = $table =~ s/\.pl//r;
576 $property = $utf8::file_to_swash_name{$property};
577 my $to_adjust = defined $property
24303724 578 && $utf8::SwashInfo{$property}{'format'} =~ / ^ a /x;
d11155ec
KW
579
580 for (split /^/m, $list) {
3a12600d
KW
581 my ($start, $end, $value) = / ^ (.+?) \t (.*?) \t (.+?)
582 \s* ( \# .* )? # Optional comment
583 $ /x;
83fd1222
KW
584 my $decimal_start = hex $start;
585 my $decimal_end = ($end eq "") ? $decimal_start : hex $end;
24303724
KW
586 $value = hex $value if $to_adjust
587 && $utf8::SwashInfo{$property}{'format'} eq 'ax';
cb366075 588 if ($return_hash) {
83fd1222 589 foreach my $i ($decimal_start .. $decimal_end) {
d11155ec
KW
590 $return{$i} = ($to_adjust)
591 ? $value + $i - $decimal_start
592 : $value;
cb366075
KW
593 }
594 }
d11155ec
KW
595 elsif (! $to_adjust
596 && @return
597 && $return[-1][1] == $decimal_start - 1
9a96c106
KW
598 && $return[-1][2] eq $value)
599 {
600 # If this is merely extending the previous range, do just that.
601 $return[-1]->[1] = $decimal_end;
602 }
cb366075 603 else {
83fd1222 604 push @return, [ $decimal_start, $decimal_end, $value ];
cb366075 605 }
3a12600d 606 }
cb366075 607 return ($return_hash) ? %return : @return;
3a12600d
KW
608}
609
10a6ecd2
JH
610sub charinrange {
611 my ($range, $arg) = @_;
612 my $code = _getcode($arg);
613 croak __PACKAGE__, "::charinrange: unknown code '$arg'"
614 unless defined $code;
615 _search($range, 0, $#$range, $code);
616}
617
12fee290
KW
618=head2 B<charprop()>
619
620 use Unicode::UCD 'charprop';
621
622 print charprop(0x41, "Gc"), "\n";
623 print charprop(0x61, "General_Category"), "\n";
624
625 prints
626 Lu
627 Ll
628
629This returns the value of the Unicode property given by the second parameter
630for the L</code point argument> given by the first.
631
632The passed-in property may be specified as any of the synonyms returned by
633L</prop_aliases()>.
634
635The return value is always a scalar, either a string or a number. For
636properties where there are synonyms for the values, the synonym returned by
637this function is the longest, most descriptive form, the one returned by
638L</prop_value_aliases()> when called in a scalar context. Of course, you can
639call L</prop_value_aliases()> on the result to get other synonyms.
640
641The return values are more "cooked" than the L</charinfo()> ones. For
642example, the C<"uc"> property value is the actual string containing the full
643uppercase mapping of the input code point. You have to go to extra trouble
644with C<charinfo> to get this value from its C<upper> hash element when the
645full mapping differs from the simple one.
646
647Special note should be made of the return values for a few properties:
648
649=over
650
651=item Block
652
653The value returned is the new-style (see L</Old-style versus new-style block
654names>).
655
656=item Decomposition_Mapping
657
658Like L</charinfo()>, the result may be an intermediate decomposition whose
659components are also decomposable. Use L<Unicode::Normalize> to get the final
660decomposition in one step.
661
662Unlike L</charinfo()>, this does not include the decomposition type. Use the
663C<Decomposition_Type> property to get that.
664
665=item Name_Alias
666
667If the input code point's name has more than one synonym, they are returned
668joined into a single comma-separated string.
669
670=item Numeric_Value
671
672If the result is a fraction, it is converted into a floating point number to
673the accuracy of your platform.
674
675=item Script_Extensions
676
677If the result is multiple script names, they are returned joined into a single
678comma-separated string.
679
680=back
681
682When called with a property that is a Perl extension that isn't expressible in
683a compound form, this function currently returns C<undef>, as the only two
684possible values are I<true> or I<false> (1 or 0 I suppose). This behavior may
685change in the future, so don't write code that relies on it. C<Present_In> is
686a Perl extension that is expressible in a bipartite or compound form (for
687example, C<\p{Present_In=4.0}>), so C<charprop> accepts it. But C<Any> is a
688Perl extension that isn't expressible that way, so C<charprop> returns
689C<undef> for it. Also C<charprop> returns C<undef> for all Perl extensions
690that are internal-only.
691
692=cut
693
694sub charprop ($$) {
695 my ($input_cp, $prop) = @_;
696
697 my $cp = _getcode($input_cp);
698 croak __PACKAGE__, "::charprop: unknown code point '$input_cp'" unless defined $cp;
699
700 my ($list_ref, $map_ref, $format, $default)
701 = prop_invmap($prop);
702 return undef unless defined $list_ref;
703
704 my $i = search_invlist($list_ref, $cp);
705 croak __PACKAGE__, "::charprop: prop_invmap return is invalid for charprop('$input_cp', '$prop)" unless defined $i;
706
707 # $i is the index into both the inversion list and map of $cp.
708 my $map = $map_ref->[$i];
709
710 # Convert enumeration values to their most complete form.
711 if (! ref $map) {
712 my $long_form = prop_value_aliases($prop, $map);
713 $map = $long_form if defined $long_form;
714 }
715
716 if ($format =~ / ^ s /x) { # Scalars
717 return join ",", @$map if ref $map; # Convert to scalar with comma
718 # separated array elements
719
720 # Resolve ambiguity as to whether an all digit value is a code point
721 # that should be converted to a character, or whether it is really
722 # just a number. To do this, look at the default. If it is a
723 # non-empty number, we can safely assume the result is also a number.
724 if ($map =~ / ^ \d+ $ /ax && $default !~ / ^ \d+ $ /ax) {
725 $map = chr $map;
726 }
727 elsif ($map =~ / ^ (?: Y | N ) $ /x) {
728
729 # prop_invmap() returns these values for properties that are Perl
730 # extensions. But this is misleading. For now, return undef for
731 # these, as currently documented.
732 undef $map unless
733 exists $Unicode::UCD::prop_aliases{utf8::_loose_name(lc $prop)};
734 }
735 return $map;
736 }
737 elsif ($format eq 'ar') { # numbers, including rationals
738 my $offset = $cp - $list_ref->[$i];
739 return $map if $map =~ /nan/i;
740 return $map + $offset if $offset != 0; # If needs adjustment
741 return eval $map; # Convert e.g., 1/2 to 0.5
742 }
743 elsif ($format =~ /^a/) { # Some entries need adjusting
744
745 # Linearize sequences into a string.
746 return join "", map { chr $_ } @$map if ref $map; # XXX && $format =~ /^ a [dl] /x;
747
748 return "" if $map eq "" && $format =~ /^a.*e/;
749
750 # These are all character mappings. Return the chr if no adjustment
751 # is needed
752 return chr $cp if $map eq "0";
753
754 # Convert special entry.
755 if ($map eq '<hangul syllable>' && $format eq 'ad') {
756 use Unicode::Normalize qw(NFD);
757 return NFD(chr $cp);
758 }
759
760 # The rest need adjustment from the first entry in the inversion list
761 # corresponding to this map.
762 my $offset = $cp - $list_ref->[$i];
763 return chr($map + $cp - $list_ref->[$i]);
764 }
765 elsif ($format eq 'n') { # The name property
766
767 # There are two special cases, handled here.
768 if ($map =~ / ( .+ ) <code\ point> $ /x) {
769 $map = sprintf("$1%04X", $cp);
770 }
771 elsif ($map eq '<hangul syllable>') {
772 $map = charnames::viacode($cp);
773 }
774 return $map;
775 }
776 else {
777 croak __PACKAGE__, "::charprop: Internal error: unknown format '$format'. Please perlbug this";
12fee290
KW
778 }
779}
780
40416981
KW
781=head2 B<charprops_all()>
782
783 use Unicode::UCD 'charprops_all';
784
785 my $%properties_of_A_hash_ref = charprops_all("U+41");
786
787This returns a reference to a hash whose keys are all the distinct Unicode (no
788Perl extension) properties, and whose values are the respective values for
789those properties for the input L</code point argument>.
790
791Each key is the property name in its longest, most descriptive form. The
792values are what L</charprop()> would return.
793
794This function is expensive in time and memory.
795
796=cut
797
798sub charprops_all($) {
799 my $input_cp = shift;
800
801 my $cp = _getcode($input_cp);
802 croak __PACKAGE__, "::charprops_all: unknown code point '$input_cp'" unless defined $cp;
803
804 my %return;
805
806 require "unicore/UCD.pl";
807
808 foreach my $prop (keys %Unicode::UCD::prop_aliases) {
809
810 # Don't return a Perl extension. (This is the only one that
811 # %prop_aliases has in it.)
812 next if $prop eq 'perldecimaldigit';
813
814 # Use long name for $prop in the hash
815 $return{scalar prop_aliases($prop)} = charprop($cp, $prop);
816 }
817
818 return \%return;
819}
820
a452d459 821=head2 B<charblock()>
561c79ed 822
55d7b906 823 use Unicode::UCD 'charblock';
561c79ed
JH
824
825 my $charblock = charblock(0x41);
10a6ecd2 826 my $charblock = charblock(1234);
a452d459 827 my $charblock = charblock(0x263a);
10a6ecd2
JH
828 my $charblock = charblock("U+263a");
829
78bf21c2 830 my $range = charblock('Armenian');
10a6ecd2 831
53cb2385 832With a L</code point argument> C<charblock()> returns the I<block> the code point
430fe03d
KW
833belongs to, e.g. C<Basic Latin>. The old-style block name is returned (see
834L</Old-style versus new-style block names>).
b08d569e
KW
835The L</prop_value_aliases()> function can be used to get all the synonyms
836of the block name.
837
a452d459 838If the code point is unassigned, this returns the block it would belong to if
fe252ba7
KW
839it were assigned. (If the Unicode version being used is so early as to not
840have blocks, all code points are considered to be in C<No_Block>.)
10a6ecd2 841
78bf21c2
JH
842See also L</Blocks versus Scripts>.
843
53cb2385 844If supplied with an argument that can't be a code point, C<charblock()> tries to
a1ae4420
KW
845do the opposite and interpret the argument as an old-style block name. On an
846ASCII platform, the return value is a I<range set> with one range: an
df2b00e8 847anonymous array with a single element that consists of another anonymous array
a1ae4420 848whose first element is the first code point in the block, and whose second
53cb2385 849element is the final code point in the block. On an EBCDIC
a1ae4420 850platform, the first two Unicode blocks are not contiguous. Their range sets
53cb2385 851are lists containing I<start-of-range>, I<end-of-range> code point pairs. You
a1ae4420 852can test whether a code point is in a range set using the L</charinrange()>
53cb2385
KW
853function. (To be precise, each I<range set> contains a third array element,
854after the range boundary ones: the old_style block name.)
855
856If the argument to C<charblock()> is not a known block, C<undef> is
857returned.
561c79ed 858
561c79ed
JH
859=cut
860
861my @BLOCKS;
10a6ecd2 862my %BLOCKS;
561c79ed 863
10a6ecd2 864sub _charblocks {
06bba7d5
KW
865
866 # Can't read from the mktables table because it loses the hyphens in the
867 # original.
561c79ed 868 unless (@BLOCKS) {
fe252ba7
KW
869 UnicodeVersion() unless defined $v_unicode_version;
870 if ($v_unicode_version lt v2.0.0) {
871 my $subrange = [ 0, 0x10FFFF, 'No_Block' ];
872 push @BLOCKS, $subrange;
25503677 873 push @{$BLOCKS{'No_Block'}}, $subrange;
fe252ba7
KW
874 }
875 elsif (openunicode(\$BLOCKSFH, "Blocks.txt")) {
6c8d78fb 876 local $_;
ce066323 877 local $/ = "\n";
10a6ecd2 878 while (<$BLOCKSFH>) {
2796c109 879 if (/^([0-9A-F]+)\.\.([0-9A-F]+);\s+(.+)/) {
10a6ecd2
JH
880 my ($lo, $hi) = (hex($1), hex($2));
881 my $subrange = [ $lo, $hi, $3 ];
882 push @BLOCKS, $subrange;
883 push @{$BLOCKS{$3}}, $subrange;
561c79ed
JH
884 }
885 }
10a6ecd2 886 close($BLOCKSFH);
a1ae4420
KW
887 if (! IS_ASCII_PLATFORM) {
888 # The first two blocks, through 0xFF, are wrong on EBCDIC
889 # platforms.
890
891 my @new_blocks = _read_table("To/Blk.pl");
892
893 # Get rid of the first two ranges in the Unicode version, and
894 # replace them with the ones computed by mktables.
895 shift @BLOCKS;
896 shift @BLOCKS;
897 delete $BLOCKS{'Basic Latin'};
898 delete $BLOCKS{'Latin-1 Supplement'};
899
900 # But there are multiple entries in the computed versions, and
901 # we change their names to (which we know) to be the old-style
902 # ones.
903 for my $i (0.. @new_blocks - 1) {
904 if ($new_blocks[$i][2] =~ s/Basic_Latin/Basic Latin/
905 or $new_blocks[$i][2] =~
906 s/Latin_1_Supplement/Latin-1 Supplement/)
907 {
908 push @{$BLOCKS{$new_blocks[$i][2]}}, $new_blocks[$i];
909 }
910 else {
911 splice @new_blocks, $i;
912 last;
913 }
914 }
915 unshift @BLOCKS, @new_blocks;
916 }
561c79ed
JH
917 }
918 }
10a6ecd2
JH
919}
920
921sub charblock {
922 my $arg = shift;
923
924 _charblocks() unless @BLOCKS;
925
926 my $code = _getcode($arg);
561c79ed 927
10a6ecd2 928 if (defined $code) {
c707cf8e
KW
929 my $result = _search(\@BLOCKS, 0, $#BLOCKS, $code);
930 return $result if defined $result;
931 return 'No_Block';
932 }
933 elsif (exists $BLOCKS{$arg}) {
cb3150f5 934 return _dclone $BLOCKS{$arg};
10a6ecd2 935 }
e882dd67
JH
936}
937
a452d459 938=head2 B<charscript()>
e882dd67 939
55d7b906 940 use Unicode::UCD 'charscript';
e882dd67
JH
941
942 my $charscript = charscript(0x41);
10a6ecd2
JH
943 my $charscript = charscript(1234);
944 my $charscript = charscript("U+263a");
e882dd67 945
78bf21c2 946 my $range = charscript('Thai');
10a6ecd2 947
53cb2385
KW
948With a L</code point argument>, C<charscript()> returns the I<script> the
949code point belongs to, e.g., C<Latin>, C<Greek>, C<Han>.
49ea58c8
KW
950If the code point is unassigned or the Unicode version being used is so early
951that it doesn't have scripts, this function returns C<"Unknown">.
b08d569e
KW
952The L</prop_value_aliases()> function can be used to get all the synonyms
953of the script name.
78bf21c2 954
eb0cc9e3 955If supplied with an argument that can't be a code point, charscript() tries
a18e976f 956to do the opposite and interpret the argument as a script name. The
df2b00e8 957return value is a I<range set>: an anonymous array of arrays that contain
eb0cc9e3 958I<start-of-range>, I<end-of-range> code point pairs. You can test whether a
53cb2385
KW
959code point is in a range set using the L</charinrange()> function.
960(To be precise, each I<range set> contains a third array element,
961after the range boundary ones: the script name.)
962
963If the C<charscript()> argument is not a known script, C<undef> is returned.
a452d459
KW
964
965See also L</Blocks versus Scripts>.
e882dd67 966
e882dd67
JH
967=cut
968
969my @SCRIPTS;
10a6ecd2 970my %SCRIPTS;
e882dd67 971
10a6ecd2 972sub _charscripts {
49ea58c8
KW
973 unless (@SCRIPTS) {
974 UnicodeVersion() unless defined $v_unicode_version;
975 if ($v_unicode_version lt v3.1.0) {
976 push @SCRIPTS, [ 0, 0x10FFFF, 'Unknown' ];
977 }
978 else {
979 @SCRIPTS =_read_table("To/Sc.pl");
980 }
981 }
7bccef0b 982 foreach my $entry (@SCRIPTS) {
f3d50ac9 983 $entry->[2] =~ s/(_\w)/\L$1/g; # Preserve old-style casing
7bccef0b 984 push @{$SCRIPTS{$entry->[2]}}, $entry;
e882dd67 985 }
10a6ecd2
JH
986}
987
988sub charscript {
989 my $arg = shift;
990
991 _charscripts() unless @SCRIPTS;
e882dd67 992
10a6ecd2
JH
993 my $code = _getcode($arg);
994
995 if (defined $code) {
7bccef0b
KW
996 my $result = _search(\@SCRIPTS, 0, $#SCRIPTS, $code);
997 return $result if defined $result;
8079ad82 998 return $utf8::SwashInfo{'ToSc'}{'missing'};
7bccef0b 999 } elsif (exists $SCRIPTS{$arg}) {
cb3150f5 1000 return _dclone $SCRIPTS{$arg};
10a6ecd2 1001 }
7bccef0b
KW
1002
1003 return;
10a6ecd2
JH
1004}
1005
a452d459 1006=head2 B<charblocks()>
10a6ecd2 1007
55d7b906 1008 use Unicode::UCD 'charblocks';
10a6ecd2 1009
b08cd201 1010 my $charblocks = charblocks();
10a6ecd2 1011
53cb2385 1012C<charblocks()> returns a reference to a hash with the known block names
a452d459 1013as the keys, and the code point ranges (see L</charblock()>) as the values.
10a6ecd2 1014
430fe03d
KW
1015The names are in the old-style (see L</Old-style versus new-style block
1016names>).
1017
62b3b855
KW
1018L<prop_invmap("block")|/prop_invmap()> can be used to get this same data in a
1019different type of data structure.
1020
9024667a
KW
1021L<prop_values("Block")|/prop_values()> can be used to get all
1022the known new-style block names as a list, without the code point ranges.
1023
78bf21c2
JH
1024See also L</Blocks versus Scripts>.
1025
10a6ecd2
JH
1026=cut
1027
1028sub charblocks {
b08cd201 1029 _charblocks() unless %BLOCKS;
cb3150f5 1030 return _dclone \%BLOCKS;
10a6ecd2
JH
1031}
1032
a452d459 1033=head2 B<charscripts()>
10a6ecd2 1034
55d7b906 1035 use Unicode::UCD 'charscripts';
10a6ecd2 1036
ea508aee 1037 my $charscripts = charscripts();
10a6ecd2 1038
53cb2385 1039C<charscripts()> returns a reference to a hash with the known script
a452d459 1040names as the keys, and the code point ranges (see L</charscript()>) as
ea508aee 1041the values.
10a6ecd2 1042
62b3b855
KW
1043L<prop_invmap("script")|/prop_invmap()> can be used to get this same data in a
1044different type of data structure.
1045
9024667a
KW
1046L<C<prop_values("Script")>|/prop_values()> can be used to get all
1047the known script names as a list, without the code point ranges.
1048
78bf21c2
JH
1049See also L</Blocks versus Scripts>.
1050
10a6ecd2
JH
1051=cut
1052
1053sub charscripts {
b08cd201 1054 _charscripts() unless %SCRIPTS;
cb3150f5 1055 return _dclone \%SCRIPTS;
561c79ed
JH
1056}
1057
a452d459 1058=head2 B<charinrange()>
10a6ecd2 1059
f200dd12 1060In addition to using the C<\p{Blk=...}> and C<\P{Blk=...}> constructs, you
10a6ecd2 1061can also test whether a code point is in the I<range> as returned by
a452d459 1062L</charblock()> and L</charscript()> or as the values of the hash returned
53cb2385 1063by L</charblocks()> and L</charscripts()> by using C<charinrange()>:
10a6ecd2 1064
55d7b906 1065 use Unicode::UCD qw(charscript charinrange);
10a6ecd2
JH
1066
1067 $range = charscript('Hiragana');
e145285f 1068 print "looks like hiragana\n" if charinrange($range, $codepoint);
10a6ecd2
JH
1069
1070=cut
1071
ea508aee
JH
1072my %GENERAL_CATEGORIES =
1073 (
1074 'L' => 'Letter',
1075 'LC' => 'CasedLetter',
1076 'Lu' => 'UppercaseLetter',
1077 'Ll' => 'LowercaseLetter',
1078 'Lt' => 'TitlecaseLetter',
1079 'Lm' => 'ModifierLetter',
1080 'Lo' => 'OtherLetter',
1081 'M' => 'Mark',
1082 'Mn' => 'NonspacingMark',
1083 'Mc' => 'SpacingMark',
1084 'Me' => 'EnclosingMark',
1085 'N' => 'Number',
1086 'Nd' => 'DecimalNumber',
1087 'Nl' => 'LetterNumber',
1088 'No' => 'OtherNumber',
1089 'P' => 'Punctuation',
1090 'Pc' => 'ConnectorPunctuation',
1091 'Pd' => 'DashPunctuation',
1092 'Ps' => 'OpenPunctuation',
1093 'Pe' => 'ClosePunctuation',
1094 'Pi' => 'InitialPunctuation',
1095 'Pf' => 'FinalPunctuation',
1096 'Po' => 'OtherPunctuation',
1097 'S' => 'Symbol',
1098 'Sm' => 'MathSymbol',
1099 'Sc' => 'CurrencySymbol',
1100 'Sk' => 'ModifierSymbol',
1101 'So' => 'OtherSymbol',
1102 'Z' => 'Separator',
1103 'Zs' => 'SpaceSeparator',
1104 'Zl' => 'LineSeparator',
1105 'Zp' => 'ParagraphSeparator',
1106 'C' => 'Other',
1107 'Cc' => 'Control',
1108 'Cf' => 'Format',
1109 'Cs' => 'Surrogate',
1110 'Co' => 'PrivateUse',
1111 'Cn' => 'Unassigned',
1112 );
1113
1114sub general_categories {
cb3150f5 1115 return _dclone \%GENERAL_CATEGORIES;
ea508aee
JH
1116}
1117
a452d459 1118=head2 B<general_categories()>
ea508aee
JH
1119
1120 use Unicode::UCD 'general_categories';
1121
1122 my $categories = general_categories();
1123
a452d459 1124This returns a reference to a hash which has short
ea508aee
JH
1125general category names (such as C<Lu>, C<Nd>, C<Zs>, C<S>) as keys and long
1126names (such as C<UppercaseLetter>, C<DecimalNumber>, C<SpaceSeparator>,
1127C<Symbol>) as values. The hash is reversible in case you need to go
1128from the long names to the short names. The general category is the
a452d459
KW
1129one returned from
1130L</charinfo()> under the C<category> key.
ea508aee 1131
9024667a
KW
1132The L</prop_values()> and L</prop_value_aliases()> functions can be used as an
1133alternative to this function; the first returning a simple list of the short
1134category names; and the second gets all the synonyms of a given category name.
7ef25837 1135
ea508aee
JH
1136=cut
1137
1138my %BIDI_TYPES =
1139 (
1140 'L' => 'Left-to-Right',
1141 'LRE' => 'Left-to-Right Embedding',
1142 'LRO' => 'Left-to-Right Override',
1143 'R' => 'Right-to-Left',
1144 'AL' => 'Right-to-Left Arabic',
1145 'RLE' => 'Right-to-Left Embedding',
1146 'RLO' => 'Right-to-Left Override',
1147 'PDF' => 'Pop Directional Format',
1148 'EN' => 'European Number',
1149 'ES' => 'European Number Separator',
1150 'ET' => 'European Number Terminator',
1151 'AN' => 'Arabic Number',
1152 'CS' => 'Common Number Separator',
1153 'NSM' => 'Non-Spacing Mark',
1154 'BN' => 'Boundary Neutral',
1155 'B' => 'Paragraph Separator',
1156 'S' => 'Segment Separator',
1157 'WS' => 'Whitespace',
1158 'ON' => 'Other Neutrals',
1159 );
1160
a452d459 1161=head2 B<bidi_types()>
ea508aee
JH
1162
1163 use Unicode::UCD 'bidi_types';
1164
1165 my $categories = bidi_types();
1166
a452d459 1167This returns a reference to a hash which has the short
ea508aee
JH
1168bidi (bidirectional) type names (such as C<L>, C<R>) as keys and long
1169names (such as C<Left-to-Right>, C<Right-to-Left>) as values. The
1170hash is reversible in case you need to go from the long names to the
a452d459
KW
1171short names. The bidi type is the one returned from
1172L</charinfo()>
ea508aee
JH
1173under the C<bidi> key. For the exact meaning of the various bidi classes
1174the Unicode TR9 is recommended reading:
a452d459 1175L<http://www.unicode.org/reports/tr9/>
ea508aee
JH
1176(as of Unicode 5.0.0)
1177
9024667a
KW
1178The L</prop_values()> and L</prop_value_aliases()> functions can be used as an
1179alternative to this function; the first returning a simple list of the short
1180bidi type names; and the second gets all the synonyms of a given bidi type
1181name.
7ef25837 1182
ea508aee
JH
1183=cut
1184
a452d459 1185sub bidi_types {
cb3150f5 1186 return _dclone \%BIDI_TYPES;
a452d459
KW
1187}
1188
1189=head2 B<compexcl()>
b08cd201 1190
55d7b906 1191 use Unicode::UCD 'compexcl';
b08cd201 1192
a452d459 1193 my $compexcl = compexcl(0x09dc);
b08cd201 1194
2afba6a4 1195This routine returns C<undef> if the Unicode version being used is so early
53cb2385
KW
1196that it doesn't have this property.
1197
1198C<compexcl()> is included for backwards
2afba6a4 1199compatibility, but as of Perl 5.12 and more modern Unicode versions, for
71a442a8
KW
1200most purposes it is probably more convenient to use one of the following
1201instead:
1202
1203 my $compexcl = chr(0x09dc) =~ /\p{Comp_Ex};
1204 my $compexcl = chr(0x09dc) =~ /\p{Full_Composition_Exclusion};
1205
1206or even
1207
1208 my $compexcl = chr(0x09dc) =~ /\p{CE};
1209 my $compexcl = chr(0x09dc) =~ /\p{Composition_Exclusion};
1210
1211The first two forms return B<true> if the L</code point argument> should not
76b05678
KW
1212be produced by composition normalization. For the final two forms to return
1213B<true>, it is additionally required that this fact not otherwise be
1214determinable from the Unicode data base.
71a442a8
KW
1215
1216This routine behaves identically to the final two forms. That is,
1217it does not return B<true> if the code point has a decomposition
a452d459
KW
1218consisting of another single code point, nor if its decomposition starts
1219with a code point whose combining class is non-zero. Code points that meet
1220either of these conditions should also not be produced by composition
71a442a8
KW
1221normalization, which is probably why you should use the
1222C<Full_Composition_Exclusion> property instead, as shown above.
b08cd201 1223
71a442a8 1224The routine returns B<false> otherwise.
b08cd201
JH
1225
1226=cut
1227
b08cd201
JH
1228sub compexcl {
1229 my $arg = shift;
1230 my $code = _getcode($arg);
74f8133e
JH
1231 croak __PACKAGE__, "::compexcl: unknown code '$arg'"
1232 unless defined $code;
b08cd201 1233
2afba6a4
KW
1234 UnicodeVersion() unless defined $v_unicode_version;
1235 return if $v_unicode_version lt v3.0.0;
1236
36c2430c 1237 no warnings "non_unicode"; # So works on non-Unicode code points
71a442a8 1238 return chr($code) =~ /\p{Composition_Exclusion}/;
b08cd201
JH
1239}
1240
a452d459 1241=head2 B<casefold()>
b08cd201 1242
55d7b906 1243 use Unicode::UCD 'casefold';
b08cd201 1244
a452d459
KW
1245 my $casefold = casefold(0xDF);
1246 if (defined $casefold) {
1247 my @full_fold_hex = split / /, $casefold->{'full'};
1248 my $full_fold_string =
1249 join "", map {chr(hex($_))} @full_fold_hex;
1250 my @turkic_fold_hex =
1251 split / /, ($casefold->{'turkic'} ne "")
1252 ? $casefold->{'turkic'}
1253 : $casefold->{'full'};
1254 my $turkic_fold_string =
1255 join "", map {chr(hex($_))} @turkic_fold_hex;
1256 }
1257 if (defined $casefold && $casefold->{'simple'} ne "") {
1258 my $simple_fold_hex = $casefold->{'simple'};
1259 my $simple_fold_string = chr(hex($simple_fold_hex));
1260 }
b08cd201 1261
a452d459 1262This returns the (almost) locale-independent case folding of the
6329003c
KW
1263character specified by the L</code point argument>. (Starting in Perl v5.16,
1264the core function C<fc()> returns the C<full> mapping (described below)
1265faster than this does, and for entire strings.)
b08cd201 1266
6329003c 1267If there is no case folding for the input code point, C<undef> is returned.
a452d459
KW
1268
1269If there is a case folding for that code point, a reference to a hash
b08cd201
JH
1270with the following fields is returned:
1271
a452d459
KW
1272=over
1273
1274=item B<code>
1275
a1ae4420
KW
1276the input native L</code point argument> expressed in hexadecimal, with
1277leading zeros
a452d459
KW
1278added if necessary to make it contain at least four hexdigits
1279
1280=item B<full>
1281
a18e976f 1282one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1283code points for the case folding for I<code>.
1284Each has at least four hexdigits.
1285
1286=item B<simple>
1287
1288is empty, or is exactly one code with at least four hexdigits which can be used
1289as an alternative case folding when the calling program cannot cope with the
1290fold being a sequence of multiple code points. If I<full> is just one code
1291point, then I<simple> equals I<full>. If there is no single code point folding
1292defined for I<code>, then I<simple> is the empty string. Otherwise, it is an
1293inferior, but still better-than-nothing alternative folding to I<full>.
1294
1295=item B<mapping>
1296
1297is the same as I<simple> if I<simple> is not empty, and it is the same as I<full>
1298otherwise. It can be considered to be the simplest possible folding for
1299I<code>. It is defined primarily for backwards compatibility.
1300
1301=item B<status>
b08cd201 1302
a452d459
KW
1303is C<C> (for C<common>) if the best possible fold is a single code point
1304(I<simple> equals I<full> equals I<mapping>). It is C<S> if there are distinct
1305folds, I<simple> and I<full> (I<mapping> equals I<simple>). And it is C<F> if
a18e976f
KW
1306there is only a I<full> fold (I<mapping> equals I<full>; I<simple> is empty).
1307Note that this
a452d459
KW
1308describes the contents of I<mapping>. It is defined primarily for backwards
1309compatibility.
b08cd201 1310
6329003c 1311For Unicode versions between 3.1 and 3.1.1 inclusive, I<status> can also be
a452d459
KW
1312C<I> which is the same as C<C> but is a special case for dotted uppercase I and
1313dotless lowercase i:
b08cd201 1314
a452d459 1315=over
b08cd201 1316
f703fc96 1317=item Z<>B<*> If you use this C<I> mapping
a452d459 1318
a18e976f 1319the result is case-insensitive,
a452d459
KW
1320but dotless and dotted I's are not distinguished
1321
f703fc96 1322=item Z<>B<*> If you exclude this C<I> mapping
a452d459 1323
a18e976f 1324the result is not fully case-insensitive, but
a452d459
KW
1325dotless and dotted I's are distinguished
1326
1327=back
1328
1329=item B<turkic>
1330
1331contains any special folding for Turkic languages. For versions of Unicode
1332starting with 3.2, this field is empty unless I<code> has a different folding
1333in Turkic languages, in which case it is one or more codes (separated by
a18e976f 1334spaces) that, taken in order, give the code points for the case folding for
a452d459
KW
1335I<code> in those languages.
1336Each code has at least four hexdigits.
1337Note that this folding does not maintain canonical equivalence without
1338additional processing.
1339
6329003c
KW
1340For Unicode versions between 3.1 and 3.1.1 inclusive, this field is empty unless
1341there is a
a452d459
KW
1342special folding for Turkic languages, in which case I<status> is C<I>, and
1343I<mapping>, I<full>, I<simple>, and I<turkic> are all equal.
1344
1345=back
1346
1347Programs that want complete generality and the best folding results should use
1348the folding contained in the I<full> field. But note that the fold for some
1349code points will be a sequence of multiple code points.
1350
1351Programs that can't cope with the fold mapping being multiple code points can
1352use the folding contained in the I<simple> field, with the loss of some
1353generality. In Unicode 5.1, about 7% of the defined foldings have no single
1354code point folding.
1355
1356The I<mapping> and I<status> fields are provided for backwards compatibility for
1357existing programs. They contain the same values as in previous versions of
1358this function.
1359
1360Locale is not completely independent. The I<turkic> field contains results to
1361use when the locale is a Turkic language.
b08cd201
JH
1362
1363For more information about case mappings see
a452d459 1364L<http://www.unicode.org/unicode/reports/tr21>
b08cd201
JH
1365
1366=cut
1367
1368my %CASEFOLD;
1369
1370sub _casefold {
727c62ff
KW
1371 unless (%CASEFOLD) { # Populate the hash
1372 my ($full_invlist_ref, $full_invmap_ref, undef, $default)
1373 = prop_invmap('Case_Folding');
1374
1375 # Use the recipe given in the prop_invmap() pod to convert the
1376 # inversion map into the hash.
1377 for my $i (0 .. @$full_invlist_ref - 1 - 1) {
1378 next if $full_invmap_ref->[$i] == $default;
1379 my $adjust = -1;
1380 for my $j ($full_invlist_ref->[$i] .. $full_invlist_ref->[$i+1] -1) {
1381 $adjust++;
1382 if (! ref $full_invmap_ref->[$i]) {
1383
1384 # This is a single character mapping
1385 $CASEFOLD{$j}{'status'} = 'C';
1386 $CASEFOLD{$j}{'simple'}
1387 = $CASEFOLD{$j}{'full'}
1388 = $CASEFOLD{$j}{'mapping'}
1389 = sprintf("%04X", $full_invmap_ref->[$i] + $adjust);
1390 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1391 $CASEFOLD{$j}{'turkic'} = "";
1392 }
1393 else { # prop_invmap ensures that $adjust is 0 for a ref
1394 $CASEFOLD{$j}{'status'} = 'F';
1395 $CASEFOLD{$j}{'full'}
1396 = $CASEFOLD{$j}{'mapping'}
1397 = join " ", map { sprintf "%04X", $_ }
1398 @{$full_invmap_ref->[$i]};
1399 $CASEFOLD{$j}{'simple'} = "";
1400 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1401 $CASEFOLD{$j}{'turkic'} = "";
1402 }
1403 }
1404 }
1405
1406 # We have filled in the full mappings above, assuming there were no
1407 # simple ones for the ones with multi-character maps. Now, we find
1408 # and fix the cases where that assumption was false.
1409 (my ($simple_invlist_ref, $simple_invmap_ref, undef), $default)
1410 = prop_invmap('Simple_Case_Folding');
1411 for my $i (0 .. @$simple_invlist_ref - 1 - 1) {
1412 next if $simple_invmap_ref->[$i] == $default;
1413 my $adjust = -1;
1414 for my $j ($simple_invlist_ref->[$i]
1415 .. $simple_invlist_ref->[$i+1] -1)
1416 {
1417 $adjust++;
1418 next if $CASEFOLD{$j}{'status'} eq 'C';
1419 $CASEFOLD{$j}{'status'} = 'S';
1420 $CASEFOLD{$j}{'simple'}
1421 = $CASEFOLD{$j}{'mapping'}
1422 = sprintf("%04X", $simple_invmap_ref->[$i] + $adjust);
1423 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1424 $CASEFOLD{$j}{'turkic'} = "";
1425 }
1426 }
1427
1428 # We hard-code in the turkish rules
1429 UnicodeVersion() unless defined $v_unicode_version;
1430 if ($v_unicode_version ge v3.2.0) {
1431
1432 # These two code points should already have regular entries, so
1433 # just fill in the turkish fields
1434 $CASEFOLD{ord('I')}{'turkic'} = '0131';
1435 $CASEFOLD{0x130}{'turkic'} = sprintf "%04X", ord('i');
1436 }
1437 elsif ($v_unicode_version ge v3.1.0) {
1438
1439 # These two code points don't have entries otherwise.
1440 $CASEFOLD{0x130}{'code'} = '0130';
1441 $CASEFOLD{0x131}{'code'} = '0131';
1442 $CASEFOLD{0x130}{'status'} = $CASEFOLD{0x131}{'status'} = 'I';
1443 $CASEFOLD{0x130}{'turkic'}
1444 = $CASEFOLD{0x130}{'mapping'}
1445 = $CASEFOLD{0x130}{'full'}
1446 = $CASEFOLD{0x130}{'simple'}
1447 = $CASEFOLD{0x131}{'turkic'}
1448 = $CASEFOLD{0x131}{'mapping'}
1449 = $CASEFOLD{0x131}{'full'}
1450 = $CASEFOLD{0x131}{'simple'}
1451 = sprintf "%04X", ord('i');
1452 }
b08cd201
JH
1453 }
1454}
1455
1456sub casefold {
1457 my $arg = shift;
1458 my $code = _getcode($arg);
74f8133e
JH
1459 croak __PACKAGE__, "::casefold: unknown code '$arg'"
1460 unless defined $code;
b08cd201
JH
1461
1462 _casefold() unless %CASEFOLD;
1463
1464 return $CASEFOLD{$code};
1465}
1466
66aa79e2
KW
1467=head2 B<all_casefolds()>
1468
1469
1470 use Unicode::UCD 'all_casefolds';
1471
1472 my $all_folds_ref = all_casefolds();
1473 foreach my $char_with_casefold (sort { $a <=> $b }
1474 keys %$all_folds_ref)
1475 {
1476 printf "%04X:", $char_with_casefold;
1477 my $casefold = $all_folds_ref->{$char_with_casefold};
1478
1479 # Get folds for $char_with_casefold
1480
1481 my @full_fold_hex = split / /, $casefold->{'full'};
1482 my $full_fold_string =
1483 join "", map {chr(hex($_))} @full_fold_hex;
1484 print " full=", join " ", @full_fold_hex;
1485 my @turkic_fold_hex =
1486 split / /, ($casefold->{'turkic'} ne "")
1487 ? $casefold->{'turkic'}
1488 : $casefold->{'full'};
1489 my $turkic_fold_string =
1490 join "", map {chr(hex($_))} @turkic_fold_hex;
1491 print "; turkic=", join " ", @turkic_fold_hex;
1492 if (defined $casefold && $casefold->{'simple'} ne "") {
1493 my $simple_fold_hex = $casefold->{'simple'};
1494 my $simple_fold_string = chr(hex($simple_fold_hex));
1495 print "; simple=$simple_fold_hex";
1496 }
1497 print "\n";
1498 }
1499
1500This returns all the case foldings in the current version of Unicode in the
1501form of a reference to a hash. Each key to the hash is the decimal
1502representation of a Unicode character that has a casefold to other than
1503itself. The casefold of a semi-colon is itself, so it isn't in the hash;
1504likewise for a lowercase "a", but there is an entry for a capital "A". The
1505hash value for each key is another hash, identical to what is returned by
1506L</casefold()> if called with that code point as its argument. So the value
1507C<< all_casefolds()->{ord("A")}' >> is equivalent to C<casefold(ord("A"))>;
1508
1509=cut
1510
1511sub all_casefolds () {
1512 _casefold() unless %CASEFOLD;
1513 return _dclone \%CASEFOLD;
1514}
1515
a452d459 1516=head2 B<casespec()>
b08cd201 1517
55d7b906 1518 use Unicode::UCD 'casespec';
b08cd201 1519
a452d459 1520 my $casespec = casespec(0xFB00);
b08cd201 1521
a452d459
KW
1522This returns the potentially locale-dependent case mappings of the L</code point
1523argument>. The mappings may be longer than a single code point (which the basic
1524Unicode case mappings as returned by L</charinfo()> never are).
b08cd201 1525
a452d459
KW
1526If there are no case mappings for the L</code point argument>, or if all three
1527possible mappings (I<lower>, I<title> and I<upper>) result in single code
a18e976f 1528points and are locale independent and unconditional, C<undef> is returned
5d8e6e41
KW
1529(which means that the case mappings, if any, for the code point are those
1530returned by L</charinfo()>).
a452d459
KW
1531
1532Otherwise, a reference to a hash giving the mappings (or a reference to a hash
5d8e6e41
KW
1533of such hashes, explained below) is returned with the following keys and their
1534meanings:
a452d459
KW
1535
1536The keys in the bottom layer hash with the meanings of their values are:
1537
1538=over
1539
1540=item B<code>
1541
a1ae4420
KW
1542the input native L</code point argument> expressed in hexadecimal, with
1543leading zeros
a452d459
KW
1544added if necessary to make it contain at least four hexdigits
1545
1546=item B<lower>
1547
a18e976f 1548one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1549code points for the lower case of I<code>.
1550Each has at least four hexdigits.
1551
1552=item B<title>
b08cd201 1553
a18e976f 1554one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1555code points for the title case of I<code>.
1556Each has at least four hexdigits.
b08cd201 1557
d2da20e3 1558=item B<upper>
b08cd201 1559
a18e976f 1560one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1561code points for the upper case of I<code>.
1562Each has at least four hexdigits.
1563
1564=item B<condition>
1565
1566the conditions for the mappings to be valid.
a18e976f 1567If C<undef>, the mappings are always valid.
a452d459
KW
1568When defined, this field is a list of conditions,
1569all of which must be true for the mappings to be valid.
1570The list consists of one or more
1571I<locales> (see below)
1572and/or I<contexts> (explained in the next paragraph),
1573separated by spaces.
1574(Other than as used to separate elements, spaces are to be ignored.)
1575Case distinctions in the condition list are not significant.
82c0b05b 1576Conditions preceded by "NON_" represent the negation of the condition.
b08cd201 1577
a452d459
KW
1578A I<context> is one of those defined in the Unicode standard.
1579For Unicode 5.1, they are defined in Section 3.13 C<Default Case Operations>
1580available at
5d8e6e41
KW
1581L<http://www.unicode.org/versions/Unicode5.1.0/>.
1582These are for context-sensitive casing.
f499c386 1583
a452d459
KW
1584=back
1585
5d8e6e41 1586The hash described above is returned for locale-independent casing, where
a18e976f 1587at least one of the mappings has length longer than one. If C<undef> is
5d8e6e41
KW
1588returned, the code point may have mappings, but if so, all are length one,
1589and are returned by L</charinfo()>.
1590Note that when this function does return a value, it will be for the complete
1591set of mappings for a code point, even those whose length is one.
1592
1593If there are additional casing rules that apply only in certain locales,
1594an additional key for each will be defined in the returned hash. Each such key
1595will be its locale name, defined as a 2-letter ISO 3166 country code, possibly
1596followed by a "_" and a 2-letter ISO language code (possibly followed by a "_"
1597and a variant code). You can find the lists of all possible locales, see
1598L<Locale::Country> and L<Locale::Language>.
89e4a205 1599(In Unicode 6.0, the only locales returned by this function
a452d459 1600are C<lt>, C<tr>, and C<az>.)
b08cd201 1601
5d8e6e41
KW
1602Each locale key is a reference to a hash that has the form above, and gives
1603the casing rules for that particular locale, which take precedence over the
1604locale-independent ones when in that locale.
1605
1606If the only casing for a code point is locale-dependent, then the returned
1607hash will not have any of the base keys, like C<code>, C<upper>, etc., but
1608will contain only locale keys.
1609
b08cd201 1610For more information about case mappings see
a452d459 1611L<http://www.unicode.org/unicode/reports/tr21/>
b08cd201
JH
1612
1613=cut
1614
1615my %CASESPEC;
1616
1617sub _casespec {
1618 unless (%CASESPEC) {
f033d3cd
KW
1619 UnicodeVersion() unless defined $v_unicode_version;
1620 if ($v_unicode_version lt v2.1.8) {
1621 %CASESPEC = {};
1622 }
1623 elsif (openunicode(\$CASESPECFH, "SpecialCasing.txt")) {
6c8d78fb 1624 local $_;
ce066323 1625 local $/ = "\n";
b08cd201
JH
1626 while (<$CASESPECFH>) {
1627 if (/^([0-9A-F]+); ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; (\w+(?: \w+)*)?/) {
f033d3cd 1628
f499c386
JH
1629 my ($hexcode, $lower, $title, $upper, $condition) =
1630 ($1, $2, $3, $4, $5);
a1ae4420
KW
1631 if (! IS_ASCII_PLATFORM) { # Remap entry to native
1632 foreach my $var_ref (\$hexcode,
1633 \$lower,
1634 \$title,
1635 \$upper)
1636 {
1637 next unless defined $$var_ref;
1638 $$var_ref = join " ",
1639 map { sprintf("%04X",
1640 utf8::unicode_to_native(hex $_)) }
1641 split " ", $$var_ref;
1642 }
1643 }
1644
f499c386 1645 my $code = hex($hexcode);
f033d3cd
KW
1646
1647 # In 2.1.8, there were duplicate entries; ignore all but
1648 # the first one -- there were no conditions in the file
1649 # anyway.
1650 if (exists $CASESPEC{$code} && $v_unicode_version ne v2.1.8)
1651 {
f499c386
JH
1652 if (exists $CASESPEC{$code}->{code}) {
1653 my ($oldlower,
1654 $oldtitle,
1655 $oldupper,
1656 $oldcondition) =
1657 @{$CASESPEC{$code}}{qw(lower
1658 title
1659 upper
1660 condition)};
822ebcc8
JH
1661 if (defined $oldcondition) {
1662 my ($oldlocale) =
f499c386 1663 ($oldcondition =~ /^([a-z][a-z](?:_\S+)?)/);
f499c386
JH
1664 delete $CASESPEC{$code};
1665 $CASESPEC{$code}->{$oldlocale} =
1666 { code => $hexcode,
1667 lower => $oldlower,
1668 title => $oldtitle,
1669 upper => $oldupper,
1670 condition => $oldcondition };
f499c386
JH
1671 }
1672 }
1673 my ($locale) =
1674 ($condition =~ /^([a-z][a-z](?:_\S+)?)/);
1675 $CASESPEC{$code}->{$locale} =
1676 { code => $hexcode,
1677 lower => $lower,
1678 title => $title,
1679 upper => $upper,
1680 condition => $condition };
1681 } else {
1682 $CASESPEC{$code} =
1683 { code => $hexcode,
1684 lower => $lower,
1685 title => $title,
1686 upper => $upper,
1687 condition => $condition };
1688 }
b08cd201
JH
1689 }
1690 }
1691 close($CASESPECFH);
1692 }
1693 }
1694}
1695
1696sub casespec {
1697 my $arg = shift;
1698 my $code = _getcode($arg);
74f8133e
JH
1699 croak __PACKAGE__, "::casespec: unknown code '$arg'"
1700 unless defined $code;
b08cd201
JH
1701
1702 _casespec() unless %CASESPEC;
1703
cb3150f5 1704 return ref $CASESPEC{$code} ? _dclone $CASESPEC{$code} : $CASESPEC{$code};
b08cd201
JH
1705}
1706
a452d459 1707=head2 B<namedseq()>
a2bd7410
JH
1708
1709 use Unicode::UCD 'namedseq';
1710
1711 my $namedseq = namedseq("KATAKANA LETTER AINU P");
1712 my @namedseq = namedseq("KATAKANA LETTER AINU P");
1713 my %namedseq = namedseq();
1714
1715If used with a single argument in a scalar context, returns the string
a18e976f 1716consisting of the code points of the named sequence, or C<undef> if no
a2bd7410 1717named sequence by that name exists. If used with a single argument in
53cb2385
KW
1718a list context, it returns the list of the ordinals of the code points.
1719
1720If used with no
1721arguments in a list context, it returns a hash with the names of all the
1722named sequences as the keys and their sequences as strings as
a18e976f 1723the values. Otherwise, it returns C<undef> or an empty list depending
a2bd7410
JH
1724on the context.
1725
a452d459
KW
1726This function only operates on officially approved (not provisional) named
1727sequences.
a2bd7410 1728
27f853a0
KW
1729Note that as of Perl 5.14, C<\N{KATAKANA LETTER AINU P}> will insert the named
1730sequence into double-quoted strings, and C<charnames::string_vianame("KATAKANA
1731LETTER AINU P")> will return the same string this function does, but will also
1732operate on character names that aren't named sequences, without you having to
1733know which are which. See L<charnames>.
1734
a2bd7410
JH
1735=cut
1736
1737my %NAMEDSEQ;
1738
1739sub _namedseq {
1740 unless (%NAMEDSEQ) {
98ef7649 1741 if (openunicode(\$NAMEDSEQFH, "Name.pl")) {
a2bd7410 1742 local $_;
ce066323 1743 local $/ = "\n";
a2bd7410 1744 while (<$NAMEDSEQFH>) {
98ef7649
KW
1745 if (/^ [0-9A-F]+ \ /x) {
1746 chomp;
1747 my ($sequence, $name) = split /\t/;
1748 my @s = map { chr(hex($_)) } split(' ', $sequence);
1749 $NAMEDSEQ{$name} = join("", @s);
a2bd7410
JH
1750 }
1751 }
1752 close($NAMEDSEQFH);
1753 }
1754 }
1755}
1756
1757sub namedseq {
98ef7649
KW
1758
1759 # Use charnames::string_vianame() which now returns this information,
1760 # unless the caller wants the hash returned, in which case we read it in,
1761 # and thereafter use it instead of calling charnames, as it is faster.
1762
a2bd7410
JH
1763 my $wantarray = wantarray();
1764 if (defined $wantarray) {
1765 if ($wantarray) {
1766 if (@_ == 0) {
98ef7649 1767 _namedseq() unless %NAMEDSEQ;
a2bd7410
JH
1768 return %NAMEDSEQ;
1769 } elsif (@_ == 1) {
98ef7649
KW
1770 my $s;
1771 if (%NAMEDSEQ) {
1772 $s = $NAMEDSEQ{ $_[0] };
1773 }
1774 else {
1775 $s = charnames::string_vianame($_[0]);
1776 }
a2bd7410
JH
1777 return defined $s ? map { ord($_) } split('', $s) : ();
1778 }
1779 } elsif (@_ == 1) {
98ef7649
KW
1780 return $NAMEDSEQ{ $_[0] } if %NAMEDSEQ;
1781 return charnames::string_vianame($_[0]);
a2bd7410
JH
1782 }
1783 }
1784 return;
1785}
1786
7319f91d
KW
1787my %NUMERIC;
1788
1789sub _numeric {
35a865d4 1790 my @numbers = _read_table("To/Nv.pl");
98025745
KW
1791 foreach my $entry (@numbers) {
1792 my ($start, $end, $value) = @$entry;
1793
05dbc6f8
KW
1794 # If value contains a slash, convert to decimal, add a reverse hash
1795 # used by charinfo.
98025745
KW
1796 if ((my @rational = split /\//, $value) == 2) {
1797 my $real = $rational[0] / $rational[1];
05dbc6f8 1798 $real_to_rational{$real} = $value;
98025745 1799 $value = $real;
98025745 1800
4f143a72
KW
1801 # Should only be single element, but just in case...
1802 for my $i ($start .. $end) {
1803 $NUMERIC{$i} = $value;
1804 }
1805 }
1806 else {
1807 # The values require adjusting, as is in 'a' format
1808 for my $i ($start .. $end) {
1809 $NUMERIC{$i} = $value + $i - $start;
1810 }
7319f91d 1811 }
7319f91d 1812 }
2dc5eb26
KW
1813
1814 # Decided unsafe to use these that aren't officially part of the Unicode
1815 # standard.
1816 #use Math::Trig;
1817 #my $pi = acos(-1.0);
98025745 1818 #$NUMERIC{0x03C0} = $pi;
7319f91d
KW
1819
1820 # Euler's constant, not to be confused with Euler's number
98025745 1821 #$NUMERIC{0x2107} = 0.57721566490153286060651209008240243104215933593992;
7319f91d
KW
1822
1823 # Euler's number
98025745 1824 #$NUMERIC{0x212F} = 2.7182818284590452353602874713526624977572;
2dc5eb26 1825
7319f91d
KW
1826 return;
1827}
1828
1829=pod
1830
67592e11 1831=head2 B<num()>
7319f91d 1832
eefd7bc2
KW
1833 use Unicode::UCD 'num';
1834
1835 my $val = num("123");
1836 my $one_quarter = num("\N{VULGAR FRACTION 1/4}");
1837
53cb2385 1838C<num()> returns the numeric value of the input Unicode string; or C<undef> if it
7319f91d
KW
1839doesn't think the entire string has a completely valid, safe numeric value.
1840
1841If the string is just one character in length, the Unicode numeric value
1842is returned if it has one, or C<undef> otherwise. Note that this need
1843not be a whole number. C<num("\N{TIBETAN DIGIT HALF ZERO}")>, for
2dc5eb26
KW
1844example returns -0.5.
1845
1846=cut
7319f91d 1847
2dc5eb26
KW
1848#A few characters to which Unicode doesn't officially
1849#assign a numeric value are considered numeric by C<num>.
1850#These are:
1851
1852# EULER CONSTANT 0.5772... (this is NOT Euler's number)
1853# SCRIPT SMALL E 2.71828... (this IS Euler's number)
1854# GREEK SMALL LETTER PI 3.14159...
1855
1856=pod
7319f91d
KW
1857
1858If the string is more than one character, C<undef> is returned unless
8bb4c8e2 1859all its characters are decimal digits (that is, they would match C<\d+>),
7319f91d
KW
1860from the same script. For example if you have an ASCII '0' and a Bengali
1861'3', mixed together, they aren't considered a valid number, and C<undef>
1862is returned. A further restriction is that the digits all have to be of
1863the same form. A half-width digit mixed with a full-width one will
1864return C<undef>. The Arabic script has two sets of digits; C<num> will
1865return C<undef> unless all the digits in the string come from the same
1866set.
1867
1868C<num> errs on the side of safety, and there may be valid strings of
1869decimal digits that it doesn't recognize. Note that Unicode defines
1870a number of "digit" characters that aren't "decimal digit" characters.
a278d14b 1871"Decimal digits" have the property that they have a positional value, i.e.,
7319f91d
KW
1872there is a units position, a 10's position, a 100's, etc, AND they are
1873arranged in Unicode in blocks of 10 contiguous code points. The Chinese
1874digits, for example, are not in such a contiguous block, and so Unicode
1875doesn't view them as decimal digits, but merely digits, and so C<\d> will not
1876match them. A single-character string containing one of these digits will
1877have its decimal value returned by C<num>, but any longer string containing
1878only these digits will return C<undef>.
1879
a278d14b
KW
1880Strings of multiple sub- and superscripts are not recognized as numbers. You
1881can use either of the compatibility decompositions in Unicode::Normalize to
7319f91d
KW
1882change these into digits, and then call C<num> on the result.
1883
1884=cut
1885
1886# To handle sub, superscripts, this could if called in list context,
1887# consider those, and return the <decomposition> type in the second
1888# array element.
1889
1890sub num {
1891 my $string = $_[0];
1892
1893 _numeric unless %NUMERIC;
1894
1895 my $length = length($string);
98025745 1896 return $NUMERIC{ord($string)} if $length == 1;
7319f91d
KW
1897 return if $string =~ /\D/;
1898 my $first_ord = ord(substr($string, 0, 1));
98025745 1899 my $value = $NUMERIC{$first_ord};
5522af1c
KW
1900
1901 # To be a valid decimal number, it should be in a block of 10 consecutive
1902 # characters, whose values are 0, 1, 2, ... 9. Therefore this digit's
1903 # value is its offset in that block from the character that means zero.
7319f91d
KW
1904 my $zero_ord = $first_ord - $value;
1905
5522af1c
KW
1906 # Unicode 6.0 instituted the rule that only digits in a consecutive
1907 # block of 10 would be considered decimal digits. If this is an earlier
1908 # release, we verify that this first character is a member of such a
1909 # block. That is, that the block of characters surrounding this one
1910 # consists of all \d characters whose numeric values are the expected
1911 # ones.
1912 UnicodeVersion() unless defined $v_unicode_version;
1913 if ($v_unicode_version lt v6.0.0) {
1914 for my $i (0 .. 9) {
1915 my $ord = $zero_ord + $i;
1916 return unless chr($ord) =~ /\d/;
1917 my $numeric = $NUMERIC{$ord};
1918 return unless defined $numeric;
1919 return unless $numeric == $i;
1920 }
1921 }
1922
7319f91d 1923 for my $i (1 .. $length -1) {
5522af1c
KW
1924
1925 # Here we know either by verifying, or by fact of the first character
1926 # being a \d in Unicode 6.0 or later, that any character between the
1927 # character that means 0, and 9 positions above it must be \d, and
1928 # must have its value correspond to its offset from the zero. Any
1929 # characters outside these 10 do not form a legal number for this
1930 # function.
7319f91d
KW
1931 my $ord = ord(substr($string, $i, 1));
1932 my $digit = $ord - $zero_ord;
1933 return unless $digit >= 0 && $digit <= 9;
1934 $value = $value * 10 + $digit;
1935 }
5522af1c 1936
7319f91d
KW
1937 return $value;
1938}
1939
7ef25837
KW
1940=pod
1941
1942=head2 B<prop_aliases()>
1943
1944 use Unicode::UCD 'prop_aliases';
1945
1946 my ($short_name, $full_name, @other_names) = prop_aliases("space");
1947 my $same_full_name = prop_aliases("Space"); # Scalar context
1948 my ($same_short_name) = prop_aliases("Space"); # gets 0th element
1949 print "The full name is $full_name\n";
1950 print "The short name is $short_name\n";
1951 print "The other aliases are: ", join(", ", @other_names), "\n";
1952
1953 prints:
1954 The full name is White_Space
1955 The short name is WSpace
1956 The other aliases are: Space
1957
1958Most Unicode properties have several synonymous names. Typically, there is at
1959least a short name, convenient to type, and a long name that more fully
1960describes the property, and hence is more easily understood.
1961
1962If you know one name for a Unicode property, you can use C<prop_aliases> to find
1963either the long name (when called in scalar context), or a list of all of the
1964names, somewhat ordered so that the short name is in the 0th element, the long
1965name in the next element, and any other synonyms are in the remaining
1966elements, in no particular order.
1967
1968The long name is returned in a form nicely capitalized, suitable for printing.
1969
1970The input parameter name is loosely matched, which means that white space,
1971hyphens, and underscores are ignored (except for the trailing underscore in
1972the old_form grandfathered-in C<"L_">, which is better written as C<"LC">, and
1973both of which mean C<General_Category=Cased Letter>).
1974
1975If the name is unknown, C<undef> is returned (or an empty list in list
1976context). Note that Perl typically recognizes property names in regular
1977expressions with an optional C<"Is_>" (with or without the underscore)
1978prefixed to them, such as C<\p{isgc=punct}>. This function does not recognize
1979those in the input, returning C<undef>. Nor are they included in the output
1980as possible synonyms.
1981
1982C<prop_aliases> does know about the Perl extensions to Unicode properties,
1983such as C<Any> and C<XPosixAlpha>, and the single form equivalents to Unicode
1984properties such as C<XDigit>, C<Greek>, C<In_Greek>, and C<Is_Greek>. The
1985final example demonstrates that the C<"Is_"> prefix is recognized for these
1986extensions; it is needed to resolve ambiguities. For example,
1987C<prop_aliases('lc')> returns the list C<(lc, Lowercase_Mapping)>, but
1988C<prop_aliases('islc')> returns C<(Is_LC, Cased_Letter)>. This is
1989because C<islc> is a Perl extension which is short for
1990C<General_Category=Cased Letter>. The lists returned for the Perl extensions
1991will not include the C<"Is_"> prefix (whether or not the input had it) unless
1992needed to resolve ambiguities, as shown in the C<"islc"> example, where the
1993returned list had one element containing C<"Is_">, and the other without.
1994
1995It is also possible for the reverse to happen: C<prop_aliases('isc')> returns
1996the list C<(isc, ISO_Comment)>; whereas C<prop_aliases('c')> returns
1997C<(C, Other)> (the latter being a Perl extension meaning
ee94c7d1
KW
1998C<General_Category=Other>.
1999L<perluniprops/Properties accessible through Unicode::UCD> lists the available
2000forms, including which ones are discouraged from use.
7ef25837
KW
2001
2002Those discouraged forms are accepted as input to C<prop_aliases>, but are not
2003returned in the lists. C<prop_aliases('isL&')> and C<prop_aliases('isL_')>,
2004which are old synonyms for C<"Is_LC"> and should not be used in new code, are
2005examples of this. These both return C<(Is_LC, Cased_Letter)>. Thus this
13d1b68a 2006function allows you to take a discouraged form, and find its acceptable
7ef25837
KW
2007alternatives. The same goes with single-form Block property equivalences.
2008Only the forms that begin with C<"In_"> are not discouraged; if you pass
2009C<prop_aliases> a discouraged form, you will get back the equivalent ones that
2010begin with C<"In_">. It will otherwise look like a new-style block name (see.
2011L</Old-style versus new-style block names>).
2012
2013C<prop_aliases> does not know about any user-defined properties, and will
2014return C<undef> if called with one of those. Likewise for Perl internal
2015properties, with the exception of "Perl_Decimal_Digit" which it does know
2016about (and which is documented below in L</prop_invmap()>).
2017
2018=cut
2019
2020# It may be that there are use cases where the discouraged forms should be
2021# returned. If that comes up, an optional boolean second parameter to the
2022# function could be created, for example.
2023
2024# These are created by mktables for this routine and stored in unicore/UCD.pl
2025# where their structures are described.
2026our %string_property_loose_to_name;
2027our %ambiguous_names;
2028our %loose_perlprop_to_name;
2029our %prop_aliases;
2030
2031sub prop_aliases ($) {
2032 my $prop = $_[0];
2033 return unless defined $prop;
2034
2035 require "unicore/UCD.pl";
2036 require "unicore/Heavy.pl";
2037 require "utf8_heavy.pl";
2038
2039 # The property name may be loosely or strictly matched; we don't know yet.
2040 # But both types use lower-case.
2041 $prop = lc $prop;
2042
2043 # It is loosely matched if its lower case isn't known to be strict.
2044 my $list_ref;
2045 if (! exists $utf8::stricter_to_file_of{$prop}) {
2046 my $loose = utf8::_loose_name($prop);
2047
2048 # There is a hash that converts from any loose name to its standard
2049 # form, mapping all synonyms for a name to one name that can be used
2050 # as a key into another hash. The whole concept is for memory
2051 # savings, as the second hash doesn't have to have all the
2052 # combinations. Actually, there are two hashes that do the
2053 # converstion. One is used in utf8_heavy.pl (stored in Heavy.pl) for
2054 # looking up properties matchable in regexes. This function needs to
2055 # access string properties, which aren't available in regexes, so a
2056 # second conversion hash is made for them (stored in UCD.pl). Look in
2057 # the string one now, as the rest can have an optional 'is' prefix,
2058 # which these don't.
2059 if (exists $string_property_loose_to_name{$loose}) {
2060
2061 # Convert to its standard loose name.
2062 $prop = $string_property_loose_to_name{$loose};
2063 }
2064 else {
2065 my $retrying = 0; # bool. ? Has an initial 'is' been stripped
2066 RETRY:
2067 if (exists $utf8::loose_property_name_of{$loose}
2068 && (! $retrying
2069 || ! exists $ambiguous_names{$loose}))
2070 {
2071 # Found an entry giving the standard form. We don't get here
2072 # (in the test above) when we've stripped off an
2073 # 'is' and the result is an ambiguous name. That is because
2074 # these are official Unicode properties (though Perl can have
2075 # an optional 'is' prefix meaning the official property), and
2076 # all ambiguous cases involve a Perl single-form extension
2077 # for the gc, script, or block properties, and the stripped
2078 # 'is' means that they mean one of those, and not one of
2079 # these
2080 $prop = $utf8::loose_property_name_of{$loose};
2081 }
2082 elsif (exists $loose_perlprop_to_name{$loose}) {
2083
2084 # This hash is specifically for this function to list Perl
2085 # extensions that aren't in the earlier hashes. If there is
2086 # only one element, the short and long names are identical.
2087 # Otherwise the form is already in the same form as
2088 # %prop_aliases, which is handled at the end of the function.
2089 $list_ref = $loose_perlprop_to_name{$loose};
2090 if (@$list_ref == 1) {
2091 my @list = ($list_ref->[0], $list_ref->[0]);
2092 $list_ref = \@list;
2093 }
2094 }
2095 elsif (! exists $utf8::loose_to_file_of{$loose}) {
2096
2097 # loose_to_file_of is a complete list of loose names. If not
2098 # there, the input is unknown.
2099 return;
2100 }
27f39eb8 2101 elsif ($loose =~ / [:=] /x) {
7ef25837
KW
2102
2103 # Here we found the name but not its aliases, so it has to
27f39eb8
KW
2104 # exist. Exclude property-value combinations. (This shows up
2105 # for something like ccc=vr which matches loosely, but is a
2106 # synonym for ccc=9 which matches only strictly.
2107 return;
2108 }
2109 else {
2110
2111 # Here it has to exist, and isn't a property-value
2112 # combination. This means it must be one of the Perl
2113 # single-form extensions. First see if it is for a
2114 # property-value combination in one of the following
2115 # properties.
7ef25837
KW
2116 my @list;
2117 foreach my $property ("gc", "script") {
2118 @list = prop_value_aliases($property, $loose);
2119 last if @list;
2120 }
2121 if (@list) {
2122
2123 # Here, it is one of those property-value combination
2124 # single-form synonyms. There are ambiguities with some
2125 # of these. Check against the list for these, and adjust
2126 # if necessary.
2127 for my $i (0 .. @list -1) {
2128 if (exists $ambiguous_names
2129 {utf8::_loose_name(lc $list[$i])})
2130 {
2131 # The ambiguity is resolved by toggling whether or
2132 # not it has an 'is' prefix
2133 $list[$i] =~ s/^Is_// or $list[$i] =~ s/^/Is_/;
2134 }
2135 }
2136 return @list;
2137 }
2138
2139 # Here, it wasn't one of the gc or script single-form
2140 # extensions. It could be a block property single-form
2141 # extension. An 'in' prefix definitely means that, and should
2a4f2769
KW
2142 # be looked up without the prefix. However, starting in
2143 # Unicode 6.1, we have to special case 'indic...', as there
2144 # is a property that begins with that name. We shouldn't
2145 # strip the 'in' from that. I'm (khw) generalizing this to
2146 # 'indic' instead of the single property, because I suspect
2147 # that others of this class may come along in the future.
2148 # However, this could backfire and a block created whose name
2149 # begins with 'dic...', and we would want to strip the 'in'.
2150 # At which point this would have to be tweaked.
2151 my $began_with_in = $loose =~ s/^in(?!dic)//;
7ef25837
KW
2152 @list = prop_value_aliases("block", $loose);
2153 if (@list) {
2154 map { $_ =~ s/^/In_/ } @list;
2155 return @list;
2156 }
2157
2158 # Here still haven't found it. The last opportunity for it
2159 # being valid is only if it began with 'is'. We retry without
2160 # the 'is', setting a flag to that effect so that we don't
2161 # accept things that begin with 'isis...'
2162 if (! $retrying && ! $began_with_in && $loose =~ s/^is//) {
2163 $retrying = 1;
2164 goto RETRY;
2165 }
2166
2167 # Here, didn't find it. Since it was in %loose_to_file_of, we
2168 # should have been able to find it.
2169 carp __PACKAGE__, "::prop_aliases: Unexpectedly could not find '$prop'. Send bug report to perlbug\@perl.org";
2170 return;
2171 }
2172 }
2173 }
2174
2175 if (! $list_ref) {
2176 # Here, we have set $prop to a standard form name of the input. Look
2177 # it up in the structure created by mktables for this purpose, which
2178 # contains both strict and loosely matched properties. Avoid
2179 # autovivifying.
2180 $list_ref = $prop_aliases{$prop} if exists $prop_aliases{$prop};
2181 return unless $list_ref;
2182 }
2183
2184 # The full name is in element 1.
2185 return $list_ref->[1] unless wantarray;
2186
cb3150f5 2187 return @{_dclone $list_ref};
7ef25837
KW
2188}
2189
2190=pod
2191
9024667a
KW
2192=head2 B<prop_values()>
2193
2194 use Unicode::UCD 'prop_values';
2195
2196 print "AHex values are: ", join(", ", prop_values("AHex")),
2197 "\n";
2198 prints:
2199 AHex values are: N, Y
2200
2201Some Unicode properties have a restricted set of legal values. For example,
2202all binary properties are restricted to just C<true> or C<false>; and there
2203are only a few dozen possible General Categories. Use C<prop_values>
2204to find out if a given property is one such, and if so, to get a list of the
2205values:
2206
2207 print join ", ", prop_values("NFC_Quick_Check");
2208 prints:
2209 M, N, Y
2210
2211If the property doesn't have such a restricted set, C<undef> is returned.
2212
2213There are usually several synonyms for each possible value. Use
2214L</prop_value_aliases()> to access those.
2215
2216Case, white space, hyphens, and underscores are ignored in the input property
2217name (except for the trailing underscore in the old-form grandfathered-in
2218general category property value C<"L_">, which is better written as C<"LC">).
2219
2220If the property name is unknown, C<undef> is returned. Note that Perl typically
2221recognizes property names in regular expressions with an optional C<"Is_>"
2222(with or without the underscore) prefixed to them, such as C<\p{isgc=punct}>.
2223This function does not recognize those in the property parameter, returning
2224C<undef>.
2225
2226For the block property, new-style block names are returned (see
2227L</Old-style versus new-style block names>).
2228
2229C<prop_values> does not know about any user-defined properties, and
2230will return C<undef> if called with one of those.
2231
2232=cut
2233
2234# These are created by mktables for this module and stored in unicore/UCD.pl
2235# where their structures are described.
2236our %loose_to_standard_value;
2237our %prop_value_aliases;
2238
2239sub prop_values ($) {
2240 my $prop = shift;
2241 return undef unless defined $prop;
2242
2243 require "unicore/UCD.pl";
2244 require "utf8_heavy.pl";
2245
2246 # Find the property name synonym that's used as the key in other hashes,
2247 # which is element 0 in the returned list.
2248 ($prop) = prop_aliases($prop);
2249 return undef if ! $prop;
2250 $prop = utf8::_loose_name(lc $prop);
2251
2252 # Here is a legal property.
2253 return undef unless exists $prop_value_aliases{$prop};
2254 my @return;
2255 foreach my $value_key (sort { lc $a cmp lc $b }
2256 keys %{$prop_value_aliases{$prop}})
2257 {
2258 push @return, $prop_value_aliases{$prop}{$value_key}[0];
2259 }
2260 return @return;
2261}
2262
2263=pod
2264
7ef25837
KW
2265=head2 B<prop_value_aliases()>
2266
2267 use Unicode::UCD 'prop_value_aliases';
2268
2269 my ($short_name, $full_name, @other_names)
2270 = prop_value_aliases("Gc", "Punct");
2271 my $same_full_name = prop_value_aliases("Gc", "P"); # Scalar cntxt
2272 my ($same_short_name) = prop_value_aliases("Gc", "P"); # gets 0th
2273 # element
2274 print "The full name is $full_name\n";
2275 print "The short name is $short_name\n";
2276 print "The other aliases are: ", join(", ", @other_names), "\n";
2277
9024667a 2278 prints:
7ef25837
KW
2279 The full name is Punctuation
2280 The short name is P
2281 The other aliases are: Punct
2282
2283Some Unicode properties have a restricted set of legal values. For example,
2284all binary properties are restricted to just C<true> or C<false>; and there
2285are only a few dozen possible General Categories.
2286
9024667a
KW
2287You can use L</prop_values()> to find out if a given property is one which has
2288a restricted set of values, and if so, what those values are. But usually
91e78470
KW
2289each value actually has several synonyms. For example, in Unicode binary
2290properties, I<truth> can be represented by any of the strings "Y", "Yes", "T",
2291or "True"; and the General Category "Punctuation" by that string, or "Punct",
2292or simply "P".
7ef25837
KW
2293
2294Like property names, there is typically at least a short name for each such
9024667a
KW
2295property-value, and a long name. If you know any name of the property-value
2296(which you can get by L</prop_values()>, you can use C<prop_value_aliases>()
2297to get the long name (when called in scalar context), or a list of all the
2298names, with the short name in the 0th element, the long name in the next
2299element, and any other synonyms in the remaining elements, in no particular
2300order, except that any all-numeric synonyms will be last.
7ef25837
KW
2301
2302The long name is returned in a form nicely capitalized, suitable for printing.
2303
2304Case, white space, hyphens, and underscores are ignored in the input parameters
2305(except for the trailing underscore in the old-form grandfathered-in general
2306category property value C<"L_">, which is better written as C<"LC">).
2307
2308If either name is unknown, C<undef> is returned. Note that Perl typically
2309recognizes property names in regular expressions with an optional C<"Is_>"
2310(with or without the underscore) prefixed to them, such as C<\p{isgc=punct}>.
2311This function does not recognize those in the property parameter, returning
2312C<undef>.
2313
2314If called with a property that doesn't have synonyms for its values, it
2315returns the input value, possibly normalized with capitalization and
91e78470 2316underscores, but not necessarily checking that the input value is valid.
7ef25837
KW
2317
2318For the block property, new-style block names are returned (see
2319L</Old-style versus new-style block names>).
2320
2321To find the synonyms for single-forms, such as C<\p{Any}>, use
2322L</prop_aliases()> instead.
2323
2324C<prop_value_aliases> does not know about any user-defined properties, and
2325will return C<undef> if called with one of those.
2326
2327=cut
2328
7ef25837
KW
2329sub prop_value_aliases ($$) {
2330 my ($prop, $value) = @_;
2331 return unless defined $prop && defined $value;
2332
2333 require "unicore/UCD.pl";
2334 require "utf8_heavy.pl";
2335
2336 # Find the property name synonym that's used as the key in other hashes,
2337 # which is element 0 in the returned list.
2338 ($prop) = prop_aliases($prop);
2339 return if ! $prop;
2340 $prop = utf8::_loose_name(lc $prop);
2341
2342 # Here is a legal property, but the hash below (created by mktables for
2343 # this purpose) only knows about the properties that have a very finite
2344 # number of potential values, that is not ones whose value could be
2345 # anything, like most (if not all) string properties. These don't have
2346 # synonyms anyway. Simply return the input. For example, there is no
2347 # synonym for ('Uppercase_Mapping', A').
5c1d8161
KW
2348 if (! exists $prop_value_aliases{$prop}) {
2349
2350 # Here, we have a legal property, but an unknown value. Since the
2351 # property is legal, if it isn't in the prop_aliases hash, it must be
2352 # a Perl-extension All perl extensions are binary, hence are
2353 # enumerateds, which means that we know that the input unknown value
2354 # is illegal.
2355 return if ! exists $Unicode::UCD::prop_aliases{$prop};
2356
2357 # Otherwise, we assume it's valid, as documented.
2358 return $value;
2359 }
7ef25837
KW
2360
2361 # The value name may be loosely or strictly matched; we don't know yet.
2362 # But both types use lower-case.
2363 $value = lc $value;
2364
2365 # If the name isn't found under loose matching, it certainly won't be
2366 # found under strict
2367 my $loose_value = utf8::_loose_name($value);
2368 return unless exists $loose_to_standard_value{"$prop=$loose_value"};
2369
2370 # Similarly if the combination under loose matching doesn't exist, it
2371 # won't exist under strict.
2372 my $standard_value = $loose_to_standard_value{"$prop=$loose_value"};
2373 return unless exists $prop_value_aliases{$prop}{$standard_value};
2374
2375 # Here we did find a combination under loose matching rules. But it could
2376 # be that is a strict property match that shouldn't have matched.
2377 # %prop_value_aliases is set up so that the strict matches will appear as
2378 # if they were in loose form. Thus, if the non-loose version is legal,
2379 # we're ok, can skip the further check.
2380 if (! exists $utf8::stricter_to_file_of{"$prop=$value"}
2381
2382 # We're also ok and skip the further check if value loosely matches.
2383 # mktables has verified that no strict name under loose rules maps to
2384 # an existing loose name. This code relies on the very limited
2385 # circumstances that strict names can be here. Strict name matching
2386 # happens under two conditions:
2387 # 1) when the name begins with an underscore. But this function
2388 # doesn't accept those, and %prop_value_aliases doesn't have
2389 # them.
2390 # 2) When the values are numeric, in which case we need to look
2391 # further, but their squeezed-out loose values will be in
2392 # %stricter_to_file_of
2393 && exists $utf8::stricter_to_file_of{"$prop=$loose_value"})
2394 {
2395 # The only thing that's legal loosely under strict is that can have an
2396 # underscore between digit pairs XXX
2397 while ($value =~ s/(\d)_(\d)/$1$2/g) {}
2398 return unless exists $utf8::stricter_to_file_of{"$prop=$value"};
2399 }
2400
2401 # Here, we know that the combination exists. Return it.
2402 my $list_ref = $prop_value_aliases{$prop}{$standard_value};
2403 if (@$list_ref > 1) {
2404 # The full name is in element 1.
2405 return $list_ref->[1] unless wantarray;
2406
cb3150f5 2407 return @{_dclone $list_ref};
7ef25837
KW
2408 }
2409
2410 return $list_ref->[0] unless wantarray;
2411
2412 # Only 1 element means that it repeats
2413 return ( $list_ref->[0], $list_ref->[0] );
2414}
7319f91d 2415
681d705c
KW
2416# All 1 bits is the largest possible UV.
2417$Unicode::UCD::MAX_CP = ~0;
2418
2419=pod
2420
2421=head2 B<prop_invlist()>
2422
2423C<prop_invlist> returns an inversion list (described below) that defines all the
2424code points for the binary Unicode property (or "property=value" pair) given
2425by the input parameter string:
2426
2427 use feature 'say';
2428 use Unicode::UCD 'prop_invlist';
2429 say join ", ", prop_invlist("Any");
2430
2431 prints:
2432 0, 1114112
2433
c8652296
KW
2434If the input is unknown C<undef> is returned in scalar context; an empty-list
2435in list context. If the input is known, the number of elements in
681d705c
KW
2436the list is returned if called in scalar context.
2437
2438L<perluniprops|perluniprops/Properties accessible through \p{} and \P{}> gives
2439the list of properties that this function accepts, as well as all the possible
2440forms for them (including with the optional "Is_" prefixes). (Except this
2441function doesn't accept any Perl-internal properties, some of which are listed
2442there.) This function uses the same loose or tighter matching rules for
2443resolving the input property's name as is done for regular expressions. These
2444are also specified in L<perluniprops|perluniprops/Properties accessible
2445through \p{} and \P{}>. Examples of using the "property=value" form are:
2446
2447 say join ", ", prop_invlist("Script=Shavian");
2448
2449 prints:
2450 66640, 66688
2451
2452 say join ", ", prop_invlist("ASCII_Hex_Digit=No");
2453
2454 prints:
2455 0, 48, 58, 65, 71, 97, 103
2456
2457 say join ", ", prop_invlist("ASCII_Hex_Digit=Yes");
2458
2459 prints:
2460 48, 58, 65, 71, 97, 103
2461
2462Inversion lists are a compact way of specifying Unicode property-value
2463definitions. The 0th item in the list is the lowest code point that has the
2464property-value. The next item (item [1]) is the lowest code point beyond that
2465one that does NOT have the property-value. And the next item beyond that
2466([2]) is the lowest code point beyond that one that does have the
2467property-value, and so on. Put another way, each element in the list gives
2468the beginning of a range that has the property-value (for even numbered
2469elements), or doesn't have the property-value (for odd numbered elements).
2470The name for this data structure stems from the fact that each element in the
2471list toggles (or inverts) whether the corresponding range is or isn't on the
2472list.
2473
2474In the final example above, the first ASCII Hex digit is code point 48, the
2475character "0", and all code points from it through 57 (a "9") are ASCII hex
2476digits. Code points 58 through 64 aren't, but 65 (an "A") through 70 (an "F")
2477are, as are 97 ("a") through 102 ("f"). 103 starts a range of code points
2478that aren't ASCII hex digits. That range extends to infinity, which on your
2479computer can be found in the variable C<$Unicode::UCD::MAX_CP>. (This
2480variable is as close to infinity as Perl can get on your platform, and may be
2481too high for some operations to work; you may wish to use a smaller number for
2482your purposes.)
2483
2484Note that the inversion lists returned by this function can possibly include
2d88a86a
KW
2485non-Unicode code points, that is anything above 0x10FFFF. Unicode properties
2486are not defined on such code points. You might wish to change the output to
2487not include these. Simply add 0x110000 at the end of the non-empty returned
2488list if it isn't already that value; and pop that value if it is; like:
681d705c
KW
2489
2490 my @list = prop_invlist("foo");
2491 if (@list) {
2492 if ($list[-1] == 0x110000) {
2493 pop @list; # Defeat the turning on for above Unicode
2494 }
2495 else {
2496 push @list, 0x110000; # Turn off for above Unicode
2497 }
2498 }
2499
2500It is a simple matter to expand out an inversion list to a full list of all
2501code points that have the property-value:
2502
2503 my @invlist = prop_invlist($property_name);
2504 die "empty" unless @invlist;
2505 my @full_list;
2506 for (my $i = 0; $i < @invlist; $i += 2) {
2507 my $upper = ($i + 1) < @invlist
2508 ? $invlist[$i+1] - 1 # In range
2509 : $Unicode::UCD::MAX_CP; # To infinity. You may want
2510 # to stop much much earlier;
2511 # going this high may expose
2512 # perl deficiencies with very
2513 # large numbers.
2514 for my $j ($invlist[$i] .. $upper) {
2515 push @full_list, $j;
2516 }
2517 }
2518
2519C<prop_invlist> does not know about any user-defined nor Perl internal-only
2520properties, and will return C<undef> if called with one of those.
2521
1fdd5e53
KW
2522The L</search_invlist()> function is provided for finding a code point within
2523an inversion list.
2524
681d705c
KW
2525=cut
2526
2527# User-defined properties could be handled with some changes to utf8_heavy.pl;
2528# and implementing here of dealing with EXTRAS. If done, consideration should
2529# be given to the fact that the user subroutine could return different results
2530# with each call; security issues need to be thought about.
2531
2532# These are created by mktables for this routine and stored in unicore/UCD.pl
2533# where their structures are described.
2534our %loose_defaults;
2535our $MAX_UNICODE_CODEPOINT;
2536
92feec86 2537sub prop_invlist ($;$) {
681d705c 2538 my $prop = $_[0];
92feec86 2539
b08d569e
KW
2540 # Undocumented way to get at Perl internal properties; it may be changed
2541 # or removed without notice at any time.
92feec86
KW
2542 my $internal_ok = defined $_[1] && $_[1] eq '_perl_core_internal_ok';
2543
681d705c
KW
2544 return if ! defined $prop;
2545
2546 require "utf8_heavy.pl";
2547
2548 # Warnings for these are only for regexes, so not applicable to us
2549 no warnings 'deprecated';
2550
2551 # Get the swash definition of the property-value.
2552 my $swash = utf8::SWASHNEW(__PACKAGE__, $prop, undef, 1, 0);
2553
2554 # Fail if not found, or isn't a boolean property-value, or is a
2555 # user-defined property, or is internal-only.
2556 return if ! $swash
2557 || ref $swash eq ""
2558 || $swash->{'BITS'} != 1
2559 || $swash->{'USER_DEFINED'}
92feec86 2560 || (! $internal_ok && $prop =~ /^\s*_/);
681d705c
KW
2561
2562 if ($swash->{'EXTRAS'}) {
2563 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has EXTRAS magic";
2564 return;
2565 }
2566 if ($swash->{'SPECIALS'}) {
2567 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has SPECIALS magic";
2568 return;
2569 }
2570
2571 my @invlist;
2572
31aa6e0b
KW
2573 if ($swash->{'LIST'} =~ /^V/) {
2574
2575 # A 'V' as the first character marks the input as already an inversion
2576 # list, in which case, all we need to do is put the remaining lines
2577 # into our array.
2578 @invlist = split "\n", $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr;
2579 shift @invlist;
2580 }
2581 else {
1784d2f9
KW
2582 # The input lines look like:
2583 # 0041\t005A # [26]
2584 # 005F
681d705c 2585
1784d2f9
KW
2586 # Split into lines, stripped of trailing comments
2587 foreach my $range (split "\n",
2588 $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr)
2589 {
2590 # And find the beginning and end of the range on the line
2591 my ($hex_begin, $hex_end) = split "\t", $range;
2592 my $begin = hex $hex_begin;
2593
2594 # If the new range merely extends the old, we remove the marker
2595 # created the last time through the loop for the old's end, which
2596 # causes the new one's end to be used instead.
2597 if (@invlist && $begin == $invlist[-1]) {
2598 pop @invlist;
2599 }
2600 else {
2601 # Add the beginning of the range
2602 push @invlist, $begin;
2603 }
681d705c 2604
1784d2f9
KW
2605 if (defined $hex_end) { # The next item starts with the code point 1
2606 # beyond the end of the range.
2607 no warnings 'portable';
2608 my $end = hex $hex_end;
2609 last if $end == $Unicode::UCD::MAX_CP;
2610 push @invlist, $end + 1;
2611 }
2612 else { # No end of range, is a single code point.
2613 push @invlist, $begin + 1;
2614 }
681d705c
KW
2615 }
2616 }
2617
681d705c 2618 # Could need to be inverted: add or subtract a 0 at the beginning of the
2d88a86a 2619 # list.
681d705c
KW
2620 if ($swash->{'INVERT_IT'}) {
2621 if (@invlist && $invlist[0] == 0) {
2622 shift @invlist;
2623 }
2624 else {
2625 unshift @invlist, 0;
2626 }
681d705c
KW
2627 }
2628
2629 return @invlist;
2630}
7319f91d 2631
62b3b855
KW
2632=pod
2633
2634=head2 B<prop_invmap()>
2635
2636 use Unicode::UCD 'prop_invmap';
ad5f730f 2637 my ($list_ref, $map_ref, $format, $default)
62b3b855
KW
2638 = prop_invmap("General Category");
2639
2640C<prop_invmap> is used to get the complete mapping definition for a property,
2641in the form of an inversion map. An inversion map consists of two parallel
2642arrays. One is an ordered list of code points that mark range beginnings, and
2643the other gives the value (or mapping) that all code points in the
2644corresponding range have.
2645
2646C<prop_invmap> is called with the name of the desired property. The name is
2647loosely matched, meaning that differences in case, white-space, hyphens, and
2648underscores are not meaningful (except for the trailing underscore in the
2649old-form grandfathered-in property C<"L_">, which is better written as C<"LC">,
2650or even better, C<"Gc=LC">).
2651
2652Many Unicode properties have more than one name (or alias). C<prop_invmap>
2653understands all of these, including Perl extensions to them. Ambiguities are
2654resolved as described above for L</prop_aliases()>. The Perl internal
2d88a86a
KW
2655property "Perl_Decimal_Digit, described below, is also accepted. An empty
2656list is returned if the property name is unknown.
ee94c7d1
KW
2657See L<perluniprops/Properties accessible through Unicode::UCD> for the
2658properties acceptable as inputs to this function.
62b3b855
KW
2659
2660It is a fatal error to call this function except in list context.
2661
13d1b68a 2662In addition to the two arrays that form the inversion map, C<prop_invmap>
62b3b855 2663returns two other values; one is a scalar that gives some details as to the
5b2b1977
KW
2664format of the entries of the map array; the other is a default value, useful
2665in maps whose format name begins with the letter C<"a">, as described
2666L<below in its subsection|/a>; and for specialized purposes, such as
2667converting to another data structure, described at the end of this main
2668section.
62b3b855
KW
2669
2670This means that C<prop_invmap> returns a 4 element list. For example,
2671
2672 my ($blocks_ranges_ref, $blocks_maps_ref, $format, $default)
2673 = prop_invmap("Block");
2674
2675In this call, the two arrays will be populated as shown below (for Unicode
26766.0):
2677
2678 Index @blocks_ranges @blocks_maps
2679 0 0x0000 Basic Latin
2680 1 0x0080 Latin-1 Supplement
2681 2 0x0100 Latin Extended-A
2682 3 0x0180 Latin Extended-B
2683 4 0x0250 IPA Extensions
2684 5 0x02B0 Spacing Modifier Letters
2685 6 0x0300 Combining Diacritical Marks
2686 7 0x0370 Greek and Coptic
2687 8 0x0400 Cyrillic
2688 ...
2689 233 0x2B820 No_Block
2690 234 0x2F800 CJK Compatibility Ideographs Supplement
2691 235 0x2FA20 No_Block
2692 236 0xE0000 Tags
2693 237 0xE0080 No_Block
2694 238 0xE0100 Variation Selectors Supplement
2695 239 0xE01F0 No_Block
2696 240 0xF0000 Supplementary Private Use Area-A
2697 241 0x100000 Supplementary Private Use Area-B
2698 242 0x110000 No_Block
2699
2700The first line (with Index [0]) means that the value for code point 0 is "Basic
2701Latin". The entry "0x0080" in the @blocks_ranges column in the second line
2702means that the value from the first line, "Basic Latin", extends to all code
2703points in the range from 0 up to but not including 0x0080, that is, through
647396da 2704127. In other words, the code points from 0 to 127 are all in the "Basic
62b3b855
KW
2705Latin" block. Similarly, all code points in the range from 0x0080 up to (but
2706not including) 0x0100 are in the block named "Latin-1 Supplement", etc.
2707(Notice that the return is the old-style block names; see L</Old-style versus
2708new-style block names>).
2709
2710The final line (with Index [242]) means that the value for all code points above
2711the legal Unicode maximum code point have the value "No_Block", which is the
2712term Unicode uses for a non-existing block.
2713
2714The arrays completely specify the mappings for all possible code points.
2715The final element in an inversion map returned by this function will always be
2716for the range that consists of all the code points that aren't legal Unicode,
2717but that are expressible on the platform. (That is, it starts with code point
27180x110000, the first code point above the legal Unicode maximum, and extends to
2719infinity.) The value for that range will be the same that any typical
2720unassigned code point has for the specified property. (Certain unassigned
2721code points are not "typical"; for example the non-character code points, or
2722those in blocks that are to be written right-to-left. The above-Unicode
2723range's value is not based on these atypical code points.) It could be argued
2724that, instead of treating these as unassigned Unicode code points, the value
2725for this range should be C<undef>. If you wish, you can change the returned
2726arrays accordingly.
2727
ad5f730f
KW
2728The maps for almost all properties are simple scalars that should be
2729interpreted as-is.
62b3b855
KW
2730These values are those given in the Unicode-supplied data files, which may be
2731inconsistent as to capitalization and as to which synonym for a property-value
2732is given. The results may be normalized by using the L</prop_value_aliases()>
2733function.
2734
2735There are exceptions to the simple scalar maps. Some properties have some
2736elements in their map list that are themselves lists of scalars; and some
2737special strings are returned that are not to be interpreted as-is. Element
2738[2] (placed into C<$format> in the example above) of the returned four element
647396da 2739list tells you if the map has any of these special elements or not, as follows:
62b3b855
KW
2740
2741=over
2742
dc8d8ea6 2743=item B<C<s>>
62b3b855
KW
2744
2745means all the elements of the map array are simple scalars, with no special
2746elements. Almost all properties are like this, like the C<block> example
2747above.
2748
dc8d8ea6 2749=item B<C<sl>>
62b3b855 2750
647396da 2751means that some of the map array elements have the form given by C<"s">, and
62b3b855
KW
2752the rest are lists of scalars. For example, here is a portion of the output
2753of calling C<prop_invmap>() with the "Script Extensions" property:
2754
2755 @scripts_ranges @scripts_maps
2756 ...
c2ca0207
KW
2757 0x0953 Devanagari
2758 0x0964 [ Bengali, Devanagari, Gurumukhi, Oriya ]
2759 0x0966 Devanagari
62b3b855
KW
2760 0x0970 Common
2761
647396da
KW
2762Here, the code points 0x964 and 0x965 are both used in Bengali,
2763Devanagari, Gurmukhi, and Oriya, but no other scripts.
62b3b855 2764
647396da 2765The Name_Alias property is also of this form. But each scalar consists of two
58b75e36 2766components: 1) the name, and 2) the type of alias this is. They are
7620cb10
KW
2767separated by a colon and a space. In Unicode 6.1, there are several alias types:
2768
2769=over
2770
2771=item C<correction>
2772
2773indicates that the name is a corrected form for the
2774original name (which remains valid) for the same code point.
2775
2776=item C<control>
2777
2778adds a new name for a control character.
2779
2780=item C<alternate>
2781
2782is an alternate name for a character
2783
2784=item C<figment>
2785
2786is a name for a character that has been documented but was never in any
2787actual standard.
2788
2789=item C<abbreviation>
2790
2791is a common abbreviation for a character
2792
2793=back
2794
2795The lists are ordered (roughly) so the most preferred names come before less
2796preferred ones.
58b75e36
KW
2797
2798For example,
2799
7620cb10
KW
2800 @aliases_ranges @alias_maps
2801 ...
2802 0x009E [ 'PRIVACY MESSAGE: control', 'PM: abbreviation' ]
2803 0x009F [ 'APPLICATION PROGRAM COMMAND: control',
2804 'APC: abbreviation'
2805 ]
2806 0x00A0 'NBSP: abbreviation'
2807 0x00A1 ""
2808 0x00AD 'SHY: abbreviation'
2809 0x00AE ""
2810 0x01A2 'LATIN CAPITAL LETTER GHA: correction'
2811 0x01A3 'LATIN SMALL LETTER GHA: correction'
2812 0x01A4 ""
58b75e36 2813 ...
58b75e36 2814
7620cb10
KW
2815A map to the empty string means that there is no alias defined for the code
2816point.
58b75e36 2817
d11155ec 2818=item B<C<a>>
62b3b855 2819
647396da 2820is like C<"s"> in that all the map array elements are scalars, but here they are
d11155ec
KW
2821restricted to all being integers, and some have to be adjusted (hence the name
2822C<"a">) to get the correct result. For example, in:
62b3b855 2823
5b2b1977 2824 my ($uppers_ranges_ref, $uppers_maps_ref, $format, $default)
62b3b855
KW
2825 = prop_invmap("Simple_Uppercase_Mapping");
2826
2827the returned arrays look like this:
2828
2829 @$uppers_ranges_ref @$uppers_maps_ref Note
bf7fe2df 2830 0 0
d11155ec 2831 97 65 'a' maps to 'A', b => B ...
bf7fe2df 2832 123 0
d11155ec 2833 181 924 MICRO SIGN => Greek Cap MU
bf7fe2df 2834 182 0
62b3b855
KW
2835 ...
2836
5b2b1977
KW
2837and C<$default> is 0.
2838
d11155ec
KW
2839Let's start with the second line. It says that the uppercase of code point 97
2840is 65; or C<uc("a")> == "A". But the line is for the entire range of code
5b2b1977
KW
2841points 97 through 122. To get the mapping for any code point in this range,
2842you take the offset it has from the beginning code point of the range, and add
d11155ec
KW
2843that to the mapping for that first code point. So, the mapping for 122 ("z")
2844is derived by taking the offset of 122 from 97 (=25) and adding that to 65,
2845yielding 90 ("z"). Likewise for everything in between.
2846
d11155ec
KW
2847Requiring this simple adjustment allows the returned arrays to be
2848significantly smaller than otherwise, up to a factor of 10, speeding up
2849searching through them.
62b3b855 2850
5b2b1977
KW
2851Ranges that map to C<$default>, C<"0">, behave somewhat differently. For
2852these, each code point maps to itself. So, in the first line in the example,
2853S<C<ord(uc(chr(0)))>> is 0, S<C<ord(uc(chr(1)))>> is 1, ..
2854S<C<ord(uc(chr(96)))>> is 96.
2855
d11155ec 2856=item B<C<al>>
62b3b855 2857
d11155ec 2858means that some of the map array elements have the form given by C<"a">, and
62b3b855
KW
2859the rest are ordered lists of code points.
2860For example, in:
2861
5b2b1977 2862 my ($uppers_ranges_ref, $uppers_maps_ref, $format, $default)
62b3b855
KW
2863 = prop_invmap("Uppercase_Mapping");
2864
2865the returned arrays look like this:
2866
2867 @$uppers_ranges_ref @$uppers_maps_ref
bf7fe2df 2868 0 0
d11155ec 2869 97 65
bf7fe2df 2870 123 0
d11155ec 2871 181 924
bf7fe2df 2872 182 0
62b3b855
KW
2873 ...
2874 0x0149 [ 0x02BC 0x004E ]
bf7fe2df 2875 0x014A 0
d11155ec 2876 0x014B 330
62b3b855
KW
2877 ...
2878
2879This is the full Uppercase_Mapping property (as opposed to the
d11155ec 2880Simple_Uppercase_Mapping given in the example for format C<"a">). The only
62b3b855
KW
2881difference between the two in the ranges shown is that the code point at
28820x0149 (LATIN SMALL LETTER N PRECEDED BY APOSTROPHE) maps to a string of two
2883characters, 0x02BC (MODIFIER LETTER APOSTROPHE) followed by 0x004E (LATIN
2884CAPITAL LETTER N).
2885
d11155ec
KW
2886No adjustments are needed to entries that are references to arrays; each such
2887entry will have exactly one element in its range, so the offset is always 0.
bf7fe2df 2888
5b2b1977
KW
2889The fourth (index [3]) element (C<$default>) in the list returned for this
2890format is 0.
2891
d11155ec 2892=item B<C<ae>>
b0b13ada 2893
d11155ec
KW
2894This is like C<"a">, but some elements are the empty string, and should not be
2895adjusted.
b0b13ada
KW
2896The one internal Perl property accessible by C<prop_invmap> is of this type:
2897"Perl_Decimal_Digit" returns an inversion map which gives the numeric values
2898that are represented by the Unicode decimal digit characters. Characters that
2899don't represent decimal digits map to the empty string, like so:
2900
2901 @digits @values
2902 0x0000 ""
d11155ec 2903 0x0030 0
b0b13ada 2904 0x003A: ""
d11155ec 2905 0x0660: 0
b0b13ada 2906 0x066A: ""
d11155ec 2907 0x06F0: 0
b0b13ada 2908 0x06FA: ""
d11155ec 2909 0x07C0: 0
b0b13ada 2910 0x07CA: ""
d11155ec 2911 0x0966: 0
b0b13ada
KW
2912 ...
2913
2914This means that the code points from 0 to 0x2F do not represent decimal digits;
d11155ec
KW
2915the code point 0x30 (DIGIT ZERO) represents 0; code point 0x31, (DIGIT ONE),
2916represents 0+1-0 = 1; ... code point 0x39, (DIGIT NINE), represents 0+9-0 = 9;
2917... code points 0x3A through 0x65F do not represent decimal digits; 0x660
2918(ARABIC-INDIC DIGIT ZERO), represents 0; ... 0x07C1 (NKO DIGIT ONE),
2919represents 0+1-0 = 1 ...
b0b13ada 2920
5b2b1977
KW
2921The fourth (index [3]) element (C<$default>) in the list returned for this
2922format is the empty string.
2923
d11155ec 2924=item B<C<ale>>
62b3b855 2925
d11155ec
KW
2926is a combination of the C<"al"> type and the C<"ae"> type. Some of
2927the map array elements have the forms given by C<"al">, and
62b3b855
KW
2928the rest are the empty string. The property C<NFKC_Casefold> has this form.
2929An example slice is:
2930
2931 @$ranges_ref @$maps_ref Note
2932 ...
d11155ec
KW
2933 0x00AA 97 FEMININE ORDINAL INDICATOR => 'a'
2934 0x00AB 0
62b3b855 2935 0x00AD SOFT HYPHEN => ""
d11155ec 2936 0x00AE 0
62b3b855 2937 0x00AF [ 0x0020, 0x0304 ] MACRON => SPACE . COMBINING MACRON
d11155ec 2938 0x00B0 0
62b3b855
KW
2939 ...
2940
5b2b1977
KW
2941The fourth (index [3]) element (C<$default>) in the list returned for this
2942format is 0.
2943
4f143a72 2944=item B<C<ar>>
6cc45523
KW
2945
2946means that all the elements of the map array are either rational numbers or
2947the string C<"NaN">, meaning "Not a Number". A rational number is either an
2948integer, or two integers separated by a solidus (C<"/">). The second integer
2949represents the denominator of the division implied by the solidus, and is
60c78852 2950actually always positive, so it is guaranteed not to be 0 and to not be
6329003c 2951signed. When the element is a plain integer (without the
4f143a72
KW
2952solidus), it may need to be adjusted to get the correct value by adding the
2953offset, just as other C<"a"> properties. No adjustment is needed for
2954fractions, as the range is guaranteed to have just a single element, and so
2955the offset is always 0.
2956
2957If you want to convert the returned map to entirely scalar numbers, you
6cc45523
KW
2958can use something like this:
2959
2960 my ($invlist_ref, $invmap_ref, $format) = prop_invmap($property);
4f143a72 2961 if ($format && $format eq "ar") {
60c78852 2962 map { $_ = eval $_ if $_ ne 'NaN' } @$map_ref;
6cc45523
KW
2963 }
2964
2965Here's some entries from the output of the property "Nv", which has format
4f143a72 2966C<"ar">.
6cc45523 2967
4f143a72 2968 @numerics_ranges @numerics_maps Note
6cc45523 2969 0x00 "NaN"
4f143a72 2970 0x30 0 DIGIT 0 .. DIGIT 9
6cc45523 2971 0x3A "NaN"
4f143a72 2972 0xB2 2 SUPERSCRIPTs 2 and 3
6cc45523 2973 0xB4 "NaN"
4f143a72 2974 0xB9 1 SUPERSCRIPT 1
6cc45523 2975 0xBA "NaN"
4f143a72
KW
2976 0xBC 1/4 VULGAR FRACTION 1/4
2977 0xBD 1/2 VULGAR FRACTION 1/2
2978 0xBE 3/4 VULGAR FRACTION 3/4
6cc45523 2979 0xBF "NaN"
4f143a72
KW
2980 0x660 0 ARABIC-INDIC DIGIT ZERO .. NINE
2981 0x66A "NaN"
6cc45523 2982
5b2b1977
KW
2983The fourth (index [3]) element (C<$default>) in the list returned for this
2984format is C<"NaN">.
2985
dc8d8ea6 2986=item B<C<n>>
62b3b855
KW
2987
2988means the Name property. All the elements of the map array are simple
2989scalars, but some of them contain special strings that require more work to
2990get the actual name.
2991
2992Entries such as:
2993
2994 CJK UNIFIED IDEOGRAPH-<code point>
2995
2996mean that the name for the code point is "CJK UNIFIED IDEOGRAPH-"
2997with the code point (expressed in hexadecimal) appended to it, like "CJK
647396da
KW
2998UNIFIED IDEOGRAPH-3403" (similarly for S<C<CJK COMPATIBILITY IDEOGRAPH-E<lt>code
2999pointE<gt>>>).
62b3b855
KW
3000
3001Also, entries like
3002
3003 <hangul syllable>
3004
3005means that the name is algorithmically calculated. This is easily done by
3006the function L<charnames/charnames::viacode(code)>.
3007
3008Note that for control characters (C<Gc=cc>), Unicode's data files have the
3009string "C<E<lt>controlE<gt>>", but the real name of each of these characters is the empty
7620cb10 3010string. This function returns that real name, the empty string. (There are
647396da
KW
3011names for these characters, but they are considered aliases, not the Name
3012property name, and are contained in the C<Name_Alias> property.)
62b3b855 3013
d11155ec 3014=item B<C<ad>>
62b3b855 3015
d11155ec 3016means the Decomposition_Mapping property. This property is like C<"al">
bea2c146 3017properties, except that one of the scalar elements is of the form:
62b3b855
KW
3018
3019 <hangul syllable>
3020
bea2c146
KW
3021This signifies that this entry should be replaced by the decompositions for
3022all the code points whose decomposition is algorithmically calculated. (All
13d1b68a 3023of them are currently in one range and no others outside the range are likely
6329003c 3024to ever be added to Unicode; the C<"n"> format
bea2c146 3025has this same entry.) These can be generated via the function
62b3b855
KW
3026L<Unicode::Normalize::NFD()|Unicode::Normalize>.
3027
62b3b855
KW
3028Note that the mapping is the one that is specified in the Unicode data files,
3029and to get the final decomposition, it may need to be applied recursively.
3030
5b2b1977
KW
3031The fourth (index [3]) element (C<$default>) in the list returned for this
3032format is 0.
3033
62b3b855
KW
3034=back
3035
d11155ec
KW
3036Note that a format begins with the letter "a" if and only the property it is
3037for requires adjustments by adding the offsets in multi-element ranges. For
3038all these properties, an entry should be adjusted only if the map is a scalar
3039which is an integer. That is, it must match the regular expression:
3040
3041 / ^ -? \d+ $ /xa
3042
3043Further, the first element in a range never needs adjustment, as the
3044adjustment would be just adding 0.
3045
1fdd5e53
KW
3046A binary search such as that provided by L</search_invlist()>, can be used to
3047quickly find a code point in the inversion list, and hence its corresponding
3048mapping.
62b3b855 3049
5b2b1977
KW
3050The final, fourth element (index [3], assigned to C<$default> in the "block"
3051example) in the four element list returned by this function is used with the
3052C<"a"> format types; it may also be useful for applications
62b3b855
KW
3053that wish to convert the returned inversion map data structure into some
3054other, such as a hash. It gives the mapping that most code points map to
3055under the property. If you establish the convention that any code point not
3056explicitly listed in your data structure maps to this value, you can
3057potentially make your data structure much smaller. As you construct your data
3058structure from the one returned by this function, simply ignore those ranges
5b2b1977 3059that map to this value. For example, to
62b3b855 3060convert to the data structure searchable by L</charinrange()>, you can follow
6329003c 3061this recipe for properties that don't require adjustments:
62b3b855 3062
ad5f730f 3063 my ($list_ref, $map_ref, $format, $default) = prop_invmap($property);
62b3b855 3064 my @range_list;
6329003c
KW
3065
3066 # Look at each element in the list, but the -2 is needed because we
3067 # look at $i+1 in the loop, and the final element is guaranteed to map
ad5f730f 3068 # to $default by prop_invmap(), so we would skip it anyway.
62b3b855 3069 for my $i (0 .. @$list_ref - 2) {
ad5f730f 3070 next if $map_ref->[$i] eq $default;
62b3b855
KW
3071 push @range_list, [ $list_ref->[$i],
3072 $list_ref->[$i+1],
3073 $map_ref->[$i]
3074 ];
3075 }
3076
3077 print charinrange(\@range_list, $code_point), "\n";
3078
62b3b855 3079With this, C<charinrange()> will return C<undef> if its input code point maps
ad5f730f 3080to C<$default>. You can avoid this by omitting the C<next> statement, and adding
62b3b855
KW
3081a line after the loop to handle the final element of the inversion map.
3082
6329003c
KW
3083Similarly, this recipe can be used for properties that do require adjustments:
3084
3085 for my $i (0 .. @$list_ref - 2) {
ad5f730f 3086 next if $map_ref->[$i] eq $default;
6329003c
KW
3087
3088 # prop_invmap() guarantees that if the mapping is to an array, the
3089 # range has just one element, so no need to worry about adjustments.
3090 if (ref $map_ref->[$i]) {
3091 push @range_list,
3092 [ $list_ref->[$i], $list_ref->[$i], $map_ref->[$i] ];
3093 }
3094 else { # Otherwise each element is actually mapped to a separate
3095 # value, so the range has to be split into single code point
3096 # ranges.
3097
3098 my $adjustment = 0;
3099
3100 # For each code point that gets mapped to something...
3101 for my $j ($list_ref->[$i] .. $list_ref->[$i+1] -1 ) {
3102
3103 # ... add a range consisting of just it mapping to the
3104 # original plus the adjustment, which is incremented for the
3105 # next time through the loop, as the offset increases by 1
3106 # for each element in the range
3107 push @range_list,
3108 [ $j, $j, $map_ref->[$i] + $adjustment++ ];
3109 }
3110 }
3111 }
62b3b855
KW
3112
3113Note that the inversion maps returned for the C<Case_Folding> and
3114C<Simple_Case_Folding> properties do not include the Turkic-locale mappings.
3115Use L</casefold()> for these.
3116
62b3b855
KW
3117C<prop_invmap> does not know about any user-defined properties, and will
3118return C<undef> if called with one of those.
3119
91e78470
KW
3120The returned values for the Perl extension properties, such as C<Any> and
3121C<Greek> are somewhat misleading. The values are either C<"Y"> or C<"N>".
3122All Unicode properties are bipartite, so you can actually use the C<"Y"> or
3123C<"N>" in a Perl regular rexpression for these, like C<qr/\p{ID_Start=Y/}> or
3124C<qr/\p{Upper=N/}>. But the Perl extensions aren't specified this way, only
3125like C</qr/\p{Any}>, I<etc>. You can't actually use the C<"Y"> and C<"N>" in
3126them.
3127
62b3b855
KW
3128=cut
3129
3130# User-defined properties could be handled with some changes to utf8_heavy.pl;
3131# if done, consideration should be given to the fact that the user subroutine
3132# could return different results with each call, which could lead to some
3133# security issues.
3134
3135# One could store things in memory so they don't have to be recalculated, but
3136# it is unlikely this will be called often, and some properties would take up
3137# significant memory.
3138
3139# These are created by mktables for this routine and stored in unicore/UCD.pl
3140# where their structures are described.
3141our @algorithmic_named_code_points;
3142our $HANGUL_BEGIN;
3143our $HANGUL_COUNT;
3144
088d3c62 3145sub prop_invmap ($;$) {
62b3b855
KW
3146
3147 croak __PACKAGE__, "::prop_invmap: must be called in list context" unless wantarray;
3148
3149 my $prop = $_[0];
3150 return unless defined $prop;
3151
088d3c62
KW
3152 # Undocumented way to get at Perl internal properties; it may be changed
3153 # or removed without notice at any time. It currently also changes the
3154 # output to use the format specified in the file rather than the one we
3155 # normally compute and return
3156 my $internal_ok = defined $_[1] && $_[1] eq '_perl_core_internal_ok';
3157
62b3b855 3158 # Fail internal properties
088d3c62 3159 return if $prop =~ /^_/ && ! $internal_ok;
62b3b855
KW
3160
3161 # The values returned by this function.
3162 my (@invlist, @invmap, $format, $missing);
3163
3164 # The swash has two components we look at, the base list, and a hash,
3165 # named 'SPECIALS', containing any additional members whose mappings don't
13d1b68a 3166 # fit into the base list scheme of things. These generally 'override'
62b3b855
KW
3167 # any value in the base list for the same code point.
3168 my $overrides;
3169
3170 require "utf8_heavy.pl";
3171 require "unicore/UCD.pl";
3172
3173RETRY:
3174
647396da
KW
3175 # If there are multiple entries for a single code point
3176 my $has_multiples = 0;
3177
62b3b855
KW
3178 # Try to get the map swash for the property. They have 'To' prepended to
3179 # the property name, and 32 means we will accept 32 bit return values.
647396da 3180 # The 0 means we aren't calling this from tr///.
62b3b855
KW
3181 my $swash = utf8::SWASHNEW(__PACKAGE__, "To$prop", undef, 32, 0);
3182
3183 # If didn't find it, could be because needs a proxy. And if was the
3184 # 'Block' or 'Name' property, use a proxy even if did find it. Finding it
647396da
KW
3185 # in these cases would be the result of the installation changing mktables
3186 # to output the Block or Name tables. The Block table gives block names
3187 # in the new-style, and this routine is supposed to return old-style block
3188 # names. The Name table is valid, but we need to execute the special code
3189 # below to add in the algorithmic-defined name entries.
34132297 3190 # And NFKCCF needs conversion, so handle that here too.
62b3b855 3191 if (ref $swash eq ""
34132297 3192 || $swash->{'TYPE'} =~ / ^ To (?: Blk | Na | NFKCCF ) $ /x)
62b3b855
KW
3193 {
3194
3195 # Get the short name of the input property, in standard form
3196 my ($second_try) = prop_aliases($prop);
3197 return unless $second_try;
3198 $second_try = utf8::_loose_name(lc $second_try);
3199
3200 if ($second_try eq "in") {
3201
3202 # This property is identical to age for inversion map purposes
3203 $prop = "age";
3204 goto RETRY;
3205 }
cfc5eb77 3206 elsif ($second_try =~ / ^ s ( cf | fc | [ltu] c ) $ /x) {
62b3b855 3207
75e7c50b
KW
3208 # These properties use just the LIST part of the full mapping,
3209 # which includes the simple maps that are otherwise overridden by
3210 # the SPECIALS. So all we need do is to not look at the SPECIALS;
3211 # set $overrides to indicate that
62b3b855 3212 $overrides = -1;
62b3b855 3213
75e7c50b 3214 # The full name is the simple name stripped of its initial 's'
cfc5eb77
KW
3215 $prop = $1;
3216
3217 # .. except for this case
3218 $prop = 'cf' if $prop eq 'fc';
3219
62b3b855
KW
3220 goto RETRY;
3221 }
3222 elsif ($second_try eq "blk") {
3223
3224 # We use the old block names. Just create a fake swash from its
3225 # data.
3226 _charblocks();
3227 my %blocks;
3228 $blocks{'LIST'} = "";
3229 $blocks{'TYPE'} = "ToBlk";
3230 $utf8::SwashInfo{ToBlk}{'missing'} = "No_Block";
3231 $utf8::SwashInfo{ToBlk}{'format'} = "s";
3232
3233 foreach my $block (@BLOCKS) {
3234 $blocks{'LIST'} .= sprintf "%x\t%x\t%s\n",
3235 $block->[0],
3236 $block->[1],
3237 $block->[2];
3238 }
3239 $swash = \%blocks;
3240 }
3241 elsif ($second_try eq "na") {
3242
3243 # Use the combo file that has all the Name-type properties in it,
3244 # extracting just the ones that are for the actual 'Name'
3245 # property. And create a fake swash from it.
3246 my %names;
3247 $names{'LIST'} = "";
3248 my $original = do "unicore/Name.pl";
62b3b855
KW
3249 my $algorithm_names = \@algorithmic_named_code_points;
3250
3b6a8189
KW
3251 # We need to remove the names from it that are aliases. For that
3252 # we need to also read in that table. Create a hash with the keys
3253 # being the code points, and the values being a list of the
3254 # aliases for the code point key.
3255 my ($aliases_code_points, $aliases_maps, undef, undef) =
3256 &prop_invmap('Name_Alias');
3257 my %aliases;
3258 for (my $i = 0; $i < @$aliases_code_points; $i++) {
3259 my $code_point = $aliases_code_points->[$i];
3260 $aliases{$code_point} = $aliases_maps->[$i];
3261
3262 # If not already a list, make it into one, so that later we
3263 # can treat things uniformly
3264 if (! ref $aliases{$code_point}) {
3265 $aliases{$code_point} = [ $aliases{$code_point} ];
3266 }
3267
3268 # Remove the alias type from the entry, retaining just the
3269 # name.
3270 map { s/:.*// } @{$aliases{$code_point}};
3271 }
3272
62b3b855
KW
3273 my $i = 0;
3274 foreach my $line (split "\n", $original) {
3275 my ($hex_code_point, $name) = split "\t", $line;
3276
3277 # Weeds out all comments, blank lines, and named sequences
df46a385 3278 next if $hex_code_point =~ /[^[:xdigit:]]/a;
62b3b855
KW
3279
3280 my $code_point = hex $hex_code_point;
3281
3282 # The name of all controls is the default: the empty string.
a1ae4420
KW
3283 # The set of controls is immutable
3284 next if chr($code_point) =~ /[[:cntrl:]]/u;
62b3b855 3285
3b6a8189
KW
3286 # If this is a name_alias, it isn't a name
3287 next if grep { $_ eq $name } @{$aliases{$code_point}};
62b3b855
KW
3288
3289 # If we are beyond where one of the special lines needs to
3290 # be inserted ...
3b6a8189 3291 while ($i < @$algorithm_names
62b3b855
KW
3292 && $code_point > $algorithm_names->[$i]->{'low'})
3293 {
3294
3295 # ... then insert it, ahead of what we were about to
3296 # output
3b6a8189 3297 $names{'LIST'} .= sprintf "%x\t%x\t%s\n",
62b3b855
KW
3298 $algorithm_names->[$i]->{'low'},
3299 $algorithm_names->[$i]->{'high'},
3300 $algorithm_names->[$i]->{'name'};
3301
62b3b855
KW
3302 # Done with this range.
3303 $i++;
3304
3b6a8189
KW
3305 # We loop until all special lines that precede the next
3306 # regular one are output.
62b3b855
KW
3307 }
3308
3b6a8189
KW
3309 # Here, is a normal name.
3310 $names{'LIST'} .= sprintf "%x\t\t%s\n", $code_point, $name;
3311 } # End of loop through all the names
62b3b855
KW
3312
3313 $names{'TYPE'} = "ToNa";
3314 $utf8::SwashInfo{ToNa}{'missing'} = "";
3315 $utf8::SwashInfo{ToNa}{'format'} = "n";
3316 $swash = \%names;
3317 }
3318 elsif ($second_try =~ / ^ ( d [mt] ) $ /x) {
3319
3320 # The file is a combination of dt and dm properties. Create a
3321 # fake swash from the portion that we want.
3322 my $original = do "unicore/Decomposition.pl";
3323 my %decomps;
3324
3325 if ($second_try eq 'dt') {
3326 $decomps{'TYPE'} = "ToDt";
3327 $utf8::SwashInfo{'ToDt'}{'missing'} = "None";
3328 $utf8::SwashInfo{'ToDt'}{'format'} = "s";
d11155ec 3329 } # 'dm' is handled below, with 'nfkccf'
62b3b855
KW
3330
3331 $decomps{'LIST'} = "";
3332
3333 # This property has one special range not in the file: for the
1a4c9760
KW
3334 # hangul syllables. But not in Unicode version 1.
3335 UnicodeVersion() unless defined $v_unicode_version;
3336 my $done_hangul = ($v_unicode_version lt v2.0.0)
3337 ? 1
3338 : 0; # Have we done the hangul range ?
62b3b855
KW
3339 foreach my $line (split "\n", $original) {
3340 my ($hex_lower, $hex_upper, $type_and_map) = split "\t", $line;
3341 my $code_point = hex $hex_lower;
3342 my $value;
bea2c146 3343 my $redo = 0;
62b3b855
KW
3344
3345 # The type, enclosed in <...>, precedes the mapping separated
3346 # by blanks
3347 if ($type_and_map =~ / ^ < ( .* ) > \s+ (.*) $ /x) {
3348 $value = ($second_try eq 'dt') ? $1 : $2
3349 }
3350 else { # If there is no type specified, it's canonical
3351 $value = ($second_try eq 'dt')
3352 ? "Canonical" :
3353 $type_and_map;
3354 }
3355
3356 # Insert the hangul range at the appropriate spot.
3357 if (! $done_hangul && $code_point > $HANGUL_BEGIN) {
3358 $done_hangul = 1;
3359 $decomps{'LIST'} .=
3360 sprintf "%x\t%x\t%s\n",
3361 $HANGUL_BEGIN,
3362 $HANGUL_BEGIN + $HANGUL_COUNT - 1,
3363 ($second_try eq 'dt')
3364 ? "Canonical"
3365 : "<hangul syllable>";
3366 }
3367
1a4c9760
KW
3368 if ($value =~ / / && $hex_upper ne "" && $hex_upper ne $hex_lower) {
3369 $line = sprintf("%04X\t%s\t%s", hex($hex_lower) + 1, $hex_upper, $value);
3370 $hex_upper = "";
3371 $redo = 1;
3372 }
3373
62b3b855
KW
3374 # And append this to our constructed LIST.
3375 $decomps{'LIST'} .= "$hex_lower\t$hex_upper\t$value\n";
bea2c146
KW
3376
3377 redo if $redo;
62b3b855
KW
3378 }
3379 $swash = \%decomps;
3380 }
d11155ec
KW
3381 elsif ($second_try ne 'nfkccf') { # Don't know this property. Fail.
3382 return;
3383 }
3384
3385 if ($second_try eq 'nfkccf' || $second_try eq 'dm') {
34132297 3386
d11155ec
KW
3387 # The 'nfkccf' property is stored in the old format for backwards
3388 # compatibility for any applications that has read its file
3389 # directly before prop_invmap() existed.
3390 # And the code above has extracted the 'dm' property from its file
3391 # yielding the same format. So here we convert them to adjusted
3392 # format for compatibility with the other properties similar to
3393 # them.
3394 my %revised_swash;
34132297 3395
d11155ec 3396 # We construct a new converted list.
34132297 3397 my $list = "";
d11155ec
KW
3398
3399 my @ranges = split "\n", $swash->{'LIST'};
3400 for (my $i = 0; $i < @ranges; $i++) {
3401 my ($hex_begin, $hex_end, $map) = split "\t", $ranges[$i];
3402
3403 # The dm property has maps that are space separated sequences
3404 # of code points, as well as the special entry "<hangul
3405 # syllable>, which also contains a blank.
3406 my @map = split " ", $map;
3407 if (@map > 1) {
3408
3409 # If it's just the special entry, append as-is.
3410 if ($map eq '<hangul syllable>') {
3411 $list .= "$ranges[$i]\n";
3412 }
3413 else {
3414
e2c04025 3415 # These should all be single-element ranges.
1a4c9760 3416 croak __PACKAGE__, "::prop_invmap: Not expecting a mapping with multiple code points in a multi-element range, $ranges[$i]" if $hex_end ne "" && $hex_end ne $hex_begin;
d11155ec
KW
3417
3418 # Convert them to decimal, as that's what's expected.
3419 $list .= "$hex_begin\t\t"
3420 . join(" ", map { hex } @map)
3421 . "\n";
3422 }
3423 next;
3424 }
3425
3426 # Here, the mapping doesn't have a blank, is for a single code
3427 # point.
34132297
KW
3428 my $begin = hex $hex_begin;
3429 my $end = (defined $hex_end && $hex_end ne "")
3430 ? hex $hex_end
3431 : $begin;
d11155ec
KW
3432
3433 # Again, the output is to be in decimal.
34132297 3434 my $decimal_map = hex $map;
d11155ec
KW
3435
3436 # We know that multi-element ranges with the same mapping
3437 # should not be adjusted, as after the adjustment
3438 # multi-element ranges are for consecutive increasing code
3439 # points. Further, the final element in the list won't be
3440 # adjusted, as there is nothing after it to include in the
3441 # adjustment
3442 if ($begin != $end || $i == @ranges -1) {
3443
3444 # So just convert these to single-element ranges
3445 foreach my $code_point ($begin .. $end) {
3446 $list .= sprintf("%04X\t\t%d\n",
3447 $code_point, $decimal_map);
3448 }
34132297 3449 }
d11155ec 3450 else {
34132297 3451
d11155ec
KW
3452 # Here, we have a candidate for adjusting. What we do is
3453 # look through the subsequent adjacent elements in the
3454 # input. If the map to the next one differs by 1 from the
3455 # one before, then we combine into a larger range with the
3456 # initial map. Loop doing this until we find one that
3457 # can't be combined.
3458
3459 my $offset = 0; # How far away are we from the initial
3460 # map
3461 my $squished = 0; # ? Did we squish at least two
3462 # elements together into one range
3463 for ( ; $i < @ranges; $i++) {
3464 my ($next_hex_begin, $next_hex_end, $next_map)
3465 = split "\t", $ranges[$i+1];
3466
3467 # In the case of 'dm', the map may be a sequence of
3468 # multiple code points, which are never combined with
3469 # another range
3470 last if $next_map =~ / /;
3471
3472 $offset++;
3473 my $next_decimal_map = hex $next_map;
3474
3475 # If the next map is not next in sequence, it
3476 # shouldn't be combined.
3477 last if $next_decimal_map != $decimal_map + $offset;
3478
3479 my $next_begin = hex $next_hex_begin;
3480
3481 # Likewise, if the next element isn't adjacent to the
3482 # previous one, it shouldn't be combined.
3483 last if $next_begin != $begin + $offset;
3484
3485 my $next_end = (defined $next_hex_end
3486 && $next_hex_end ne "")
3487 ? hex $next_hex_end
3488 : $next_begin;
3489
3490 # And finally, if the next element is a multi-element
3491 # range, it shouldn't be combined.
3492 last if $next_end != $next_begin;
3493
3494 # Here, we will combine. Loop to see if we should
3495 # combine the next element too.
3496 $squished = 1;
3497 }
3498
3499 if ($squished) {
3500
3501 # Here, 'i' is the element number of the last element to
3502 # be combined, and the range is single-element, or we
3503 # wouldn't be combining. Get it's code point.
3504 my ($hex_end, undef, undef) = split "\t", $ranges[$i];
3505 $list .= "$hex_begin\t$hex_end\t$decimal_map\n";
3506 } else {
3507
13d1b68a 3508 # Here, no combining done. Just append the initial
d11155ec
KW
3509 # (and current) values.
3510 $list .= "$hex_begin\t\t$decimal_map\n";
3511 }
3512 }
3513 } # End of loop constructing the converted list
3514
3515 # Finish up the data structure for our converted swash
3516 my $type = ($second_try eq 'nfkccf') ? 'ToNFKCCF' : 'ToDm';
3517 $revised_swash{'LIST'} = $list;
3518 $revised_swash{'TYPE'} = $type;
3519 $revised_swash{'SPECIALS'} = $swash->{'SPECIALS'};
3520 $swash = \%revised_swash;
3521
3522 $utf8::SwashInfo{$type}{'missing'} = 0;
3523 $utf8::SwashInfo{$type}{'format'} = 'a';
62b3b855
KW
3524 }
3525 }
3526
3527 if ($swash->{'EXTRAS'}) {
3528 carp __PACKAGE__, "::prop_invmap: swash returned for $prop unexpectedly has EXTRAS magic";
3529 return;
3530 }
3531
3532 # Here, have a valid swash return. Examine it.
34132297 3533 my $returned_prop = $swash->{'TYPE'};
62b3b855
KW
3534
3535 # All properties but binary ones should have 'missing' and 'format'
3536 # entries
3537 $missing = $utf8::SwashInfo{$returned_prop}{'missing'};
3538 $missing = 'N' unless defined $missing;
3539
3540 $format = $utf8::SwashInfo{$returned_prop}{'format'};
3541 $format = 'b' unless defined $format;
3542
d11155ec
KW
3543 my $requires_adjustment = $format =~ /^a/;
3544
31aa6e0b
KW
3545 if ($swash->{'LIST'} =~ /^V/) {
3546 @invlist = split "\n", $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr;
3547 shift @invlist;
3548 foreach my $i (0 .. @invlist - 1) {
3549 $invmap[$i] = ($i % 2 == 0) ? 'Y' : 'N'
3550 }
3551
3552 # The map includes lines for all code points; add one for the range
3553 # from 0 to the first Y.
3554 if ($invlist[0] != 0) {
3555 unshift @invlist, 0;
3556 unshift @invmap, 'N';
3557 }
3558 }
3559 else {
1784d2f9
KW
3560 # The LIST input lines look like:
3561 # ...
3562 # 0374\t\tCommon
3563 # 0375\t0377\tGreek # [3]
3564 # 037A\t037D\tGreek # [4]
3565 # 037E\t\tCommon
3566 # 0384\t\tGreek
3567 # ...
92bcf67b 3568 #
1784d2f9
KW
3569 # Convert them to like
3570 # 0374 => Common
3571 # 0375 => Greek
3572 # 0378 => $missing
3573 # 037A => Greek
3574 # 037E => Common
3575 # 037F => $missing
3576 # 0384 => Greek
3577 #
3578 # For binary properties, the final non-comment column is absent, and
3579 # assumed to be 'Y'.
3580
3581 foreach my $range (split "\n", $swash->{'LIST'}) {
3582 $range =~ s/ \s* (?: \# .* )? $ //xg; # rmv trailing space, comments
3583
3584 # Find the beginning and end of the range on the line
3585 my ($hex_begin, $hex_end, $map) = split "\t", $range;
3586 my $begin = hex $hex_begin;
3587 no warnings 'portable';
3588 my $end = (defined $hex_end && $hex_end ne "")
3589 ? hex $hex_end
3590 : $begin;
3591
3592 # Each time through the loop (after the first):
3593 # $invlist[-2] contains the beginning of the previous range processed
3594 # $invlist[-1] contains the end+1 of the previous range processed
3595 # $invmap[-2] contains the value of the previous range processed
3596 # $invmap[-1] contains the default value for missing ranges
3597 # ($missing)
62b3b855 3598 #
1784d2f9
KW
3599 # Thus, things are set up for the typical case of a new
3600 # non-adjacent range of non-missings to be added. But, if the new
3601 # range is adjacent, it needs to replace the [-1] element; and if
3602 # the new range is a multiple value of the previous one, it needs
3603 # to be added to the [-2] map element.
3604
3605 # The first time through, everything will be empty. If the
3606 # property doesn't have a range that begins at 0, add one that
3607 # maps to $missing
3608 if (! @invlist) {
3609 if ($begin != 0) {
3610 push @invlist, 0;
3611 push @invmap, $missing;
3612 }
3613 }
3614 elsif (@invlist > 1 && $invlist[-2] == $begin) {
3615
3616 # Here we handle the case where the input has multiple entries
3617 # for each code point. mktables should have made sure that
3618 # each such range contains only one code point. At this
3619 # point, $invlist[-1] is the $missing that was added at the
3620 # end of the last loop iteration, and [-2] is the last real
3621 # input code point, and that code point is the same as the one
3622 # we are adding now, making the new one a multiple entry. Add
3623 # it to the existing entry, either by pushing it to the
3624 # existing list of multiple entries, or converting the single
3625 # current entry into a list with both on it. This is all we
3626 # need do for this iteration.
3627
3628 if ($end != $begin) {
3629 croak __PACKAGE__, ":prop_invmap: Multiple maps per code point in '$prop' require single-element ranges: begin=$begin, end=$end, map=$map";
3630 }
3631 if (! ref $invmap[-2]) {
3632 $invmap[-2] = [ $invmap[-2], $map ];
3633 }
3634 else {
3635 push @{$invmap[-2]}, $map;
3636 }
3637 $has_multiples = 1;
62b3b855
KW
3638 next;
3639 }
1784d2f9
KW
3640 elsif ($invlist[-1] == $begin) {
3641
3642 # If the input isn't in the most compact form, so that there
3643 # are two adjacent ranges that map to the same thing, they
3644 # should be combined (EXCEPT where the arrays require
3645 # adjustments, in which case everything is already set up
3646 # correctly). This happens in our constructed dt mapping, as
3647 # Element [-2] is the map for the latest range so far
3648 # processed. Just set the beginning point of the map to
3649 # $missing (in invlist[-1]) to 1 beyond where this range ends.
3650 # For example, in
3651 # 12\t13\tXYZ
3652 # 14\t17\tXYZ
3653 # we have set it up so that it looks like
3654 # 12 => XYZ
3655 # 14 => $missing
3656 #
3657 # We now see that it should be
3658 # 12 => XYZ
3659 # 18 => $missing
3660 if (! $requires_adjustment && @invlist > 1 && ( (defined $map)
3661 ? $invmap[-2] eq $map
3662 : $invmap[-2] eq 'Y'))
3663 {
3664 $invlist[-1] = $end + 1;
3665 next;
3666 }
62b3b855 3667
1784d2f9
KW
3668 # Here, the range started in the previous iteration that maps
3669 # to $missing starts at the same code point as this range.
3670 # That means there is no gap to fill that that range was
3671 # intended for, so we just pop it off the parallel arrays.
3672 pop @invlist;
3673 pop @invmap;
62b3b855 3674 }
1784d2f9
KW
3675
3676 # Add the range beginning, and the range's map.
3677 push @invlist, $begin;
3678 if ($returned_prop eq 'ToDm') {
3679
3680 # The decomposition maps are either a line like <hangul
3681 # syllable> which are to be taken as is; or a sequence of code
3682 # points in hex and separated by blanks. Convert them to
3683 # decimal, and if there is more than one, use an anonymous
3684 # array as the map.