This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
lib/Unicode/UCD.pm: Clarify pod
[perl5.git] / lib / Unicode / UCD.pm
CommitLineData
55d7b906 1package Unicode::UCD;
561c79ed
JH
2
3use strict;
4use warnings;
36c2430c 5no warnings 'surrogate'; # surrogates can be inputs to this
98ef7649 6use charnames ();
561c79ed 7
12cf36fe 8our $VERSION = '0.49';
561c79ed
JH
9
10require Exporter;
11
12our @ISA = qw(Exporter);
74f8133e 13
10a6ecd2
JH
14our @EXPORT_OK = qw(charinfo
15 charblock charscript
16 charblocks charscripts
b08cd201 17 charinrange
ea508aee 18 general_categories bidi_types
b08cd201 19 compexcl
66aa79e2 20 casefold all_casefolds casespec
7319f91d
KW
21 namedseq
22 num
7ef25837
KW
23 prop_aliases
24 prop_value_aliases
681d705c 25 prop_invlist
62b3b855 26 prop_invmap
681d705c 27 MAX_CP
7319f91d 28 );
561c79ed
JH
29
30use Carp;
31
32=head1 NAME
33
55d7b906 34Unicode::UCD - Unicode character database
561c79ed
JH
35
36=head1 SYNOPSIS
37
55d7b906 38 use Unicode::UCD 'charinfo';
b08cd201 39 my $charinfo = charinfo($codepoint);
561c79ed 40
956cae9a
KW
41 use Unicode::UCD 'casefold';
42 my $casefold = casefold(0xFB00);
43
66aa79e2
KW
44 use Unicode::UCD 'all_casefolds';
45 my $all_casefolds_ref = all_casefolds();
46
5d8e6e41
KW
47 use Unicode::UCD 'casespec';
48 my $casespec = casespec(0xFB00);
49
55d7b906 50 use Unicode::UCD 'charblock';
e882dd67
JH
51 my $charblock = charblock($codepoint);
52
55d7b906 53 use Unicode::UCD 'charscript';
65044554 54 my $charscript = charscript($codepoint);
561c79ed 55
55d7b906 56 use Unicode::UCD 'charblocks';
e145285f
JH
57 my $charblocks = charblocks();
58
55d7b906 59 use Unicode::UCD 'charscripts';
ea508aee 60 my $charscripts = charscripts();
e145285f 61
55d7b906 62 use Unicode::UCD qw(charscript charinrange);
e145285f
JH
63 my $range = charscript($script);
64 print "looks like $script\n" if charinrange($range, $codepoint);
65
ea508aee
JH
66 use Unicode::UCD qw(general_categories bidi_types);
67 my $categories = general_categories();
68 my $types = bidi_types();
69
7ef25837
KW
70 use Unicode::UCD 'prop_aliases';
71 my @space_names = prop_aliases("space");
72
73 use Unicode::UCD 'prop_value_aliases';
74 my @gc_punct_names = prop_value_aliases("Gc", "Punct");
75
681d705c
KW
76 use Unicode::UCD 'prop_invlist';
77 my @puncts = prop_invlist("gc=punctuation");
78
62b3b855
KW
79 use Unicode::UCD 'prop_invmap';
80 my ($list_ref, $map_ref, $format, $missing)
81 = prop_invmap("General Category");
82
55d7b906 83 use Unicode::UCD 'compexcl';
e145285f
JH
84 my $compexcl = compexcl($codepoint);
85
a2bd7410
JH
86 use Unicode::UCD 'namedseq';
87 my $namedseq = namedseq($named_sequence_name);
88
55d7b906 89 my $unicode_version = Unicode::UCD::UnicodeVersion();
e145285f 90
7319f91d 91 my $convert_to_numeric =
62a8c8c2 92 Unicode::UCD::num("\N{RUMI DIGIT ONE}\N{RUMI DIGIT TWO}");
7319f91d 93
561c79ed
JH
94=head1 DESCRIPTION
95
a452d459
KW
96The Unicode::UCD module offers a series of functions that
97provide a simple interface to the Unicode
8b731da2 98Character Database.
561c79ed 99
a452d459
KW
100=head2 code point argument
101
102Some of the functions are called with a I<code point argument>, which is either
103a decimal or a hexadecimal scalar designating a Unicode code point, or C<U+>
104followed by hexadecimals designating a Unicode code point. In other words, if
105you want a code point to be interpreted as a hexadecimal number, you must
106prefix it with either C<0x> or C<U+>, because a string like e.g. C<123> will be
f200dd12
KW
107interpreted as a decimal code point. Note that the largest code point in
108Unicode is U+10FFFF.
c3e5bc54 109
561c79ed
JH
110=cut
111
10a6ecd2 112my $BLOCKSFH;
10a6ecd2 113my $VERSIONFH;
b08cd201
JH
114my $CASEFOLDFH;
115my $CASESPECFH;
a2bd7410 116my $NAMEDSEQFH;
e80c2d9d 117my $v_unicode_version; # v-string.
561c79ed
JH
118
119sub openunicode {
120 my ($rfh, @path) = @_;
121 my $f;
122 unless (defined $$rfh) {
123 for my $d (@INC) {
124 use File::Spec;
55d7b906 125 $f = File::Spec->catfile($d, "unicore", @path);
32c16050 126 last if open($$rfh, $f);
e882dd67 127 undef $f;
561c79ed 128 }
e882dd67
JH
129 croak __PACKAGE__, ": failed to find ",
130 File::Spec->catfile(@path), " in @INC"
131 unless defined $f;
561c79ed
JH
132 }
133 return $f;
134}
135
cb3150f5
KW
136sub _dclone ($) { # Use Storable::dclone if available; otherwise emulate it.
137
138 use if defined &DynaLoader::boot_DynaLoader, Storable => qw(dclone);
139
140 return dclone(shift) if defined &dclone;
141
142 my $arg = shift;
143 my $type = ref $arg;
144 return $arg unless $type; # No deep cloning needed for scalars
145
146 if ($type eq 'ARRAY') {
147 my @return;
148 foreach my $element (@$arg) {
149 push @return, &_dclone($element);
150 }
151 return \@return;
152 }
153 elsif ($type eq 'HASH') {
154 my %return;
155 foreach my $key (keys %$arg) {
156 $return{$key} = &_dclone($arg->{$key});
157 }
158 return \%return;
159 }
160 else {
161 croak "_dclone can't handle " . $type;
162 }
163}
164
a452d459 165=head2 B<charinfo()>
561c79ed 166
55d7b906 167 use Unicode::UCD 'charinfo';
561c79ed 168
b08cd201 169 my $charinfo = charinfo(0x41);
561c79ed 170
a452d459
KW
171This returns information about the input L</code point argument>
172as a reference to a hash of fields as defined by the Unicode
173standard. If the L</code point argument> is not assigned in the standard
174(i.e., has the general category C<Cn> meaning C<Unassigned>)
175or is a non-character (meaning it is guaranteed to never be assigned in
176the standard),
a18e976f 177C<undef> is returned.
a452d459
KW
178
179Fields that aren't applicable to the particular code point argument exist in the
180returned hash, and are empty.
181
182The keys in the hash with the meanings of their values are:
183
184=over
185
186=item B<code>
187
188the input L</code point argument> expressed in hexadecimal, with leading zeros
189added if necessary to make it contain at least four hexdigits
190
191=item B<name>
192
193name of I<code>, all IN UPPER CASE.
194Some control-type code points do not have names.
195This field will be empty for C<Surrogate> and C<Private Use> code points,
196and for the others without a name,
197it will contain a description enclosed in angle brackets, like
198C<E<lt>controlE<gt>>.
199
200
201=item B<category>
202
203The short name of the general category of I<code>.
204This will match one of the keys in the hash returned by L</general_categories()>.
205
7ef25837
KW
206The L</prop_value_aliases()> function can be used to get all the synonyms
207of the category name.
208
a452d459
KW
209=item B<combining>
210
211the combining class number for I<code> used in the Canonical Ordering Algorithm.
212For Unicode 5.1, this is described in Section 3.11 C<Canonical Ordering Behavior>
213available at
214L<http://www.unicode.org/versions/Unicode5.1.0/>
215
7ef25837
KW
216The L</prop_value_aliases()> function can be used to get all the synonyms
217of the combining class number.
218
a452d459
KW
219=item B<bidi>
220
221bidirectional type of I<code>.
222This will match one of the keys in the hash returned by L</bidi_types()>.
223
7ef25837
KW
224The L</prop_value_aliases()> function can be used to get all the synonyms
225of the bidi type name.
226
a452d459
KW
227=item B<decomposition>
228
229is empty if I<code> has no decomposition; or is one or more codes
a18e976f 230(separated by spaces) that, taken in order, represent a decomposition for
a452d459
KW
231I<code>. Each has at least four hexdigits.
232The codes may be preceded by a word enclosed in angle brackets then a space,
233like C<E<lt>compatE<gt> >, giving the type of decomposition
234
06bba7d5
KW
235This decomposition may be an intermediate one whose components are also
236decomposable. Use L<Unicode::Normalize> to get the final decomposition.
237
a452d459
KW
238=item B<decimal>
239
240if I<code> is a decimal digit this is its integer numeric value
241
242=item B<digit>
243
89e4a205
KW
244if I<code> represents some other digit-like number, this is its integer
245numeric value
a452d459
KW
246
247=item B<numeric>
248
249if I<code> represents a whole or rational number, this is its numeric value.
250Rational values are expressed as a string like C<1/4>.
251
252=item B<mirrored>
253
254C<Y> or C<N> designating if I<code> is mirrored in bidirectional text
255
256=item B<unicode10>
257
258name of I<code> in the Unicode 1.0 standard if one
259existed for this code point and is different from the current name
260
261=item B<comment>
262
89e4a205 263As of Unicode 6.0, this is always empty.
a452d459
KW
264
265=item B<upper>
266
06bba7d5 267is empty if there is no single code point uppercase mapping for I<code>
4f66642e 268(its uppercase mapping is itself);
a452d459
KW
269otherwise it is that mapping expressed as at least four hexdigits.
270(L</casespec()> should be used in addition to B<charinfo()>
271for case mappings when the calling program can cope with multiple code point
272mappings.)
273
274=item B<lower>
275
06bba7d5 276is empty if there is no single code point lowercase mapping for I<code>
4f66642e 277(its lowercase mapping is itself);
a452d459
KW
278otherwise it is that mapping expressed as at least four hexdigits.
279(L</casespec()> should be used in addition to B<charinfo()>
280for case mappings when the calling program can cope with multiple code point
281mappings.)
282
283=item B<title>
284
06bba7d5 285is empty if there is no single code point titlecase mapping for I<code>
4f66642e 286(its titlecase mapping is itself);
a452d459
KW
287otherwise it is that mapping expressed as at least four hexdigits.
288(L</casespec()> should be used in addition to B<charinfo()>
289for case mappings when the calling program can cope with multiple code point
290mappings.)
291
292=item B<block>
293
a18e976f 294the block I<code> belongs to (used in C<\p{Blk=...}>).
a452d459
KW
295See L</Blocks versus Scripts>.
296
297
298=item B<script>
299
a18e976f 300the script I<code> belongs to.
a452d459
KW
301See L</Blocks versus Scripts>.
302
303=back
32c16050
JH
304
305Note that you cannot do (de)composition and casing based solely on the
a452d459
KW
306I<decomposition>, I<combining>, I<lower>, I<upper>, and I<title> fields;
307you will need also the L</compexcl()>, and L</casespec()> functions.
561c79ed
JH
308
309=cut
310
e10d7780 311# NB: This function is nearly duplicated in charnames.pm
10a6ecd2
JH
312sub _getcode {
313 my $arg = shift;
314
dc0a4417 315 if ($arg =~ /^[1-9]\d*$/) {
10a6ecd2 316 return $arg;
dc0a4417 317 } elsif ($arg =~ /^(?:[Uu]\+|0[xX])?([[:xdigit:]]+)$/) {
10a6ecd2
JH
318 return hex($1);
319 }
320
321 return;
322}
323
05dbc6f8
KW
324# Populated by _num. Converts real number back to input rational
325my %real_to_rational;
326
327# To store the contents of files found on disk.
328my @BIDIS;
329my @CATEGORIES;
330my @DECOMPOSITIONS;
331my @NUMERIC_TYPES;
5c3b35c9
KW
332my %SIMPLE_LOWER;
333my %SIMPLE_TITLE;
334my %SIMPLE_UPPER;
335my %UNICODE_1_NAMES;
72fcb9f0 336my %ISO_COMMENT;
05dbc6f8 337
05dbc6f8 338sub charinfo {
a6fa416b 339
05dbc6f8
KW
340 # This function has traditionally mimicked what is in UnicodeData.txt,
341 # warts and all. This is a re-write that avoids UnicodeData.txt so that
342 # it can be removed to save disk space. Instead, this assembles
343 # information gotten by other methods that get data from various other
344 # files. It uses charnames to get the character name; and various
345 # mktables tables.
324f9e44 346
05dbc6f8 347 use feature 'unicode_strings';
a6fa416b 348
cb3150f5
KW
349 # Will fail if called under minitest
350 use if defined &DynaLoader::boot_DynaLoader, "Unicode::Normalize" => qw(getCombinClass NFD);
351
10a6ecd2
JH
352 my $arg = shift;
353 my $code = _getcode($arg);
05dbc6f8
KW
354 croak __PACKAGE__, "::charinfo: unknown code '$arg'" unless defined $code;
355
356 # Non-unicode implies undef.
357 return if $code > 0x10FFFF;
358
359 my %prop;
360 my $char = chr($code);
361
35a865d4 362 @CATEGORIES =_read_table("To/Gc.pl") unless @CATEGORIES;
05dbc6f8
KW
363 $prop{'category'} = _search(\@CATEGORIES, 0, $#CATEGORIES, $code)
364 // $utf8::SwashInfo{'ToGc'}{'missing'};
365
366 return if $prop{'category'} eq 'Cn'; # Unassigned code points are undef
367
368 $prop{'code'} = sprintf "%04X", $code;
369 $prop{'name'} = ($char =~ /\p{Cntrl}/) ? '<control>'
370 : (charnames::viacode($code) // "");
371
372 $prop{'combining'} = getCombinClass($code);
373
35a865d4 374 @BIDIS =_read_table("To/Bc.pl") unless @BIDIS;
05dbc6f8
KW
375 $prop{'bidi'} = _search(\@BIDIS, 0, $#BIDIS, $code)
376 // $utf8::SwashInfo{'ToBc'}{'missing'};
377
378 # For most code points, we can just read in "unicore/Decomposition.pl", as
379 # its contents are exactly what should be output. But that file doesn't
380 # contain the data for the Hangul syllable decompositions, which can be
94c91ffc
KW
381 # algorithmically computed, and NFD() does that, so we call NFD() for
382 # those. We can't use NFD() for everything, as it does a complete
05dbc6f8 383 # recursive decomposition, and what this function has always done is to
94c91ffc
KW
384 # return what's in UnicodeData.txt which doesn't show that recursiveness.
385 # Fortunately, the NFD() of the Hanguls doesn't have any recursion
386 # issues.
387 # Having no decomposition implies an empty field; otherwise, all but
388 # "Canonical" imply a compatible decomposition, and the type is prefixed
389 # to that, as it is in UnicodeData.txt
9abdc62b
KW
390 UnicodeVersion() unless defined $v_unicode_version;
391 if ($v_unicode_version ge v2.0.0 && $char =~ /\p{Block=Hangul_Syllables}/) {
05dbc6f8
KW
392 # The code points of the decomposition are output in standard Unicode
393 # hex format, separated by blanks.
394 $prop{'decomposition'} = join " ", map { sprintf("%04X", $_)}
94c91ffc 395 unpack "U*", NFD($char);
a6fa416b 396 }
05dbc6f8 397 else {
35a865d4 398 @DECOMPOSITIONS = _read_table("Decomposition.pl")
05dbc6f8
KW
399 unless @DECOMPOSITIONS;
400 $prop{'decomposition'} = _search(\@DECOMPOSITIONS, 0, $#DECOMPOSITIONS,
401 $code) // "";
561c79ed 402 }
05dbc6f8
KW
403
404 # Can use num() to get the numeric values, if any.
405 if (! defined (my $value = num($char))) {
406 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = "";
407 }
408 else {
409 if ($char =~ /\d/) {
410 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = $value;
411 }
412 else {
413
414 # For non-decimal-digits, we have to read in the Numeric type
415 # to distinguish them. It is not just a matter of integer vs.
416 # rational, as some whole number values are not considered digits,
417 # e.g., TAMIL NUMBER TEN.
418 $prop{'decimal'} = "";
419
35a865d4 420 @NUMERIC_TYPES =_read_table("To/Nt.pl") unless @NUMERIC_TYPES;
05dbc6f8
KW
421 if ((_search(\@NUMERIC_TYPES, 0, $#NUMERIC_TYPES, $code) // "")
422 eq 'Digit')
423 {
424 $prop{'digit'} = $prop{'numeric'} = $value;
425 }
426 else {
427 $prop{'digit'} = "";
428 $prop{'numeric'} = $real_to_rational{$value} // $value;
429 }
430 }
431 }
432
433 $prop{'mirrored'} = ($char =~ /\p{Bidi_Mirrored}/) ? 'Y' : 'N';
434
35a865d4 435 %UNICODE_1_NAMES =_read_table("To/Na1.pl", "use_hash") unless %UNICODE_1_NAMES;
5c3b35c9 436 $prop{'unicode10'} = $UNICODE_1_NAMES{$code} // "";
05dbc6f8 437
72fcb9f0
KW
438 UnicodeVersion() unless defined $v_unicode_version;
439 if ($v_unicode_version ge v6.0.0) {
440 $prop{'comment'} = "";
441 }
442 else {
443 %ISO_COMMENT = _read_table("To/Isc.pl", "use_hash") unless %ISO_COMMENT;
444 $prop{'comment'} = (defined $ISO_COMMENT{$code})
445 ? $ISO_COMMENT{$code}
446 : "";
447 }
05dbc6f8 448
35a865d4 449 %SIMPLE_UPPER = _read_table("To/Uc.pl", "use_hash") unless %SIMPLE_UPPER;
bf7fe2df 450 $prop{'upper'} = (defined $SIMPLE_UPPER{$code})
d11155ec 451 ? sprintf("%04X", $SIMPLE_UPPER{$code})
bf7fe2df 452 : "";
75e7c50b 453
35a865d4 454 %SIMPLE_LOWER = _read_table("To/Lc.pl", "use_hash") unless %SIMPLE_LOWER;
bf7fe2df 455 $prop{'lower'} = (defined $SIMPLE_LOWER{$code})
d11155ec 456 ? sprintf("%04X", $SIMPLE_LOWER{$code})
bf7fe2df 457 : "";
75e7c50b 458
35a865d4 459 %SIMPLE_TITLE = _read_table("To/Tc.pl", "use_hash") unless %SIMPLE_TITLE;
bf7fe2df 460 $prop{'title'} = (defined $SIMPLE_TITLE{$code})
d11155ec 461 ? sprintf("%04X", $SIMPLE_TITLE{$code})
bf7fe2df 462 : "";
05dbc6f8
KW
463
464 $prop{block} = charblock($code);
465 $prop{script} = charscript($code);
466 return \%prop;
561c79ed
JH
467}
468
e882dd67
JH
469sub _search { # Binary search in a [[lo,hi,prop],[...],...] table.
470 my ($table, $lo, $hi, $code) = @_;
471
472 return if $lo > $hi;
473
474 my $mid = int(($lo+$hi) / 2);
475
476 if ($table->[$mid]->[0] < $code) {
10a6ecd2 477 if ($table->[$mid]->[1] >= $code) {
e882dd67
JH
478 return $table->[$mid]->[2];
479 } else {
480 _search($table, $mid + 1, $hi, $code);
481 }
482 } elsif ($table->[$mid]->[0] > $code) {
483 _search($table, $lo, $mid - 1, $code);
484 } else {
485 return $table->[$mid]->[2];
486 }
487}
488
cb366075 489sub _read_table ($;$) {
3a12600d
KW
490
491 # Returns the contents of the mktables generated table file located at $1
cb366075
KW
492 # in the form of either an array of arrays or a hash, depending on if the
493 # optional second parameter is true (for hash return) or not. In the case
494 # of a hash return, each key is a code point, and its corresponding value
495 # is what the table gives as the code point's corresponding value. In the
496 # case of an array return, each outer array denotes a range with [0] the
497 # start point of that range; [1] the end point; and [2] the value that
498 # every code point in the range has. The hash return is useful for fast
499 # lookup when the table contains only single code point ranges. The array
500 # return takes much less memory when there are large ranges.
3a12600d 501 #
cb366075 502 # This function has the side effect of setting
3a12600d
KW
503 # $utf8::SwashInfo{$property}{'format'} to be the mktables format of the
504 # table; and
505 # $utf8::SwashInfo{$property}{'missing'} to be the value for all entries
506 # not listed in the table.
507 # where $property is the Unicode property name, preceded by 'To' for map
508 # properties., e.g., 'ToSc'.
509 #
510 # Table entries look like one of:
511 # 0000 0040 Common # [65]
512 # 00AA Latin
513
514 my $table = shift;
cb366075
KW
515 my $return_hash = shift;
516 $return_hash = 0 unless defined $return_hash;
3a12600d 517 my @return;
cb366075 518 my %return;
3a12600d 519 local $_;
d11155ec 520 my $list = do "unicore/$table";
3a12600d 521
d11155ec
KW
522 # Look up if this property requires adjustments, which we do below if it
523 # does.
524 require "unicore/Heavy.pl";
525 my $property = $table =~ s/\.pl//r;
526 $property = $utf8::file_to_swash_name{$property};
527 my $to_adjust = defined $property
528 && $utf8::SwashInfo{$property}{'format'} eq 'a';
529
530 for (split /^/m, $list) {
3a12600d
KW
531 my ($start, $end, $value) = / ^ (.+?) \t (.*?) \t (.+?)
532 \s* ( \# .* )? # Optional comment
533 $ /x;
83fd1222
KW
534 my $decimal_start = hex $start;
535 my $decimal_end = ($end eq "") ? $decimal_start : hex $end;
cb366075 536 if ($return_hash) {
83fd1222 537 foreach my $i ($decimal_start .. $decimal_end) {
d11155ec
KW
538 $return{$i} = ($to_adjust)
539 ? $value + $i - $decimal_start
540 : $value;
cb366075
KW
541 }
542 }
d11155ec
KW
543 elsif (! $to_adjust
544 && @return
545 && $return[-1][1] == $decimal_start - 1
9a96c106
KW
546 && $return[-1][2] eq $value)
547 {
548 # If this is merely extending the previous range, do just that.
549 $return[-1]->[1] = $decimal_end;
550 }
cb366075 551 else {
83fd1222 552 push @return, [ $decimal_start, $decimal_end, $value ];
cb366075 553 }
3a12600d 554 }
cb366075 555 return ($return_hash) ? %return : @return;
3a12600d
KW
556}
557
10a6ecd2
JH
558sub charinrange {
559 my ($range, $arg) = @_;
560 my $code = _getcode($arg);
561 croak __PACKAGE__, "::charinrange: unknown code '$arg'"
562 unless defined $code;
563 _search($range, 0, $#$range, $code);
564}
565
a452d459 566=head2 B<charblock()>
561c79ed 567
55d7b906 568 use Unicode::UCD 'charblock';
561c79ed
JH
569
570 my $charblock = charblock(0x41);
10a6ecd2 571 my $charblock = charblock(1234);
a452d459 572 my $charblock = charblock(0x263a);
10a6ecd2
JH
573 my $charblock = charblock("U+263a");
574
78bf21c2 575 my $range = charblock('Armenian');
10a6ecd2 576
a452d459 577With a L</code point argument> charblock() returns the I<block> the code point
430fe03d
KW
578belongs to, e.g. C<Basic Latin>. The old-style block name is returned (see
579L</Old-style versus new-style block names>).
a452d459 580If the code point is unassigned, this returns the block it would belong to if
fe252ba7
KW
581it were assigned. (If the Unicode version being used is so early as to not
582have blocks, all code points are considered to be in C<No_Block>.)
10a6ecd2 583
78bf21c2
JH
584See also L</Blocks versus Scripts>.
585
18972f4b 586If supplied with an argument that can't be a code point, charblock() tries to
430fe03d
KW
587do the opposite and interpret the argument as an old-style block name. The
588return value
a18e976f
KW
589is a I<range set> with one range: an anonymous list with a single element that
590consists of another anonymous list whose first element is the first code point
591in the block, and whose second (and final) element is the final code point in
592the block. (The extra list consisting of just one element is so that the same
593program logic can be used to handle both this return, and the return from
594L</charscript()> which can have multiple ranges.) You can test whether a code
595point is in a range using the L</charinrange()> function. If the argument is
596not a known block, C<undef> is returned.
561c79ed 597
561c79ed
JH
598=cut
599
600my @BLOCKS;
10a6ecd2 601my %BLOCKS;
561c79ed 602
10a6ecd2 603sub _charblocks {
06bba7d5
KW
604
605 # Can't read from the mktables table because it loses the hyphens in the
606 # original.
561c79ed 607 unless (@BLOCKS) {
fe252ba7
KW
608 UnicodeVersion() unless defined $v_unicode_version;
609 if ($v_unicode_version lt v2.0.0) {
610 my $subrange = [ 0, 0x10FFFF, 'No_Block' ];
611 push @BLOCKS, $subrange;
612 push @{$BLOCKS{$3}}, $subrange;
613 }
614 elsif (openunicode(\$BLOCKSFH, "Blocks.txt")) {
6c8d78fb 615 local $_;
ce066323 616 local $/ = "\n";
10a6ecd2 617 while (<$BLOCKSFH>) {
2796c109 618 if (/^([0-9A-F]+)\.\.([0-9A-F]+);\s+(.+)/) {
10a6ecd2
JH
619 my ($lo, $hi) = (hex($1), hex($2));
620 my $subrange = [ $lo, $hi, $3 ];
621 push @BLOCKS, $subrange;
622 push @{$BLOCKS{$3}}, $subrange;
561c79ed
JH
623 }
624 }
10a6ecd2 625 close($BLOCKSFH);
561c79ed
JH
626 }
627 }
10a6ecd2
JH
628}
629
630sub charblock {
631 my $arg = shift;
632
633 _charblocks() unless @BLOCKS;
634
635 my $code = _getcode($arg);
561c79ed 636
10a6ecd2 637 if (defined $code) {
c707cf8e
KW
638 my $result = _search(\@BLOCKS, 0, $#BLOCKS, $code);
639 return $result if defined $result;
640 return 'No_Block';
641 }
642 elsif (exists $BLOCKS{$arg}) {
cb3150f5 643 return _dclone $BLOCKS{$arg};
10a6ecd2 644 }
e882dd67
JH
645}
646
a452d459 647=head2 B<charscript()>
e882dd67 648
55d7b906 649 use Unicode::UCD 'charscript';
e882dd67
JH
650
651 my $charscript = charscript(0x41);
10a6ecd2
JH
652 my $charscript = charscript(1234);
653 my $charscript = charscript("U+263a");
e882dd67 654
78bf21c2 655 my $range = charscript('Thai');
10a6ecd2 656
a452d459
KW
657With a L</code point argument> charscript() returns the I<script> the
658code point belongs to, e.g. C<Latin>, C<Greek>, C<Han>.
49ea58c8
KW
659If the code point is unassigned or the Unicode version being used is so early
660that it doesn't have scripts, this function returns C<"Unknown">.
78bf21c2 661
eb0cc9e3 662If supplied with an argument that can't be a code point, charscript() tries
a18e976f
KW
663to do the opposite and interpret the argument as a script name. The
664return value is a I<range set>: an anonymous list of lists that contain
eb0cc9e3 665I<start-of-range>, I<end-of-range> code point pairs. You can test whether a
a18e976f
KW
666code point is in a range set using the L</charinrange()> function. If the
667argument is not a known script, C<undef> is returned.
a452d459
KW
668
669See also L</Blocks versus Scripts>.
e882dd67 670
e882dd67
JH
671=cut
672
673my @SCRIPTS;
10a6ecd2 674my %SCRIPTS;
e882dd67 675
10a6ecd2 676sub _charscripts {
49ea58c8
KW
677 unless (@SCRIPTS) {
678 UnicodeVersion() unless defined $v_unicode_version;
679 if ($v_unicode_version lt v3.1.0) {
680 push @SCRIPTS, [ 0, 0x10FFFF, 'Unknown' ];
681 }
682 else {
683 @SCRIPTS =_read_table("To/Sc.pl");
684 }
685 }
7bccef0b 686 foreach my $entry (@SCRIPTS) {
f3d50ac9 687 $entry->[2] =~ s/(_\w)/\L$1/g; # Preserve old-style casing
7bccef0b 688 push @{$SCRIPTS{$entry->[2]}}, $entry;
e882dd67 689 }
10a6ecd2
JH
690}
691
692sub charscript {
693 my $arg = shift;
694
695 _charscripts() unless @SCRIPTS;
e882dd67 696
10a6ecd2
JH
697 my $code = _getcode($arg);
698
699 if (defined $code) {
7bccef0b
KW
700 my $result = _search(\@SCRIPTS, 0, $#SCRIPTS, $code);
701 return $result if defined $result;
8079ad82 702 return $utf8::SwashInfo{'ToSc'}{'missing'};
7bccef0b 703 } elsif (exists $SCRIPTS{$arg}) {
cb3150f5 704 return _dclone $SCRIPTS{$arg};
10a6ecd2 705 }
7bccef0b
KW
706
707 return;
10a6ecd2
JH
708}
709
a452d459 710=head2 B<charblocks()>
10a6ecd2 711
55d7b906 712 use Unicode::UCD 'charblocks';
10a6ecd2 713
b08cd201 714 my $charblocks = charblocks();
10a6ecd2 715
b08cd201 716charblocks() returns a reference to a hash with the known block names
a452d459 717as the keys, and the code point ranges (see L</charblock()>) as the values.
10a6ecd2 718
430fe03d
KW
719The names are in the old-style (see L</Old-style versus new-style block
720names>).
721
62b3b855
KW
722L<prop_invmap("block")|/prop_invmap()> can be used to get this same data in a
723different type of data structure.
724
78bf21c2
JH
725See also L</Blocks versus Scripts>.
726
10a6ecd2
JH
727=cut
728
729sub charblocks {
b08cd201 730 _charblocks() unless %BLOCKS;
cb3150f5 731 return _dclone \%BLOCKS;
10a6ecd2
JH
732}
733
a452d459 734=head2 B<charscripts()>
10a6ecd2 735
55d7b906 736 use Unicode::UCD 'charscripts';
10a6ecd2 737
ea508aee 738 my $charscripts = charscripts();
10a6ecd2 739
ea508aee 740charscripts() returns a reference to a hash with the known script
a452d459 741names as the keys, and the code point ranges (see L</charscript()>) as
ea508aee 742the values.
10a6ecd2 743
62b3b855
KW
744L<prop_invmap("script")|/prop_invmap()> can be used to get this same data in a
745different type of data structure.
746
78bf21c2
JH
747See also L</Blocks versus Scripts>.
748
10a6ecd2
JH
749=cut
750
751sub charscripts {
b08cd201 752 _charscripts() unless %SCRIPTS;
cb3150f5 753 return _dclone \%SCRIPTS;
561c79ed
JH
754}
755
a452d459 756=head2 B<charinrange()>
10a6ecd2 757
f200dd12 758In addition to using the C<\p{Blk=...}> and C<\P{Blk=...}> constructs, you
10a6ecd2 759can also test whether a code point is in the I<range> as returned by
a452d459
KW
760L</charblock()> and L</charscript()> or as the values of the hash returned
761by L</charblocks()> and L</charscripts()> by using charinrange():
10a6ecd2 762
55d7b906 763 use Unicode::UCD qw(charscript charinrange);
10a6ecd2
JH
764
765 $range = charscript('Hiragana');
e145285f 766 print "looks like hiragana\n" if charinrange($range, $codepoint);
10a6ecd2
JH
767
768=cut
769
ea508aee
JH
770my %GENERAL_CATEGORIES =
771 (
772 'L' => 'Letter',
773 'LC' => 'CasedLetter',
774 'Lu' => 'UppercaseLetter',
775 'Ll' => 'LowercaseLetter',
776 'Lt' => 'TitlecaseLetter',
777 'Lm' => 'ModifierLetter',
778 'Lo' => 'OtherLetter',
779 'M' => 'Mark',
780 'Mn' => 'NonspacingMark',
781 'Mc' => 'SpacingMark',
782 'Me' => 'EnclosingMark',
783 'N' => 'Number',
784 'Nd' => 'DecimalNumber',
785 'Nl' => 'LetterNumber',
786 'No' => 'OtherNumber',
787 'P' => 'Punctuation',
788 'Pc' => 'ConnectorPunctuation',
789 'Pd' => 'DashPunctuation',
790 'Ps' => 'OpenPunctuation',
791 'Pe' => 'ClosePunctuation',
792 'Pi' => 'InitialPunctuation',
793 'Pf' => 'FinalPunctuation',
794 'Po' => 'OtherPunctuation',
795 'S' => 'Symbol',
796 'Sm' => 'MathSymbol',
797 'Sc' => 'CurrencySymbol',
798 'Sk' => 'ModifierSymbol',
799 'So' => 'OtherSymbol',
800 'Z' => 'Separator',
801 'Zs' => 'SpaceSeparator',
802 'Zl' => 'LineSeparator',
803 'Zp' => 'ParagraphSeparator',
804 'C' => 'Other',
805 'Cc' => 'Control',
806 'Cf' => 'Format',
807 'Cs' => 'Surrogate',
808 'Co' => 'PrivateUse',
809 'Cn' => 'Unassigned',
810 );
811
812sub general_categories {
cb3150f5 813 return _dclone \%GENERAL_CATEGORIES;
ea508aee
JH
814}
815
a452d459 816=head2 B<general_categories()>
ea508aee
JH
817
818 use Unicode::UCD 'general_categories';
819
820 my $categories = general_categories();
821
a452d459 822This returns a reference to a hash which has short
ea508aee
JH
823general category names (such as C<Lu>, C<Nd>, C<Zs>, C<S>) as keys and long
824names (such as C<UppercaseLetter>, C<DecimalNumber>, C<SpaceSeparator>,
825C<Symbol>) as values. The hash is reversible in case you need to go
826from the long names to the short names. The general category is the
a452d459
KW
827one returned from
828L</charinfo()> under the C<category> key.
ea508aee 829
7ef25837
KW
830The L</prop_value_aliases()> function can be used to get all the synonyms of
831the category name.
832
ea508aee
JH
833=cut
834
835my %BIDI_TYPES =
836 (
837 'L' => 'Left-to-Right',
838 'LRE' => 'Left-to-Right Embedding',
839 'LRO' => 'Left-to-Right Override',
840 'R' => 'Right-to-Left',
841 'AL' => 'Right-to-Left Arabic',
842 'RLE' => 'Right-to-Left Embedding',
843 'RLO' => 'Right-to-Left Override',
844 'PDF' => 'Pop Directional Format',
845 'EN' => 'European Number',
846 'ES' => 'European Number Separator',
847 'ET' => 'European Number Terminator',
848 'AN' => 'Arabic Number',
849 'CS' => 'Common Number Separator',
850 'NSM' => 'Non-Spacing Mark',
851 'BN' => 'Boundary Neutral',
852 'B' => 'Paragraph Separator',
853 'S' => 'Segment Separator',
854 'WS' => 'Whitespace',
855 'ON' => 'Other Neutrals',
856 );
857
a452d459 858=head2 B<bidi_types()>
ea508aee
JH
859
860 use Unicode::UCD 'bidi_types';
861
862 my $categories = bidi_types();
863
a452d459 864This returns a reference to a hash which has the short
ea508aee
JH
865bidi (bidirectional) type names (such as C<L>, C<R>) as keys and long
866names (such as C<Left-to-Right>, C<Right-to-Left>) as values. The
867hash is reversible in case you need to go from the long names to the
a452d459
KW
868short names. The bidi type is the one returned from
869L</charinfo()>
ea508aee
JH
870under the C<bidi> key. For the exact meaning of the various bidi classes
871the Unicode TR9 is recommended reading:
a452d459 872L<http://www.unicode.org/reports/tr9/>
ea508aee
JH
873(as of Unicode 5.0.0)
874
7ef25837
KW
875The L</prop_value_aliases()> function can be used to get all the synonyms of
876the bidi type name.
877
ea508aee
JH
878=cut
879
a452d459 880sub bidi_types {
cb3150f5 881 return _dclone \%BIDI_TYPES;
a452d459
KW
882}
883
884=head2 B<compexcl()>
b08cd201 885
55d7b906 886 use Unicode::UCD 'compexcl';
b08cd201 887
a452d459 888 my $compexcl = compexcl(0x09dc);
b08cd201 889
2afba6a4
KW
890This routine returns C<undef> if the Unicode version being used is so early
891that it doesn't have this property. It is included for backwards
892compatibility, but as of Perl 5.12 and more modern Unicode versions, for
71a442a8
KW
893most purposes it is probably more convenient to use one of the following
894instead:
895
896 my $compexcl = chr(0x09dc) =~ /\p{Comp_Ex};
897 my $compexcl = chr(0x09dc) =~ /\p{Full_Composition_Exclusion};
898
899or even
900
901 my $compexcl = chr(0x09dc) =~ /\p{CE};
902 my $compexcl = chr(0x09dc) =~ /\p{Composition_Exclusion};
903
904The first two forms return B<true> if the L</code point argument> should not
76b05678
KW
905be produced by composition normalization. For the final two forms to return
906B<true>, it is additionally required that this fact not otherwise be
907determinable from the Unicode data base.
71a442a8
KW
908
909This routine behaves identically to the final two forms. That is,
910it does not return B<true> if the code point has a decomposition
a452d459
KW
911consisting of another single code point, nor if its decomposition starts
912with a code point whose combining class is non-zero. Code points that meet
913either of these conditions should also not be produced by composition
71a442a8
KW
914normalization, which is probably why you should use the
915C<Full_Composition_Exclusion> property instead, as shown above.
b08cd201 916
71a442a8 917The routine returns B<false> otherwise.
b08cd201
JH
918
919=cut
920
b08cd201
JH
921sub compexcl {
922 my $arg = shift;
923 my $code = _getcode($arg);
74f8133e
JH
924 croak __PACKAGE__, "::compexcl: unknown code '$arg'"
925 unless defined $code;
b08cd201 926
2afba6a4
KW
927 UnicodeVersion() unless defined $v_unicode_version;
928 return if $v_unicode_version lt v3.0.0;
929
36c2430c 930 no warnings "non_unicode"; # So works on non-Unicode code points
71a442a8 931 return chr($code) =~ /\p{Composition_Exclusion}/;
b08cd201
JH
932}
933
a452d459 934=head2 B<casefold()>
b08cd201 935
55d7b906 936 use Unicode::UCD 'casefold';
b08cd201 937
a452d459
KW
938 my $casefold = casefold(0xDF);
939 if (defined $casefold) {
940 my @full_fold_hex = split / /, $casefold->{'full'};
941 my $full_fold_string =
942 join "", map {chr(hex($_))} @full_fold_hex;
943 my @turkic_fold_hex =
944 split / /, ($casefold->{'turkic'} ne "")
945 ? $casefold->{'turkic'}
946 : $casefold->{'full'};
947 my $turkic_fold_string =
948 join "", map {chr(hex($_))} @turkic_fold_hex;
949 }
950 if (defined $casefold && $casefold->{'simple'} ne "") {
951 my $simple_fold_hex = $casefold->{'simple'};
952 my $simple_fold_string = chr(hex($simple_fold_hex));
953 }
b08cd201 954
a452d459 955This returns the (almost) locale-independent case folding of the
6329003c
KW
956character specified by the L</code point argument>. (Starting in Perl v5.16,
957the core function C<fc()> returns the C<full> mapping (described below)
958faster than this does, and for entire strings.)
b08cd201 959
6329003c 960If there is no case folding for the input code point, C<undef> is returned.
a452d459
KW
961
962If there is a case folding for that code point, a reference to a hash
b08cd201
JH
963with the following fields is returned:
964
a452d459
KW
965=over
966
967=item B<code>
968
969the input L</code point argument> expressed in hexadecimal, with leading zeros
970added if necessary to make it contain at least four hexdigits
971
972=item B<full>
973
a18e976f 974one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
975code points for the case folding for I<code>.
976Each has at least four hexdigits.
977
978=item B<simple>
979
980is empty, or is exactly one code with at least four hexdigits which can be used
981as an alternative case folding when the calling program cannot cope with the
982fold being a sequence of multiple code points. If I<full> is just one code
983point, then I<simple> equals I<full>. If there is no single code point folding
984defined for I<code>, then I<simple> is the empty string. Otherwise, it is an
985inferior, but still better-than-nothing alternative folding to I<full>.
986
987=item B<mapping>
988
989is the same as I<simple> if I<simple> is not empty, and it is the same as I<full>
990otherwise. It can be considered to be the simplest possible folding for
991I<code>. It is defined primarily for backwards compatibility.
992
993=item B<status>
b08cd201 994
a452d459
KW
995is C<C> (for C<common>) if the best possible fold is a single code point
996(I<simple> equals I<full> equals I<mapping>). It is C<S> if there are distinct
997folds, I<simple> and I<full> (I<mapping> equals I<simple>). And it is C<F> if
a18e976f
KW
998there is only a I<full> fold (I<mapping> equals I<full>; I<simple> is empty).
999Note that this
a452d459
KW
1000describes the contents of I<mapping>. It is defined primarily for backwards
1001compatibility.
b08cd201 1002
6329003c 1003For Unicode versions between 3.1 and 3.1.1 inclusive, I<status> can also be
a452d459
KW
1004C<I> which is the same as C<C> but is a special case for dotted uppercase I and
1005dotless lowercase i:
b08cd201 1006
a452d459 1007=over
b08cd201 1008
f703fc96 1009=item Z<>B<*> If you use this C<I> mapping
a452d459 1010
a18e976f 1011the result is case-insensitive,
a452d459
KW
1012but dotless and dotted I's are not distinguished
1013
f703fc96 1014=item Z<>B<*> If you exclude this C<I> mapping
a452d459 1015
a18e976f 1016the result is not fully case-insensitive, but
a452d459
KW
1017dotless and dotted I's are distinguished
1018
1019=back
1020
1021=item B<turkic>
1022
1023contains any special folding for Turkic languages. For versions of Unicode
1024starting with 3.2, this field is empty unless I<code> has a different folding
1025in Turkic languages, in which case it is one or more codes (separated by
a18e976f 1026spaces) that, taken in order, give the code points for the case folding for
a452d459
KW
1027I<code> in those languages.
1028Each code has at least four hexdigits.
1029Note that this folding does not maintain canonical equivalence without
1030additional processing.
1031
6329003c
KW
1032For Unicode versions between 3.1 and 3.1.1 inclusive, this field is empty unless
1033there is a
a452d459
KW
1034special folding for Turkic languages, in which case I<status> is C<I>, and
1035I<mapping>, I<full>, I<simple>, and I<turkic> are all equal.
1036
1037=back
1038
1039Programs that want complete generality and the best folding results should use
1040the folding contained in the I<full> field. But note that the fold for some
1041code points will be a sequence of multiple code points.
1042
1043Programs that can't cope with the fold mapping being multiple code points can
1044use the folding contained in the I<simple> field, with the loss of some
1045generality. In Unicode 5.1, about 7% of the defined foldings have no single
1046code point folding.
1047
1048The I<mapping> and I<status> fields are provided for backwards compatibility for
1049existing programs. They contain the same values as in previous versions of
1050this function.
1051
1052Locale is not completely independent. The I<turkic> field contains results to
1053use when the locale is a Turkic language.
b08cd201
JH
1054
1055For more information about case mappings see
a452d459 1056L<http://www.unicode.org/unicode/reports/tr21>
b08cd201
JH
1057
1058=cut
1059
1060my %CASEFOLD;
1061
1062sub _casefold {
727c62ff
KW
1063 unless (%CASEFOLD) { # Populate the hash
1064 my ($full_invlist_ref, $full_invmap_ref, undef, $default)
1065 = prop_invmap('Case_Folding');
1066
1067 # Use the recipe given in the prop_invmap() pod to convert the
1068 # inversion map into the hash.
1069 for my $i (0 .. @$full_invlist_ref - 1 - 1) {
1070 next if $full_invmap_ref->[$i] == $default;
1071 my $adjust = -1;
1072 for my $j ($full_invlist_ref->[$i] .. $full_invlist_ref->[$i+1] -1) {
1073 $adjust++;
1074 if (! ref $full_invmap_ref->[$i]) {
1075
1076 # This is a single character mapping
1077 $CASEFOLD{$j}{'status'} = 'C';
1078 $CASEFOLD{$j}{'simple'}
1079 = $CASEFOLD{$j}{'full'}
1080 = $CASEFOLD{$j}{'mapping'}
1081 = sprintf("%04X", $full_invmap_ref->[$i] + $adjust);
1082 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1083 $CASEFOLD{$j}{'turkic'} = "";
1084 }
1085 else { # prop_invmap ensures that $adjust is 0 for a ref
1086 $CASEFOLD{$j}{'status'} = 'F';
1087 $CASEFOLD{$j}{'full'}
1088 = $CASEFOLD{$j}{'mapping'}
1089 = join " ", map { sprintf "%04X", $_ }
1090 @{$full_invmap_ref->[$i]};
1091 $CASEFOLD{$j}{'simple'} = "";
1092 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1093 $CASEFOLD{$j}{'turkic'} = "";
1094 }
1095 }
1096 }
1097
1098 # We have filled in the full mappings above, assuming there were no
1099 # simple ones for the ones with multi-character maps. Now, we find
1100 # and fix the cases where that assumption was false.
1101 (my ($simple_invlist_ref, $simple_invmap_ref, undef), $default)
1102 = prop_invmap('Simple_Case_Folding');
1103 for my $i (0 .. @$simple_invlist_ref - 1 - 1) {
1104 next if $simple_invmap_ref->[$i] == $default;
1105 my $adjust = -1;
1106 for my $j ($simple_invlist_ref->[$i]
1107 .. $simple_invlist_ref->[$i+1] -1)
1108 {
1109 $adjust++;
1110 next if $CASEFOLD{$j}{'status'} eq 'C';
1111 $CASEFOLD{$j}{'status'} = 'S';
1112 $CASEFOLD{$j}{'simple'}
1113 = $CASEFOLD{$j}{'mapping'}
1114 = sprintf("%04X", $simple_invmap_ref->[$i] + $adjust);
1115 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1116 $CASEFOLD{$j}{'turkic'} = "";
1117 }
1118 }
1119
1120 # We hard-code in the turkish rules
1121 UnicodeVersion() unless defined $v_unicode_version;
1122 if ($v_unicode_version ge v3.2.0) {
1123
1124 # These two code points should already have regular entries, so
1125 # just fill in the turkish fields
1126 $CASEFOLD{ord('I')}{'turkic'} = '0131';
1127 $CASEFOLD{0x130}{'turkic'} = sprintf "%04X", ord('i');
1128 }
1129 elsif ($v_unicode_version ge v3.1.0) {
1130
1131 # These two code points don't have entries otherwise.
1132 $CASEFOLD{0x130}{'code'} = '0130';
1133 $CASEFOLD{0x131}{'code'} = '0131';
1134 $CASEFOLD{0x130}{'status'} = $CASEFOLD{0x131}{'status'} = 'I';
1135 $CASEFOLD{0x130}{'turkic'}
1136 = $CASEFOLD{0x130}{'mapping'}
1137 = $CASEFOLD{0x130}{'full'}
1138 = $CASEFOLD{0x130}{'simple'}
1139 = $CASEFOLD{0x131}{'turkic'}
1140 = $CASEFOLD{0x131}{'mapping'}
1141 = $CASEFOLD{0x131}{'full'}
1142 = $CASEFOLD{0x131}{'simple'}
1143 = sprintf "%04X", ord('i');
1144 }
b08cd201
JH
1145 }
1146}
1147
1148sub casefold {
1149 my $arg = shift;
1150 my $code = _getcode($arg);
74f8133e
JH
1151 croak __PACKAGE__, "::casefold: unknown code '$arg'"
1152 unless defined $code;
b08cd201
JH
1153
1154 _casefold() unless %CASEFOLD;
1155
1156 return $CASEFOLD{$code};
1157}
1158
66aa79e2
KW
1159=head2 B<all_casefolds()>
1160
1161
1162 use Unicode::UCD 'all_casefolds';
1163
1164 my $all_folds_ref = all_casefolds();
1165 foreach my $char_with_casefold (sort { $a <=> $b }
1166 keys %$all_folds_ref)
1167 {
1168 printf "%04X:", $char_with_casefold;
1169 my $casefold = $all_folds_ref->{$char_with_casefold};
1170
1171 # Get folds for $char_with_casefold
1172
1173 my @full_fold_hex = split / /, $casefold->{'full'};
1174 my $full_fold_string =
1175 join "", map {chr(hex($_))} @full_fold_hex;
1176 print " full=", join " ", @full_fold_hex;
1177 my @turkic_fold_hex =
1178 split / /, ($casefold->{'turkic'} ne "")
1179 ? $casefold->{'turkic'}
1180 : $casefold->{'full'};
1181 my $turkic_fold_string =
1182 join "", map {chr(hex($_))} @turkic_fold_hex;
1183 print "; turkic=", join " ", @turkic_fold_hex;
1184 if (defined $casefold && $casefold->{'simple'} ne "") {
1185 my $simple_fold_hex = $casefold->{'simple'};
1186 my $simple_fold_string = chr(hex($simple_fold_hex));
1187 print "; simple=$simple_fold_hex";
1188 }
1189 print "\n";
1190 }
1191
1192This returns all the case foldings in the current version of Unicode in the
1193form of a reference to a hash. Each key to the hash is the decimal
1194representation of a Unicode character that has a casefold to other than
1195itself. The casefold of a semi-colon is itself, so it isn't in the hash;
1196likewise for a lowercase "a", but there is an entry for a capital "A". The
1197hash value for each key is another hash, identical to what is returned by
1198L</casefold()> if called with that code point as its argument. So the value
1199C<< all_casefolds()->{ord("A")}' >> is equivalent to C<casefold(ord("A"))>;
1200
1201=cut
1202
1203sub all_casefolds () {
1204 _casefold() unless %CASEFOLD;
1205 return _dclone \%CASEFOLD;
1206}
1207
a452d459 1208=head2 B<casespec()>
b08cd201 1209
55d7b906 1210 use Unicode::UCD 'casespec';
b08cd201 1211
a452d459 1212 my $casespec = casespec(0xFB00);
b08cd201 1213
a452d459
KW
1214This returns the potentially locale-dependent case mappings of the L</code point
1215argument>. The mappings may be longer than a single code point (which the basic
1216Unicode case mappings as returned by L</charinfo()> never are).
b08cd201 1217
a452d459
KW
1218If there are no case mappings for the L</code point argument>, or if all three
1219possible mappings (I<lower>, I<title> and I<upper>) result in single code
a18e976f 1220points and are locale independent and unconditional, C<undef> is returned
5d8e6e41
KW
1221(which means that the case mappings, if any, for the code point are those
1222returned by L</charinfo()>).
a452d459
KW
1223
1224Otherwise, a reference to a hash giving the mappings (or a reference to a hash
5d8e6e41
KW
1225of such hashes, explained below) is returned with the following keys and their
1226meanings:
a452d459
KW
1227
1228The keys in the bottom layer hash with the meanings of their values are:
1229
1230=over
1231
1232=item B<code>
1233
1234the input L</code point argument> expressed in hexadecimal, with leading zeros
1235added if necessary to make it contain at least four hexdigits
1236
1237=item B<lower>
1238
a18e976f 1239one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1240code points for the lower case of I<code>.
1241Each has at least four hexdigits.
1242
1243=item B<title>
b08cd201 1244
a18e976f 1245one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1246code points for the title case of I<code>.
1247Each has at least four hexdigits.
b08cd201 1248
d2da20e3 1249=item B<upper>
b08cd201 1250
a18e976f 1251one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1252code points for the upper case of I<code>.
1253Each has at least four hexdigits.
1254
1255=item B<condition>
1256
1257the conditions for the mappings to be valid.
a18e976f 1258If C<undef>, the mappings are always valid.
a452d459
KW
1259When defined, this field is a list of conditions,
1260all of which must be true for the mappings to be valid.
1261The list consists of one or more
1262I<locales> (see below)
1263and/or I<contexts> (explained in the next paragraph),
1264separated by spaces.
1265(Other than as used to separate elements, spaces are to be ignored.)
1266Case distinctions in the condition list are not significant.
82c0b05b 1267Conditions preceded by "NON_" represent the negation of the condition.
b08cd201 1268
a452d459
KW
1269A I<context> is one of those defined in the Unicode standard.
1270For Unicode 5.1, they are defined in Section 3.13 C<Default Case Operations>
1271available at
5d8e6e41
KW
1272L<http://www.unicode.org/versions/Unicode5.1.0/>.
1273These are for context-sensitive casing.
f499c386 1274
a452d459
KW
1275=back
1276
5d8e6e41 1277The hash described above is returned for locale-independent casing, where
a18e976f 1278at least one of the mappings has length longer than one. If C<undef> is
5d8e6e41
KW
1279returned, the code point may have mappings, but if so, all are length one,
1280and are returned by L</charinfo()>.
1281Note that when this function does return a value, it will be for the complete
1282set of mappings for a code point, even those whose length is one.
1283
1284If there are additional casing rules that apply only in certain locales,
1285an additional key for each will be defined in the returned hash. Each such key
1286will be its locale name, defined as a 2-letter ISO 3166 country code, possibly
1287followed by a "_" and a 2-letter ISO language code (possibly followed by a "_"
1288and a variant code). You can find the lists of all possible locales, see
1289L<Locale::Country> and L<Locale::Language>.
89e4a205 1290(In Unicode 6.0, the only locales returned by this function
a452d459 1291are C<lt>, C<tr>, and C<az>.)
b08cd201 1292
5d8e6e41
KW
1293Each locale key is a reference to a hash that has the form above, and gives
1294the casing rules for that particular locale, which take precedence over the
1295locale-independent ones when in that locale.
1296
1297If the only casing for a code point is locale-dependent, then the returned
1298hash will not have any of the base keys, like C<code>, C<upper>, etc., but
1299will contain only locale keys.
1300
b08cd201 1301For more information about case mappings see
a452d459 1302L<http://www.unicode.org/unicode/reports/tr21/>
b08cd201
JH
1303
1304=cut
1305
1306my %CASESPEC;
1307
1308sub _casespec {
1309 unless (%CASESPEC) {
f033d3cd
KW
1310 UnicodeVersion() unless defined $v_unicode_version;
1311 if ($v_unicode_version lt v2.1.8) {
1312 %CASESPEC = {};
1313 }
1314 elsif (openunicode(\$CASESPECFH, "SpecialCasing.txt")) {
6c8d78fb 1315 local $_;
ce066323 1316 local $/ = "\n";
b08cd201
JH
1317 while (<$CASESPECFH>) {
1318 if (/^([0-9A-F]+); ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; (\w+(?: \w+)*)?/) {
f033d3cd 1319
f499c386
JH
1320 my ($hexcode, $lower, $title, $upper, $condition) =
1321 ($1, $2, $3, $4, $5);
1322 my $code = hex($hexcode);
f033d3cd
KW
1323
1324 # In 2.1.8, there were duplicate entries; ignore all but
1325 # the first one -- there were no conditions in the file
1326 # anyway.
1327 if (exists $CASESPEC{$code} && $v_unicode_version ne v2.1.8)
1328 {
f499c386
JH
1329 if (exists $CASESPEC{$code}->{code}) {
1330 my ($oldlower,
1331 $oldtitle,
1332 $oldupper,
1333 $oldcondition) =
1334 @{$CASESPEC{$code}}{qw(lower
1335 title
1336 upper
1337 condition)};
822ebcc8
JH
1338 if (defined $oldcondition) {
1339 my ($oldlocale) =
f499c386 1340 ($oldcondition =~ /^([a-z][a-z](?:_\S+)?)/);
f499c386
JH
1341 delete $CASESPEC{$code};
1342 $CASESPEC{$code}->{$oldlocale} =
1343 { code => $hexcode,
1344 lower => $oldlower,
1345 title => $oldtitle,
1346 upper => $oldupper,
1347 condition => $oldcondition };
f499c386
JH
1348 }
1349 }
1350 my ($locale) =
1351 ($condition =~ /^([a-z][a-z](?:_\S+)?)/);
1352 $CASESPEC{$code}->{$locale} =
1353 { code => $hexcode,
1354 lower => $lower,
1355 title => $title,
1356 upper => $upper,
1357 condition => $condition };
1358 } else {
1359 $CASESPEC{$code} =
1360 { code => $hexcode,
1361 lower => $lower,
1362 title => $title,
1363 upper => $upper,
1364 condition => $condition };
1365 }
b08cd201
JH
1366 }
1367 }
1368 close($CASESPECFH);
1369 }
1370 }
1371}
1372
1373sub casespec {
1374 my $arg = shift;
1375 my $code = _getcode($arg);
74f8133e
JH
1376 croak __PACKAGE__, "::casespec: unknown code '$arg'"
1377 unless defined $code;
b08cd201
JH
1378
1379 _casespec() unless %CASESPEC;
1380
cb3150f5 1381 return ref $CASESPEC{$code} ? _dclone $CASESPEC{$code} : $CASESPEC{$code};
b08cd201
JH
1382}
1383
a452d459 1384=head2 B<namedseq()>
a2bd7410
JH
1385
1386 use Unicode::UCD 'namedseq';
1387
1388 my $namedseq = namedseq("KATAKANA LETTER AINU P");
1389 my @namedseq = namedseq("KATAKANA LETTER AINU P");
1390 my %namedseq = namedseq();
1391
1392If used with a single argument in a scalar context, returns the string
a18e976f 1393consisting of the code points of the named sequence, or C<undef> if no
a2bd7410 1394named sequence by that name exists. If used with a single argument in
956cae9a
KW
1395a list context, it returns the list of the ordinals of the code points. If used
1396with no
a2bd7410
JH
1397arguments in a list context, returns a hash with the names of the
1398named sequences as the keys and the named sequences as strings as
a18e976f 1399the values. Otherwise, it returns C<undef> or an empty list depending
a2bd7410
JH
1400on the context.
1401
a452d459
KW
1402This function only operates on officially approved (not provisional) named
1403sequences.
a2bd7410 1404
27f853a0
KW
1405Note that as of Perl 5.14, C<\N{KATAKANA LETTER AINU P}> will insert the named
1406sequence into double-quoted strings, and C<charnames::string_vianame("KATAKANA
1407LETTER AINU P")> will return the same string this function does, but will also
1408operate on character names that aren't named sequences, without you having to
1409know which are which. See L<charnames>.
1410
a2bd7410
JH
1411=cut
1412
1413my %NAMEDSEQ;
1414
1415sub _namedseq {
1416 unless (%NAMEDSEQ) {
98ef7649 1417 if (openunicode(\$NAMEDSEQFH, "Name.pl")) {
a2bd7410 1418 local $_;
ce066323 1419 local $/ = "\n";
a2bd7410 1420 while (<$NAMEDSEQFH>) {
98ef7649
KW
1421 if (/^ [0-9A-F]+ \ /x) {
1422 chomp;
1423 my ($sequence, $name) = split /\t/;
1424 my @s = map { chr(hex($_)) } split(' ', $sequence);
1425 $NAMEDSEQ{$name} = join("", @s);
a2bd7410
JH
1426 }
1427 }
1428 close($NAMEDSEQFH);
1429 }
1430 }
1431}
1432
1433sub namedseq {
98ef7649
KW
1434
1435 # Use charnames::string_vianame() which now returns this information,
1436 # unless the caller wants the hash returned, in which case we read it in,
1437 # and thereafter use it instead of calling charnames, as it is faster.
1438
a2bd7410
JH
1439 my $wantarray = wantarray();
1440 if (defined $wantarray) {
1441 if ($wantarray) {
1442 if (@_ == 0) {
98ef7649 1443 _namedseq() unless %NAMEDSEQ;
a2bd7410
JH
1444 return %NAMEDSEQ;
1445 } elsif (@_ == 1) {
98ef7649
KW
1446 my $s;
1447 if (%NAMEDSEQ) {
1448 $s = $NAMEDSEQ{ $_[0] };
1449 }
1450 else {
1451 $s = charnames::string_vianame($_[0]);
1452 }
a2bd7410
JH
1453 return defined $s ? map { ord($_) } split('', $s) : ();
1454 }
1455 } elsif (@_ == 1) {
98ef7649
KW
1456 return $NAMEDSEQ{ $_[0] } if %NAMEDSEQ;
1457 return charnames::string_vianame($_[0]);
a2bd7410
JH
1458 }
1459 }
1460 return;
1461}
1462
7319f91d
KW
1463my %NUMERIC;
1464
1465sub _numeric {
35a865d4 1466 my @numbers = _read_table("To/Nv.pl");
98025745
KW
1467 foreach my $entry (@numbers) {
1468 my ($start, $end, $value) = @$entry;
1469
05dbc6f8
KW
1470 # If value contains a slash, convert to decimal, add a reverse hash
1471 # used by charinfo.
98025745
KW
1472 if ((my @rational = split /\//, $value) == 2) {
1473 my $real = $rational[0] / $rational[1];
05dbc6f8 1474 $real_to_rational{$real} = $value;
98025745 1475 $value = $real;
98025745 1476
4f143a72
KW
1477 # Should only be single element, but just in case...
1478 for my $i ($start .. $end) {
1479 $NUMERIC{$i} = $value;
1480 }
1481 }
1482 else {
1483 # The values require adjusting, as is in 'a' format
1484 for my $i ($start .. $end) {
1485 $NUMERIC{$i} = $value + $i - $start;
1486 }
7319f91d 1487 }
7319f91d 1488 }
2dc5eb26
KW
1489
1490 # Decided unsafe to use these that aren't officially part of the Unicode
1491 # standard.
1492 #use Math::Trig;
1493 #my $pi = acos(-1.0);
98025745 1494 #$NUMERIC{0x03C0} = $pi;
7319f91d
KW
1495
1496 # Euler's constant, not to be confused with Euler's number
98025745 1497 #$NUMERIC{0x2107} = 0.57721566490153286060651209008240243104215933593992;
7319f91d
KW
1498
1499 # Euler's number
98025745 1500 #$NUMERIC{0x212F} = 2.7182818284590452353602874713526624977572;
2dc5eb26 1501
7319f91d
KW
1502 return;
1503}
1504
1505=pod
1506
67592e11 1507=head2 B<num()>
7319f91d 1508
eefd7bc2
KW
1509 use Unicode::UCD 'num';
1510
1511 my $val = num("123");
1512 my $one_quarter = num("\N{VULGAR FRACTION 1/4}");
1513
7319f91d
KW
1514C<num> returns the numeric value of the input Unicode string; or C<undef> if it
1515doesn't think the entire string has a completely valid, safe numeric value.
1516
1517If the string is just one character in length, the Unicode numeric value
1518is returned if it has one, or C<undef> otherwise. Note that this need
1519not be a whole number. C<num("\N{TIBETAN DIGIT HALF ZERO}")>, for
2dc5eb26
KW
1520example returns -0.5.
1521
1522=cut
7319f91d 1523
2dc5eb26
KW
1524#A few characters to which Unicode doesn't officially
1525#assign a numeric value are considered numeric by C<num>.
1526#These are:
1527
1528# EULER CONSTANT 0.5772... (this is NOT Euler's number)
1529# SCRIPT SMALL E 2.71828... (this IS Euler's number)
1530# GREEK SMALL LETTER PI 3.14159...
1531
1532=pod
7319f91d
KW
1533
1534If the string is more than one character, C<undef> is returned unless
8bb4c8e2 1535all its characters are decimal digits (that is, they would match C<\d+>),
7319f91d
KW
1536from the same script. For example if you have an ASCII '0' and a Bengali
1537'3', mixed together, they aren't considered a valid number, and C<undef>
1538is returned. A further restriction is that the digits all have to be of
1539the same form. A half-width digit mixed with a full-width one will
1540return C<undef>. The Arabic script has two sets of digits; C<num> will
1541return C<undef> unless all the digits in the string come from the same
1542set.
1543
1544C<num> errs on the side of safety, and there may be valid strings of
1545decimal digits that it doesn't recognize. Note that Unicode defines
1546a number of "digit" characters that aren't "decimal digit" characters.
a278d14b 1547"Decimal digits" have the property that they have a positional value, i.e.,
7319f91d
KW
1548there is a units position, a 10's position, a 100's, etc, AND they are
1549arranged in Unicode in blocks of 10 contiguous code points. The Chinese
1550digits, for example, are not in such a contiguous block, and so Unicode
1551doesn't view them as decimal digits, but merely digits, and so C<\d> will not
1552match them. A single-character string containing one of these digits will
1553have its decimal value returned by C<num>, but any longer string containing
1554only these digits will return C<undef>.
1555
a278d14b
KW
1556Strings of multiple sub- and superscripts are not recognized as numbers. You
1557can use either of the compatibility decompositions in Unicode::Normalize to
7319f91d
KW
1558change these into digits, and then call C<num> on the result.
1559
1560=cut
1561
1562# To handle sub, superscripts, this could if called in list context,
1563# consider those, and return the <decomposition> type in the second
1564# array element.
1565
1566sub num {
1567 my $string = $_[0];
1568
1569 _numeric unless %NUMERIC;
1570
1571 my $length = length($string);
98025745 1572 return $NUMERIC{ord($string)} if $length == 1;
7319f91d
KW
1573 return if $string =~ /\D/;
1574 my $first_ord = ord(substr($string, 0, 1));
98025745 1575 my $value = $NUMERIC{$first_ord};
5522af1c
KW
1576
1577 # To be a valid decimal number, it should be in a block of 10 consecutive
1578 # characters, whose values are 0, 1, 2, ... 9. Therefore this digit's
1579 # value is its offset in that block from the character that means zero.
7319f91d
KW
1580 my $zero_ord = $first_ord - $value;
1581
5522af1c
KW
1582 # Unicode 6.0 instituted the rule that only digits in a consecutive
1583 # block of 10 would be considered decimal digits. If this is an earlier
1584 # release, we verify that this first character is a member of such a
1585 # block. That is, that the block of characters surrounding this one
1586 # consists of all \d characters whose numeric values are the expected
1587 # ones.
1588 UnicodeVersion() unless defined $v_unicode_version;
1589 if ($v_unicode_version lt v6.0.0) {
1590 for my $i (0 .. 9) {
1591 my $ord = $zero_ord + $i;
1592 return unless chr($ord) =~ /\d/;
1593 my $numeric = $NUMERIC{$ord};
1594 return unless defined $numeric;
1595 return unless $numeric == $i;
1596 }
1597 }
1598
7319f91d 1599 for my $i (1 .. $length -1) {
5522af1c
KW
1600
1601 # Here we know either by verifying, or by fact of the first character
1602 # being a \d in Unicode 6.0 or later, that any character between the
1603 # character that means 0, and 9 positions above it must be \d, and
1604 # must have its value correspond to its offset from the zero. Any
1605 # characters outside these 10 do not form a legal number for this
1606 # function.
7319f91d
KW
1607 my $ord = ord(substr($string, $i, 1));
1608 my $digit = $ord - $zero_ord;
1609 return unless $digit >= 0 && $digit <= 9;
1610 $value = $value * 10 + $digit;
1611 }
5522af1c 1612
7319f91d
KW
1613 return $value;
1614}
1615
7ef25837
KW
1616=pod
1617
1618=head2 B<prop_aliases()>
1619
1620 use Unicode::UCD 'prop_aliases';
1621
1622 my ($short_name, $full_name, @other_names) = prop_aliases("space");
1623 my $same_full_name = prop_aliases("Space"); # Scalar context
1624 my ($same_short_name) = prop_aliases("Space"); # gets 0th element
1625 print "The full name is $full_name\n";
1626 print "The short name is $short_name\n";
1627 print "The other aliases are: ", join(", ", @other_names), "\n";
1628
1629 prints:
1630 The full name is White_Space
1631 The short name is WSpace
1632 The other aliases are: Space
1633
1634Most Unicode properties have several synonymous names. Typically, there is at
1635least a short name, convenient to type, and a long name that more fully
1636describes the property, and hence is more easily understood.
1637
1638If you know one name for a Unicode property, you can use C<prop_aliases> to find
1639either the long name (when called in scalar context), or a list of all of the
1640names, somewhat ordered so that the short name is in the 0th element, the long
1641name in the next element, and any other synonyms are in the remaining
1642elements, in no particular order.
1643
1644The long name is returned in a form nicely capitalized, suitable for printing.
1645
1646The input parameter name is loosely matched, which means that white space,
1647hyphens, and underscores are ignored (except for the trailing underscore in
1648the old_form grandfathered-in C<"L_">, which is better written as C<"LC">, and
1649both of which mean C<General_Category=Cased Letter>).
1650
1651If the name is unknown, C<undef> is returned (or an empty list in list
1652context). Note that Perl typically recognizes property names in regular
1653expressions with an optional C<"Is_>" (with or without the underscore)
1654prefixed to them, such as C<\p{isgc=punct}>. This function does not recognize
1655those in the input, returning C<undef>. Nor are they included in the output
1656as possible synonyms.
1657
1658C<prop_aliases> does know about the Perl extensions to Unicode properties,
1659such as C<Any> and C<XPosixAlpha>, and the single form equivalents to Unicode
1660properties such as C<XDigit>, C<Greek>, C<In_Greek>, and C<Is_Greek>. The
1661final example demonstrates that the C<"Is_"> prefix is recognized for these
1662extensions; it is needed to resolve ambiguities. For example,
1663C<prop_aliases('lc')> returns the list C<(lc, Lowercase_Mapping)>, but
1664C<prop_aliases('islc')> returns C<(Is_LC, Cased_Letter)>. This is
1665because C<islc> is a Perl extension which is short for
1666C<General_Category=Cased Letter>. The lists returned for the Perl extensions
1667will not include the C<"Is_"> prefix (whether or not the input had it) unless
1668needed to resolve ambiguities, as shown in the C<"islc"> example, where the
1669returned list had one element containing C<"Is_">, and the other without.
1670
1671It is also possible for the reverse to happen: C<prop_aliases('isc')> returns
1672the list C<(isc, ISO_Comment)>; whereas C<prop_aliases('c')> returns
1673C<(C, Other)> (the latter being a Perl extension meaning
ee94c7d1
KW
1674C<General_Category=Other>.
1675L<perluniprops/Properties accessible through Unicode::UCD> lists the available
1676forms, including which ones are discouraged from use.
7ef25837
KW
1677
1678Those discouraged forms are accepted as input to C<prop_aliases>, but are not
1679returned in the lists. C<prop_aliases('isL&')> and C<prop_aliases('isL_')>,
1680which are old synonyms for C<"Is_LC"> and should not be used in new code, are
1681examples of this. These both return C<(Is_LC, Cased_Letter)>. Thus this
1682function allows you to take a discourarged form, and find its acceptable
1683alternatives. The same goes with single-form Block property equivalences.
1684Only the forms that begin with C<"In_"> are not discouraged; if you pass
1685C<prop_aliases> a discouraged form, you will get back the equivalent ones that
1686begin with C<"In_">. It will otherwise look like a new-style block name (see.
1687L</Old-style versus new-style block names>).
1688
1689C<prop_aliases> does not know about any user-defined properties, and will
1690return C<undef> if called with one of those. Likewise for Perl internal
1691properties, with the exception of "Perl_Decimal_Digit" which it does know
1692about (and which is documented below in L</prop_invmap()>).
1693
1694=cut
1695
1696# It may be that there are use cases where the discouraged forms should be
1697# returned. If that comes up, an optional boolean second parameter to the
1698# function could be created, for example.
1699
1700# These are created by mktables for this routine and stored in unicore/UCD.pl
1701# where their structures are described.
1702our %string_property_loose_to_name;
1703our %ambiguous_names;
1704our %loose_perlprop_to_name;
1705our %prop_aliases;
1706
1707sub prop_aliases ($) {
1708 my $prop = $_[0];
1709 return unless defined $prop;
1710
1711 require "unicore/UCD.pl";
1712 require "unicore/Heavy.pl";
1713 require "utf8_heavy.pl";
1714
1715 # The property name may be loosely or strictly matched; we don't know yet.
1716 # But both types use lower-case.
1717 $prop = lc $prop;
1718
1719 # It is loosely matched if its lower case isn't known to be strict.
1720 my $list_ref;
1721 if (! exists $utf8::stricter_to_file_of{$prop}) {
1722 my $loose = utf8::_loose_name($prop);
1723
1724 # There is a hash that converts from any loose name to its standard
1725 # form, mapping all synonyms for a name to one name that can be used
1726 # as a key into another hash. The whole concept is for memory
1727 # savings, as the second hash doesn't have to have all the
1728 # combinations. Actually, there are two hashes that do the
1729 # converstion. One is used in utf8_heavy.pl (stored in Heavy.pl) for
1730 # looking up properties matchable in regexes. This function needs to
1731 # access string properties, which aren't available in regexes, so a
1732 # second conversion hash is made for them (stored in UCD.pl). Look in
1733 # the string one now, as the rest can have an optional 'is' prefix,
1734 # which these don't.
1735 if (exists $string_property_loose_to_name{$loose}) {
1736
1737 # Convert to its standard loose name.
1738 $prop = $string_property_loose_to_name{$loose};
1739 }
1740 else {
1741 my $retrying = 0; # bool. ? Has an initial 'is' been stripped
1742 RETRY:
1743 if (exists $utf8::loose_property_name_of{$loose}
1744 && (! $retrying
1745 || ! exists $ambiguous_names{$loose}))
1746 {
1747 # Found an entry giving the standard form. We don't get here
1748 # (in the test above) when we've stripped off an
1749 # 'is' and the result is an ambiguous name. That is because
1750 # these are official Unicode properties (though Perl can have
1751 # an optional 'is' prefix meaning the official property), and
1752 # all ambiguous cases involve a Perl single-form extension
1753 # for the gc, script, or block properties, and the stripped
1754 # 'is' means that they mean one of those, and not one of
1755 # these
1756 $prop = $utf8::loose_property_name_of{$loose};
1757 }
1758 elsif (exists $loose_perlprop_to_name{$loose}) {
1759
1760 # This hash is specifically for this function to list Perl
1761 # extensions that aren't in the earlier hashes. If there is
1762 # only one element, the short and long names are identical.
1763 # Otherwise the form is already in the same form as
1764 # %prop_aliases, which is handled at the end of the function.
1765 $list_ref = $loose_perlprop_to_name{$loose};
1766 if (@$list_ref == 1) {
1767 my @list = ($list_ref->[0], $list_ref->[0]);
1768 $list_ref = \@list;
1769 }
1770 }
1771 elsif (! exists $utf8::loose_to_file_of{$loose}) {
1772
1773 # loose_to_file_of is a complete list of loose names. If not
1774 # there, the input is unknown.
1775 return;
1776 }
1777 else {
1778
1779 # Here we found the name but not its aliases, so it has to
1780 # exist. This means it must be one of the Perl single-form
1781 # extensions. First see if it is for a property-value
1782 # combination in one of the following properties.
1783 my @list;
1784 foreach my $property ("gc", "script") {
1785 @list = prop_value_aliases($property, $loose);
1786 last if @list;
1787 }
1788 if (@list) {
1789
1790 # Here, it is one of those property-value combination
1791 # single-form synonyms. There are ambiguities with some
1792 # of these. Check against the list for these, and adjust
1793 # if necessary.
1794 for my $i (0 .. @list -1) {
1795 if (exists $ambiguous_names
1796 {utf8::_loose_name(lc $list[$i])})
1797 {
1798 # The ambiguity is resolved by toggling whether or
1799 # not it has an 'is' prefix
1800 $list[$i] =~ s/^Is_// or $list[$i] =~ s/^/Is_/;
1801 }
1802 }
1803 return @list;
1804 }
1805
1806 # Here, it wasn't one of the gc or script single-form
1807 # extensions. It could be a block property single-form
1808 # extension. An 'in' prefix definitely means that, and should
2a4f2769
KW
1809 # be looked up without the prefix. However, starting in
1810 # Unicode 6.1, we have to special case 'indic...', as there
1811 # is a property that begins with that name. We shouldn't
1812 # strip the 'in' from that. I'm (khw) generalizing this to
1813 # 'indic' instead of the single property, because I suspect
1814 # that others of this class may come along in the future.
1815 # However, this could backfire and a block created whose name
1816 # begins with 'dic...', and we would want to strip the 'in'.
1817 # At which point this would have to be tweaked.
1818 my $began_with_in = $loose =~ s/^in(?!dic)//;
7ef25837
KW
1819 @list = prop_value_aliases("block", $loose);
1820 if (@list) {
1821 map { $_ =~ s/^/In_/ } @list;
1822 return @list;
1823 }
1824
1825 # Here still haven't found it. The last opportunity for it
1826 # being valid is only if it began with 'is'. We retry without
1827 # the 'is', setting a flag to that effect so that we don't
1828 # accept things that begin with 'isis...'
1829 if (! $retrying && ! $began_with_in && $loose =~ s/^is//) {
1830 $retrying = 1;
1831 goto RETRY;
1832 }
1833
1834 # Here, didn't find it. Since it was in %loose_to_file_of, we
1835 # should have been able to find it.
1836 carp __PACKAGE__, "::prop_aliases: Unexpectedly could not find '$prop'. Send bug report to perlbug\@perl.org";
1837 return;
1838 }
1839 }
1840 }
1841
1842 if (! $list_ref) {
1843 # Here, we have set $prop to a standard form name of the input. Look
1844 # it up in the structure created by mktables for this purpose, which
1845 # contains both strict and loosely matched properties. Avoid
1846 # autovivifying.
1847 $list_ref = $prop_aliases{$prop} if exists $prop_aliases{$prop};
1848 return unless $list_ref;
1849 }
1850
1851 # The full name is in element 1.
1852 return $list_ref->[1] unless wantarray;
1853
cb3150f5 1854 return @{_dclone $list_ref};
7ef25837
KW
1855}
1856
1857=pod
1858
1859=head2 B<prop_value_aliases()>
1860
1861 use Unicode::UCD 'prop_value_aliases';
1862
1863 my ($short_name, $full_name, @other_names)
1864 = prop_value_aliases("Gc", "Punct");
1865 my $same_full_name = prop_value_aliases("Gc", "P"); # Scalar cntxt
1866 my ($same_short_name) = prop_value_aliases("Gc", "P"); # gets 0th
1867 # element
1868 print "The full name is $full_name\n";
1869 print "The short name is $short_name\n";
1870 print "The other aliases are: ", join(", ", @other_names), "\n";
1871
1872 prints:
1873 The full name is Punctuation
1874 The short name is P
1875 The other aliases are: Punct
1876
1877Some Unicode properties have a restricted set of legal values. For example,
1878all binary properties are restricted to just C<true> or C<false>; and there
1879are only a few dozen possible General Categories.
1880
1881For such properties, there are usually several synonyms for each possible
1882value. For example, in binary properties, I<truth> can be represented by any of
1883the strings "Y", "Yes", "T", or "True"; and the General Category
1884"Punctuation" by that string, or "Punct", or simply "P".
1885
1886Like property names, there is typically at least a short name for each such
1887property-value, and a long name. If you know any name of the property-value,
1888you can use C<prop_value_aliases>() to get the long name (when called in
1889scalar context), or a list of all the names, with the short name in the 0th
1890element, the long name in the next element, and any other synonyms in the
1891remaining elements, in no particular order, except that any all-numeric
1892synonyms will be last.
1893
1894The long name is returned in a form nicely capitalized, suitable for printing.
1895
1896Case, white space, hyphens, and underscores are ignored in the input parameters
1897(except for the trailing underscore in the old-form grandfathered-in general
1898category property value C<"L_">, which is better written as C<"LC">).
1899
1900If either name is unknown, C<undef> is returned. Note that Perl typically
1901recognizes property names in regular expressions with an optional C<"Is_>"
1902(with or without the underscore) prefixed to them, such as C<\p{isgc=punct}>.
1903This function does not recognize those in the property parameter, returning
1904C<undef>.
1905
1906If called with a property that doesn't have synonyms for its values, it
1907returns the input value, possibly normalized with capitalization and
1908underscores.
1909
1910For the block property, new-style block names are returned (see
1911L</Old-style versus new-style block names>).
1912
1913To find the synonyms for single-forms, such as C<\p{Any}>, use
1914L</prop_aliases()> instead.
1915
1916C<prop_value_aliases> does not know about any user-defined properties, and
1917will return C<undef> if called with one of those.
1918
1919=cut
1920
1921# These are created by mktables for this routine and stored in unicore/UCD.pl
1922# where their structures are described.
1923our %loose_to_standard_value;
1924our %prop_value_aliases;
1925
1926sub prop_value_aliases ($$) {
1927 my ($prop, $value) = @_;
1928 return unless defined $prop && defined $value;
1929
1930 require "unicore/UCD.pl";
1931 require "utf8_heavy.pl";
1932
1933 # Find the property name synonym that's used as the key in other hashes,
1934 # which is element 0 in the returned list.
1935 ($prop) = prop_aliases($prop);
1936 return if ! $prop;
1937 $prop = utf8::_loose_name(lc $prop);
1938
1939 # Here is a legal property, but the hash below (created by mktables for
1940 # this purpose) only knows about the properties that have a very finite
1941 # number of potential values, that is not ones whose value could be
1942 # anything, like most (if not all) string properties. These don't have
1943 # synonyms anyway. Simply return the input. For example, there is no
1944 # synonym for ('Uppercase_Mapping', A').
1945 return $value if ! exists $prop_value_aliases{$prop};
1946
1947 # The value name may be loosely or strictly matched; we don't know yet.
1948 # But both types use lower-case.
1949 $value = lc $value;
1950
1951 # If the name isn't found under loose matching, it certainly won't be
1952 # found under strict
1953 my $loose_value = utf8::_loose_name($value);
1954 return unless exists $loose_to_standard_value{"$prop=$loose_value"};
1955
1956 # Similarly if the combination under loose matching doesn't exist, it
1957 # won't exist under strict.
1958 my $standard_value = $loose_to_standard_value{"$prop=$loose_value"};
1959 return unless exists $prop_value_aliases{$prop}{$standard_value};
1960
1961 # Here we did find a combination under loose matching rules. But it could
1962 # be that is a strict property match that shouldn't have matched.
1963 # %prop_value_aliases is set up so that the strict matches will appear as
1964 # if they were in loose form. Thus, if the non-loose version is legal,
1965 # we're ok, can skip the further check.
1966 if (! exists $utf8::stricter_to_file_of{"$prop=$value"}
1967
1968 # We're also ok and skip the further check if value loosely matches.
1969 # mktables has verified that no strict name under loose rules maps to
1970 # an existing loose name. This code relies on the very limited
1971 # circumstances that strict names can be here. Strict name matching
1972 # happens under two conditions:
1973 # 1) when the name begins with an underscore. But this function
1974 # doesn't accept those, and %prop_value_aliases doesn't have
1975 # them.
1976 # 2) When the values are numeric, in which case we need to look
1977 # further, but their squeezed-out loose values will be in
1978 # %stricter_to_file_of
1979 && exists $utf8::stricter_to_file_of{"$prop=$loose_value"})
1980 {
1981 # The only thing that's legal loosely under strict is that can have an
1982 # underscore between digit pairs XXX
1983 while ($value =~ s/(\d)_(\d)/$1$2/g) {}
1984 return unless exists $utf8::stricter_to_file_of{"$prop=$value"};
1985 }
1986
1987 # Here, we know that the combination exists. Return it.
1988 my $list_ref = $prop_value_aliases{$prop}{$standard_value};
1989 if (@$list_ref > 1) {
1990 # The full name is in element 1.
1991 return $list_ref->[1] unless wantarray;
1992
cb3150f5 1993 return @{_dclone $list_ref};
7ef25837
KW
1994 }
1995
1996 return $list_ref->[0] unless wantarray;
1997
1998 # Only 1 element means that it repeats
1999 return ( $list_ref->[0], $list_ref->[0] );
2000}
7319f91d 2001
681d705c
KW
2002# All 1 bits is the largest possible UV.
2003$Unicode::UCD::MAX_CP = ~0;
2004
2005=pod
2006
2007=head2 B<prop_invlist()>
2008
2009C<prop_invlist> returns an inversion list (described below) that defines all the
2010code points for the binary Unicode property (or "property=value" pair) given
2011by the input parameter string:
2012
2013 use feature 'say';
2014 use Unicode::UCD 'prop_invlist';
2015 say join ", ", prop_invlist("Any");
2016
2017 prints:
2018 0, 1114112
2019
c8652296
KW
2020If the input is unknown C<undef> is returned in scalar context; an empty-list
2021in list context. If the input is known, the number of elements in
681d705c
KW
2022the list is returned if called in scalar context.
2023
2024L<perluniprops|perluniprops/Properties accessible through \p{} and \P{}> gives
2025the list of properties that this function accepts, as well as all the possible
2026forms for them (including with the optional "Is_" prefixes). (Except this
2027function doesn't accept any Perl-internal properties, some of which are listed
2028there.) This function uses the same loose or tighter matching rules for
2029resolving the input property's name as is done for regular expressions. These
2030are also specified in L<perluniprops|perluniprops/Properties accessible
2031through \p{} and \P{}>. Examples of using the "property=value" form are:
2032
2033 say join ", ", prop_invlist("Script=Shavian");
2034
2035 prints:
2036 66640, 66688
2037
2038 say join ", ", prop_invlist("ASCII_Hex_Digit=No");
2039
2040 prints:
2041 0, 48, 58, 65, 71, 97, 103
2042
2043 say join ", ", prop_invlist("ASCII_Hex_Digit=Yes");
2044
2045 prints:
2046 48, 58, 65, 71, 97, 103
2047
2048Inversion lists are a compact way of specifying Unicode property-value
2049definitions. The 0th item in the list is the lowest code point that has the
2050property-value. The next item (item [1]) is the lowest code point beyond that
2051one that does NOT have the property-value. And the next item beyond that
2052([2]) is the lowest code point beyond that one that does have the
2053property-value, and so on. Put another way, each element in the list gives
2054the beginning of a range that has the property-value (for even numbered
2055elements), or doesn't have the property-value (for odd numbered elements).
2056The name for this data structure stems from the fact that each element in the
2057list toggles (or inverts) whether the corresponding range is or isn't on the
2058list.
2059
2060In the final example above, the first ASCII Hex digit is code point 48, the
2061character "0", and all code points from it through 57 (a "9") are ASCII hex
2062digits. Code points 58 through 64 aren't, but 65 (an "A") through 70 (an "F")
2063are, as are 97 ("a") through 102 ("f"). 103 starts a range of code points
2064that aren't ASCII hex digits. That range extends to infinity, which on your
2065computer can be found in the variable C<$Unicode::UCD::MAX_CP>. (This
2066variable is as close to infinity as Perl can get on your platform, and may be
2067too high for some operations to work; you may wish to use a smaller number for
2068your purposes.)
2069
2070Note that the inversion lists returned by this function can possibly include
2071non-Unicode code points, that is anything above 0x10FFFF. This is in
2072contrast to Perl regular expression matches on those code points, in which a
2073non-Unicode code point always fails to match. For example, both of these have
2074the same result:
2075
2076 chr(0x110000) =~ \p{ASCII_Hex_Digit=True} # Fails.
2077 chr(0x110000) =~ \p{ASCII_Hex_Digit=False} # Fails!
2078
2079And both raise a warning that a Unicode property is being used on a
2080non-Unicode code point. It is arguable as to which is the correct thing to do
2081here. This function has chosen the way opposite to the Perl regular
2082expression behavior. This allows you to easily flip to to the Perl regular
2083expression way (for you to go in the other direction would be far harder).
2084Simply add 0x110000 at the end of the non-empty returned list if it isn't
2085already that value; and pop that value if it is; like:
2086
2087 my @list = prop_invlist("foo");
2088 if (@list) {
2089 if ($list[-1] == 0x110000) {
2090 pop @list; # Defeat the turning on for above Unicode
2091 }
2092 else {
2093 push @list, 0x110000; # Turn off for above Unicode
2094 }
2095 }
2096
2097It is a simple matter to expand out an inversion list to a full list of all
2098code points that have the property-value:
2099
2100 my @invlist = prop_invlist($property_name);
2101 die "empty" unless @invlist;
2102 my @full_list;
2103 for (my $i = 0; $i < @invlist; $i += 2) {
2104 my $upper = ($i + 1) < @invlist
2105 ? $invlist[$i+1] - 1 # In range
2106 : $Unicode::UCD::MAX_CP; # To infinity. You may want
2107 # to stop much much earlier;
2108 # going this high may expose
2109 # perl deficiencies with very
2110 # large numbers.
2111 for my $j ($invlist[$i] .. $upper) {
2112 push @full_list, $j;
2113 }
2114 }
2115
2116C<prop_invlist> does not know about any user-defined nor Perl internal-only
2117properties, and will return C<undef> if called with one of those.
2118
2119=cut
2120
2121# User-defined properties could be handled with some changes to utf8_heavy.pl;
2122# and implementing here of dealing with EXTRAS. If done, consideration should
2123# be given to the fact that the user subroutine could return different results
2124# with each call; security issues need to be thought about.
2125
2126# These are created by mktables for this routine and stored in unicore/UCD.pl
2127# where their structures are described.
2128our %loose_defaults;
2129our $MAX_UNICODE_CODEPOINT;
2130
92feec86 2131sub prop_invlist ($;$) {
681d705c 2132 my $prop = $_[0];
92feec86
KW
2133
2134 # Undocumented way to get at Perl internal properties
2135 my $internal_ok = defined $_[1] && $_[1] eq '_perl_core_internal_ok';
2136
681d705c
KW
2137 return if ! defined $prop;
2138
2139 require "utf8_heavy.pl";
2140
2141 # Warnings for these are only for regexes, so not applicable to us
2142 no warnings 'deprecated';
2143
2144 # Get the swash definition of the property-value.
2145 my $swash = utf8::SWASHNEW(__PACKAGE__, $prop, undef, 1, 0);
2146
2147 # Fail if not found, or isn't a boolean property-value, or is a
2148 # user-defined property, or is internal-only.
2149 return if ! $swash
2150 || ref $swash eq ""
2151 || $swash->{'BITS'} != 1
2152 || $swash->{'USER_DEFINED'}
92feec86 2153 || (! $internal_ok && $prop =~ /^\s*_/);
681d705c
KW
2154
2155 if ($swash->{'EXTRAS'}) {
2156 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has EXTRAS magic";
2157 return;
2158 }
2159 if ($swash->{'SPECIALS'}) {
2160 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has SPECIALS magic";
2161 return;
2162 }
2163
2164 my @invlist;
2165
2166 # The input lines look like:
2167 # 0041\t005A # [26]
2168 # 005F
2169
2170 # Split into lines, stripped of trailing comments
2171 foreach my $range (split "\n",
2172 $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr)
2173 {
2174 # And find the beginning and end of the range on the line
2175 my ($hex_begin, $hex_end) = split "\t", $range;
2176 my $begin = hex $hex_begin;
2177
a39cc031
KW
2178 # If the new range merely extends the old, we remove the marker
2179 # created the last time through the loop for the old's end, which
2180 # causes the new one's end to be used instead.
2181 if (@invlist && $begin == $invlist[-1]) {
2182 pop @invlist;
2183 }
2184 else {
2f3f243e
KW
2185 # Add the beginning of the range
2186 push @invlist, $begin;
a39cc031 2187 }
681d705c
KW
2188
2189 if (defined $hex_end) { # The next item starts with the code point 1
2190 # beyond the end of the range.
2191 push @invlist, hex($hex_end) + 1;
2192 }
2193 else { # No end of range, is a single code point.
2194 push @invlist, $begin + 1;
2195 }
2196 }
2197
2198 require "unicore/UCD.pl";
2199 my $FIRST_NON_UNICODE = $MAX_UNICODE_CODEPOINT + 1;
2200
2201 # Could need to be inverted: add or subtract a 0 at the beginning of the
2202 # list. And to keep it from matching non-Unicode, add or subtract the
2203 # first non-unicode code point.
2204 if ($swash->{'INVERT_IT'}) {
2205 if (@invlist && $invlist[0] == 0) {
2206 shift @invlist;
2207 }
2208 else {
2209 unshift @invlist, 0;
2210 }
2211 if (@invlist && $invlist[-1] == $FIRST_NON_UNICODE) {
2212 pop @invlist;
2213 }
2214 else {
2215 push @invlist, $FIRST_NON_UNICODE;
2216 }
2217 }
2218
2219 # Here, the list is set up to include only Unicode code points. But, if
2220 # the table is the default one for the property, it should contain all
2221 # non-Unicode code points. First calculate the loose name for the
2222 # property. This is done even for strict-name properties, as the data
2223 # structure that mktables generates for us is set up so that we don't have
2224 # to worry about that. The property-value needs to be split if compound,
2225 # as the loose rules need to be independently calculated on each part. We
2226 # know that it is syntactically valid, or SWASHNEW would have failed.
2227
2228 $prop = lc $prop;
2229 my ($prop_only, $table) = split /\s*[:=]\s*/, $prop;
2230 if ($table) {
2231
2232 # May have optional prefixed 'is'
2233 $prop = utf8::_loose_name($prop_only) =~ s/^is//r;
2234 $prop = $utf8::loose_property_name_of{$prop};
2235 $prop .= "=" . utf8::_loose_name($table);
2236 }
2237 else {
2238 $prop = utf8::_loose_name($prop);
2239 }
2240 if (exists $loose_defaults{$prop}) {
2241
2242 # Here, is the default table. If a range ended with 10ffff, instead
2243 # continue that range to infinity, by popping the 110000; otherwise,
2244 # add the range from 11000 to infinity
2245 if (! @invlist || $invlist[-1] != $FIRST_NON_UNICODE) {
2246 push @invlist, $FIRST_NON_UNICODE;
2247 }
2248 else {
2249 pop @invlist;
2250 }
2251 }
2252
2253 return @invlist;
2254}
7319f91d 2255
62b3b855
KW
2256sub _search_invlist {
2257 # Find the range in the inversion list which contains a code point; that
de13b7b1
KW
2258 # is, find i such that l[i] <= code_point < l[i+1]. Returns undef if no
2259 # such i.
62b3b855
KW
2260
2261 # If this is ever made public, could use to speed up .t specials. Would
2262 # need to use code point argument, as in other functions in this pm
2263
2264 my $list_ref = shift;
2265 my $code_point = shift;
2266 # Verify non-neg numeric XXX
2267
2268 my $max_element = @$list_ref - 1;
de13b7b1
KW
2269
2270 # Return undef if list is empty or requested item is before the first element.
2271 return if $max_element < 0;
2272 return if $code_point < $list_ref->[0];
62b3b855
KW
2273
2274 # Short cut something at the far-end of the table. This also allows us to
2275 # refer to element [$i+1] without fear of being out-of-bounds in the loop
2276 # below.
2277 return $max_element if $code_point >= $list_ref->[$max_element];
2278
2279 use integer; # want integer division
2280
2281 my $i = $max_element / 2;
2282
2283 my $lower = 0;
2284 my $upper = $max_element;
2285 while (1) {
2286
2287 if ($code_point >= $list_ref->[$i]) {
2288
2289 # Here we have met the lower constraint. We can quit if we
2290 # also meet the upper one.
2291 last if $code_point < $list_ref->[$i+1];
2292
2293 $lower = $i; # Still too low.
2294
2295 }
2296 else {
2297
2298 # Here, $code_point < $list_ref[$i], so look lower down.
2299 $upper = $i;
2300 }
2301
2302 # Split search domain in half to try again.
2303 my $temp = ($upper + $lower) / 2;
2304
2305 # No point in continuing unless $i changes for next time
2306 # in the loop.
2307 return $i if $temp == $i;
2308 $i = $temp;
2309 } # End of while loop
2310
2311 # Here we have found the offset
2312 return $i;
2313}
2314
2315=pod
2316
2317=head2 B<prop_invmap()>
2318
2319 use Unicode::UCD 'prop_invmap';
2320 my ($list_ref, $map_ref, $format, $missing)
2321 = prop_invmap("General Category");
2322
2323C<prop_invmap> is used to get the complete mapping definition for a property,
2324in the form of an inversion map. An inversion map consists of two parallel
2325arrays. One is an ordered list of code points that mark range beginnings, and
2326the other gives the value (or mapping) that all code points in the
2327corresponding range have.
2328
2329C<prop_invmap> is called with the name of the desired property. The name is
2330loosely matched, meaning that differences in case, white-space, hyphens, and
2331underscores are not meaningful (except for the trailing underscore in the
2332old-form grandfathered-in property C<"L_">, which is better written as C<"LC">,
2333or even better, C<"Gc=LC">).
2334
2335Many Unicode properties have more than one name (or alias). C<prop_invmap>
2336understands all of these, including Perl extensions to them. Ambiguities are
2337resolved as described above for L</prop_aliases()>. The Perl internal
2338property "Perl_Decimal_Digit, described below, is also accepted. C<undef> is
2339returned if the property name is unknown.
ee94c7d1
KW
2340See L<perluniprops/Properties accessible through Unicode::UCD> for the
2341properties acceptable as inputs to this function.
62b3b855
KW
2342
2343It is a fatal error to call this function except in list context.
2344
2345In addition to the the two arrays that form the inversion map, C<prop_invmap>
2346returns two other values; one is a scalar that gives some details as to the
2347format of the entries of the map array; the other is used for specialized
2348purposes, described at the end of this section.
2349
2350This means that C<prop_invmap> returns a 4 element list. For example,
2351
2352 my ($blocks_ranges_ref, $blocks_maps_ref, $format, $default)
2353 = prop_invmap("Block");
2354
2355In this call, the two arrays will be populated as shown below (for Unicode
23566.0):
2357
2358 Index @blocks_ranges @blocks_maps
2359 0 0x0000 Basic Latin
2360 1 0x0080 Latin-1 Supplement
2361 2 0x0100 Latin Extended-A
2362 3 0x0180 Latin Extended-B
2363 4 0x0250 IPA Extensions
2364 5 0x02B0 Spacing Modifier Letters
2365 6 0x0300 Combining Diacritical Marks
2366 7 0x0370 Greek and Coptic
2367 8 0x0400 Cyrillic
2368 ...
2369 233 0x2B820 No_Block
2370 234 0x2F800 CJK Compatibility Ideographs Supplement
2371 235 0x2FA20 No_Block
2372 236 0xE0000 Tags
2373 237 0xE0080 No_Block
2374 238 0xE0100 Variation Selectors Supplement
2375 239 0xE01F0 No_Block
2376 240 0xF0000 Supplementary Private Use Area-A
2377 241 0x100000 Supplementary Private Use Area-B
2378 242 0x110000 No_Block
2379
2380The first line (with Index [0]) means that the value for code point 0 is "Basic
2381Latin". The entry "0x0080" in the @blocks_ranges column in the second line
2382means that the value from the first line, "Basic Latin", extends to all code
2383points in the range from 0 up to but not including 0x0080, that is, through
647396da 2384127. In other words, the code points from 0 to 127 are all in the "Basic
62b3b855
KW
2385Latin" block. Similarly, all code points in the range from 0x0080 up to (but
2386not including) 0x0100 are in the block named "Latin-1 Supplement", etc.
2387(Notice that the return is the old-style block names; see L</Old-style versus
2388new-style block names>).
2389
2390The final line (with Index [242]) means that the value for all code points above
2391the legal Unicode maximum code point have the value "No_Block", which is the
2392term Unicode uses for a non-existing block.
2393
2394The arrays completely specify the mappings for all possible code points.
2395The final element in an inversion map returned by this function will always be
2396for the range that consists of all the code points that aren't legal Unicode,
2397but that are expressible on the platform. (That is, it starts with code point
23980x110000, the first code point above the legal Unicode maximum, and extends to
2399infinity.) The value for that range will be the same that any typical
2400unassigned code point has for the specified property. (Certain unassigned
2401code points are not "typical"; for example the non-character code points, or
2402those in blocks that are to be written right-to-left. The above-Unicode
2403range's value is not based on these atypical code points.) It could be argued
2404that, instead of treating these as unassigned Unicode code points, the value
2405for this range should be C<undef>. If you wish, you can change the returned
2406arrays accordingly.
2407
2408The maps are almost always simple scalars that should be interpreted as-is.
2409These values are those given in the Unicode-supplied data files, which may be
2410inconsistent as to capitalization and as to which synonym for a property-value
2411is given. The results may be normalized by using the L</prop_value_aliases()>
2412function.
2413
2414There are exceptions to the simple scalar maps. Some properties have some
2415elements in their map list that are themselves lists of scalars; and some
2416special strings are returned that are not to be interpreted as-is. Element
2417[2] (placed into C<$format> in the example above) of the returned four element
647396da 2418list tells you if the map has any of these special elements or not, as follows:
62b3b855
KW
2419
2420=over
2421
dc8d8ea6 2422=item B<C<s>>
62b3b855
KW
2423
2424means all the elements of the map array are simple scalars, with no special
2425elements. Almost all properties are like this, like the C<block> example
2426above.
2427
dc8d8ea6 2428=item B<C<sl>>
62b3b855 2429
647396da 2430means that some of the map array elements have the form given by C<"s">, and
62b3b855
KW
2431the rest are lists of scalars. For example, here is a portion of the output
2432of calling C<prop_invmap>() with the "Script Extensions" property:
2433
2434 @scripts_ranges @scripts_maps
2435 ...
c2ca0207
KW
2436 0x0953 Devanagari
2437 0x0964 [ Bengali, Devanagari, Gurumukhi, Oriya ]
2438 0x0966 Devanagari
62b3b855
KW
2439 0x0970 Common
2440
647396da
KW
2441Here, the code points 0x964 and 0x965 are both used in Bengali,
2442Devanagari, Gurmukhi, and Oriya, but no other scripts.
62b3b855 2443
647396da 2444The Name_Alias property is also of this form. But each scalar consists of two
58b75e36 2445components: 1) the name, and 2) the type of alias this is. They are
7620cb10
KW
2446separated by a colon and a space. In Unicode 6.1, there are several alias types:
2447
2448=over
2449
2450=item C<correction>
2451
2452indicates that the name is a corrected form for the
2453original name (which remains valid) for the same code point.
2454
2455=item C<control>
2456
2457adds a new name for a control character.
2458
2459=item C<alternate>
2460
2461is an alternate name for a character
2462
2463=item C<figment>
2464
2465is a name for a character that has been documented but was never in any
2466actual standard.
2467
2468=item C<abbreviation>
2469
2470is a common abbreviation for a character
2471
2472=back
2473
2474The lists are ordered (roughly) so the most preferred names come before less
2475preferred ones.
58b75e36
KW
2476
2477For example,
2478
7620cb10
KW
2479 @aliases_ranges @alias_maps
2480 ...
2481 0x009E [ 'PRIVACY MESSAGE: control', 'PM: abbreviation' ]
2482 0x009F [ 'APPLICATION PROGRAM COMMAND: control',
2483 'APC: abbreviation'
2484 ]
2485 0x00A0 'NBSP: abbreviation'
2486 0x00A1 ""
2487 0x00AD 'SHY: abbreviation'
2488 0x00AE ""
2489 0x01A2 'LATIN CAPITAL LETTER GHA: correction'
2490 0x01A3 'LATIN SMALL LETTER GHA: correction'
2491 0x01A4 ""
58b75e36 2492 ...
58b75e36 2493
7620cb10
KW
2494A map to the empty string means that there is no alias defined for the code
2495point.
58b75e36 2496
d11155ec 2497=item B<C<a>>
62b3b855 2498
647396da 2499is like C<"s"> in that all the map array elements are scalars, but here they are
d11155ec
KW
2500restricted to all being integers, and some have to be adjusted (hence the name
2501C<"a">) to get the correct result. For example, in:
62b3b855
KW
2502
2503 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2504 = prop_invmap("Simple_Uppercase_Mapping");
2505
2506the returned arrays look like this:
2507
2508 @$uppers_ranges_ref @$uppers_maps_ref Note
bf7fe2df 2509 0 0
d11155ec 2510 97 65 'a' maps to 'A', b => B ...
bf7fe2df 2511 123 0
d11155ec 2512 181 924 MICRO SIGN => Greek Cap MU
bf7fe2df 2513 182 0
62b3b855
KW
2514 ...
2515
d11155ec
KW
2516Let's start with the second line. It says that the uppercase of code point 97
2517is 65; or C<uc("a")> == "A". But the line is for the entire range of code
2518points 97 through 122. To get the mapping for any code point in a range, you
2519take the offset it has from the beginning code point of the range, and add
2520that to the mapping for that first code point. So, the mapping for 122 ("z")
2521is derived by taking the offset of 122 from 97 (=25) and adding that to 65,
2522yielding 90 ("z"). Likewise for everything in between.
2523
2524The first line works the same way. The first map in a range is always the
2525correct value for its code point (because the adjustment is 0). Thus the
2526C<uc(chr(0))> is just itself. Also, C<uc(chr(1))> is also itself, as the
2527adjustment is 0+1-0 .. C<uc(chr(96))> is 96.
bf7fe2df 2528
d11155ec
KW
2529Requiring this simple adjustment allows the returned arrays to be
2530significantly smaller than otherwise, up to a factor of 10, speeding up
2531searching through them.
62b3b855 2532
d11155ec 2533=item B<C<al>>
62b3b855 2534
d11155ec 2535means that some of the map array elements have the form given by C<"a">, and
62b3b855
KW
2536the rest are ordered lists of code points.
2537For example, in:
2538
2539 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2540 = prop_invmap("Uppercase_Mapping");
2541
2542the returned arrays look like this:
2543
2544 @$uppers_ranges_ref @$uppers_maps_ref
bf7fe2df 2545 0 0
d11155ec 2546 97 65
bf7fe2df 2547 123 0
d11155ec 2548 181 924
bf7fe2df 2549 182 0
62b3b855
KW
2550 ...
2551 0x0149 [ 0x02BC 0x004E ]
bf7fe2df 2552 0x014A 0
d11155ec 2553 0x014B 330
62b3b855
KW
2554 ...
2555
2556This is the full Uppercase_Mapping property (as opposed to the
d11155ec 2557Simple_Uppercase_Mapping given in the example for format C<"a">). The only
62b3b855
KW
2558difference between the two in the ranges shown is that the code point at
25590x0149 (LATIN SMALL LETTER N PRECEDED BY APOSTROPHE) maps to a string of two
2560characters, 0x02BC (MODIFIER LETTER APOSTROPHE) followed by 0x004E (LATIN
2561CAPITAL LETTER N).
2562
d11155ec
KW
2563No adjustments are needed to entries that are references to arrays; each such
2564entry will have exactly one element in its range, so the offset is always 0.
bf7fe2df 2565
d11155ec 2566=item B<C<ae>>
b0b13ada 2567
d11155ec
KW
2568This is like C<"a">, but some elements are the empty string, and should not be
2569adjusted.
b0b13ada
KW
2570The one internal Perl property accessible by C<prop_invmap> is of this type:
2571"Perl_Decimal_Digit" returns an inversion map which gives the numeric values
2572that are represented by the Unicode decimal digit characters. Characters that
2573don't represent decimal digits map to the empty string, like so:
2574
2575 @digits @values
2576 0x0000 ""
d11155ec 2577 0x0030 0
b0b13ada 2578 0x003A: ""
d11155ec 2579 0x0660: 0
b0b13ada 2580 0x066A: ""
d11155ec 2581 0x06F0: 0
b0b13ada 2582 0x06FA: ""
d11155ec 2583 0x07C0: 0
b0b13ada 2584 0x07CA: ""
d11155ec 2585 0x0966: 0
b0b13ada
KW
2586 ...
2587
2588This means that the code points from 0 to 0x2F do not represent decimal digits;
d11155ec
KW
2589the code point 0x30 (DIGIT ZERO) represents 0; code point 0x31, (DIGIT ONE),
2590represents 0+1-0 = 1; ... code point 0x39, (DIGIT NINE), represents 0+9-0 = 9;
2591... code points 0x3A through 0x65F do not represent decimal digits; 0x660
2592(ARABIC-INDIC DIGIT ZERO), represents 0; ... 0x07C1 (NKO DIGIT ONE),
2593represents 0+1-0 = 1 ...
b0b13ada 2594
d11155ec 2595=item B<C<ale>>
62b3b855 2596
d11155ec
KW
2597is a combination of the C<"al"> type and the C<"ae"> type. Some of
2598the map array elements have the forms given by C<"al">, and
62b3b855
KW
2599the rest are the empty string. The property C<NFKC_Casefold> has this form.
2600An example slice is:
2601
2602 @$ranges_ref @$maps_ref Note
2603 ...
d11155ec
KW
2604 0x00AA 97 FEMININE ORDINAL INDICATOR => 'a'
2605 0x00AB 0
62b3b855 2606 0x00AD SOFT HYPHEN => ""
d11155ec 2607 0x00AE 0
62b3b855 2608 0x00AF [ 0x0020, 0x0304 ] MACRON => SPACE . COMBINING MACRON
d11155ec 2609 0x00B0 0
62b3b855
KW
2610 ...
2611
4f143a72 2612=item B<C<ar>>
6cc45523
KW
2613
2614means that all the elements of the map array are either rational numbers or
2615the string C<"NaN">, meaning "Not a Number". A rational number is either an
2616integer, or two integers separated by a solidus (C<"/">). The second integer
2617represents the denominator of the division implied by the solidus, and is
60c78852 2618actually always positive, so it is guaranteed not to be 0 and to not be
6329003c 2619signed. When the element is a plain integer (without the
4f143a72
KW
2620solidus), it may need to be adjusted to get the correct value by adding the
2621offset, just as other C<"a"> properties. No adjustment is needed for
2622fractions, as the range is guaranteed to have just a single element, and so
2623the offset is always 0.
2624
2625If you want to convert the returned map to entirely scalar numbers, you
6cc45523
KW
2626can use something like this:
2627
2628 my ($invlist_ref, $invmap_ref, $format) = prop_invmap($property);
4f143a72 2629 if ($format && $format eq "ar") {
60c78852 2630 map { $_ = eval $_ if $_ ne 'NaN' } @$map_ref;
6cc45523
KW
2631 }
2632
2633Here's some entries from the output of the property "Nv", which has format
4f143a72 2634C<"ar">.
6cc45523 2635
4f143a72 2636 @numerics_ranges @numerics_maps Note
6cc45523 2637 0x00 "NaN"
4f143a72 2638 0x30 0 DIGIT 0 .. DIGIT 9
6cc45523 2639 0x3A "NaN"
4f143a72 2640 0xB2 2 SUPERSCRIPTs 2 and 3
6cc45523 2641 0xB4 "NaN"
4f143a72 2642 0xB9 1 SUPERSCRIPT 1
6cc45523 2643 0xBA "NaN"
4f143a72
KW
2644 0xBC 1/4 VULGAR FRACTION 1/4
2645 0xBD 1/2 VULGAR FRACTION 1/2
2646 0xBE 3/4 VULGAR FRACTION 3/4
6cc45523 2647 0xBF "NaN"
4f143a72
KW
2648 0x660 0 ARABIC-INDIC DIGIT ZERO .. NINE
2649 0x66A "NaN"
6cc45523 2650
dc8d8ea6 2651=item B<C<n>>
62b3b855
KW
2652
2653means the Name property. All the elements of the map array are simple
2654scalars, but some of them contain special strings that require more work to
2655get the actual name.
2656
2657Entries such as:
2658
2659 CJK UNIFIED IDEOGRAPH-<code point>
2660
2661mean that the name for the code point is "CJK UNIFIED IDEOGRAPH-"
2662with the code point (expressed in hexadecimal) appended to it, like "CJK
647396da
KW
2663UNIFIED IDEOGRAPH-3403" (similarly for S<C<CJK COMPATIBILITY IDEOGRAPH-E<lt>code
2664pointE<gt>>>).
62b3b855
KW
2665
2666Also, entries like
2667
2668 <hangul syllable>
2669
2670means that the name is algorithmically calculated. This is easily done by
2671the function L<charnames/charnames::viacode(code)>.
2672
2673Note that for control characters (C<Gc=cc>), Unicode's data files have the
2674string "C<E<lt>controlE<gt>>", but the real name of each of these characters is the empty
7620cb10 2675string. This function returns that real name, the empty string. (There are
647396da
KW
2676names for these characters, but they are considered aliases, not the Name
2677property name, and are contained in the C<Name_Alias> property.)
62b3b855 2678
d11155ec 2679=item B<C<ad>>
62b3b855 2680
d11155ec 2681means the Decomposition_Mapping property. This property is like C<"al">
bea2c146 2682properties, except that one of the scalar elements is of the form:
62b3b855
KW
2683
2684 <hangul syllable>
2685
bea2c146
KW
2686This signifies that this entry should be replaced by the decompositions for
2687all the code points whose decomposition is algorithmically calculated. (All
6329003c
KW
2688of them are currently in one range and no others outisde the range are likely
2689to ever be added to Unicode; the C<"n"> format
bea2c146 2690has this same entry.) These can be generated via the function
62b3b855
KW
2691L<Unicode::Normalize::NFD()|Unicode::Normalize>.
2692
62b3b855
KW
2693Note that the mapping is the one that is specified in the Unicode data files,
2694and to get the final decomposition, it may need to be applied recursively.
2695
2696=back
2697
d11155ec
KW
2698Note that a format begins with the letter "a" if and only the property it is
2699for requires adjustments by adding the offsets in multi-element ranges. For
2700all these properties, an entry should be adjusted only if the map is a scalar
2701which is an integer. That is, it must match the regular expression:
2702
2703 / ^ -? \d+ $ /xa
2704
2705Further, the first element in a range never needs adjustment, as the
2706adjustment would be just adding 0.
2707
62b3b855
KW
2708A binary search can be used to quickly find a code point in the inversion
2709list, and hence its corresponding mapping.
2710
2711The final element (index [3], assigned to C<$default> in the "block" example) in
2712the four element list returned by this function may be useful for applications
2713that wish to convert the returned inversion map data structure into some
2714other, such as a hash. It gives the mapping that most code points map to
2715under the property. If you establish the convention that any code point not
2716explicitly listed in your data structure maps to this value, you can
2717potentially make your data structure much smaller. As you construct your data
2718structure from the one returned by this function, simply ignore those ranges
2719that map to this value, generally called the "default" value. For example, to
2720convert to the data structure searchable by L</charinrange()>, you can follow
6329003c 2721this recipe for properties that don't require adjustments:
62b3b855
KW
2722
2723 my ($list_ref, $map_ref, $format, $missing) = prop_invmap($property);
2724 my @range_list;
6329003c
KW
2725
2726 # Look at each element in the list, but the -2 is needed because we
2727 # look at $i+1 in the loop, and the final element is guaranteed to map
2728 # to $missing by prop_invmap(), so we would skip it anyway.
62b3b855
KW
2729 for my $i (0 .. @$list_ref - 2) {
2730 next if $map_ref->[$i] eq $missing;
2731 push @range_list, [ $list_ref->[$i],
2732 $list_ref->[$i+1],
2733 $map_ref->[$i]
2734 ];
2735 }
2736
2737 print charinrange(\@range_list, $code_point), "\n";
2738
62b3b855
KW
2739With this, C<charinrange()> will return C<undef> if its input code point maps
2740to C<$missing>. You can avoid this by omitting the C<next> statement, and adding
2741a line after the loop to handle the final element of the inversion map.
2742
6329003c
KW
2743Similarly, this recipe can be used for properties that do require adjustments:
2744
2745 for my $i (0 .. @$list_ref - 2) {
2746 next if $map_ref->[$i] eq $missing;
2747
2748 # prop_invmap() guarantees that if the mapping is to an array, the
2749 # range has just one element, so no need to worry about adjustments.
2750 if (ref $map_ref->[$i]) {
2751 push @range_list,
2752 [ $list_ref->[$i], $list_ref->[$i], $map_ref->[$i] ];
2753 }
2754 else { # Otherwise each element is actually mapped to a separate
2755 # value, so the range has to be split into single code point
2756 # ranges.
2757
2758 my $adjustment = 0;
2759
2760 # For each code point that gets mapped to something...
2761 for my $j ($list_ref->[$i] .. $list_ref->[$i+1] -1 ) {
2762
2763 # ... add a range consisting of just it mapping to the
2764 # original plus the adjustment, which is incremented for the
2765 # next time through the loop, as the offset increases by 1
2766 # for each element in the range
2767 push @range_list,
2768 [ $j, $j, $map_ref->[$i] + $adjustment++ ];
2769 }
2770 }
2771 }
62b3b855
KW
2772
2773Note that the inversion maps returned for the C<Case_Folding> and
2774C<Simple_Case_Folding> properties do not include the Turkic-locale mappings.
2775Use L</casefold()> for these.
2776
62b3b855
KW
2777C<prop_invmap> does not know about any user-defined properties, and will
2778return C<undef> if called with one of those.
2779
2780=cut
2781
2782# User-defined properties could be handled with some changes to utf8_heavy.pl;
2783# if done, consideration should be given to the fact that the user subroutine
2784# could return different results with each call, which could lead to some
2785# security issues.
2786
2787# One could store things in memory so they don't have to be recalculated, but
2788# it is unlikely this will be called often, and some properties would take up
2789# significant memory.
2790
2791# These are created by mktables for this routine and stored in unicore/UCD.pl
2792# where their structures are described.
2793our @algorithmic_named_code_points;
2794our $HANGUL_BEGIN;
2795our $HANGUL_COUNT;
2796
2797sub prop_invmap ($) {
2798
2799 croak __PACKAGE__, "::prop_invmap: must be called in list context" unless wantarray;
2800
2801 my $prop = $_[0];
2802 return unless defined $prop;
2803
2804 # Fail internal properties
2805 return if $prop =~ /^_/;
2806
2807 # The values returned by this function.
2808 my (@invlist, @invmap, $format, $missing);
2809
2810 # The swash has two components we look at, the base list, and a hash,
2811 # named 'SPECIALS', containing any additional members whose mappings don't
2812 # fit into the the base list scheme of things. These generally 'override'
2813 # any value in the base list for the same code point.
2814 my $overrides;
2815
2816 require "utf8_heavy.pl";
2817 require "unicore/UCD.pl";
2818
2819RETRY:
2820
647396da
KW
2821 # If there are multiple entries for a single code point
2822 my $has_multiples = 0;
2823
62b3b855
KW
2824 # Try to get the map swash for the property. They have 'To' prepended to
2825 # the property name, and 32 means we will accept 32 bit return values.
647396da 2826 # The 0 means we aren't calling this from tr///.
62b3b855
KW
2827 my $swash = utf8::SWASHNEW(__PACKAGE__, "To$prop", undef, 32, 0);
2828
2829 # If didn't find it, could be because needs a proxy. And if was the
2830 # 'Block' or 'Name' property, use a proxy even if did find it. Finding it
647396da
KW
2831 # in these cases would be the result of the installation changing mktables
2832 # to output the Block or Name tables. The Block table gives block names
2833 # in the new-style, and this routine is supposed to return old-style block
2834 # names. The Name table is valid, but we need to execute the special code
2835 # below to add in the algorithmic-defined name entries.
34132297 2836 # And NFKCCF needs conversion, so handle that here too.
62b3b855 2837 if (ref $swash eq ""
34132297 2838 || $swash->{'TYPE'} =~ / ^ To (?: Blk | Na | NFKCCF ) $ /x)
62b3b855
KW
2839 {
2840
2841 # Get the short name of the input property, in standard form
2842 my ($second_try) = prop_aliases($prop);
2843 return unless $second_try;
2844 $second_try = utf8::_loose_name(lc $second_try);
2845
2846 if ($second_try eq "in") {
2847
2848 # This property is identical to age for inversion map purposes
2849 $prop = "age";
2850 goto RETRY;
2851 }
cfc5eb77 2852 elsif ($second_try =~ / ^ s ( cf | fc | [ltu] c ) $ /x) {
62b3b855 2853
75e7c50b
KW
2854 # These properties use just the LIST part of the full mapping,
2855 # which includes the simple maps that are otherwise overridden by
2856 # the SPECIALS. So all we need do is to not look at the SPECIALS;
2857 # set $overrides to indicate that
62b3b855 2858 $overrides = -1;
62b3b855 2859
75e7c50b 2860 # The full name is the simple name stripped of its initial 's'
cfc5eb77
KW
2861 $prop = $1;
2862
2863 # .. except for this case
2864 $prop = 'cf' if $prop eq 'fc';
2865
62b3b855
KW
2866 goto RETRY;
2867 }
2868 elsif ($second_try eq "blk") {
2869
2870 # We use the old block names. Just create a fake swash from its
2871 # data.
2872 _charblocks();
2873 my %blocks;
2874 $blocks{'LIST'} = "";
2875 $blocks{'TYPE'} = "ToBlk";
2876 $utf8::SwashInfo{ToBlk}{'missing'} = "No_Block";
2877 $utf8::SwashInfo{ToBlk}{'format'} = "s";
2878
2879 foreach my $block (@BLOCKS) {
2880 $blocks{'LIST'} .= sprintf "%x\t%x\t%s\n",
2881 $block->[0],
2882 $block->[1],
2883 $block->[2];
2884 }
2885 $swash = \%blocks;
2886 }
2887 elsif ($second_try eq "na") {
2888
2889 # Use the combo file that has all the Name-type properties in it,
2890 # extracting just the ones that are for the actual 'Name'
2891 # property. And create a fake swash from it.
2892 my %names;
2893 $names{'LIST'} = "";
2894 my $original = do "unicore/Name.pl";
62b3b855
KW
2895 my $algorithm_names = \@algorithmic_named_code_points;
2896
3b6a8189
KW
2897 # We need to remove the names from it that are aliases. For that
2898 # we need to also read in that table. Create a hash with the keys
2899 # being the code points, and the values being a list of the
2900 # aliases for the code point key.
2901 my ($aliases_code_points, $aliases_maps, undef, undef) =
2902 &prop_invmap('Name_Alias');
2903 my %aliases;
2904 for (my $i = 0; $i < @$aliases_code_points; $i++) {
2905 my $code_point = $aliases_code_points->[$i];
2906 $aliases{$code_point} = $aliases_maps->[$i];
2907
2908 # If not already a list, make it into one, so that later we
2909 # can treat things uniformly
2910 if (! ref $aliases{$code_point}) {
2911 $aliases{$code_point} = [ $aliases{$code_point} ];
2912 }
2913
2914 # Remove the alias type from the entry, retaining just the
2915 # name.
2916 map { s/:.*// } @{$aliases{$code_point}};
2917 }
2918
62b3b855
KW
2919 my $i = 0;
2920 foreach my $line (split "\n", $original) {
2921 my ($hex_code_point, $name) = split "\t", $line;
2922
2923 # Weeds out all comments, blank lines, and named sequences
df46a385 2924 next if $hex_code_point =~ /[^[:xdigit:]]/a;
62b3b855
KW
2925
2926 my $code_point = hex $hex_code_point;
2927
2928 # The name of all controls is the default: the empty string.
2929 # The set of controls is immutable, so these hard-coded
2930 # constants work.
2931 next if $code_point <= 0x9F
2932 && ($code_point <= 0x1F || $code_point >= 0x7F);
2933
3b6a8189
KW
2934 # If this is a name_alias, it isn't a name
2935 next if grep { $_ eq $name } @{$aliases{$code_point}};
62b3b855
KW
2936
2937 # If we are beyond where one of the special lines needs to
2938 # be inserted ...
3b6a8189 2939 while ($i < @$algorithm_names
62b3b855
KW
2940 && $code_point > $algorithm_names->[$i]->{'low'})
2941 {
2942
2943 # ... then insert it, ahead of what we were about to
2944 # output
3b6a8189 2945 $names{'LIST'} .= sprintf "%x\t%x\t%s\n",
62b3b855
KW
2946 $algorithm_names->[$i]->{'low'},
2947 $algorithm_names->[$i]->{'high'},
2948 $algorithm_names->[$i]->{'name'};
2949
62b3b855
KW
2950 # Done with this range.
2951 $i++;
2952
3b6a8189
KW
2953 # We loop until all special lines that precede the next
2954 # regular one are output.
62b3b855
KW
2955 }
2956
3b6a8189
KW
2957 # Here, is a normal name.
2958 $names{'LIST'} .= sprintf "%x\t\t%s\n", $code_point, $name;
2959 } # End of loop through all the names
62b3b855
KW
2960
2961 $names{'TYPE'} = "ToNa";
2962 $utf8::SwashInfo{ToNa}{'missing'} = "";
2963 $utf8::SwashInfo{ToNa}{'format'} = "n";
2964 $swash = \%names;
2965 }
2966 elsif ($second_try =~ / ^ ( d [mt] ) $ /x) {
2967
2968 # The file is a combination of dt and dm properties. Create a
2969 # fake swash from the portion that we want.
2970 my $original = do "unicore/Decomposition.pl";
2971 my %decomps;
2972
2973 if ($second_try eq 'dt') {
2974 $decomps{'TYPE'} = "ToDt";
2975 $utf8::SwashInfo{'ToDt'}{'missing'} = "None";
2976 $utf8::SwashInfo{'ToDt'}{'format'} = "s";
d11155ec 2977 } # 'dm' is handled below, with 'nfkccf'
62b3b855
KW
2978
2979 $decomps{'LIST'} = "";
2980
2981 # This property has one special range not in the file: for the
1a4c9760
KW
2982 # hangul syllables. But not in Unicode version 1.
2983 UnicodeVersion() unless defined $v_unicode_version;
2984 my $done_hangul = ($v_unicode_version lt v2.0.0)
2985 ? 1
2986 : 0; # Have we done the hangul range ?
62b3b855
KW
2987 foreach my $line (split "\n", $original) {
2988 my ($hex_lower, $hex_upper, $type_and_map) = split "\t", $line;
2989 my $code_point = hex $hex_lower;
2990 my $value;
bea2c146 2991 my $redo = 0;
62b3b855
KW
2992
2993 # The type, enclosed in <...>, precedes the mapping separated
2994 # by blanks
2995 if ($type_and_map =~ / ^ < ( .* ) > \s+ (.*) $ /x) {
2996 $value = ($second_try eq 'dt') ? $1 : $2
2997 }
2998 else { # If there is no type specified, it's canonical
2999 $value = ($second_try eq 'dt')
3000 ? "Canonical" :
3001 $type_and_map;
3002 }
3003
3004 # Insert the hangul range at the appropriate spot.
3005 if (! $done_hangul && $code_point > $HANGUL_BEGIN) {
3006 $done_hangul = 1;
3007 $decomps{'LIST'} .=
3008 sprintf "%x\t%x\t%s\n",
3009 $HANGUL_BEGIN,
3010 $HANGUL_BEGIN + $HANGUL_COUNT - 1,
3011 ($second_try eq 'dt')
3012 ? "Canonical"
3013 : "<hangul syllable>";
3014 }
3015
1a4c9760
KW
3016 if ($value =~ / / && $hex_upper ne "" && $hex_upper ne $hex_lower) {
3017 $line = sprintf("%04X\t%s\t%s", hex($hex_lower) + 1, $hex_upper, $value);
3018 $hex_upper = "";
3019 $redo = 1;
3020 }
3021
62b3b855
KW
3022 # And append this to our constructed LIST.
3023 $decomps{'LIST'} .= "$hex_lower\t$hex_upper\t$value\n";
bea2c146
KW
3024
3025 redo if $redo;
62b3b855
KW
3026 }
3027 $swash = \%decomps;
3028 }
d11155ec
KW
3029 elsif ($second_try ne 'nfkccf') { # Don't know this property. Fail.
3030 return;
3031 }
3032
3033 if ($second_try eq 'nfkccf' || $second_try eq 'dm') {
34132297 3034
d11155ec
KW
3035 # The 'nfkccf' property is stored in the old format for backwards
3036 # compatibility for any applications that has read its file
3037 # directly before prop_invmap() existed.
3038 # And the code above has extracted the 'dm' property from its file
3039 # yielding the same format. So here we convert them to adjusted
3040 # format for compatibility with the other properties similar to
3041 # them.
3042 my %revised_swash;
34132297 3043
d11155ec 3044 # We construct a new converted list.
34132297 3045 my $list = "";
d11155ec
KW
3046
3047 my @ranges = split "\n", $swash->{'LIST'};
3048 for (my $i = 0; $i < @ranges; $i++) {
3049 my ($hex_begin, $hex_end, $map) = split "\t", $ranges[$i];
3050
3051 # The dm property has maps that are space separated sequences
3052 # of code points, as well as the special entry "<hangul
3053 # syllable>, which also contains a blank.
3054 my @map = split " ", $map;
3055 if (@map > 1) {
3056
3057 # If it's just the special entry, append as-is.
3058 if ($map eq '<hangul syllable>') {
3059 $list .= "$ranges[$i]\n";
3060 }
3061 else {
3062
e2c04025 3063 # These should all be single-element ranges.
1a4c9760 3064 croak __PACKAGE__, "::prop_invmap: Not expecting a mapping with multiple code points in a multi-element range, $ranges[$i]" if $hex_end ne "" && $hex_end ne $hex_begin;
d11155ec
KW
3065
3066 # Convert them to decimal, as that's what's expected.
3067 $list .= "$hex_begin\t\t"
3068 . join(" ", map { hex } @map)
3069 . "\n";
3070 }
3071 next;
3072 }
3073
3074 # Here, the mapping doesn't have a blank, is for a single code
3075 # point.
34132297
KW
3076 my $begin = hex $hex_begin;
3077 my $end = (defined $hex_end && $hex_end ne "")
3078 ? hex $hex_end
3079 : $begin;
d11155ec
KW
3080
3081 # Again, the output is to be in decimal.
34132297 3082 my $decimal_map = hex $map;
d11155ec
KW
3083
3084 # We know that multi-element ranges with the same mapping
3085 # should not be adjusted, as after the adjustment
3086 # multi-element ranges are for consecutive increasing code
3087 # points. Further, the final element in the list won't be
3088 # adjusted, as there is nothing after it to include in the
3089 # adjustment
3090 if ($begin != $end || $i == @ranges -1) {
3091
3092 # So just convert these to single-element ranges
3093 foreach my $code_point ($begin .. $end) {
3094 $list .= sprintf("%04X\t\t%d\n",
3095 $code_point, $decimal_map);
3096 }
34132297 3097 }
d11155ec 3098 else {
34132297 3099
d11155ec
KW
3100 # Here, we have a candidate for adjusting. What we do is
3101 # look through the subsequent adjacent elements in the
3102 # input. If the map to the next one differs by 1 from the
3103 # one before, then we combine into a larger range with the
3104 # initial map. Loop doing this until we find one that
3105 # can't be combined.
3106
3107 my $offset = 0; # How far away are we from the initial
3108 # map
3109 my $squished = 0; # ? Did we squish at least two
3110 # elements together into one range
3111 for ( ; $i < @ranges; $i++) {
3112 my ($next_hex_begin, $next_hex_end, $next_map)
3113 = split "\t", $ranges[$i+1];
3114
3115 # In the case of 'dm', the map may be a sequence of
3116 # multiple code points, which are never combined with
3117 # another range
3118 last if $next_map =~ / /;
3119
3120 $offset++;
3121 my $next_decimal_map = hex $next_map;
3122
3123 # If the next map is not next in sequence, it
3124 # shouldn't be combined.
3125 last if $next_decimal_map != $decimal_map + $offset;
3126
3127 my $next_begin = hex $next_hex_begin;
3128
3129 # Likewise, if the next element isn't adjacent to the
3130 # previous one, it shouldn't be combined.
3131 last if $next_begin != $begin + $offset;
3132
3133 my $next_end = (defined $next_hex_end
3134 && $next_hex_end ne "")
3135 ? hex $next_hex_end
3136 : $next_begin;
3137
3138 # And finally, if the next element is a multi-element
3139 # range, it shouldn't be combined.
3140 last if $next_end != $next_begin;
3141
3142 # Here, we will combine. Loop to see if we should
3143 # combine the next element too.
3144 $squished = 1;
3145 }
3146
3147 if ($squished) {
3148
3149 # Here, 'i' is the element number of the last element to
3150 # be combined, and the range is single-element, or we
3151 # wouldn't be combining. Get it's code point.
3152 my ($hex_end, undef, undef) = split "\t", $ranges[$i];
3153 $list .= "$hex_begin\t$hex_end\t$decimal_map\n";
3154 } else {
3155
3156 # Here, no combining done. Just appen the initial
3157 # (and current) values.
3158 $list .= "$hex_begin\t\t$decimal_map\n";
3159 }
3160 }
3161 } # End of loop constructing the converted list
3162
3163 # Finish up the data structure for our converted swash
3164 my $type = ($second_try eq 'nfkccf') ? 'ToNFKCCF' : 'ToDm';
3165 $revised_swash{'LIST'} = $list;
3166 $revised_swash{'TYPE'} = $type;
3167 $revised_swash{'SPECIALS'} = $swash->{'SPECIALS'};
3168 $swash = \%revised_swash;
3169
3170 $utf8::SwashInfo{$type}{'missing'} = 0;
3171 $utf8::SwashInfo{$type}{'format'} = 'a';
62b3b855
KW
3172 }
3173 }
3174
3175 if ($swash->{'EXTRAS'}) {
3176 carp __PACKAGE__, "::prop_invmap: swash returned for $prop unexpectedly has EXTRAS magic";
3177 return;
3178 }
3179
3180 # Here, have a valid swash return. Examine it.
34132297 3181 my $returned_prop = $swash->{'TYPE'};
62b3b855
KW
3182
3183 # All properties but binary ones should have 'missing' and 'format'
3184 # entries
3185 $missing = $utf8::SwashInfo{$returned_prop}{'missing'};
3186 $missing = 'N' unless defined $missing;
3187
3188 $format = $utf8::SwashInfo{$returned_prop}{'format'};
3189 $format = 'b' unless defined $format;
3190
d11155ec
KW
3191 my $requires_adjustment = $format =~ /^a/;
3192
62b3b855
KW
3193 # The LIST input lines look like:
3194 # ...
3195 # 0374\t\tCommon
3196 # 0375\t0377\tGreek # [3]
3197 # 037A\t037D\tGreek # [4]
3198 # 037E\t\tCommon
3199 # 0384\t\tGreek
3200 # ...
3201 #
3202 # Convert them to like
3203 # 0374 => Common
3204 # 0375 => Greek
3205 # 0378 => $missing
3206 # 037A => Greek
3207 # 037E => Common
3208 # 037F => $missing
3209 # 0384 => Greek
3210 #
3211 # For binary properties, the final non-comment column is absent, and
3212 # assumed to be 'Y'.
3213
3214 foreach my $range (split "\n", $swash->{'LIST'}) {
3215 $range =~ s/ \s* (?: \# .* )? $ //xg; # rmv trailing space, comments
3216
3217 # Find the beginning and end of the range on the line
3218 my ($hex_begin, $hex_end, $map) = split "\t", $range;
3219 my $begin = hex $hex_begin;
3220 my $end = (defined $hex_end && $hex_end ne "")
3221 ? hex $hex_end
3222 : $begin;
3223
92bcf67b
KW
3224 # Each time through the loop (after the first):
3225 # $invlist[-2] contains the beginning of the previous range processed
3226 # $invlist[-1] contains the end+1 of the previous range processed
3227 # $invmap[-2] contains the value of the previous range processed
3228 # $invmap[-1] contains the default value for missing ranges ($missing)
3229 #
3230 # Thus, things are set up for the typical case of a new non-adjacent
3231 # range of non-missings to be added. But, if the new range is
dc8d8ea6 3232 # adjacent, it needs to replace the [-1] element; and if the new
92bcf67b
KW
3233 # range is a multiple value of the previous one, it needs to be added
3234 # to the [-2] map element.
3235
3236 # The first time through, everything will be empty. If the property
3237 # doesn't have a range that begins at 0, add one that maps to $missing
62b3b855
KW
3238 if (! @invlist) {
3239 if ($begin != 0) {
3240 push @invlist, 0;
3241 push @invmap, $missing;
3242 }
3243 }
e35c6019
KW
3244 elsif (@invlist > 1 && $invlist[-2] == $begin) {
3245
3246 # Here we handle the case where the input has multiple entries for
3247 # each code point. mktables should have made sure that each such
3248 # range contains only one code point. At this point, $invlist[-1]
3249 # is the $missing that was added at the end of the last loop
3250 # iteration, and [-2] is the last real input code point, and that
3251 # code point is the same as the one we are adding now, making the
3252 # new one a multiple entry. Add it to the existing entry, either
3253 # by pushing it to the existing list of multiple entries, or
3254 # converting the single current entry into a list with both on it.
3255 # This is all we need do for this iteration.
3256
3257 if ($end != $begin) {
294705a8 3258 croak __PACKAGE__, ":prop_invmap: Multiple maps per code point in '$prop' require single-element ranges: begin=$begin, end=$end, map=$map";
e35c6019
KW
3259 }
3260 if (! ref $invmap[-2]) {
3261 $invmap[-2] = [ $invmap[-2], $map ];
3262 }
3263 else {
3264 push @{$invmap[-2]}, $map;
3265 }
3266 $has_multiples = 1;
3267 next;
3268 }
62b3b855
KW
3269 elsif ($invlist[-1] == $begin) {
3270
3271 # If the input isn't in the most compact form, so that there are
3272 # two adjacent ranges that map to the same thing, they should be
d11155ec
KW
3273 # combined (EXCEPT where the arrays require adjustments, in which
3274 # case everything is already set up correctly). This happens in
3275 # our constructed dt mapping, as Element [-2] is the map for the
3276 # latest range so far processed. Just set the beginning point of
3277 # the map to $missing (in invlist[-1]) to 1 beyond where this
3278 # range ends. For example, in
62b3b855
KW
3279 # 12\t13\tXYZ
3280 # 14\t17\tXYZ
3281 # we have set it up so that it looks like
3282 # 12 => XYZ
3283 # 14 => $missing
3284 #
3285 # We now see that it should be
3286 # 12 => XYZ
3287 # 18 => $missing
d11155ec 3288 if (! $requires_adjustment && @invlist > 1 && ( (defined $map)
c887f93f
KW
3289 ? $invmap[-2] eq $map
3290 : $invmap[-2] eq 'Y'))
3291 {
62b3b855
KW
3292 $invlist[-1] = $end + 1;
3293 next;
3294 }
3295
3296 # Here, the range started in the previous iteration that maps to
3297 # $missing starts at the same code point as this range. That
3298 # means there is no gap to fill that that range was intended for,
3299 # so we just pop it off the parallel arrays.
3300 pop @invlist;
3301 pop @invmap;
3302 }
3303
3304 # Add the range beginning, and the range's map.
3305 push @invlist, $begin;
d11155ec 3306 if ($returned_prop eq 'ToDm') {
62b3b855
KW
3307
3308 # The decomposition maps are either a line like <hangul syllable>
3309 # which are to be taken as is; or a sequence of code points in hex
3310 # and separated by blanks. Convert them to decimal, and if there
3311 # is more than one, use an anonymous array as the map.
3312 if ($map =~ /^ < /x) {
3313 push @invmap, $map;
3314 }
3315 else {
bea2c146 3316 my @map = split " ", $map;
62b3b855
KW
3317 if (@map == 1) {
3318 push @invmap, $map[0];
3319 }
3320 else {
3321 push @invmap, \@map;
3322 }
3323 }
3324 }
3325 else {
3326
3327 # Otherwise, convert hex formatted list entries to decimal; add a
3328 # 'Y' map for the missing value in binary properties, or
3329 # otherwise, use the input map unchanged.
3330 $map = ($format eq 'x')
3331 ? hex $map
3332 : $format eq 'b'
3333 ? 'Y'
3334 : $map;
3335 push @invmap, $map;
3336 }
3337
3338 # We just started a range. It ends with $end. The gap between it and
3339 # the next element in the list must be filled with a range that maps
3340 # to the default value. If there is no gap, the next iteration will
3341 # pop this, unless there is no next iteration, and we have filled all
3342 # of the Unicode code space, so check for that and skip.
3343 if ($end < $MAX_UNICODE_CODEPOINT) {
3344 push @invlist, $end + 1;
3345 push @invmap, $missing;
3346 }
3347 }
3348
3349 # If the property is empty, make all code points use the value for missing
3350 # ones.
3351 if (! @invlist) {
3352 push @invlist, 0;
3353 push @invmap, $missing;
3354 }
3355
647396da 3356 # And add in standard element that all non-Unicode code points map to:
62b3b855
KW
3357 # $missing
3358 push @invlist, $MAX_UNICODE_CODEPOINT + 1;
3359 push @invmap, $missing;
3360
3361 # The second component of the map are those values that require
3362 # non-standard specification, stored in SPECIALS. These override any
3363 # duplicate code points in LIST. If we are using a proxy, we may have
3364 # already set $overrides based on the proxy.
3365 $overrides = $swash->{'SPECIALS'} unless defined $overrides;
3366 if ($overrides) {
3367
3368 # A negative $overrides implies that the SPECIALS should be ignored,
d11155ec 3369 # and a simple 'a' list is the value.
62b3b855 3370 if ($overrides < 0) {
d11155ec 3371 $format = 'a';
62b3b855
KW
3372 }
3373 else {
3374
3375 # Currently, all overrides are for properties that normally map to
3376 # single code points, but now some will map to lists of code
3377 # points (but there is an exception case handled below).
d11155ec 3378 $format = 'al';
62b3b855
KW
3379
3380 # Look through the overrides.
3381 foreach my $cp_maybe_utf8 (keys %$overrides) {
3382 my $cp;
3383 my @map;
3384
3385 # If the overrides came from SPECIALS, the code point keys are
3386 # packed UTF-8.
3387 if ($overrides == $swash->{'SPECIALS'}) {
3388 $cp = unpack("C0U", $cp_maybe_utf8);
3389 @map = unpack "U0U*", $swash->{'SPECIALS'}{$cp_maybe_utf8};
3390
3391 # The empty string will show up unpacked as an empty
3392 # array.
d11155ec 3393 $format = 'ale' if @map == 0;
62b3b855
KW
3394 }
3395 else {
3396
3397 # But if we generated the overrides, we didn't bother to
3398 # pack them, and we, so far, do this only for properties
d11155ec 3399 # that are 'a' ones.
62b3b855
KW
3400 $cp = $cp_maybe_utf8;
3401 @map = hex $overrides->{$cp};
d11155ec 3402 $format = 'a';
62b3b855
KW
3403 }
3404
3405 # Find the range that the override applies to.
3406 my $i = _search_invlist(\@invlist, $cp);
3407 if ($cp < $invlist[$i] || $cp >= $invlist[$i + 1]) {
294705a8 3408 croak __PACKAGE__, "::prop_invmap: wrong_range, cp=$cp; i=$i, current=$invlist[$i]; next=$invlist[$i + 1]"
62b3b855
KW
3409 }
3410
3411 # And what that range currently maps to
3412 my $cur_map = $invmap[$i];
3413
3414 # If there is a gap between the next range and the code point
3415 # we are overriding, we have to add elements to both arrays to
3416 # fill that gap, using the map that applies to it, which is
3417 # $cur_map, since it is part of the current range.
3418 if ($invlist[$i + 1] > $cp + 1) {
3419 #use feature 'say';
3420 #say "Before splice:";
3421 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3422 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3423 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3424 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3425 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3426
3427 splice @invlist, $i + 1, 0, $cp + 1;
3428 splice @invmap, $i + 1, 0, $cur_map;
3429
3430 #say "After splice:";
3431 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3432 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3433 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3434 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3435 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3436 }
3437
3438 # If the remaining portion of the range is multiple code
3439 # points (ending with the one we are replacing, guaranteed by
3440 # the earlier splice). We must split it into two
3441 if ($invlist[$i] < $cp) {
3442 $i++; # Compensate for the new element
3443
3444 #use feature 'say';
3445 #say "Before splice:";
3446 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3447 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3448 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3449 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3450 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3451
3452 splice @invlist, $i, 0, $cp;
3453 splice @invmap, $i, 0, 'dummy';
3454
3455 #say "After splice:";
3456 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3457 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3458 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3459 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3460 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3461 }
3462
3463 # Here, the range we are overriding contains a single code
3464 # point. The result could be the empty string, a single
3465 # value, or a list. If the last case, we use an anonymous
3466 # array.
3467 $invmap[$i] = (scalar @map == 0)
3468 ? ""
3469 : (scalar @map > 1)
3470 ? \@map
3471 : $map[0];
3472 }
3473 }
3474 }
3475 elsif ($format eq 'x') {
3476
647396da
KW
3477 # All hex-valued properties are really to code points, and have been
3478 # converted to decimal.
5bbfa552 3479 $format = 's';
62b3b855 3480 }
d11155ec
KW
3481 elsif ($returned_prop eq 'ToDm') {
3482 $format = 'ad';
62b3b855
KW
3483 }
3484 elsif ($format eq 'sw') { # blank-separated elements to form a list.
3485 map { $_ = [ split " ", $_ ] if $_ =~ / / } @invmap;
3486 $format = 'sl';
3487 }
3488 elsif ($returned_prop eq 'ToNameAlias') {
3489
3490 # This property currently doesn't have any lists, but theoretically
3491 # could
3492 $format = 'sl';
3493 }
b0b13ada 3494 elsif ($returned_prop eq 'ToPerlDecimalDigit') {
d11155ec 3495 $format = 'ae';
b0b13ada 3496 }
4f143a72
KW
3497 elsif ($returned_prop eq 'ToNv') {
3498
3499 # The one property that has this format is stored as a delta, so needs
3500 # to indicate that need to add code point to it.
3501 $format = 'ar';
3502 }
b577d4a6 3503 elsif ($format ne 'n' && $format ne 'a') {
62b3b855
KW
3504
3505 # All others are simple scalars
3506 $format = 's';
3507 }
e35c6019 3508 if ($has_multiples && $format !~ /l/) {
294705a8 3509 croak __PACKAGE__, "::prop_invmap: Wrong format '$format' for prop_invmap('$prop'); should indicate has lists";
e35c6019 3510 }
62b3b855
KW
3511
3512 return (\@invlist, \@invmap, $format, $missing);
3513}
3514
55d7b906 3515=head2 Unicode::UCD::UnicodeVersion
10a6ecd2 3516
a452d459
KW
3517This returns the version of the Unicode Character Database, in other words, the
3518version of the Unicode standard the database implements. The version is a
3519string of numbers delimited by dots (C<'.'>).
10a6ecd2
JH
3520
3521=cut
3522
3523my $UNICODEVERSION;
3524
3525sub UnicodeVersion {
3526 unless (defined $UNICODEVERSION) {
3527 openunicode(\$VERSIONFH, "version");
ce066323 3528 local $/ = "\n";
10a6ecd2
JH
3529 chomp($UNICODEVERSION = <$VERSIONFH>);
3530 close($VERSIONFH);
3531 croak __PACKAGE__, "::VERSION: strange version '$UNICODEVERSION'"
3532 unless $UNICODEVERSION =~ /^\d+(?:\.\d+)+$/;
3533 }
e80c2d9d 3534 $v_unicode_version = pack "C*", split /\./, $UNICODEVERSION;
10a6ecd2
JH
3535 return $UNICODEVERSION;
3536}
3aa957f9 3537
a452d459
KW
3538=head2 B<Blocks versus Scripts>
3539
3540The difference between a block and a script is that scripts are closer
3541to the linguistic notion of a set of code points required to present
3542languages, while block is more of an artifact of the Unicode code point
12cf36fe
KW
3543numbering and separation into blocks of consecutive code points (so far the
3544size of a block is some multiple of 16, like 128 or 256).
a452d459
KW
3545
3546For example the Latin B<script> is spread over several B<blocks>, such
3547as C<Basic Latin>, C<Latin 1 Supplement>, C<Latin Extended-A>, and
3548C<Latin Extended-B>. On the other hand, the Latin script does not
3549contain all the characters of the C<Basic Latin> block (also known as
3550ASCII): it includes only the letters, and not, for example, the digits
3551or the punctuation.
3552
3553For blocks see L<http://www.unicode.org/Public/UNIDATA/Blocks.txt>
3554
3555For scripts see UTR #24: L<http://www.unicode.org/unicode/reports/tr24/>
3556
3557=head2 B<Matching Scripts and Blocks>
3558
3559Scripts are matched with the regular-expression construct
3560C<\p{...}> (e.g. C<\p{Tibetan}> matches characters of the Tibetan script),
f200dd12 3561while C<\p{Blk=...}> is used for blocks (e.g. C<\p{Blk=Tibetan}> matches
a452d459
KW
3562any of the 256 code points in the Tibetan block).
3563
430fe03d
KW
3564=head2 Old-style versus new-style block names
3565
3566Unicode publishes the names of blocks in two different styles, though the two
3567are equivalent under Unicode's loose matching rules.
3568
3569The original style uses blanks and hyphens in the block names (except for
3570C<No_Block>), like so:
3571
3572 Miscellaneous Mathematical Symbols-B
3573
3574The newer style replaces these with underscores, like this:
3575
3576 Miscellaneous_Mathematical_Symbols_B
3577
3578This newer style is consistent with the values of other Unicode properties.
3579To preserve backward compatibility, all the functions in Unicode::UCD that
3580return block names (except one) return the old-style ones. That one function,
3581L</prop_value_aliases()> can be used to convert from old-style to new-style:
3582
3583 my $new_style = prop_values_aliases("block", $old_style);
3584
3585Perl also has single-form extensions that refer to blocks, C<In_Cyrillic>,
3586meaning C<Block=Cyrillic>. These have always been written in the new style.
3587
3588To convert from new-style to old-style, follow this recipe:
3589
3590 $old_style = charblock((prop_invlist("block=$new_style"))[0]);
3591
3592(which finds the range of code points in the block using C<prop_invlist>,
3593gets the lower end of the range (0th element) and then looks up the old name
3594for its block using C<charblock>).
3595
7620cb10
KW
3596Note that starting in Unicode 6.1, many of the block names have shorter
3597synonyms. These are always given in the new style.
3598
8b731da2
JH
3599=head1 BUGS
3600
3601Does not yet support EBCDIC platforms.
3602
561c79ed
JH
3603=head1 AUTHOR
3604
a18e976f 3605Jarkko Hietaniemi. Now maintained by perl5 porters.
561c79ed
JH
3606
3607=cut
3608
36091;