This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Unicode/UCD.pm: Clarify pod
[perl5.git] / lib / Unicode / UCD.pm
CommitLineData
55d7b906 1package Unicode::UCD;
561c79ed
JH
2
3use strict;
4use warnings;
36c2430c 5no warnings 'surrogate'; # surrogates can be inputs to this
98ef7649 6use charnames ();
561c79ed 7
c8652296 8our $VERSION = '0.46';
561c79ed
JH
9
10require Exporter;
11
12our @ISA = qw(Exporter);
74f8133e 13
10a6ecd2
JH
14our @EXPORT_OK = qw(charinfo
15 charblock charscript
16 charblocks charscripts
b08cd201 17 charinrange
ea508aee 18 general_categories bidi_types
b08cd201 19 compexcl
66aa79e2 20 casefold all_casefolds casespec
7319f91d
KW
21 namedseq
22 num
7ef25837
KW
23 prop_aliases
24 prop_value_aliases
681d705c 25 prop_invlist
62b3b855 26 prop_invmap
681d705c 27 MAX_CP
7319f91d 28 );
561c79ed
JH
29
30use Carp;
31
32=head1 NAME
33
55d7b906 34Unicode::UCD - Unicode character database
561c79ed
JH
35
36=head1 SYNOPSIS
37
55d7b906 38 use Unicode::UCD 'charinfo';
b08cd201 39 my $charinfo = charinfo($codepoint);
561c79ed 40
956cae9a
KW
41 use Unicode::UCD 'casefold';
42 my $casefold = casefold(0xFB00);
43
66aa79e2
KW
44 use Unicode::UCD 'all_casefolds';
45 my $all_casefolds_ref = all_casefolds();
46
5d8e6e41
KW
47 use Unicode::UCD 'casespec';
48 my $casespec = casespec(0xFB00);
49
55d7b906 50 use Unicode::UCD 'charblock';
e882dd67
JH
51 my $charblock = charblock($codepoint);
52
55d7b906 53 use Unicode::UCD 'charscript';
65044554 54 my $charscript = charscript($codepoint);
561c79ed 55
55d7b906 56 use Unicode::UCD 'charblocks';
e145285f
JH
57 my $charblocks = charblocks();
58
55d7b906 59 use Unicode::UCD 'charscripts';
ea508aee 60 my $charscripts = charscripts();
e145285f 61
55d7b906 62 use Unicode::UCD qw(charscript charinrange);
e145285f
JH
63 my $range = charscript($script);
64 print "looks like $script\n" if charinrange($range, $codepoint);
65
ea508aee
JH
66 use Unicode::UCD qw(general_categories bidi_types);
67 my $categories = general_categories();
68 my $types = bidi_types();
69
7ef25837
KW
70 use Unicode::UCD 'prop_aliases';
71 my @space_names = prop_aliases("space");
72
73 use Unicode::UCD 'prop_value_aliases';
74 my @gc_punct_names = prop_value_aliases("Gc", "Punct");
75
681d705c
KW
76 use Unicode::UCD 'prop_invlist';
77 my @puncts = prop_invlist("gc=punctuation");
78
62b3b855
KW
79 use Unicode::UCD 'prop_invmap';
80 my ($list_ref, $map_ref, $format, $missing)
81 = prop_invmap("General Category");
82
55d7b906 83 use Unicode::UCD 'compexcl';
e145285f
JH
84 my $compexcl = compexcl($codepoint);
85
a2bd7410
JH
86 use Unicode::UCD 'namedseq';
87 my $namedseq = namedseq($named_sequence_name);
88
55d7b906 89 my $unicode_version = Unicode::UCD::UnicodeVersion();
e145285f 90
7319f91d 91 my $convert_to_numeric =
62a8c8c2 92 Unicode::UCD::num("\N{RUMI DIGIT ONE}\N{RUMI DIGIT TWO}");
7319f91d 93
561c79ed
JH
94=head1 DESCRIPTION
95
a452d459
KW
96The Unicode::UCD module offers a series of functions that
97provide a simple interface to the Unicode
8b731da2 98Character Database.
561c79ed 99
a452d459
KW
100=head2 code point argument
101
102Some of the functions are called with a I<code point argument>, which is either
103a decimal or a hexadecimal scalar designating a Unicode code point, or C<U+>
104followed by hexadecimals designating a Unicode code point. In other words, if
105you want a code point to be interpreted as a hexadecimal number, you must
106prefix it with either C<0x> or C<U+>, because a string like e.g. C<123> will be
f200dd12
KW
107interpreted as a decimal code point. Note that the largest code point in
108Unicode is U+10FFFF.
c3e5bc54 109
561c79ed
JH
110=cut
111
10a6ecd2 112my $BLOCKSFH;
10a6ecd2 113my $VERSIONFH;
b08cd201
JH
114my $CASEFOLDFH;
115my $CASESPECFH;
a2bd7410 116my $NAMEDSEQFH;
e80c2d9d 117my $v_unicode_version; # v-string.
561c79ed
JH
118
119sub openunicode {
120 my ($rfh, @path) = @_;
121 my $f;
122 unless (defined $$rfh) {
123 for my $d (@INC) {
124 use File::Spec;
55d7b906 125 $f = File::Spec->catfile($d, "unicore", @path);
32c16050 126 last if open($$rfh, $f);
e882dd67 127 undef $f;
561c79ed 128 }
e882dd67
JH
129 croak __PACKAGE__, ": failed to find ",
130 File::Spec->catfile(@path), " in @INC"
131 unless defined $f;
561c79ed
JH
132 }
133 return $f;
134}
135
cb3150f5
KW
136sub _dclone ($) { # Use Storable::dclone if available; otherwise emulate it.
137
138 use if defined &DynaLoader::boot_DynaLoader, Storable => qw(dclone);
139
140 return dclone(shift) if defined &dclone;
141
142 my $arg = shift;
143 my $type = ref $arg;
144 return $arg unless $type; # No deep cloning needed for scalars
145
146 if ($type eq 'ARRAY') {
147 my @return;
148 foreach my $element (@$arg) {
149 push @return, &_dclone($element);
150 }
151 return \@return;
152 }
153 elsif ($type eq 'HASH') {
154 my %return;
155 foreach my $key (keys %$arg) {
156 $return{$key} = &_dclone($arg->{$key});
157 }
158 return \%return;
159 }
160 else {
161 croak "_dclone can't handle " . $type;
162 }
163}
164
a452d459 165=head2 B<charinfo()>
561c79ed 166
55d7b906 167 use Unicode::UCD 'charinfo';
561c79ed 168
b08cd201 169 my $charinfo = charinfo(0x41);
561c79ed 170
a452d459
KW
171This returns information about the input L</code point argument>
172as a reference to a hash of fields as defined by the Unicode
173standard. If the L</code point argument> is not assigned in the standard
174(i.e., has the general category C<Cn> meaning C<Unassigned>)
175or is a non-character (meaning it is guaranteed to never be assigned in
176the standard),
a18e976f 177C<undef> is returned.
a452d459
KW
178
179Fields that aren't applicable to the particular code point argument exist in the
180returned hash, and are empty.
181
182The keys in the hash with the meanings of their values are:
183
184=over
185
186=item B<code>
187
188the input L</code point argument> expressed in hexadecimal, with leading zeros
189added if necessary to make it contain at least four hexdigits
190
191=item B<name>
192
193name of I<code>, all IN UPPER CASE.
194Some control-type code points do not have names.
195This field will be empty for C<Surrogate> and C<Private Use> code points,
196and for the others without a name,
197it will contain a description enclosed in angle brackets, like
198C<E<lt>controlE<gt>>.
199
200
201=item B<category>
202
203The short name of the general category of I<code>.
204This will match one of the keys in the hash returned by L</general_categories()>.
205
7ef25837
KW
206The L</prop_value_aliases()> function can be used to get all the synonyms
207of the category name.
208
a452d459
KW
209=item B<combining>
210
211the combining class number for I<code> used in the Canonical Ordering Algorithm.
212For Unicode 5.1, this is described in Section 3.11 C<Canonical Ordering Behavior>
213available at
214L<http://www.unicode.org/versions/Unicode5.1.0/>
215
7ef25837
KW
216The L</prop_value_aliases()> function can be used to get all the synonyms
217of the combining class number.
218
a452d459
KW
219=item B<bidi>
220
221bidirectional type of I<code>.
222This will match one of the keys in the hash returned by L</bidi_types()>.
223
7ef25837
KW
224The L</prop_value_aliases()> function can be used to get all the synonyms
225of the bidi type name.
226
a452d459
KW
227=item B<decomposition>
228
229is empty if I<code> has no decomposition; or is one or more codes
a18e976f 230(separated by spaces) that, taken in order, represent a decomposition for
a452d459
KW
231I<code>. Each has at least four hexdigits.
232The codes may be preceded by a word enclosed in angle brackets then a space,
233like C<E<lt>compatE<gt> >, giving the type of decomposition
234
06bba7d5
KW
235This decomposition may be an intermediate one whose components are also
236decomposable. Use L<Unicode::Normalize> to get the final decomposition.
237
a452d459
KW
238=item B<decimal>
239
240if I<code> is a decimal digit this is its integer numeric value
241
242=item B<digit>
243
89e4a205
KW
244if I<code> represents some other digit-like number, this is its integer
245numeric value
a452d459
KW
246
247=item B<numeric>
248
249if I<code> represents a whole or rational number, this is its numeric value.
250Rational values are expressed as a string like C<1/4>.
251
252=item B<mirrored>
253
254C<Y> or C<N> designating if I<code> is mirrored in bidirectional text
255
256=item B<unicode10>
257
258name of I<code> in the Unicode 1.0 standard if one
259existed for this code point and is different from the current name
260
261=item B<comment>
262
89e4a205 263As of Unicode 6.0, this is always empty.
a452d459
KW
264
265=item B<upper>
266
06bba7d5 267is empty if there is no single code point uppercase mapping for I<code>
4f66642e 268(its uppercase mapping is itself);
a452d459
KW
269otherwise it is that mapping expressed as at least four hexdigits.
270(L</casespec()> should be used in addition to B<charinfo()>
271for case mappings when the calling program can cope with multiple code point
272mappings.)
273
274=item B<lower>
275
06bba7d5 276is empty if there is no single code point lowercase mapping for I<code>
4f66642e 277(its lowercase mapping is itself);
a452d459
KW
278otherwise it is that mapping expressed as at least four hexdigits.
279(L</casespec()> should be used in addition to B<charinfo()>
280for case mappings when the calling program can cope with multiple code point
281mappings.)
282
283=item B<title>
284
06bba7d5 285is empty if there is no single code point titlecase mapping for I<code>
4f66642e 286(its titlecase mapping is itself);
a452d459
KW
287otherwise it is that mapping expressed as at least four hexdigits.
288(L</casespec()> should be used in addition to B<charinfo()>
289for case mappings when the calling program can cope with multiple code point
290mappings.)
291
292=item B<block>
293
a18e976f 294the block I<code> belongs to (used in C<\p{Blk=...}>).
a452d459
KW
295See L</Blocks versus Scripts>.
296
297
298=item B<script>
299
a18e976f 300the script I<code> belongs to.
a452d459
KW
301See L</Blocks versus Scripts>.
302
303=back
32c16050
JH
304
305Note that you cannot do (de)composition and casing based solely on the
a452d459
KW
306I<decomposition>, I<combining>, I<lower>, I<upper>, and I<title> fields;
307you will need also the L</compexcl()>, and L</casespec()> functions.
561c79ed
JH
308
309=cut
310
e10d7780 311# NB: This function is nearly duplicated in charnames.pm
10a6ecd2
JH
312sub _getcode {
313 my $arg = shift;
314
dc0a4417 315 if ($arg =~ /^[1-9]\d*$/) {
10a6ecd2 316 return $arg;
dc0a4417 317 } elsif ($arg =~ /^(?:[Uu]\+|0[xX])?([[:xdigit:]]+)$/) {
10a6ecd2
JH
318 return hex($1);
319 }
320
321 return;
322}
323
05dbc6f8
KW
324# Populated by _num. Converts real number back to input rational
325my %real_to_rational;
326
327# To store the contents of files found on disk.
328my @BIDIS;
329my @CATEGORIES;
330my @DECOMPOSITIONS;
331my @NUMERIC_TYPES;
5c3b35c9
KW
332my %SIMPLE_LOWER;
333my %SIMPLE_TITLE;
334my %SIMPLE_UPPER;
335my %UNICODE_1_NAMES;
72fcb9f0 336my %ISO_COMMENT;
05dbc6f8 337
05dbc6f8 338sub charinfo {
a6fa416b 339
05dbc6f8
KW
340 # This function has traditionally mimicked what is in UnicodeData.txt,
341 # warts and all. This is a re-write that avoids UnicodeData.txt so that
342 # it can be removed to save disk space. Instead, this assembles
343 # information gotten by other methods that get data from various other
344 # files. It uses charnames to get the character name; and various
345 # mktables tables.
324f9e44 346
05dbc6f8 347 use feature 'unicode_strings';
a6fa416b 348
cb3150f5
KW
349 # Will fail if called under minitest
350 use if defined &DynaLoader::boot_DynaLoader, "Unicode::Normalize" => qw(getCombinClass NFD);
351
10a6ecd2
JH
352 my $arg = shift;
353 my $code = _getcode($arg);
05dbc6f8
KW
354 croak __PACKAGE__, "::charinfo: unknown code '$arg'" unless defined $code;
355
356 # Non-unicode implies undef.
357 return if $code > 0x10FFFF;
358
359 my %prop;
360 my $char = chr($code);
361
35a865d4 362 @CATEGORIES =_read_table("To/Gc.pl") unless @CATEGORIES;
05dbc6f8
KW
363 $prop{'category'} = _search(\@CATEGORIES, 0, $#CATEGORIES, $code)
364 // $utf8::SwashInfo{'ToGc'}{'missing'};
365
366 return if $prop{'category'} eq 'Cn'; # Unassigned code points are undef
367
368 $prop{'code'} = sprintf "%04X", $code;
369 $prop{'name'} = ($char =~ /\p{Cntrl}/) ? '<control>'
370 : (charnames::viacode($code) // "");
371
372 $prop{'combining'} = getCombinClass($code);
373
35a865d4 374 @BIDIS =_read_table("To/Bc.pl") unless @BIDIS;
05dbc6f8
KW
375 $prop{'bidi'} = _search(\@BIDIS, 0, $#BIDIS, $code)
376 // $utf8::SwashInfo{'ToBc'}{'missing'};
377
378 # For most code points, we can just read in "unicore/Decomposition.pl", as
379 # its contents are exactly what should be output. But that file doesn't
380 # contain the data for the Hangul syllable decompositions, which can be
94c91ffc
KW
381 # algorithmically computed, and NFD() does that, so we call NFD() for
382 # those. We can't use NFD() for everything, as it does a complete
05dbc6f8 383 # recursive decomposition, and what this function has always done is to
94c91ffc
KW
384 # return what's in UnicodeData.txt which doesn't show that recursiveness.
385 # Fortunately, the NFD() of the Hanguls doesn't have any recursion
386 # issues.
387 # Having no decomposition implies an empty field; otherwise, all but
388 # "Canonical" imply a compatible decomposition, and the type is prefixed
389 # to that, as it is in UnicodeData.txt
9abdc62b
KW
390 UnicodeVersion() unless defined $v_unicode_version;
391 if ($v_unicode_version ge v2.0.0 && $char =~ /\p{Block=Hangul_Syllables}/) {
05dbc6f8
KW
392 # The code points of the decomposition are output in standard Unicode
393 # hex format, separated by blanks.
394 $prop{'decomposition'} = join " ", map { sprintf("%04X", $_)}
94c91ffc 395 unpack "U*", NFD($char);
a6fa416b 396 }
05dbc6f8 397 else {
35a865d4 398 @DECOMPOSITIONS = _read_table("Decomposition.pl")
05dbc6f8
KW
399 unless @DECOMPOSITIONS;
400 $prop{'decomposition'} = _search(\@DECOMPOSITIONS, 0, $#DECOMPOSITIONS,
401 $code) // "";
561c79ed 402 }
05dbc6f8
KW
403
404 # Can use num() to get the numeric values, if any.
405 if (! defined (my $value = num($char))) {
406 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = "";
407 }
408 else {
409 if ($char =~ /\d/) {
410 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = $value;
411 }
412 else {
413
414 # For non-decimal-digits, we have to read in the Numeric type
415 # to distinguish them. It is not just a matter of integer vs.
416 # rational, as some whole number values are not considered digits,
417 # e.g., TAMIL NUMBER TEN.
418 $prop{'decimal'} = "";
419
35a865d4 420 @NUMERIC_TYPES =_read_table("To/Nt.pl") unless @NUMERIC_TYPES;
05dbc6f8
KW
421 if ((_search(\@NUMERIC_TYPES, 0, $#NUMERIC_TYPES, $code) // "")
422 eq 'Digit')
423 {
424 $prop{'digit'} = $prop{'numeric'} = $value;
425 }
426 else {
427 $prop{'digit'} = "";
428 $prop{'numeric'} = $real_to_rational{$value} // $value;
429 }
430 }
431 }
432
433 $prop{'mirrored'} = ($char =~ /\p{Bidi_Mirrored}/) ? 'Y' : 'N';
434
35a865d4 435 %UNICODE_1_NAMES =_read_table("To/Na1.pl", "use_hash") unless %UNICODE_1_NAMES;
5c3b35c9 436 $prop{'unicode10'} = $UNICODE_1_NAMES{$code} // "";
05dbc6f8 437
72fcb9f0
KW
438 UnicodeVersion() unless defined $v_unicode_version;
439 if ($v_unicode_version ge v6.0.0) {
440 $prop{'comment'} = "";
441 }
442 else {
443 %ISO_COMMENT = _read_table("To/Isc.pl", "use_hash") unless %ISO_COMMENT;
444 $prop{'comment'} = (defined $ISO_COMMENT{$code})
445 ? $ISO_COMMENT{$code}
446 : "";
447 }
05dbc6f8 448
35a865d4 449 %SIMPLE_UPPER = _read_table("To/Uc.pl", "use_hash") unless %SIMPLE_UPPER;
bf7fe2df 450 $prop{'upper'} = (defined $SIMPLE_UPPER{$code})
d11155ec 451 ? sprintf("%04X", $SIMPLE_UPPER{$code})
bf7fe2df 452 : "";
75e7c50b 453
35a865d4 454 %SIMPLE_LOWER = _read_table("To/Lc.pl", "use_hash") unless %SIMPLE_LOWER;
bf7fe2df 455 $prop{'lower'} = (defined $SIMPLE_LOWER{$code})
d11155ec 456 ? sprintf("%04X", $SIMPLE_LOWER{$code})
bf7fe2df 457 : "";
75e7c50b 458
35a865d4 459 %SIMPLE_TITLE = _read_table("To/Tc.pl", "use_hash") unless %SIMPLE_TITLE;
bf7fe2df 460 $prop{'title'} = (defined $SIMPLE_TITLE{$code})
d11155ec 461 ? sprintf("%04X", $SIMPLE_TITLE{$code})
bf7fe2df 462 : "";
05dbc6f8
KW
463
464 $prop{block} = charblock($code);
465 $prop{script} = charscript($code);
466 return \%prop;
561c79ed
JH
467}
468
e882dd67
JH
469sub _search { # Binary search in a [[lo,hi,prop],[...],...] table.
470 my ($table, $lo, $hi, $code) = @_;
471
472 return if $lo > $hi;
473
474 my $mid = int(($lo+$hi) / 2);
475
476 if ($table->[$mid]->[0] < $code) {
10a6ecd2 477 if ($table->[$mid]->[1] >= $code) {
e882dd67
JH
478 return $table->[$mid]->[2];
479 } else {
480 _search($table, $mid + 1, $hi, $code);
481 }
482 } elsif ($table->[$mid]->[0] > $code) {
483 _search($table, $lo, $mid - 1, $code);
484 } else {
485 return $table->[$mid]->[2];
486 }
487}
488
cb366075 489sub _read_table ($;$) {
3a12600d
KW
490
491 # Returns the contents of the mktables generated table file located at $1
cb366075
KW
492 # in the form of either an array of arrays or a hash, depending on if the
493 # optional second parameter is true (for hash return) or not. In the case
494 # of a hash return, each key is a code point, and its corresponding value
495 # is what the table gives as the code point's corresponding value. In the
496 # case of an array return, each outer array denotes a range with [0] the
497 # start point of that range; [1] the end point; and [2] the value that
498 # every code point in the range has. The hash return is useful for fast
499 # lookup when the table contains only single code point ranges. The array
500 # return takes much less memory when there are large ranges.
3a12600d 501 #
cb366075 502 # This function has the side effect of setting
3a12600d
KW
503 # $utf8::SwashInfo{$property}{'format'} to be the mktables format of the
504 # table; and
505 # $utf8::SwashInfo{$property}{'missing'} to be the value for all entries
506 # not listed in the table.
507 # where $property is the Unicode property name, preceded by 'To' for map
508 # properties., e.g., 'ToSc'.
509 #
510 # Table entries look like one of:
511 # 0000 0040 Common # [65]
512 # 00AA Latin
513
514 my $table = shift;
cb366075
KW
515 my $return_hash = shift;
516 $return_hash = 0 unless defined $return_hash;
3a12600d 517 my @return;
cb366075 518 my %return;
3a12600d 519 local $_;
d11155ec 520 my $list = do "unicore/$table";
3a12600d 521
d11155ec
KW
522 # Look up if this property requires adjustments, which we do below if it
523 # does.
524 require "unicore/Heavy.pl";
525 my $property = $table =~ s/\.pl//r;
526 $property = $utf8::file_to_swash_name{$property};
527 my $to_adjust = defined $property
528 && $utf8::SwashInfo{$property}{'format'} eq 'a';
529
530 for (split /^/m, $list) {
3a12600d
KW
531 my ($start, $end, $value) = / ^ (.+?) \t (.*?) \t (.+?)
532 \s* ( \# .* )? # Optional comment
533 $ /x;
83fd1222
KW
534 my $decimal_start = hex $start;
535 my $decimal_end = ($end eq "") ? $decimal_start : hex $end;
cb366075 536 if ($return_hash) {
83fd1222 537 foreach my $i ($decimal_start .. $decimal_end) {
d11155ec
KW
538 $return{$i} = ($to_adjust)
539 ? $value + $i - $decimal_start
540 : $value;
cb366075
KW
541 }
542 }
d11155ec
KW
543 elsif (! $to_adjust
544 && @return
545 && $return[-1][1] == $decimal_start - 1
9a96c106
KW
546 && $return[-1][2] eq $value)
547 {
548 # If this is merely extending the previous range, do just that.
549 $return[-1]->[1] = $decimal_end;
550 }
cb366075 551 else {
83fd1222 552 push @return, [ $decimal_start, $decimal_end, $value ];
cb366075 553 }
3a12600d 554 }
cb366075 555 return ($return_hash) ? %return : @return;
3a12600d
KW
556}
557
10a6ecd2
JH
558sub charinrange {
559 my ($range, $arg) = @_;
560 my $code = _getcode($arg);
561 croak __PACKAGE__, "::charinrange: unknown code '$arg'"
562 unless defined $code;
563 _search($range, 0, $#$range, $code);
564}
565
a452d459 566=head2 B<charblock()>
561c79ed 567
55d7b906 568 use Unicode::UCD 'charblock';
561c79ed
JH
569
570 my $charblock = charblock(0x41);
10a6ecd2 571 my $charblock = charblock(1234);
a452d459 572 my $charblock = charblock(0x263a);
10a6ecd2
JH
573 my $charblock = charblock("U+263a");
574
78bf21c2 575 my $range = charblock('Armenian');
10a6ecd2 576
a452d459 577With a L</code point argument> charblock() returns the I<block> the code point
430fe03d
KW
578belongs to, e.g. C<Basic Latin>. The old-style block name is returned (see
579L</Old-style versus new-style block names>).
a452d459 580If the code point is unassigned, this returns the block it would belong to if
fe252ba7
KW
581it were assigned. (If the Unicode version being used is so early as to not
582have blocks, all code points are considered to be in C<No_Block>.)
10a6ecd2 583
78bf21c2
JH
584See also L</Blocks versus Scripts>.
585
18972f4b 586If supplied with an argument that can't be a code point, charblock() tries to
430fe03d
KW
587do the opposite and interpret the argument as an old-style block name. The
588return value
a18e976f
KW
589is a I<range set> with one range: an anonymous list with a single element that
590consists of another anonymous list whose first element is the first code point
591in the block, and whose second (and final) element is the final code point in
592the block. (The extra list consisting of just one element is so that the same
593program logic can be used to handle both this return, and the return from
594L</charscript()> which can have multiple ranges.) You can test whether a code
595point is in a range using the L</charinrange()> function. If the argument is
596not a known block, C<undef> is returned.
561c79ed 597
561c79ed
JH
598=cut
599
600my @BLOCKS;
10a6ecd2 601my %BLOCKS;
561c79ed 602
10a6ecd2 603sub _charblocks {
06bba7d5
KW
604
605 # Can't read from the mktables table because it loses the hyphens in the
606 # original.
561c79ed 607 unless (@BLOCKS) {
fe252ba7
KW
608 UnicodeVersion() unless defined $v_unicode_version;
609 if ($v_unicode_version lt v2.0.0) {
610 my $subrange = [ 0, 0x10FFFF, 'No_Block' ];
611 push @BLOCKS, $subrange;
612 push @{$BLOCKS{$3}}, $subrange;
613 }
614 elsif (openunicode(\$BLOCKSFH, "Blocks.txt")) {
6c8d78fb 615 local $_;
ce066323 616 local $/ = "\n";
10a6ecd2 617 while (<$BLOCKSFH>) {
2796c109 618 if (/^([0-9A-F]+)\.\.([0-9A-F]+);\s+(.+)/) {
10a6ecd2
JH
619 my ($lo, $hi) = (hex($1), hex($2));
620 my $subrange = [ $lo, $hi, $3 ];
621 push @BLOCKS, $subrange;
622 push @{$BLOCKS{$3}}, $subrange;
561c79ed
JH
623 }
624 }
10a6ecd2 625 close($BLOCKSFH);
561c79ed
JH
626 }
627 }
10a6ecd2
JH
628}
629
630sub charblock {
631 my $arg = shift;
632
633 _charblocks() unless @BLOCKS;
634
635 my $code = _getcode($arg);
561c79ed 636
10a6ecd2 637 if (defined $code) {
c707cf8e
KW
638 my $result = _search(\@BLOCKS, 0, $#BLOCKS, $code);
639 return $result if defined $result;
640 return 'No_Block';
641 }
642 elsif (exists $BLOCKS{$arg}) {
cb3150f5 643 return _dclone $BLOCKS{$arg};
10a6ecd2 644 }
e882dd67
JH
645}
646
a452d459 647=head2 B<charscript()>
e882dd67 648
55d7b906 649 use Unicode::UCD 'charscript';
e882dd67
JH
650
651 my $charscript = charscript(0x41);
10a6ecd2
JH
652 my $charscript = charscript(1234);
653 my $charscript = charscript("U+263a");
e882dd67 654
78bf21c2 655 my $range = charscript('Thai');
10a6ecd2 656
a452d459
KW
657With a L</code point argument> charscript() returns the I<script> the
658code point belongs to, e.g. C<Latin>, C<Greek>, C<Han>.
49ea58c8
KW
659If the code point is unassigned or the Unicode version being used is so early
660that it doesn't have scripts, this function returns C<"Unknown">.
78bf21c2 661
eb0cc9e3 662If supplied with an argument that can't be a code point, charscript() tries
a18e976f
KW
663to do the opposite and interpret the argument as a script name. The
664return value is a I<range set>: an anonymous list of lists that contain
eb0cc9e3 665I<start-of-range>, I<end-of-range> code point pairs. You can test whether a
a18e976f
KW
666code point is in a range set using the L</charinrange()> function. If the
667argument is not a known script, C<undef> is returned.
a452d459
KW
668
669See also L</Blocks versus Scripts>.
e882dd67 670
e882dd67
JH
671=cut
672
673my @SCRIPTS;
10a6ecd2 674my %SCRIPTS;
e882dd67 675
10a6ecd2 676sub _charscripts {
49ea58c8
KW
677 unless (@SCRIPTS) {
678 UnicodeVersion() unless defined $v_unicode_version;
679 if ($v_unicode_version lt v3.1.0) {
680 push @SCRIPTS, [ 0, 0x10FFFF, 'Unknown' ];
681 }
682 else {
683 @SCRIPTS =_read_table("To/Sc.pl");
684 }
685 }
7bccef0b 686 foreach my $entry (@SCRIPTS) {
f3d50ac9 687 $entry->[2] =~ s/(_\w)/\L$1/g; # Preserve old-style casing
7bccef0b 688 push @{$SCRIPTS{$entry->[2]}}, $entry;
e882dd67 689 }
10a6ecd2
JH
690}
691
692sub charscript {
693 my $arg = shift;
694
695 _charscripts() unless @SCRIPTS;
e882dd67 696
10a6ecd2
JH
697 my $code = _getcode($arg);
698
699 if (defined $code) {
7bccef0b
KW
700 my $result = _search(\@SCRIPTS, 0, $#SCRIPTS, $code);
701 return $result if defined $result;
8079ad82 702 return $utf8::SwashInfo{'ToSc'}{'missing'};
7bccef0b 703 } elsif (exists $SCRIPTS{$arg}) {
cb3150f5 704 return _dclone $SCRIPTS{$arg};
10a6ecd2 705 }
7bccef0b
KW
706
707 return;
10a6ecd2
JH
708}
709
a452d459 710=head2 B<charblocks()>
10a6ecd2 711
55d7b906 712 use Unicode::UCD 'charblocks';
10a6ecd2 713
b08cd201 714 my $charblocks = charblocks();
10a6ecd2 715
b08cd201 716charblocks() returns a reference to a hash with the known block names
a452d459 717as the keys, and the code point ranges (see L</charblock()>) as the values.
10a6ecd2 718
430fe03d
KW
719The names are in the old-style (see L</Old-style versus new-style block
720names>).
721
62b3b855
KW
722L<prop_invmap("block")|/prop_invmap()> can be used to get this same data in a
723different type of data structure.
724
78bf21c2
JH
725See also L</Blocks versus Scripts>.
726
10a6ecd2
JH
727=cut
728
729sub charblocks {
b08cd201 730 _charblocks() unless %BLOCKS;
cb3150f5 731 return _dclone \%BLOCKS;
10a6ecd2
JH
732}
733
a452d459 734=head2 B<charscripts()>
10a6ecd2 735
55d7b906 736 use Unicode::UCD 'charscripts';
10a6ecd2 737
ea508aee 738 my $charscripts = charscripts();
10a6ecd2 739
ea508aee 740charscripts() returns a reference to a hash with the known script
a452d459 741names as the keys, and the code point ranges (see L</charscript()>) as
ea508aee 742the values.
10a6ecd2 743
62b3b855
KW
744L<prop_invmap("script")|/prop_invmap()> can be used to get this same data in a
745different type of data structure.
746
78bf21c2
JH
747See also L</Blocks versus Scripts>.
748
10a6ecd2
JH
749=cut
750
751sub charscripts {
b08cd201 752 _charscripts() unless %SCRIPTS;
cb3150f5 753 return _dclone \%SCRIPTS;
561c79ed
JH
754}
755
a452d459 756=head2 B<charinrange()>
10a6ecd2 757
f200dd12 758In addition to using the C<\p{Blk=...}> and C<\P{Blk=...}> constructs, you
10a6ecd2 759can also test whether a code point is in the I<range> as returned by
a452d459
KW
760L</charblock()> and L</charscript()> or as the values of the hash returned
761by L</charblocks()> and L</charscripts()> by using charinrange():
10a6ecd2 762
55d7b906 763 use Unicode::UCD qw(charscript charinrange);
10a6ecd2
JH
764
765 $range = charscript('Hiragana');
e145285f 766 print "looks like hiragana\n" if charinrange($range, $codepoint);
10a6ecd2
JH
767
768=cut
769
ea508aee
JH
770my %GENERAL_CATEGORIES =
771 (
772 'L' => 'Letter',
773 'LC' => 'CasedLetter',
774 'Lu' => 'UppercaseLetter',
775 'Ll' => 'LowercaseLetter',
776 'Lt' => 'TitlecaseLetter',
777 'Lm' => 'ModifierLetter',
778 'Lo' => 'OtherLetter',
779 'M' => 'Mark',
780 'Mn' => 'NonspacingMark',
781 'Mc' => 'SpacingMark',
782 'Me' => 'EnclosingMark',
783 'N' => 'Number',
784 'Nd' => 'DecimalNumber',
785 'Nl' => 'LetterNumber',
786 'No' => 'OtherNumber',
787 'P' => 'Punctuation',
788 'Pc' => 'ConnectorPunctuation',
789 'Pd' => 'DashPunctuation',
790 'Ps' => 'OpenPunctuation',
791 'Pe' => 'ClosePunctuation',
792 'Pi' => 'InitialPunctuation',
793 'Pf' => 'FinalPunctuation',
794 'Po' => 'OtherPunctuation',
795 'S' => 'Symbol',
796 'Sm' => 'MathSymbol',
797 'Sc' => 'CurrencySymbol',
798 'Sk' => 'ModifierSymbol',
799 'So' => 'OtherSymbol',
800 'Z' => 'Separator',
801 'Zs' => 'SpaceSeparator',
802 'Zl' => 'LineSeparator',
803 'Zp' => 'ParagraphSeparator',
804 'C' => 'Other',
805 'Cc' => 'Control',
806 'Cf' => 'Format',
807 'Cs' => 'Surrogate',
808 'Co' => 'PrivateUse',
809 'Cn' => 'Unassigned',
810 );
811
812sub general_categories {
cb3150f5 813 return _dclone \%GENERAL_CATEGORIES;
ea508aee
JH
814}
815
a452d459 816=head2 B<general_categories()>
ea508aee
JH
817
818 use Unicode::UCD 'general_categories';
819
820 my $categories = general_categories();
821
a452d459 822This returns a reference to a hash which has short
ea508aee
JH
823general category names (such as C<Lu>, C<Nd>, C<Zs>, C<S>) as keys and long
824names (such as C<UppercaseLetter>, C<DecimalNumber>, C<SpaceSeparator>,
825C<Symbol>) as values. The hash is reversible in case you need to go
826from the long names to the short names. The general category is the
a452d459
KW
827one returned from
828L</charinfo()> under the C<category> key.
ea508aee 829
7ef25837
KW
830The L</prop_value_aliases()> function can be used to get all the synonyms of
831the category name.
832
ea508aee
JH
833=cut
834
835my %BIDI_TYPES =
836 (
837 'L' => 'Left-to-Right',
838 'LRE' => 'Left-to-Right Embedding',
839 'LRO' => 'Left-to-Right Override',
840 'R' => 'Right-to-Left',
841 'AL' => 'Right-to-Left Arabic',
842 'RLE' => 'Right-to-Left Embedding',
843 'RLO' => 'Right-to-Left Override',
844 'PDF' => 'Pop Directional Format',
845 'EN' => 'European Number',
846 'ES' => 'European Number Separator',
847 'ET' => 'European Number Terminator',
848 'AN' => 'Arabic Number',
849 'CS' => 'Common Number Separator',
850 'NSM' => 'Non-Spacing Mark',
851 'BN' => 'Boundary Neutral',
852 'B' => 'Paragraph Separator',
853 'S' => 'Segment Separator',
854 'WS' => 'Whitespace',
855 'ON' => 'Other Neutrals',
856 );
857
a452d459 858=head2 B<bidi_types()>
ea508aee
JH
859
860 use Unicode::UCD 'bidi_types';
861
862 my $categories = bidi_types();
863
a452d459 864This returns a reference to a hash which has the short
ea508aee
JH
865bidi (bidirectional) type names (such as C<L>, C<R>) as keys and long
866names (such as C<Left-to-Right>, C<Right-to-Left>) as values. The
867hash is reversible in case you need to go from the long names to the
a452d459
KW
868short names. The bidi type is the one returned from
869L</charinfo()>
ea508aee
JH
870under the C<bidi> key. For the exact meaning of the various bidi classes
871the Unicode TR9 is recommended reading:
a452d459 872L<http://www.unicode.org/reports/tr9/>
ea508aee
JH
873(as of Unicode 5.0.0)
874
7ef25837
KW
875The L</prop_value_aliases()> function can be used to get all the synonyms of
876the bidi type name.
877
ea508aee
JH
878=cut
879
a452d459 880sub bidi_types {
cb3150f5 881 return _dclone \%BIDI_TYPES;
a452d459
KW
882}
883
884=head2 B<compexcl()>
b08cd201 885
55d7b906 886 use Unicode::UCD 'compexcl';
b08cd201 887
a452d459 888 my $compexcl = compexcl(0x09dc);
b08cd201 889
2afba6a4
KW
890This routine returns C<undef> if the Unicode version being used is so early
891that it doesn't have this property. It is included for backwards
892compatibility, but as of Perl 5.12 and more modern Unicode versions, for
71a442a8
KW
893most purposes it is probably more convenient to use one of the following
894instead:
895
896 my $compexcl = chr(0x09dc) =~ /\p{Comp_Ex};
897 my $compexcl = chr(0x09dc) =~ /\p{Full_Composition_Exclusion};
898
899or even
900
901 my $compexcl = chr(0x09dc) =~ /\p{CE};
902 my $compexcl = chr(0x09dc) =~ /\p{Composition_Exclusion};
903
904The first two forms return B<true> if the L</code point argument> should not
76b05678
KW
905be produced by composition normalization. For the final two forms to return
906B<true>, it is additionally required that this fact not otherwise be
907determinable from the Unicode data base.
71a442a8
KW
908
909This routine behaves identically to the final two forms. That is,
910it does not return B<true> if the code point has a decomposition
a452d459
KW
911consisting of another single code point, nor if its decomposition starts
912with a code point whose combining class is non-zero. Code points that meet
913either of these conditions should also not be produced by composition
71a442a8
KW
914normalization, which is probably why you should use the
915C<Full_Composition_Exclusion> property instead, as shown above.
b08cd201 916
71a442a8 917The routine returns B<false> otherwise.
b08cd201
JH
918
919=cut
920
b08cd201
JH
921sub compexcl {
922 my $arg = shift;
923 my $code = _getcode($arg);
74f8133e
JH
924 croak __PACKAGE__, "::compexcl: unknown code '$arg'"
925 unless defined $code;
b08cd201 926
2afba6a4
KW
927 UnicodeVersion() unless defined $v_unicode_version;
928 return if $v_unicode_version lt v3.0.0;
929
36c2430c 930 no warnings "non_unicode"; # So works on non-Unicode code points
71a442a8 931 return chr($code) =~ /\p{Composition_Exclusion}/;
b08cd201
JH
932}
933
a452d459 934=head2 B<casefold()>
b08cd201 935
55d7b906 936 use Unicode::UCD 'casefold';
b08cd201 937
a452d459
KW
938 my $casefold = casefold(0xDF);
939 if (defined $casefold) {
940 my @full_fold_hex = split / /, $casefold->{'full'};
941 my $full_fold_string =
942 join "", map {chr(hex($_))} @full_fold_hex;
943 my @turkic_fold_hex =
944 split / /, ($casefold->{'turkic'} ne "")
945 ? $casefold->{'turkic'}
946 : $casefold->{'full'};
947 my $turkic_fold_string =
948 join "", map {chr(hex($_))} @turkic_fold_hex;
949 }
950 if (defined $casefold && $casefold->{'simple'} ne "") {
951 my $simple_fold_hex = $casefold->{'simple'};
952 my $simple_fold_string = chr(hex($simple_fold_hex));
953 }
b08cd201 954
a452d459 955This returns the (almost) locale-independent case folding of the
6329003c
KW
956character specified by the L</code point argument>. (Starting in Perl v5.16,
957the core function C<fc()> returns the C<full> mapping (described below)
958faster than this does, and for entire strings.)
b08cd201 959
6329003c 960If there is no case folding for the input code point, C<undef> is returned.
a452d459
KW
961
962If there is a case folding for that code point, a reference to a hash
b08cd201
JH
963with the following fields is returned:
964
a452d459
KW
965=over
966
967=item B<code>
968
969the input L</code point argument> expressed in hexadecimal, with leading zeros
970added if necessary to make it contain at least four hexdigits
971
972=item B<full>
973
a18e976f 974one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
975code points for the case folding for I<code>.
976Each has at least four hexdigits.
977
978=item B<simple>
979
980is empty, or is exactly one code with at least four hexdigits which can be used
981as an alternative case folding when the calling program cannot cope with the
982fold being a sequence of multiple code points. If I<full> is just one code
983point, then I<simple> equals I<full>. If there is no single code point folding
984defined for I<code>, then I<simple> is the empty string. Otherwise, it is an
985inferior, but still better-than-nothing alternative folding to I<full>.
986
987=item B<mapping>
988
989is the same as I<simple> if I<simple> is not empty, and it is the same as I<full>
990otherwise. It can be considered to be the simplest possible folding for
991I<code>. It is defined primarily for backwards compatibility.
992
993=item B<status>
b08cd201 994
a452d459
KW
995is C<C> (for C<common>) if the best possible fold is a single code point
996(I<simple> equals I<full> equals I<mapping>). It is C<S> if there are distinct
997folds, I<simple> and I<full> (I<mapping> equals I<simple>). And it is C<F> if
a18e976f
KW
998there is only a I<full> fold (I<mapping> equals I<full>; I<simple> is empty).
999Note that this
a452d459
KW
1000describes the contents of I<mapping>. It is defined primarily for backwards
1001compatibility.
b08cd201 1002
6329003c 1003For Unicode versions between 3.1 and 3.1.1 inclusive, I<status> can also be
a452d459
KW
1004C<I> which is the same as C<C> but is a special case for dotted uppercase I and
1005dotless lowercase i:
b08cd201 1006
a452d459 1007=over
b08cd201 1008
a18e976f 1009=item B<*> If you use this C<I> mapping
a452d459 1010
a18e976f 1011the result is case-insensitive,
a452d459
KW
1012but dotless and dotted I's are not distinguished
1013
a18e976f 1014=item B<*> If you exclude this C<I> mapping
a452d459 1015
a18e976f 1016the result is not fully case-insensitive, but
a452d459
KW
1017dotless and dotted I's are distinguished
1018
1019=back
1020
1021=item B<turkic>
1022
1023contains any special folding for Turkic languages. For versions of Unicode
1024starting with 3.2, this field is empty unless I<code> has a different folding
1025in Turkic languages, in which case it is one or more codes (separated by
a18e976f 1026spaces) that, taken in order, give the code points for the case folding for
a452d459
KW
1027I<code> in those languages.
1028Each code has at least four hexdigits.
1029Note that this folding does not maintain canonical equivalence without
1030additional processing.
1031
6329003c
KW
1032For Unicode versions between 3.1 and 3.1.1 inclusive, this field is empty unless
1033there is a
a452d459
KW
1034special folding for Turkic languages, in which case I<status> is C<I>, and
1035I<mapping>, I<full>, I<simple>, and I<turkic> are all equal.
1036
1037=back
1038
1039Programs that want complete generality and the best folding results should use
1040the folding contained in the I<full> field. But note that the fold for some
1041code points will be a sequence of multiple code points.
1042
1043Programs that can't cope with the fold mapping being multiple code points can
1044use the folding contained in the I<simple> field, with the loss of some
1045generality. In Unicode 5.1, about 7% of the defined foldings have no single
1046code point folding.
1047
1048The I<mapping> and I<status> fields are provided for backwards compatibility for
1049existing programs. They contain the same values as in previous versions of
1050this function.
1051
1052Locale is not completely independent. The I<turkic> field contains results to
1053use when the locale is a Turkic language.
b08cd201
JH
1054
1055For more information about case mappings see
a452d459 1056L<http://www.unicode.org/unicode/reports/tr21>
b08cd201
JH
1057
1058=cut
1059
1060my %CASEFOLD;
1061
1062sub _casefold {
727c62ff
KW
1063 unless (%CASEFOLD) { # Populate the hash
1064 my ($full_invlist_ref, $full_invmap_ref, undef, $default)
1065 = prop_invmap('Case_Folding');
1066
1067 # Use the recipe given in the prop_invmap() pod to convert the
1068 # inversion map into the hash.
1069 for my $i (0 .. @$full_invlist_ref - 1 - 1) {
1070 next if $full_invmap_ref->[$i] == $default;
1071 my $adjust = -1;
1072 for my $j ($full_invlist_ref->[$i] .. $full_invlist_ref->[$i+1] -1) {
1073 $adjust++;
1074 if (! ref $full_invmap_ref->[$i]) {
1075
1076 # This is a single character mapping
1077 $CASEFOLD{$j}{'status'} = 'C';
1078 $CASEFOLD{$j}{'simple'}
1079 = $CASEFOLD{$j}{'full'}
1080 = $CASEFOLD{$j}{'mapping'}
1081 = sprintf("%04X", $full_invmap_ref->[$i] + $adjust);
1082 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1083 $CASEFOLD{$j}{'turkic'} = "";
1084 }
1085 else { # prop_invmap ensures that $adjust is 0 for a ref
1086 $CASEFOLD{$j}{'status'} = 'F';
1087 $CASEFOLD{$j}{'full'}
1088 = $CASEFOLD{$j}{'mapping'}
1089 = join " ", map { sprintf "%04X", $_ }
1090 @{$full_invmap_ref->[$i]};
1091 $CASEFOLD{$j}{'simple'} = "";
1092 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1093 $CASEFOLD{$j}{'turkic'} = "";
1094 }
1095 }
1096 }
1097
1098 # We have filled in the full mappings above, assuming there were no
1099 # simple ones for the ones with multi-character maps. Now, we find
1100 # and fix the cases where that assumption was false.
1101 (my ($simple_invlist_ref, $simple_invmap_ref, undef), $default)
1102 = prop_invmap('Simple_Case_Folding');
1103 for my $i (0 .. @$simple_invlist_ref - 1 - 1) {
1104 next if $simple_invmap_ref->[$i] == $default;
1105 my $adjust = -1;
1106 for my $j ($simple_invlist_ref->[$i]
1107 .. $simple_invlist_ref->[$i+1] -1)
1108 {
1109 $adjust++;
1110 next if $CASEFOLD{$j}{'status'} eq 'C';
1111 $CASEFOLD{$j}{'status'} = 'S';
1112 $CASEFOLD{$j}{'simple'}
1113 = $CASEFOLD{$j}{'mapping'}
1114 = sprintf("%04X", $simple_invmap_ref->[$i] + $adjust);
1115 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1116 $CASEFOLD{$j}{'turkic'} = "";
1117 }
1118 }
1119
1120 # We hard-code in the turkish rules
1121 UnicodeVersion() unless defined $v_unicode_version;
1122 if ($v_unicode_version ge v3.2.0) {
1123
1124 # These two code points should already have regular entries, so
1125 # just fill in the turkish fields
1126 $CASEFOLD{ord('I')}{'turkic'} = '0131';
1127 $CASEFOLD{0x130}{'turkic'} = sprintf "%04X", ord('i');
1128 }
1129 elsif ($v_unicode_version ge v3.1.0) {
1130
1131 # These two code points don't have entries otherwise.
1132 $CASEFOLD{0x130}{'code'} = '0130';
1133 $CASEFOLD{0x131}{'code'} = '0131';
1134 $CASEFOLD{0x130}{'status'} = $CASEFOLD{0x131}{'status'} = 'I';
1135 $CASEFOLD{0x130}{'turkic'}
1136 = $CASEFOLD{0x130}{'mapping'}
1137 = $CASEFOLD{0x130}{'full'}
1138 = $CASEFOLD{0x130}{'simple'}
1139 = $CASEFOLD{0x131}{'turkic'}
1140 = $CASEFOLD{0x131}{'mapping'}
1141 = $CASEFOLD{0x131}{'full'}
1142 = $CASEFOLD{0x131}{'simple'}
1143 = sprintf "%04X", ord('i');
1144 }
b08cd201
JH
1145 }
1146}
1147
1148sub casefold {
1149 my $arg = shift;
1150 my $code = _getcode($arg);
74f8133e
JH
1151 croak __PACKAGE__, "::casefold: unknown code '$arg'"
1152 unless defined $code;
b08cd201
JH
1153
1154 _casefold() unless %CASEFOLD;
1155
1156 return $CASEFOLD{$code};
1157}
1158
66aa79e2
KW
1159=head2 B<all_casefolds()>
1160
1161
1162 use Unicode::UCD 'all_casefolds';
1163
1164 my $all_folds_ref = all_casefolds();
1165 foreach my $char_with_casefold (sort { $a <=> $b }
1166 keys %$all_folds_ref)
1167 {
1168 printf "%04X:", $char_with_casefold;
1169 my $casefold = $all_folds_ref->{$char_with_casefold};
1170
1171 # Get folds for $char_with_casefold
1172
1173 my @full_fold_hex = split / /, $casefold->{'full'};
1174 my $full_fold_string =
1175 join "", map {chr(hex($_))} @full_fold_hex;
1176 print " full=", join " ", @full_fold_hex;
1177 my @turkic_fold_hex =
1178 split / /, ($casefold->{'turkic'} ne "")
1179 ? $casefold->{'turkic'}
1180 : $casefold->{'full'};
1181 my $turkic_fold_string =
1182 join "", map {chr(hex($_))} @turkic_fold_hex;
1183 print "; turkic=", join " ", @turkic_fold_hex;
1184 if (defined $casefold && $casefold->{'simple'} ne "") {
1185 my $simple_fold_hex = $casefold->{'simple'};
1186 my $simple_fold_string = chr(hex($simple_fold_hex));
1187 print "; simple=$simple_fold_hex";
1188 }
1189 print "\n";
1190 }
1191
1192This returns all the case foldings in the current version of Unicode in the
1193form of a reference to a hash. Each key to the hash is the decimal
1194representation of a Unicode character that has a casefold to other than
1195itself. The casefold of a semi-colon is itself, so it isn't in the hash;
1196likewise for a lowercase "a", but there is an entry for a capital "A". The
1197hash value for each key is another hash, identical to what is returned by
1198L</casefold()> if called with that code point as its argument. So the value
1199C<< all_casefolds()->{ord("A")}' >> is equivalent to C<casefold(ord("A"))>;
1200
1201=cut
1202
1203sub all_casefolds () {
1204 _casefold() unless %CASEFOLD;
1205 return _dclone \%CASEFOLD;
1206}
1207
a452d459 1208=head2 B<casespec()>
b08cd201 1209
55d7b906 1210 use Unicode::UCD 'casespec';
b08cd201 1211
a452d459 1212 my $casespec = casespec(0xFB00);
b08cd201 1213
a452d459
KW
1214This returns the potentially locale-dependent case mappings of the L</code point
1215argument>. The mappings may be longer than a single code point (which the basic
1216Unicode case mappings as returned by L</charinfo()> never are).
b08cd201 1217
a452d459
KW
1218If there are no case mappings for the L</code point argument>, or if all three
1219possible mappings (I<lower>, I<title> and I<upper>) result in single code
a18e976f 1220points and are locale independent and unconditional, C<undef> is returned
5d8e6e41
KW
1221(which means that the case mappings, if any, for the code point are those
1222returned by L</charinfo()>).
a452d459
KW
1223
1224Otherwise, a reference to a hash giving the mappings (or a reference to a hash
5d8e6e41
KW
1225of such hashes, explained below) is returned with the following keys and their
1226meanings:
a452d459
KW
1227
1228The keys in the bottom layer hash with the meanings of their values are:
1229
1230=over
1231
1232=item B<code>
1233
1234the input L</code point argument> expressed in hexadecimal, with leading zeros
1235added if necessary to make it contain at least four hexdigits
1236
1237=item B<lower>
1238
a18e976f 1239one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1240code points for the lower case of I<code>.
1241Each has at least four hexdigits.
1242
1243=item B<title>
b08cd201 1244
a18e976f 1245one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1246code points for the title case of I<code>.
1247Each has at least four hexdigits.
b08cd201 1248
d2da20e3 1249=item B<upper>
b08cd201 1250
a18e976f 1251one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1252code points for the upper case of I<code>.
1253Each has at least four hexdigits.
1254
1255=item B<condition>
1256
1257the conditions for the mappings to be valid.
a18e976f 1258If C<undef>, the mappings are always valid.
a452d459
KW
1259When defined, this field is a list of conditions,
1260all of which must be true for the mappings to be valid.
1261The list consists of one or more
1262I<locales> (see below)
1263and/or I<contexts> (explained in the next paragraph),
1264separated by spaces.
1265(Other than as used to separate elements, spaces are to be ignored.)
1266Case distinctions in the condition list are not significant.
82c0b05b 1267Conditions preceded by "NON_" represent the negation of the condition.
b08cd201 1268
a452d459
KW
1269A I<context> is one of those defined in the Unicode standard.
1270For Unicode 5.1, they are defined in Section 3.13 C<Default Case Operations>
1271available at
5d8e6e41
KW
1272L<http://www.unicode.org/versions/Unicode5.1.0/>.
1273These are for context-sensitive casing.
f499c386 1274
a452d459
KW
1275=back
1276
5d8e6e41 1277The hash described above is returned for locale-independent casing, where
a18e976f 1278at least one of the mappings has length longer than one. If C<undef> is
5d8e6e41
KW
1279returned, the code point may have mappings, but if so, all are length one,
1280and are returned by L</charinfo()>.
1281Note that when this function does return a value, it will be for the complete
1282set of mappings for a code point, even those whose length is one.
1283
1284If there are additional casing rules that apply only in certain locales,
1285an additional key for each will be defined in the returned hash. Each such key
1286will be its locale name, defined as a 2-letter ISO 3166 country code, possibly
1287followed by a "_" and a 2-letter ISO language code (possibly followed by a "_"
1288and a variant code). You can find the lists of all possible locales, see
1289L<Locale::Country> and L<Locale::Language>.
89e4a205 1290(In Unicode 6.0, the only locales returned by this function
a452d459 1291are C<lt>, C<tr>, and C<az>.)
b08cd201 1292
5d8e6e41
KW
1293Each locale key is a reference to a hash that has the form above, and gives
1294the casing rules for that particular locale, which take precedence over the
1295locale-independent ones when in that locale.
1296
1297If the only casing for a code point is locale-dependent, then the returned
1298hash will not have any of the base keys, like C<code>, C<upper>, etc., but
1299will contain only locale keys.
1300
b08cd201 1301For more information about case mappings see
a452d459 1302L<http://www.unicode.org/unicode/reports/tr21/>
b08cd201
JH
1303
1304=cut
1305
1306my %CASESPEC;
1307
1308sub _casespec {
1309 unless (%CASESPEC) {
f033d3cd
KW
1310 UnicodeVersion() unless defined $v_unicode_version;
1311 if ($v_unicode_version lt v2.1.8) {
1312 %CASESPEC = {};
1313 }
1314 elsif (openunicode(\$CASESPECFH, "SpecialCasing.txt")) {
6c8d78fb 1315 local $_;
ce066323 1316 local $/ = "\n";
b08cd201
JH
1317 while (<$CASESPECFH>) {
1318 if (/^([0-9A-F]+); ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; (\w+(?: \w+)*)?/) {
f033d3cd 1319
f499c386
JH
1320 my ($hexcode, $lower, $title, $upper, $condition) =
1321 ($1, $2, $3, $4, $5);
1322 my $code = hex($hexcode);
f033d3cd
KW
1323
1324 # In 2.1.8, there were duplicate entries; ignore all but
1325 # the first one -- there were no conditions in the file
1326 # anyway.
1327 if (exists $CASESPEC{$code} && $v_unicode_version ne v2.1.8)
1328 {
f499c386
JH
1329 if (exists $CASESPEC{$code}->{code}) {
1330 my ($oldlower,
1331 $oldtitle,
1332 $oldupper,
1333 $oldcondition) =
1334 @{$CASESPEC{$code}}{qw(lower
1335 title
1336 upper
1337 condition)};
822ebcc8
JH
1338 if (defined $oldcondition) {
1339 my ($oldlocale) =
f499c386 1340 ($oldcondition =~ /^([a-z][a-z](?:_\S+)?)/);
f499c386
JH
1341 delete $CASESPEC{$code};
1342 $CASESPEC{$code}->{$oldlocale} =
1343 { code => $hexcode,
1344 lower => $oldlower,
1345 title => $oldtitle,
1346 upper => $oldupper,
1347 condition => $oldcondition };
f499c386
JH
1348 }
1349 }
1350 my ($locale) =
1351 ($condition =~ /^([a-z][a-z](?:_\S+)?)/);
1352 $CASESPEC{$code}->{$locale} =
1353 { code => $hexcode,
1354 lower => $lower,
1355 title => $title,
1356 upper => $upper,
1357 condition => $condition };
1358 } else {
1359 $CASESPEC{$code} =
1360 { code => $hexcode,
1361 lower => $lower,
1362 title => $title,
1363 upper => $upper,
1364 condition => $condition };
1365 }
b08cd201
JH
1366 }
1367 }
1368 close($CASESPECFH);
1369 }
1370 }
1371}
1372
1373sub casespec {
1374 my $arg = shift;
1375 my $code = _getcode($arg);
74f8133e
JH
1376 croak __PACKAGE__, "::casespec: unknown code '$arg'"
1377 unless defined $code;
b08cd201
JH
1378
1379 _casespec() unless %CASESPEC;
1380
cb3150f5 1381 return ref $CASESPEC{$code} ? _dclone $CASESPEC{$code} : $CASESPEC{$code};
b08cd201
JH
1382}
1383
a452d459 1384=head2 B<namedseq()>
a2bd7410
JH
1385
1386 use Unicode::UCD 'namedseq';
1387
1388 my $namedseq = namedseq("KATAKANA LETTER AINU P");
1389 my @namedseq = namedseq("KATAKANA LETTER AINU P");
1390 my %namedseq = namedseq();
1391
1392If used with a single argument in a scalar context, returns the string
a18e976f 1393consisting of the code points of the named sequence, or C<undef> if no
a2bd7410 1394named sequence by that name exists. If used with a single argument in
956cae9a
KW
1395a list context, it returns the list of the ordinals of the code points. If used
1396with no
a2bd7410
JH
1397arguments in a list context, returns a hash with the names of the
1398named sequences as the keys and the named sequences as strings as
a18e976f 1399the values. Otherwise, it returns C<undef> or an empty list depending
a2bd7410
JH
1400on the context.
1401
a452d459
KW
1402This function only operates on officially approved (not provisional) named
1403sequences.
a2bd7410 1404
27f853a0
KW
1405Note that as of Perl 5.14, C<\N{KATAKANA LETTER AINU P}> will insert the named
1406sequence into double-quoted strings, and C<charnames::string_vianame("KATAKANA
1407LETTER AINU P")> will return the same string this function does, but will also
1408operate on character names that aren't named sequences, without you having to
1409know which are which. See L<charnames>.
1410
a2bd7410
JH
1411=cut
1412
1413my %NAMEDSEQ;
1414
1415sub _namedseq {
1416 unless (%NAMEDSEQ) {
98ef7649 1417 if (openunicode(\$NAMEDSEQFH, "Name.pl")) {
a2bd7410 1418 local $_;
ce066323 1419 local $/ = "\n";
a2bd7410 1420 while (<$NAMEDSEQFH>) {
98ef7649
KW
1421 if (/^ [0-9A-F]+ \ /x) {
1422 chomp;
1423 my ($sequence, $name) = split /\t/;
1424 my @s = map { chr(hex($_)) } split(' ', $sequence);
1425 $NAMEDSEQ{$name} = join("", @s);
a2bd7410
JH
1426 }
1427 }
1428 close($NAMEDSEQFH);
1429 }
1430 }
1431}
1432
1433sub namedseq {
98ef7649
KW
1434
1435 # Use charnames::string_vianame() which now returns this information,
1436 # unless the caller wants the hash returned, in which case we read it in,
1437 # and thereafter use it instead of calling charnames, as it is faster.
1438
a2bd7410
JH
1439 my $wantarray = wantarray();
1440 if (defined $wantarray) {
1441 if ($wantarray) {
1442 if (@_ == 0) {
98ef7649 1443 _namedseq() unless %NAMEDSEQ;
a2bd7410
JH
1444 return %NAMEDSEQ;
1445 } elsif (@_ == 1) {
98ef7649
KW
1446 my $s;
1447 if (%NAMEDSEQ) {
1448 $s = $NAMEDSEQ{ $_[0] };
1449 }
1450 else {
1451 $s = charnames::string_vianame($_[0]);
1452 }
a2bd7410
JH
1453 return defined $s ? map { ord($_) } split('', $s) : ();
1454 }
1455 } elsif (@_ == 1) {
98ef7649
KW
1456 return $NAMEDSEQ{ $_[0] } if %NAMEDSEQ;
1457 return charnames::string_vianame($_[0]);
a2bd7410
JH
1458 }
1459 }
1460 return;
1461}
1462
7319f91d
KW
1463my %NUMERIC;
1464
1465sub _numeric {
35a865d4 1466 my @numbers = _read_table("To/Nv.pl");
98025745
KW
1467 foreach my $entry (@numbers) {
1468 my ($start, $end, $value) = @$entry;
1469
05dbc6f8
KW
1470 # If value contains a slash, convert to decimal, add a reverse hash
1471 # used by charinfo.
98025745
KW
1472 if ((my @rational = split /\//, $value) == 2) {
1473 my $real = $rational[0] / $rational[1];
05dbc6f8 1474 $real_to_rational{$real} = $value;
98025745 1475 $value = $real;
98025745 1476
4f143a72
KW
1477 # Should only be single element, but just in case...
1478 for my $i ($start .. $end) {
1479 $NUMERIC{$i} = $value;
1480 }
1481 }
1482 else {
1483 # The values require adjusting, as is in 'a' format
1484 for my $i ($start .. $end) {
1485 $NUMERIC{$i} = $value + $i - $start;
1486 }
7319f91d 1487 }
7319f91d 1488 }
2dc5eb26
KW
1489
1490 # Decided unsafe to use these that aren't officially part of the Unicode
1491 # standard.
1492 #use Math::Trig;
1493 #my $pi = acos(-1.0);
98025745 1494 #$NUMERIC{0x03C0} = $pi;
7319f91d
KW
1495
1496 # Euler's constant, not to be confused with Euler's number
98025745 1497 #$NUMERIC{0x2107} = 0.57721566490153286060651209008240243104215933593992;
7319f91d
KW
1498
1499 # Euler's number
98025745 1500 #$NUMERIC{0x212F} = 2.7182818284590452353602874713526624977572;
2dc5eb26 1501
7319f91d
KW
1502 return;
1503}
1504
1505=pod
1506
67592e11 1507=head2 B<num()>
7319f91d 1508
eefd7bc2
KW
1509 use Unicode::UCD 'num';
1510
1511 my $val = num("123");
1512 my $one_quarter = num("\N{VULGAR FRACTION 1/4}");
1513
7319f91d
KW
1514C<num> returns the numeric value of the input Unicode string; or C<undef> if it
1515doesn't think the entire string has a completely valid, safe numeric value.
1516
1517If the string is just one character in length, the Unicode numeric value
1518is returned if it has one, or C<undef> otherwise. Note that this need
1519not be a whole number. C<num("\N{TIBETAN DIGIT HALF ZERO}")>, for
2dc5eb26
KW
1520example returns -0.5.
1521
1522=cut
7319f91d 1523
2dc5eb26
KW
1524#A few characters to which Unicode doesn't officially
1525#assign a numeric value are considered numeric by C<num>.
1526#These are:
1527
1528# EULER CONSTANT 0.5772... (this is NOT Euler's number)
1529# SCRIPT SMALL E 2.71828... (this IS Euler's number)
1530# GREEK SMALL LETTER PI 3.14159...
1531
1532=pod
7319f91d
KW
1533
1534If the string is more than one character, C<undef> is returned unless
8bb4c8e2 1535all its characters are decimal digits (that is, they would match C<\d+>),
7319f91d
KW
1536from the same script. For example if you have an ASCII '0' and a Bengali
1537'3', mixed together, they aren't considered a valid number, and C<undef>
1538is returned. A further restriction is that the digits all have to be of
1539the same form. A half-width digit mixed with a full-width one will
1540return C<undef>. The Arabic script has two sets of digits; C<num> will
1541return C<undef> unless all the digits in the string come from the same
1542set.
1543
1544C<num> errs on the side of safety, and there may be valid strings of
1545decimal digits that it doesn't recognize. Note that Unicode defines
1546a number of "digit" characters that aren't "decimal digit" characters.
a278d14b 1547"Decimal digits" have the property that they have a positional value, i.e.,
7319f91d
KW
1548there is a units position, a 10's position, a 100's, etc, AND they are
1549arranged in Unicode in blocks of 10 contiguous code points. The Chinese
1550digits, for example, are not in such a contiguous block, and so Unicode
1551doesn't view them as decimal digits, but merely digits, and so C<\d> will not
1552match them. A single-character string containing one of these digits will
1553have its decimal value returned by C<num>, but any longer string containing
1554only these digits will return C<undef>.
1555
a278d14b
KW
1556Strings of multiple sub- and superscripts are not recognized as numbers. You
1557can use either of the compatibility decompositions in Unicode::Normalize to
7319f91d
KW
1558change these into digits, and then call C<num> on the result.
1559
1560=cut
1561
1562# To handle sub, superscripts, this could if called in list context,
1563# consider those, and return the <decomposition> type in the second
1564# array element.
1565
1566sub num {
1567 my $string = $_[0];
1568
1569 _numeric unless %NUMERIC;
1570
1571 my $length = length($string);
98025745 1572 return $NUMERIC{ord($string)} if $length == 1;
7319f91d
KW
1573 return if $string =~ /\D/;
1574 my $first_ord = ord(substr($string, 0, 1));
98025745 1575 my $value = $NUMERIC{$first_ord};
5522af1c
KW
1576
1577 # To be a valid decimal number, it should be in a block of 10 consecutive
1578 # characters, whose values are 0, 1, 2, ... 9. Therefore this digit's
1579 # value is its offset in that block from the character that means zero.
7319f91d
KW
1580 my $zero_ord = $first_ord - $value;
1581
5522af1c
KW
1582 # Unicode 6.0 instituted the rule that only digits in a consecutive
1583 # block of 10 would be considered decimal digits. If this is an earlier
1584 # release, we verify that this first character is a member of such a
1585 # block. That is, that the block of characters surrounding this one
1586 # consists of all \d characters whose numeric values are the expected
1587 # ones.
1588 UnicodeVersion() unless defined $v_unicode_version;
1589 if ($v_unicode_version lt v6.0.0) {
1590 for my $i (0 .. 9) {
1591 my $ord = $zero_ord + $i;
1592 return unless chr($ord) =~ /\d/;
1593 my $numeric = $NUMERIC{$ord};
1594 return unless defined $numeric;
1595 return unless $numeric == $i;
1596 }
1597 }
1598
7319f91d 1599 for my $i (1 .. $length -1) {
5522af1c
KW
1600
1601 # Here we know either by verifying, or by fact of the first character
1602 # being a \d in Unicode 6.0 or later, that any character between the
1603 # character that means 0, and 9 positions above it must be \d, and
1604 # must have its value correspond to its offset from the zero. Any
1605 # characters outside these 10 do not form a legal number for this
1606 # function.
7319f91d
KW
1607 my $ord = ord(substr($string, $i, 1));
1608 my $digit = $ord - $zero_ord;
1609 return unless $digit >= 0 && $digit <= 9;
1610 $value = $value * 10 + $digit;
1611 }
5522af1c 1612
7319f91d
KW
1613 return $value;
1614}
1615
7ef25837
KW
1616=pod
1617
1618=head2 B<prop_aliases()>
1619
1620 use Unicode::UCD 'prop_aliases';
1621
1622 my ($short_name, $full_name, @other_names) = prop_aliases("space");
1623 my $same_full_name = prop_aliases("Space"); # Scalar context
1624 my ($same_short_name) = prop_aliases("Space"); # gets 0th element
1625 print "The full name is $full_name\n";
1626 print "The short name is $short_name\n";
1627 print "The other aliases are: ", join(", ", @other_names), "\n";
1628
1629 prints:
1630 The full name is White_Space
1631 The short name is WSpace
1632 The other aliases are: Space
1633
1634Most Unicode properties have several synonymous names. Typically, there is at
1635least a short name, convenient to type, and a long name that more fully
1636describes the property, and hence is more easily understood.
1637
1638If you know one name for a Unicode property, you can use C<prop_aliases> to find
1639either the long name (when called in scalar context), or a list of all of the
1640names, somewhat ordered so that the short name is in the 0th element, the long
1641name in the next element, and any other synonyms are in the remaining
1642elements, in no particular order.
1643
1644The long name is returned in a form nicely capitalized, suitable for printing.
1645
1646The input parameter name is loosely matched, which means that white space,
1647hyphens, and underscores are ignored (except for the trailing underscore in
1648the old_form grandfathered-in C<"L_">, which is better written as C<"LC">, and
1649both of which mean C<General_Category=Cased Letter>).
1650
1651If the name is unknown, C<undef> is returned (or an empty list in list
1652context). Note that Perl typically recognizes property names in regular
1653expressions with an optional C<"Is_>" (with or without the underscore)
1654prefixed to them, such as C<\p{isgc=punct}>. This function does not recognize
1655those in the input, returning C<undef>. Nor are they included in the output
1656as possible synonyms.
1657
1658C<prop_aliases> does know about the Perl extensions to Unicode properties,
1659such as C<Any> and C<XPosixAlpha>, and the single form equivalents to Unicode
1660properties such as C<XDigit>, C<Greek>, C<In_Greek>, and C<Is_Greek>. The
1661final example demonstrates that the C<"Is_"> prefix is recognized for these
1662extensions; it is needed to resolve ambiguities. For example,
1663C<prop_aliases('lc')> returns the list C<(lc, Lowercase_Mapping)>, but
1664C<prop_aliases('islc')> returns C<(Is_LC, Cased_Letter)>. This is
1665because C<islc> is a Perl extension which is short for
1666C<General_Category=Cased Letter>. The lists returned for the Perl extensions
1667will not include the C<"Is_"> prefix (whether or not the input had it) unless
1668needed to resolve ambiguities, as shown in the C<"islc"> example, where the
1669returned list had one element containing C<"Is_">, and the other without.
1670
1671It is also possible for the reverse to happen: C<prop_aliases('isc')> returns
1672the list C<(isc, ISO_Comment)>; whereas C<prop_aliases('c')> returns
1673C<(C, Other)> (the latter being a Perl extension meaning
ee94c7d1
KW
1674C<General_Category=Other>.
1675L<perluniprops/Properties accessible through Unicode::UCD> lists the available
1676forms, including which ones are discouraged from use.
7ef25837
KW
1677
1678Those discouraged forms are accepted as input to C<prop_aliases>, but are not
1679returned in the lists. C<prop_aliases('isL&')> and C<prop_aliases('isL_')>,
1680which are old synonyms for C<"Is_LC"> and should not be used in new code, are
1681examples of this. These both return C<(Is_LC, Cased_Letter)>. Thus this
1682function allows you to take a discourarged form, and find its acceptable
1683alternatives. The same goes with single-form Block property equivalences.
1684Only the forms that begin with C<"In_"> are not discouraged; if you pass
1685C<prop_aliases> a discouraged form, you will get back the equivalent ones that
1686begin with C<"In_">. It will otherwise look like a new-style block name (see.
1687L</Old-style versus new-style block names>).
1688
1689C<prop_aliases> does not know about any user-defined properties, and will
1690return C<undef> if called with one of those. Likewise for Perl internal
1691properties, with the exception of "Perl_Decimal_Digit" which it does know
1692about (and which is documented below in L</prop_invmap()>).
1693
1694=cut
1695
1696# It may be that there are use cases where the discouraged forms should be
1697# returned. If that comes up, an optional boolean second parameter to the
1698# function could be created, for example.
1699
1700# These are created by mktables for this routine and stored in unicore/UCD.pl
1701# where their structures are described.
1702our %string_property_loose_to_name;
1703our %ambiguous_names;
1704our %loose_perlprop_to_name;
1705our %prop_aliases;
1706
1707sub prop_aliases ($) {
1708 my $prop = $_[0];
1709 return unless defined $prop;
1710
1711 require "unicore/UCD.pl";
1712 require "unicore/Heavy.pl";
1713 require "utf8_heavy.pl";
1714
1715 # The property name may be loosely or strictly matched; we don't know yet.
1716 # But both types use lower-case.
1717 $prop = lc $prop;
1718
1719 # It is loosely matched if its lower case isn't known to be strict.
1720 my $list_ref;
1721 if (! exists $utf8::stricter_to_file_of{$prop}) {
1722 my $loose = utf8::_loose_name($prop);
1723
1724 # There is a hash that converts from any loose name to its standard
1725 # form, mapping all synonyms for a name to one name that can be used
1726 # as a key into another hash. The whole concept is for memory
1727 # savings, as the second hash doesn't have to have all the
1728 # combinations. Actually, there are two hashes that do the
1729 # converstion. One is used in utf8_heavy.pl (stored in Heavy.pl) for
1730 # looking up properties matchable in regexes. This function needs to
1731 # access string properties, which aren't available in regexes, so a
1732 # second conversion hash is made for them (stored in UCD.pl). Look in
1733 # the string one now, as the rest can have an optional 'is' prefix,
1734 # which these don't.
1735 if (exists $string_property_loose_to_name{$loose}) {
1736
1737 # Convert to its standard loose name.
1738 $prop = $string_property_loose_to_name{$loose};
1739 }
1740 else {
1741 my $retrying = 0; # bool. ? Has an initial 'is' been stripped
1742 RETRY:
1743 if (exists $utf8::loose_property_name_of{$loose}
1744 && (! $retrying
1745 || ! exists $ambiguous_names{$loose}))
1746 {
1747 # Found an entry giving the standard form. We don't get here
1748 # (in the test above) when we've stripped off an
1749 # 'is' and the result is an ambiguous name. That is because
1750 # these are official Unicode properties (though Perl can have
1751 # an optional 'is' prefix meaning the official property), and
1752 # all ambiguous cases involve a Perl single-form extension
1753 # for the gc, script, or block properties, and the stripped
1754 # 'is' means that they mean one of those, and not one of
1755 # these
1756 $prop = $utf8::loose_property_name_of{$loose};
1757 }
1758 elsif (exists $loose_perlprop_to_name{$loose}) {
1759
1760 # This hash is specifically for this function to list Perl
1761 # extensions that aren't in the earlier hashes. If there is
1762 # only one element, the short and long names are identical.
1763 # Otherwise the form is already in the same form as
1764 # %prop_aliases, which is handled at the end of the function.
1765 $list_ref = $loose_perlprop_to_name{$loose};
1766 if (@$list_ref == 1) {
1767 my @list = ($list_ref->[0], $list_ref->[0]);
1768 $list_ref = \@list;
1769 }
1770 }
1771 elsif (! exists $utf8::loose_to_file_of{$loose}) {
1772
1773 # loose_to_file_of is a complete list of loose names. If not
1774 # there, the input is unknown.
1775 return;
1776 }
1777 else {
1778
1779 # Here we found the name but not its aliases, so it has to
1780 # exist. This means it must be one of the Perl single-form
1781 # extensions. First see if it is for a property-value
1782 # combination in one of the following properties.
1783 my @list;
1784 foreach my $property ("gc", "script") {
1785 @list = prop_value_aliases($property, $loose);
1786 last if @list;
1787 }
1788 if (@list) {
1789
1790 # Here, it is one of those property-value combination
1791 # single-form synonyms. There are ambiguities with some
1792 # of these. Check against the list for these, and adjust
1793 # if necessary.
1794 for my $i (0 .. @list -1) {
1795 if (exists $ambiguous_names
1796 {utf8::_loose_name(lc $list[$i])})
1797 {
1798 # The ambiguity is resolved by toggling whether or
1799 # not it has an 'is' prefix
1800 $list[$i] =~ s/^Is_// or $list[$i] =~ s/^/Is_/;
1801 }
1802 }
1803 return @list;
1804 }
1805
1806 # Here, it wasn't one of the gc or script single-form
1807 # extensions. It could be a block property single-form
1808 # extension. An 'in' prefix definitely means that, and should
2a4f2769
KW
1809 # be looked up without the prefix. However, starting in
1810 # Unicode 6.1, we have to special case 'indic...', as there
1811 # is a property that begins with that name. We shouldn't
1812 # strip the 'in' from that. I'm (khw) generalizing this to
1813 # 'indic' instead of the single property, because I suspect
1814 # that others of this class may come along in the future.
1815 # However, this could backfire and a block created whose name
1816 # begins with 'dic...', and we would want to strip the 'in'.
1817 # At which point this would have to be tweaked.
1818 my $began_with_in = $loose =~ s/^in(?!dic)//;
7ef25837
KW
1819 @list = prop_value_aliases("block", $loose);
1820 if (@list) {
1821 map { $_ =~ s/^/In_/ } @list;
1822 return @list;
1823 }
1824
1825 # Here still haven't found it. The last opportunity for it
1826 # being valid is only if it began with 'is'. We retry without
1827 # the 'is', setting a flag to that effect so that we don't
1828 # accept things that begin with 'isis...'
1829 if (! $retrying && ! $began_with_in && $loose =~ s/^is//) {
1830 $retrying = 1;
1831 goto RETRY;
1832 }
1833
1834 # Here, didn't find it. Since it was in %loose_to_file_of, we
1835 # should have been able to find it.
1836 carp __PACKAGE__, "::prop_aliases: Unexpectedly could not find '$prop'. Send bug report to perlbug\@perl.org";
1837 return;
1838 }
1839 }
1840 }
1841
1842 if (! $list_ref) {
1843 # Here, we have set $prop to a standard form name of the input. Look
1844 # it up in the structure created by mktables for this purpose, which
1845 # contains both strict and loosely matched properties. Avoid
1846 # autovivifying.
1847 $list_ref = $prop_aliases{$prop} if exists $prop_aliases{$prop};
1848 return unless $list_ref;
1849 }
1850
1851 # The full name is in element 1.
1852 return $list_ref->[1] unless wantarray;
1853
cb3150f5 1854 return @{_dclone $list_ref};
7ef25837
KW
1855}
1856
1857=pod
1858
1859=head2 B<prop_value_aliases()>
1860
1861 use Unicode::UCD 'prop_value_aliases';
1862
1863 my ($short_name, $full_name, @other_names)
1864 = prop_value_aliases("Gc", "Punct");
1865 my $same_full_name = prop_value_aliases("Gc", "P"); # Scalar cntxt
1866 my ($same_short_name) = prop_value_aliases("Gc", "P"); # gets 0th
1867 # element
1868 print "The full name is $full_name\n";
1869 print "The short name is $short_name\n";
1870 print "The other aliases are: ", join(", ", @other_names), "\n";
1871
1872 prints:
1873 The full name is Punctuation
1874 The short name is P
1875 The other aliases are: Punct
1876
1877Some Unicode properties have a restricted set of legal values. For example,
1878all binary properties are restricted to just C<true> or C<false>; and there
1879are only a few dozen possible General Categories.
1880
1881For such properties, there are usually several synonyms for each possible
1882value. For example, in binary properties, I<truth> can be represented by any of
1883the strings "Y", "Yes", "T", or "True"; and the General Category
1884"Punctuation" by that string, or "Punct", or simply "P".
1885
1886Like property names, there is typically at least a short name for each such
1887property-value, and a long name. If you know any name of the property-value,
1888you can use C<prop_value_aliases>() to get the long name (when called in
1889scalar context), or a list of all the names, with the short name in the 0th
1890element, the long name in the next element, and any other synonyms in the
1891remaining elements, in no particular order, except that any all-numeric
1892synonyms will be last.
1893
1894The long name is returned in a form nicely capitalized, suitable for printing.
1895
1896Case, white space, hyphens, and underscores are ignored in the input parameters
1897(except for the trailing underscore in the old-form grandfathered-in general
1898category property value C<"L_">, which is better written as C<"LC">).
1899
1900If either name is unknown, C<undef> is returned. Note that Perl typically
1901recognizes property names in regular expressions with an optional C<"Is_>"
1902(with or without the underscore) prefixed to them, such as C<\p{isgc=punct}>.
1903This function does not recognize those in the property parameter, returning
1904C<undef>.
1905
1906If called with a property that doesn't have synonyms for its values, it
1907returns the input value, possibly normalized with capitalization and
1908underscores.
1909
1910For the block property, new-style block names are returned (see
1911L</Old-style versus new-style block names>).
1912
1913To find the synonyms for single-forms, such as C<\p{Any}>, use
1914L</prop_aliases()> instead.
1915
1916C<prop_value_aliases> does not know about any user-defined properties, and
1917will return C<undef> if called with one of those.
1918
1919=cut
1920
1921# These are created by mktables for this routine and stored in unicore/UCD.pl
1922# where their structures are described.
1923our %loose_to_standard_value;
1924our %prop_value_aliases;
1925
1926sub prop_value_aliases ($$) {
1927 my ($prop, $value) = @_;
1928 return unless defined $prop && defined $value;
1929
1930 require "unicore/UCD.pl";
1931 require "utf8_heavy.pl";
1932
1933 # Find the property name synonym that's used as the key in other hashes,
1934 # which is element 0 in the returned list.
1935 ($prop) = prop_aliases($prop);
1936 return if ! $prop;
1937 $prop = utf8::_loose_name(lc $prop);
1938
1939 # Here is a legal property, but the hash below (created by mktables for
1940 # this purpose) only knows about the properties that have a very finite
1941 # number of potential values, that is not ones whose value could be
1942 # anything, like most (if not all) string properties. These don't have
1943 # synonyms anyway. Simply return the input. For example, there is no
1944 # synonym for ('Uppercase_Mapping', A').
1945 return $value if ! exists $prop_value_aliases{$prop};
1946
1947 # The value name may be loosely or strictly matched; we don't know yet.
1948 # But both types use lower-case.
1949 $value = lc $value;
1950
1951 # If the name isn't found under loose matching, it certainly won't be
1952 # found under strict
1953 my $loose_value = utf8::_loose_name($value);
1954 return unless exists $loose_to_standard_value{"$prop=$loose_value"};
1955
1956 # Similarly if the combination under loose matching doesn't exist, it
1957 # won't exist under strict.
1958 my $standard_value = $loose_to_standard_value{"$prop=$loose_value"};
1959 return unless exists $prop_value_aliases{$prop}{$standard_value};
1960
1961 # Here we did find a combination under loose matching rules. But it could
1962 # be that is a strict property match that shouldn't have matched.
1963 # %prop_value_aliases is set up so that the strict matches will appear as
1964 # if they were in loose form. Thus, if the non-loose version is legal,
1965 # we're ok, can skip the further check.
1966 if (! exists $utf8::stricter_to_file_of{"$prop=$value"}
1967
1968 # We're also ok and skip the further check if value loosely matches.
1969 # mktables has verified that no strict name under loose rules maps to
1970 # an existing loose name. This code relies on the very limited
1971 # circumstances that strict names can be here. Strict name matching
1972 # happens under two conditions:
1973 # 1) when the name begins with an underscore. But this function
1974 # doesn't accept those, and %prop_value_aliases doesn't have
1975 # them.
1976 # 2) When the values are numeric, in which case we need to look
1977 # further, but their squeezed-out loose values will be in
1978 # %stricter_to_file_of
1979 && exists $utf8::stricter_to_file_of{"$prop=$loose_value"})
1980 {
1981 # The only thing that's legal loosely under strict is that can have an
1982 # underscore between digit pairs XXX
1983 while ($value =~ s/(\d)_(\d)/$1$2/g) {}
1984 return unless exists $utf8::stricter_to_file_of{"$prop=$value"};
1985 }
1986
1987 # Here, we know that the combination exists. Return it.
1988 my $list_ref = $prop_value_aliases{$prop}{$standard_value};
1989 if (@$list_ref > 1) {
1990 # The full name is in element 1.
1991 return $list_ref->[1] unless wantarray;
1992
cb3150f5 1993 return @{_dclone $list_ref};
7ef25837
KW
1994 }
1995
1996 return $list_ref->[0] unless wantarray;
1997
1998 # Only 1 element means that it repeats
1999 return ( $list_ref->[0], $list_ref->[0] );
2000}
7319f91d 2001
681d705c
KW
2002# All 1 bits is the largest possible UV.
2003$Unicode::UCD::MAX_CP = ~0;
2004
2005=pod
2006
2007=head2 B<prop_invlist()>
2008
2009C<prop_invlist> returns an inversion list (described below) that defines all the
2010code points for the binary Unicode property (or "property=value" pair) given
2011by the input parameter string:
2012
2013 use feature 'say';
2014 use Unicode::UCD 'prop_invlist';
2015 say join ", ", prop_invlist("Any");
2016
2017 prints:
2018 0, 1114112
2019
c8652296
KW
2020If the input is unknown C<undef> is returned in scalar context; an empty-list
2021in list context. If the input is known, the number of elements in
681d705c
KW
2022the list is returned if called in scalar context.
2023
2024L<perluniprops|perluniprops/Properties accessible through \p{} and \P{}> gives
2025the list of properties that this function accepts, as well as all the possible
2026forms for them (including with the optional "Is_" prefixes). (Except this
2027function doesn't accept any Perl-internal properties, some of which are listed
2028there.) This function uses the same loose or tighter matching rules for
2029resolving the input property's name as is done for regular expressions. These
2030are also specified in L<perluniprops|perluniprops/Properties accessible
2031through \p{} and \P{}>. Examples of using the "property=value" form are:
2032
2033 say join ", ", prop_invlist("Script=Shavian");
2034
2035 prints:
2036 66640, 66688
2037
2038 say join ", ", prop_invlist("ASCII_Hex_Digit=No");
2039
2040 prints:
2041 0, 48, 58, 65, 71, 97, 103
2042
2043 say join ", ", prop_invlist("ASCII_Hex_Digit=Yes");
2044
2045 prints:
2046 48, 58, 65, 71, 97, 103
2047
2048Inversion lists are a compact way of specifying Unicode property-value
2049definitions. The 0th item in the list is the lowest code point that has the
2050property-value. The next item (item [1]) is the lowest code point beyond that
2051one that does NOT have the property-value. And the next item beyond that
2052([2]) is the lowest code point beyond that one that does have the
2053property-value, and so on. Put another way, each element in the list gives
2054the beginning of a range that has the property-value (for even numbered
2055elements), or doesn't have the property-value (for odd numbered elements).
2056The name for this data structure stems from the fact that each element in the
2057list toggles (or inverts) whether the corresponding range is or isn't on the
2058list.
2059
2060In the final example above, the first ASCII Hex digit is code point 48, the
2061character "0", and all code points from it through 57 (a "9") are ASCII hex
2062digits. Code points 58 through 64 aren't, but 65 (an "A") through 70 (an "F")
2063are, as are 97 ("a") through 102 ("f"). 103 starts a range of code points
2064that aren't ASCII hex digits. That range extends to infinity, which on your
2065computer can be found in the variable C<$Unicode::UCD::MAX_CP>. (This
2066variable is as close to infinity as Perl can get on your platform, and may be
2067too high for some operations to work; you may wish to use a smaller number for
2068your purposes.)
2069
2070Note that the inversion lists returned by this function can possibly include
2071non-Unicode code points, that is anything above 0x10FFFF. This is in
2072contrast to Perl regular expression matches on those code points, in which a
2073non-Unicode code point always fails to match. For example, both of these have
2074the same result:
2075
2076 chr(0x110000) =~ \p{ASCII_Hex_Digit=True} # Fails.
2077 chr(0x110000) =~ \p{ASCII_Hex_Digit=False} # Fails!
2078
2079And both raise a warning that a Unicode property is being used on a
2080non-Unicode code point. It is arguable as to which is the correct thing to do
2081here. This function has chosen the way opposite to the Perl regular
2082expression behavior. This allows you to easily flip to to the Perl regular
2083expression way (for you to go in the other direction would be far harder).
2084Simply add 0x110000 at the end of the non-empty returned list if it isn't
2085already that value; and pop that value if it is; like:
2086
2087 my @list = prop_invlist("foo");
2088 if (@list) {
2089 if ($list[-1] == 0x110000) {
2090 pop @list; # Defeat the turning on for above Unicode
2091 }
2092 else {
2093 push @list, 0x110000; # Turn off for above Unicode
2094 }
2095 }
2096
2097It is a simple matter to expand out an inversion list to a full list of all
2098code points that have the property-value:
2099
2100 my @invlist = prop_invlist($property_name);
2101 die "empty" unless @invlist;
2102 my @full_list;
2103 for (my $i = 0; $i < @invlist; $i += 2) {
2104 my $upper = ($i + 1) < @invlist
2105 ? $invlist[$i+1] - 1 # In range
2106 : $Unicode::UCD::MAX_CP; # To infinity. You may want
2107 # to stop much much earlier;
2108 # going this high may expose
2109 # perl deficiencies with very
2110 # large numbers.
2111 for my $j ($invlist[$i] .. $upper) {
2112 push @full_list, $j;
2113 }
2114 }
2115
2116C<prop_invlist> does not know about any user-defined nor Perl internal-only
2117properties, and will return C<undef> if called with one of those.
2118
2119=cut
2120
2121# User-defined properties could be handled with some changes to utf8_heavy.pl;
2122# and implementing here of dealing with EXTRAS. If done, consideration should
2123# be given to the fact that the user subroutine could return different results
2124# with each call; security issues need to be thought about.
2125
2126# These are created by mktables for this routine and stored in unicore/UCD.pl
2127# where their structures are described.
2128our %loose_defaults;
2129our $MAX_UNICODE_CODEPOINT;
2130
92feec86 2131sub prop_invlist ($;$) {
681d705c 2132 my $prop = $_[0];
92feec86
KW
2133
2134 # Undocumented way to get at Perl internal properties
2135 my $internal_ok = defined $_[1] && $_[1] eq '_perl_core_internal_ok';
2136
681d705c
KW
2137 return if ! defined $prop;
2138
2139 require "utf8_heavy.pl";
2140
2141 # Warnings for these are only for regexes, so not applicable to us
2142 no warnings 'deprecated';
2143
2144 # Get the swash definition of the property-value.
2145 my $swash = utf8::SWASHNEW(__PACKAGE__, $prop, undef, 1, 0);
2146
2147 # Fail if not found, or isn't a boolean property-value, or is a
2148 # user-defined property, or is internal-only.
2149 return if ! $swash
2150 || ref $swash eq ""
2151 || $swash->{'BITS'} != 1
2152 || $swash->{'USER_DEFINED'}
92feec86 2153 || (! $internal_ok && $prop =~ /^\s*_/);
681d705c
KW
2154
2155 if ($swash->{'EXTRAS'}) {
2156 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has EXTRAS magic";
2157 return;
2158 }
2159 if ($swash->{'SPECIALS'}) {
2160 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has SPECIALS magic";
2161 return;
2162 }
2163
2164 my @invlist;
2165
2166 # The input lines look like:
2167 # 0041\t005A # [26]
2168 # 005F
2169
2170 # Split into lines, stripped of trailing comments
2171 foreach my $range (split "\n",
2172 $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr)
2173 {
2174 # And find the beginning and end of the range on the line
2175 my ($hex_begin, $hex_end) = split "\t", $range;
2176 my $begin = hex $hex_begin;
2177
a39cc031
KW
2178 # If the new range merely extends the old, we remove the marker
2179 # created the last time through the loop for the old's end, which
2180 # causes the new one's end to be used instead.
2181 if (@invlist && $begin == $invlist[-1]) {
2182 pop @invlist;
2183 }
2184 else {
2f3f243e
KW
2185 # Add the beginning of the range
2186 push @invlist, $begin;
a39cc031 2187 }
681d705c
KW
2188
2189 if (defined $hex_end) { # The next item starts with the code point 1
2190 # beyond the end of the range.
2191 push @invlist, hex($hex_end) + 1;
2192 }
2193 else { # No end of range, is a single code point.
2194 push @invlist, $begin + 1;
2195 }
2196 }
2197
2198 require "unicore/UCD.pl";
2199 my $FIRST_NON_UNICODE = $MAX_UNICODE_CODEPOINT + 1;
2200
2201 # Could need to be inverted: add or subtract a 0 at the beginning of the
2202 # list. And to keep it from matching non-Unicode, add or subtract the
2203 # first non-unicode code point.
2204 if ($swash->{'INVERT_IT'}) {
2205 if (@invlist && $invlist[0] == 0) {
2206 shift @invlist;
2207 }
2208 else {
2209 unshift @invlist, 0;
2210 }
2211 if (@invlist && $invlist[-1] == $FIRST_NON_UNICODE) {
2212 pop @invlist;
2213 }
2214 else {
2215 push @invlist, $FIRST_NON_UNICODE;
2216 }
2217 }
2218
2219 # Here, the list is set up to include only Unicode code points. But, if
2220 # the table is the default one for the property, it should contain all
2221 # non-Unicode code points. First calculate the loose name for the
2222 # property. This is done even for strict-name properties, as the data
2223 # structure that mktables generates for us is set up so that we don't have
2224 # to worry about that. The property-value needs to be split if compound,
2225 # as the loose rules need to be independently calculated on each part. We
2226 # know that it is syntactically valid, or SWASHNEW would have failed.
2227
2228 $prop = lc $prop;
2229 my ($prop_only, $table) = split /\s*[:=]\s*/, $prop;
2230 if ($table) {
2231
2232 # May have optional prefixed 'is'
2233 $prop = utf8::_loose_name($prop_only) =~ s/^is//r;
2234 $prop = $utf8::loose_property_name_of{$prop};
2235 $prop .= "=" . utf8::_loose_name($table);
2236 }
2237 else {
2238 $prop = utf8::_loose_name($prop);
2239 }
2240 if (exists $loose_defaults{$prop}) {
2241
2242 # Here, is the default table. If a range ended with 10ffff, instead
2243 # continue that range to infinity, by popping the 110000; otherwise,
2244 # add the range from 11000 to infinity
2245 if (! @invlist || $invlist[-1] != $FIRST_NON_UNICODE) {
2246 push @invlist, $FIRST_NON_UNICODE;
2247 }
2248 else {
2249 pop @invlist;
2250 }
2251 }
2252
2253 return @invlist;
2254}
7319f91d 2255
62b3b855
KW
2256sub _search_invlist {
2257 # Find the range in the inversion list which contains a code point; that
2258 # is, find i such that l[i] <= code_point < l[i+1]
2259
2260 # If this is ever made public, could use to speed up .t specials. Would
2261 # need to use code point argument, as in other functions in this pm
2262
2263 my $list_ref = shift;
2264 my $code_point = shift;
2265 # Verify non-neg numeric XXX
2266
2267 my $max_element = @$list_ref - 1;
2268 return if ! $max_element < 0; # Undef if list is empty.
2269
2270 # Short cut something at the far-end of the table. This also allows us to
2271 # refer to element [$i+1] without fear of being out-of-bounds in the loop
2272 # below.
2273 return $max_element if $code_point >= $list_ref->[$max_element];
2274
2275 use integer; # want integer division
2276
2277 my $i = $max_element / 2;
2278
2279 my $lower = 0;
2280 my $upper = $max_element;
2281 while (1) {
2282
2283 if ($code_point >= $list_ref->[$i]) {
2284
2285 # Here we have met the lower constraint. We can quit if we
2286 # also meet the upper one.
2287 last if $code_point < $list_ref->[$i+1];
2288
2289 $lower = $i; # Still too low.
2290
2291 }
2292 else {
2293
2294 # Here, $code_point < $list_ref[$i], so look lower down.
2295 $upper = $i;
2296 }
2297
2298 # Split search domain in half to try again.
2299 my $temp = ($upper + $lower) / 2;
2300
2301 # No point in continuing unless $i changes for next time
2302 # in the loop.
2303 return $i if $temp == $i;
2304 $i = $temp;
2305 } # End of while loop
2306
2307 # Here we have found the offset
2308 return $i;
2309}
2310
2311=pod
2312
2313=head2 B<prop_invmap()>
2314
2315 use Unicode::UCD 'prop_invmap';
2316 my ($list_ref, $map_ref, $format, $missing)
2317 = prop_invmap("General Category");
2318
2319C<prop_invmap> is used to get the complete mapping definition for a property,
2320in the form of an inversion map. An inversion map consists of two parallel
2321arrays. One is an ordered list of code points that mark range beginnings, and
2322the other gives the value (or mapping) that all code points in the
2323corresponding range have.
2324
2325C<prop_invmap> is called with the name of the desired property. The name is
2326loosely matched, meaning that differences in case, white-space, hyphens, and
2327underscores are not meaningful (except for the trailing underscore in the
2328old-form grandfathered-in property C<"L_">, which is better written as C<"LC">,
2329or even better, C<"Gc=LC">).
2330
2331Many Unicode properties have more than one name (or alias). C<prop_invmap>
2332understands all of these, including Perl extensions to them. Ambiguities are
2333resolved as described above for L</prop_aliases()>. The Perl internal
2334property "Perl_Decimal_Digit, described below, is also accepted. C<undef> is
2335returned if the property name is unknown.
ee94c7d1
KW
2336See L<perluniprops/Properties accessible through Unicode::UCD> for the
2337properties acceptable as inputs to this function.
62b3b855
KW
2338
2339It is a fatal error to call this function except in list context.
2340
2341In addition to the the two arrays that form the inversion map, C<prop_invmap>
2342returns two other values; one is a scalar that gives some details as to the
2343format of the entries of the map array; the other is used for specialized
2344purposes, described at the end of this section.
2345
2346This means that C<prop_invmap> returns a 4 element list. For example,
2347
2348 my ($blocks_ranges_ref, $blocks_maps_ref, $format, $default)
2349 = prop_invmap("Block");
2350
2351In this call, the two arrays will be populated as shown below (for Unicode
23526.0):
2353
2354 Index @blocks_ranges @blocks_maps
2355 0 0x0000 Basic Latin
2356 1 0x0080 Latin-1 Supplement
2357 2 0x0100 Latin Extended-A
2358 3 0x0180 Latin Extended-B
2359 4 0x0250 IPA Extensions
2360 5 0x02B0 Spacing Modifier Letters
2361 6 0x0300 Combining Diacritical Marks
2362 7 0x0370 Greek and Coptic
2363 8 0x0400 Cyrillic
2364 ...
2365 233 0x2B820 No_Block
2366 234 0x2F800 CJK Compatibility Ideographs Supplement
2367 235 0x2FA20 No_Block
2368 236 0xE0000 Tags
2369 237 0xE0080 No_Block
2370 238 0xE0100 Variation Selectors Supplement
2371 239 0xE01F0 No_Block
2372 240 0xF0000 Supplementary Private Use Area-A
2373 241 0x100000 Supplementary Private Use Area-B
2374 242 0x110000 No_Block
2375
2376The first line (with Index [0]) means that the value for code point 0 is "Basic
2377Latin". The entry "0x0080" in the @blocks_ranges column in the second line
2378means that the value from the first line, "Basic Latin", extends to all code
2379points in the range from 0 up to but not including 0x0080, that is, through
647396da 2380127. In other words, the code points from 0 to 127 are all in the "Basic
62b3b855
KW
2381Latin" block. Similarly, all code points in the range from 0x0080 up to (but
2382not including) 0x0100 are in the block named "Latin-1 Supplement", etc.
2383(Notice that the return is the old-style block names; see L</Old-style versus
2384new-style block names>).
2385
2386The final line (with Index [242]) means that the value for all code points above
2387the legal Unicode maximum code point have the value "No_Block", which is the
2388term Unicode uses for a non-existing block.
2389
2390The arrays completely specify the mappings for all possible code points.
2391The final element in an inversion map returned by this function will always be
2392for the range that consists of all the code points that aren't legal Unicode,
2393but that are expressible on the platform. (That is, it starts with code point
23940x110000, the first code point above the legal Unicode maximum, and extends to
2395infinity.) The value for that range will be the same that any typical
2396unassigned code point has for the specified property. (Certain unassigned
2397code points are not "typical"; for example the non-character code points, or
2398those in blocks that are to be written right-to-left. The above-Unicode
2399range's value is not based on these atypical code points.) It could be argued
2400that, instead of treating these as unassigned Unicode code points, the value
2401for this range should be C<undef>. If you wish, you can change the returned
2402arrays accordingly.
2403
2404The maps are almost always simple scalars that should be interpreted as-is.
2405These values are those given in the Unicode-supplied data files, which may be
2406inconsistent as to capitalization and as to which synonym for a property-value
2407is given. The results may be normalized by using the L</prop_value_aliases()>
2408function.
2409
2410There are exceptions to the simple scalar maps. Some properties have some
2411elements in their map list that are themselves lists of scalars; and some
2412special strings are returned that are not to be interpreted as-is. Element
2413[2] (placed into C<$format> in the example above) of the returned four element
647396da 2414list tells you if the map has any of these special elements or not, as follows:
62b3b855
KW
2415
2416=over
2417
dc8d8ea6 2418=item B<C<s>>
62b3b855
KW
2419
2420means all the elements of the map array are simple scalars, with no special
2421elements. Almost all properties are like this, like the C<block> example
2422above.
2423
dc8d8ea6 2424=item B<C<sl>>
62b3b855 2425
647396da 2426means that some of the map array elements have the form given by C<"s">, and
62b3b855
KW
2427the rest are lists of scalars. For example, here is a portion of the output
2428of calling C<prop_invmap>() with the "Script Extensions" property:
2429
2430 @scripts_ranges @scripts_maps
2431 ...
c2ca0207
KW
2432 0x0953 Devanagari
2433 0x0964 [ Bengali, Devanagari, Gurumukhi, Oriya ]
2434 0x0966 Devanagari
62b3b855
KW
2435 0x0970 Common
2436
647396da
KW
2437Here, the code points 0x964 and 0x965 are both used in Bengali,
2438Devanagari, Gurmukhi, and Oriya, but no other scripts.
62b3b855 2439
647396da 2440The Name_Alias property is also of this form. But each scalar consists of two
58b75e36 2441components: 1) the name, and 2) the type of alias this is. They are
7620cb10
KW
2442separated by a colon and a space. In Unicode 6.1, there are several alias types:
2443
2444=over
2445
2446=item C<correction>
2447
2448indicates that the name is a corrected form for the
2449original name (which remains valid) for the same code point.
2450
2451=item C<control>
2452
2453adds a new name for a control character.
2454
2455=item C<alternate>
2456
2457is an alternate name for a character
2458
2459=item C<figment>
2460
2461is a name for a character that has been documented but was never in any
2462actual standard.
2463
2464=item C<abbreviation>
2465
2466is a common abbreviation for a character
2467
2468=back
2469
2470The lists are ordered (roughly) so the most preferred names come before less
2471preferred ones.
58b75e36
KW
2472
2473For example,
2474
7620cb10
KW
2475 @aliases_ranges @alias_maps
2476 ...
2477 0x009E [ 'PRIVACY MESSAGE: control', 'PM: abbreviation' ]
2478 0x009F [ 'APPLICATION PROGRAM COMMAND: control',
2479 'APC: abbreviation'
2480 ]
2481 0x00A0 'NBSP: abbreviation'
2482 0x00A1 ""
2483 0x00AD 'SHY: abbreviation'
2484 0x00AE ""
2485 0x01A2 'LATIN CAPITAL LETTER GHA: correction'
2486 0x01A3 'LATIN SMALL LETTER GHA: correction'
2487 0x01A4 ""
58b75e36 2488 ...
58b75e36 2489
7620cb10
KW
2490A map to the empty string means that there is no alias defined for the code
2491point.
58b75e36 2492
d11155ec 2493=item B<C<a>>
62b3b855 2494
647396da 2495is like C<"s"> in that all the map array elements are scalars, but here they are
d11155ec
KW
2496restricted to all being integers, and some have to be adjusted (hence the name
2497C<"a">) to get the correct result. For example, in:
62b3b855
KW
2498
2499 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2500 = prop_invmap("Simple_Uppercase_Mapping");
2501
2502the returned arrays look like this:
2503
2504 @$uppers_ranges_ref @$uppers_maps_ref Note
bf7fe2df 2505 0 0
d11155ec 2506 97 65 'a' maps to 'A', b => B ...
bf7fe2df 2507 123 0
d11155ec 2508 181 924 MICRO SIGN => Greek Cap MU
bf7fe2df 2509 182 0
62b3b855
KW
2510 ...
2511
d11155ec
KW
2512Let's start with the second line. It says that the uppercase of code point 97
2513is 65; or C<uc("a")> == "A". But the line is for the entire range of code
2514points 97 through 122. To get the mapping for any code point in a range, you
2515take the offset it has from the beginning code point of the range, and add
2516that to the mapping for that first code point. So, the mapping for 122 ("z")
2517is derived by taking the offset of 122 from 97 (=25) and adding that to 65,
2518yielding 90 ("z"). Likewise for everything in between.
2519
2520The first line works the same way. The first map in a range is always the
2521correct value for its code point (because the adjustment is 0). Thus the
2522C<uc(chr(0))> is just itself. Also, C<uc(chr(1))> is also itself, as the
2523adjustment is 0+1-0 .. C<uc(chr(96))> is 96.
bf7fe2df 2524
d11155ec
KW
2525Requiring this simple adjustment allows the returned arrays to be
2526significantly smaller than otherwise, up to a factor of 10, speeding up
2527searching through them.
62b3b855 2528
d11155ec 2529=item B<C<al>>
62b3b855 2530
d11155ec 2531means that some of the map array elements have the form given by C<"a">, and
62b3b855
KW
2532the rest are ordered lists of code points.
2533For example, in:
2534
2535 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2536 = prop_invmap("Uppercase_Mapping");
2537
2538the returned arrays look like this:
2539
2540 @$uppers_ranges_ref @$uppers_maps_ref
bf7fe2df 2541 0 0
d11155ec 2542 97 65
bf7fe2df 2543 123 0
d11155ec 2544 181 924
bf7fe2df 2545 182 0
62b3b855
KW
2546 ...
2547 0x0149 [ 0x02BC 0x004E ]
bf7fe2df 2548 0x014A 0
d11155ec 2549 0x014B 330
62b3b855
KW
2550 ...
2551
2552This is the full Uppercase_Mapping property (as opposed to the
d11155ec 2553Simple_Uppercase_Mapping given in the example for format C<"a">). The only
62b3b855
KW
2554difference between the two in the ranges shown is that the code point at
25550x0149 (LATIN SMALL LETTER N PRECEDED BY APOSTROPHE) maps to a string of two
2556characters, 0x02BC (MODIFIER LETTER APOSTROPHE) followed by 0x004E (LATIN
2557CAPITAL LETTER N).
2558
d11155ec
KW
2559No adjustments are needed to entries that are references to arrays; each such
2560entry will have exactly one element in its range, so the offset is always 0.
bf7fe2df 2561
d11155ec 2562=item B<C<ae>>
b0b13ada 2563
d11155ec
KW
2564This is like C<"a">, but some elements are the empty string, and should not be
2565adjusted.
b0b13ada
KW
2566The one internal Perl property accessible by C<prop_invmap> is of this type:
2567"Perl_Decimal_Digit" returns an inversion map which gives the numeric values
2568that are represented by the Unicode decimal digit characters. Characters that
2569don't represent decimal digits map to the empty string, like so:
2570
2571 @digits @values
2572 0x0000 ""
d11155ec 2573 0x0030 0
b0b13ada 2574 0x003A: ""
d11155ec 2575 0x0660: 0
b0b13ada 2576 0x066A: ""
d11155ec 2577 0x06F0: 0
b0b13ada 2578 0x06FA: ""
d11155ec 2579 0x07C0: 0
b0b13ada 2580 0x07CA: ""
d11155ec 2581 0x0966: 0
b0b13ada
KW
2582 ...
2583
2584This means that the code points from 0 to 0x2F do not represent decimal digits;
d11155ec
KW
2585the code point 0x30 (DIGIT ZERO) represents 0; code point 0x31, (DIGIT ONE),
2586represents 0+1-0 = 1; ... code point 0x39, (DIGIT NINE), represents 0+9-0 = 9;
2587... code points 0x3A through 0x65F do not represent decimal digits; 0x660
2588(ARABIC-INDIC DIGIT ZERO), represents 0; ... 0x07C1 (NKO DIGIT ONE),
2589represents 0+1-0 = 1 ...
b0b13ada 2590
d11155ec 2591=item B<C<ale>>
62b3b855 2592
d11155ec
KW
2593is a combination of the C<"al"> type and the C<"ae"> type. Some of
2594the map array elements have the forms given by C<"al">, and
62b3b855
KW
2595the rest are the empty string. The property C<NFKC_Casefold> has this form.
2596An example slice is:
2597
2598 @$ranges_ref @$maps_ref Note
2599 ...
d11155ec
KW
2600 0x00AA 97 FEMININE ORDINAL INDICATOR => 'a'
2601 0x00AB 0
62b3b855 2602 0x00AD SOFT HYPHEN => ""
d11155ec 2603 0x00AE 0
62b3b855 2604 0x00AF [ 0x0020, 0x0304 ] MACRON => SPACE . COMBINING MACRON
d11155ec 2605 0x00B0 0
62b3b855
KW
2606 ...
2607
4f143a72 2608=item B<C<ar>>
6cc45523
KW
2609
2610means that all the elements of the map array are either rational numbers or
2611the string C<"NaN">, meaning "Not a Number". A rational number is either an
2612integer, or two integers separated by a solidus (C<"/">). The second integer
2613represents the denominator of the division implied by the solidus, and is
60c78852 2614actually always positive, so it is guaranteed not to be 0 and to not be
6329003c 2615signed. When the element is a plain integer (without the
4f143a72
KW
2616solidus), it may need to be adjusted to get the correct value by adding the
2617offset, just as other C<"a"> properties. No adjustment is needed for
2618fractions, as the range is guaranteed to have just a single element, and so
2619the offset is always 0.
2620
2621If you want to convert the returned map to entirely scalar numbers, you
6cc45523
KW
2622can use something like this:
2623
2624 my ($invlist_ref, $invmap_ref, $format) = prop_invmap($property);
4f143a72 2625 if ($format && $format eq "ar") {
60c78852 2626 map { $_ = eval $_ if $_ ne 'NaN' } @$map_ref;
6cc45523
KW
2627 }
2628
2629Here's some entries from the output of the property "Nv", which has format
4f143a72 2630C<"ar">.
6cc45523 2631
4f143a72 2632 @numerics_ranges @numerics_maps Note
6cc45523 2633 0x00 "NaN"
4f143a72 2634 0x30 0 DIGIT 0 .. DIGIT 9
6cc45523 2635 0x3A "NaN"
4f143a72 2636 0xB2 2 SUPERSCRIPTs 2 and 3
6cc45523 2637 0xB4 "NaN"
4f143a72 2638 0xB9 1 SUPERSCRIPT 1
6cc45523 2639 0xBA "NaN"
4f143a72
KW
2640 0xBC 1/4 VULGAR FRACTION 1/4
2641 0xBD 1/2 VULGAR FRACTION 1/2
2642 0xBE 3/4 VULGAR FRACTION 3/4
6cc45523 2643 0xBF "NaN"
4f143a72
KW
2644 0x660 0 ARABIC-INDIC DIGIT ZERO .. NINE
2645 0x66A "NaN"
6cc45523 2646
dc8d8ea6 2647=item B<C<n>>
62b3b855
KW
2648
2649means the Name property. All the elements of the map array are simple
2650scalars, but some of them contain special strings that require more work to
2651get the actual name.
2652
2653Entries such as:
2654
2655 CJK UNIFIED IDEOGRAPH-<code point>
2656
2657mean that the name for the code point is "CJK UNIFIED IDEOGRAPH-"
2658with the code point (expressed in hexadecimal) appended to it, like "CJK
647396da
KW
2659UNIFIED IDEOGRAPH-3403" (similarly for S<C<CJK COMPATIBILITY IDEOGRAPH-E<lt>code
2660pointE<gt>>>).
62b3b855
KW
2661
2662Also, entries like
2663
2664 <hangul syllable>
2665
2666means that the name is algorithmically calculated. This is easily done by
2667the function L<charnames/charnames::viacode(code)>.
2668
2669Note that for control characters (C<Gc=cc>), Unicode's data files have the
2670string "C<E<lt>controlE<gt>>", but the real name of each of these characters is the empty
7620cb10 2671string. This function returns that real name, the empty string. (There are
647396da
KW
2672names for these characters, but they are considered aliases, not the Name
2673property name, and are contained in the C<Name_Alias> property.)
62b3b855 2674
d11155ec 2675=item B<C<ad>>
62b3b855 2676
d11155ec 2677means the Decomposition_Mapping property. This property is like C<"al">
bea2c146 2678properties, except that one of the scalar elements is of the form:
62b3b855
KW
2679
2680 <hangul syllable>
2681
bea2c146
KW
2682This signifies that this entry should be replaced by the decompositions for
2683all the code points whose decomposition is algorithmically calculated. (All
6329003c
KW
2684of them are currently in one range and no others outisde the range are likely
2685to ever be added to Unicode; the C<"n"> format
bea2c146 2686has this same entry.) These can be generated via the function
62b3b855
KW
2687L<Unicode::Normalize::NFD()|Unicode::Normalize>.
2688
62b3b855
KW
2689Note that the mapping is the one that is specified in the Unicode data files,
2690and to get the final decomposition, it may need to be applied recursively.
2691
2692=back
2693
d11155ec
KW
2694Note that a format begins with the letter "a" if and only the property it is
2695for requires adjustments by adding the offsets in multi-element ranges. For
2696all these properties, an entry should be adjusted only if the map is a scalar
2697which is an integer. That is, it must match the regular expression:
2698
2699 / ^ -? \d+ $ /xa
2700
2701Further, the first element in a range never needs adjustment, as the
2702adjustment would be just adding 0.
2703
62b3b855
KW
2704A binary search can be used to quickly find a code point in the inversion
2705list, and hence its corresponding mapping.
2706
2707The final element (index [3], assigned to C<$default> in the "block" example) in
2708the four element list returned by this function may be useful for applications
2709that wish to convert the returned inversion map data structure into some
2710other, such as a hash. It gives the mapping that most code points map to
2711under the property. If you establish the convention that any code point not
2712explicitly listed in your data structure maps to this value, you can
2713potentially make your data structure much smaller. As you construct your data
2714structure from the one returned by this function, simply ignore those ranges
2715that map to this value, generally called the "default" value. For example, to
2716convert to the data structure searchable by L</charinrange()>, you can follow
6329003c 2717this recipe for properties that don't require adjustments:
62b3b855
KW
2718
2719 my ($list_ref, $map_ref, $format, $missing) = prop_invmap($property);
2720 my @range_list;
6329003c
KW
2721
2722 # Look at each element in the list, but the -2 is needed because we
2723 # look at $i+1 in the loop, and the final element is guaranteed to map
2724 # to $missing by prop_invmap(), so we would skip it anyway.
62b3b855
KW
2725 for my $i (0 .. @$list_ref - 2) {
2726 next if $map_ref->[$i] eq $missing;
2727 push @range_list, [ $list_ref->[$i],
2728 $list_ref->[$i+1],
2729 $map_ref->[$i]
2730 ];
2731 }
2732
2733 print charinrange(\@range_list, $code_point), "\n";
2734
62b3b855
KW
2735With this, C<charinrange()> will return C<undef> if its input code point maps
2736to C<$missing>. You can avoid this by omitting the C<next> statement, and adding
2737a line after the loop to handle the final element of the inversion map.
2738
6329003c
KW
2739Similarly, this recipe can be used for properties that do require adjustments:
2740
2741 for my $i (0 .. @$list_ref - 2) {
2742 next if $map_ref->[$i] eq $missing;
2743
2744 # prop_invmap() guarantees that if the mapping is to an array, the
2745 # range has just one element, so no need to worry about adjustments.
2746 if (ref $map_ref->[$i]) {
2747 push @range_list,
2748 [ $list_ref->[$i], $list_ref->[$i], $map_ref->[$i] ];
2749 }
2750 else { # Otherwise each element is actually mapped to a separate
2751 # value, so the range has to be split into single code point
2752 # ranges.
2753
2754 my $adjustment = 0;
2755
2756 # For each code point that gets mapped to something...
2757 for my $j ($list_ref->[$i] .. $list_ref->[$i+1] -1 ) {
2758
2759 # ... add a range consisting of just it mapping to the
2760 # original plus the adjustment, which is incremented for the
2761 # next time through the loop, as the offset increases by 1
2762 # for each element in the range
2763 push @range_list,
2764 [ $j, $j, $map_ref->[$i] + $adjustment++ ];
2765 }
2766 }
2767 }
62b3b855
KW
2768
2769Note that the inversion maps returned for the C<Case_Folding> and
2770C<Simple_Case_Folding> properties do not include the Turkic-locale mappings.
2771Use L</casefold()> for these.
2772
62b3b855
KW
2773C<prop_invmap> does not know about any user-defined properties, and will
2774return C<undef> if called with one of those.
2775
2776=cut
2777
2778# User-defined properties could be handled with some changes to utf8_heavy.pl;
2779# if done, consideration should be given to the fact that the user subroutine
2780# could return different results with each call, which could lead to some
2781# security issues.
2782
2783# One could store things in memory so they don't have to be recalculated, but
2784# it is unlikely this will be called often, and some properties would take up
2785# significant memory.
2786
2787# These are created by mktables for this routine and stored in unicore/UCD.pl
2788# where their structures are described.
2789our @algorithmic_named_code_points;
2790our $HANGUL_BEGIN;
2791our $HANGUL_COUNT;
2792
2793sub prop_invmap ($) {
2794
2795 croak __PACKAGE__, "::prop_invmap: must be called in list context" unless wantarray;
2796
2797 my $prop = $_[0];
2798 return unless defined $prop;
2799
2800 # Fail internal properties
2801 return if $prop =~ /^_/;
2802
2803 # The values returned by this function.
2804 my (@invlist, @invmap, $format, $missing);
2805
2806 # The swash has two components we look at, the base list, and a hash,
2807 # named 'SPECIALS', containing any additional members whose mappings don't
2808 # fit into the the base list scheme of things. These generally 'override'
2809 # any value in the base list for the same code point.
2810 my $overrides;
2811
2812 require "utf8_heavy.pl";
2813 require "unicore/UCD.pl";
2814
2815RETRY:
2816
647396da
KW
2817 # If there are multiple entries for a single code point
2818 my $has_multiples = 0;
2819
62b3b855
KW
2820 # Try to get the map swash for the property. They have 'To' prepended to
2821 # the property name, and 32 means we will accept 32 bit return values.
647396da 2822 # The 0 means we aren't calling this from tr///.
62b3b855
KW
2823 my $swash = utf8::SWASHNEW(__PACKAGE__, "To$prop", undef, 32, 0);
2824
2825 # If didn't find it, could be because needs a proxy. And if was the
2826 # 'Block' or 'Name' property, use a proxy even if did find it. Finding it
647396da
KW
2827 # in these cases would be the result of the installation changing mktables
2828 # to output the Block or Name tables. The Block table gives block names
2829 # in the new-style, and this routine is supposed to return old-style block
2830 # names. The Name table is valid, but we need to execute the special code
2831 # below to add in the algorithmic-defined name entries.
34132297 2832 # And NFKCCF needs conversion, so handle that here too.
62b3b855 2833 if (ref $swash eq ""
34132297 2834 || $swash->{'TYPE'} =~ / ^ To (?: Blk | Na | NFKCCF ) $ /x)
62b3b855
KW
2835 {
2836
2837 # Get the short name of the input property, in standard form
2838 my ($second_try) = prop_aliases($prop);
2839 return unless $second_try;
2840 $second_try = utf8::_loose_name(lc $second_try);
2841
2842 if ($second_try eq "in") {
2843
2844 # This property is identical to age for inversion map purposes
2845 $prop = "age";
2846 goto RETRY;
2847 }
cfc5eb77 2848 elsif ($second_try =~ / ^ s ( cf | fc | [ltu] c ) $ /x) {
62b3b855 2849
75e7c50b
KW
2850 # These properties use just the LIST part of the full mapping,
2851 # which includes the simple maps that are otherwise overridden by
2852 # the SPECIALS. So all we need do is to not look at the SPECIALS;
2853 # set $overrides to indicate that
62b3b855 2854 $overrides = -1;
62b3b855 2855
75e7c50b 2856 # The full name is the simple name stripped of its initial 's'
cfc5eb77
KW
2857 $prop = $1;
2858
2859 # .. except for this case
2860 $prop = 'cf' if $prop eq 'fc';
2861
62b3b855
KW
2862 goto RETRY;
2863 }
2864 elsif ($second_try eq "blk") {
2865
2866 # We use the old block names. Just create a fake swash from its
2867 # data.
2868 _charblocks();
2869 my %blocks;
2870 $blocks{'LIST'} = "";
2871 $blocks{'TYPE'} = "ToBlk";
2872 $utf8::SwashInfo{ToBlk}{'missing'} = "No_Block";
2873 $utf8::SwashInfo{ToBlk}{'format'} = "s";
2874
2875 foreach my $block (@BLOCKS) {
2876 $blocks{'LIST'} .= sprintf "%x\t%x\t%s\n",
2877 $block->[0],
2878 $block->[1],
2879 $block->[2];
2880 }
2881 $swash = \%blocks;
2882 }
2883 elsif ($second_try eq "na") {
2884
2885 # Use the combo file that has all the Name-type properties in it,
2886 # extracting just the ones that are for the actual 'Name'
2887 # property. And create a fake swash from it.
2888 my %names;
2889 $names{'LIST'} = "";
2890 my $original = do "unicore/Name.pl";
62b3b855
KW
2891 my $algorithm_names = \@algorithmic_named_code_points;
2892
3b6a8189
KW
2893 # We need to remove the names from it that are aliases. For that
2894 # we need to also read in that table. Create a hash with the keys
2895 # being the code points, and the values being a list of the
2896 # aliases for the code point key.
2897 my ($aliases_code_points, $aliases_maps, undef, undef) =
2898 &prop_invmap('Name_Alias');
2899 my %aliases;
2900 for (my $i = 0; $i < @$aliases_code_points; $i++) {
2901 my $code_point = $aliases_code_points->[$i];
2902 $aliases{$code_point} = $aliases_maps->[$i];
2903
2904 # If not already a list, make it into one, so that later we
2905 # can treat things uniformly
2906 if (! ref $aliases{$code_point}) {
2907 $aliases{$code_point} = [ $aliases{$code_point} ];
2908 }
2909
2910 # Remove the alias type from the entry, retaining just the
2911 # name.
2912 map { s/:.*// } @{$aliases{$code_point}};
2913 }
2914
62b3b855
KW
2915 my $i = 0;
2916 foreach my $line (split "\n", $original) {
2917 my ($hex_code_point, $name) = split "\t", $line;
2918
2919 # Weeds out all comments, blank lines, and named sequences
df46a385 2920 next if $hex_code_point =~ /[^[:xdigit:]]/a;
62b3b855
KW
2921
2922 my $code_point = hex $hex_code_point;
2923
2924 # The name of all controls is the default: the empty string.
2925 # The set of controls is immutable, so these hard-coded
2926 # constants work.
2927 next if $code_point <= 0x9F
2928 && ($code_point <= 0x1F || $code_point >= 0x7F);
2929
3b6a8189
KW
2930 # If this is a name_alias, it isn't a name
2931 next if grep { $_ eq $name } @{$aliases{$code_point}};
62b3b855
KW
2932
2933 # If we are beyond where one of the special lines needs to
2934 # be inserted ...
3b6a8189 2935 while ($i < @$algorithm_names
62b3b855
KW
2936 && $code_point > $algorithm_names->[$i]->{'low'})
2937 {
2938
2939 # ... then insert it, ahead of what we were about to
2940 # output
3b6a8189 2941 $names{'LIST'} .= sprintf "%x\t%x\t%s\n",
62b3b855
KW
2942 $algorithm_names->[$i]->{'low'},
2943 $algorithm_names->[$i]->{'high'},
2944 $algorithm_names->[$i]->{'name'};
2945
62b3b855
KW
2946 # Done with this range.
2947 $i++;
2948
3b6a8189
KW
2949 # We loop until all special lines that precede the next
2950 # regular one are output.
62b3b855
KW
2951 }
2952
3b6a8189
KW
2953 # Here, is a normal name.
2954 $names{'LIST'} .= sprintf "%x\t\t%s\n", $code_point, $name;
2955 } # End of loop through all the names
62b3b855
KW
2956
2957 $names{'TYPE'} = "ToNa";
2958 $utf8::SwashInfo{ToNa}{'missing'} = "";
2959 $utf8::SwashInfo{ToNa}{'format'} = "n";
2960 $swash = \%names;
2961 }
2962 elsif ($second_try =~ / ^ ( d [mt] ) $ /x) {
2963
2964 # The file is a combination of dt and dm properties. Create a
2965 # fake swash from the portion that we want.
2966 my $original = do "unicore/Decomposition.pl";
2967 my %decomps;
2968
2969 if ($second_try eq 'dt') {
2970 $decomps{'TYPE'} = "ToDt";
2971 $utf8::SwashInfo{'ToDt'}{'missing'} = "None";
2972 $utf8::SwashInfo{'ToDt'}{'format'} = "s";
d11155ec 2973 } # 'dm' is handled below, with 'nfkccf'
62b3b855
KW
2974
2975 $decomps{'LIST'} = "";
2976
2977 # This property has one special range not in the file: for the
1a4c9760
KW
2978 # hangul syllables. But not in Unicode version 1.
2979 UnicodeVersion() unless defined $v_unicode_version;
2980 my $done_hangul = ($v_unicode_version lt v2.0.0)
2981 ? 1
2982 : 0; # Have we done the hangul range ?
62b3b855
KW
2983 foreach my $line (split "\n", $original) {
2984 my ($hex_lower, $hex_upper, $type_and_map) = split "\t", $line;
2985 my $code_point = hex $hex_lower;
2986 my $value;
bea2c146 2987 my $redo = 0;
62b3b855
KW
2988
2989 # The type, enclosed in <...>, precedes the mapping separated
2990 # by blanks
2991 if ($type_and_map =~ / ^ < ( .* ) > \s+ (.*) $ /x) {
2992 $value = ($second_try eq 'dt') ? $1 : $2
2993 }
2994 else { # If there is no type specified, it's canonical
2995 $value = ($second_try eq 'dt')
2996 ? "Canonical" :
2997 $type_and_map;
2998 }
2999
3000 # Insert the hangul range at the appropriate spot.
3001 if (! $done_hangul && $code_point > $HANGUL_BEGIN) {
3002 $done_hangul = 1;
3003 $decomps{'LIST'} .=
3004 sprintf "%x\t%x\t%s\n",
3005 $HANGUL_BEGIN,
3006 $HANGUL_BEGIN + $HANGUL_COUNT - 1,
3007 ($second_try eq 'dt')
3008 ? "Canonical"
3009 : "<hangul syllable>";
3010 }
3011
1a4c9760
KW
3012 if ($value =~ / / && $hex_upper ne "" && $hex_upper ne $hex_lower) {
3013 $line = sprintf("%04X\t%s\t%s", hex($hex_lower) + 1, $hex_upper, $value);
3014 $hex_upper = "";
3015 $redo = 1;
3016 }
3017
62b3b855
KW
3018 # And append this to our constructed LIST.
3019 $decomps{'LIST'} .= "$hex_lower\t$hex_upper\t$value\n";
bea2c146
KW
3020
3021 redo if $redo;
62b3b855
KW
3022 }
3023 $swash = \%decomps;
3024 }
d11155ec
KW
3025 elsif ($second_try ne 'nfkccf') { # Don't know this property. Fail.
3026 return;
3027 }
3028
3029 if ($second_try eq 'nfkccf' || $second_try eq 'dm') {
34132297 3030
d11155ec
KW
3031 # The 'nfkccf' property is stored in the old format for backwards
3032 # compatibility for any applications that has read its file
3033 # directly before prop_invmap() existed.
3034 # And the code above has extracted the 'dm' property from its file
3035 # yielding the same format. So here we convert them to adjusted
3036 # format for compatibility with the other properties similar to
3037 # them.
3038 my %revised_swash;
34132297 3039
d11155ec 3040 # We construct a new converted list.
34132297 3041 my $list = "";
d11155ec
KW
3042
3043 my @ranges = split "\n", $swash->{'LIST'};
3044 for (my $i = 0; $i < @ranges; $i++) {
3045 my ($hex_begin, $hex_end, $map) = split "\t", $ranges[$i];
3046
3047 # The dm property has maps that are space separated sequences
3048 # of code points, as well as the special entry "<hangul
3049 # syllable>, which also contains a blank.
3050 my @map = split " ", $map;
3051 if (@map > 1) {
3052
3053 # If it's just the special entry, append as-is.
3054 if ($map eq '<hangul syllable>') {
3055 $list .= "$ranges[$i]\n";
3056 }
3057 else {
3058
e2c04025 3059 # These should all be single-element ranges.
1a4c9760 3060 croak __PACKAGE__, "::prop_invmap: Not expecting a mapping with multiple code points in a multi-element range, $ranges[$i]" if $hex_end ne "" && $hex_end ne $hex_begin;
d11155ec
KW
3061
3062 # Convert them to decimal, as that's what's expected.
3063 $list .= "$hex_begin\t\t"
3064 . join(" ", map { hex } @map)
3065 . "\n";
3066 }
3067 next;
3068 }
3069
3070 # Here, the mapping doesn't have a blank, is for a single code
3071 # point.
34132297
KW
3072 my $begin = hex $hex_begin;
3073 my $end = (defined $hex_end && $hex_end ne "")
3074 ? hex $hex_end
3075 : $begin;
d11155ec
KW
3076
3077 # Again, the output is to be in decimal.
34132297 3078 my $decimal_map = hex $map;
d11155ec
KW
3079
3080 # We know that multi-element ranges with the same mapping
3081 # should not be adjusted, as after the adjustment
3082 # multi-element ranges are for consecutive increasing code
3083 # points. Further, the final element in the list won't be
3084 # adjusted, as there is nothing after it to include in the
3085 # adjustment
3086 if ($begin != $end || $i == @ranges -1) {
3087
3088 # So just convert these to single-element ranges
3089 foreach my $code_point ($begin .. $end) {
3090 $list .= sprintf("%04X\t\t%d\n",
3091 $code_point, $decimal_map);
3092 }
34132297 3093 }
d11155ec 3094 else {
34132297 3095
d11155ec
KW
3096 # Here, we have a candidate for adjusting. What we do is
3097 # look through the subsequent adjacent elements in the
3098 # input. If the map to the next one differs by 1 from the
3099 # one before, then we combine into a larger range with the
3100 # initial map. Loop doing this until we find one that
3101 # can't be combined.
3102
3103 my $offset = 0; # How far away are we from the initial
3104 # map
3105 my $squished = 0; # ? Did we squish at least two
3106 # elements together into one range
3107 for ( ; $i < @ranges; $i++) {
3108 my ($next_hex_begin, $next_hex_end, $next_map)
3109 = split "\t", $ranges[$i+1];
3110
3111 # In the case of 'dm', the map may be a sequence of
3112 # multiple code points, which are never combined with
3113 # another range
3114 last if $next_map =~ / /;
3115
3116 $offset++;
3117 my $next_decimal_map = hex $next_map;
3118
3119 # If the next map is not next in sequence, it
3120 # shouldn't be combined.
3121 last if $next_decimal_map != $decimal_map + $offset;
3122
3123 my $next_begin = hex $next_hex_begin;
3124
3125 # Likewise, if the next element isn't adjacent to the
3126 # previous one, it shouldn't be combined.
3127 last if $next_begin != $begin + $offset;
3128
3129 my $next_end = (defined $next_hex_end
3130 && $next_hex_end ne "")
3131 ? hex $next_hex_end
3132 : $next_begin;
3133
3134 # And finally, if the next element is a multi-element
3135 # range, it shouldn't be combined.
3136 last if $next_end != $next_begin;
3137
3138 # Here, we will combine. Loop to see if we should
3139 # combine the next element too.
3140 $squished = 1;
3141 }
3142
3143 if ($squished) {
3144
3145 # Here, 'i' is the element number of the last element to
3146 # be combined, and the range is single-element, or we
3147 # wouldn't be combining. Get it's code point.
3148 my ($hex_end, undef, undef) = split "\t", $ranges[$i];
3149 $list .= "$hex_begin\t$hex_end\t$decimal_map\n";
3150 } else {
3151
3152 # Here, no combining done. Just appen the initial
3153 # (and current) values.
3154 $list .= "$hex_begin\t\t$decimal_map\n";
3155 }
3156 }
3157 } # End of loop constructing the converted list
3158
3159 # Finish up the data structure for our converted swash
3160 my $type = ($second_try eq 'nfkccf') ? 'ToNFKCCF' : 'ToDm';
3161 $revised_swash{'LIST'} = $list;
3162 $revised_swash{'TYPE'} = $type;
3163 $revised_swash{'SPECIALS'} = $swash->{'SPECIALS'};
3164 $swash = \%revised_swash;
3165
3166 $utf8::SwashInfo{$type}{'missing'} = 0;
3167 $utf8::SwashInfo{$type}{'format'} = 'a';
62b3b855
KW
3168 }
3169 }
3170
3171 if ($swash->{'EXTRAS'}) {
3172 carp __PACKAGE__, "::prop_invmap: swash returned for $prop unexpectedly has EXTRAS magic";
3173 return;
3174 }
3175
3176 # Here, have a valid swash return. Examine it.
34132297 3177 my $returned_prop = $swash->{'TYPE'};
62b3b855
KW
3178
3179 # All properties but binary ones should have 'missing' and 'format'
3180 # entries
3181 $missing = $utf8::SwashInfo{$returned_prop}{'missing'};
3182 $missing = 'N' unless defined $missing;
3183
3184 $format = $utf8::SwashInfo{$returned_prop}{'format'};
3185 $format = 'b' unless defined $format;
3186
d11155ec
KW
3187 my $requires_adjustment = $format =~ /^a/;
3188
62b3b855
KW
3189 # The LIST input lines look like:
3190 # ...
3191 # 0374\t\tCommon
3192 # 0375\t0377\tGreek # [3]
3193 # 037A\t037D\tGreek # [4]
3194 # 037E\t\tCommon
3195 # 0384\t\tGreek
3196 # ...
3197 #
3198 # Convert them to like
3199 # 0374 => Common
3200 # 0375 => Greek
3201 # 0378 => $missing
3202 # 037A => Greek
3203 # 037E => Common
3204 # 037F => $missing
3205 # 0384 => Greek
3206 #
3207 # For binary properties, the final non-comment column is absent, and
3208 # assumed to be 'Y'.
3209
3210 foreach my $range (split "\n", $swash->{'LIST'}) {
3211 $range =~ s/ \s* (?: \# .* )? $ //xg; # rmv trailing space, comments
3212
3213 # Find the beginning and end of the range on the line
3214 my ($hex_begin, $hex_end, $map) = split "\t", $range;
3215 my $begin = hex $hex_begin;
3216 my $end = (defined $hex_end && $hex_end ne "")
3217 ? hex $hex_end
3218 : $begin;
3219
92bcf67b
KW
3220 # Each time through the loop (after the first):
3221 # $invlist[-2] contains the beginning of the previous range processed
3222 # $invlist[-1] contains the end+1 of the previous range processed
3223 # $invmap[-2] contains the value of the previous range processed
3224 # $invmap[-1] contains the default value for missing ranges ($missing)
3225 #
3226 # Thus, things are set up for the typical case of a new non-adjacent
3227 # range of non-missings to be added. But, if the new range is
dc8d8ea6 3228 # adjacent, it needs to replace the [-1] element; and if the new
92bcf67b
KW
3229 # range is a multiple value of the previous one, it needs to be added
3230 # to the [-2] map element.
3231
3232 # The first time through, everything will be empty. If the property
3233 # doesn't have a range that begins at 0, add one that maps to $missing
62b3b855
KW
3234 if (! @invlist) {
3235 if ($begin != 0) {
3236 push @invlist, 0;
3237 push @invmap, $missing;
3238 }
3239 }
e35c6019
KW
3240 elsif (@invlist > 1 && $invlist[-2] == $begin) {
3241
3242 # Here we handle the case where the input has multiple entries for
3243 # each code point. mktables should have made sure that each such
3244 # range contains only one code point. At this point, $invlist[-1]
3245 # is the $missing that was added at the end of the last loop
3246 # iteration, and [-2] is the last real input code point, and that
3247 # code point is the same as the one we are adding now, making the
3248 # new one a multiple entry. Add it to the existing entry, either
3249 # by pushing it to the existing list of multiple entries, or
3250 # converting the single current entry into a list with both on it.
3251 # This is all we need do for this iteration.
3252
3253 if ($end != $begin) {
294705a8 3254 croak __PACKAGE__, ":prop_invmap: Multiple maps per code point in '$prop' require single-element ranges: begin=$begin, end=$end, map=$map";
e35c6019
KW
3255 }
3256 if (! ref $invmap[-2]) {
3257 $invmap[-2] = [ $invmap[-2], $map ];
3258 }
3259 else {
3260 push @{$invmap[-2]}, $map;
3261 }
3262 $has_multiples = 1;
3263 next;
3264 }
62b3b855
KW
3265 elsif ($invlist[-1] == $begin) {
3266
3267 # If the input isn't in the most compact form, so that there are
3268 # two adjacent ranges that map to the same thing, they should be
d11155ec
KW
3269 # combined (EXCEPT where the arrays require adjustments, in which
3270 # case everything is already set up correctly). This happens in
3271 # our constructed dt mapping, as Element [-2] is the map for the
3272 # latest range so far processed. Just set the beginning point of
3273 # the map to $missing (in invlist[-1]) to 1 beyond where this
3274 # range ends. For example, in
62b3b855
KW
3275 # 12\t13\tXYZ
3276 # 14\t17\tXYZ
3277 # we have set it up so that it looks like
3278 # 12 => XYZ
3279 # 14 => $missing
3280 #
3281 # We now see that it should be
3282 # 12 => XYZ
3283 # 18 => $missing
d11155ec 3284 if (! $requires_adjustment && @invlist > 1 && ( (defined $map)
c887f93f
KW
3285 ? $invmap[-2] eq $map
3286 : $invmap[-2] eq 'Y'))
3287 {
62b3b855
KW
3288 $invlist[-1] = $end + 1;
3289 next;
3290 }
3291
3292 # Here, the range started in the previous iteration that maps to
3293 # $missing starts at the same code point as this range. That
3294 # means there is no gap to fill that that range was intended for,
3295 # so we just pop it off the parallel arrays.
3296 pop @invlist;
3297 pop @invmap;
3298 }
3299
3300 # Add the range beginning, and the range's map.
3301 push @invlist, $begin;
d11155ec 3302 if ($returned_prop eq 'ToDm') {
62b3b855
KW
3303
3304 # The decomposition maps are either a line like <hangul syllable>
3305 # which are to be taken as is; or a sequence of code points in hex
3306 # and separated by blanks. Convert them to decimal, and if there
3307 # is more than one, use an anonymous array as the map.
3308 if ($map =~ /^ < /x) {
3309 push @invmap, $map;
3310 }
3311 else {
bea2c146 3312 my @map = split " ", $map;
62b3b855
KW
3313 if (@map == 1) {
3314 push @invmap, $map[0];
3315 }
3316 else {
3317 push @invmap, \@map;
3318 }
3319 }
3320 }
3321 else {
3322
3323 # Otherwise, convert hex formatted list entries to decimal; add a
3324 # 'Y' map for the missing value in binary properties, or
3325 # otherwise, use the input map unchanged.
3326 $map = ($format eq 'x')
3327 ? hex $map
3328 : $format eq 'b'
3329 ? 'Y'
3330 : $map;
3331 push @invmap, $map;
3332 }
3333
3334 # We just started a range. It ends with $end. The gap between it and
3335 # the next element in the list must be filled with a range that maps
3336 # to the default value. If there is no gap, the next iteration will
3337 # pop this, unless there is no next iteration, and we have filled all
3338 # of the Unicode code space, so check for that and skip.
3339 if ($end < $MAX_UNICODE_CODEPOINT) {
3340 push @invlist, $end + 1;
3341 push @invmap, $missing;
3342 }
3343 }
3344
3345 # If the property is empty, make all code points use the value for missing
3346 # ones.
3347 if (! @invlist) {
3348 push @invlist, 0;
3349 push @invmap, $missing;
3350 }
3351
647396da 3352 # And add in standard element that all non-Unicode code points map to:
62b3b855
KW
3353 # $missing
3354 push @invlist, $MAX_UNICODE_CODEPOINT + 1;
3355 push @invmap, $missing;
3356
3357 # The second component of the map are those values that require
3358 # non-standard specification, stored in SPECIALS. These override any
3359 # duplicate code points in LIST. If we are using a proxy, we may have
3360 # already set $overrides based on the proxy.
3361 $overrides = $swash->{'SPECIALS'} unless defined $overrides;
3362 if ($overrides) {
3363
3364 # A negative $overrides implies that the SPECIALS should be ignored,
d11155ec 3365 # and a simple 'a' list is the value.
62b3b855 3366 if ($overrides < 0) {
d11155ec 3367 $format = 'a';
62b3b855
KW
3368 }
3369 else {
3370
3371 # Currently, all overrides are for properties that normally map to
3372 # single code points, but now some will map to lists of code
3373 # points (but there is an exception case handled below).
d11155ec 3374 $format = 'al';
62b3b855
KW
3375
3376 # Look through the overrides.
3377 foreach my $cp_maybe_utf8 (keys %$overrides) {
3378 my $cp;
3379 my @map;
3380
3381 # If the overrides came from SPECIALS, the code point keys are
3382 # packed UTF-8.
3383 if ($overrides == $swash->{'SPECIALS'}) {
3384 $cp = unpack("C0U", $cp_maybe_utf8);
3385 @map = unpack "U0U*", $swash->{'SPECIALS'}{$cp_maybe_utf8};
3386
3387 # The empty string will show up unpacked as an empty
3388 # array.
d11155ec 3389 $format = 'ale' if @map == 0;
62b3b855
KW
3390 }
3391 else {
3392
3393 # But if we generated the overrides, we didn't bother to
3394 # pack them, and we, so far, do this only for properties
d11155ec 3395 # that are 'a' ones.
62b3b855
KW
3396 $cp = $cp_maybe_utf8;
3397 @map = hex $overrides->{$cp};
d11155ec 3398 $format = 'a';
62b3b855
KW
3399 }
3400
3401 # Find the range that the override applies to.
3402 my $i = _search_invlist(\@invlist, $cp);
3403 if ($cp < $invlist[$i] || $cp >= $invlist[$i + 1]) {
294705a8 3404 croak __PACKAGE__, "::prop_invmap: wrong_range, cp=$cp; i=$i, current=$invlist[$i]; next=$invlist[$i + 1]"
62b3b855
KW
3405 }
3406
3407 # And what that range currently maps to
3408 my $cur_map = $invmap[$i];
3409
3410 # If there is a gap between the next range and the code point
3411 # we are overriding, we have to add elements to both arrays to
3412 # fill that gap, using the map that applies to it, which is
3413 # $cur_map, since it is part of the current range.
3414 if ($invlist[$i + 1] > $cp + 1) {
3415 #use feature 'say';
3416 #say "Before splice:";
3417 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3418 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3419 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3420 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3421 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3422
3423 splice @invlist, $i + 1, 0, $cp + 1;
3424 splice @invmap, $i + 1, 0, $cur_map;
3425
3426 #say "After splice:";
3427 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3428 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3429 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3430 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3431 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3432 }
3433
3434 # If the remaining portion of the range is multiple code
3435 # points (ending with the one we are replacing, guaranteed by
3436 # the earlier splice). We must split it into two
3437 if ($invlist[$i] < $cp) {
3438 $i++; # Compensate for the new element
3439
3440 #use feature 'say';
3441 #say "Before splice:";
3442 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3443 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3444 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3445 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3446 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3447
3448 splice @invlist, $i, 0, $cp;
3449 splice @invmap, $i, 0, 'dummy';
3450
3451 #say "After splice:";
3452 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3453 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3454 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3455 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3456 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3457 }
3458
3459 # Here, the range we are overriding contains a single code
3460 # point. The result could be the empty string, a single
3461 # value, or a list. If the last case, we use an anonymous
3462 # array.
3463 $invmap[$i] = (scalar @map == 0)
3464 ? ""
3465 : (scalar @map > 1)
3466 ? \@map
3467 : $map[0];
3468 }
3469 }
3470 }
3471 elsif ($format eq 'x') {
3472
647396da
KW
3473 # All hex-valued properties are really to code points, and have been
3474 # converted to decimal.
5bbfa552 3475 $format = 's';
62b3b855 3476 }
d11155ec
KW
3477 elsif ($returned_prop eq 'ToDm') {
3478 $format = 'ad';
62b3b855
KW
3479 }
3480 elsif ($format eq 'sw') { # blank-separated elements to form a list.
3481 map { $_ = [ split " ", $_ ] if $_ =~ / / } @invmap;
3482 $format = 'sl';
3483 }
3484 elsif ($returned_prop eq 'ToNameAlias') {
3485
3486 # This property currently doesn't have any lists, but theoretically
3487 # could
3488 $format = 'sl';
3489 }
b0b13ada 3490 elsif ($returned_prop eq 'ToPerlDecimalDigit') {
d11155ec 3491 $format = 'ae';
b0b13ada 3492 }
4f143a72
KW
3493 elsif ($returned_prop eq 'ToNv') {
3494
3495 # The one property that has this format is stored as a delta, so needs
3496 # to indicate that need to add code point to it.
3497 $format = 'ar';
3498 }
b577d4a6 3499 elsif ($format ne 'n' && $format ne 'a') {
62b3b855
KW
3500
3501 # All others are simple scalars
3502 $format = 's';
3503 }
e35c6019 3504 if ($has_multiples && $format !~ /l/) {
294705a8 3505 croak __PACKAGE__, "::prop_invmap: Wrong format '$format' for prop_invmap('$prop'); should indicate has lists";
e35c6019 3506 }
62b3b855
KW
3507
3508 return (\@invlist, \@invmap, $format, $missing);
3509}
3510
55d7b906 3511=head2 Unicode::UCD::UnicodeVersion
10a6ecd2 3512
a452d459
KW
3513This returns the version of the Unicode Character Database, in other words, the
3514version of the Unicode standard the database implements. The version is a
3515string of numbers delimited by dots (C<'.'>).
10a6ecd2
JH
3516
3517=cut
3518
3519my $UNICODEVERSION;
3520
3521sub UnicodeVersion {
3522 unless (defined $UNICODEVERSION) {
3523 openunicode(\$VERSIONFH, "version");
ce066323 3524 local $/ = "\n";
10a6ecd2
JH
3525 chomp($UNICODEVERSION = <$VERSIONFH>);
3526 close($VERSIONFH);
3527 croak __PACKAGE__, "::VERSION: strange version '$UNICODEVERSION'"
3528 unless $UNICODEVERSION =~ /^\d+(?:\.\d+)+$/;
3529 }
e80c2d9d 3530 $v_unicode_version = pack "C*", split /\./, $UNICODEVERSION;
10a6ecd2
JH
3531 return $UNICODEVERSION;
3532}
3aa957f9 3533
a452d459
KW
3534=head2 B<Blocks versus Scripts>
3535
3536The difference between a block and a script is that scripts are closer
3537to the linguistic notion of a set of code points required to present
3538languages, while block is more of an artifact of the Unicode code point
3539numbering and separation into blocks of (mostly) 256 code points.
3540
3541For example the Latin B<script> is spread over several B<blocks>, such
3542as C<Basic Latin>, C<Latin 1 Supplement>, C<Latin Extended-A>, and
3543C<Latin Extended-B>. On the other hand, the Latin script does not
3544contain all the characters of the C<Basic Latin> block (also known as
3545ASCII): it includes only the letters, and not, for example, the digits
3546or the punctuation.
3547
3548For blocks see L<http://www.unicode.org/Public/UNIDATA/Blocks.txt>
3549
3550For scripts see UTR #24: L<http://www.unicode.org/unicode/reports/tr24/>
3551
3552=head2 B<Matching Scripts and Blocks>
3553
3554Scripts are matched with the regular-expression construct
3555C<\p{...}> (e.g. C<\p{Tibetan}> matches characters of the Tibetan script),
f200dd12 3556while C<\p{Blk=...}> is used for blocks (e.g. C<\p{Blk=Tibetan}> matches
a452d459
KW
3557any of the 256 code points in the Tibetan block).
3558
430fe03d
KW
3559=head2 Old-style versus new-style block names
3560
3561Unicode publishes the names of blocks in two different styles, though the two
3562are equivalent under Unicode's loose matching rules.
3563
3564The original style uses blanks and hyphens in the block names (except for
3565C<No_Block>), like so:
3566
3567 Miscellaneous Mathematical Symbols-B
3568
3569The newer style replaces these with underscores, like this:
3570
3571 Miscellaneous_Mathematical_Symbols_B
3572
3573This newer style is consistent with the values of other Unicode properties.
3574To preserve backward compatibility, all the functions in Unicode::UCD that
3575return block names (except one) return the old-style ones. That one function,
3576L</prop_value_aliases()> can be used to convert from old-style to new-style:
3577
3578 my $new_style = prop_values_aliases("block", $old_style);
3579
3580Perl also has single-form extensions that refer to blocks, C<In_Cyrillic>,
3581meaning C<Block=Cyrillic>. These have always been written in the new style.
3582
3583To convert from new-style to old-style, follow this recipe:
3584
3585 $old_style = charblock((prop_invlist("block=$new_style"))[0]);
3586
3587(which finds the range of code points in the block using C<prop_invlist>,
3588gets the lower end of the range (0th element) and then looks up the old name
3589for its block using C<charblock>).
3590
7620cb10
KW
3591Note that starting in Unicode 6.1, many of the block names have shorter
3592synonyms. These are always given in the new style.
3593
8b731da2
JH
3594=head1 BUGS
3595
3596Does not yet support EBCDIC platforms.
3597
561c79ed
JH
3598=head1 AUTHOR
3599
a18e976f 3600Jarkko Hietaniemi. Now maintained by perl5 porters.
561c79ed
JH
3601
3602=cut
3603
36041;