This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
reg_fold.t: Fix test to work on EBCDIC platform
[perl5.git] / lib / Unicode / UCD.pm
CommitLineData
55d7b906 1package Unicode::UCD;
561c79ed
JH
2
3use strict;
4use warnings;
36c2430c 5no warnings 'surrogate'; # surrogates can be inputs to this
98ef7649 6use charnames ();
94c91ffc 7use Unicode::Normalize qw(getCombinClass NFD);
561c79ed 8
e80c2d9d 9our $VERSION = '0.44';
561c79ed 10
741297c1
JH
11use Storable qw(dclone);
12
561c79ed
JH
13require Exporter;
14
15our @ISA = qw(Exporter);
74f8133e 16
10a6ecd2
JH
17our @EXPORT_OK = qw(charinfo
18 charblock charscript
19 charblocks charscripts
b08cd201 20 charinrange
ea508aee 21 general_categories bidi_types
b08cd201 22 compexcl
a2bd7410 23 casefold casespec
7319f91d
KW
24 namedseq
25 num
7ef25837
KW
26 prop_aliases
27 prop_value_aliases
681d705c 28 prop_invlist
62b3b855 29 prop_invmap
681d705c 30 MAX_CP
7319f91d 31 );
561c79ed
JH
32
33use Carp;
34
35=head1 NAME
36
55d7b906 37Unicode::UCD - Unicode character database
561c79ed
JH
38
39=head1 SYNOPSIS
40
55d7b906 41 use Unicode::UCD 'charinfo';
b08cd201 42 my $charinfo = charinfo($codepoint);
561c79ed 43
956cae9a
KW
44 use Unicode::UCD 'casefold';
45 my $casefold = casefold(0xFB00);
46
5d8e6e41
KW
47 use Unicode::UCD 'casespec';
48 my $casespec = casespec(0xFB00);
49
55d7b906 50 use Unicode::UCD 'charblock';
e882dd67
JH
51 my $charblock = charblock($codepoint);
52
55d7b906 53 use Unicode::UCD 'charscript';
65044554 54 my $charscript = charscript($codepoint);
561c79ed 55
55d7b906 56 use Unicode::UCD 'charblocks';
e145285f
JH
57 my $charblocks = charblocks();
58
55d7b906 59 use Unicode::UCD 'charscripts';
ea508aee 60 my $charscripts = charscripts();
e145285f 61
55d7b906 62 use Unicode::UCD qw(charscript charinrange);
e145285f
JH
63 my $range = charscript($script);
64 print "looks like $script\n" if charinrange($range, $codepoint);
65
ea508aee
JH
66 use Unicode::UCD qw(general_categories bidi_types);
67 my $categories = general_categories();
68 my $types = bidi_types();
69
7ef25837
KW
70 use Unicode::UCD 'prop_aliases';
71 my @space_names = prop_aliases("space");
72
73 use Unicode::UCD 'prop_value_aliases';
74 my @gc_punct_names = prop_value_aliases("Gc", "Punct");
75
681d705c
KW
76 use Unicode::UCD 'prop_invlist';
77 my @puncts = prop_invlist("gc=punctuation");
78
62b3b855
KW
79 use Unicode::UCD 'prop_invmap';
80 my ($list_ref, $map_ref, $format, $missing)
81 = prop_invmap("General Category");
82
55d7b906 83 use Unicode::UCD 'compexcl';
e145285f
JH
84 my $compexcl = compexcl($codepoint);
85
a2bd7410
JH
86 use Unicode::UCD 'namedseq';
87 my $namedseq = namedseq($named_sequence_name);
88
55d7b906 89 my $unicode_version = Unicode::UCD::UnicodeVersion();
e145285f 90
7319f91d 91 my $convert_to_numeric =
62a8c8c2 92 Unicode::UCD::num("\N{RUMI DIGIT ONE}\N{RUMI DIGIT TWO}");
7319f91d 93
561c79ed
JH
94=head1 DESCRIPTION
95
a452d459
KW
96The Unicode::UCD module offers a series of functions that
97provide a simple interface to the Unicode
8b731da2 98Character Database.
561c79ed 99
a452d459
KW
100=head2 code point argument
101
102Some of the functions are called with a I<code point argument>, which is either
103a decimal or a hexadecimal scalar designating a Unicode code point, or C<U+>
104followed by hexadecimals designating a Unicode code point. In other words, if
105you want a code point to be interpreted as a hexadecimal number, you must
106prefix it with either C<0x> or C<U+>, because a string like e.g. C<123> will be
f200dd12
KW
107interpreted as a decimal code point. Note that the largest code point in
108Unicode is U+10FFFF.
c3e5bc54 109
561c79ed
JH
110=cut
111
10a6ecd2 112my $BLOCKSFH;
10a6ecd2 113my $VERSIONFH;
b08cd201
JH
114my $CASEFOLDFH;
115my $CASESPECFH;
a2bd7410 116my $NAMEDSEQFH;
e80c2d9d 117my $v_unicode_version; # v-string.
561c79ed
JH
118
119sub openunicode {
120 my ($rfh, @path) = @_;
121 my $f;
122 unless (defined $$rfh) {
123 for my $d (@INC) {
124 use File::Spec;
55d7b906 125 $f = File::Spec->catfile($d, "unicore", @path);
32c16050 126 last if open($$rfh, $f);
e882dd67 127 undef $f;
561c79ed 128 }
e882dd67
JH
129 croak __PACKAGE__, ": failed to find ",
130 File::Spec->catfile(@path), " in @INC"
131 unless defined $f;
561c79ed
JH
132 }
133 return $f;
134}
135
a452d459 136=head2 B<charinfo()>
561c79ed 137
55d7b906 138 use Unicode::UCD 'charinfo';
561c79ed 139
b08cd201 140 my $charinfo = charinfo(0x41);
561c79ed 141
a452d459
KW
142This returns information about the input L</code point argument>
143as a reference to a hash of fields as defined by the Unicode
144standard. If the L</code point argument> is not assigned in the standard
145(i.e., has the general category C<Cn> meaning C<Unassigned>)
146or is a non-character (meaning it is guaranteed to never be assigned in
147the standard),
a18e976f 148C<undef> is returned.
a452d459
KW
149
150Fields that aren't applicable to the particular code point argument exist in the
151returned hash, and are empty.
152
153The keys in the hash with the meanings of their values are:
154
155=over
156
157=item B<code>
158
159the input L</code point argument> expressed in hexadecimal, with leading zeros
160added if necessary to make it contain at least four hexdigits
161
162=item B<name>
163
164name of I<code>, all IN UPPER CASE.
165Some control-type code points do not have names.
166This field will be empty for C<Surrogate> and C<Private Use> code points,
167and for the others without a name,
168it will contain a description enclosed in angle brackets, like
169C<E<lt>controlE<gt>>.
170
171
172=item B<category>
173
174The short name of the general category of I<code>.
175This will match one of the keys in the hash returned by L</general_categories()>.
176
7ef25837
KW
177The L</prop_value_aliases()> function can be used to get all the synonyms
178of the category name.
179
a452d459
KW
180=item B<combining>
181
182the combining class number for I<code> used in the Canonical Ordering Algorithm.
183For Unicode 5.1, this is described in Section 3.11 C<Canonical Ordering Behavior>
184available at
185L<http://www.unicode.org/versions/Unicode5.1.0/>
186
7ef25837
KW
187The L</prop_value_aliases()> function can be used to get all the synonyms
188of the combining class number.
189
a452d459
KW
190=item B<bidi>
191
192bidirectional type of I<code>.
193This will match one of the keys in the hash returned by L</bidi_types()>.
194
7ef25837
KW
195The L</prop_value_aliases()> function can be used to get all the synonyms
196of the bidi type name.
197
a452d459
KW
198=item B<decomposition>
199
200is empty if I<code> has no decomposition; or is one or more codes
a18e976f 201(separated by spaces) that, taken in order, represent a decomposition for
a452d459
KW
202I<code>. Each has at least four hexdigits.
203The codes may be preceded by a word enclosed in angle brackets then a space,
204like C<E<lt>compatE<gt> >, giving the type of decomposition
205
06bba7d5
KW
206This decomposition may be an intermediate one whose components are also
207decomposable. Use L<Unicode::Normalize> to get the final decomposition.
208
a452d459
KW
209=item B<decimal>
210
211if I<code> is a decimal digit this is its integer numeric value
212
213=item B<digit>
214
89e4a205
KW
215if I<code> represents some other digit-like number, this is its integer
216numeric value
a452d459
KW
217
218=item B<numeric>
219
220if I<code> represents a whole or rational number, this is its numeric value.
221Rational values are expressed as a string like C<1/4>.
222
223=item B<mirrored>
224
225C<Y> or C<N> designating if I<code> is mirrored in bidirectional text
226
227=item B<unicode10>
228
229name of I<code> in the Unicode 1.0 standard if one
230existed for this code point and is different from the current name
231
232=item B<comment>
233
89e4a205 234As of Unicode 6.0, this is always empty.
a452d459
KW
235
236=item B<upper>
237
06bba7d5 238is empty if there is no single code point uppercase mapping for I<code>
4f66642e 239(its uppercase mapping is itself);
a452d459
KW
240otherwise it is that mapping expressed as at least four hexdigits.
241(L</casespec()> should be used in addition to B<charinfo()>
242for case mappings when the calling program can cope with multiple code point
243mappings.)
244
245=item B<lower>
246
06bba7d5 247is empty if there is no single code point lowercase mapping for I<code>
4f66642e 248(its lowercase mapping is itself);
a452d459
KW
249otherwise it is that mapping expressed as at least four hexdigits.
250(L</casespec()> should be used in addition to B<charinfo()>
251for case mappings when the calling program can cope with multiple code point
252mappings.)
253
254=item B<title>
255
06bba7d5 256is empty if there is no single code point titlecase mapping for I<code>
4f66642e 257(its titlecase mapping is itself);
a452d459
KW
258otherwise it is that mapping expressed as at least four hexdigits.
259(L</casespec()> should be used in addition to B<charinfo()>
260for case mappings when the calling program can cope with multiple code point
261mappings.)
262
263=item B<block>
264
a18e976f 265the block I<code> belongs to (used in C<\p{Blk=...}>).
a452d459
KW
266See L</Blocks versus Scripts>.
267
268
269=item B<script>
270
a18e976f 271the script I<code> belongs to.
a452d459
KW
272See L</Blocks versus Scripts>.
273
274=back
32c16050
JH
275
276Note that you cannot do (de)composition and casing based solely on the
a452d459
KW
277I<decomposition>, I<combining>, I<lower>, I<upper>, and I<title> fields;
278you will need also the L</compexcl()>, and L</casespec()> functions.
561c79ed
JH
279
280=cut
281
e10d7780 282# NB: This function is nearly duplicated in charnames.pm
10a6ecd2
JH
283sub _getcode {
284 my $arg = shift;
285
dc0a4417 286 if ($arg =~ /^[1-9]\d*$/) {
10a6ecd2 287 return $arg;
dc0a4417 288 } elsif ($arg =~ /^(?:[Uu]\+|0[xX])?([[:xdigit:]]+)$/) {
10a6ecd2
JH
289 return hex($1);
290 }
291
292 return;
293}
294
05dbc6f8
KW
295# Populated by _num. Converts real number back to input rational
296my %real_to_rational;
297
298# To store the contents of files found on disk.
299my @BIDIS;
300my @CATEGORIES;
301my @DECOMPOSITIONS;
302my @NUMERIC_TYPES;
5c3b35c9
KW
303my %SIMPLE_LOWER;
304my %SIMPLE_TITLE;
305my %SIMPLE_UPPER;
306my %UNICODE_1_NAMES;
72fcb9f0 307my %ISO_COMMENT;
05dbc6f8 308
05dbc6f8 309sub charinfo {
a6fa416b 310
05dbc6f8
KW
311 # This function has traditionally mimicked what is in UnicodeData.txt,
312 # warts and all. This is a re-write that avoids UnicodeData.txt so that
313 # it can be removed to save disk space. Instead, this assembles
314 # information gotten by other methods that get data from various other
315 # files. It uses charnames to get the character name; and various
316 # mktables tables.
324f9e44 317
05dbc6f8 318 use feature 'unicode_strings';
a6fa416b 319
10a6ecd2
JH
320 my $arg = shift;
321 my $code = _getcode($arg);
05dbc6f8
KW
322 croak __PACKAGE__, "::charinfo: unknown code '$arg'" unless defined $code;
323
324 # Non-unicode implies undef.
325 return if $code > 0x10FFFF;
326
327 my %prop;
328 my $char = chr($code);
329
35a865d4 330 @CATEGORIES =_read_table("To/Gc.pl") unless @CATEGORIES;
05dbc6f8
KW
331 $prop{'category'} = _search(\@CATEGORIES, 0, $#CATEGORIES, $code)
332 // $utf8::SwashInfo{'ToGc'}{'missing'};
333
334 return if $prop{'category'} eq 'Cn'; # Unassigned code points are undef
335
336 $prop{'code'} = sprintf "%04X", $code;
337 $prop{'name'} = ($char =~ /\p{Cntrl}/) ? '<control>'
338 : (charnames::viacode($code) // "");
339
340 $prop{'combining'} = getCombinClass($code);
341
35a865d4 342 @BIDIS =_read_table("To/Bc.pl") unless @BIDIS;
05dbc6f8
KW
343 $prop{'bidi'} = _search(\@BIDIS, 0, $#BIDIS, $code)
344 // $utf8::SwashInfo{'ToBc'}{'missing'};
345
346 # For most code points, we can just read in "unicore/Decomposition.pl", as
347 # its contents are exactly what should be output. But that file doesn't
348 # contain the data for the Hangul syllable decompositions, which can be
94c91ffc
KW
349 # algorithmically computed, and NFD() does that, so we call NFD() for
350 # those. We can't use NFD() for everything, as it does a complete
05dbc6f8 351 # recursive decomposition, and what this function has always done is to
94c91ffc
KW
352 # return what's in UnicodeData.txt which doesn't show that recursiveness.
353 # Fortunately, the NFD() of the Hanguls doesn't have any recursion
354 # issues.
355 # Having no decomposition implies an empty field; otherwise, all but
356 # "Canonical" imply a compatible decomposition, and the type is prefixed
357 # to that, as it is in UnicodeData.txt
9abdc62b
KW
358 UnicodeVersion() unless defined $v_unicode_version;
359 if ($v_unicode_version ge v2.0.0 && $char =~ /\p{Block=Hangul_Syllables}/) {
05dbc6f8
KW
360 # The code points of the decomposition are output in standard Unicode
361 # hex format, separated by blanks.
362 $prop{'decomposition'} = join " ", map { sprintf("%04X", $_)}
94c91ffc 363 unpack "U*", NFD($char);
a6fa416b 364 }
05dbc6f8 365 else {
35a865d4 366 @DECOMPOSITIONS = _read_table("Decomposition.pl")
05dbc6f8
KW
367 unless @DECOMPOSITIONS;
368 $prop{'decomposition'} = _search(\@DECOMPOSITIONS, 0, $#DECOMPOSITIONS,
369 $code) // "";
561c79ed 370 }
05dbc6f8
KW
371
372 # Can use num() to get the numeric values, if any.
373 if (! defined (my $value = num($char))) {
374 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = "";
375 }
376 else {
377 if ($char =~ /\d/) {
378 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = $value;
379 }
380 else {
381
382 # For non-decimal-digits, we have to read in the Numeric type
383 # to distinguish them. It is not just a matter of integer vs.
384 # rational, as some whole number values are not considered digits,
385 # e.g., TAMIL NUMBER TEN.
386 $prop{'decimal'} = "";
387
35a865d4 388 @NUMERIC_TYPES =_read_table("To/Nt.pl") unless @NUMERIC_TYPES;
05dbc6f8
KW
389 if ((_search(\@NUMERIC_TYPES, 0, $#NUMERIC_TYPES, $code) // "")
390 eq 'Digit')
391 {
392 $prop{'digit'} = $prop{'numeric'} = $value;
393 }
394 else {
395 $prop{'digit'} = "";
396 $prop{'numeric'} = $real_to_rational{$value} // $value;
397 }
398 }
399 }
400
401 $prop{'mirrored'} = ($char =~ /\p{Bidi_Mirrored}/) ? 'Y' : 'N';
402
35a865d4 403 %UNICODE_1_NAMES =_read_table("To/Na1.pl", "use_hash") unless %UNICODE_1_NAMES;
5c3b35c9 404 $prop{'unicode10'} = $UNICODE_1_NAMES{$code} // "";
05dbc6f8 405
72fcb9f0
KW
406 UnicodeVersion() unless defined $v_unicode_version;
407 if ($v_unicode_version ge v6.0.0) {
408 $prop{'comment'} = "";
409 }
410 else {
411 %ISO_COMMENT = _read_table("To/Isc.pl", "use_hash") unless %ISO_COMMENT;
412 $prop{'comment'} = (defined $ISO_COMMENT{$code})
413 ? $ISO_COMMENT{$code}
414 : "";
415 }
05dbc6f8 416
35a865d4 417 %SIMPLE_UPPER = _read_table("To/Uc.pl", "use_hash") unless %SIMPLE_UPPER;
bf7fe2df 418 $prop{'upper'} = (defined $SIMPLE_UPPER{$code})
d11155ec 419 ? sprintf("%04X", $SIMPLE_UPPER{$code})
bf7fe2df 420 : "";
75e7c50b 421
35a865d4 422 %SIMPLE_LOWER = _read_table("To/Lc.pl", "use_hash") unless %SIMPLE_LOWER;
bf7fe2df 423 $prop{'lower'} = (defined $SIMPLE_LOWER{$code})
d11155ec 424 ? sprintf("%04X", $SIMPLE_LOWER{$code})
bf7fe2df 425 : "";
75e7c50b 426
35a865d4 427 %SIMPLE_TITLE = _read_table("To/Tc.pl", "use_hash") unless %SIMPLE_TITLE;
bf7fe2df 428 $prop{'title'} = (defined $SIMPLE_TITLE{$code})
d11155ec 429 ? sprintf("%04X", $SIMPLE_TITLE{$code})
bf7fe2df 430 : "";
05dbc6f8
KW
431
432 $prop{block} = charblock($code);
433 $prop{script} = charscript($code);
434 return \%prop;
561c79ed
JH
435}
436
e882dd67
JH
437sub _search { # Binary search in a [[lo,hi,prop],[...],...] table.
438 my ($table, $lo, $hi, $code) = @_;
439
440 return if $lo > $hi;
441
442 my $mid = int(($lo+$hi) / 2);
443
444 if ($table->[$mid]->[0] < $code) {
10a6ecd2 445 if ($table->[$mid]->[1] >= $code) {
e882dd67
JH
446 return $table->[$mid]->[2];
447 } else {
448 _search($table, $mid + 1, $hi, $code);
449 }
450 } elsif ($table->[$mid]->[0] > $code) {
451 _search($table, $lo, $mid - 1, $code);
452 } else {
453 return $table->[$mid]->[2];
454 }
455}
456
cb366075 457sub _read_table ($;$) {
3a12600d
KW
458
459 # Returns the contents of the mktables generated table file located at $1
cb366075
KW
460 # in the form of either an array of arrays or a hash, depending on if the
461 # optional second parameter is true (for hash return) or not. In the case
462 # of a hash return, each key is a code point, and its corresponding value
463 # is what the table gives as the code point's corresponding value. In the
464 # case of an array return, each outer array denotes a range with [0] the
465 # start point of that range; [1] the end point; and [2] the value that
466 # every code point in the range has. The hash return is useful for fast
467 # lookup when the table contains only single code point ranges. The array
468 # return takes much less memory when there are large ranges.
3a12600d 469 #
cb366075 470 # This function has the side effect of setting
3a12600d
KW
471 # $utf8::SwashInfo{$property}{'format'} to be the mktables format of the
472 # table; and
473 # $utf8::SwashInfo{$property}{'missing'} to be the value for all entries
474 # not listed in the table.
475 # where $property is the Unicode property name, preceded by 'To' for map
476 # properties., e.g., 'ToSc'.
477 #
478 # Table entries look like one of:
479 # 0000 0040 Common # [65]
480 # 00AA Latin
481
482 my $table = shift;
cb366075
KW
483 my $return_hash = shift;
484 $return_hash = 0 unless defined $return_hash;
3a12600d 485 my @return;
cb366075 486 my %return;
3a12600d 487 local $_;
d11155ec 488 my $list = do "unicore/$table";
3a12600d 489
d11155ec
KW
490 # Look up if this property requires adjustments, which we do below if it
491 # does.
492 require "unicore/Heavy.pl";
493 my $property = $table =~ s/\.pl//r;
494 $property = $utf8::file_to_swash_name{$property};
495 my $to_adjust = defined $property
496 && $utf8::SwashInfo{$property}{'format'} eq 'a';
497
498 for (split /^/m, $list) {
3a12600d
KW
499 my ($start, $end, $value) = / ^ (.+?) \t (.*?) \t (.+?)
500 \s* ( \# .* )? # Optional comment
501 $ /x;
83fd1222
KW
502 my $decimal_start = hex $start;
503 my $decimal_end = ($end eq "") ? $decimal_start : hex $end;
cb366075 504 if ($return_hash) {
83fd1222 505 foreach my $i ($decimal_start .. $decimal_end) {
d11155ec
KW
506 $return{$i} = ($to_adjust)
507 ? $value + $i - $decimal_start
508 : $value;
cb366075
KW
509 }
510 }
d11155ec
KW
511 elsif (! $to_adjust
512 && @return
513 && $return[-1][1] == $decimal_start - 1
9a96c106
KW
514 && $return[-1][2] eq $value)
515 {
516 # If this is merely extending the previous range, do just that.
517 $return[-1]->[1] = $decimal_end;
518 }
cb366075 519 else {
83fd1222 520 push @return, [ $decimal_start, $decimal_end, $value ];
cb366075 521 }
3a12600d 522 }
cb366075 523 return ($return_hash) ? %return : @return;
3a12600d
KW
524}
525
10a6ecd2
JH
526sub charinrange {
527 my ($range, $arg) = @_;
528 my $code = _getcode($arg);
529 croak __PACKAGE__, "::charinrange: unknown code '$arg'"
530 unless defined $code;
531 _search($range, 0, $#$range, $code);
532}
533
a452d459 534=head2 B<charblock()>
561c79ed 535
55d7b906 536 use Unicode::UCD 'charblock';
561c79ed
JH
537
538 my $charblock = charblock(0x41);
10a6ecd2 539 my $charblock = charblock(1234);
a452d459 540 my $charblock = charblock(0x263a);
10a6ecd2
JH
541 my $charblock = charblock("U+263a");
542
78bf21c2 543 my $range = charblock('Armenian');
10a6ecd2 544
a452d459 545With a L</code point argument> charblock() returns the I<block> the code point
430fe03d
KW
546belongs to, e.g. C<Basic Latin>. The old-style block name is returned (see
547L</Old-style versus new-style block names>).
a452d459 548If the code point is unassigned, this returns the block it would belong to if
fe252ba7
KW
549it were assigned. (If the Unicode version being used is so early as to not
550have blocks, all code points are considered to be in C<No_Block>.)
10a6ecd2 551
78bf21c2
JH
552See also L</Blocks versus Scripts>.
553
18972f4b 554If supplied with an argument that can't be a code point, charblock() tries to
430fe03d
KW
555do the opposite and interpret the argument as an old-style block name. The
556return value
a18e976f
KW
557is a I<range set> with one range: an anonymous list with a single element that
558consists of another anonymous list whose first element is the first code point
559in the block, and whose second (and final) element is the final code point in
560the block. (The extra list consisting of just one element is so that the same
561program logic can be used to handle both this return, and the return from
562L</charscript()> which can have multiple ranges.) You can test whether a code
563point is in a range using the L</charinrange()> function. If the argument is
564not a known block, C<undef> is returned.
561c79ed 565
561c79ed
JH
566=cut
567
568my @BLOCKS;
10a6ecd2 569my %BLOCKS;
561c79ed 570
10a6ecd2 571sub _charblocks {
06bba7d5
KW
572
573 # Can't read from the mktables table because it loses the hyphens in the
574 # original.
561c79ed 575 unless (@BLOCKS) {
fe252ba7
KW
576 UnicodeVersion() unless defined $v_unicode_version;
577 if ($v_unicode_version lt v2.0.0) {
578 my $subrange = [ 0, 0x10FFFF, 'No_Block' ];
579 push @BLOCKS, $subrange;
580 push @{$BLOCKS{$3}}, $subrange;
581 }
582 elsif (openunicode(\$BLOCKSFH, "Blocks.txt")) {
6c8d78fb 583 local $_;
ce066323 584 local $/ = "\n";
10a6ecd2 585 while (<$BLOCKSFH>) {
2796c109 586 if (/^([0-9A-F]+)\.\.([0-9A-F]+);\s+(.+)/) {
10a6ecd2
JH
587 my ($lo, $hi) = (hex($1), hex($2));
588 my $subrange = [ $lo, $hi, $3 ];
589 push @BLOCKS, $subrange;
590 push @{$BLOCKS{$3}}, $subrange;
561c79ed
JH
591 }
592 }
10a6ecd2 593 close($BLOCKSFH);
561c79ed
JH
594 }
595 }
10a6ecd2
JH
596}
597
598sub charblock {
599 my $arg = shift;
600
601 _charblocks() unless @BLOCKS;
602
603 my $code = _getcode($arg);
561c79ed 604
10a6ecd2 605 if (defined $code) {
c707cf8e
KW
606 my $result = _search(\@BLOCKS, 0, $#BLOCKS, $code);
607 return $result if defined $result;
608 return 'No_Block';
609 }
610 elsif (exists $BLOCKS{$arg}) {
611 return dclone $BLOCKS{$arg};
10a6ecd2 612 }
e882dd67
JH
613}
614
a452d459 615=head2 B<charscript()>
e882dd67 616
55d7b906 617 use Unicode::UCD 'charscript';
e882dd67
JH
618
619 my $charscript = charscript(0x41);
10a6ecd2
JH
620 my $charscript = charscript(1234);
621 my $charscript = charscript("U+263a");
e882dd67 622
78bf21c2 623 my $range = charscript('Thai');
10a6ecd2 624
a452d459
KW
625With a L</code point argument> charscript() returns the I<script> the
626code point belongs to, e.g. C<Latin>, C<Greek>, C<Han>.
49ea58c8
KW
627If the code point is unassigned or the Unicode version being used is so early
628that it doesn't have scripts, this function returns C<"Unknown">.
78bf21c2 629
eb0cc9e3 630If supplied with an argument that can't be a code point, charscript() tries
a18e976f
KW
631to do the opposite and interpret the argument as a script name. The
632return value is a I<range set>: an anonymous list of lists that contain
eb0cc9e3 633I<start-of-range>, I<end-of-range> code point pairs. You can test whether a
a18e976f
KW
634code point is in a range set using the L</charinrange()> function. If the
635argument is not a known script, C<undef> is returned.
a452d459
KW
636
637See also L</Blocks versus Scripts>.
e882dd67 638
e882dd67
JH
639=cut
640
641my @SCRIPTS;
10a6ecd2 642my %SCRIPTS;
e882dd67 643
10a6ecd2 644sub _charscripts {
49ea58c8
KW
645 unless (@SCRIPTS) {
646 UnicodeVersion() unless defined $v_unicode_version;
647 if ($v_unicode_version lt v3.1.0) {
648 push @SCRIPTS, [ 0, 0x10FFFF, 'Unknown' ];
649 }
650 else {
651 @SCRIPTS =_read_table("To/Sc.pl");
652 }
653 }
7bccef0b 654 foreach my $entry (@SCRIPTS) {
f3d50ac9 655 $entry->[2] =~ s/(_\w)/\L$1/g; # Preserve old-style casing
7bccef0b 656 push @{$SCRIPTS{$entry->[2]}}, $entry;
e882dd67 657 }
10a6ecd2
JH
658}
659
660sub charscript {
661 my $arg = shift;
662
663 _charscripts() unless @SCRIPTS;
e882dd67 664
10a6ecd2
JH
665 my $code = _getcode($arg);
666
667 if (defined $code) {
7bccef0b
KW
668 my $result = _search(\@SCRIPTS, 0, $#SCRIPTS, $code);
669 return $result if defined $result;
8079ad82 670 return $utf8::SwashInfo{'ToSc'}{'missing'};
7bccef0b
KW
671 } elsif (exists $SCRIPTS{$arg}) {
672 return dclone $SCRIPTS{$arg};
10a6ecd2 673 }
7bccef0b
KW
674
675 return;
10a6ecd2
JH
676}
677
a452d459 678=head2 B<charblocks()>
10a6ecd2 679
55d7b906 680 use Unicode::UCD 'charblocks';
10a6ecd2 681
b08cd201 682 my $charblocks = charblocks();
10a6ecd2 683
b08cd201 684charblocks() returns a reference to a hash with the known block names
a452d459 685as the keys, and the code point ranges (see L</charblock()>) as the values.
10a6ecd2 686
430fe03d
KW
687The names are in the old-style (see L</Old-style versus new-style block
688names>).
689
62b3b855
KW
690L<prop_invmap("block")|/prop_invmap()> can be used to get this same data in a
691different type of data structure.
692
78bf21c2
JH
693See also L</Blocks versus Scripts>.
694
10a6ecd2
JH
695=cut
696
697sub charblocks {
b08cd201 698 _charblocks() unless %BLOCKS;
741297c1 699 return dclone \%BLOCKS;
10a6ecd2
JH
700}
701
a452d459 702=head2 B<charscripts()>
10a6ecd2 703
55d7b906 704 use Unicode::UCD 'charscripts';
10a6ecd2 705
ea508aee 706 my $charscripts = charscripts();
10a6ecd2 707
ea508aee 708charscripts() returns a reference to a hash with the known script
a452d459 709names as the keys, and the code point ranges (see L</charscript()>) as
ea508aee 710the values.
10a6ecd2 711
62b3b855
KW
712L<prop_invmap("script")|/prop_invmap()> can be used to get this same data in a
713different type of data structure.
714
78bf21c2
JH
715See also L</Blocks versus Scripts>.
716
10a6ecd2
JH
717=cut
718
719sub charscripts {
b08cd201 720 _charscripts() unless %SCRIPTS;
741297c1 721 return dclone \%SCRIPTS;
561c79ed
JH
722}
723
a452d459 724=head2 B<charinrange()>
10a6ecd2 725
f200dd12 726In addition to using the C<\p{Blk=...}> and C<\P{Blk=...}> constructs, you
10a6ecd2 727can also test whether a code point is in the I<range> as returned by
a452d459
KW
728L</charblock()> and L</charscript()> or as the values of the hash returned
729by L</charblocks()> and L</charscripts()> by using charinrange():
10a6ecd2 730
55d7b906 731 use Unicode::UCD qw(charscript charinrange);
10a6ecd2
JH
732
733 $range = charscript('Hiragana');
e145285f 734 print "looks like hiragana\n" if charinrange($range, $codepoint);
10a6ecd2
JH
735
736=cut
737
ea508aee
JH
738my %GENERAL_CATEGORIES =
739 (
740 'L' => 'Letter',
741 'LC' => 'CasedLetter',
742 'Lu' => 'UppercaseLetter',
743 'Ll' => 'LowercaseLetter',
744 'Lt' => 'TitlecaseLetter',
745 'Lm' => 'ModifierLetter',
746 'Lo' => 'OtherLetter',
747 'M' => 'Mark',
748 'Mn' => 'NonspacingMark',
749 'Mc' => 'SpacingMark',
750 'Me' => 'EnclosingMark',
751 'N' => 'Number',
752 'Nd' => 'DecimalNumber',
753 'Nl' => 'LetterNumber',
754 'No' => 'OtherNumber',
755 'P' => 'Punctuation',
756 'Pc' => 'ConnectorPunctuation',
757 'Pd' => 'DashPunctuation',
758 'Ps' => 'OpenPunctuation',
759 'Pe' => 'ClosePunctuation',
760 'Pi' => 'InitialPunctuation',
761 'Pf' => 'FinalPunctuation',
762 'Po' => 'OtherPunctuation',
763 'S' => 'Symbol',
764 'Sm' => 'MathSymbol',
765 'Sc' => 'CurrencySymbol',
766 'Sk' => 'ModifierSymbol',
767 'So' => 'OtherSymbol',
768 'Z' => 'Separator',
769 'Zs' => 'SpaceSeparator',
770 'Zl' => 'LineSeparator',
771 'Zp' => 'ParagraphSeparator',
772 'C' => 'Other',
773 'Cc' => 'Control',
774 'Cf' => 'Format',
775 'Cs' => 'Surrogate',
776 'Co' => 'PrivateUse',
777 'Cn' => 'Unassigned',
778 );
779
780sub general_categories {
781 return dclone \%GENERAL_CATEGORIES;
782}
783
a452d459 784=head2 B<general_categories()>
ea508aee
JH
785
786 use Unicode::UCD 'general_categories';
787
788 my $categories = general_categories();
789
a452d459 790This returns a reference to a hash which has short
ea508aee
JH
791general category names (such as C<Lu>, C<Nd>, C<Zs>, C<S>) as keys and long
792names (such as C<UppercaseLetter>, C<DecimalNumber>, C<SpaceSeparator>,
793C<Symbol>) as values. The hash is reversible in case you need to go
794from the long names to the short names. The general category is the
a452d459
KW
795one returned from
796L</charinfo()> under the C<category> key.
ea508aee 797
7ef25837
KW
798The L</prop_value_aliases()> function can be used to get all the synonyms of
799the category name.
800
ea508aee
JH
801=cut
802
803my %BIDI_TYPES =
804 (
805 'L' => 'Left-to-Right',
806 'LRE' => 'Left-to-Right Embedding',
807 'LRO' => 'Left-to-Right Override',
808 'R' => 'Right-to-Left',
809 'AL' => 'Right-to-Left Arabic',
810 'RLE' => 'Right-to-Left Embedding',
811 'RLO' => 'Right-to-Left Override',
812 'PDF' => 'Pop Directional Format',
813 'EN' => 'European Number',
814 'ES' => 'European Number Separator',
815 'ET' => 'European Number Terminator',
816 'AN' => 'Arabic Number',
817 'CS' => 'Common Number Separator',
818 'NSM' => 'Non-Spacing Mark',
819 'BN' => 'Boundary Neutral',
820 'B' => 'Paragraph Separator',
821 'S' => 'Segment Separator',
822 'WS' => 'Whitespace',
823 'ON' => 'Other Neutrals',
824 );
825
a452d459 826=head2 B<bidi_types()>
ea508aee
JH
827
828 use Unicode::UCD 'bidi_types';
829
830 my $categories = bidi_types();
831
a452d459 832This returns a reference to a hash which has the short
ea508aee
JH
833bidi (bidirectional) type names (such as C<L>, C<R>) as keys and long
834names (such as C<Left-to-Right>, C<Right-to-Left>) as values. The
835hash is reversible in case you need to go from the long names to the
a452d459
KW
836short names. The bidi type is the one returned from
837L</charinfo()>
ea508aee
JH
838under the C<bidi> key. For the exact meaning of the various bidi classes
839the Unicode TR9 is recommended reading:
a452d459 840L<http://www.unicode.org/reports/tr9/>
ea508aee
JH
841(as of Unicode 5.0.0)
842
7ef25837
KW
843The L</prop_value_aliases()> function can be used to get all the synonyms of
844the bidi type name.
845
ea508aee
JH
846=cut
847
a452d459
KW
848sub bidi_types {
849 return dclone \%BIDI_TYPES;
850}
851
852=head2 B<compexcl()>
b08cd201 853
55d7b906 854 use Unicode::UCD 'compexcl';
b08cd201 855
a452d459 856 my $compexcl = compexcl(0x09dc);
b08cd201 857
2afba6a4
KW
858This routine returns C<undef> if the Unicode version being used is so early
859that it doesn't have this property. It is included for backwards
860compatibility, but as of Perl 5.12 and more modern Unicode versions, for
71a442a8
KW
861most purposes it is probably more convenient to use one of the following
862instead:
863
864 my $compexcl = chr(0x09dc) =~ /\p{Comp_Ex};
865 my $compexcl = chr(0x09dc) =~ /\p{Full_Composition_Exclusion};
866
867or even
868
869 my $compexcl = chr(0x09dc) =~ /\p{CE};
870 my $compexcl = chr(0x09dc) =~ /\p{Composition_Exclusion};
871
872The first two forms return B<true> if the L</code point argument> should not
76b05678
KW
873be produced by composition normalization. For the final two forms to return
874B<true>, it is additionally required that this fact not otherwise be
875determinable from the Unicode data base.
71a442a8
KW
876
877This routine behaves identically to the final two forms. That is,
878it does not return B<true> if the code point has a decomposition
a452d459
KW
879consisting of another single code point, nor if its decomposition starts
880with a code point whose combining class is non-zero. Code points that meet
881either of these conditions should also not be produced by composition
71a442a8
KW
882normalization, which is probably why you should use the
883C<Full_Composition_Exclusion> property instead, as shown above.
b08cd201 884
71a442a8 885The routine returns B<false> otherwise.
b08cd201
JH
886
887=cut
888
b08cd201
JH
889sub compexcl {
890 my $arg = shift;
891 my $code = _getcode($arg);
74f8133e
JH
892 croak __PACKAGE__, "::compexcl: unknown code '$arg'"
893 unless defined $code;
b08cd201 894
2afba6a4
KW
895 UnicodeVersion() unless defined $v_unicode_version;
896 return if $v_unicode_version lt v3.0.0;
897
36c2430c 898 no warnings "non_unicode"; # So works on non-Unicode code points
71a442a8 899 return chr($code) =~ /\p{Composition_Exclusion}/;
b08cd201
JH
900}
901
a452d459 902=head2 B<casefold()>
b08cd201 903
55d7b906 904 use Unicode::UCD 'casefold';
b08cd201 905
a452d459
KW
906 my $casefold = casefold(0xDF);
907 if (defined $casefold) {
908 my @full_fold_hex = split / /, $casefold->{'full'};
909 my $full_fold_string =
910 join "", map {chr(hex($_))} @full_fold_hex;
911 my @turkic_fold_hex =
912 split / /, ($casefold->{'turkic'} ne "")
913 ? $casefold->{'turkic'}
914 : $casefold->{'full'};
915 my $turkic_fold_string =
916 join "", map {chr(hex($_))} @turkic_fold_hex;
917 }
918 if (defined $casefold && $casefold->{'simple'} ne "") {
919 my $simple_fold_hex = $casefold->{'simple'};
920 my $simple_fold_string = chr(hex($simple_fold_hex));
921 }
b08cd201 922
a452d459 923This returns the (almost) locale-independent case folding of the
6329003c
KW
924character specified by the L</code point argument>. (Starting in Perl v5.16,
925the core function C<fc()> returns the C<full> mapping (described below)
926faster than this does, and for entire strings.)
b08cd201 927
6329003c 928If there is no case folding for the input code point, C<undef> is returned.
a452d459
KW
929
930If there is a case folding for that code point, a reference to a hash
b08cd201
JH
931with the following fields is returned:
932
a452d459
KW
933=over
934
935=item B<code>
936
937the input L</code point argument> expressed in hexadecimal, with leading zeros
938added if necessary to make it contain at least four hexdigits
939
940=item B<full>
941
a18e976f 942one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
943code points for the case folding for I<code>.
944Each has at least four hexdigits.
945
946=item B<simple>
947
948is empty, or is exactly one code with at least four hexdigits which can be used
949as an alternative case folding when the calling program cannot cope with the
950fold being a sequence of multiple code points. If I<full> is just one code
951point, then I<simple> equals I<full>. If there is no single code point folding
952defined for I<code>, then I<simple> is the empty string. Otherwise, it is an
953inferior, but still better-than-nothing alternative folding to I<full>.
954
955=item B<mapping>
956
957is the same as I<simple> if I<simple> is not empty, and it is the same as I<full>
958otherwise. It can be considered to be the simplest possible folding for
959I<code>. It is defined primarily for backwards compatibility.
960
961=item B<status>
b08cd201 962
a452d459
KW
963is C<C> (for C<common>) if the best possible fold is a single code point
964(I<simple> equals I<full> equals I<mapping>). It is C<S> if there are distinct
965folds, I<simple> and I<full> (I<mapping> equals I<simple>). And it is C<F> if
a18e976f
KW
966there is only a I<full> fold (I<mapping> equals I<full>; I<simple> is empty).
967Note that this
a452d459
KW
968describes the contents of I<mapping>. It is defined primarily for backwards
969compatibility.
b08cd201 970
6329003c 971For Unicode versions between 3.1 and 3.1.1 inclusive, I<status> can also be
a452d459
KW
972C<I> which is the same as C<C> but is a special case for dotted uppercase I and
973dotless lowercase i:
b08cd201 974
a452d459 975=over
b08cd201 976
a18e976f 977=item B<*> If you use this C<I> mapping
a452d459 978
a18e976f 979the result is case-insensitive,
a452d459
KW
980but dotless and dotted I's are not distinguished
981
a18e976f 982=item B<*> If you exclude this C<I> mapping
a452d459 983
a18e976f 984the result is not fully case-insensitive, but
a452d459
KW
985dotless and dotted I's are distinguished
986
987=back
988
989=item B<turkic>
990
991contains any special folding for Turkic languages. For versions of Unicode
992starting with 3.2, this field is empty unless I<code> has a different folding
993in Turkic languages, in which case it is one or more codes (separated by
a18e976f 994spaces) that, taken in order, give the code points for the case folding for
a452d459
KW
995I<code> in those languages.
996Each code has at least four hexdigits.
997Note that this folding does not maintain canonical equivalence without
998additional processing.
999
6329003c
KW
1000For Unicode versions between 3.1 and 3.1.1 inclusive, this field is empty unless
1001there is a
a452d459
KW
1002special folding for Turkic languages, in which case I<status> is C<I>, and
1003I<mapping>, I<full>, I<simple>, and I<turkic> are all equal.
1004
1005=back
1006
1007Programs that want complete generality and the best folding results should use
1008the folding contained in the I<full> field. But note that the fold for some
1009code points will be a sequence of multiple code points.
1010
1011Programs that can't cope with the fold mapping being multiple code points can
1012use the folding contained in the I<simple> field, with the loss of some
1013generality. In Unicode 5.1, about 7% of the defined foldings have no single
1014code point folding.
1015
1016The I<mapping> and I<status> fields are provided for backwards compatibility for
1017existing programs. They contain the same values as in previous versions of
1018this function.
1019
1020Locale is not completely independent. The I<turkic> field contains results to
1021use when the locale is a Turkic language.
b08cd201
JH
1022
1023For more information about case mappings see
a452d459 1024L<http://www.unicode.org/unicode/reports/tr21>
b08cd201
JH
1025
1026=cut
1027
1028my %CASEFOLD;
1029
1030sub _casefold {
727c62ff
KW
1031 unless (%CASEFOLD) { # Populate the hash
1032 my ($full_invlist_ref, $full_invmap_ref, undef, $default)
1033 = prop_invmap('Case_Folding');
1034
1035 # Use the recipe given in the prop_invmap() pod to convert the
1036 # inversion map into the hash.
1037 for my $i (0 .. @$full_invlist_ref - 1 - 1) {
1038 next if $full_invmap_ref->[$i] == $default;
1039 my $adjust = -1;
1040 for my $j ($full_invlist_ref->[$i] .. $full_invlist_ref->[$i+1] -1) {
1041 $adjust++;
1042 if (! ref $full_invmap_ref->[$i]) {
1043
1044 # This is a single character mapping
1045 $CASEFOLD{$j}{'status'} = 'C';
1046 $CASEFOLD{$j}{'simple'}
1047 = $CASEFOLD{$j}{'full'}
1048 = $CASEFOLD{$j}{'mapping'}
1049 = sprintf("%04X", $full_invmap_ref->[$i] + $adjust);
1050 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1051 $CASEFOLD{$j}{'turkic'} = "";
1052 }
1053 else { # prop_invmap ensures that $adjust is 0 for a ref
1054 $CASEFOLD{$j}{'status'} = 'F';
1055 $CASEFOLD{$j}{'full'}
1056 = $CASEFOLD{$j}{'mapping'}
1057 = join " ", map { sprintf "%04X", $_ }
1058 @{$full_invmap_ref->[$i]};
1059 $CASEFOLD{$j}{'simple'} = "";
1060 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1061 $CASEFOLD{$j}{'turkic'} = "";
1062 }
1063 }
1064 }
1065
1066 # We have filled in the full mappings above, assuming there were no
1067 # simple ones for the ones with multi-character maps. Now, we find
1068 # and fix the cases where that assumption was false.
1069 (my ($simple_invlist_ref, $simple_invmap_ref, undef), $default)
1070 = prop_invmap('Simple_Case_Folding');
1071 for my $i (0 .. @$simple_invlist_ref - 1 - 1) {
1072 next if $simple_invmap_ref->[$i] == $default;
1073 my $adjust = -1;
1074 for my $j ($simple_invlist_ref->[$i]
1075 .. $simple_invlist_ref->[$i+1] -1)
1076 {
1077 $adjust++;
1078 next if $CASEFOLD{$j}{'status'} eq 'C';
1079 $CASEFOLD{$j}{'status'} = 'S';
1080 $CASEFOLD{$j}{'simple'}
1081 = $CASEFOLD{$j}{'mapping'}
1082 = sprintf("%04X", $simple_invmap_ref->[$i] + $adjust);
1083 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1084 $CASEFOLD{$j}{'turkic'} = "";
1085 }
1086 }
1087
1088 # We hard-code in the turkish rules
1089 UnicodeVersion() unless defined $v_unicode_version;
1090 if ($v_unicode_version ge v3.2.0) {
1091
1092 # These two code points should already have regular entries, so
1093 # just fill in the turkish fields
1094 $CASEFOLD{ord('I')}{'turkic'} = '0131';
1095 $CASEFOLD{0x130}{'turkic'} = sprintf "%04X", ord('i');
1096 }
1097 elsif ($v_unicode_version ge v3.1.0) {
1098
1099 # These two code points don't have entries otherwise.
1100 $CASEFOLD{0x130}{'code'} = '0130';
1101 $CASEFOLD{0x131}{'code'} = '0131';
1102 $CASEFOLD{0x130}{'status'} = $CASEFOLD{0x131}{'status'} = 'I';
1103 $CASEFOLD{0x130}{'turkic'}
1104 = $CASEFOLD{0x130}{'mapping'}
1105 = $CASEFOLD{0x130}{'full'}
1106 = $CASEFOLD{0x130}{'simple'}
1107 = $CASEFOLD{0x131}{'turkic'}
1108 = $CASEFOLD{0x131}{'mapping'}
1109 = $CASEFOLD{0x131}{'full'}
1110 = $CASEFOLD{0x131}{'simple'}
1111 = sprintf "%04X", ord('i');
1112 }
b08cd201
JH
1113 }
1114}
1115
1116sub casefold {
1117 my $arg = shift;
1118 my $code = _getcode($arg);
74f8133e
JH
1119 croak __PACKAGE__, "::casefold: unknown code '$arg'"
1120 unless defined $code;
b08cd201
JH
1121
1122 _casefold() unless %CASEFOLD;
1123
1124 return $CASEFOLD{$code};
1125}
1126
a452d459 1127=head2 B<casespec()>
b08cd201 1128
55d7b906 1129 use Unicode::UCD 'casespec';
b08cd201 1130
a452d459 1131 my $casespec = casespec(0xFB00);
b08cd201 1132
a452d459
KW
1133This returns the potentially locale-dependent case mappings of the L</code point
1134argument>. The mappings may be longer than a single code point (which the basic
1135Unicode case mappings as returned by L</charinfo()> never are).
b08cd201 1136
a452d459
KW
1137If there are no case mappings for the L</code point argument>, or if all three
1138possible mappings (I<lower>, I<title> and I<upper>) result in single code
a18e976f 1139points and are locale independent and unconditional, C<undef> is returned
5d8e6e41
KW
1140(which means that the case mappings, if any, for the code point are those
1141returned by L</charinfo()>).
a452d459
KW
1142
1143Otherwise, a reference to a hash giving the mappings (or a reference to a hash
5d8e6e41
KW
1144of such hashes, explained below) is returned with the following keys and their
1145meanings:
a452d459
KW
1146
1147The keys in the bottom layer hash with the meanings of their values are:
1148
1149=over
1150
1151=item B<code>
1152
1153the input L</code point argument> expressed in hexadecimal, with leading zeros
1154added if necessary to make it contain at least four hexdigits
1155
1156=item B<lower>
1157
a18e976f 1158one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1159code points for the lower case of I<code>.
1160Each has at least four hexdigits.
1161
1162=item B<title>
b08cd201 1163
a18e976f 1164one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1165code points for the title case of I<code>.
1166Each has at least four hexdigits.
b08cd201 1167
d2da20e3 1168=item B<upper>
b08cd201 1169
a18e976f 1170one or more codes (separated by spaces) that, taken in order, give the
a452d459
KW
1171code points for the upper case of I<code>.
1172Each has at least four hexdigits.
1173
1174=item B<condition>
1175
1176the conditions for the mappings to be valid.
a18e976f 1177If C<undef>, the mappings are always valid.
a452d459
KW
1178When defined, this field is a list of conditions,
1179all of which must be true for the mappings to be valid.
1180The list consists of one or more
1181I<locales> (see below)
1182and/or I<contexts> (explained in the next paragraph),
1183separated by spaces.
1184(Other than as used to separate elements, spaces are to be ignored.)
1185Case distinctions in the condition list are not significant.
82c0b05b 1186Conditions preceded by "NON_" represent the negation of the condition.
b08cd201 1187
a452d459
KW
1188A I<context> is one of those defined in the Unicode standard.
1189For Unicode 5.1, they are defined in Section 3.13 C<Default Case Operations>
1190available at
5d8e6e41
KW
1191L<http://www.unicode.org/versions/Unicode5.1.0/>.
1192These are for context-sensitive casing.
f499c386 1193
a452d459
KW
1194=back
1195
5d8e6e41 1196The hash described above is returned for locale-independent casing, where
a18e976f 1197at least one of the mappings has length longer than one. If C<undef> is
5d8e6e41
KW
1198returned, the code point may have mappings, but if so, all are length one,
1199and are returned by L</charinfo()>.
1200Note that when this function does return a value, it will be for the complete
1201set of mappings for a code point, even those whose length is one.
1202
1203If there are additional casing rules that apply only in certain locales,
1204an additional key for each will be defined in the returned hash. Each such key
1205will be its locale name, defined as a 2-letter ISO 3166 country code, possibly
1206followed by a "_" and a 2-letter ISO language code (possibly followed by a "_"
1207and a variant code). You can find the lists of all possible locales, see
1208L<Locale::Country> and L<Locale::Language>.
89e4a205 1209(In Unicode 6.0, the only locales returned by this function
a452d459 1210are C<lt>, C<tr>, and C<az>.)
b08cd201 1211
5d8e6e41
KW
1212Each locale key is a reference to a hash that has the form above, and gives
1213the casing rules for that particular locale, which take precedence over the
1214locale-independent ones when in that locale.
1215
1216If the only casing for a code point is locale-dependent, then the returned
1217hash will not have any of the base keys, like C<code>, C<upper>, etc., but
1218will contain only locale keys.
1219
b08cd201 1220For more information about case mappings see
a452d459 1221L<http://www.unicode.org/unicode/reports/tr21/>
b08cd201
JH
1222
1223=cut
1224
1225my %CASESPEC;
1226
1227sub _casespec {
1228 unless (%CASESPEC) {
551b6b6f 1229 if (openunicode(\$CASESPECFH, "SpecialCasing.txt")) {
6c8d78fb 1230 local $_;
ce066323 1231 local $/ = "\n";
b08cd201
JH
1232 while (<$CASESPECFH>) {
1233 if (/^([0-9A-F]+); ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; (\w+(?: \w+)*)?/) {
f499c386
JH
1234 my ($hexcode, $lower, $title, $upper, $condition) =
1235 ($1, $2, $3, $4, $5);
1236 my $code = hex($hexcode);
1237 if (exists $CASESPEC{$code}) {
1238 if (exists $CASESPEC{$code}->{code}) {
1239 my ($oldlower,
1240 $oldtitle,
1241 $oldupper,
1242 $oldcondition) =
1243 @{$CASESPEC{$code}}{qw(lower
1244 title
1245 upper
1246 condition)};
822ebcc8
JH
1247 if (defined $oldcondition) {
1248 my ($oldlocale) =
f499c386 1249 ($oldcondition =~ /^([a-z][a-z](?:_\S+)?)/);
f499c386
JH
1250 delete $CASESPEC{$code};
1251 $CASESPEC{$code}->{$oldlocale} =
1252 { code => $hexcode,
1253 lower => $oldlower,
1254 title => $oldtitle,
1255 upper => $oldupper,
1256 condition => $oldcondition };
f499c386
JH
1257 }
1258 }
1259 my ($locale) =
1260 ($condition =~ /^([a-z][a-z](?:_\S+)?)/);
1261 $CASESPEC{$code}->{$locale} =
1262 { code => $hexcode,
1263 lower => $lower,
1264 title => $title,
1265 upper => $upper,
1266 condition => $condition };
1267 } else {
1268 $CASESPEC{$code} =
1269 { code => $hexcode,
1270 lower => $lower,
1271 title => $title,
1272 upper => $upper,
1273 condition => $condition };
1274 }
b08cd201
JH
1275 }
1276 }
1277 close($CASESPECFH);
1278 }
1279 }
1280}
1281
1282sub casespec {
1283 my $arg = shift;
1284 my $code = _getcode($arg);
74f8133e
JH
1285 croak __PACKAGE__, "::casespec: unknown code '$arg'"
1286 unless defined $code;
b08cd201
JH
1287
1288 _casespec() unless %CASESPEC;
1289
741297c1 1290 return ref $CASESPEC{$code} ? dclone $CASESPEC{$code} : $CASESPEC{$code};
b08cd201
JH
1291}
1292
a452d459 1293=head2 B<namedseq()>
a2bd7410
JH
1294
1295 use Unicode::UCD 'namedseq';
1296
1297 my $namedseq = namedseq("KATAKANA LETTER AINU P");
1298 my @namedseq = namedseq("KATAKANA LETTER AINU P");
1299 my %namedseq = namedseq();
1300
1301If used with a single argument in a scalar context, returns the string
a18e976f 1302consisting of the code points of the named sequence, or C<undef> if no
a2bd7410 1303named sequence by that name exists. If used with a single argument in
956cae9a
KW
1304a list context, it returns the list of the ordinals of the code points. If used
1305with no
a2bd7410
JH
1306arguments in a list context, returns a hash with the names of the
1307named sequences as the keys and the named sequences as strings as
a18e976f 1308the values. Otherwise, it returns C<undef> or an empty list depending
a2bd7410
JH
1309on the context.
1310
a452d459
KW
1311This function only operates on officially approved (not provisional) named
1312sequences.
a2bd7410 1313
27f853a0
KW
1314Note that as of Perl 5.14, C<\N{KATAKANA LETTER AINU P}> will insert the named
1315sequence into double-quoted strings, and C<charnames::string_vianame("KATAKANA
1316LETTER AINU P")> will return the same string this function does, but will also
1317operate on character names that aren't named sequences, without you having to
1318know which are which. See L<charnames>.
1319
a2bd7410
JH
1320=cut
1321
1322my %NAMEDSEQ;
1323
1324sub _namedseq {
1325 unless (%NAMEDSEQ) {
98ef7649 1326 if (openunicode(\$NAMEDSEQFH, "Name.pl")) {
a2bd7410 1327 local $_;
ce066323 1328 local $/ = "\n";
a2bd7410 1329 while (<$NAMEDSEQFH>) {
98ef7649
KW
1330 if (/^ [0-9A-F]+ \ /x) {
1331 chomp;
1332 my ($sequence, $name) = split /\t/;
1333 my @s = map { chr(hex($_)) } split(' ', $sequence);
1334 $NAMEDSEQ{$name} = join("", @s);
a2bd7410
JH
1335 }
1336 }
1337 close($NAMEDSEQFH);
1338 }
1339 }
1340}
1341
1342sub namedseq {
98ef7649
KW
1343
1344 # Use charnames::string_vianame() which now returns this information,
1345 # unless the caller wants the hash returned, in which case we read it in,
1346 # and thereafter use it instead of calling charnames, as it is faster.
1347
a2bd7410
JH
1348 my $wantarray = wantarray();
1349 if (defined $wantarray) {
1350 if ($wantarray) {
1351 if (@_ == 0) {
98ef7649 1352 _namedseq() unless %NAMEDSEQ;
a2bd7410
JH
1353 return %NAMEDSEQ;
1354 } elsif (@_ == 1) {
98ef7649
KW
1355 my $s;
1356 if (%NAMEDSEQ) {
1357 $s = $NAMEDSEQ{ $_[0] };
1358 }
1359 else {
1360 $s = charnames::string_vianame($_[0]);
1361 }
a2bd7410
JH
1362 return defined $s ? map { ord($_) } split('', $s) : ();
1363 }
1364 } elsif (@_ == 1) {
98ef7649
KW
1365 return $NAMEDSEQ{ $_[0] } if %NAMEDSEQ;
1366 return charnames::string_vianame($_[0]);
a2bd7410
JH
1367 }
1368 }
1369 return;
1370}
1371
7319f91d
KW
1372my %NUMERIC;
1373
1374sub _numeric {
35a865d4 1375 my @numbers = _read_table("To/Nv.pl");
98025745
KW
1376 foreach my $entry (@numbers) {
1377 my ($start, $end, $value) = @$entry;
1378
05dbc6f8
KW
1379 # If value contains a slash, convert to decimal, add a reverse hash
1380 # used by charinfo.
98025745
KW
1381 if ((my @rational = split /\//, $value) == 2) {
1382 my $real = $rational[0] / $rational[1];
05dbc6f8 1383 $real_to_rational{$real} = $value;
98025745 1384 $value = $real;
98025745 1385
4f143a72
KW
1386 # Should only be single element, but just in case...
1387 for my $i ($start .. $end) {
1388 $NUMERIC{$i} = $value;
1389 }
1390 }
1391 else {
1392 # The values require adjusting, as is in 'a' format
1393 for my $i ($start .. $end) {
1394 $NUMERIC{$i} = $value + $i - $start;
1395 }
7319f91d 1396 }
7319f91d 1397 }
2dc5eb26
KW
1398
1399 # Decided unsafe to use these that aren't officially part of the Unicode
1400 # standard.
1401 #use Math::Trig;
1402 #my $pi = acos(-1.0);
98025745 1403 #$NUMERIC{0x03C0} = $pi;
7319f91d
KW
1404
1405 # Euler's constant, not to be confused with Euler's number
98025745 1406 #$NUMERIC{0x2107} = 0.57721566490153286060651209008240243104215933593992;
7319f91d
KW
1407
1408 # Euler's number
98025745 1409 #$NUMERIC{0x212F} = 2.7182818284590452353602874713526624977572;
2dc5eb26 1410
7319f91d
KW
1411 return;
1412}
1413
1414=pod
1415
67592e11 1416=head2 B<num()>
7319f91d 1417
eefd7bc2
KW
1418 use Unicode::UCD 'num';
1419
1420 my $val = num("123");
1421 my $one_quarter = num("\N{VULGAR FRACTION 1/4}");
1422
7319f91d
KW
1423C<num> returns the numeric value of the input Unicode string; or C<undef> if it
1424doesn't think the entire string has a completely valid, safe numeric value.
1425
1426If the string is just one character in length, the Unicode numeric value
1427is returned if it has one, or C<undef> otherwise. Note that this need
1428not be a whole number. C<num("\N{TIBETAN DIGIT HALF ZERO}")>, for
2dc5eb26
KW
1429example returns -0.5.
1430
1431=cut
7319f91d 1432
2dc5eb26
KW
1433#A few characters to which Unicode doesn't officially
1434#assign a numeric value are considered numeric by C<num>.
1435#These are:
1436
1437# EULER CONSTANT 0.5772... (this is NOT Euler's number)
1438# SCRIPT SMALL E 2.71828... (this IS Euler's number)
1439# GREEK SMALL LETTER PI 3.14159...
1440
1441=pod
7319f91d
KW
1442
1443If the string is more than one character, C<undef> is returned unless
8bb4c8e2 1444all its characters are decimal digits (that is, they would match C<\d+>),
7319f91d
KW
1445from the same script. For example if you have an ASCII '0' and a Bengali
1446'3', mixed together, they aren't considered a valid number, and C<undef>
1447is returned. A further restriction is that the digits all have to be of
1448the same form. A half-width digit mixed with a full-width one will
1449return C<undef>. The Arabic script has two sets of digits; C<num> will
1450return C<undef> unless all the digits in the string come from the same
1451set.
1452
1453C<num> errs on the side of safety, and there may be valid strings of
1454decimal digits that it doesn't recognize. Note that Unicode defines
1455a number of "digit" characters that aren't "decimal digit" characters.
a278d14b 1456"Decimal digits" have the property that they have a positional value, i.e.,
7319f91d
KW
1457there is a units position, a 10's position, a 100's, etc, AND they are
1458arranged in Unicode in blocks of 10 contiguous code points. The Chinese
1459digits, for example, are not in such a contiguous block, and so Unicode
1460doesn't view them as decimal digits, but merely digits, and so C<\d> will not
1461match them. A single-character string containing one of these digits will
1462have its decimal value returned by C<num>, but any longer string containing
1463only these digits will return C<undef>.
1464
a278d14b
KW
1465Strings of multiple sub- and superscripts are not recognized as numbers. You
1466can use either of the compatibility decompositions in Unicode::Normalize to
7319f91d
KW
1467change these into digits, and then call C<num> on the result.
1468
1469=cut
1470
1471# To handle sub, superscripts, this could if called in list context,
1472# consider those, and return the <decomposition> type in the second
1473# array element.
1474
1475sub num {
1476 my $string = $_[0];
1477
1478 _numeric unless %NUMERIC;
1479
1480 my $length = length($string);
98025745 1481 return $NUMERIC{ord($string)} if $length == 1;
7319f91d
KW
1482 return if $string =~ /\D/;
1483 my $first_ord = ord(substr($string, 0, 1));
98025745 1484 my $value = $NUMERIC{$first_ord};
5522af1c
KW
1485
1486 # To be a valid decimal number, it should be in a block of 10 consecutive
1487 # characters, whose values are 0, 1, 2, ... 9. Therefore this digit's
1488 # value is its offset in that block from the character that means zero.
7319f91d
KW
1489 my $zero_ord = $first_ord - $value;
1490
5522af1c
KW
1491 # Unicode 6.0 instituted the rule that only digits in a consecutive
1492 # block of 10 would be considered decimal digits. If this is an earlier
1493 # release, we verify that this first character is a member of such a
1494 # block. That is, that the block of characters surrounding this one
1495 # consists of all \d characters whose numeric values are the expected
1496 # ones.
1497 UnicodeVersion() unless defined $v_unicode_version;
1498 if ($v_unicode_version lt v6.0.0) {
1499 for my $i (0 .. 9) {
1500 my $ord = $zero_ord + $i;
1501 return unless chr($ord) =~ /\d/;
1502 my $numeric = $NUMERIC{$ord};
1503 return unless defined $numeric;
1504 return unless $numeric == $i;
1505 }
1506 }
1507
7319f91d 1508 for my $i (1 .. $length -1) {
5522af1c
KW
1509
1510 # Here we know either by verifying, or by fact of the first character
1511 # being a \d in Unicode 6.0 or later, that any character between the
1512 # character that means 0, and 9 positions above it must be \d, and
1513 # must have its value correspond to its offset from the zero. Any
1514 # characters outside these 10 do not form a legal number for this
1515 # function.
7319f91d
KW
1516 my $ord = ord(substr($string, $i, 1));
1517 my $digit = $ord - $zero_ord;
1518 return unless $digit >= 0 && $digit <= 9;
1519 $value = $value * 10 + $digit;
1520 }
5522af1c 1521
7319f91d
KW
1522 return $value;
1523}
1524
7ef25837
KW
1525=pod
1526
1527=head2 B<prop_aliases()>
1528
1529 use Unicode::UCD 'prop_aliases';
1530
1531 my ($short_name, $full_name, @other_names) = prop_aliases("space");
1532 my $same_full_name = prop_aliases("Space"); # Scalar context
1533 my ($same_short_name) = prop_aliases("Space"); # gets 0th element
1534 print "The full name is $full_name\n";
1535 print "The short name is $short_name\n";
1536 print "The other aliases are: ", join(", ", @other_names), "\n";
1537
1538 prints:
1539 The full name is White_Space
1540 The short name is WSpace
1541 The other aliases are: Space
1542
1543Most Unicode properties have several synonymous names. Typically, there is at
1544least a short name, convenient to type, and a long name that more fully
1545describes the property, and hence is more easily understood.
1546
1547If you know one name for a Unicode property, you can use C<prop_aliases> to find
1548either the long name (when called in scalar context), or a list of all of the
1549names, somewhat ordered so that the short name is in the 0th element, the long
1550name in the next element, and any other synonyms are in the remaining
1551elements, in no particular order.
1552
1553The long name is returned in a form nicely capitalized, suitable for printing.
1554
1555The input parameter name is loosely matched, which means that white space,
1556hyphens, and underscores are ignored (except for the trailing underscore in
1557the old_form grandfathered-in C<"L_">, which is better written as C<"LC">, and
1558both of which mean C<General_Category=Cased Letter>).
1559
1560If the name is unknown, C<undef> is returned (or an empty list in list
1561context). Note that Perl typically recognizes property names in regular
1562expressions with an optional C<"Is_>" (with or without the underscore)
1563prefixed to them, such as C<\p{isgc=punct}>. This function does not recognize
1564those in the input, returning C<undef>. Nor are they included in the output
1565as possible synonyms.
1566
1567C<prop_aliases> does know about the Perl extensions to Unicode properties,
1568such as C<Any> and C<XPosixAlpha>, and the single form equivalents to Unicode
1569properties such as C<XDigit>, C<Greek>, C<In_Greek>, and C<Is_Greek>. The
1570final example demonstrates that the C<"Is_"> prefix is recognized for these
1571extensions; it is needed to resolve ambiguities. For example,
1572C<prop_aliases('lc')> returns the list C<(lc, Lowercase_Mapping)>, but
1573C<prop_aliases('islc')> returns C<(Is_LC, Cased_Letter)>. This is
1574because C<islc> is a Perl extension which is short for
1575C<General_Category=Cased Letter>. The lists returned for the Perl extensions
1576will not include the C<"Is_"> prefix (whether or not the input had it) unless
1577needed to resolve ambiguities, as shown in the C<"islc"> example, where the
1578returned list had one element containing C<"Is_">, and the other without.
1579
1580It is also possible for the reverse to happen: C<prop_aliases('isc')> returns
1581the list C<(isc, ISO_Comment)>; whereas C<prop_aliases('c')> returns
1582C<(C, Other)> (the latter being a Perl extension meaning
ee94c7d1
KW
1583C<General_Category=Other>.
1584L<perluniprops/Properties accessible through Unicode::UCD> lists the available
1585forms, including which ones are discouraged from use.
7ef25837
KW
1586
1587Those discouraged forms are accepted as input to C<prop_aliases>, but are not
1588returned in the lists. C<prop_aliases('isL&')> and C<prop_aliases('isL_')>,
1589which are old synonyms for C<"Is_LC"> and should not be used in new code, are
1590examples of this. These both return C<(Is_LC, Cased_Letter)>. Thus this
1591function allows you to take a discourarged form, and find its acceptable
1592alternatives. The same goes with single-form Block property equivalences.
1593Only the forms that begin with C<"In_"> are not discouraged; if you pass
1594C<prop_aliases> a discouraged form, you will get back the equivalent ones that
1595begin with C<"In_">. It will otherwise look like a new-style block name (see.
1596L</Old-style versus new-style block names>).
1597
1598C<prop_aliases> does not know about any user-defined properties, and will
1599return C<undef> if called with one of those. Likewise for Perl internal
1600properties, with the exception of "Perl_Decimal_Digit" which it does know
1601about (and which is documented below in L</prop_invmap()>).
1602
1603=cut
1604
1605# It may be that there are use cases where the discouraged forms should be
1606# returned. If that comes up, an optional boolean second parameter to the
1607# function could be created, for example.
1608
1609# These are created by mktables for this routine and stored in unicore/UCD.pl
1610# where their structures are described.
1611our %string_property_loose_to_name;
1612our %ambiguous_names;
1613our %loose_perlprop_to_name;
1614our %prop_aliases;
1615
1616sub prop_aliases ($) {
1617 my $prop = $_[0];
1618 return unless defined $prop;
1619
1620 require "unicore/UCD.pl";
1621 require "unicore/Heavy.pl";
1622 require "utf8_heavy.pl";
1623
1624 # The property name may be loosely or strictly matched; we don't know yet.
1625 # But both types use lower-case.
1626 $prop = lc $prop;
1627
1628 # It is loosely matched if its lower case isn't known to be strict.
1629 my $list_ref;
1630 if (! exists $utf8::stricter_to_file_of{$prop}) {
1631 my $loose = utf8::_loose_name($prop);
1632
1633 # There is a hash that converts from any loose name to its standard
1634 # form, mapping all synonyms for a name to one name that can be used
1635 # as a key into another hash. The whole concept is for memory
1636 # savings, as the second hash doesn't have to have all the
1637 # combinations. Actually, there are two hashes that do the
1638 # converstion. One is used in utf8_heavy.pl (stored in Heavy.pl) for
1639 # looking up properties matchable in regexes. This function needs to
1640 # access string properties, which aren't available in regexes, so a
1641 # second conversion hash is made for them (stored in UCD.pl). Look in
1642 # the string one now, as the rest can have an optional 'is' prefix,
1643 # which these don't.
1644 if (exists $string_property_loose_to_name{$loose}) {
1645
1646 # Convert to its standard loose name.
1647 $prop = $string_property_loose_to_name{$loose};
1648 }
1649 else {
1650 my $retrying = 0; # bool. ? Has an initial 'is' been stripped
1651 RETRY:
1652 if (exists $utf8::loose_property_name_of{$loose}
1653 && (! $retrying
1654 || ! exists $ambiguous_names{$loose}))
1655 {
1656 # Found an entry giving the standard form. We don't get here
1657 # (in the test above) when we've stripped off an
1658 # 'is' and the result is an ambiguous name. That is because
1659 # these are official Unicode properties (though Perl can have
1660 # an optional 'is' prefix meaning the official property), and
1661 # all ambiguous cases involve a Perl single-form extension
1662 # for the gc, script, or block properties, and the stripped
1663 # 'is' means that they mean one of those, and not one of
1664 # these
1665 $prop = $utf8::loose_property_name_of{$loose};
1666 }
1667 elsif (exists $loose_perlprop_to_name{$loose}) {
1668
1669 # This hash is specifically for this function to list Perl
1670 # extensions that aren't in the earlier hashes. If there is
1671 # only one element, the short and long names are identical.
1672 # Otherwise the form is already in the same form as
1673 # %prop_aliases, which is handled at the end of the function.
1674 $list_ref = $loose_perlprop_to_name{$loose};
1675 if (@$list_ref == 1) {
1676 my @list = ($list_ref->[0], $list_ref->[0]);
1677 $list_ref = \@list;
1678 }
1679 }
1680 elsif (! exists $utf8::loose_to_file_of{$loose}) {
1681
1682 # loose_to_file_of is a complete list of loose names. If not
1683 # there, the input is unknown.
1684 return;
1685 }
1686 else {
1687
1688 # Here we found the name but not its aliases, so it has to
1689 # exist. This means it must be one of the Perl single-form
1690 # extensions. First see if it is for a property-value
1691 # combination in one of the following properties.
1692 my @list;
1693 foreach my $property ("gc", "script") {
1694 @list = prop_value_aliases($property, $loose);
1695 last if @list;
1696 }
1697 if (@list) {
1698
1699 # Here, it is one of those property-value combination
1700 # single-form synonyms. There are ambiguities with some
1701 # of these. Check against the list for these, and adjust
1702 # if necessary.
1703 for my $i (0 .. @list -1) {
1704 if (exists $ambiguous_names
1705 {utf8::_loose_name(lc $list[$i])})
1706 {
1707 # The ambiguity is resolved by toggling whether or
1708 # not it has an 'is' prefix
1709 $list[$i] =~ s/^Is_// or $list[$i] =~ s/^/Is_/;
1710 }
1711 }
1712 return @list;
1713 }
1714
1715 # Here, it wasn't one of the gc or script single-form
1716 # extensions. It could be a block property single-form
1717 # extension. An 'in' prefix definitely means that, and should
2a4f2769
KW
1718 # be looked up without the prefix. However, starting in
1719 # Unicode 6.1, we have to special case 'indic...', as there
1720 # is a property that begins with that name. We shouldn't
1721 # strip the 'in' from that. I'm (khw) generalizing this to
1722 # 'indic' instead of the single property, because I suspect
1723 # that others of this class may come along in the future.
1724 # However, this could backfire and a block created whose name
1725 # begins with 'dic...', and we would want to strip the 'in'.
1726 # At which point this would have to be tweaked.
1727 my $began_with_in = $loose =~ s/^in(?!dic)//;
7ef25837
KW
1728 @list = prop_value_aliases("block", $loose);
1729 if (@list) {
1730 map { $_ =~ s/^/In_/ } @list;
1731 return @list;
1732 }
1733
1734 # Here still haven't found it. The last opportunity for it
1735 # being valid is only if it began with 'is'. We retry without
1736 # the 'is', setting a flag to that effect so that we don't
1737 # accept things that begin with 'isis...'
1738 if (! $retrying && ! $began_with_in && $loose =~ s/^is//) {
1739 $retrying = 1;
1740 goto RETRY;
1741 }
1742
1743 # Here, didn't find it. Since it was in %loose_to_file_of, we
1744 # should have been able to find it.
1745 carp __PACKAGE__, "::prop_aliases: Unexpectedly could not find '$prop'. Send bug report to perlbug\@perl.org";
1746 return;
1747 }
1748 }
1749 }
1750
1751 if (! $list_ref) {
1752 # Here, we have set $prop to a standard form name of the input. Look
1753 # it up in the structure created by mktables for this purpose, which
1754 # contains both strict and loosely matched properties. Avoid
1755 # autovivifying.
1756 $list_ref = $prop_aliases{$prop} if exists $prop_aliases{$prop};
1757 return unless $list_ref;
1758 }
1759
1760 # The full name is in element 1.
1761 return $list_ref->[1] unless wantarray;
1762
1763 return @{dclone $list_ref};
1764}
1765
1766=pod
1767
1768=head2 B<prop_value_aliases()>
1769
1770 use Unicode::UCD 'prop_value_aliases';
1771
1772 my ($short_name, $full_name, @other_names)
1773 = prop_value_aliases("Gc", "Punct");
1774 my $same_full_name = prop_value_aliases("Gc", "P"); # Scalar cntxt
1775 my ($same_short_name) = prop_value_aliases("Gc", "P"); # gets 0th
1776 # element
1777 print "The full name is $full_name\n";
1778 print "The short name is $short_name\n";
1779 print "The other aliases are: ", join(", ", @other_names), "\n";
1780
1781 prints:
1782 The full name is Punctuation
1783 The short name is P
1784 The other aliases are: Punct
1785
1786Some Unicode properties have a restricted set of legal values. For example,
1787all binary properties are restricted to just C<true> or C<false>; and there
1788are only a few dozen possible General Categories.
1789
1790For such properties, there are usually several synonyms for each possible
1791value. For example, in binary properties, I<truth> can be represented by any of
1792the strings "Y", "Yes", "T", or "True"; and the General Category
1793"Punctuation" by that string, or "Punct", or simply "P".
1794
1795Like property names, there is typically at least a short name for each such
1796property-value, and a long name. If you know any name of the property-value,
1797you can use C<prop_value_aliases>() to get the long name (when called in
1798scalar context), or a list of all the names, with the short name in the 0th
1799element, the long name in the next element, and any other synonyms in the
1800remaining elements, in no particular order, except that any all-numeric
1801synonyms will be last.
1802
1803The long name is returned in a form nicely capitalized, suitable for printing.
1804
1805Case, white space, hyphens, and underscores are ignored in the input parameters
1806(except for the trailing underscore in the old-form grandfathered-in general
1807category property value C<"L_">, which is better written as C<"LC">).
1808
1809If either name is unknown, C<undef> is returned. Note that Perl typically
1810recognizes property names in regular expressions with an optional C<"Is_>"
1811(with or without the underscore) prefixed to them, such as C<\p{isgc=punct}>.
1812This function does not recognize those in the property parameter, returning
1813C<undef>.
1814
1815If called with a property that doesn't have synonyms for its values, it
1816returns the input value, possibly normalized with capitalization and
1817underscores.
1818
1819For the block property, new-style block names are returned (see
1820L</Old-style versus new-style block names>).
1821
1822To find the synonyms for single-forms, such as C<\p{Any}>, use
1823L</prop_aliases()> instead.
1824
1825C<prop_value_aliases> does not know about any user-defined properties, and
1826will return C<undef> if called with one of those.
1827
1828=cut
1829
1830# These are created by mktables for this routine and stored in unicore/UCD.pl
1831# where their structures are described.
1832our %loose_to_standard_value;
1833our %prop_value_aliases;
1834
1835sub prop_value_aliases ($$) {
1836 my ($prop, $value) = @_;
1837 return unless defined $prop && defined $value;
1838
1839 require "unicore/UCD.pl";
1840 require "utf8_heavy.pl";
1841
1842 # Find the property name synonym that's used as the key in other hashes,
1843 # which is element 0 in the returned list.
1844 ($prop) = prop_aliases($prop);
1845 return if ! $prop;
1846 $prop = utf8::_loose_name(lc $prop);
1847
1848 # Here is a legal property, but the hash below (created by mktables for
1849 # this purpose) only knows about the properties that have a very finite
1850 # number of potential values, that is not ones whose value could be
1851 # anything, like most (if not all) string properties. These don't have
1852 # synonyms anyway. Simply return the input. For example, there is no
1853 # synonym for ('Uppercase_Mapping', A').
1854 return $value if ! exists $prop_value_aliases{$prop};
1855
1856 # The value name may be loosely or strictly matched; we don't know yet.
1857 # But both types use lower-case.
1858 $value = lc $value;
1859
1860 # If the name isn't found under loose matching, it certainly won't be
1861 # found under strict
1862 my $loose_value = utf8::_loose_name($value);
1863 return unless exists $loose_to_standard_value{"$prop=$loose_value"};
1864
1865 # Similarly if the combination under loose matching doesn't exist, it
1866 # won't exist under strict.
1867 my $standard_value = $loose_to_standard_value{"$prop=$loose_value"};
1868 return unless exists $prop_value_aliases{$prop}{$standard_value};
1869
1870 # Here we did find a combination under loose matching rules. But it could
1871 # be that is a strict property match that shouldn't have matched.
1872 # %prop_value_aliases is set up so that the strict matches will appear as
1873 # if they were in loose form. Thus, if the non-loose version is legal,
1874 # we're ok, can skip the further check.
1875 if (! exists $utf8::stricter_to_file_of{"$prop=$value"}
1876
1877 # We're also ok and skip the further check if value loosely matches.
1878 # mktables has verified that no strict name under loose rules maps to
1879 # an existing loose name. This code relies on the very limited
1880 # circumstances that strict names can be here. Strict name matching
1881 # happens under two conditions:
1882 # 1) when the name begins with an underscore. But this function
1883 # doesn't accept those, and %prop_value_aliases doesn't have
1884 # them.
1885 # 2) When the values are numeric, in which case we need to look
1886 # further, but their squeezed-out loose values will be in
1887 # %stricter_to_file_of
1888 && exists $utf8::stricter_to_file_of{"$prop=$loose_value"})
1889 {
1890 # The only thing that's legal loosely under strict is that can have an
1891 # underscore between digit pairs XXX
1892 while ($value =~ s/(\d)_(\d)/$1$2/g) {}
1893 return unless exists $utf8::stricter_to_file_of{"$prop=$value"};
1894 }
1895
1896 # Here, we know that the combination exists. Return it.
1897 my $list_ref = $prop_value_aliases{$prop}{$standard_value};
1898 if (@$list_ref > 1) {
1899 # The full name is in element 1.
1900 return $list_ref->[1] unless wantarray;
1901
1902 return @{dclone $list_ref};
1903 }
1904
1905 return $list_ref->[0] unless wantarray;
1906
1907 # Only 1 element means that it repeats
1908 return ( $list_ref->[0], $list_ref->[0] );
1909}
7319f91d 1910
681d705c
KW
1911# All 1 bits is the largest possible UV.
1912$Unicode::UCD::MAX_CP = ~0;
1913
1914=pod
1915
1916=head2 B<prop_invlist()>
1917
1918C<prop_invlist> returns an inversion list (described below) that defines all the
1919code points for the binary Unicode property (or "property=value" pair) given
1920by the input parameter string:
1921
1922 use feature 'say';
1923 use Unicode::UCD 'prop_invlist';
1924 say join ", ", prop_invlist("Any");
1925
1926 prints:
1927 0, 1114112
1928
1929An empty list is returned if the input is unknown; the number of elements in
1930the list is returned if called in scalar context.
1931
1932L<perluniprops|perluniprops/Properties accessible through \p{} and \P{}> gives
1933the list of properties that this function accepts, as well as all the possible
1934forms for them (including with the optional "Is_" prefixes). (Except this
1935function doesn't accept any Perl-internal properties, some of which are listed
1936there.) This function uses the same loose or tighter matching rules for
1937resolving the input property's name as is done for regular expressions. These
1938are also specified in L<perluniprops|perluniprops/Properties accessible
1939through \p{} and \P{}>. Examples of using the "property=value" form are:
1940
1941 say join ", ", prop_invlist("Script=Shavian");
1942
1943 prints:
1944 66640, 66688
1945
1946 say join ", ", prop_invlist("ASCII_Hex_Digit=No");
1947
1948 prints:
1949 0, 48, 58, 65, 71, 97, 103
1950
1951 say join ", ", prop_invlist("ASCII_Hex_Digit=Yes");
1952
1953 prints:
1954 48, 58, 65, 71, 97, 103
1955
1956Inversion lists are a compact way of specifying Unicode property-value
1957definitions. The 0th item in the list is the lowest code point that has the
1958property-value. The next item (item [1]) is the lowest code point beyond that
1959one that does NOT have the property-value. And the next item beyond that
1960([2]) is the lowest code point beyond that one that does have the
1961property-value, and so on. Put another way, each element in the list gives
1962the beginning of a range that has the property-value (for even numbered
1963elements), or doesn't have the property-value (for odd numbered elements).
1964The name for this data structure stems from the fact that each element in the
1965list toggles (or inverts) whether the corresponding range is or isn't on the
1966list.
1967
1968In the final example above, the first ASCII Hex digit is code point 48, the
1969character "0", and all code points from it through 57 (a "9") are ASCII hex
1970digits. Code points 58 through 64 aren't, but 65 (an "A") through 70 (an "F")
1971are, as are 97 ("a") through 102 ("f"). 103 starts a range of code points
1972that aren't ASCII hex digits. That range extends to infinity, which on your
1973computer can be found in the variable C<$Unicode::UCD::MAX_CP>. (This
1974variable is as close to infinity as Perl can get on your platform, and may be
1975too high for some operations to work; you may wish to use a smaller number for
1976your purposes.)
1977
1978Note that the inversion lists returned by this function can possibly include
1979non-Unicode code points, that is anything above 0x10FFFF. This is in
1980contrast to Perl regular expression matches on those code points, in which a
1981non-Unicode code point always fails to match. For example, both of these have
1982the same result:
1983
1984 chr(0x110000) =~ \p{ASCII_Hex_Digit=True} # Fails.
1985 chr(0x110000) =~ \p{ASCII_Hex_Digit=False} # Fails!
1986
1987And both raise a warning that a Unicode property is being used on a
1988non-Unicode code point. It is arguable as to which is the correct thing to do
1989here. This function has chosen the way opposite to the Perl regular
1990expression behavior. This allows you to easily flip to to the Perl regular
1991expression way (for you to go in the other direction would be far harder).
1992Simply add 0x110000 at the end of the non-empty returned list if it isn't
1993already that value; and pop that value if it is; like:
1994
1995 my @list = prop_invlist("foo");
1996 if (@list) {
1997 if ($list[-1] == 0x110000) {
1998 pop @list; # Defeat the turning on for above Unicode
1999 }
2000 else {
2001 push @list, 0x110000; # Turn off for above Unicode
2002 }
2003 }
2004
2005It is a simple matter to expand out an inversion list to a full list of all
2006code points that have the property-value:
2007
2008 my @invlist = prop_invlist($property_name);
2009 die "empty" unless @invlist;
2010 my @full_list;
2011 for (my $i = 0; $i < @invlist; $i += 2) {
2012 my $upper = ($i + 1) < @invlist
2013 ? $invlist[$i+1] - 1 # In range
2014 : $Unicode::UCD::MAX_CP; # To infinity. You may want
2015 # to stop much much earlier;
2016 # going this high may expose
2017 # perl deficiencies with very
2018 # large numbers.
2019 for my $j ($invlist[$i] .. $upper) {
2020 push @full_list, $j;
2021 }
2022 }
2023
2024C<prop_invlist> does not know about any user-defined nor Perl internal-only
2025properties, and will return C<undef> if called with one of those.
2026
2027=cut
2028
2029# User-defined properties could be handled with some changes to utf8_heavy.pl;
2030# and implementing here of dealing with EXTRAS. If done, consideration should
2031# be given to the fact that the user subroutine could return different results
2032# with each call; security issues need to be thought about.
2033
2034# These are created by mktables for this routine and stored in unicore/UCD.pl
2035# where their structures are described.
2036our %loose_defaults;
2037our $MAX_UNICODE_CODEPOINT;
2038
2039sub prop_invlist ($) {
2040 my $prop = $_[0];
2041 return if ! defined $prop;
2042
2043 require "utf8_heavy.pl";
2044
2045 # Warnings for these are only for regexes, so not applicable to us
2046 no warnings 'deprecated';
2047
2048 # Get the swash definition of the property-value.
2049 my $swash = utf8::SWASHNEW(__PACKAGE__, $prop, undef, 1, 0);
2050
2051 # Fail if not found, or isn't a boolean property-value, or is a
2052 # user-defined property, or is internal-only.
2053 return if ! $swash
2054 || ref $swash eq ""
2055 || $swash->{'BITS'} != 1
2056 || $swash->{'USER_DEFINED'}
2057 || $prop =~ /^\s*_/;
2058
2059 if ($swash->{'EXTRAS'}) {
2060 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has EXTRAS magic";
2061 return;
2062 }
2063 if ($swash->{'SPECIALS'}) {
2064 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has SPECIALS magic";
2065 return;
2066 }
2067
2068 my @invlist;
2069
2070 # The input lines look like:
2071 # 0041\t005A # [26]
2072 # 005F
2073
2074 # Split into lines, stripped of trailing comments
2075 foreach my $range (split "\n",
2076 $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr)
2077 {
2078 # And find the beginning and end of the range on the line
2079 my ($hex_begin, $hex_end) = split "\t", $range;
2080 my $begin = hex $hex_begin;
2081
a39cc031
KW
2082 # If the new range merely extends the old, we remove the marker
2083 # created the last time through the loop for the old's end, which
2084 # causes the new one's end to be used instead.
2085 if (@invlist && $begin == $invlist[-1]) {
2086 pop @invlist;
2087 }
2088 else {
2f3f243e
KW
2089 # Add the beginning of the range
2090 push @invlist, $begin;
a39cc031 2091 }
681d705c
KW
2092
2093 if (defined $hex_end) { # The next item starts with the code point 1
2094 # beyond the end of the range.
2095 push @invlist, hex($hex_end) + 1;
2096 }
2097 else { # No end of range, is a single code point.
2098 push @invlist, $begin + 1;
2099 }
2100 }
2101
2102 require "unicore/UCD.pl";
2103 my $FIRST_NON_UNICODE = $MAX_UNICODE_CODEPOINT + 1;
2104
2105 # Could need to be inverted: add or subtract a 0 at the beginning of the
2106 # list. And to keep it from matching non-Unicode, add or subtract the
2107 # first non-unicode code point.
2108 if ($swash->{'INVERT_IT'}) {
2109 if (@invlist && $invlist[0] == 0) {
2110 shift @invlist;
2111 }
2112 else {
2113 unshift @invlist, 0;
2114 }
2115 if (@invlist && $invlist[-1] == $FIRST_NON_UNICODE) {
2116 pop @invlist;
2117 }
2118 else {
2119 push @invlist, $FIRST_NON_UNICODE;
2120 }
2121 }
2122
2123 # Here, the list is set up to include only Unicode code points. But, if
2124 # the table is the default one for the property, it should contain all
2125 # non-Unicode code points. First calculate the loose name for the
2126 # property. This is done even for strict-name properties, as the data
2127 # structure that mktables generates for us is set up so that we don't have
2128 # to worry about that. The property-value needs to be split if compound,
2129 # as the loose rules need to be independently calculated on each part. We
2130 # know that it is syntactically valid, or SWASHNEW would have failed.
2131
2132 $prop = lc $prop;
2133 my ($prop_only, $table) = split /\s*[:=]\s*/, $prop;
2134 if ($table) {
2135
2136 # May have optional prefixed 'is'
2137 $prop = utf8::_loose_name($prop_only) =~ s/^is//r;
2138 $prop = $utf8::loose_property_name_of{$prop};
2139 $prop .= "=" . utf8::_loose_name($table);
2140 }
2141 else {
2142 $prop = utf8::_loose_name($prop);
2143 }
2144 if (exists $loose_defaults{$prop}) {
2145
2146 # Here, is the default table. If a range ended with 10ffff, instead
2147 # continue that range to infinity, by popping the 110000; otherwise,
2148 # add the range from 11000 to infinity
2149 if (! @invlist || $invlist[-1] != $FIRST_NON_UNICODE) {
2150 push @invlist, $FIRST_NON_UNICODE;
2151 }
2152 else {
2153 pop @invlist;
2154 }
2155 }
2156
2157 return @invlist;
2158}
7319f91d 2159
62b3b855
KW
2160sub _search_invlist {
2161 # Find the range in the inversion list which contains a code point; that
2162 # is, find i such that l[i] <= code_point < l[i+1]
2163
2164 # If this is ever made public, could use to speed up .t specials. Would
2165 # need to use code point argument, as in other functions in this pm
2166
2167 my $list_ref = shift;
2168 my $code_point = shift;
2169 # Verify non-neg numeric XXX
2170
2171 my $max_element = @$list_ref - 1;
2172 return if ! $max_element < 0; # Undef if list is empty.
2173
2174 # Short cut something at the far-end of the table. This also allows us to
2175 # refer to element [$i+1] without fear of being out-of-bounds in the loop
2176 # below.
2177 return $max_element if $code_point >= $list_ref->[$max_element];
2178
2179 use integer; # want integer division
2180
2181 my $i = $max_element / 2;
2182
2183 my $lower = 0;
2184 my $upper = $max_element;
2185 while (1) {
2186
2187 if ($code_point >= $list_ref->[$i]) {
2188
2189 # Here we have met the lower constraint. We can quit if we
2190 # also meet the upper one.
2191 last if $code_point < $list_ref->[$i+1];
2192
2193 $lower = $i; # Still too low.
2194
2195 }
2196 else {
2197
2198 # Here, $code_point < $list_ref[$i], so look lower down.
2199 $upper = $i;
2200 }
2201
2202 # Split search domain in half to try again.
2203 my $temp = ($upper + $lower) / 2;
2204
2205 # No point in continuing unless $i changes for next time
2206 # in the loop.
2207 return $i if $temp == $i;
2208 $i = $temp;
2209 } # End of while loop
2210
2211 # Here we have found the offset
2212 return $i;
2213}
2214
2215=pod
2216
2217=head2 B<prop_invmap()>
2218
2219 use Unicode::UCD 'prop_invmap';
2220 my ($list_ref, $map_ref, $format, $missing)
2221 = prop_invmap("General Category");
2222
2223C<prop_invmap> is used to get the complete mapping definition for a property,
2224in the form of an inversion map. An inversion map consists of two parallel
2225arrays. One is an ordered list of code points that mark range beginnings, and
2226the other gives the value (or mapping) that all code points in the
2227corresponding range have.
2228
2229C<prop_invmap> is called with the name of the desired property. The name is
2230loosely matched, meaning that differences in case, white-space, hyphens, and
2231underscores are not meaningful (except for the trailing underscore in the
2232old-form grandfathered-in property C<"L_">, which is better written as C<"LC">,
2233or even better, C<"Gc=LC">).
2234
2235Many Unicode properties have more than one name (or alias). C<prop_invmap>
2236understands all of these, including Perl extensions to them. Ambiguities are
2237resolved as described above for L</prop_aliases()>. The Perl internal
2238property "Perl_Decimal_Digit, described below, is also accepted. C<undef> is
2239returned if the property name is unknown.
ee94c7d1
KW
2240See L<perluniprops/Properties accessible through Unicode::UCD> for the
2241properties acceptable as inputs to this function.
62b3b855
KW
2242
2243It is a fatal error to call this function except in list context.
2244
2245In addition to the the two arrays that form the inversion map, C<prop_invmap>
2246returns two other values; one is a scalar that gives some details as to the
2247format of the entries of the map array; the other is used for specialized
2248purposes, described at the end of this section.
2249
2250This means that C<prop_invmap> returns a 4 element list. For example,
2251
2252 my ($blocks_ranges_ref, $blocks_maps_ref, $format, $default)
2253 = prop_invmap("Block");
2254
2255In this call, the two arrays will be populated as shown below (for Unicode
22566.0):
2257
2258 Index @blocks_ranges @blocks_maps
2259 0 0x0000 Basic Latin
2260 1 0x0080 Latin-1 Supplement
2261 2 0x0100 Latin Extended-A
2262 3 0x0180 Latin Extended-B
2263 4 0x0250 IPA Extensions
2264 5 0x02B0 Spacing Modifier Letters
2265 6 0x0300 Combining Diacritical Marks
2266 7 0x0370 Greek and Coptic
2267 8 0x0400 Cyrillic
2268 ...
2269 233 0x2B820 No_Block
2270 234 0x2F800 CJK Compatibility Ideographs Supplement
2271 235 0x2FA20 No_Block
2272 236 0xE0000 Tags
2273 237 0xE0080 No_Block
2274 238 0xE0100 Variation Selectors Supplement
2275 239 0xE01F0 No_Block
2276 240 0xF0000 Supplementary Private Use Area-A
2277 241 0x100000 Supplementary Private Use Area-B
2278 242 0x110000 No_Block
2279
2280The first line (with Index [0]) means that the value for code point 0 is "Basic
2281Latin". The entry "0x0080" in the @blocks_ranges column in the second line
2282means that the value from the first line, "Basic Latin", extends to all code
2283points in the range from 0 up to but not including 0x0080, that is, through
647396da 2284127. In other words, the code points from 0 to 127 are all in the "Basic
62b3b855
KW
2285Latin" block. Similarly, all code points in the range from 0x0080 up to (but
2286not including) 0x0100 are in the block named "Latin-1 Supplement", etc.
2287(Notice that the return is the old-style block names; see L</Old-style versus
2288new-style block names>).
2289
2290The final line (with Index [242]) means that the value for all code points above
2291the legal Unicode maximum code point have the value "No_Block", which is the
2292term Unicode uses for a non-existing block.
2293
2294The arrays completely specify the mappings for all possible code points.
2295The final element in an inversion map returned by this function will always be
2296for the range that consists of all the code points that aren't legal Unicode,
2297but that are expressible on the platform. (That is, it starts with code point
22980x110000, the first code point above the legal Unicode maximum, and extends to
2299infinity.) The value for that range will be the same that any typical
2300unassigned code point has for the specified property. (Certain unassigned
2301code points are not "typical"; for example the non-character code points, or
2302those in blocks that are to be written right-to-left. The above-Unicode
2303range's value is not based on these atypical code points.) It could be argued
2304that, instead of treating these as unassigned Unicode code points, the value
2305for this range should be C<undef>. If you wish, you can change the returned
2306arrays accordingly.
2307
2308The maps are almost always simple scalars that should be interpreted as-is.
2309These values are those given in the Unicode-supplied data files, which may be
2310inconsistent as to capitalization and as to which synonym for a property-value
2311is given. The results may be normalized by using the L</prop_value_aliases()>
2312function.
2313
2314There are exceptions to the simple scalar maps. Some properties have some
2315elements in their map list that are themselves lists of scalars; and some
2316special strings are returned that are not to be interpreted as-is. Element
2317[2] (placed into C<$format> in the example above) of the returned four element
647396da 2318list tells you if the map has any of these special elements or not, as follows:
62b3b855
KW
2319
2320=over
2321
dc8d8ea6 2322=item B<C<s>>
62b3b855
KW
2323
2324means all the elements of the map array are simple scalars, with no special
2325elements. Almost all properties are like this, like the C<block> example
2326above.
2327
dc8d8ea6 2328=item B<C<sl>>
62b3b855 2329
647396da 2330means that some of the map array elements have the form given by C<"s">, and
62b3b855
KW
2331the rest are lists of scalars. For example, here is a portion of the output
2332of calling C<prop_invmap>() with the "Script Extensions" property:
2333
2334 @scripts_ranges @scripts_maps
2335 ...
c2ca0207
KW
2336 0x0953 Devanagari
2337 0x0964 [ Bengali, Devanagari, Gurumukhi, Oriya ]
2338 0x0966 Devanagari
62b3b855
KW
2339 0x0970 Common
2340
647396da
KW
2341Here, the code points 0x964 and 0x965 are both used in Bengali,
2342Devanagari, Gurmukhi, and Oriya, but no other scripts.
62b3b855 2343
647396da 2344The Name_Alias property is also of this form. But each scalar consists of two
58b75e36 2345components: 1) the name, and 2) the type of alias this is. They are
7620cb10
KW
2346separated by a colon and a space. In Unicode 6.1, there are several alias types:
2347
2348=over
2349
2350=item C<correction>
2351
2352indicates that the name is a corrected form for the
2353original name (which remains valid) for the same code point.
2354
2355=item C<control>
2356
2357adds a new name for a control character.
2358
2359=item C<alternate>
2360
2361is an alternate name for a character
2362
2363=item C<figment>
2364
2365is a name for a character that has been documented but was never in any
2366actual standard.
2367
2368=item C<abbreviation>
2369
2370is a common abbreviation for a character
2371
2372=back
2373
2374The lists are ordered (roughly) so the most preferred names come before less
2375preferred ones.
58b75e36
KW
2376
2377For example,
2378
7620cb10
KW
2379 @aliases_ranges @alias_maps
2380 ...
2381 0x009E [ 'PRIVACY MESSAGE: control', 'PM: abbreviation' ]
2382 0x009F [ 'APPLICATION PROGRAM COMMAND: control',
2383 'APC: abbreviation'
2384 ]
2385 0x00A0 'NBSP: abbreviation'
2386 0x00A1 ""
2387 0x00AD 'SHY: abbreviation'
2388 0x00AE ""
2389 0x01A2 'LATIN CAPITAL LETTER GHA: correction'
2390 0x01A3 'LATIN SMALL LETTER GHA: correction'
2391 0x01A4 ""
58b75e36 2392 ...
58b75e36 2393
7620cb10
KW
2394A map to the empty string means that there is no alias defined for the code
2395point.
58b75e36 2396
d11155ec 2397=item B<C<a>>
62b3b855 2398
647396da 2399is like C<"s"> in that all the map array elements are scalars, but here they are
d11155ec
KW
2400restricted to all being integers, and some have to be adjusted (hence the name
2401C<"a">) to get the correct result. For example, in:
62b3b855
KW
2402
2403 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2404 = prop_invmap("Simple_Uppercase_Mapping");
2405
2406the returned arrays look like this:
2407
2408 @$uppers_ranges_ref @$uppers_maps_ref Note
bf7fe2df 2409 0 0
d11155ec 2410 97 65 'a' maps to 'A', b => B ...
bf7fe2df 2411 123 0
d11155ec 2412 181 924 MICRO SIGN => Greek Cap MU
bf7fe2df 2413 182 0
62b3b855
KW
2414 ...
2415
d11155ec
KW
2416Let's start with the second line. It says that the uppercase of code point 97
2417is 65; or C<uc("a")> == "A". But the line is for the entire range of code
2418points 97 through 122. To get the mapping for any code point in a range, you
2419take the offset it has from the beginning code point of the range, and add
2420that to the mapping for that first code point. So, the mapping for 122 ("z")
2421is derived by taking the offset of 122 from 97 (=25) and adding that to 65,
2422yielding 90 ("z"). Likewise for everything in between.
2423
2424The first line works the same way. The first map in a range is always the
2425correct value for its code point (because the adjustment is 0). Thus the
2426C<uc(chr(0))> is just itself. Also, C<uc(chr(1))> is also itself, as the
2427adjustment is 0+1-0 .. C<uc(chr(96))> is 96.
bf7fe2df 2428
d11155ec
KW
2429Requiring this simple adjustment allows the returned arrays to be
2430significantly smaller than otherwise, up to a factor of 10, speeding up
2431searching through them.
62b3b855 2432
d11155ec 2433=item B<C<al>>
62b3b855 2434
d11155ec 2435means that some of the map array elements have the form given by C<"a">, and
62b3b855
KW
2436the rest are ordered lists of code points.
2437For example, in:
2438
2439 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2440 = prop_invmap("Uppercase_Mapping");
2441
2442the returned arrays look like this:
2443
2444 @$uppers_ranges_ref @$uppers_maps_ref
bf7fe2df 2445 0 0
d11155ec 2446 97 65
bf7fe2df 2447 123 0
d11155ec 2448 181 924
bf7fe2df 2449 182 0
62b3b855
KW
2450 ...
2451 0x0149 [ 0x02BC 0x004E ]
bf7fe2df 2452 0x014A 0
d11155ec 2453 0x014B 330
62b3b855
KW
2454 ...
2455
2456This is the full Uppercase_Mapping property (as opposed to the
d11155ec 2457Simple_Uppercase_Mapping given in the example for format C<"a">). The only
62b3b855
KW
2458difference between the two in the ranges shown is that the code point at
24590x0149 (LATIN SMALL LETTER N PRECEDED BY APOSTROPHE) maps to a string of two
2460characters, 0x02BC (MODIFIER LETTER APOSTROPHE) followed by 0x004E (LATIN
2461CAPITAL LETTER N).
2462
d11155ec
KW
2463No adjustments are needed to entries that are references to arrays; each such
2464entry will have exactly one element in its range, so the offset is always 0.
bf7fe2df 2465
d11155ec 2466=item B<C<ae>>
b0b13ada 2467
d11155ec
KW
2468This is like C<"a">, but some elements are the empty string, and should not be
2469adjusted.
b0b13ada
KW
2470The one internal Perl property accessible by C<prop_invmap> is of this type:
2471"Perl_Decimal_Digit" returns an inversion map which gives the numeric values
2472that are represented by the Unicode decimal digit characters. Characters that
2473don't represent decimal digits map to the empty string, like so:
2474
2475 @digits @values
2476 0x0000 ""
d11155ec 2477 0x0030 0
b0b13ada 2478 0x003A: ""
d11155ec 2479 0x0660: 0
b0b13ada 2480 0x066A: ""
d11155ec 2481 0x06F0: 0
b0b13ada 2482 0x06FA: ""
d11155ec 2483 0x07C0: 0
b0b13ada 2484 0x07CA: ""
d11155ec 2485 0x0966: 0
b0b13ada
KW
2486 ...
2487
2488This means that the code points from 0 to 0x2F do not represent decimal digits;
d11155ec
KW
2489the code point 0x30 (DIGIT ZERO) represents 0; code point 0x31, (DIGIT ONE),
2490represents 0+1-0 = 1; ... code point 0x39, (DIGIT NINE), represents 0+9-0 = 9;
2491... code points 0x3A through 0x65F do not represent decimal digits; 0x660
2492(ARABIC-INDIC DIGIT ZERO), represents 0; ... 0x07C1 (NKO DIGIT ONE),
2493represents 0+1-0 = 1 ...
b0b13ada 2494
d11155ec 2495=item B<C<ale>>
62b3b855 2496
d11155ec
KW
2497is a combination of the C<"al"> type and the C<"ae"> type. Some of
2498the map array elements have the forms given by C<"al">, and
62b3b855
KW
2499the rest are the empty string. The property C<NFKC_Casefold> has this form.
2500An example slice is:
2501
2502 @$ranges_ref @$maps_ref Note
2503 ...
d11155ec
KW
2504 0x00AA 97 FEMININE ORDINAL INDICATOR => 'a'
2505 0x00AB 0
62b3b855 2506 0x00AD SOFT HYPHEN => ""
d11155ec 2507 0x00AE 0
62b3b855 2508 0x00AF [ 0x0020, 0x0304 ] MACRON => SPACE . COMBINING MACRON
d11155ec 2509 0x00B0 0
62b3b855
KW
2510 ...
2511
4f143a72 2512=item B<C<ar>>
6cc45523
KW
2513
2514means that all the elements of the map array are either rational numbers or
2515the string C<"NaN">, meaning "Not a Number". A rational number is either an
2516integer, or two integers separated by a solidus (C<"/">). The second integer
2517represents the denominator of the division implied by the solidus, and is
6329003c
KW
2518actually always positive, so it is guaranteed not to be 0 and to not to be
2519signed. When the element is a plain integer (without the
4f143a72
KW
2520solidus), it may need to be adjusted to get the correct value by adding the
2521offset, just as other C<"a"> properties. No adjustment is needed for
2522fractions, as the range is guaranteed to have just a single element, and so
2523the offset is always 0.
2524
2525If you want to convert the returned map to entirely scalar numbers, you
6cc45523
KW
2526can use something like this:
2527
2528 my ($invlist_ref, $invmap_ref, $format) = prop_invmap($property);
4f143a72 2529 if ($format && $format eq "ar") {
6cc45523
KW
2530 map { $_ = eval $_ } @$invmap_ref;
2531 }
2532
2533Here's some entries from the output of the property "Nv", which has format
4f143a72 2534C<"ar">.
6cc45523 2535
4f143a72 2536 @numerics_ranges @numerics_maps Note
6cc45523 2537 0x00 "NaN"
4f143a72 2538 0x30 0 DIGIT 0 .. DIGIT 9
6cc45523 2539 0x3A "NaN"
4f143a72 2540 0xB2 2 SUPERSCRIPTs 2 and 3
6cc45523 2541 0xB4 "NaN"
4f143a72 2542 0xB9 1 SUPERSCRIPT 1
6cc45523 2543 0xBA "NaN"
4f143a72
KW
2544 0xBC 1/4 VULGAR FRACTION 1/4
2545 0xBD 1/2 VULGAR FRACTION 1/2
2546 0xBE 3/4 VULGAR FRACTION 3/4
6cc45523 2547 0xBF "NaN"
4f143a72
KW
2548 0x660 0 ARABIC-INDIC DIGIT ZERO .. NINE
2549 0x66A "NaN"
6cc45523 2550
dc8d8ea6 2551=item B<C<n>>
62b3b855
KW
2552
2553means the Name property. All the elements of the map array are simple
2554scalars, but some of them contain special strings that require more work to
2555get the actual name.
2556
2557Entries such as:
2558
2559 CJK UNIFIED IDEOGRAPH-<code point>
2560
2561mean that the name for the code point is "CJK UNIFIED IDEOGRAPH-"
2562with the code point (expressed in hexadecimal) appended to it, like "CJK
647396da
KW
2563UNIFIED IDEOGRAPH-3403" (similarly for S<C<CJK COMPATIBILITY IDEOGRAPH-E<lt>code
2564pointE<gt>>>).
62b3b855
KW
2565
2566Also, entries like
2567
2568 <hangul syllable>
2569
2570means that the name is algorithmically calculated. This is easily done by
2571the function L<charnames/charnames::viacode(code)>.
2572
2573Note that for control characters (C<Gc=cc>), Unicode's data files have the
2574string "C<E<lt>controlE<gt>>", but the real name of each of these characters is the empty
7620cb10 2575string. This function returns that real name, the empty string. (There are
647396da
KW
2576names for these characters, but they are considered aliases, not the Name
2577property name, and are contained in the C<Name_Alias> property.)
62b3b855 2578
d11155ec 2579=item B<C<ad>>
62b3b855 2580
d11155ec 2581means the Decomposition_Mapping property. This property is like C<"al">
bea2c146 2582properties, except that one of the scalar elements is of the form:
62b3b855
KW
2583
2584 <hangul syllable>
2585
bea2c146
KW
2586This signifies that this entry should be replaced by the decompositions for
2587all the code points whose decomposition is algorithmically calculated. (All
6329003c
KW
2588of them are currently in one range and no others outisde the range are likely
2589to ever be added to Unicode; the C<"n"> format
bea2c146 2590has this same entry.) These can be generated via the function
62b3b855
KW
2591L<Unicode::Normalize::NFD()|Unicode::Normalize>.
2592
62b3b855
KW
2593Note that the mapping is the one that is specified in the Unicode data files,
2594and to get the final decomposition, it may need to be applied recursively.
2595
2596=back
2597
d11155ec
KW
2598Note that a format begins with the letter "a" if and only the property it is
2599for requires adjustments by adding the offsets in multi-element ranges. For
2600all these properties, an entry should be adjusted only if the map is a scalar
2601which is an integer. That is, it must match the regular expression:
2602
2603 / ^ -? \d+ $ /xa
2604
2605Further, the first element in a range never needs adjustment, as the
2606adjustment would be just adding 0.
2607
62b3b855
KW
2608A binary search can be used to quickly find a code point in the inversion
2609list, and hence its corresponding mapping.
2610
2611The final element (index [3], assigned to C<$default> in the "block" example) in
2612the four element list returned by this function may be useful for applications
2613that wish to convert the returned inversion map data structure into some
2614other, such as a hash. It gives the mapping that most code points map to
2615under the property. If you establish the convention that any code point not
2616explicitly listed in your data structure maps to this value, you can
2617potentially make your data structure much smaller. As you construct your data
2618structure from the one returned by this function, simply ignore those ranges
2619that map to this value, generally called the "default" value. For example, to
2620convert to the data structure searchable by L</charinrange()>, you can follow
6329003c 2621this recipe for properties that don't require adjustments:
62b3b855
KW
2622
2623 my ($list_ref, $map_ref, $format, $missing) = prop_invmap($property);
2624 my @range_list;
6329003c
KW
2625
2626 # Look at each element in the list, but the -2 is needed because we
2627 # look at $i+1 in the loop, and the final element is guaranteed to map
2628 # to $missing by prop_invmap(), so we would skip it anyway.
62b3b855
KW
2629 for my $i (0 .. @$list_ref - 2) {
2630 next if $map_ref->[$i] eq $missing;
2631 push @range_list, [ $list_ref->[$i],
2632 $list_ref->[$i+1],
2633 $map_ref->[$i]
2634 ];
2635 }
2636
2637 print charinrange(\@range_list, $code_point), "\n";
2638
62b3b855
KW
2639With this, C<charinrange()> will return C<undef> if its input code point maps
2640to C<$missing>. You can avoid this by omitting the C<next> statement, and adding
2641a line after the loop to handle the final element of the inversion map.
2642
6329003c
KW
2643Similarly, this recipe can be used for properties that do require adjustments:
2644
2645 for my $i (0 .. @$list_ref - 2) {
2646 next if $map_ref->[$i] eq $missing;
2647
2648 # prop_invmap() guarantees that if the mapping is to an array, the
2649 # range has just one element, so no need to worry about adjustments.
2650 if (ref $map_ref->[$i]) {
2651 push @range_list,
2652 [ $list_ref->[$i], $list_ref->[$i], $map_ref->[$i] ];
2653 }
2654 else { # Otherwise each element is actually mapped to a separate
2655 # value, so the range has to be split into single code point
2656 # ranges.
2657
2658 my $adjustment = 0;
2659
2660 # For each code point that gets mapped to something...
2661 for my $j ($list_ref->[$i] .. $list_ref->[$i+1] -1 ) {
2662
2663 # ... add a range consisting of just it mapping to the
2664 # original plus the adjustment, which is incremented for the
2665 # next time through the loop, as the offset increases by 1
2666 # for each element in the range
2667 push @range_list,
2668 [ $j, $j, $map_ref->[$i] + $adjustment++ ];
2669 }
2670 }
2671 }
62b3b855
KW
2672
2673Note that the inversion maps returned for the C<Case_Folding> and
2674C<Simple_Case_Folding> properties do not include the Turkic-locale mappings.
2675Use L</casefold()> for these.
2676
62b3b855
KW
2677C<prop_invmap> does not know about any user-defined properties, and will
2678return C<undef> if called with one of those.
2679
2680=cut
2681
2682# User-defined properties could be handled with some changes to utf8_heavy.pl;
2683# if done, consideration should be given to the fact that the user subroutine
2684# could return different results with each call, which could lead to some
2685# security issues.
2686
2687# One could store things in memory so they don't have to be recalculated, but
2688# it is unlikely this will be called often, and some properties would take up
2689# significant memory.
2690
2691# These are created by mktables for this routine and stored in unicore/UCD.pl
2692# where their structures are described.
2693our @algorithmic_named_code_points;
2694our $HANGUL_BEGIN;
2695our $HANGUL_COUNT;
2696
2697sub prop_invmap ($) {
2698
2699 croak __PACKAGE__, "::prop_invmap: must be called in list context" unless wantarray;
2700
2701 my $prop = $_[0];
2702 return unless defined $prop;
2703
2704 # Fail internal properties
2705 return if $prop =~ /^_/;
2706
2707 # The values returned by this function.
2708 my (@invlist, @invmap, $format, $missing);
2709
2710 # The swash has two components we look at, the base list, and a hash,
2711 # named 'SPECIALS', containing any additional members whose mappings don't
2712 # fit into the the base list scheme of things. These generally 'override'
2713 # any value in the base list for the same code point.
2714 my $overrides;
2715
2716 require "utf8_heavy.pl";
2717 require "unicore/UCD.pl";
2718
2719RETRY:
2720
647396da
KW
2721 # If there are multiple entries for a single code point
2722 my $has_multiples = 0;
2723
62b3b855
KW
2724 # Try to get the map swash for the property. They have 'To' prepended to
2725 # the property name, and 32 means we will accept 32 bit return values.
647396da 2726 # The 0 means we aren't calling this from tr///.
62b3b855
KW
2727 my $swash = utf8::SWASHNEW(__PACKAGE__, "To$prop", undef, 32, 0);
2728
2729 # If didn't find it, could be because needs a proxy. And if was the
2730 # 'Block' or 'Name' property, use a proxy even if did find it. Finding it
647396da
KW
2731 # in these cases would be the result of the installation changing mktables
2732 # to output the Block or Name tables. The Block table gives block names
2733 # in the new-style, and this routine is supposed to return old-style block
2734 # names. The Name table is valid, but we need to execute the special code
2735 # below to add in the algorithmic-defined name entries.
34132297 2736 # And NFKCCF needs conversion, so handle that here too.
62b3b855 2737 if (ref $swash eq ""
34132297 2738 || $swash->{'TYPE'} =~ / ^ To (?: Blk | Na | NFKCCF ) $ /x)
62b3b855
KW
2739 {
2740
2741 # Get the short name of the input property, in standard form
2742 my ($second_try) = prop_aliases($prop);
2743 return unless $second_try;
2744 $second_try = utf8::_loose_name(lc $second_try);
2745
2746 if ($second_try eq "in") {
2747
2748 # This property is identical to age for inversion map purposes
2749 $prop = "age";
2750 goto RETRY;
2751 }
cfc5eb77 2752 elsif ($second_try =~ / ^ s ( cf | fc | [ltu] c ) $ /x) {
62b3b855 2753
75e7c50b
KW
2754 # These properties use just the LIST part of the full mapping,
2755 # which includes the simple maps that are otherwise overridden by
2756 # the SPECIALS. So all we need do is to not look at the SPECIALS;
2757 # set $overrides to indicate that
62b3b855 2758 $overrides = -1;
62b3b855 2759
75e7c50b 2760 # The full name is the simple name stripped of its initial 's'
cfc5eb77
KW
2761 $prop = $1;
2762
2763 # .. except for this case
2764 $prop = 'cf' if $prop eq 'fc';
2765
62b3b855
KW
2766 goto RETRY;
2767 }
2768 elsif ($second_try eq "blk") {
2769
2770 # We use the old block names. Just create a fake swash from its
2771 # data.
2772 _charblocks();
2773 my %blocks;
2774 $blocks{'LIST'} = "";
2775 $blocks{'TYPE'} = "ToBlk";
2776 $utf8::SwashInfo{ToBlk}{'missing'} = "No_Block";
2777 $utf8::SwashInfo{ToBlk}{'format'} = "s";
2778
2779 foreach my $block (@BLOCKS) {
2780 $blocks{'LIST'} .= sprintf "%x\t%x\t%s\n",
2781 $block->[0],
2782 $block->[1],
2783 $block->[2];
2784 }
2785 $swash = \%blocks;
2786 }
2787 elsif ($second_try eq "na") {
2788
2789 # Use the combo file that has all the Name-type properties in it,
2790 # extracting just the ones that are for the actual 'Name'
2791 # property. And create a fake swash from it.
2792 my %names;
2793 $names{'LIST'} = "";
2794 my $original = do "unicore/Name.pl";
62b3b855
KW
2795 my $algorithm_names = \@algorithmic_named_code_points;
2796
3b6a8189
KW
2797 # We need to remove the names from it that are aliases. For that
2798 # we need to also read in that table. Create a hash with the keys
2799 # being the code points, and the values being a list of the
2800 # aliases for the code point key.
2801 my ($aliases_code_points, $aliases_maps, undef, undef) =
2802 &prop_invmap('Name_Alias');
2803 my %aliases;
2804 for (my $i = 0; $i < @$aliases_code_points; $i++) {
2805 my $code_point = $aliases_code_points->[$i];
2806 $aliases{$code_point} = $aliases_maps->[$i];
2807
2808 # If not already a list, make it into one, so that later we
2809 # can treat things uniformly
2810 if (! ref $aliases{$code_point}) {
2811 $aliases{$code_point} = [ $aliases{$code_point} ];
2812 }
2813
2814 # Remove the alias type from the entry, retaining just the
2815 # name.
2816 map { s/:.*// } @{$aliases{$code_point}};
2817 }
2818
62b3b855
KW
2819 my $i = 0;
2820 foreach my $line (split "\n", $original) {
2821 my ($hex_code_point, $name) = split "\t", $line;
2822
2823 # Weeds out all comments, blank lines, and named sequences
df46a385 2824 next if $hex_code_point =~ /[^[:xdigit:]]/a;
62b3b855
KW
2825
2826 my $code_point = hex $hex_code_point;
2827
2828 # The name of all controls is the default: the empty string.
2829 # The set of controls is immutable, so these hard-coded
2830 # constants work.
2831 next if $code_point <= 0x9F
2832 && ($code_point <= 0x1F || $code_point >= 0x7F);
2833
3b6a8189
KW
2834 # If this is a name_alias, it isn't a name
2835 next if grep { $_ eq $name } @{$aliases{$code_point}};
62b3b855
KW
2836
2837 # If we are beyond where one of the special lines needs to
2838 # be inserted ...
3b6a8189 2839 while ($i < @$algorithm_names
62b3b855
KW
2840 && $code_point > $algorithm_names->[$i]->{'low'})
2841 {
2842
2843 # ... then insert it, ahead of what we were about to
2844 # output
3b6a8189 2845 $names{'LIST'} .= sprintf "%x\t%x\t%s\n",
62b3b855
KW
2846 $algorithm_names->[$i]->{'low'},
2847 $algorithm_names->[$i]->{'high'},
2848 $algorithm_names->[$i]->{'name'};
2849
62b3b855
KW
2850 # Done with this range.
2851 $i++;
2852
3b6a8189
KW
2853 # We loop until all special lines that precede the next
2854 # regular one are output.
62b3b855
KW
2855 }
2856
3b6a8189
KW
2857 # Here, is a normal name.
2858 $names{'LIST'} .= sprintf "%x\t\t%s\n", $code_point, $name;
2859 } # End of loop through all the names
62b3b855
KW
2860
2861 $names{'TYPE'} = "ToNa";
2862 $utf8::SwashInfo{ToNa}{'missing'} = "";
2863 $utf8::SwashInfo{ToNa}{'format'} = "n";
2864 $swash = \%names;
2865 }
2866 elsif ($second_try =~ / ^ ( d [mt] ) $ /x) {
2867
2868 # The file is a combination of dt and dm properties. Create a
2869 # fake swash from the portion that we want.
2870 my $original = do "unicore/Decomposition.pl";
2871 my %decomps;
2872
2873 if ($second_try eq 'dt') {
2874 $decomps{'TYPE'} = "ToDt";
2875 $utf8::SwashInfo{'ToDt'}{'missing'} = "None";
2876 $utf8::SwashInfo{'ToDt'}{'format'} = "s";
d11155ec 2877 } # 'dm' is handled below, with 'nfkccf'
62b3b855
KW
2878
2879 $decomps{'LIST'} = "";
2880
2881 # This property has one special range not in the file: for the
1a4c9760
KW
2882 # hangul syllables. But not in Unicode version 1.
2883 UnicodeVersion() unless defined $v_unicode_version;
2884 my $done_hangul = ($v_unicode_version lt v2.0.0)
2885 ? 1
2886 : 0; # Have we done the hangul range ?
62b3b855
KW
2887 foreach my $line (split "\n", $original) {
2888 my ($hex_lower, $hex_upper, $type_and_map) = split "\t", $line;
2889 my $code_point = hex $hex_lower;
2890 my $value;
bea2c146 2891 my $redo = 0;
62b3b855
KW
2892
2893 # The type, enclosed in <...>, precedes the mapping separated
2894 # by blanks
2895 if ($type_and_map =~ / ^ < ( .* ) > \s+ (.*) $ /x) {
2896 $value = ($second_try eq 'dt') ? $1 : $2
2897 }
2898 else { # If there is no type specified, it's canonical
2899 $value = ($second_try eq 'dt')
2900 ? "Canonical" :
2901 $type_and_map;
2902 }
2903
2904 # Insert the hangul range at the appropriate spot.
2905 if (! $done_hangul && $code_point > $HANGUL_BEGIN) {
2906 $done_hangul = 1;
2907 $decomps{'LIST'} .=
2908 sprintf "%x\t%x\t%s\n",
2909 $HANGUL_BEGIN,
2910 $HANGUL_BEGIN + $HANGUL_COUNT - 1,
2911 ($second_try eq 'dt')
2912 ? "Canonical"
2913 : "<hangul syllable>";
2914 }
2915
1a4c9760
KW
2916 if ($value =~ / / && $hex_upper ne "" && $hex_upper ne $hex_lower) {
2917 $line = sprintf("%04X\t%s\t%s", hex($hex_lower) + 1, $hex_upper, $value);
2918 $hex_upper = "";
2919 $redo = 1;
2920 }
2921
62b3b855
KW
2922 # And append this to our constructed LIST.
2923 $decomps{'LIST'} .= "$hex_lower\t$hex_upper\t$value\n";
bea2c146
KW
2924
2925 redo if $redo;
62b3b855
KW
2926 }
2927 $swash = \%decomps;
2928 }
d11155ec
KW
2929 elsif ($second_try ne 'nfkccf') { # Don't know this property. Fail.
2930 return;
2931 }
2932
2933 if ($second_try eq 'nfkccf' || $second_try eq 'dm') {
34132297 2934
d11155ec
KW
2935 # The 'nfkccf' property is stored in the old format for backwards
2936 # compatibility for any applications that has read its file
2937 # directly before prop_invmap() existed.
2938 # And the code above has extracted the 'dm' property from its file
2939 # yielding the same format. So here we convert them to adjusted
2940 # format for compatibility with the other properties similar to
2941 # them.
2942 my %revised_swash;
34132297 2943
d11155ec 2944 # We construct a new converted list.
34132297 2945 my $list = "";
d11155ec
KW
2946
2947 my @ranges = split "\n", $swash->{'LIST'};
2948 for (my $i = 0; $i < @ranges; $i++) {
2949 my ($hex_begin, $hex_end, $map) = split "\t", $ranges[$i];
2950
2951 # The dm property has maps that are space separated sequences
2952 # of code points, as well as the special entry "<hangul
2953 # syllable>, which also contains a blank.
2954 my @map = split " ", $map;
2955 if (@map > 1) {
2956
2957 # If it's just the special entry, append as-is.
2958 if ($map eq '<hangul syllable>') {
2959 $list .= "$ranges[$i]\n";
2960 }
2961 else {
2962
2963 # These should all single-element ranges.
1a4c9760 2964 croak __PACKAGE__, "::prop_invmap: Not expecting a mapping with multiple code points in a multi-element range, $ranges[$i]" if $hex_end ne "" && $hex_end ne $hex_begin;
d11155ec
KW
2965
2966 # Convert them to decimal, as that's what's expected.
2967 $list .= "$hex_begin\t\t"
2968 . join(" ", map { hex } @map)
2969 . "\n";
2970 }
2971 next;
2972 }
2973
2974 # Here, the mapping doesn't have a blank, is for a single code
2975 # point.
34132297
KW
2976 my $begin = hex $hex_begin;
2977 my $end = (defined $hex_end && $hex_end ne "")
2978 ? hex $hex_end
2979 : $begin;
d11155ec
KW
2980
2981 # Again, the output is to be in decimal.
34132297 2982 my $decimal_map = hex $map;
d11155ec
KW
2983
2984 # We know that multi-element ranges with the same mapping
2985 # should not be adjusted, as after the adjustment
2986 # multi-element ranges are for consecutive increasing code
2987 # points. Further, the final element in the list won't be
2988 # adjusted, as there is nothing after it to include in the
2989 # adjustment
2990 if ($begin != $end || $i == @ranges -1) {
2991
2992 # So just convert these to single-element ranges
2993 foreach my $code_point ($begin .. $end) {
2994 $list .= sprintf("%04X\t\t%d\n",
2995 $code_point, $decimal_map);
2996 }
34132297 2997 }
d11155ec 2998 else {
34132297 2999
d11155ec
KW
3000 # Here, we have a candidate for adjusting. What we do is
3001 # look through the subsequent adjacent elements in the
3002 # input. If the map to the next one differs by 1 from the
3003 # one before, then we combine into a larger range with the
3004 # initial map. Loop doing this until we find one that
3005 # can't be combined.
3006
3007 my $offset = 0; # How far away are we from the initial
3008 # map
3009 my $squished = 0; # ? Did we squish at least two
3010 # elements together into one range
3011 for ( ; $i < @ranges; $i++) {
3012 my ($next_hex_begin, $next_hex_end, $next_map)
3013 = split "\t", $ranges[$i+1];
3014
3015 # In the case of 'dm', the map may be a sequence of
3016 # multiple code points, which are never combined with
3017 # another range
3018 last if $next_map =~ / /;
3019
3020 $offset++;
3021 my $next_decimal_map = hex $next_map;
3022
3023 # If the next map is not next in sequence, it
3024 # shouldn't be combined.
3025 last if $next_decimal_map != $decimal_map + $offset;
3026
3027 my $next_begin = hex $next_hex_begin;
3028
3029 # Likewise, if the next element isn't adjacent to the
3030 # previous one, it shouldn't be combined.
3031 last if $next_begin != $begin + $offset;
3032
3033 my $next_end = (defined $next_hex_end
3034 && $next_hex_end ne "")
3035 ? hex $next_hex_end
3036 : $next_begin;
3037
3038 # And finally, if the next element is a multi-element
3039 # range, it shouldn't be combined.
3040 last if $next_end != $next_begin;
3041
3042 # Here, we will combine. Loop to see if we should
3043 # combine the next element too.
3044 $squished = 1;
3045 }
3046
3047 if ($squished) {
3048
3049 # Here, 'i' is the element number of the last element to
3050 # be combined, and the range is single-element, or we
3051 # wouldn't be combining. Get it's code point.
3052 my ($hex_end, undef, undef) = split "\t", $ranges[$i];
3053 $list .= "$hex_begin\t$hex_end\t$decimal_map\n";
3054 } else {
3055
3056 # Here, no combining done. Just appen the initial
3057 # (and current) values.
3058 $list .= "$hex_begin\t\t$decimal_map\n";
3059 }
3060 }
3061 } # End of loop constructing the converted list
3062
3063 # Finish up the data structure for our converted swash
3064 my $type = ($second_try eq 'nfkccf') ? 'ToNFKCCF' : 'ToDm';
3065 $revised_swash{'LIST'} = $list;
3066 $revised_swash{'TYPE'} = $type;
3067 $revised_swash{'SPECIALS'} = $swash->{'SPECIALS'};
3068 $swash = \%revised_swash;
3069
3070 $utf8::SwashInfo{$type}{'missing'} = 0;
3071 $utf8::SwashInfo{$type}{'format'} = 'a';
62b3b855
KW
3072 }
3073 }
3074
3075 if ($swash->{'EXTRAS'}) {
3076 carp __PACKAGE__, "::prop_invmap: swash returned for $prop unexpectedly has EXTRAS magic";
3077 return;
3078 }
3079
3080 # Here, have a valid swash return. Examine it.
34132297 3081 my $returned_prop = $swash->{'TYPE'};
62b3b855
KW
3082
3083 # All properties but binary ones should have 'missing' and 'format'
3084 # entries
3085 $missing = $utf8::SwashInfo{$returned_prop}{'missing'};
3086 $missing = 'N' unless defined $missing;
3087
3088 $format = $utf8::SwashInfo{$returned_prop}{'format'};
3089 $format = 'b' unless defined $format;
3090
d11155ec
KW
3091 my $requires_adjustment = $format =~ /^a/;
3092
62b3b855
KW
3093 # The LIST input lines look like:
3094 # ...
3095 # 0374\t\tCommon
3096 # 0375\t0377\tGreek # [3]
3097 # 037A\t037D\tGreek # [4]
3098 # 037E\t\tCommon
3099 # 0384\t\tGreek
3100 # ...
3101 #
3102 # Convert them to like
3103 # 0374 => Common
3104 # 0375 => Greek
3105 # 0378 => $missing
3106 # 037A => Greek
3107 # 037E => Common
3108 # 037F => $missing
3109 # 0384 => Greek
3110 #
3111 # For binary properties, the final non-comment column is absent, and
3112 # assumed to be 'Y'.
3113
3114 foreach my $range (split "\n", $swash->{'LIST'}) {
3115 $range =~ s/ \s* (?: \# .* )? $ //xg; # rmv trailing space, comments
3116
3117 # Find the beginning and end of the range on the line
3118 my ($hex_begin, $hex_end, $map) = split "\t", $range;
3119 my $begin = hex $hex_begin;
3120 my $end = (defined $hex_end && $hex_end ne "")
3121 ? hex $hex_end
3122 : $begin;
3123
92bcf67b
KW
3124 # Each time through the loop (after the first):
3125 # $invlist[-2] contains the beginning of the previous range processed
3126 # $invlist[-1] contains the end+1 of the previous range processed
3127 # $invmap[-2] contains the value of the previous range processed
3128 # $invmap[-1] contains the default value for missing ranges ($missing)
3129 #
3130 # Thus, things are set up for the typical case of a new non-adjacent
3131 # range of non-missings to be added. But, if the new range is
dc8d8ea6 3132 # adjacent, it needs to replace the [-1] element; and if the new
92bcf67b
KW
3133 # range is a multiple value of the previous one, it needs to be added
3134 # to the [-2] map element.
3135
3136 # The first time through, everything will be empty. If the property
3137 # doesn't have a range that begins at 0, add one that maps to $missing
62b3b855
KW
3138 if (! @invlist) {
3139 if ($begin != 0) {
3140 push @invlist, 0;
3141 push @invmap, $missing;
3142 }
3143 }
e35c6019
KW
3144 elsif (@invlist > 1 && $invlist[-2] == $begin) {
3145
3146 # Here we handle the case where the input has multiple entries for
3147 # each code point. mktables should have made sure that each such
3148 # range contains only one code point. At this point, $invlist[-1]
3149 # is the $missing that was added at the end of the last loop
3150 # iteration, and [-2] is the last real input code point, and that
3151 # code point is the same as the one we are adding now, making the
3152 # new one a multiple entry. Add it to the existing entry, either
3153 # by pushing it to the existing list of multiple entries, or
3154 # converting the single current entry into a list with both on it.
3155 # This is all we need do for this iteration.
3156
3157 if ($end != $begin) {
294705a8 3158 croak __PACKAGE__, ":prop_invmap: Multiple maps per code point in '$prop' require single-element ranges: begin=$begin, end=$end, map=$map";
e35c6019
KW
3159 }
3160 if (! ref $invmap[-2]) {
3161 $invmap[-2] = [ $invmap[-2], $map ];
3162 }
3163 else {
3164 push @{$invmap[-2]}, $map;
3165 }
3166 $has_multiples = 1;
3167 next;
3168 }
62b3b855
KW
3169 elsif ($invlist[-1] == $begin) {
3170
3171 # If the input isn't in the most compact form, so that there are
3172 # two adjacent ranges that map to the same thing, they should be
d11155ec
KW
3173 # combined (EXCEPT where the arrays require adjustments, in which
3174 # case everything is already set up correctly). This happens in
3175 # our constructed dt mapping, as Element [-2] is the map for the
3176 # latest range so far processed. Just set the beginning point of
3177 # the map to $missing (in invlist[-1]) to 1 beyond where this
3178 # range ends. For example, in
62b3b855
KW
3179 # 12\t13\tXYZ
3180 # 14\t17\tXYZ
3181 # we have set it up so that it looks like
3182 # 12 => XYZ
3183 # 14 => $missing
3184 #
3185 # We now see that it should be
3186 # 12 => XYZ
3187 # 18 => $missing
d11155ec 3188 if (! $requires_adjustment && @invlist > 1 && ( (defined $map)
c887f93f
KW
3189 ? $invmap[-2] eq $map
3190 : $invmap[-2] eq 'Y'))
3191 {
62b3b855
KW
3192 $invlist[-1] = $end + 1;
3193 next;
3194 }
3195
3196 # Here, the range started in the previous iteration that maps to
3197 # $missing starts at the same code point as this range. That
3198 # means there is no gap to fill that that range was intended for,
3199 # so we just pop it off the parallel arrays.
3200 pop @invlist;
3201 pop @invmap;
3202 }
3203
3204 # Add the range beginning, and the range's map.
3205 push @invlist, $begin;
d11155ec 3206 if ($returned_prop eq 'ToDm') {
62b3b855
KW
3207
3208 # The decomposition maps are either a line like <hangul syllable>
3209 # which are to be taken as is; or a sequence of code points in hex
3210 # and separated by blanks. Convert them to decimal, and if there
3211 # is more than one, use an anonymous array as the map.
3212 if ($map =~ /^ < /x) {
3213 push @invmap, $map;
3214 }
3215 else {
bea2c146 3216 my @map = split " ", $map;
62b3b855
KW
3217 if (@map == 1) {
3218 push @invmap, $map[0];
3219 }
3220 else {
3221 push @invmap, \@map;
3222 }
3223 }
3224 }
3225 else {
3226
3227 # Otherwise, convert hex formatted list entries to decimal; add a
3228 # 'Y' map for the missing value in binary properties, or
3229 # otherwise, use the input map unchanged.
3230 $map = ($format eq 'x')
3231 ? hex $map
3232 : $format eq 'b'
3233 ? 'Y'
3234 : $map;
3235 push @invmap, $map;
3236 }
3237
3238 # We just started a range. It ends with $end. The gap between it and
3239 # the next element in the list must be filled with a range that maps
3240 # to the default value. If there is no gap, the next iteration will
3241 # pop this, unless there is no next iteration, and we have filled all
3242 # of the Unicode code space, so check for that and skip.
3243 if ($end < $MAX_UNICODE_CODEPOINT) {
3244 push @invlist, $end + 1;
3245 push @invmap, $missing;
3246 }
3247 }
3248
3249 # If the property is empty, make all code points use the value for missing
3250 # ones.
3251 if (! @invlist) {
3252 push @invlist, 0;
3253 push @invmap, $missing;
3254 }
3255
647396da 3256 # And add in standard element that all non-Unicode code points map to:
62b3b855
KW
3257 # $missing
3258 push @invlist, $MAX_UNICODE_CODEPOINT + 1;
3259 push @invmap, $missing;
3260
3261 # The second component of the map are those values that require
3262 # non-standard specification, stored in SPECIALS. These override any
3263 # duplicate code points in LIST. If we are using a proxy, we may have
3264 # already set $overrides based on the proxy.
3265 $overrides = $swash->{'SPECIALS'} unless defined $overrides;
3266 if ($overrides) {
3267
3268 # A negative $overrides implies that the SPECIALS should be ignored,
d11155ec 3269 # and a simple 'a' list is the value.
62b3b855 3270 if ($overrides < 0) {
d11155ec 3271 $format = 'a';
62b3b855
KW
3272 }
3273 else {
3274
3275 # Currently, all overrides are for properties that normally map to
3276 # single code points, but now some will map to lists of code
3277 # points (but there is an exception case handled below).
d11155ec 3278 $format = 'al';
62b3b855
KW
3279
3280 # Look through the overrides.
3281 foreach my $cp_maybe_utf8 (keys %$overrides) {
3282 my $cp;
3283 my @map;
3284
3285 # If the overrides came from SPECIALS, the code point keys are
3286 # packed UTF-8.
3287 if ($overrides == $swash->{'SPECIALS'}) {
3288 $cp = unpack("C0U", $cp_maybe_utf8);
3289 @map = unpack "U0U*", $swash->{'SPECIALS'}{$cp_maybe_utf8};
3290
3291 # The empty string will show up unpacked as an empty
3292 # array.
d11155ec 3293 $format = 'ale' if @map == 0;
62b3b855
KW
3294 }
3295 else {
3296
3297 # But if we generated the overrides, we didn't bother to
3298 # pack them, and we, so far, do this only for properties
d11155ec 3299 # that are 'a' ones.
62b3b855
KW
3300 $cp = $cp_maybe_utf8;
3301 @map = hex $overrides->{$cp};
d11155ec 3302 $format = 'a';
62b3b855
KW
3303 }
3304
3305 # Find the range that the override applies to.
3306 my $i = _search_invlist(\@invlist, $cp);
3307 if ($cp < $invlist[$i] || $cp >= $invlist[$i + 1]) {
294705a8 3308 croak __PACKAGE__, "::prop_invmap: wrong_range, cp=$cp; i=$i, current=$invlist[$i]; next=$invlist[$i + 1]"
62b3b855
KW
3309 }
3310
3311 # And what that range currently maps to
3312 my $cur_map = $invmap[$i];
3313
3314 # If there is a gap between the next range and the code point
3315 # we are overriding, we have to add elements to both arrays to
3316 # fill that gap, using the map that applies to it, which is
3317 # $cur_map, since it is part of the current range.
3318 if ($invlist[$i + 1] > $cp + 1) {
3319 #use feature 'say';
3320 #say "Before splice:";
3321 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3322 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3323 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3324 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3325 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3326
3327 splice @invlist, $i + 1, 0, $cp + 1;
3328 splice @invmap, $i + 1, 0, $cur_map;
3329
3330 #say "After splice:";
3331 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3332 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3333 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3334 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3335 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3336 }
3337
3338 # If the remaining portion of the range is multiple code
3339 # points (ending with the one we are replacing, guaranteed by
3340 # the earlier splice). We must split it into two
3341 if ($invlist[$i] < $cp) {
3342 $i++; # Compensate for the new element
3343
3344 #use feature 'say';
3345 #say "Before splice:";
3346 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3347 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3348 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3349 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3350 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3351
3352 splice @invlist, $i, 0, $cp;
3353 splice @invmap, $i, 0, 'dummy';
3354
3355 #say "After splice:";
3356 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3357 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3358 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3359 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3360 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3361 }
3362
3363 # Here, the range we are overriding contains a single code
3364 # point. The result could be the empty string, a single
3365 # value, or a list. If the last case, we use an anonymous
3366 # array.
3367 $invmap[$i] = (scalar @map == 0)
3368 ? ""
3369 : (scalar @map > 1)
3370 ? \@map
3371 : $map[0];
3372 }
3373 }
3374 }
3375 elsif ($format eq 'x') {
3376
647396da
KW
3377 # All hex-valued properties are really to code points, and have been
3378 # converted to decimal.
5bbfa552 3379 $format = 's';
62b3b855 3380 }
d11155ec
KW
3381 elsif ($returned_prop eq 'ToDm') {
3382 $format = 'ad';
62b3b855
KW
3383 }
3384 elsif ($format eq 'sw') { # blank-separated elements to form a list.
3385 map { $_ = [ split " ", $_ ] if $_ =~ / / } @invmap;
3386 $format = 'sl';
3387 }
3388 elsif ($returned_prop eq 'ToNameAlias') {
3389
3390 # This property currently doesn't have any lists, but theoretically
3391 # could
3392 $format = 'sl';
3393 }
b0b13ada 3394 elsif ($returned_prop eq 'ToPerlDecimalDigit') {
d11155ec 3395 $format = 'ae';
b0b13ada 3396 }
4f143a72
KW
3397 elsif ($returned_prop eq 'ToNv') {
3398
3399 # The one property that has this format is stored as a delta, so needs
3400 # to indicate that need to add code point to it.
3401 $format = 'ar';
3402 }
b577d4a6 3403 elsif ($format ne 'n' && $format ne 'a') {
62b3b855
KW
3404
3405 # All others are simple scalars
3406 $format = 's';
3407 }
e35c6019 3408 if ($has_multiples && $format !~ /l/) {
294705a8 3409 croak __PACKAGE__, "::prop_invmap: Wrong format '$format' for prop_invmap('$prop'); should indicate has lists";
e35c6019 3410 }
62b3b855
KW
3411
3412 return (\@invlist, \@invmap, $format, $missing);
3413}
3414
55d7b906 3415=head2 Unicode::UCD::UnicodeVersion
10a6ecd2 3416
a452d459
KW
3417This returns the version of the Unicode Character Database, in other words, the
3418version of the Unicode standard the database implements. The version is a
3419string of numbers delimited by dots (C<'.'>).
10a6ecd2
JH
3420
3421=cut
3422
3423my $UNICODEVERSION;
3424
3425sub UnicodeVersion {
3426 unless (defined $UNICODEVERSION) {
3427 openunicode(\$VERSIONFH, "version");
ce066323 3428 local $/ = "\n";
10a6ecd2
JH
3429 chomp($UNICODEVERSION = <$VERSIONFH>);
3430 close($VERSIONFH);
3431 croak __PACKAGE__, "::VERSION: strange version '$UNICODEVERSION'"
3432 unless $UNICODEVERSION =~ /^\d+(?:\.\d+)+$/;
3433 }
e80c2d9d 3434 $v_unicode_version = pack "C*", split /\./, $UNICODEVERSION;
10a6ecd2
JH
3435 return $UNICODEVERSION;
3436}
3aa957f9 3437
a452d459
KW
3438=head2 B<Blocks versus Scripts>
3439
3440The difference between a block and a script is that scripts are closer
3441to the linguistic notion of a set of code points required to present
3442languages, while block is more of an artifact of the Unicode code point
3443numbering and separation into blocks of (mostly) 256 code points.
3444
3445For example the Latin B<script> is spread over several B<blocks>, such
3446as C<Basic Latin>, C<Latin 1 Supplement>, C<Latin Extended-A>, and
3447C<Latin Extended-B>. On the other hand, the Latin script does not
3448contain all the characters of the C<Basic Latin> block (also known as
3449ASCII): it includes only the letters, and not, for example, the digits
3450or the punctuation.
3451
3452For blocks see L<http://www.unicode.org/Public/UNIDATA/Blocks.txt>
3453
3454For scripts see UTR #24: L<http://www.unicode.org/unicode/reports/tr24/>
3455
3456=head2 B<Matching Scripts and Blocks>
3457
3458Scripts are matched with the regular-expression construct
3459C<\p{...}> (e.g. C<\p{Tibetan}> matches characters of the Tibetan script),
f200dd12 3460while C<\p{Blk=...}> is used for blocks (e.g. C<\p{Blk=Tibetan}> matches
a452d459
KW
3461any of the 256 code points in the Tibetan block).
3462
430fe03d
KW
3463=head2 Old-style versus new-style block names
3464
3465Unicode publishes the names of blocks in two different styles, though the two
3466are equivalent under Unicode's loose matching rules.
3467
3468The original style uses blanks and hyphens in the block names (except for
3469C<No_Block>), like so:
3470
3471 Miscellaneous Mathematical Symbols-B
3472
3473The newer style replaces these with underscores, like this:
3474
3475 Miscellaneous_Mathematical_Symbols_B
3476
3477This newer style is consistent with the values of other Unicode properties.
3478To preserve backward compatibility, all the functions in Unicode::UCD that
3479return block names (except one) return the old-style ones. That one function,
3480L</prop_value_aliases()> can be used to convert from old-style to new-style:
3481
3482 my $new_style = prop_values_aliases("block", $old_style);
3483
3484Perl also has single-form extensions that refer to blocks, C<In_Cyrillic>,
3485meaning C<Block=Cyrillic>. These have always been written in the new style.
3486
3487To convert from new-style to old-style, follow this recipe:
3488
3489 $old_style = charblock((prop_invlist("block=$new_style"))[0]);
3490
3491(which finds the range of code points in the block using C<prop_invlist>,
3492gets the lower end of the range (0th element) and then looks up the old name
3493for its block using C<charblock>).
3494
7620cb10
KW
3495Note that starting in Unicode 6.1, many of the block names have shorter
3496synonyms. These are always given in the new style.
3497
8b731da2
JH
3498=head1 BUGS
3499
3500Does not yet support EBCDIC platforms.
3501
561c79ed
JH
3502=head1 AUTHOR
3503
a18e976f 3504Jarkko Hietaniemi. Now maintained by perl5 porters.
561c79ed
JH
3505
3506=cut
3507
35081;