5 no warnings 'surrogate'; # surrogates can be inputs to this
7 use Unicode::Normalize qw(getCombinClass NFD);
11 use Storable qw(dclone);
15 our @ISA = qw(Exporter);
17 our @EXPORT_OK = qw(charinfo
19 charblocks charscripts
21 general_categories bidi_types
37 Unicode::UCD - Unicode character database
41 use Unicode::UCD 'charinfo';
42 my $charinfo = charinfo($codepoint);
44 use Unicode::UCD 'casefold';
45 my $casefold = casefold(0xFB00);
47 use Unicode::UCD 'casespec';
48 my $casespec = casespec(0xFB00);
50 use Unicode::UCD 'charblock';
51 my $charblock = charblock($codepoint);
53 use Unicode::UCD 'charscript';
54 my $charscript = charscript($codepoint);
56 use Unicode::UCD 'charblocks';
57 my $charblocks = charblocks();
59 use Unicode::UCD 'charscripts';
60 my $charscripts = charscripts();
62 use Unicode::UCD qw(charscript charinrange);
63 my $range = charscript($script);
64 print "looks like $script\n" if charinrange($range, $codepoint);
66 use Unicode::UCD qw(general_categories bidi_types);
67 my $categories = general_categories();
68 my $types = bidi_types();
70 use Unicode::UCD 'prop_aliases';
71 my @space_names = prop_aliases("space");
73 use Unicode::UCD 'prop_value_aliases';
74 my @gc_punct_names = prop_value_aliases("Gc", "Punct");
76 use Unicode::UCD 'prop_invlist';
77 my @puncts = prop_invlist("gc=punctuation");
79 use Unicode::UCD 'prop_invmap';
80 my ($list_ref, $map_ref, $format, $missing)
81 = prop_invmap("General Category");
83 use Unicode::UCD 'compexcl';
84 my $compexcl = compexcl($codepoint);
86 use Unicode::UCD 'namedseq';
87 my $namedseq = namedseq($named_sequence_name);
89 my $unicode_version = Unicode::UCD::UnicodeVersion();
91 my $convert_to_numeric =
92 Unicode::UCD::num("\N{RUMI DIGIT ONE}\N{RUMI DIGIT TWO}");
96 The Unicode::UCD module offers a series of functions that
97 provide a simple interface to the Unicode
100 =head2 code point argument
102 Some of the functions are called with a I<code point argument>, which is either
103 a decimal or a hexadecimal scalar designating a Unicode code point, or C<U+>
104 followed by hexadecimals designating a Unicode code point. In other words, if
105 you want a code point to be interpreted as a hexadecimal number, you must
106 prefix it with either C<0x> or C<U+>, because a string like e.g. C<123> will be
107 interpreted as a decimal code point. Note that the largest code point in
117 my $v_unicode_version; # v-string.
120 my ($rfh, @path) = @_;
122 unless (defined $$rfh) {
125 $f = File::Spec->catfile($d, "unicore", @path);
126 last if open($$rfh, $f);
129 croak __PACKAGE__, ": failed to find ",
130 File::Spec->catfile(@path), " in @INC"
138 use Unicode::UCD 'charinfo';
140 my $charinfo = charinfo(0x41);
142 This returns information about the input L</code point argument>
143 as a reference to a hash of fields as defined by the Unicode
144 standard. If the L</code point argument> is not assigned in the standard
145 (i.e., has the general category C<Cn> meaning C<Unassigned>)
146 or is a non-character (meaning it is guaranteed to never be assigned in
148 C<undef> is returned.
150 Fields that aren't applicable to the particular code point argument exist in the
151 returned hash, and are empty.
153 The keys in the hash with the meanings of their values are:
159 the input L</code point argument> expressed in hexadecimal, with leading zeros
160 added if necessary to make it contain at least four hexdigits
164 name of I<code>, all IN UPPER CASE.
165 Some control-type code points do not have names.
166 This field will be empty for C<Surrogate> and C<Private Use> code points,
167 and for the others without a name,
168 it will contain a description enclosed in angle brackets, like
169 C<E<lt>controlE<gt>>.
174 The short name of the general category of I<code>.
175 This will match one of the keys in the hash returned by L</general_categories()>.
177 The L</prop_value_aliases()> function can be used to get all the synonyms
178 of the category name.
182 the combining class number for I<code> used in the Canonical Ordering Algorithm.
183 For Unicode 5.1, this is described in Section 3.11 C<Canonical Ordering Behavior>
185 L<http://www.unicode.org/versions/Unicode5.1.0/>
187 The L</prop_value_aliases()> function can be used to get all the synonyms
188 of the combining class number.
192 bidirectional type of I<code>.
193 This will match one of the keys in the hash returned by L</bidi_types()>.
195 The L</prop_value_aliases()> function can be used to get all the synonyms
196 of the bidi type name.
198 =item B<decomposition>
200 is empty if I<code> has no decomposition; or is one or more codes
201 (separated by spaces) that, taken in order, represent a decomposition for
202 I<code>. Each has at least four hexdigits.
203 The codes may be preceded by a word enclosed in angle brackets then a space,
204 like C<E<lt>compatE<gt> >, giving the type of decomposition
206 This decomposition may be an intermediate one whose components are also
207 decomposable. Use L<Unicode::Normalize> to get the final decomposition.
211 if I<code> is a decimal digit this is its integer numeric value
215 if I<code> represents some other digit-like number, this is its integer
220 if I<code> represents a whole or rational number, this is its numeric value.
221 Rational values are expressed as a string like C<1/4>.
225 C<Y> or C<N> designating if I<code> is mirrored in bidirectional text
229 name of I<code> in the Unicode 1.0 standard if one
230 existed for this code point and is different from the current name
234 As of Unicode 6.0, this is always empty.
238 is empty if there is no single code point uppercase mapping for I<code>
239 (its uppercase mapping is itself);
240 otherwise it is that mapping expressed as at least four hexdigits.
241 (L</casespec()> should be used in addition to B<charinfo()>
242 for case mappings when the calling program can cope with multiple code point
247 is empty if there is no single code point lowercase mapping for I<code>
248 (its lowercase mapping is itself);
249 otherwise it is that mapping expressed as at least four hexdigits.
250 (L</casespec()> should be used in addition to B<charinfo()>
251 for case mappings when the calling program can cope with multiple code point
256 is empty if there is no single code point titlecase mapping for I<code>
257 (its titlecase mapping is itself);
258 otherwise it is that mapping expressed as at least four hexdigits.
259 (L</casespec()> should be used in addition to B<charinfo()>
260 for case mappings when the calling program can cope with multiple code point
265 the block I<code> belongs to (used in C<\p{Blk=...}>).
266 See L</Blocks versus Scripts>.
271 the script I<code> belongs to.
272 See L</Blocks versus Scripts>.
276 Note that you cannot do (de)composition and casing based solely on the
277 I<decomposition>, I<combining>, I<lower>, I<upper>, and I<title> fields;
278 you will need also the L</compexcl()>, and L</casespec()> functions.
282 # NB: This function is nearly duplicated in charnames.pm
286 if ($arg =~ /^[1-9]\d*$/) {
288 } elsif ($arg =~ /^(?:[Uu]\+|0[xX])?([[:xdigit:]]+)$/) {
295 # Populated by _num. Converts real number back to input rational
296 my %real_to_rational;
298 # To store the contents of files found on disk.
311 # This function has traditionally mimicked what is in UnicodeData.txt,
312 # warts and all. This is a re-write that avoids UnicodeData.txt so that
313 # it can be removed to save disk space. Instead, this assembles
314 # information gotten by other methods that get data from various other
315 # files. It uses charnames to get the character name; and various
318 use feature 'unicode_strings';
321 my $code = _getcode($arg);
322 croak __PACKAGE__, "::charinfo: unknown code '$arg'" unless defined $code;
324 # Non-unicode implies undef.
325 return if $code > 0x10FFFF;
328 my $char = chr($code);
330 @CATEGORIES =_read_table("To/Gc.pl") unless @CATEGORIES;
331 $prop{'category'} = _search(\@CATEGORIES, 0, $#CATEGORIES, $code)
332 // $utf8::SwashInfo{'ToGc'}{'missing'};
334 return if $prop{'category'} eq 'Cn'; # Unassigned code points are undef
336 $prop{'code'} = sprintf "%04X", $code;
337 $prop{'name'} = ($char =~ /\p{Cntrl}/) ? '<control>'
338 : (charnames::viacode($code) // "");
340 $prop{'combining'} = getCombinClass($code);
342 @BIDIS =_read_table("To/Bc.pl") unless @BIDIS;
343 $prop{'bidi'} = _search(\@BIDIS, 0, $#BIDIS, $code)
344 // $utf8::SwashInfo{'ToBc'}{'missing'};
346 # For most code points, we can just read in "unicore/Decomposition.pl", as
347 # its contents are exactly what should be output. But that file doesn't
348 # contain the data for the Hangul syllable decompositions, which can be
349 # algorithmically computed, and NFD() does that, so we call NFD() for
350 # those. We can't use NFD() for everything, as it does a complete
351 # recursive decomposition, and what this function has always done is to
352 # return what's in UnicodeData.txt which doesn't show that recursiveness.
353 # Fortunately, the NFD() of the Hanguls doesn't have any recursion
355 # Having no decomposition implies an empty field; otherwise, all but
356 # "Canonical" imply a compatible decomposition, and the type is prefixed
357 # to that, as it is in UnicodeData.txt
358 UnicodeVersion() unless defined $v_unicode_version;
359 if ($v_unicode_version ge v2.0.0 && $char =~ /\p{Block=Hangul_Syllables}/) {
360 # The code points of the decomposition are output in standard Unicode
361 # hex format, separated by blanks.
362 $prop{'decomposition'} = join " ", map { sprintf("%04X", $_)}
363 unpack "U*", NFD($char);
366 @DECOMPOSITIONS = _read_table("Decomposition.pl")
367 unless @DECOMPOSITIONS;
368 $prop{'decomposition'} = _search(\@DECOMPOSITIONS, 0, $#DECOMPOSITIONS,
372 # Can use num() to get the numeric values, if any.
373 if (! defined (my $value = num($char))) {
374 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = "";
378 $prop{'decimal'} = $prop{'digit'} = $prop{'numeric'} = $value;
382 # For non-decimal-digits, we have to read in the Numeric type
383 # to distinguish them. It is not just a matter of integer vs.
384 # rational, as some whole number values are not considered digits,
385 # e.g., TAMIL NUMBER TEN.
386 $prop{'decimal'} = "";
388 @NUMERIC_TYPES =_read_table("To/Nt.pl") unless @NUMERIC_TYPES;
389 if ((_search(\@NUMERIC_TYPES, 0, $#NUMERIC_TYPES, $code) // "")
392 $prop{'digit'} = $prop{'numeric'} = $value;
396 $prop{'numeric'} = $real_to_rational{$value} // $value;
401 $prop{'mirrored'} = ($char =~ /\p{Bidi_Mirrored}/) ? 'Y' : 'N';
403 %UNICODE_1_NAMES =_read_table("To/Na1.pl", "use_hash") unless %UNICODE_1_NAMES;
404 $prop{'unicode10'} = $UNICODE_1_NAMES{$code} // "";
406 UnicodeVersion() unless defined $v_unicode_version;
407 if ($v_unicode_version ge v6.0.0) {
408 $prop{'comment'} = "";
411 %ISO_COMMENT = _read_table("To/Isc.pl", "use_hash") unless %ISO_COMMENT;
412 $prop{'comment'} = (defined $ISO_COMMENT{$code})
413 ? $ISO_COMMENT{$code}
417 %SIMPLE_UPPER = _read_table("To/Uc.pl", "use_hash") unless %SIMPLE_UPPER;
418 $prop{'upper'} = (defined $SIMPLE_UPPER{$code})
419 ? sprintf("%04X", $SIMPLE_UPPER{$code})
422 %SIMPLE_LOWER = _read_table("To/Lc.pl", "use_hash") unless %SIMPLE_LOWER;
423 $prop{'lower'} = (defined $SIMPLE_LOWER{$code})
424 ? sprintf("%04X", $SIMPLE_LOWER{$code})
427 %SIMPLE_TITLE = _read_table("To/Tc.pl", "use_hash") unless %SIMPLE_TITLE;
428 $prop{'title'} = (defined $SIMPLE_TITLE{$code})
429 ? sprintf("%04X", $SIMPLE_TITLE{$code})
432 $prop{block} = charblock($code);
433 $prop{script} = charscript($code);
437 sub _search { # Binary search in a [[lo,hi,prop],[...],...] table.
438 my ($table, $lo, $hi, $code) = @_;
442 my $mid = int(($lo+$hi) / 2);
444 if ($table->[$mid]->[0] < $code) {
445 if ($table->[$mid]->[1] >= $code) {
446 return $table->[$mid]->[2];
448 _search($table, $mid + 1, $hi, $code);
450 } elsif ($table->[$mid]->[0] > $code) {
451 _search($table, $lo, $mid - 1, $code);
453 return $table->[$mid]->[2];
457 sub _read_table ($;$) {
459 # Returns the contents of the mktables generated table file located at $1
460 # in the form of either an array of arrays or a hash, depending on if the
461 # optional second parameter is true (for hash return) or not. In the case
462 # of a hash return, each key is a code point, and its corresponding value
463 # is what the table gives as the code point's corresponding value. In the
464 # case of an array return, each outer array denotes a range with [0] the
465 # start point of that range; [1] the end point; and [2] the value that
466 # every code point in the range has. The hash return is useful for fast
467 # lookup when the table contains only single code point ranges. The array
468 # return takes much less memory when there are large ranges.
470 # This function has the side effect of setting
471 # $utf8::SwashInfo{$property}{'format'} to be the mktables format of the
473 # $utf8::SwashInfo{$property}{'missing'} to be the value for all entries
474 # not listed in the table.
475 # where $property is the Unicode property name, preceded by 'To' for map
476 # properties., e.g., 'ToSc'.
478 # Table entries look like one of:
479 # 0000 0040 Common # [65]
483 my $return_hash = shift;
484 $return_hash = 0 unless defined $return_hash;
488 my $list = do "unicore/$table";
490 # Look up if this property requires adjustments, which we do below if it
492 require "unicore/Heavy.pl";
493 my $property = $table =~ s/\.pl//r;
494 $property = $utf8::file_to_swash_name{$property};
495 my $to_adjust = defined $property
496 && $utf8::SwashInfo{$property}{'format'} eq 'a';
498 for (split /^/m, $list) {
499 my ($start, $end, $value) = / ^ (.+?) \t (.*?) \t (.+?)
500 \s* ( \# .* )? # Optional comment
502 my $decimal_start = hex $start;
503 my $decimal_end = ($end eq "") ? $decimal_start : hex $end;
505 foreach my $i ($decimal_start .. $decimal_end) {
506 $return{$i} = ($to_adjust)
507 ? $value + $i - $decimal_start
513 && $return[-1][1] == $decimal_start - 1
514 && $return[-1][2] eq $value)
516 # If this is merely extending the previous range, do just that.
517 $return[-1]->[1] = $decimal_end;
520 push @return, [ $decimal_start, $decimal_end, $value ];
523 return ($return_hash) ? %return : @return;
527 my ($range, $arg) = @_;
528 my $code = _getcode($arg);
529 croak __PACKAGE__, "::charinrange: unknown code '$arg'"
530 unless defined $code;
531 _search($range, 0, $#$range, $code);
534 =head2 B<charblock()>
536 use Unicode::UCD 'charblock';
538 my $charblock = charblock(0x41);
539 my $charblock = charblock(1234);
540 my $charblock = charblock(0x263a);
541 my $charblock = charblock("U+263a");
543 my $range = charblock('Armenian');
545 With a L</code point argument> charblock() returns the I<block> the code point
546 belongs to, e.g. C<Basic Latin>. The old-style block name is returned (see
547 L</Old-style versus new-style block names>).
548 If the code point is unassigned, this returns the block it would belong to if
549 it were assigned. (If the Unicode version being used is so early as to not
550 have blocks, all code points are considered to be in C<No_Block>.)
552 See also L</Blocks versus Scripts>.
554 If supplied with an argument that can't be a code point, charblock() tries to
555 do the opposite and interpret the argument as an old-style block name. The
557 is a I<range set> with one range: an anonymous list with a single element that
558 consists of another anonymous list whose first element is the first code point
559 in the block, and whose second (and final) element is the final code point in
560 the block. (The extra list consisting of just one element is so that the same
561 program logic can be used to handle both this return, and the return from
562 L</charscript()> which can have multiple ranges.) You can test whether a code
563 point is in a range using the L</charinrange()> function. If the argument is
564 not a known block, C<undef> is returned.
573 # Can't read from the mktables table because it loses the hyphens in the
576 UnicodeVersion() unless defined $v_unicode_version;
577 if ($v_unicode_version lt v2.0.0) {
578 my $subrange = [ 0, 0x10FFFF, 'No_Block' ];
579 push @BLOCKS, $subrange;
580 push @{$BLOCKS{$3}}, $subrange;
582 elsif (openunicode(\$BLOCKSFH, "Blocks.txt")) {
585 while (<$BLOCKSFH>) {
586 if (/^([0-9A-F]+)\.\.([0-9A-F]+);\s+(.+)/) {
587 my ($lo, $hi) = (hex($1), hex($2));
588 my $subrange = [ $lo, $hi, $3 ];
589 push @BLOCKS, $subrange;
590 push @{$BLOCKS{$3}}, $subrange;
601 _charblocks() unless @BLOCKS;
603 my $code = _getcode($arg);
606 my $result = _search(\@BLOCKS, 0, $#BLOCKS, $code);
607 return $result if defined $result;
610 elsif (exists $BLOCKS{$arg}) {
611 return dclone $BLOCKS{$arg};
615 =head2 B<charscript()>
617 use Unicode::UCD 'charscript';
619 my $charscript = charscript(0x41);
620 my $charscript = charscript(1234);
621 my $charscript = charscript("U+263a");
623 my $range = charscript('Thai');
625 With a L</code point argument> charscript() returns the I<script> the
626 code point belongs to, e.g. C<Latin>, C<Greek>, C<Han>.
627 If the code point is unassigned or the Unicode version being used is so early
628 that it doesn't have scripts, this function returns C<"Unknown">.
630 If supplied with an argument that can't be a code point, charscript() tries
631 to do the opposite and interpret the argument as a script name. The
632 return value is a I<range set>: an anonymous list of lists that contain
633 I<start-of-range>, I<end-of-range> code point pairs. You can test whether a
634 code point is in a range set using the L</charinrange()> function. If the
635 argument is not a known script, C<undef> is returned.
637 See also L</Blocks versus Scripts>.
646 UnicodeVersion() unless defined $v_unicode_version;
647 if ($v_unicode_version lt v3.1.0) {
648 push @SCRIPTS, [ 0, 0x10FFFF, 'Unknown' ];
651 @SCRIPTS =_read_table("To/Sc.pl");
654 foreach my $entry (@SCRIPTS) {
655 $entry->[2] =~ s/(_\w)/\L$1/g; # Preserve old-style casing
656 push @{$SCRIPTS{$entry->[2]}}, $entry;
663 _charscripts() unless @SCRIPTS;
665 my $code = _getcode($arg);
668 my $result = _search(\@SCRIPTS, 0, $#SCRIPTS, $code);
669 return $result if defined $result;
670 return $utf8::SwashInfo{'ToSc'}{'missing'};
671 } elsif (exists $SCRIPTS{$arg}) {
672 return dclone $SCRIPTS{$arg};
678 =head2 B<charblocks()>
680 use Unicode::UCD 'charblocks';
682 my $charblocks = charblocks();
684 charblocks() returns a reference to a hash with the known block names
685 as the keys, and the code point ranges (see L</charblock()>) as the values.
687 The names are in the old-style (see L</Old-style versus new-style block
690 L<prop_invmap("block")|/prop_invmap()> can be used to get this same data in a
691 different type of data structure.
693 See also L</Blocks versus Scripts>.
698 _charblocks() unless %BLOCKS;
699 return dclone \%BLOCKS;
702 =head2 B<charscripts()>
704 use Unicode::UCD 'charscripts';
706 my $charscripts = charscripts();
708 charscripts() returns a reference to a hash with the known script
709 names as the keys, and the code point ranges (see L</charscript()>) as
712 L<prop_invmap("script")|/prop_invmap()> can be used to get this same data in a
713 different type of data structure.
715 See also L</Blocks versus Scripts>.
720 _charscripts() unless %SCRIPTS;
721 return dclone \%SCRIPTS;
724 =head2 B<charinrange()>
726 In addition to using the C<\p{Blk=...}> and C<\P{Blk=...}> constructs, you
727 can also test whether a code point is in the I<range> as returned by
728 L</charblock()> and L</charscript()> or as the values of the hash returned
729 by L</charblocks()> and L</charscripts()> by using charinrange():
731 use Unicode::UCD qw(charscript charinrange);
733 $range = charscript('Hiragana');
734 print "looks like hiragana\n" if charinrange($range, $codepoint);
738 my %GENERAL_CATEGORIES =
741 'LC' => 'CasedLetter',
742 'Lu' => 'UppercaseLetter',
743 'Ll' => 'LowercaseLetter',
744 'Lt' => 'TitlecaseLetter',
745 'Lm' => 'ModifierLetter',
746 'Lo' => 'OtherLetter',
748 'Mn' => 'NonspacingMark',
749 'Mc' => 'SpacingMark',
750 'Me' => 'EnclosingMark',
752 'Nd' => 'DecimalNumber',
753 'Nl' => 'LetterNumber',
754 'No' => 'OtherNumber',
755 'P' => 'Punctuation',
756 'Pc' => 'ConnectorPunctuation',
757 'Pd' => 'DashPunctuation',
758 'Ps' => 'OpenPunctuation',
759 'Pe' => 'ClosePunctuation',
760 'Pi' => 'InitialPunctuation',
761 'Pf' => 'FinalPunctuation',
762 'Po' => 'OtherPunctuation',
764 'Sm' => 'MathSymbol',
765 'Sc' => 'CurrencySymbol',
766 'Sk' => 'ModifierSymbol',
767 'So' => 'OtherSymbol',
769 'Zs' => 'SpaceSeparator',
770 'Zl' => 'LineSeparator',
771 'Zp' => 'ParagraphSeparator',
776 'Co' => 'PrivateUse',
777 'Cn' => 'Unassigned',
780 sub general_categories {
781 return dclone \%GENERAL_CATEGORIES;
784 =head2 B<general_categories()>
786 use Unicode::UCD 'general_categories';
788 my $categories = general_categories();
790 This returns a reference to a hash which has short
791 general category names (such as C<Lu>, C<Nd>, C<Zs>, C<S>) as keys and long
792 names (such as C<UppercaseLetter>, C<DecimalNumber>, C<SpaceSeparator>,
793 C<Symbol>) as values. The hash is reversible in case you need to go
794 from the long names to the short names. The general category is the
796 L</charinfo()> under the C<category> key.
798 The L</prop_value_aliases()> function can be used to get all the synonyms of
805 'L' => 'Left-to-Right',
806 'LRE' => 'Left-to-Right Embedding',
807 'LRO' => 'Left-to-Right Override',
808 'R' => 'Right-to-Left',
809 'AL' => 'Right-to-Left Arabic',
810 'RLE' => 'Right-to-Left Embedding',
811 'RLO' => 'Right-to-Left Override',
812 'PDF' => 'Pop Directional Format',
813 'EN' => 'European Number',
814 'ES' => 'European Number Separator',
815 'ET' => 'European Number Terminator',
816 'AN' => 'Arabic Number',
817 'CS' => 'Common Number Separator',
818 'NSM' => 'Non-Spacing Mark',
819 'BN' => 'Boundary Neutral',
820 'B' => 'Paragraph Separator',
821 'S' => 'Segment Separator',
822 'WS' => 'Whitespace',
823 'ON' => 'Other Neutrals',
826 =head2 B<bidi_types()>
828 use Unicode::UCD 'bidi_types';
830 my $categories = bidi_types();
832 This returns a reference to a hash which has the short
833 bidi (bidirectional) type names (such as C<L>, C<R>) as keys and long
834 names (such as C<Left-to-Right>, C<Right-to-Left>) as values. The
835 hash is reversible in case you need to go from the long names to the
836 short names. The bidi type is the one returned from
838 under the C<bidi> key. For the exact meaning of the various bidi classes
839 the Unicode TR9 is recommended reading:
840 L<http://www.unicode.org/reports/tr9/>
841 (as of Unicode 5.0.0)
843 The L</prop_value_aliases()> function can be used to get all the synonyms of
849 return dclone \%BIDI_TYPES;
854 use Unicode::UCD 'compexcl';
856 my $compexcl = compexcl(0x09dc);
858 This routine returns C<undef> if the Unicode version being used is so early
859 that it doesn't have this property. It is included for backwards
860 compatibility, but as of Perl 5.12 and more modern Unicode versions, for
861 most purposes it is probably more convenient to use one of the following
864 my $compexcl = chr(0x09dc) =~ /\p{Comp_Ex};
865 my $compexcl = chr(0x09dc) =~ /\p{Full_Composition_Exclusion};
869 my $compexcl = chr(0x09dc) =~ /\p{CE};
870 my $compexcl = chr(0x09dc) =~ /\p{Composition_Exclusion};
872 The first two forms return B<true> if the L</code point argument> should not
873 be produced by composition normalization. For the final two forms to return
874 B<true>, it is additionally required that this fact not otherwise be
875 determinable from the Unicode data base.
877 This routine behaves identically to the final two forms. That is,
878 it does not return B<true> if the code point has a decomposition
879 consisting of another single code point, nor if its decomposition starts
880 with a code point whose combining class is non-zero. Code points that meet
881 either of these conditions should also not be produced by composition
882 normalization, which is probably why you should use the
883 C<Full_Composition_Exclusion> property instead, as shown above.
885 The routine returns B<false> otherwise.
891 my $code = _getcode($arg);
892 croak __PACKAGE__, "::compexcl: unknown code '$arg'"
893 unless defined $code;
895 UnicodeVersion() unless defined $v_unicode_version;
896 return if $v_unicode_version lt v3.0.0;
898 no warnings "non_unicode"; # So works on non-Unicode code points
899 return chr($code) =~ /\p{Composition_Exclusion}/;
904 use Unicode::UCD 'casefold';
906 my $casefold = casefold(0xDF);
907 if (defined $casefold) {
908 my @full_fold_hex = split / /, $casefold->{'full'};
909 my $full_fold_string =
910 join "", map {chr(hex($_))} @full_fold_hex;
911 my @turkic_fold_hex =
912 split / /, ($casefold->{'turkic'} ne "")
913 ? $casefold->{'turkic'}
914 : $casefold->{'full'};
915 my $turkic_fold_string =
916 join "", map {chr(hex($_))} @turkic_fold_hex;
918 if (defined $casefold && $casefold->{'simple'} ne "") {
919 my $simple_fold_hex = $casefold->{'simple'};
920 my $simple_fold_string = chr(hex($simple_fold_hex));
923 This returns the (almost) locale-independent case folding of the
924 character specified by the L</code point argument>. (Starting in Perl v5.16,
925 the core function C<fc()> returns the C<full> mapping (described below)
926 faster than this does, and for entire strings.)
928 If there is no case folding for the input code point, C<undef> is returned.
930 If there is a case folding for that code point, a reference to a hash
931 with the following fields is returned:
937 the input L</code point argument> expressed in hexadecimal, with leading zeros
938 added if necessary to make it contain at least four hexdigits
942 one or more codes (separated by spaces) that, taken in order, give the
943 code points for the case folding for I<code>.
944 Each has at least four hexdigits.
948 is empty, or is exactly one code with at least four hexdigits which can be used
949 as an alternative case folding when the calling program cannot cope with the
950 fold being a sequence of multiple code points. If I<full> is just one code
951 point, then I<simple> equals I<full>. If there is no single code point folding
952 defined for I<code>, then I<simple> is the empty string. Otherwise, it is an
953 inferior, but still better-than-nothing alternative folding to I<full>.
957 is the same as I<simple> if I<simple> is not empty, and it is the same as I<full>
958 otherwise. It can be considered to be the simplest possible folding for
959 I<code>. It is defined primarily for backwards compatibility.
963 is C<C> (for C<common>) if the best possible fold is a single code point
964 (I<simple> equals I<full> equals I<mapping>). It is C<S> if there are distinct
965 folds, I<simple> and I<full> (I<mapping> equals I<simple>). And it is C<F> if
966 there is only a I<full> fold (I<mapping> equals I<full>; I<simple> is empty).
968 describes the contents of I<mapping>. It is defined primarily for backwards
971 For Unicode versions between 3.1 and 3.1.1 inclusive, I<status> can also be
972 C<I> which is the same as C<C> but is a special case for dotted uppercase I and
977 =item B<*> If you use this C<I> mapping
979 the result is case-insensitive,
980 but dotless and dotted I's are not distinguished
982 =item B<*> If you exclude this C<I> mapping
984 the result is not fully case-insensitive, but
985 dotless and dotted I's are distinguished
991 contains any special folding for Turkic languages. For versions of Unicode
992 starting with 3.2, this field is empty unless I<code> has a different folding
993 in Turkic languages, in which case it is one or more codes (separated by
994 spaces) that, taken in order, give the code points for the case folding for
995 I<code> in those languages.
996 Each code has at least four hexdigits.
997 Note that this folding does not maintain canonical equivalence without
998 additional processing.
1000 For Unicode versions between 3.1 and 3.1.1 inclusive, this field is empty unless
1002 special folding for Turkic languages, in which case I<status> is C<I>, and
1003 I<mapping>, I<full>, I<simple>, and I<turkic> are all equal.
1007 Programs that want complete generality and the best folding results should use
1008 the folding contained in the I<full> field. But note that the fold for some
1009 code points will be a sequence of multiple code points.
1011 Programs that can't cope with the fold mapping being multiple code points can
1012 use the folding contained in the I<simple> field, with the loss of some
1013 generality. In Unicode 5.1, about 7% of the defined foldings have no single
1016 The I<mapping> and I<status> fields are provided for backwards compatibility for
1017 existing programs. They contain the same values as in previous versions of
1020 Locale is not completely independent. The I<turkic> field contains results to
1021 use when the locale is a Turkic language.
1023 For more information about case mappings see
1024 L<http://www.unicode.org/unicode/reports/tr21>
1031 unless (%CASEFOLD) { # Populate the hash
1032 my ($full_invlist_ref, $full_invmap_ref, undef, $default)
1033 = prop_invmap('Case_Folding');
1035 # Use the recipe given in the prop_invmap() pod to convert the
1036 # inversion map into the hash.
1037 for my $i (0 .. @$full_invlist_ref - 1 - 1) {
1038 next if $full_invmap_ref->[$i] == $default;
1040 for my $j ($full_invlist_ref->[$i] .. $full_invlist_ref->[$i+1] -1) {
1042 if (! ref $full_invmap_ref->[$i]) {
1044 # This is a single character mapping
1045 $CASEFOLD{$j}{'status'} = 'C';
1046 $CASEFOLD{$j}{'simple'}
1047 = $CASEFOLD{$j}{'full'}
1048 = $CASEFOLD{$j}{'mapping'}
1049 = sprintf("%04X", $full_invmap_ref->[$i] + $adjust);
1050 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1051 $CASEFOLD{$j}{'turkic'} = "";
1053 else { # prop_invmap ensures that $adjust is 0 for a ref
1054 $CASEFOLD{$j}{'status'} = 'F';
1055 $CASEFOLD{$j}{'full'}
1056 = $CASEFOLD{$j}{'mapping'}
1057 = join " ", map { sprintf "%04X", $_ }
1058 @{$full_invmap_ref->[$i]};
1059 $CASEFOLD{$j}{'simple'} = "";
1060 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1061 $CASEFOLD{$j}{'turkic'} = "";
1066 # We have filled in the full mappings above, assuming there were no
1067 # simple ones for the ones with multi-character maps. Now, we find
1068 # and fix the cases where that assumption was false.
1069 (my ($simple_invlist_ref, $simple_invmap_ref, undef), $default)
1070 = prop_invmap('Simple_Case_Folding');
1071 for my $i (0 .. @$simple_invlist_ref - 1 - 1) {
1072 next if $simple_invmap_ref->[$i] == $default;
1074 for my $j ($simple_invlist_ref->[$i]
1075 .. $simple_invlist_ref->[$i+1] -1)
1078 next if $CASEFOLD{$j}{'status'} eq 'C';
1079 $CASEFOLD{$j}{'status'} = 'S';
1080 $CASEFOLD{$j}{'simple'}
1081 = $CASEFOLD{$j}{'mapping'}
1082 = sprintf("%04X", $simple_invmap_ref->[$i] + $adjust);
1083 $CASEFOLD{$j}{'code'} = sprintf("%04X", $j);
1084 $CASEFOLD{$j}{'turkic'} = "";
1088 # We hard-code in the turkish rules
1089 UnicodeVersion() unless defined $v_unicode_version;
1090 if ($v_unicode_version ge v3.2.0) {
1092 # These two code points should already have regular entries, so
1093 # just fill in the turkish fields
1094 $CASEFOLD{ord('I')}{'turkic'} = '0131';
1095 $CASEFOLD{0x130}{'turkic'} = sprintf "%04X", ord('i');
1097 elsif ($v_unicode_version ge v3.1.0) {
1099 # These two code points don't have entries otherwise.
1100 $CASEFOLD{0x130}{'code'} = '0130';
1101 $CASEFOLD{0x131}{'code'} = '0131';
1102 $CASEFOLD{0x130}{'status'} = $CASEFOLD{0x131}{'status'} = 'I';
1103 $CASEFOLD{0x130}{'turkic'}
1104 = $CASEFOLD{0x130}{'mapping'}
1105 = $CASEFOLD{0x130}{'full'}
1106 = $CASEFOLD{0x130}{'simple'}
1107 = $CASEFOLD{0x131}{'turkic'}
1108 = $CASEFOLD{0x131}{'mapping'}
1109 = $CASEFOLD{0x131}{'full'}
1110 = $CASEFOLD{0x131}{'simple'}
1111 = sprintf "%04X", ord('i');
1118 my $code = _getcode($arg);
1119 croak __PACKAGE__, "::casefold: unknown code '$arg'"
1120 unless defined $code;
1122 _casefold() unless %CASEFOLD;
1124 return $CASEFOLD{$code};
1127 =head2 B<casespec()>
1129 use Unicode::UCD 'casespec';
1131 my $casespec = casespec(0xFB00);
1133 This returns the potentially locale-dependent case mappings of the L</code point
1134 argument>. The mappings may be longer than a single code point (which the basic
1135 Unicode case mappings as returned by L</charinfo()> never are).
1137 If there are no case mappings for the L</code point argument>, or if all three
1138 possible mappings (I<lower>, I<title> and I<upper>) result in single code
1139 points and are locale independent and unconditional, C<undef> is returned
1140 (which means that the case mappings, if any, for the code point are those
1141 returned by L</charinfo()>).
1143 Otherwise, a reference to a hash giving the mappings (or a reference to a hash
1144 of such hashes, explained below) is returned with the following keys and their
1147 The keys in the bottom layer hash with the meanings of their values are:
1153 the input L</code point argument> expressed in hexadecimal, with leading zeros
1154 added if necessary to make it contain at least four hexdigits
1158 one or more codes (separated by spaces) that, taken in order, give the
1159 code points for the lower case of I<code>.
1160 Each has at least four hexdigits.
1164 one or more codes (separated by spaces) that, taken in order, give the
1165 code points for the title case of I<code>.
1166 Each has at least four hexdigits.
1170 one or more codes (separated by spaces) that, taken in order, give the
1171 code points for the upper case of I<code>.
1172 Each has at least four hexdigits.
1176 the conditions for the mappings to be valid.
1177 If C<undef>, the mappings are always valid.
1178 When defined, this field is a list of conditions,
1179 all of which must be true for the mappings to be valid.
1180 The list consists of one or more
1181 I<locales> (see below)
1182 and/or I<contexts> (explained in the next paragraph),
1183 separated by spaces.
1184 (Other than as used to separate elements, spaces are to be ignored.)
1185 Case distinctions in the condition list are not significant.
1186 Conditions preceded by "NON_" represent the negation of the condition.
1188 A I<context> is one of those defined in the Unicode standard.
1189 For Unicode 5.1, they are defined in Section 3.13 C<Default Case Operations>
1191 L<http://www.unicode.org/versions/Unicode5.1.0/>.
1192 These are for context-sensitive casing.
1196 The hash described above is returned for locale-independent casing, where
1197 at least one of the mappings has length longer than one. If C<undef> is
1198 returned, the code point may have mappings, but if so, all are length one,
1199 and are returned by L</charinfo()>.
1200 Note that when this function does return a value, it will be for the complete
1201 set of mappings for a code point, even those whose length is one.
1203 If there are additional casing rules that apply only in certain locales,
1204 an additional key for each will be defined in the returned hash. Each such key
1205 will be its locale name, defined as a 2-letter ISO 3166 country code, possibly
1206 followed by a "_" and a 2-letter ISO language code (possibly followed by a "_"
1207 and a variant code). You can find the lists of all possible locales, see
1208 L<Locale::Country> and L<Locale::Language>.
1209 (In Unicode 6.0, the only locales returned by this function
1210 are C<lt>, C<tr>, and C<az>.)
1212 Each locale key is a reference to a hash that has the form above, and gives
1213 the casing rules for that particular locale, which take precedence over the
1214 locale-independent ones when in that locale.
1216 If the only casing for a code point is locale-dependent, then the returned
1217 hash will not have any of the base keys, like C<code>, C<upper>, etc., but
1218 will contain only locale keys.
1220 For more information about case mappings see
1221 L<http://www.unicode.org/unicode/reports/tr21/>
1228 unless (%CASESPEC) {
1229 if (openunicode(\$CASESPECFH, "SpecialCasing.txt")) {
1232 while (<$CASESPECFH>) {
1233 if (/^([0-9A-F]+); ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; ([0-9A-F]+(?: [0-9A-F]+)*)?; (\w+(?: \w+)*)?/) {
1234 my ($hexcode, $lower, $title, $upper, $condition) =
1235 ($1, $2, $3, $4, $5);
1236 my $code = hex($hexcode);
1237 if (exists $CASESPEC{$code}) {
1238 if (exists $CASESPEC{$code}->{code}) {
1243 @{$CASESPEC{$code}}{qw(lower
1247 if (defined $oldcondition) {
1249 ($oldcondition =~ /^([a-z][a-z](?:_\S+)?)/);
1250 delete $CASESPEC{$code};
1251 $CASESPEC{$code}->{$oldlocale} =
1256 condition => $oldcondition };
1260 ($condition =~ /^([a-z][a-z](?:_\S+)?)/);
1261 $CASESPEC{$code}->{$locale} =
1266 condition => $condition };
1273 condition => $condition };
1284 my $code = _getcode($arg);
1285 croak __PACKAGE__, "::casespec: unknown code '$arg'"
1286 unless defined $code;
1288 _casespec() unless %CASESPEC;
1290 return ref $CASESPEC{$code} ? dclone $CASESPEC{$code} : $CASESPEC{$code};
1293 =head2 B<namedseq()>
1295 use Unicode::UCD 'namedseq';
1297 my $namedseq = namedseq("KATAKANA LETTER AINU P");
1298 my @namedseq = namedseq("KATAKANA LETTER AINU P");
1299 my %namedseq = namedseq();
1301 If used with a single argument in a scalar context, returns the string
1302 consisting of the code points of the named sequence, or C<undef> if no
1303 named sequence by that name exists. If used with a single argument in
1304 a list context, it returns the list of the ordinals of the code points. If used
1306 arguments in a list context, returns a hash with the names of the
1307 named sequences as the keys and the named sequences as strings as
1308 the values. Otherwise, it returns C<undef> or an empty list depending
1311 This function only operates on officially approved (not provisional) named
1314 Note that as of Perl 5.14, C<\N{KATAKANA LETTER AINU P}> will insert the named
1315 sequence into double-quoted strings, and C<charnames::string_vianame("KATAKANA
1316 LETTER AINU P")> will return the same string this function does, but will also
1317 operate on character names that aren't named sequences, without you having to
1318 know which are which. See L<charnames>.
1325 unless (%NAMEDSEQ) {
1326 if (openunicode(\$NAMEDSEQFH, "Name.pl")) {
1329 while (<$NAMEDSEQFH>) {
1330 if (/^ [0-9A-F]+ \ /x) {
1332 my ($sequence, $name) = split /\t/;
1333 my @s = map { chr(hex($_)) } split(' ', $sequence);
1334 $NAMEDSEQ{$name} = join("", @s);
1344 # Use charnames::string_vianame() which now returns this information,
1345 # unless the caller wants the hash returned, in which case we read it in,
1346 # and thereafter use it instead of calling charnames, as it is faster.
1348 my $wantarray = wantarray();
1349 if (defined $wantarray) {
1352 _namedseq() unless %NAMEDSEQ;
1357 $s = $NAMEDSEQ{ $_[0] };
1360 $s = charnames::string_vianame($_[0]);
1362 return defined $s ? map { ord($_) } split('', $s) : ();
1365 return $NAMEDSEQ{ $_[0] } if %NAMEDSEQ;
1366 return charnames::string_vianame($_[0]);
1375 my @numbers = _read_table("To/Nv.pl");
1376 foreach my $entry (@numbers) {
1377 my ($start, $end, $value) = @$entry;
1379 # If value contains a slash, convert to decimal, add a reverse hash
1381 if ((my @rational = split /\//, $value) == 2) {
1382 my $real = $rational[0] / $rational[1];
1383 $real_to_rational{$real} = $value;
1386 # Should only be single element, but just in case...
1387 for my $i ($start .. $end) {
1388 $NUMERIC{$i} = $value;
1392 # The values require adjusting, as is in 'a' format
1393 for my $i ($start .. $end) {
1394 $NUMERIC{$i} = $value + $i - $start;
1399 # Decided unsafe to use these that aren't officially part of the Unicode
1402 #my $pi = acos(-1.0);
1403 #$NUMERIC{0x03C0} = $pi;
1405 # Euler's constant, not to be confused with Euler's number
1406 #$NUMERIC{0x2107} = 0.57721566490153286060651209008240243104215933593992;
1409 #$NUMERIC{0x212F} = 2.7182818284590452353602874713526624977572;
1418 use Unicode::UCD 'num';
1420 my $val = num("123");
1421 my $one_quarter = num("\N{VULGAR FRACTION 1/4}");
1423 C<num> returns the numeric value of the input Unicode string; or C<undef> if it
1424 doesn't think the entire string has a completely valid, safe numeric value.
1426 If the string is just one character in length, the Unicode numeric value
1427 is returned if it has one, or C<undef> otherwise. Note that this need
1428 not be a whole number. C<num("\N{TIBETAN DIGIT HALF ZERO}")>, for
1429 example returns -0.5.
1433 #A few characters to which Unicode doesn't officially
1434 #assign a numeric value are considered numeric by C<num>.
1437 # EULER CONSTANT 0.5772... (this is NOT Euler's number)
1438 # SCRIPT SMALL E 2.71828... (this IS Euler's number)
1439 # GREEK SMALL LETTER PI 3.14159...
1443 If the string is more than one character, C<undef> is returned unless
1444 all its characters are decimal digits (that is, they would match C<\d+>),
1445 from the same script. For example if you have an ASCII '0' and a Bengali
1446 '3', mixed together, they aren't considered a valid number, and C<undef>
1447 is returned. A further restriction is that the digits all have to be of
1448 the same form. A half-width digit mixed with a full-width one will
1449 return C<undef>. The Arabic script has two sets of digits; C<num> will
1450 return C<undef> unless all the digits in the string come from the same
1453 C<num> errs on the side of safety, and there may be valid strings of
1454 decimal digits that it doesn't recognize. Note that Unicode defines
1455 a number of "digit" characters that aren't "decimal digit" characters.
1456 "Decimal digits" have the property that they have a positional value, i.e.,
1457 there is a units position, a 10's position, a 100's, etc, AND they are
1458 arranged in Unicode in blocks of 10 contiguous code points. The Chinese
1459 digits, for example, are not in such a contiguous block, and so Unicode
1460 doesn't view them as decimal digits, but merely digits, and so C<\d> will not
1461 match them. A single-character string containing one of these digits will
1462 have its decimal value returned by C<num>, but any longer string containing
1463 only these digits will return C<undef>.
1465 Strings of multiple sub- and superscripts are not recognized as numbers. You
1466 can use either of the compatibility decompositions in Unicode::Normalize to
1467 change these into digits, and then call C<num> on the result.
1471 # To handle sub, superscripts, this could if called in list context,
1472 # consider those, and return the <decomposition> type in the second
1478 _numeric unless %NUMERIC;
1480 my $length = length($string);
1481 return $NUMERIC{ord($string)} if $length == 1;
1482 return if $string =~ /\D/;
1483 my $first_ord = ord(substr($string, 0, 1));
1484 my $value = $NUMERIC{$first_ord};
1486 # To be a valid decimal number, it should be in a block of 10 consecutive
1487 # characters, whose values are 0, 1, 2, ... 9. Therefore this digit's
1488 # value is its offset in that block from the character that means zero.
1489 my $zero_ord = $first_ord - $value;
1491 # Unicode 6.0 instituted the rule that only digits in a consecutive
1492 # block of 10 would be considered decimal digits. If this is an earlier
1493 # release, we verify that this first character is a member of such a
1494 # block. That is, that the block of characters surrounding this one
1495 # consists of all \d characters whose numeric values are the expected
1497 UnicodeVersion() unless defined $v_unicode_version;
1498 if ($v_unicode_version lt v6.0.0) {
1499 for my $i (0 .. 9) {
1500 my $ord = $zero_ord + $i;
1501 return unless chr($ord) =~ /\d/;
1502 my $numeric = $NUMERIC{$ord};
1503 return unless defined $numeric;
1504 return unless $numeric == $i;
1508 for my $i (1 .. $length -1) {
1510 # Here we know either by verifying, or by fact of the first character
1511 # being a \d in Unicode 6.0 or later, that any character between the
1512 # character that means 0, and 9 positions above it must be \d, and
1513 # must have its value correspond to its offset from the zero. Any
1514 # characters outside these 10 do not form a legal number for this
1516 my $ord = ord(substr($string, $i, 1));
1517 my $digit = $ord - $zero_ord;
1518 return unless $digit >= 0 && $digit <= 9;
1519 $value = $value * 10 + $digit;
1527 =head2 B<prop_aliases()>
1529 use Unicode::UCD 'prop_aliases';
1531 my ($short_name, $full_name, @other_names) = prop_aliases("space");
1532 my $same_full_name = prop_aliases("Space"); # Scalar context
1533 my ($same_short_name) = prop_aliases("Space"); # gets 0th element
1534 print "The full name is $full_name\n";
1535 print "The short name is $short_name\n";
1536 print "The other aliases are: ", join(", ", @other_names), "\n";
1539 The full name is White_Space
1540 The short name is WSpace
1541 The other aliases are: Space
1543 Most Unicode properties have several synonymous names. Typically, there is at
1544 least a short name, convenient to type, and a long name that more fully
1545 describes the property, and hence is more easily understood.
1547 If you know one name for a Unicode property, you can use C<prop_aliases> to find
1548 either the long name (when called in scalar context), or a list of all of the
1549 names, somewhat ordered so that the short name is in the 0th element, the long
1550 name in the next element, and any other synonyms are in the remaining
1551 elements, in no particular order.
1553 The long name is returned in a form nicely capitalized, suitable for printing.
1555 The input parameter name is loosely matched, which means that white space,
1556 hyphens, and underscores are ignored (except for the trailing underscore in
1557 the old_form grandfathered-in C<"L_">, which is better written as C<"LC">, and
1558 both of which mean C<General_Category=Cased Letter>).
1560 If the name is unknown, C<undef> is returned (or an empty list in list
1561 context). Note that Perl typically recognizes property names in regular
1562 expressions with an optional C<"Is_>" (with or without the underscore)
1563 prefixed to them, such as C<\p{isgc=punct}>. This function does not recognize
1564 those in the input, returning C<undef>. Nor are they included in the output
1565 as possible synonyms.
1567 C<prop_aliases> does know about the Perl extensions to Unicode properties,
1568 such as C<Any> and C<XPosixAlpha>, and the single form equivalents to Unicode
1569 properties such as C<XDigit>, C<Greek>, C<In_Greek>, and C<Is_Greek>. The
1570 final example demonstrates that the C<"Is_"> prefix is recognized for these
1571 extensions; it is needed to resolve ambiguities. For example,
1572 C<prop_aliases('lc')> returns the list C<(lc, Lowercase_Mapping)>, but
1573 C<prop_aliases('islc')> returns C<(Is_LC, Cased_Letter)>. This is
1574 because C<islc> is a Perl extension which is short for
1575 C<General_Category=Cased Letter>. The lists returned for the Perl extensions
1576 will not include the C<"Is_"> prefix (whether or not the input had it) unless
1577 needed to resolve ambiguities, as shown in the C<"islc"> example, where the
1578 returned list had one element containing C<"Is_">, and the other without.
1580 It is also possible for the reverse to happen: C<prop_aliases('isc')> returns
1581 the list C<(isc, ISO_Comment)>; whereas C<prop_aliases('c')> returns
1582 C<(C, Other)> (the latter being a Perl extension meaning
1583 C<General_Category=Other>.
1584 L<perluniprops/Properties accessible through Unicode::UCD> lists the available
1585 forms, including which ones are discouraged from use.
1587 Those discouraged forms are accepted as input to C<prop_aliases>, but are not
1588 returned in the lists. C<prop_aliases('isL&')> and C<prop_aliases('isL_')>,
1589 which are old synonyms for C<"Is_LC"> and should not be used in new code, are
1590 examples of this. These both return C<(Is_LC, Cased_Letter)>. Thus this
1591 function allows you to take a discourarged form, and find its acceptable
1592 alternatives. The same goes with single-form Block property equivalences.
1593 Only the forms that begin with C<"In_"> are not discouraged; if you pass
1594 C<prop_aliases> a discouraged form, you will get back the equivalent ones that
1595 begin with C<"In_">. It will otherwise look like a new-style block name (see.
1596 L</Old-style versus new-style block names>).
1598 C<prop_aliases> does not know about any user-defined properties, and will
1599 return C<undef> if called with one of those. Likewise for Perl internal
1600 properties, with the exception of "Perl_Decimal_Digit" which it does know
1601 about (and which is documented below in L</prop_invmap()>).
1605 # It may be that there are use cases where the discouraged forms should be
1606 # returned. If that comes up, an optional boolean second parameter to the
1607 # function could be created, for example.
1609 # These are created by mktables for this routine and stored in unicore/UCD.pl
1610 # where their structures are described.
1611 our %string_property_loose_to_name;
1612 our %ambiguous_names;
1613 our %loose_perlprop_to_name;
1616 sub prop_aliases ($) {
1618 return unless defined $prop;
1620 require "unicore/UCD.pl";
1621 require "unicore/Heavy.pl";
1622 require "utf8_heavy.pl";
1624 # The property name may be loosely or strictly matched; we don't know yet.
1625 # But both types use lower-case.
1628 # It is loosely matched if its lower case isn't known to be strict.
1630 if (! exists $utf8::stricter_to_file_of{$prop}) {
1631 my $loose = utf8::_loose_name($prop);
1633 # There is a hash that converts from any loose name to its standard
1634 # form, mapping all synonyms for a name to one name that can be used
1635 # as a key into another hash. The whole concept is for memory
1636 # savings, as the second hash doesn't have to have all the
1637 # combinations. Actually, there are two hashes that do the
1638 # converstion. One is used in utf8_heavy.pl (stored in Heavy.pl) for
1639 # looking up properties matchable in regexes. This function needs to
1640 # access string properties, which aren't available in regexes, so a
1641 # second conversion hash is made for them (stored in UCD.pl). Look in
1642 # the string one now, as the rest can have an optional 'is' prefix,
1643 # which these don't.
1644 if (exists $string_property_loose_to_name{$loose}) {
1646 # Convert to its standard loose name.
1647 $prop = $string_property_loose_to_name{$loose};
1650 my $retrying = 0; # bool. ? Has an initial 'is' been stripped
1652 if (exists $utf8::loose_property_name_of{$loose}
1654 || ! exists $ambiguous_names{$loose}))
1656 # Found an entry giving the standard form. We don't get here
1657 # (in the test above) when we've stripped off an
1658 # 'is' and the result is an ambiguous name. That is because
1659 # these are official Unicode properties (though Perl can have
1660 # an optional 'is' prefix meaning the official property), and
1661 # all ambiguous cases involve a Perl single-form extension
1662 # for the gc, script, or block properties, and the stripped
1663 # 'is' means that they mean one of those, and not one of
1665 $prop = $utf8::loose_property_name_of{$loose};
1667 elsif (exists $loose_perlprop_to_name{$loose}) {
1669 # This hash is specifically for this function to list Perl
1670 # extensions that aren't in the earlier hashes. If there is
1671 # only one element, the short and long names are identical.
1672 # Otherwise the form is already in the same form as
1673 # %prop_aliases, which is handled at the end of the function.
1674 $list_ref = $loose_perlprop_to_name{$loose};
1675 if (@$list_ref == 1) {
1676 my @list = ($list_ref->[0], $list_ref->[0]);
1680 elsif (! exists $utf8::loose_to_file_of{$loose}) {
1682 # loose_to_file_of is a complete list of loose names. If not
1683 # there, the input is unknown.
1688 # Here we found the name but not its aliases, so it has to
1689 # exist. This means it must be one of the Perl single-form
1690 # extensions. First see if it is for a property-value
1691 # combination in one of the following properties.
1693 foreach my $property ("gc", "script") {
1694 @list = prop_value_aliases($property, $loose);
1699 # Here, it is one of those property-value combination
1700 # single-form synonyms. There are ambiguities with some
1701 # of these. Check against the list for these, and adjust
1703 for my $i (0 .. @list -1) {
1704 if (exists $ambiguous_names
1705 {utf8::_loose_name(lc $list[$i])})
1707 # The ambiguity is resolved by toggling whether or
1708 # not it has an 'is' prefix
1709 $list[$i] =~ s/^Is_// or $list[$i] =~ s/^/Is_/;
1715 # Here, it wasn't one of the gc or script single-form
1716 # extensions. It could be a block property single-form
1717 # extension. An 'in' prefix definitely means that, and should
1718 # be looked up without the prefix. However, starting in
1719 # Unicode 6.1, we have to special case 'indic...', as there
1720 # is a property that begins with that name. We shouldn't
1721 # strip the 'in' from that. I'm (khw) generalizing this to
1722 # 'indic' instead of the single property, because I suspect
1723 # that others of this class may come along in the future.
1724 # However, this could backfire and a block created whose name
1725 # begins with 'dic...', and we would want to strip the 'in'.
1726 # At which point this would have to be tweaked.
1727 my $began_with_in = $loose =~ s/^in(?!dic)//;
1728 @list = prop_value_aliases("block", $loose);
1730 map { $_ =~ s/^/In_/ } @list;
1734 # Here still haven't found it. The last opportunity for it
1735 # being valid is only if it began with 'is'. We retry without
1736 # the 'is', setting a flag to that effect so that we don't
1737 # accept things that begin with 'isis...'
1738 if (! $retrying && ! $began_with_in && $loose =~ s/^is//) {
1743 # Here, didn't find it. Since it was in %loose_to_file_of, we
1744 # should have been able to find it.
1745 carp __PACKAGE__, "::prop_aliases: Unexpectedly could not find '$prop'. Send bug report to perlbug\@perl.org";
1752 # Here, we have set $prop to a standard form name of the input. Look
1753 # it up in the structure created by mktables for this purpose, which
1754 # contains both strict and loosely matched properties. Avoid
1756 $list_ref = $prop_aliases{$prop} if exists $prop_aliases{$prop};
1757 return unless $list_ref;
1760 # The full name is in element 1.
1761 return $list_ref->[1] unless wantarray;
1763 return @{dclone $list_ref};
1768 =head2 B<prop_value_aliases()>
1770 use Unicode::UCD 'prop_value_aliases';
1772 my ($short_name, $full_name, @other_names)
1773 = prop_value_aliases("Gc", "Punct");
1774 my $same_full_name = prop_value_aliases("Gc", "P"); # Scalar cntxt
1775 my ($same_short_name) = prop_value_aliases("Gc", "P"); # gets 0th
1777 print "The full name is $full_name\n";
1778 print "The short name is $short_name\n";
1779 print "The other aliases are: ", join(", ", @other_names), "\n";
1782 The full name is Punctuation
1784 The other aliases are: Punct
1786 Some Unicode properties have a restricted set of legal values. For example,
1787 all binary properties are restricted to just C<true> or C<false>; and there
1788 are only a few dozen possible General Categories.
1790 For such properties, there are usually several synonyms for each possible
1791 value. For example, in binary properties, I<truth> can be represented by any of
1792 the strings "Y", "Yes", "T", or "True"; and the General Category
1793 "Punctuation" by that string, or "Punct", or simply "P".
1795 Like property names, there is typically at least a short name for each such
1796 property-value, and a long name. If you know any name of the property-value,
1797 you can use C<prop_value_aliases>() to get the long name (when called in
1798 scalar context), or a list of all the names, with the short name in the 0th
1799 element, the long name in the next element, and any other synonyms in the
1800 remaining elements, in no particular order, except that any all-numeric
1801 synonyms will be last.
1803 The long name is returned in a form nicely capitalized, suitable for printing.
1805 Case, white space, hyphens, and underscores are ignored in the input parameters
1806 (except for the trailing underscore in the old-form grandfathered-in general
1807 category property value C<"L_">, which is better written as C<"LC">).
1809 If either name is unknown, C<undef> is returned. Note that Perl typically
1810 recognizes property names in regular expressions with an optional C<"Is_>"
1811 (with or without the underscore) prefixed to them, such as C<\p{isgc=punct}>.
1812 This function does not recognize those in the property parameter, returning
1815 If called with a property that doesn't have synonyms for its values, it
1816 returns the input value, possibly normalized with capitalization and
1819 For the block property, new-style block names are returned (see
1820 L</Old-style versus new-style block names>).
1822 To find the synonyms for single-forms, such as C<\p{Any}>, use
1823 L</prop_aliases()> instead.
1825 C<prop_value_aliases> does not know about any user-defined properties, and
1826 will return C<undef> if called with one of those.
1830 # These are created by mktables for this routine and stored in unicore/UCD.pl
1831 # where their structures are described.
1832 our %loose_to_standard_value;
1833 our %prop_value_aliases;
1835 sub prop_value_aliases ($$) {
1836 my ($prop, $value) = @_;
1837 return unless defined $prop && defined $value;
1839 require "unicore/UCD.pl";
1840 require "utf8_heavy.pl";
1842 # Find the property name synonym that's used as the key in other hashes,
1843 # which is element 0 in the returned list.
1844 ($prop) = prop_aliases($prop);
1846 $prop = utf8::_loose_name(lc $prop);
1848 # Here is a legal property, but the hash below (created by mktables for
1849 # this purpose) only knows about the properties that have a very finite
1850 # number of potential values, that is not ones whose value could be
1851 # anything, like most (if not all) string properties. These don't have
1852 # synonyms anyway. Simply return the input. For example, there is no
1853 # synonym for ('Uppercase_Mapping', A').
1854 return $value if ! exists $prop_value_aliases{$prop};
1856 # The value name may be loosely or strictly matched; we don't know yet.
1857 # But both types use lower-case.
1860 # If the name isn't found under loose matching, it certainly won't be
1861 # found under strict
1862 my $loose_value = utf8::_loose_name($value);
1863 return unless exists $loose_to_standard_value{"$prop=$loose_value"};
1865 # Similarly if the combination under loose matching doesn't exist, it
1866 # won't exist under strict.
1867 my $standard_value = $loose_to_standard_value{"$prop=$loose_value"};
1868 return unless exists $prop_value_aliases{$prop}{$standard_value};
1870 # Here we did find a combination under loose matching rules. But it could
1871 # be that is a strict property match that shouldn't have matched.
1872 # %prop_value_aliases is set up so that the strict matches will appear as
1873 # if they were in loose form. Thus, if the non-loose version is legal,
1874 # we're ok, can skip the further check.
1875 if (! exists $utf8::stricter_to_file_of{"$prop=$value"}
1877 # We're also ok and skip the further check if value loosely matches.
1878 # mktables has verified that no strict name under loose rules maps to
1879 # an existing loose name. This code relies on the very limited
1880 # circumstances that strict names can be here. Strict name matching
1881 # happens under two conditions:
1882 # 1) when the name begins with an underscore. But this function
1883 # doesn't accept those, and %prop_value_aliases doesn't have
1885 # 2) When the values are numeric, in which case we need to look
1886 # further, but their squeezed-out loose values will be in
1887 # %stricter_to_file_of
1888 && exists $utf8::stricter_to_file_of{"$prop=$loose_value"})
1890 # The only thing that's legal loosely under strict is that can have an
1891 # underscore between digit pairs XXX
1892 while ($value =~ s/(\d)_(\d)/$1$2/g) {}
1893 return unless exists $utf8::stricter_to_file_of{"$prop=$value"};
1896 # Here, we know that the combination exists. Return it.
1897 my $list_ref = $prop_value_aliases{$prop}{$standard_value};
1898 if (@$list_ref > 1) {
1899 # The full name is in element 1.
1900 return $list_ref->[1] unless wantarray;
1902 return @{dclone $list_ref};
1905 return $list_ref->[0] unless wantarray;
1907 # Only 1 element means that it repeats
1908 return ( $list_ref->[0], $list_ref->[0] );
1911 # All 1 bits is the largest possible UV.
1912 $Unicode::UCD::MAX_CP = ~0;
1916 =head2 B<prop_invlist()>
1918 C<prop_invlist> returns an inversion list (described below) that defines all the
1919 code points for the binary Unicode property (or "property=value" pair) given
1920 by the input parameter string:
1923 use Unicode::UCD 'prop_invlist';
1924 say join ", ", prop_invlist("Any");
1929 An empty list is returned if the input is unknown; the number of elements in
1930 the list is returned if called in scalar context.
1932 L<perluniprops|perluniprops/Properties accessible through \p{} and \P{}> gives
1933 the list of properties that this function accepts, as well as all the possible
1934 forms for them (including with the optional "Is_" prefixes). (Except this
1935 function doesn't accept any Perl-internal properties, some of which are listed
1936 there.) This function uses the same loose or tighter matching rules for
1937 resolving the input property's name as is done for regular expressions. These
1938 are also specified in L<perluniprops|perluniprops/Properties accessible
1939 through \p{} and \P{}>. Examples of using the "property=value" form are:
1941 say join ", ", prop_invlist("Script=Shavian");
1946 say join ", ", prop_invlist("ASCII_Hex_Digit=No");
1949 0, 48, 58, 65, 71, 97, 103
1951 say join ", ", prop_invlist("ASCII_Hex_Digit=Yes");
1954 48, 58, 65, 71, 97, 103
1956 Inversion lists are a compact way of specifying Unicode property-value
1957 definitions. The 0th item in the list is the lowest code point that has the
1958 property-value. The next item (item [1]) is the lowest code point beyond that
1959 one that does NOT have the property-value. And the next item beyond that
1960 ([2]) is the lowest code point beyond that one that does have the
1961 property-value, and so on. Put another way, each element in the list gives
1962 the beginning of a range that has the property-value (for even numbered
1963 elements), or doesn't have the property-value (for odd numbered elements).
1964 The name for this data structure stems from the fact that each element in the
1965 list toggles (or inverts) whether the corresponding range is or isn't on the
1968 In the final example above, the first ASCII Hex digit is code point 48, the
1969 character "0", and all code points from it through 57 (a "9") are ASCII hex
1970 digits. Code points 58 through 64 aren't, but 65 (an "A") through 70 (an "F")
1971 are, as are 97 ("a") through 102 ("f"). 103 starts a range of code points
1972 that aren't ASCII hex digits. That range extends to infinity, which on your
1973 computer can be found in the variable C<$Unicode::UCD::MAX_CP>. (This
1974 variable is as close to infinity as Perl can get on your platform, and may be
1975 too high for some operations to work; you may wish to use a smaller number for
1978 Note that the inversion lists returned by this function can possibly include
1979 non-Unicode code points, that is anything above 0x10FFFF. This is in
1980 contrast to Perl regular expression matches on those code points, in which a
1981 non-Unicode code point always fails to match. For example, both of these have
1984 chr(0x110000) =~ \p{ASCII_Hex_Digit=True} # Fails.
1985 chr(0x110000) =~ \p{ASCII_Hex_Digit=False} # Fails!
1987 And both raise a warning that a Unicode property is being used on a
1988 non-Unicode code point. It is arguable as to which is the correct thing to do
1989 here. This function has chosen the way opposite to the Perl regular
1990 expression behavior. This allows you to easily flip to to the Perl regular
1991 expression way (for you to go in the other direction would be far harder).
1992 Simply add 0x110000 at the end of the non-empty returned list if it isn't
1993 already that value; and pop that value if it is; like:
1995 my @list = prop_invlist("foo");
1997 if ($list[-1] == 0x110000) {
1998 pop @list; # Defeat the turning on for above Unicode
2001 push @list, 0x110000; # Turn off for above Unicode
2005 It is a simple matter to expand out an inversion list to a full list of all
2006 code points that have the property-value:
2008 my @invlist = prop_invlist($property_name);
2009 die "empty" unless @invlist;
2011 for (my $i = 0; $i < @invlist; $i += 2) {
2012 my $upper = ($i + 1) < @invlist
2013 ? $invlist[$i+1] - 1 # In range
2014 : $Unicode::UCD::MAX_CP; # To infinity. You may want
2015 # to stop much much earlier;
2016 # going this high may expose
2017 # perl deficiencies with very
2019 for my $j ($invlist[$i] .. $upper) {
2020 push @full_list, $j;
2024 C<prop_invlist> does not know about any user-defined nor Perl internal-only
2025 properties, and will return C<undef> if called with one of those.
2029 # User-defined properties could be handled with some changes to utf8_heavy.pl;
2030 # and implementing here of dealing with EXTRAS. If done, consideration should
2031 # be given to the fact that the user subroutine could return different results
2032 # with each call; security issues need to be thought about.
2034 # These are created by mktables for this routine and stored in unicore/UCD.pl
2035 # where their structures are described.
2036 our %loose_defaults;
2037 our $MAX_UNICODE_CODEPOINT;
2039 sub prop_invlist ($) {
2041 return if ! defined $prop;
2043 require "utf8_heavy.pl";
2045 # Warnings for these are only for regexes, so not applicable to us
2046 no warnings 'deprecated';
2048 # Get the swash definition of the property-value.
2049 my $swash = utf8::SWASHNEW(__PACKAGE__, $prop, undef, 1, 0);
2051 # Fail if not found, or isn't a boolean property-value, or is a
2052 # user-defined property, or is internal-only.
2055 || $swash->{'BITS'} != 1
2056 || $swash->{'USER_DEFINED'}
2057 || $prop =~ /^\s*_/;
2059 if ($swash->{'EXTRAS'}) {
2060 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has EXTRAS magic";
2063 if ($swash->{'SPECIALS'}) {
2064 carp __PACKAGE__, "::prop_invlist: swash returned for $prop unexpectedly has SPECIALS magic";
2070 # The input lines look like:
2074 # Split into lines, stripped of trailing comments
2075 foreach my $range (split "\n",
2076 $swash->{'LIST'} =~ s/ \s* (?: \# .* )? $ //xmgr)
2078 # And find the beginning and end of the range on the line
2079 my ($hex_begin, $hex_end) = split "\t", $range;
2080 my $begin = hex $hex_begin;
2082 # If the new range merely extends the old, we remove the marker
2083 # created the last time through the loop for the old's end, which
2084 # causes the new one's end to be used instead.
2085 if (@invlist && $begin == $invlist[-1]) {
2089 # Add the beginning of the range
2090 push @invlist, $begin;
2093 if (defined $hex_end) { # The next item starts with the code point 1
2094 # beyond the end of the range.
2095 push @invlist, hex($hex_end) + 1;
2097 else { # No end of range, is a single code point.
2098 push @invlist, $begin + 1;
2102 require "unicore/UCD.pl";
2103 my $FIRST_NON_UNICODE = $MAX_UNICODE_CODEPOINT + 1;
2105 # Could need to be inverted: add or subtract a 0 at the beginning of the
2106 # list. And to keep it from matching non-Unicode, add or subtract the
2107 # first non-unicode code point.
2108 if ($swash->{'INVERT_IT'}) {
2109 if (@invlist && $invlist[0] == 0) {
2113 unshift @invlist, 0;
2115 if (@invlist && $invlist[-1] == $FIRST_NON_UNICODE) {
2119 push @invlist, $FIRST_NON_UNICODE;
2123 # Here, the list is set up to include only Unicode code points. But, if
2124 # the table is the default one for the property, it should contain all
2125 # non-Unicode code points. First calculate the loose name for the
2126 # property. This is done even for strict-name properties, as the data
2127 # structure that mktables generates for us is set up so that we don't have
2128 # to worry about that. The property-value needs to be split if compound,
2129 # as the loose rules need to be independently calculated on each part. We
2130 # know that it is syntactically valid, or SWASHNEW would have failed.
2133 my ($prop_only, $table) = split /\s*[:=]\s*/, $prop;
2136 # May have optional prefixed 'is'
2137 $prop = utf8::_loose_name($prop_only) =~ s/^is//r;
2138 $prop = $utf8::loose_property_name_of{$prop};
2139 $prop .= "=" . utf8::_loose_name($table);
2142 $prop = utf8::_loose_name($prop);
2144 if (exists $loose_defaults{$prop}) {
2146 # Here, is the default table. If a range ended with 10ffff, instead
2147 # continue that range to infinity, by popping the 110000; otherwise,
2148 # add the range from 11000 to infinity
2149 if (! @invlist || $invlist[-1] != $FIRST_NON_UNICODE) {
2150 push @invlist, $FIRST_NON_UNICODE;
2160 sub _search_invlist {
2161 # Find the range in the inversion list which contains a code point; that
2162 # is, find i such that l[i] <= code_point < l[i+1]
2164 # If this is ever made public, could use to speed up .t specials. Would
2165 # need to use code point argument, as in other functions in this pm
2167 my $list_ref = shift;
2168 my $code_point = shift;
2169 # Verify non-neg numeric XXX
2171 my $max_element = @$list_ref - 1;
2172 return if ! $max_element < 0; # Undef if list is empty.
2174 # Short cut something at the far-end of the table. This also allows us to
2175 # refer to element [$i+1] without fear of being out-of-bounds in the loop
2177 return $max_element if $code_point >= $list_ref->[$max_element];
2179 use integer; # want integer division
2181 my $i = $max_element / 2;
2184 my $upper = $max_element;
2187 if ($code_point >= $list_ref->[$i]) {
2189 # Here we have met the lower constraint. We can quit if we
2190 # also meet the upper one.
2191 last if $code_point < $list_ref->[$i+1];
2193 $lower = $i; # Still too low.
2198 # Here, $code_point < $list_ref[$i], so look lower down.
2202 # Split search domain in half to try again.
2203 my $temp = ($upper + $lower) / 2;
2205 # No point in continuing unless $i changes for next time
2207 return $i if $temp == $i;
2209 } # End of while loop
2211 # Here we have found the offset
2217 =head2 B<prop_invmap()>
2219 use Unicode::UCD 'prop_invmap';
2220 my ($list_ref, $map_ref, $format, $missing)
2221 = prop_invmap("General Category");
2223 C<prop_invmap> is used to get the complete mapping definition for a property,
2224 in the form of an inversion map. An inversion map consists of two parallel
2225 arrays. One is an ordered list of code points that mark range beginnings, and
2226 the other gives the value (or mapping) that all code points in the
2227 corresponding range have.
2229 C<prop_invmap> is called with the name of the desired property. The name is
2230 loosely matched, meaning that differences in case, white-space, hyphens, and
2231 underscores are not meaningful (except for the trailing underscore in the
2232 old-form grandfathered-in property C<"L_">, which is better written as C<"LC">,
2233 or even better, C<"Gc=LC">).
2235 Many Unicode properties have more than one name (or alias). C<prop_invmap>
2236 understands all of these, including Perl extensions to them. Ambiguities are
2237 resolved as described above for L</prop_aliases()>. The Perl internal
2238 property "Perl_Decimal_Digit, described below, is also accepted. C<undef> is
2239 returned if the property name is unknown.
2240 See L<perluniprops/Properties accessible through Unicode::UCD> for the
2241 properties acceptable as inputs to this function.
2243 It is a fatal error to call this function except in list context.
2245 In addition to the the two arrays that form the inversion map, C<prop_invmap>
2246 returns two other values; one is a scalar that gives some details as to the
2247 format of the entries of the map array; the other is used for specialized
2248 purposes, described at the end of this section.
2250 This means that C<prop_invmap> returns a 4 element list. For example,
2252 my ($blocks_ranges_ref, $blocks_maps_ref, $format, $default)
2253 = prop_invmap("Block");
2255 In this call, the two arrays will be populated as shown below (for Unicode
2258 Index @blocks_ranges @blocks_maps
2259 0 0x0000 Basic Latin
2260 1 0x0080 Latin-1 Supplement
2261 2 0x0100 Latin Extended-A
2262 3 0x0180 Latin Extended-B
2263 4 0x0250 IPA Extensions
2264 5 0x02B0 Spacing Modifier Letters
2265 6 0x0300 Combining Diacritical Marks
2266 7 0x0370 Greek and Coptic
2269 233 0x2B820 No_Block
2270 234 0x2F800 CJK Compatibility Ideographs Supplement
2271 235 0x2FA20 No_Block
2273 237 0xE0080 No_Block
2274 238 0xE0100 Variation Selectors Supplement
2275 239 0xE01F0 No_Block
2276 240 0xF0000 Supplementary Private Use Area-A
2277 241 0x100000 Supplementary Private Use Area-B
2278 242 0x110000 No_Block
2280 The first line (with Index [0]) means that the value for code point 0 is "Basic
2281 Latin". The entry "0x0080" in the @blocks_ranges column in the second line
2282 means that the value from the first line, "Basic Latin", extends to all code
2283 points in the range from 0 up to but not including 0x0080, that is, through
2284 127. In other words, the code points from 0 to 127 are all in the "Basic
2285 Latin" block. Similarly, all code points in the range from 0x0080 up to (but
2286 not including) 0x0100 are in the block named "Latin-1 Supplement", etc.
2287 (Notice that the return is the old-style block names; see L</Old-style versus
2288 new-style block names>).
2290 The final line (with Index [242]) means that the value for all code points above
2291 the legal Unicode maximum code point have the value "No_Block", which is the
2292 term Unicode uses for a non-existing block.
2294 The arrays completely specify the mappings for all possible code points.
2295 The final element in an inversion map returned by this function will always be
2296 for the range that consists of all the code points that aren't legal Unicode,
2297 but that are expressible on the platform. (That is, it starts with code point
2298 0x110000, the first code point above the legal Unicode maximum, and extends to
2299 infinity.) The value for that range will be the same that any typical
2300 unassigned code point has for the specified property. (Certain unassigned
2301 code points are not "typical"; for example the non-character code points, or
2302 those in blocks that are to be written right-to-left. The above-Unicode
2303 range's value is not based on these atypical code points.) It could be argued
2304 that, instead of treating these as unassigned Unicode code points, the value
2305 for this range should be C<undef>. If you wish, you can change the returned
2308 The maps are almost always simple scalars that should be interpreted as-is.
2309 These values are those given in the Unicode-supplied data files, which may be
2310 inconsistent as to capitalization and as to which synonym for a property-value
2311 is given. The results may be normalized by using the L</prop_value_aliases()>
2314 There are exceptions to the simple scalar maps. Some properties have some
2315 elements in their map list that are themselves lists of scalars; and some
2316 special strings are returned that are not to be interpreted as-is. Element
2317 [2] (placed into C<$format> in the example above) of the returned four element
2318 list tells you if the map has any of these special elements or not, as follows:
2324 means all the elements of the map array are simple scalars, with no special
2325 elements. Almost all properties are like this, like the C<block> example
2330 means that some of the map array elements have the form given by C<"s">, and
2331 the rest are lists of scalars. For example, here is a portion of the output
2332 of calling C<prop_invmap>() with the "Script Extensions" property:
2334 @scripts_ranges @scripts_maps
2337 0x0964 [ Bengali, Devanagari, Gurumukhi, Oriya ]
2341 Here, the code points 0x964 and 0x965 are both used in Bengali,
2342 Devanagari, Gurmukhi, and Oriya, but no other scripts.
2344 The Name_Alias property is also of this form. But each scalar consists of two
2345 components: 1) the name, and 2) the type of alias this is. They are
2346 separated by a colon and a space. In Unicode 6.1, there are several alias types:
2352 indicates that the name is a corrected form for the
2353 original name (which remains valid) for the same code point.
2357 adds a new name for a control character.
2361 is an alternate name for a character
2365 is a name for a character that has been documented but was never in any
2368 =item C<abbreviation>
2370 is a common abbreviation for a character
2374 The lists are ordered (roughly) so the most preferred names come before less
2379 @aliases_ranges @alias_maps
2381 0x009E [ 'PRIVACY MESSAGE: control', 'PM: abbreviation' ]
2382 0x009F [ 'APPLICATION PROGRAM COMMAND: control',
2385 0x00A0 'NBSP: abbreviation'
2387 0x00AD 'SHY: abbreviation'
2389 0x01A2 'LATIN CAPITAL LETTER GHA: correction'
2390 0x01A3 'LATIN SMALL LETTER GHA: correction'
2394 A map to the empty string means that there is no alias defined for the code
2399 is like C<"s"> in that all the map array elements are scalars, but here they are
2400 restricted to all being integers, and some have to be adjusted (hence the name
2401 C<"a">) to get the correct result. For example, in:
2403 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2404 = prop_invmap("Simple_Uppercase_Mapping");
2406 the returned arrays look like this:
2408 @$uppers_ranges_ref @$uppers_maps_ref Note
2410 97 65 'a' maps to 'A', b => B ...
2412 181 924 MICRO SIGN => Greek Cap MU
2416 Let's start with the second line. It says that the uppercase of code point 97
2417 is 65; or C<uc("a")> == "A". But the line is for the entire range of code
2418 points 97 through 122. To get the mapping for any code point in a range, you
2419 take the offset it has from the beginning code point of the range, and add
2420 that to the mapping for that first code point. So, the mapping for 122 ("z")
2421 is derived by taking the offset of 122 from 97 (=25) and adding that to 65,
2422 yielding 90 ("z"). Likewise for everything in between.
2424 The first line works the same way. The first map in a range is always the
2425 correct value for its code point (because the adjustment is 0). Thus the
2426 C<uc(chr(0))> is just itself. Also, C<uc(chr(1))> is also itself, as the
2427 adjustment is 0+1-0 .. C<uc(chr(96))> is 96.
2429 Requiring this simple adjustment allows the returned arrays to be
2430 significantly smaller than otherwise, up to a factor of 10, speeding up
2431 searching through them.
2435 means that some of the map array elements have the form given by C<"a">, and
2436 the rest are ordered lists of code points.
2439 my ($uppers_ranges_ref, $uppers_maps_ref, $format)
2440 = prop_invmap("Uppercase_Mapping");
2442 the returned arrays look like this:
2444 @$uppers_ranges_ref @$uppers_maps_ref
2451 0x0149 [ 0x02BC 0x004E ]
2456 This is the full Uppercase_Mapping property (as opposed to the
2457 Simple_Uppercase_Mapping given in the example for format C<"a">). The only
2458 difference between the two in the ranges shown is that the code point at
2459 0x0149 (LATIN SMALL LETTER N PRECEDED BY APOSTROPHE) maps to a string of two
2460 characters, 0x02BC (MODIFIER LETTER APOSTROPHE) followed by 0x004E (LATIN
2463 No adjustments are needed to entries that are references to arrays; each such
2464 entry will have exactly one element in its range, so the offset is always 0.
2468 This is like C<"a">, but some elements are the empty string, and should not be
2470 The one internal Perl property accessible by C<prop_invmap> is of this type:
2471 "Perl_Decimal_Digit" returns an inversion map which gives the numeric values
2472 that are represented by the Unicode decimal digit characters. Characters that
2473 don't represent decimal digits map to the empty string, like so:
2488 This means that the code points from 0 to 0x2F do not represent decimal digits;
2489 the code point 0x30 (DIGIT ZERO) represents 0; code point 0x31, (DIGIT ONE),
2490 represents 0+1-0 = 1; ... code point 0x39, (DIGIT NINE), represents 0+9-0 = 9;
2491 ... code points 0x3A through 0x65F do not represent decimal digits; 0x660
2492 (ARABIC-INDIC DIGIT ZERO), represents 0; ... 0x07C1 (NKO DIGIT ONE),
2493 represents 0+1-0 = 1 ...
2497 is a combination of the C<"al"> type and the C<"ae"> type. Some of
2498 the map array elements have the forms given by C<"al">, and
2499 the rest are the empty string. The property C<NFKC_Casefold> has this form.
2500 An example slice is:
2502 @$ranges_ref @$maps_ref Note
2504 0x00AA 97 FEMININE ORDINAL INDICATOR => 'a'
2506 0x00AD SOFT HYPHEN => ""
2508 0x00AF [ 0x0020, 0x0304 ] MACRON => SPACE . COMBINING MACRON
2514 means that all the elements of the map array are either rational numbers or
2515 the string C<"NaN">, meaning "Not a Number". A rational number is either an
2516 integer, or two integers separated by a solidus (C<"/">). The second integer
2517 represents the denominator of the division implied by the solidus, and is
2518 actually always positive, so it is guaranteed not to be 0 and to not to be
2519 signed. When the element is a plain integer (without the
2520 solidus), it may need to be adjusted to get the correct value by adding the
2521 offset, just as other C<"a"> properties. No adjustment is needed for
2522 fractions, as the range is guaranteed to have just a single element, and so
2523 the offset is always 0.
2525 If you want to convert the returned map to entirely scalar numbers, you
2526 can use something like this:
2528 my ($invlist_ref, $invmap_ref, $format) = prop_invmap($property);
2529 if ($format && $format eq "ar") {
2530 map { $_ = eval $_ } @$invmap_ref;
2533 Here's some entries from the output of the property "Nv", which has format
2536 @numerics_ranges @numerics_maps Note
2538 0x30 0 DIGIT 0 .. DIGIT 9
2540 0xB2 2 SUPERSCRIPTs 2 and 3
2542 0xB9 1 SUPERSCRIPT 1
2544 0xBC 1/4 VULGAR FRACTION 1/4
2545 0xBD 1/2 VULGAR FRACTION 1/2
2546 0xBE 3/4 VULGAR FRACTION 3/4
2548 0x660 0 ARABIC-INDIC DIGIT ZERO .. NINE
2553 means the Name property. All the elements of the map array are simple
2554 scalars, but some of them contain special strings that require more work to
2555 get the actual name.
2559 CJK UNIFIED IDEOGRAPH-<code point>
2561 mean that the name for the code point is "CJK UNIFIED IDEOGRAPH-"
2562 with the code point (expressed in hexadecimal) appended to it, like "CJK
2563 UNIFIED IDEOGRAPH-3403" (similarly for S<C<CJK COMPATIBILITY IDEOGRAPH-E<lt>code
2570 means that the name is algorithmically calculated. This is easily done by
2571 the function L<charnames/charnames::viacode(code)>.
2573 Note that for control characters (C<Gc=cc>), Unicode's data files have the
2574 string "C<E<lt>controlE<gt>>", but the real name of each of these characters is the empty
2575 string. This function returns that real name, the empty string. (There are
2576 names for these characters, but they are considered aliases, not the Name
2577 property name, and are contained in the C<Name_Alias> property.)
2581 means the Decomposition_Mapping property. This property is like C<"al">
2582 properties, except that one of the scalar elements is of the form:
2586 This signifies that this entry should be replaced by the decompositions for
2587 all the code points whose decomposition is algorithmically calculated. (All
2588 of them are currently in one range and no others outisde the range are likely
2589 to ever be added to Unicode; the C<"n"> format
2590 has this same entry.) These can be generated via the function
2591 L<Unicode::Normalize::NFD()|Unicode::Normalize>.
2593 Note that the mapping is the one that is specified in the Unicode data files,
2594 and to get the final decomposition, it may need to be applied recursively.
2598 Note that a format begins with the letter "a" if and only the property it is
2599 for requires adjustments by adding the offsets in multi-element ranges. For
2600 all these properties, an entry should be adjusted only if the map is a scalar
2601 which is an integer. That is, it must match the regular expression:
2605 Further, the first element in a range never needs adjustment, as the
2606 adjustment would be just adding 0.
2608 A binary search can be used to quickly find a code point in the inversion
2609 list, and hence its corresponding mapping.
2611 The final element (index [3], assigned to C<$default> in the "block" example) in
2612 the four element list returned by this function may be useful for applications
2613 that wish to convert the returned inversion map data structure into some
2614 other, such as a hash. It gives the mapping that most code points map to
2615 under the property. If you establish the convention that any code point not
2616 explicitly listed in your data structure maps to this value, you can
2617 potentially make your data structure much smaller. As you construct your data
2618 structure from the one returned by this function, simply ignore those ranges
2619 that map to this value, generally called the "default" value. For example, to
2620 convert to the data structure searchable by L</charinrange()>, you can follow
2621 this recipe for properties that don't require adjustments:
2623 my ($list_ref, $map_ref, $format, $missing) = prop_invmap($property);
2626 # Look at each element in the list, but the -2 is needed because we
2627 # look at $i+1 in the loop, and the final element is guaranteed to map
2628 # to $missing by prop_invmap(), so we would skip it anyway.
2629 for my $i (0 .. @$list_ref - 2) {
2630 next if $map_ref->[$i] eq $missing;
2631 push @range_list, [ $list_ref->[$i],
2637 print charinrange(\@range_list, $code_point), "\n";
2639 With this, C<charinrange()> will return C<undef> if its input code point maps
2640 to C<$missing>. You can avoid this by omitting the C<next> statement, and adding
2641 a line after the loop to handle the final element of the inversion map.
2643 Similarly, this recipe can be used for properties that do require adjustments:
2645 for my $i (0 .. @$list_ref - 2) {
2646 next if $map_ref->[$i] eq $missing;
2648 # prop_invmap() guarantees that if the mapping is to an array, the
2649 # range has just one element, so no need to worry about adjustments.
2650 if (ref $map_ref->[$i]) {
2652 [ $list_ref->[$i], $list_ref->[$i], $map_ref->[$i] ];
2654 else { # Otherwise each element is actually mapped to a separate
2655 # value, so the range has to be split into single code point
2660 # For each code point that gets mapped to something...
2661 for my $j ($list_ref->[$i] .. $list_ref->[$i+1] -1 ) {
2663 # ... add a range consisting of just it mapping to the
2664 # original plus the adjustment, which is incremented for the
2665 # next time through the loop, as the offset increases by 1
2666 # for each element in the range
2668 [ $j, $j, $map_ref->[$i] + $adjustment++ ];
2673 Note that the inversion maps returned for the C<Case_Folding> and
2674 C<Simple_Case_Folding> properties do not include the Turkic-locale mappings.
2675 Use L</casefold()> for these.
2677 C<prop_invmap> does not know about any user-defined properties, and will
2678 return C<undef> if called with one of those.
2682 # User-defined properties could be handled with some changes to utf8_heavy.pl;
2683 # if done, consideration should be given to the fact that the user subroutine
2684 # could return different results with each call, which could lead to some
2687 # One could store things in memory so they don't have to be recalculated, but
2688 # it is unlikely this will be called often, and some properties would take up
2689 # significant memory.
2691 # These are created by mktables for this routine and stored in unicore/UCD.pl
2692 # where their structures are described.
2693 our @algorithmic_named_code_points;
2697 sub prop_invmap ($) {
2699 croak __PACKAGE__, "::prop_invmap: must be called in list context" unless wantarray;
2702 return unless defined $prop;
2704 # Fail internal properties
2705 return if $prop =~ /^_/;
2707 # The values returned by this function.
2708 my (@invlist, @invmap, $format, $missing);
2710 # The swash has two components we look at, the base list, and a hash,
2711 # named 'SPECIALS', containing any additional members whose mappings don't
2712 # fit into the the base list scheme of things. These generally 'override'
2713 # any value in the base list for the same code point.
2716 require "utf8_heavy.pl";
2717 require "unicore/UCD.pl";
2721 # If there are multiple entries for a single code point
2722 my $has_multiples = 0;
2724 # Try to get the map swash for the property. They have 'To' prepended to
2725 # the property name, and 32 means we will accept 32 bit return values.
2726 # The 0 means we aren't calling this from tr///.
2727 my $swash = utf8::SWASHNEW(__PACKAGE__, "To$prop", undef, 32, 0);
2729 # If didn't find it, could be because needs a proxy. And if was the
2730 # 'Block' or 'Name' property, use a proxy even if did find it. Finding it
2731 # in these cases would be the result of the installation changing mktables
2732 # to output the Block or Name tables. The Block table gives block names
2733 # in the new-style, and this routine is supposed to return old-style block
2734 # names. The Name table is valid, but we need to execute the special code
2735 # below to add in the algorithmic-defined name entries.
2736 # And NFKCCF needs conversion, so handle that here too.
2737 if (ref $swash eq ""
2738 || $swash->{'TYPE'} =~ / ^ To (?: Blk | Na | NFKCCF ) $ /x)
2741 # Get the short name of the input property, in standard form
2742 my ($second_try) = prop_aliases($prop);
2743 return unless $second_try;
2744 $second_try = utf8::_loose_name(lc $second_try);
2746 if ($second_try eq "in") {
2748 # This property is identical to age for inversion map purposes
2752 elsif ($second_try =~ / ^ s ( cf | fc | [ltu] c ) $ /x) {
2754 # These properties use just the LIST part of the full mapping,
2755 # which includes the simple maps that are otherwise overridden by
2756 # the SPECIALS. So all we need do is to not look at the SPECIALS;
2757 # set $overrides to indicate that
2760 # The full name is the simple name stripped of its initial 's'
2763 # .. except for this case
2764 $prop = 'cf' if $prop eq 'fc';
2768 elsif ($second_try eq "blk") {
2770 # We use the old block names. Just create a fake swash from its
2774 $blocks{'LIST'} = "";
2775 $blocks{'TYPE'} = "ToBlk";
2776 $utf8::SwashInfo{ToBlk}{'missing'} = "No_Block";
2777 $utf8::SwashInfo{ToBlk}{'format'} = "s";
2779 foreach my $block (@BLOCKS) {
2780 $blocks{'LIST'} .= sprintf "%x\t%x\t%s\n",
2787 elsif ($second_try eq "na") {
2789 # Use the combo file that has all the Name-type properties in it,
2790 # extracting just the ones that are for the actual 'Name'
2791 # property. And create a fake swash from it.
2793 $names{'LIST'} = "";
2794 my $original = do "unicore/Name.pl";
2795 my $algorithm_names = \@algorithmic_named_code_points;
2797 # We need to remove the names from it that are aliases. For that
2798 # we need to also read in that table. Create a hash with the keys
2799 # being the code points, and the values being a list of the
2800 # aliases for the code point key.
2801 my ($aliases_code_points, $aliases_maps, undef, undef) =
2802 &prop_invmap('Name_Alias');
2804 for (my $i = 0; $i < @$aliases_code_points; $i++) {
2805 my $code_point = $aliases_code_points->[$i];
2806 $aliases{$code_point} = $aliases_maps->[$i];
2808 # If not already a list, make it into one, so that later we
2809 # can treat things uniformly
2810 if (! ref $aliases{$code_point}) {
2811 $aliases{$code_point} = [ $aliases{$code_point} ];
2814 # Remove the alias type from the entry, retaining just the
2816 map { s/:.*// } @{$aliases{$code_point}};
2820 foreach my $line (split "\n", $original) {
2821 my ($hex_code_point, $name) = split "\t", $line;
2823 # Weeds out all comments, blank lines, and named sequences
2824 next if $hex_code_point =~ /[^[:xdigit:]]/a;
2826 my $code_point = hex $hex_code_point;
2828 # The name of all controls is the default: the empty string.
2829 # The set of controls is immutable, so these hard-coded
2831 next if $code_point <= 0x9F
2832 && ($code_point <= 0x1F || $code_point >= 0x7F);
2834 # If this is a name_alias, it isn't a name
2835 next if grep { $_ eq $name } @{$aliases{$code_point}};
2837 # If we are beyond where one of the special lines needs to
2839 while ($i < @$algorithm_names
2840 && $code_point > $algorithm_names->[$i]->{'low'})
2843 # ... then insert it, ahead of what we were about to
2845 $names{'LIST'} .= sprintf "%x\t%x\t%s\n",
2846 $algorithm_names->[$i]->{'low'},
2847 $algorithm_names->[$i]->{'high'},
2848 $algorithm_names->[$i]->{'name'};
2850 # Done with this range.
2853 # We loop until all special lines that precede the next
2854 # regular one are output.
2857 # Here, is a normal name.
2858 $names{'LIST'} .= sprintf "%x\t\t%s\n", $code_point, $name;
2859 } # End of loop through all the names
2861 $names{'TYPE'} = "ToNa";
2862 $utf8::SwashInfo{ToNa}{'missing'} = "";
2863 $utf8::SwashInfo{ToNa}{'format'} = "n";
2866 elsif ($second_try =~ / ^ ( d [mt] ) $ /x) {
2868 # The file is a combination of dt and dm properties. Create a
2869 # fake swash from the portion that we want.
2870 my $original = do "unicore/Decomposition.pl";
2873 if ($second_try eq 'dt') {
2874 $decomps{'TYPE'} = "ToDt";
2875 $utf8::SwashInfo{'ToDt'}{'missing'} = "None";
2876 $utf8::SwashInfo{'ToDt'}{'format'} = "s";
2877 } # 'dm' is handled below, with 'nfkccf'
2879 $decomps{'LIST'} = "";
2881 # This property has one special range not in the file: for the
2882 # hangul syllables. But not in Unicode version 1.
2883 UnicodeVersion() unless defined $v_unicode_version;
2884 my $done_hangul = ($v_unicode_version lt v2.0.0)
2886 : 0; # Have we done the hangul range ?
2887 foreach my $line (split "\n", $original) {
2888 my ($hex_lower, $hex_upper, $type_and_map) = split "\t", $line;
2889 my $code_point = hex $hex_lower;
2893 # The type, enclosed in <...>, precedes the mapping separated
2895 if ($type_and_map =~ / ^ < ( .* ) > \s+ (.*) $ /x) {
2896 $value = ($second_try eq 'dt') ? $1 : $2
2898 else { # If there is no type specified, it's canonical
2899 $value = ($second_try eq 'dt')
2904 # Insert the hangul range at the appropriate spot.
2905 if (! $done_hangul && $code_point > $HANGUL_BEGIN) {
2908 sprintf "%x\t%x\t%s\n",
2910 $HANGUL_BEGIN + $HANGUL_COUNT - 1,
2911 ($second_try eq 'dt')
2913 : "<hangul syllable>";
2916 if ($value =~ / / && $hex_upper ne "" && $hex_upper ne $hex_lower) {
2917 $line = sprintf("%04X\t%s\t%s", hex($hex_lower) + 1, $hex_upper, $value);
2922 # And append this to our constructed LIST.
2923 $decomps{'LIST'} .= "$hex_lower\t$hex_upper\t$value\n";
2929 elsif ($second_try ne 'nfkccf') { # Don't know this property. Fail.
2933 if ($second_try eq 'nfkccf' || $second_try eq 'dm') {
2935 # The 'nfkccf' property is stored in the old format for backwards
2936 # compatibility for any applications that has read its file
2937 # directly before prop_invmap() existed.
2938 # And the code above has extracted the 'dm' property from its file
2939 # yielding the same format. So here we convert them to adjusted
2940 # format for compatibility with the other properties similar to
2944 # We construct a new converted list.
2947 my @ranges = split "\n", $swash->{'LIST'};
2948 for (my $i = 0; $i < @ranges; $i++) {
2949 my ($hex_begin, $hex_end, $map) = split "\t", $ranges[$i];
2951 # The dm property has maps that are space separated sequences
2952 # of code points, as well as the special entry "<hangul
2953 # syllable>, which also contains a blank.
2954 my @map = split " ", $map;
2957 # If it's just the special entry, append as-is.
2958 if ($map eq '<hangul syllable>') {
2959 $list .= "$ranges[$i]\n";
2963 # These should all single-element ranges.
2964 croak __PACKAGE__, "::prop_invmap: Not expecting a mapping with multiple code points in a multi-element range, $ranges[$i]" if $hex_end ne "" && $hex_end ne $hex_begin;
2966 # Convert them to decimal, as that's what's expected.
2967 $list .= "$hex_begin\t\t"
2968 . join(" ", map { hex } @map)
2974 # Here, the mapping doesn't have a blank, is for a single code
2976 my $begin = hex $hex_begin;
2977 my $end = (defined $hex_end && $hex_end ne "")
2981 # Again, the output is to be in decimal.
2982 my $decimal_map = hex $map;
2984 # We know that multi-element ranges with the same mapping
2985 # should not be adjusted, as after the adjustment
2986 # multi-element ranges are for consecutive increasing code
2987 # points. Further, the final element in the list won't be
2988 # adjusted, as there is nothing after it to include in the
2990 if ($begin != $end || $i == @ranges -1) {
2992 # So just convert these to single-element ranges
2993 foreach my $code_point ($begin .. $end) {
2994 $list .= sprintf("%04X\t\t%d\n",
2995 $code_point, $decimal_map);
3000 # Here, we have a candidate for adjusting. What we do is
3001 # look through the subsequent adjacent elements in the
3002 # input. If the map to the next one differs by 1 from the
3003 # one before, then we combine into a larger range with the
3004 # initial map. Loop doing this until we find one that
3005 # can't be combined.
3007 my $offset = 0; # How far away are we from the initial
3009 my $squished = 0; # ? Did we squish at least two
3010 # elements together into one range
3011 for ( ; $i < @ranges; $i++) {
3012 my ($next_hex_begin, $next_hex_end, $next_map)
3013 = split "\t", $ranges[$i+1];
3015 # In the case of 'dm', the map may be a sequence of
3016 # multiple code points, which are never combined with
3018 last if $next_map =~ / /;
3021 my $next_decimal_map = hex $next_map;
3023 # If the next map is not next in sequence, it
3024 # shouldn't be combined.
3025 last if $next_decimal_map != $decimal_map + $offset;
3027 my $next_begin = hex $next_hex_begin;
3029 # Likewise, if the next element isn't adjacent to the
3030 # previous one, it shouldn't be combined.
3031 last if $next_begin != $begin + $offset;
3033 my $next_end = (defined $next_hex_end
3034 && $next_hex_end ne "")
3038 # And finally, if the next element is a multi-element
3039 # range, it shouldn't be combined.
3040 last if $next_end != $next_begin;
3042 # Here, we will combine. Loop to see if we should
3043 # combine the next element too.
3049 # Here, 'i' is the element number of the last element to
3050 # be combined, and the range is single-element, or we
3051 # wouldn't be combining. Get it's code point.
3052 my ($hex_end, undef, undef) = split "\t", $ranges[$i];
3053 $list .= "$hex_begin\t$hex_end\t$decimal_map\n";
3056 # Here, no combining done. Just appen the initial
3057 # (and current) values.
3058 $list .= "$hex_begin\t\t$decimal_map\n";
3061 } # End of loop constructing the converted list
3063 # Finish up the data structure for our converted swash
3064 my $type = ($second_try eq 'nfkccf') ? 'ToNFKCCF' : 'ToDm';
3065 $revised_swash{'LIST'} = $list;
3066 $revised_swash{'TYPE'} = $type;
3067 $revised_swash{'SPECIALS'} = $swash->{'SPECIALS'};
3068 $swash = \%revised_swash;
3070 $utf8::SwashInfo{$type}{'missing'} = 0;
3071 $utf8::SwashInfo{$type}{'format'} = 'a';
3075 if ($swash->{'EXTRAS'}) {
3076 carp __PACKAGE__, "::prop_invmap: swash returned for $prop unexpectedly has EXTRAS magic";
3080 # Here, have a valid swash return. Examine it.
3081 my $returned_prop = $swash->{'TYPE'};
3083 # All properties but binary ones should have 'missing' and 'format'
3085 $missing = $utf8::SwashInfo{$returned_prop}{'missing'};
3086 $missing = 'N' unless defined $missing;
3088 $format = $utf8::SwashInfo{$returned_prop}{'format'};
3089 $format = 'b' unless defined $format;
3091 my $requires_adjustment = $format =~ /^a/;
3093 # The LIST input lines look like:
3096 # 0375\t0377\tGreek # [3]
3097 # 037A\t037D\tGreek # [4]
3102 # Convert them to like
3111 # For binary properties, the final non-comment column is absent, and
3112 # assumed to be 'Y'.
3114 foreach my $range (split "\n", $swash->{'LIST'}) {
3115 $range =~ s/ \s* (?: \# .* )? $ //xg; # rmv trailing space, comments
3117 # Find the beginning and end of the range on the line
3118 my ($hex_begin, $hex_end, $map) = split "\t", $range;
3119 my $begin = hex $hex_begin;
3120 my $end = (defined $hex_end && $hex_end ne "")
3124 # Each time through the loop (after the first):
3125 # $invlist[-2] contains the beginning of the previous range processed
3126 # $invlist[-1] contains the end+1 of the previous range processed
3127 # $invmap[-2] contains the value of the previous range processed
3128 # $invmap[-1] contains the default value for missing ranges ($missing)
3130 # Thus, things are set up for the typical case of a new non-adjacent
3131 # range of non-missings to be added. But, if the new range is
3132 # adjacent, it needs to replace the [-1] element; and if the new
3133 # range is a multiple value of the previous one, it needs to be added
3134 # to the [-2] map element.
3136 # The first time through, everything will be empty. If the property
3137 # doesn't have a range that begins at 0, add one that maps to $missing
3141 push @invmap, $missing;
3144 elsif (@invlist > 1 && $invlist[-2] == $begin) {
3146 # Here we handle the case where the input has multiple entries for
3147 # each code point. mktables should have made sure that each such
3148 # range contains only one code point. At this point, $invlist[-1]
3149 # is the $missing that was added at the end of the last loop
3150 # iteration, and [-2] is the last real input code point, and that
3151 # code point is the same as the one we are adding now, making the
3152 # new one a multiple entry. Add it to the existing entry, either
3153 # by pushing it to the existing list of multiple entries, or
3154 # converting the single current entry into a list with both on it.
3155 # This is all we need do for this iteration.
3157 if ($end != $begin) {
3158 croak __PACKAGE__, ":prop_invmap: Multiple maps per code point in '$prop' require single-element ranges: begin=$begin, end=$end, map=$map";
3160 if (! ref $invmap[-2]) {
3161 $invmap[-2] = [ $invmap[-2], $map ];
3164 push @{$invmap[-2]}, $map;
3169 elsif ($invlist[-1] == $begin) {
3171 # If the input isn't in the most compact form, so that there are
3172 # two adjacent ranges that map to the same thing, they should be
3173 # combined (EXCEPT where the arrays require adjustments, in which
3174 # case everything is already set up correctly). This happens in
3175 # our constructed dt mapping, as Element [-2] is the map for the
3176 # latest range so far processed. Just set the beginning point of
3177 # the map to $missing (in invlist[-1]) to 1 beyond where this
3178 # range ends. For example, in
3181 # we have set it up so that it looks like
3185 # We now see that it should be
3188 if (! $requires_adjustment && @invlist > 1 && ( (defined $map)
3189 ? $invmap[-2] eq $map
3190 : $invmap[-2] eq 'Y'))
3192 $invlist[-1] = $end + 1;
3196 # Here, the range started in the previous iteration that maps to
3197 # $missing starts at the same code point as this range. That
3198 # means there is no gap to fill that that range was intended for,
3199 # so we just pop it off the parallel arrays.
3204 # Add the range beginning, and the range's map.
3205 push @invlist, $begin;
3206 if ($returned_prop eq 'ToDm') {
3208 # The decomposition maps are either a line like <hangul syllable>
3209 # which are to be taken as is; or a sequence of code points in hex
3210 # and separated by blanks. Convert them to decimal, and if there
3211 # is more than one, use an anonymous array as the map.
3212 if ($map =~ /^ < /x) {
3216 my @map = split " ", $map;
3218 push @invmap, $map[0];
3221 push @invmap, \@map;
3227 # Otherwise, convert hex formatted list entries to decimal; add a
3228 # 'Y' map for the missing value in binary properties, or
3229 # otherwise, use the input map unchanged.
3230 $map = ($format eq 'x')
3238 # We just started a range. It ends with $end. The gap between it and
3239 # the next element in the list must be filled with a range that maps
3240 # to the default value. If there is no gap, the next iteration will
3241 # pop this, unless there is no next iteration, and we have filled all
3242 # of the Unicode code space, so check for that and skip.
3243 if ($end < $MAX_UNICODE_CODEPOINT) {
3244 push @invlist, $end + 1;
3245 push @invmap, $missing;
3249 # If the property is empty, make all code points use the value for missing
3253 push @invmap, $missing;
3256 # And add in standard element that all non-Unicode code points map to:
3258 push @invlist, $MAX_UNICODE_CODEPOINT + 1;
3259 push @invmap, $missing;
3261 # The second component of the map are those values that require
3262 # non-standard specification, stored in SPECIALS. These override any
3263 # duplicate code points in LIST. If we are using a proxy, we may have
3264 # already set $overrides based on the proxy.
3265 $overrides = $swash->{'SPECIALS'} unless defined $overrides;
3268 # A negative $overrides implies that the SPECIALS should be ignored,
3269 # and a simple 'a' list is the value.
3270 if ($overrides < 0) {
3275 # Currently, all overrides are for properties that normally map to
3276 # single code points, but now some will map to lists of code
3277 # points (but there is an exception case handled below).
3280 # Look through the overrides.
3281 foreach my $cp_maybe_utf8 (keys %$overrides) {
3285 # If the overrides came from SPECIALS, the code point keys are
3287 if ($overrides == $swash->{'SPECIALS'}) {
3288 $cp = unpack("C0U", $cp_maybe_utf8);
3289 @map = unpack "U0U*", $swash->{'SPECIALS'}{$cp_maybe_utf8};
3291 # The empty string will show up unpacked as an empty
3293 $format = 'ale' if @map == 0;
3297 # But if we generated the overrides, we didn't bother to
3298 # pack them, and we, so far, do this only for properties
3299 # that are 'a' ones.
3300 $cp = $cp_maybe_utf8;
3301 @map = hex $overrides->{$cp};
3305 # Find the range that the override applies to.
3306 my $i = _search_invlist(\@invlist, $cp);
3307 if ($cp < $invlist[$i] || $cp >= $invlist[$i + 1]) {
3308 croak __PACKAGE__, "::prop_invmap: wrong_range, cp=$cp; i=$i, current=$invlist[$i]; next=$invlist[$i + 1]"
3311 # And what that range currently maps to
3312 my $cur_map = $invmap[$i];
3314 # If there is a gap between the next range and the code point
3315 # we are overriding, we have to add elements to both arrays to
3316 # fill that gap, using the map that applies to it, which is
3317 # $cur_map, since it is part of the current range.
3318 if ($invlist[$i + 1] > $cp + 1) {
3320 #say "Before splice:";
3321 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3322 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3323 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3324 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3325 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3327 splice @invlist, $i + 1, 0, $cp + 1;
3328 splice @invmap, $i + 1, 0, $cur_map;
3330 #say "After splice:";
3331 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3332 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3333 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3334 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3335 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3338 # If the remaining portion of the range is multiple code
3339 # points (ending with the one we are replacing, guaranteed by
3340 # the earlier splice). We must split it into two
3341 if ($invlist[$i] < $cp) {
3342 $i++; # Compensate for the new element
3345 #say "Before splice:";
3346 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3347 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3348 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3349 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3350 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3352 splice @invlist, $i, 0, $cp;
3353 splice @invmap, $i, 0, 'dummy';
3355 #say "After splice:";
3356 #say 'i-2=[', $i-2, ']', sprintf("%04X maps to %s", $invlist[$i-2], $invmap[$i-2]) if $i >= 2;
3357 #say 'i-1=[', $i-1, ']', sprintf("%04X maps to %s", $invlist[$i-1], $invmap[$i-1]) if $i >= 1;
3358 #say 'i =[', $i, ']', sprintf("%04X maps to %s", $invlist[$i], $invmap[$i]);
3359 #say 'i+1=[', $i+1, ']', sprintf("%04X maps to %s", $invlist[$i+1], $invmap[$i+1]) if $i < @invlist + 1;
3360 #say 'i+2=[', $i+2, ']', sprintf("%04X maps to %s", $invlist[$i+2], $invmap[$i+2]) if $i < @invlist + 2;
3363 # Here, the range we are overriding contains a single code
3364 # point. The result could be the empty string, a single
3365 # value, or a list. If the last case, we use an anonymous
3367 $invmap[$i] = (scalar @map == 0)
3375 elsif ($format eq 'x') {
3377 # All hex-valued properties are really to code points, and have been
3378 # converted to decimal.
3381 elsif ($returned_prop eq 'ToDm') {
3384 elsif ($format eq 'sw') { # blank-separated elements to form a list.
3385 map { $_ = [ split " ", $_ ] if $_ =~ / / } @invmap;
3388 elsif ($returned_prop eq 'ToNameAlias') {
3390 # This property currently doesn't have any lists, but theoretically
3394 elsif ($returned_prop eq 'ToPerlDecimalDigit') {
3397 elsif ($returned_prop eq 'ToNv') {
3399 # The one property that has this format is stored as a delta, so needs
3400 # to indicate that need to add code point to it.
3403 elsif ($format ne 'n' && $format ne 'a') {
3405 # All others are simple scalars
3408 if ($has_multiples && $format !~ /l/) {
3409 croak __PACKAGE__, "::prop_invmap: Wrong format '$format' for prop_invmap('$prop'); should indicate has lists";
3412 return (\@invlist, \@invmap, $format, $missing);
3415 =head2 Unicode::UCD::UnicodeVersion
3417 This returns the version of the Unicode Character Database, in other words, the
3418 version of the Unicode standard the database implements. The version is a
3419 string of numbers delimited by dots (C<'.'>).
3425 sub UnicodeVersion {
3426 unless (defined $UNICODEVERSION) {
3427 openunicode(\$VERSIONFH, "version");
3429 chomp($UNICODEVERSION = <$VERSIONFH>);
3431 croak __PACKAGE__, "::VERSION: strange version '$UNICODEVERSION'"
3432 unless $UNICODEVERSION =~ /^\d+(?:\.\d+)+$/;
3434 $v_unicode_version = pack "C*", split /\./, $UNICODEVERSION;
3435 return $UNICODEVERSION;
3438 =head2 B<Blocks versus Scripts>
3440 The difference between a block and a script is that scripts are closer
3441 to the linguistic notion of a set of code points required to present
3442 languages, while block is more of an artifact of the Unicode code point
3443 numbering and separation into blocks of (mostly) 256 code points.
3445 For example the Latin B<script> is spread over several B<blocks>, such
3446 as C<Basic Latin>, C<Latin 1 Supplement>, C<Latin Extended-A>, and
3447 C<Latin Extended-B>. On the other hand, the Latin script does not
3448 contain all the characters of the C<Basic Latin> block (also known as
3449 ASCII): it includes only the letters, and not, for example, the digits
3452 For blocks see L<http://www.unicode.org/Public/UNIDATA/Blocks.txt>
3454 For scripts see UTR #24: L<http://www.unicode.org/unicode/reports/tr24/>
3456 =head2 B<Matching Scripts and Blocks>
3458 Scripts are matched with the regular-expression construct
3459 C<\p{...}> (e.g. C<\p{Tibetan}> matches characters of the Tibetan script),
3460 while C<\p{Blk=...}> is used for blocks (e.g. C<\p{Blk=Tibetan}> matches
3461 any of the 256 code points in the Tibetan block).
3463 =head2 Old-style versus new-style block names
3465 Unicode publishes the names of blocks in two different styles, though the two
3466 are equivalent under Unicode's loose matching rules.
3468 The original style uses blanks and hyphens in the block names (except for
3469 C<No_Block>), like so:
3471 Miscellaneous Mathematical Symbols-B
3473 The newer style replaces these with underscores, like this:
3475 Miscellaneous_Mathematical_Symbols_B
3477 This newer style is consistent with the values of other Unicode properties.
3478 To preserve backward compatibility, all the functions in Unicode::UCD that
3479 return block names (except one) return the old-style ones. That one function,
3480 L</prop_value_aliases()> can be used to convert from old-style to new-style:
3482 my $new_style = prop_values_aliases("block", $old_style);
3484 Perl also has single-form extensions that refer to blocks, C<In_Cyrillic>,
3485 meaning C<Block=Cyrillic>. These have always been written in the new style.
3487 To convert from new-style to old-style, follow this recipe:
3489 $old_style = charblock((prop_invlist("block=$new_style"))[0]);
3491 (which finds the range of code points in the block using C<prop_invlist>,
3492 gets the lower end of the range (0th element) and then looks up the old name
3493 for its block using C<charblock>).
3495 Note that starting in Unicode 6.1, many of the block names have shorter
3496 synonyms. These are always given in the new style.
3500 Does not yet support EBCDIC platforms.
3504 Jarkko Hietaniemi. Now maintained by perl5 porters.