no warnings 'surrogate'; # surrogates can be inputs to this
use charnames ();
-our $VERSION = '0.45';
+our $VERSION = '0.52';
require Exporter;
prop_value_aliases
prop_invlist
prop_invmap
+ search_invlist
MAX_CP
);
my ($list_ref, $map_ref, $format, $missing)
= prop_invmap("General Category");
+ use Unicode::UCD 'search_invlist';
+ my $index = search_invlist(\@invlist, $code_point);
+
use Unicode::UCD 'compexcl';
my $compexcl = compexcl($codepoint);
followed by hexadecimals designating a Unicode code point. In other words, if
you want a code point to be interpreted as a hexadecimal number, you must
prefix it with either C<0x> or C<U+>, because a string like e.g. C<123> will be
-interpreted as a decimal code point. Note that the largest code point in
-Unicode is U+10FFFF.
+interpreted as a decimal code point.
+
+Examples:
+
+ 223 # Decimal 223
+ 0223 # Hexadecimal 223 (= 547 decimal)
+ 0xDF # Hexadecimal DF (= 223 decimal
+ U+DF # Hexadecimal DF
+
+Note that the largest code point in Unicode is U+10FFFF.
=cut
if ($v_unicode_version lt v2.0.0) {
my $subrange = [ 0, 0x10FFFF, 'No_Block' ];
push @BLOCKS, $subrange;
- push @{$BLOCKS{$3}}, $subrange;
+ push @{$BLOCKS{'No_Block'}}, $subrange;
}
elsif (openunicode(\$BLOCKSFH, "Blocks.txt")) {
local $_;
=over
-=item B<*> If you use this C<I> mapping
+=item Z<>B<*> If you use this C<I> mapping
the result is case-insensitive,
but dotless and dotted I's are not distinguished
-=item B<*> If you exclude this C<I> mapping
+=item Z<>B<*> If you exclude this C<I> mapping
the result is not fully case-insensitive, but
dotless and dotted I's are distinguished
prints:
0, 1114112
-An empty list is returned if the input is unknown; the number of elements in
+If the input is unknown C<undef> is returned in scalar context; an empty-list
+in list context. If the input is known, the number of elements in
the list is returned if called in scalar context.
L<perluniprops|perluniprops/Properties accessible through \p{} and \P{}> gives
C<prop_invlist> does not know about any user-defined nor Perl internal-only
properties, and will return C<undef> if called with one of those.
+The L</search_invlist()> function is provided for finding a code point within
+an inversion list.
+
=cut
# User-defined properties could be handled with some changes to utf8_heavy.pl;
return @invlist;
}
-sub _search_invlist {
- # Find the range in the inversion list which contains a code point; that
- # is, find i such that l[i] <= code_point < l[i+1]
-
- # If this is ever made public, could use to speed up .t specials. Would
- # need to use code point argument, as in other functions in this pm
-
- my $list_ref = shift;
- my $code_point = shift;
- # Verify non-neg numeric XXX
-
- my $max_element = @$list_ref - 1;
- return if ! $max_element < 0; # Undef if list is empty.
-
- # Short cut something at the far-end of the table. This also allows us to
- # refer to element [$i+1] without fear of being out-of-bounds in the loop
- # below.
- return $max_element if $code_point >= $list_ref->[$max_element];
-
- use integer; # want integer division
-
- my $i = $max_element / 2;
-
- my $lower = 0;
- my $upper = $max_element;
- while (1) {
-
- if ($code_point >= $list_ref->[$i]) {
-
- # Here we have met the lower constraint. We can quit if we
- # also meet the upper one.
- last if $code_point < $list_ref->[$i+1];
-
- $lower = $i; # Still too low.
-
- }
- else {
-
- # Here, $code_point < $list_ref[$i], so look lower down.
- $upper = $i;
- }
-
- # Split search domain in half to try again.
- my $temp = ($upper + $lower) / 2;
-
- # No point in continuing unless $i changes for next time
- # in the loop.
- return $i if $temp == $i;
- $i = $temp;
- } # End of while loop
-
- # Here we have found the offset
- return $i;
-}
-
=pod
=head2 B<prop_invmap()>
use Unicode::UCD 'prop_invmap';
- my ($list_ref, $map_ref, $format, $missing)
+ my ($list_ref, $map_ref, $format, $default)
= prop_invmap("General Category");
C<prop_invmap> is used to get the complete mapping definition for a property,
In addition to the the two arrays that form the inversion map, C<prop_invmap>
returns two other values; one is a scalar that gives some details as to the
-format of the entries of the map array; the other is used for specialized
-purposes, described at the end of this section.
+format of the entries of the map array; the other is a default value, useful
+in maps whose format name begins with the letter C<"a">, as described
+L<below in its subsection|/a>; and for specialized purposes, such as
+converting to another data structure, described at the end of this main
+section.
This means that C<prop_invmap> returns a 4 element list. For example,
for this range should be C<undef>. If you wish, you can change the returned
arrays accordingly.
-The maps are almost always simple scalars that should be interpreted as-is.
+The maps for almost all properties are simple scalars that should be
+interpreted as-is.
These values are those given in the Unicode-supplied data files, which may be
inconsistent as to capitalization and as to which synonym for a property-value
is given. The results may be normalized by using the L</prop_value_aliases()>
restricted to all being integers, and some have to be adjusted (hence the name
C<"a">) to get the correct result. For example, in:
- my ($uppers_ranges_ref, $uppers_maps_ref, $format)
+ my ($uppers_ranges_ref, $uppers_maps_ref, $format, $default)
= prop_invmap("Simple_Uppercase_Mapping");
the returned arrays look like this:
182 0
...
+and C<$default> is 0.
+
Let's start with the second line. It says that the uppercase of code point 97
is 65; or C<uc("a")> == "A". But the line is for the entire range of code
-points 97 through 122. To get the mapping for any code point in a range, you
-take the offset it has from the beginning code point of the range, and add
+points 97 through 122. To get the mapping for any code point in this range,
+you take the offset it has from the beginning code point of the range, and add
that to the mapping for that first code point. So, the mapping for 122 ("z")
is derived by taking the offset of 122 from 97 (=25) and adding that to 65,
yielding 90 ("z"). Likewise for everything in between.
-The first line works the same way. The first map in a range is always the
-correct value for its code point (because the adjustment is 0). Thus the
-C<uc(chr(0))> is just itself. Also, C<uc(chr(1))> is also itself, as the
-adjustment is 0+1-0 .. C<uc(chr(96))> is 96.
-
Requiring this simple adjustment allows the returned arrays to be
significantly smaller than otherwise, up to a factor of 10, speeding up
searching through them.
+Ranges that map to C<$default>, C<"0">, behave somewhat differently. For
+these, each code point maps to itself. So, in the first line in the example,
+S<C<ord(uc(chr(0)))>> is 0, S<C<ord(uc(chr(1)))>> is 1, ..
+S<C<ord(uc(chr(96)))>> is 96.
+
=item B<C<al>>
means that some of the map array elements have the form given by C<"a">, and
the rest are ordered lists of code points.
For example, in:
- my ($uppers_ranges_ref, $uppers_maps_ref, $format)
+ my ($uppers_ranges_ref, $uppers_maps_ref, $format, $default)
= prop_invmap("Uppercase_Mapping");
the returned arrays look like this:
No adjustments are needed to entries that are references to arrays; each such
entry will have exactly one element in its range, so the offset is always 0.
+The fourth (index [3]) element (C<$default>) in the list returned for this
+format is 0.
+
=item B<C<ae>>
This is like C<"a">, but some elements are the empty string, and should not be
(ARABIC-INDIC DIGIT ZERO), represents 0; ... 0x07C1 (NKO DIGIT ONE),
represents 0+1-0 = 1 ...
+The fourth (index [3]) element (C<$default>) in the list returned for this
+format is the empty string.
+
=item B<C<ale>>
is a combination of the C<"al"> type and the C<"ae"> type. Some of
0x00B0 0
...
+The fourth (index [3]) element (C<$default>) in the list returned for this
+format is 0.
+
=item B<C<ar>>
means that all the elements of the map array are either rational numbers or
0x660 0 ARABIC-INDIC DIGIT ZERO .. NINE
0x66A "NaN"
+The fourth (index [3]) element (C<$default>) in the list returned for this
+format is C<"NaN">.
+
=item B<C<n>>
means the Name property. All the elements of the map array are simple
Note that the mapping is the one that is specified in the Unicode data files,
and to get the final decomposition, it may need to be applied recursively.
+The fourth (index [3]) element (C<$default>) in the list returned for this
+format is 0.
+
=back
Note that a format begins with the letter "a" if and only the property it is
Further, the first element in a range never needs adjustment, as the
adjustment would be just adding 0.
-A binary search can be used to quickly find a code point in the inversion
-list, and hence its corresponding mapping.
+A binary search such as that provided by L</search_invlist()>, can be used to
+quickly find a code point in the inversion list, and hence its corresponding
+mapping.
-The final element (index [3], assigned to C<$default> in the "block" example) in
-the four element list returned by this function may be useful for applications
+The final, fourth element (index [3], assigned to C<$default> in the "block"
+example) in the four element list returned by this function is used with the
+C<"a"> format types; it may also be useful for applications
that wish to convert the returned inversion map data structure into some
other, such as a hash. It gives the mapping that most code points map to
under the property. If you establish the convention that any code point not
explicitly listed in your data structure maps to this value, you can
potentially make your data structure much smaller. As you construct your data
structure from the one returned by this function, simply ignore those ranges
-that map to this value, generally called the "default" value. For example, to
+that map to this value. For example, to
convert to the data structure searchable by L</charinrange()>, you can follow
this recipe for properties that don't require adjustments:
- my ($list_ref, $map_ref, $format, $missing) = prop_invmap($property);
+ my ($list_ref, $map_ref, $format, $default) = prop_invmap($property);
my @range_list;
# Look at each element in the list, but the -2 is needed because we
# look at $i+1 in the loop, and the final element is guaranteed to map
- # to $missing by prop_invmap(), so we would skip it anyway.
+ # to $default by prop_invmap(), so we would skip it anyway.
for my $i (0 .. @$list_ref - 2) {
- next if $map_ref->[$i] eq $missing;
+ next if $map_ref->[$i] eq $default;
push @range_list, [ $list_ref->[$i],
$list_ref->[$i+1],
$map_ref->[$i]
print charinrange(\@range_list, $code_point), "\n";
With this, C<charinrange()> will return C<undef> if its input code point maps
-to C<$missing>. You can avoid this by omitting the C<next> statement, and adding
+to C<$default>. You can avoid this by omitting the C<next> statement, and adding
a line after the loop to handle the final element of the inversion map.
Similarly, this recipe can be used for properties that do require adjustments:
for my $i (0 .. @$list_ref - 2) {
- next if $map_ref->[$i] eq $missing;
+ next if $map_ref->[$i] eq $default;
# prop_invmap() guarantees that if the mapping is to an array, the
# range has just one element, so no need to worry about adjustments.
}
# Find the range that the override applies to.
- my $i = _search_invlist(\@invlist, $cp);
+ my $i = search_invlist(\@invlist, $cp);
if ($cp < $invlist[$i] || $cp >= $invlist[$i + 1]) {
croak __PACKAGE__, "::prop_invmap: wrong_range, cp=$cp; i=$i, current=$invlist[$i]; next=$invlist[$i + 1]"
}
return (\@invlist, \@invmap, $format, $missing);
}
+sub search_invlist {
+
+=pod
+
+=head2 B<search_invlist()>
+
+ use Unicode::UCD qw(prop_invmap prop_invlist);
+ use Unicode::UCD 'search_invlist';
+
+ my @invlist = prop_invlist($property_name);
+ print $code_point, ((search_invlist(\@invlist, $code_point) // -1) % 2)
+ ? " isn't"
+ : " is",
+ " in $property_name\n";
+
+ my ($blocks_ranges_ref, $blocks_map_ref) = prop_invmap("Block");
+ my $index = search_invlist($blocks_ranges_ref, $code_point);
+ print "$code_point is in block ", $blocks_map_ref->[$index], "\n";
+
+C<search_invlist> is used to search an inversion list returned by
+C<prop_invlist> or C<prop_invmap> for a particular L</code point argument>.
+C<undef> is returned if the code point is not found in the inversion list
+(this happens only when it is not a legal L<code point argument>, or is less
+than the list's first element). A warning is raised in the first instance.
+
+Otherwise, it returns the index into the list of the range that contains the
+code point.; that is, find C<i> such that
+
+ list[i]<= code_point < list[i+1].
+
+As explained in L</prop_invlist()>, whether a code point is in the list or not
+depends on if the index is even (in) or odd (not in). And as explained in
+L</prop_invmap()>, the index is used with the returned parallel array to find
+the mapping.
+
+=cut
+
+
+ my $list_ref = shift;
+ my $input_code_point = shift;
+ my $code_point = _getcode($input_code_point);
+
+ if (! defined $code_point) {
+ carp __PACKAGE__, "::search_invlist: unknown code '$input_code_point'";
+ return;
+ }
+
+ my $max_element = @$list_ref - 1;
+
+ # Return undef if list is empty or requested item is before the first element.
+ return if $max_element < 0;
+ return if $code_point < $list_ref->[0];
+
+ # Short cut something at the far-end of the table. This also allows us to
+ # refer to element [$i+1] without fear of being out-of-bounds in the loop
+ # below.
+ return $max_element if $code_point >= $list_ref->[$max_element];
+
+ use integer; # want integer division
+
+ my $i = $max_element / 2;
+
+ my $lower = 0;
+ my $upper = $max_element;
+ while (1) {
+
+ if ($code_point >= $list_ref->[$i]) {
+
+ # Here we have met the lower constraint. We can quit if we
+ # also meet the upper one.
+ last if $code_point < $list_ref->[$i+1];
+
+ $lower = $i; # Still too low.
+
+ }
+ else {
+
+ # Here, $code_point < $list_ref[$i], so look lower down.
+ $upper = $i;
+ }
+
+ # Split search domain in half to try again.
+ my $temp = ($upper + $lower) / 2;
+
+ # No point in continuing unless $i changes for next time
+ # in the loop.
+ return $i if $temp == $i;
+ $i = $temp;
+ } # End of while loop
+
+ # Here we have found the offset
+ return $i;
+}
+
=head2 Unicode::UCD::UnicodeVersion
This returns the version of the Unicode Character Database, in other words, the
The difference between a block and a script is that scripts are closer
to the linguistic notion of a set of code points required to present
languages, while block is more of an artifact of the Unicode code point
-numbering and separation into blocks of (mostly) 256 code points.
+numbering and separation into blocks of consecutive code points (so far the
+size of a block is some multiple of 16, like 128 or 256).
For example the Latin B<script> is spread over several B<blocks>, such
as C<Basic Latin>, C<Latin 1 Supplement>, C<Latin Extended-A>, and
https://perl5.git.perl.org / perl5.git / blobdiff