use 5.015;
use strict;
use warnings;
-use Unicode::UCD qw(prop_invlist prop_invmap);
-require 'regen/regen_lib.pl';
-require 'regen/charset_translations.pl';
+use Unicode::UCD qw(prop_aliases
+ prop_values
+ prop_value_aliases
+ prop_invlist
+ prop_invmap search_invlist
+ );
+require './regen/regen_lib.pl';
+require './regen/charset_translations.pl';
# This program outputs charclass_invlists.h, which contains various inversion
# lists in the form of C arrays that are to be used as-is for inversion lists.
# in the headers is used to minimize the possibility of things getting
# out-of-sync, or the wrong data structure being passed. Currently that
# random number is:
+
+# charclass_invlists.h now also has a partial implementation of inversion
+# maps; enough to generate tables for the line break properties, such as GCB
+
my $VERSION_DATA_STRUCTURE_TYPE = 148565664;
+# integer or float
+my $numeric_re = qr/ ^ -? \d+ (:? \. \d+ )? $ /ax;
+
+# Matches valid C language enum names: begins with ASCII alphabetic, then any
+# ASCII \w
+my $enum_name_re = qr / ^ [[:alpha:]] \w* $ /ax;
+
my $out_fh = open_new('charclass_invlists.h', '>',
{style => '*', by => $0,
from => "Unicode::UCD"});
-my $is_in_ifndef_ext_re = 0;
+my $in_file_pound_if = 0;
+
+my $max_hdr_len = 3; # In headings, how wide a name is allowed?
print $out_fh "/* See the generating file for comments */\n\n";
-my %include_in_ext_re = ( NonL1_Perl_Non_Final_Folds => 1 );
+# The symbols generated by this program are all currently defined only in a
+# single dot c each. The code knows where most of them go, but this hash
+# gives overrides for the exceptions to the typical place
+my %exceptions_to_where_to_define =
+ ( NonL1_Perl_Non_Final_Folds => 'PERL_IN_REGCOMP_C',
+ AboveLatin1 => 'PERL_IN_REGCOMP_C',
+ Latin1 => 'PERL_IN_REGCOMP_C',
+ UpperLatin1 => 'PERL_IN_REGCOMP_C',
+ _Perl_Any_Folds => 'PERL_IN_REGCOMP_C',
+ _Perl_Folds_To_Multi_Char => 'PERL_IN_REGCOMP_C',
+ _Perl_IDCont => 'PERL_IN_UTF8_C',
+ _Perl_IDStart => 'PERL_IN_UTF8_C',
+ );
+
+# This hash contains the properties with enums that have hard-coded references
+# to them in C code. It is neeed to make sure that if perl is compiled
+# with an older Unicode data set, that all the enum values the code is
+# expecting will still be in the enum typedef. Thus the code doesn't have to
+# change. The Unicode version won't have any code points that have the enum
+# values not in that version, so the code that handles them will not get
+# exercised. This is far better than having to #ifdef things. The names here
+# should be the long names of the respective property values. The reason for
+# this is because regexec.c uses them as case labels, and the long name is
+# generally more understandable than the short.
+my %hard_coded_enums =
+ ( gcb => [
+ 'Control',
+ 'CR',
+ 'E_Base',
+ 'E_Base_GAZ',
+ 'E_Modifier',
+ 'Extend',
+ 'Glue_After_Zwj',
+ 'L',
+ 'LF',
+ 'LV',
+ 'LVT',
+ 'Other',
+ 'Prepend',
+ 'Regional_Indicator',
+ 'SpacingMark',
+ 'T',
+ 'V',
+ 'ZWJ',
+ ],
+ lb => [
+ 'Alphabetic',
+ 'Break_After',
+ 'Break_Before',
+ 'Break_Both',
+ 'Break_Symbols',
+ 'Carriage_Return',
+ 'Close_Parenthesis',
+ 'Close_Punctuation',
+ 'Combining_Mark',
+ 'Contingent_Break',
+ 'E_Base',
+ 'E_Modifier',
+ 'Exclamation',
+ 'Glue',
+ 'H2',
+ 'H3',
+ 'Hebrew_Letter',
+ 'Hyphen',
+ 'Ideographic',
+ 'Infix_Numeric',
+ 'Inseparable',
+ 'JL',
+ 'JT',
+ 'JV',
+ 'Line_Feed',
+ 'Mandatory_Break',
+ 'Next_Line',
+ 'Nonstarter',
+ 'Numeric',
+ 'Open_Punctuation',
+ 'Postfix_Numeric',
+ 'Prefix_Numeric',
+ 'Quotation',
+ 'Regional_Indicator',
+ 'Space',
+ 'Word_Joiner',
+ 'ZWJ',
+ 'ZWSpace',
+ ],
+ sb => [
+ 'ATerm',
+ 'Close',
+ 'CR',
+ 'Extend',
+ 'Format',
+ 'LF',
+ 'Lower',
+ 'Numeric',
+ 'OLetter',
+ 'Other',
+ 'SContinue',
+ 'Sep',
+ 'Sp',
+ 'STerm',
+ 'Upper',
+ ],
+ wb => [
+ 'ALetter',
+ 'CR',
+ 'Double_Quote',
+ 'E_Base',
+ 'E_Base_GAZ',
+ 'E_Modifier',
+ 'Extend',
+ 'ExtendNumLet',
+ 'Format',
+ 'Glue_After_Zwj',
+ 'Hebrew_Letter',
+ 'Katakana',
+ 'LF',
+ 'MidLetter',
+ 'MidNum',
+ 'MidNumLet',
+ 'Newline',
+ 'Numeric',
+ 'Other',
+ 'Perl_Tailored_HSpace',
+ 'Regional_Indicator',
+ 'Single_Quote',
+ 'ZWJ',
+ ],
+);
+
+my %gcb_enums;
+my @gcb_short_enums;
+my %gcb_abbreviations;
+my %lb_enums;
+my @lb_short_enums;
+my %lb_abbreviations;
+my %wb_enums;
+my @wb_short_enums;
+my %wb_abbreviations;
+
+my @a2n;
+
+sub uniques {
+ # Returns non-duplicated input values. From "Perl Best Practices:
+ # Encapsulated Cleverness". p. 455 in first edition.
+
+ my %seen;
+ return grep { ! $seen{$_}++ } @_;
+}
+
+sub a2n($) {
+ my $cp = shift;
+
+ # Returns the input Unicode code point translated to native.
+
+ return $cp if $cp !~ $numeric_re || $cp > 255;
+ return $a2n[$cp];
+}
-sub end_ifndef_ext_re {
- if ($is_in_ifndef_ext_re) {
- print $out_fh "\n#endif\t/* #ifndef PERL_IN_XSUB_RE */\n";
- $is_in_ifndef_ext_re = 0;
+sub end_file_pound_if {
+ if ($in_file_pound_if) {
+ print $out_fh "\n#endif\t/* $in_file_pound_if */\n";
+ $in_file_pound_if = 0;
+ }
+}
+
+sub switch_pound_if ($$) {
+ my $name = shift;
+ my $new_pound_if = shift;
+
+ # Switch to new #if given by the 2nd argument. If there is an override
+ # for this, it instead switches to that. The 1st argument is the
+ # static's name, used to look up the overrides
+
+ if (exists $exceptions_to_where_to_define{$name}) {
+ $new_pound_if = $exceptions_to_where_to_define{$name};
+ }
+
+ # Exit current #if if the new one is different from the old
+ if ($in_file_pound_if
+ && $in_file_pound_if !~ /$new_pound_if/)
+ {
+ end_file_pound_if;
+ }
+
+ # Enter new #if, if not already in it.
+ if (! $in_file_pound_if) {
+ $in_file_pound_if = "defined($new_pound_if)";
+ print $out_fh "\n#if $in_file_pound_if\n";
}
}
my $charset = shift // ""; # name of character set for comment
die "No inversion list for $name" unless defined $invlist
- && ref $invlist eq 'ARRAY'
- && @$invlist;
+ && ref $invlist eq 'ARRAY';
# Output the inversion list $invlist using the name $name for it.
# It is output in the exact internal form for inversion lists.
# Is the last element of the header 0, or 1 ?
my $zero_or_one = 0;
- if ($invlist->[0] != 0) {
+ if (@$invlist && $invlist->[0] != 0) {
unshift @$invlist, 0;
$zero_or_one = 1;
}
my $count = @$invlist;
- if ($is_in_ifndef_ext_re) {
- if (exists $include_in_ext_re{$name}) {
- end_ifndef_ext_re;
- }
- }
- elsif (! exists $include_in_ext_re{$name}) {
- print $out_fh "\n#ifndef PERL_IN_XSUB_RE\n" unless exists $include_in_ext_re{$name};
- $is_in_ifndef_ext_re = 1;
- }
+ switch_pound_if ($name, 'PERL_IN_PERL_C');
print $out_fh "\nstatic const UV ${name}_invlist[] = {";
print $out_fh " /* for $charset */" if $charset;
# The main body are the UVs passed in to this routine. Do the final
# element separately
- for my $i (0 .. @$invlist - 1 - 1) {
- print $out_fh "\t$invlist->[$i],\n";
+ for my $i (0 .. @$invlist - 1) {
+ printf $out_fh "\t0x%X", $invlist->[$i];
+ print $out_fh "," if $i < @$invlist - 1;
+ print $out_fh "\n";
}
- # The final element does not have a trailing comma, as C can't handle it.
- print $out_fh "\t$invlist->[-1]\n";
+ print $out_fh "};\n";
+}
+
+sub output_invmap ($$$$$$$) {
+ my $name = shift;
+ my $invmap = shift; # Reference to inversion map array
+ my $prop_name = shift;
+ my $input_format = shift; # The inversion map's format
+ my $default = shift; # The property value for code points who
+ # otherwise don't have a value specified.
+ my $extra_enums = shift; # comma-separated list of our additions to the
+ # property's standard possible values
+ my $charset = shift // ""; # name of character set for comment
+ # Output the inversion map $invmap for property $prop_name, but use $name
+ # as the actual data structure's name.
+
+ my $count = @$invmap;
+
+ my $output_format;
+ my $declaration_type;
+ my %enums;
+ my $name_prefix;
+
+ if ($input_format eq 's') {
+ my $orig_prop_name = $prop_name;
+ $prop_name = (prop_aliases($prop_name))[1] // $prop_name =~ s/^_Perl_//r; # Get full name
+ my $short_name = (prop_aliases($prop_name))[0] // $prop_name;
+ my @enums;
+ if ($orig_prop_name eq $prop_name) {
+ @enums = prop_values($prop_name);
+ }
+ else {
+ @enums = uniques(@$invmap);
+ }
+
+ if (! @enums) {
+ die "Only enum properties are currently handled; '$prop_name' isn't one";
+ }
+ else {
+ my @expected_enums = @{$hard_coded_enums{lc $short_name}};
+ my @canonical_input_enums;
+ if (@expected_enums) {
+ if (@expected_enums < @enums) {
+ die 'You need to update %hard_coded_enums to reflect new'
+ . " entries in this Unicode version\n"
+ . "Expected: " . join(", ", sort @expected_enums) . "\n"
+ . " Got: " . join(", ", sort @enums);
+ }
+
+ if (! defined prop_aliases($prop_name)) {
+
+ # Convert the input enums into canonical form and
+ # save for use below
+ @canonical_input_enums = map { lc ($_ =~ s/_//gr) }
+ @enums;
+ }
+ @enums = sort @expected_enums;
+ }
+
+ # The internal enums come last, and in the order specified
+ my @extras;
+ if ($extra_enums ne "") {
+ @extras = split /,/, $extra_enums;
+ push @enums, @extras;
+ }
+
+ # Assign a value to each element of the enum. The default
+ # value always gets 0; the others are arbitrarily assigned.
+ my $enum_val = 0;
+ my $canonical_default = prop_value_aliases($prop_name, $default);
+ $default = $canonical_default if defined $canonical_default;
+ $enums{$default} = $enum_val++;
+ for my $enum (@enums) {
+ $enums{$enum} = $enum_val++ unless exists $enums{$enum};
+ }
+
+ # Calculate the enum values for certain properties like
+ # _Perl_GCB and _Perl_LB, because we output special tables for
+ # them.
+ if ($name =~ / ^ _Perl_ (?: GCB | LB | WB ) $ /x) {
+
+ # We use string evals to allow the same code to work on
+ # all tables we're doing.
+ my $type = lc $prop_name;
+
+ # We use lowercase single letter names for any property
+ # values not in the release of Unicode being compiled now.
+ my $placeholder = "a";
+
+ # Skip if we've already done this code, which populated
+ # this hash
+ if (eval "! \%${type}_enums") {
+
+ # For each enum ...
+ foreach my $enum (sort keys %enums) {
+ my $value = $enums{$enum};
+ my $short;
+ my $abbreviated_from;
+
+ # Special case this wb property value to make the
+ # name more clear
+ if ($enum eq 'Perl_Tailored_HSpace') {
+ $short = 'hs';
+ $abbreviated_from = $enum;
+ }
+ elsif (grep { $_ eq $enum } @extras) {
+
+ # The 'short' name for one of the property
+ # values added by this file is just the
+ # lowercase of it
+ $short = lc $enum;
+ }
+ elsif (grep {$_ eq lc ( $enum =~ s/_//gr) }
+ @canonical_input_enums)
+ { # On Unicode versions that predate the
+ # official property, we have set up this array
+ # to be the canonical form of each enum in the
+ # substitute property. If the enum we're
+ # looking at is canonically the same as one of
+ # these, use its name instead of generating a
+ # placeholder one in the next clause (which
+ # will happen because prop_value_aliases()
+ # will fail because it only works on official
+ # properties)
+ $short = $enum;
+ }
+ else {
+ # Use the official short name for the other
+ # property values, which should all be
+ # official ones.
+ ($short) = prop_value_aliases($type, $enum);
+
+ # But create a placeholder for ones not in
+ # this Unicode version.
+ $short = $placeholder++ unless defined $short;
+ }
+
+ # If our short name is too long, or we already
+ # know that the name is an abbreviation, truncate
+ # to make sure it's short enough, and remember
+ # that we did this so we can later place in a
+ # comment in the generated file
+ if ( $abbreviated_from
+ || length $short > $max_hdr_len)
+ {
+ $short = substr($short, 0, $max_hdr_len);
+ $abbreviated_from = $enum
+ unless $abbreviated_from;
+ # If the name we are to display conflicts, try
+ # another.
+ while (eval "exists
+ \$${type}_abbreviations{$short}")
+ {
+ die $@ if $@;
+ $short++;
+ }
+
+ eval "\$${type}_abbreviations{$short} = '$enum'";
+ die $@ if $@;
+ }
+
+ # Remember the mapping from the property value
+ # (enum) name to its value.
+ eval "\$${type}_enums{$enum} = $value";
+ die $@ if $@;
+
+ # Remember the inverse mapping to the short name
+ # so that we can properly label the generated
+ # table's rows and columns
+ eval "\$${type}_short_enums[$value] = '$short'";
+ die $@ if $@;
+ }
+ }
+ }
+ }
+
+ # Inversion map stuff is currently used only by regexec
+ switch_pound_if($name, 'PERL_IN_REGEXEC_C');
+ {
+
+ # The short names tend to be two lower case letters, but it looks
+ # better for those if they are upper. XXX
+ $short_name = uc($short_name) if length($short_name) < 3
+ || substr($short_name, 0, 1) =~ /[[:lower:]]/;
+ $name_prefix = "${short_name}_";
+ my $enum_count = keys %enums;
+ print $out_fh "\n#define ${name_prefix}ENUM_COUNT ", scalar keys %enums, "\n";
+
+ print $out_fh "\ntypedef enum {\n";
+ my @enum_list;
+ foreach my $enum (keys %enums) {
+ $enum_list[$enums{$enum}] = $enum;
+ }
+ foreach my $i (0 .. @enum_list - 1) {
+ my $name = $enum_list[$i];
+ print $out_fh "\t${name_prefix}$name = $i";
+ print $out_fh "," if $i < $enum_count - 1;
+ print $out_fh "\n";
+ }
+ $declaration_type = "${name_prefix}enum";
+ print $out_fh "} $declaration_type;\n";
+
+ $output_format = "${name_prefix}%s";
+ }
+ }
+ else {
+ die "'$input_format' invmap() format for '$prop_name' unimplemented";
+ }
+
+ die "No inversion map for $prop_name" unless defined $invmap
+ && ref $invmap eq 'ARRAY'
+ && $count;
+
+ print $out_fh "\nstatic const $declaration_type ${name}_invmap[] = {";
+ print $out_fh " /* for $charset */" if $charset;
+ print $out_fh "\n";
+
+ # The main body are the scalars passed in to this routine.
+ for my $i (0 .. $count - 1) {
+ my $element = $invmap->[$i];
+ my $full_element_name = prop_value_aliases($prop_name, $element);
+ $element = $full_element_name if defined $full_element_name;
+ $element = $name_prefix . $element;
+ print $out_fh "\t$element";
+ print $out_fh "," if $i < $count - 1;
+ print $out_fh "\n";
+ }
print $out_fh "};\n";
}
-sub mk_invlist_from_cp_list {
+sub mk_invlist_from_sorted_cp_list {
# Returns an inversion list constructed from the sorted input array of
# code points
my $list_ref = shift;
+ return unless @$list_ref;
+
# Initialize to just the first element
my @invlist = ( $list_ref->[0], $list_ref->[0] + 1);
my ($cp_ref, $folds_ref, $format) = prop_invmap("Case_Folding");
die "Could not find inversion map for Case_Folding" unless defined $format;
die "Incorrect format '$format' for Case_Folding inversion map"
- unless $format eq 'al';
+ unless $format eq 'al'
+ || $format eq 'a';
my @has_multi_char_fold;
my @is_non_final_fold;
sub _Perl_Non_Final_Folds {
@is_non_final_fold = sort { $a <=> $b } @is_non_final_fold;
- return mk_invlist_from_cp_list(\@is_non_final_fold);
+ return mk_invlist_from_sorted_cp_list(\@is_non_final_fold);
+}
+
+sub prop_name_for_cmp ($) { # Sort helper
+ my $name = shift;
+
+ # Returns the input lowercased, with non-alphas removed, as well as
+ # everything starting with a comma
+
+ $name =~ s/,.*//;
+ $name =~ s/[[:^alpha:]]//g;
+ return lc $name;
}
sub UpperLatin1 {
- return mk_invlist_from_cp_list([ 128 .. 255 ]);
+ return mk_invlist_from_sorted_cp_list([ 128 .. 255 ]);
+}
+
+sub output_table_common {
+
+ # Common subroutine to actually output the generated rules table.
+
+ my ($property,
+ $table_value_defines_ref,
+ $table_ref,
+ $names_ref,
+ $abbreviations_ref) = @_;
+ my $size = @$table_ref;
+
+ # Output the #define list, sorted by numeric value
+ if ($table_value_defines_ref) {
+ my $max_name_length = 0;
+ my @defines;
+
+ # Put in order, and at the same time find the longest name
+ while (my ($enum, $value) = each %$table_value_defines_ref) {
+ $defines[$value] = $enum;
+
+ my $length = length $enum;
+ $max_name_length = $length if $length > $max_name_length;
+ }
+
+ print $out_fh "\n";
+
+ # Output, so that the values are vertically aligned in a column after
+ # the longest name
+ foreach my $i (0 .. @defines - 1) {
+ next unless defined $defines[$i];
+ printf $out_fh "#define %-*s %2d\n",
+ $max_name_length,
+ $defines[$i],
+ $i;
+ }
+ }
+
+ my $column_width = 2; # We currently allow 2 digits for the number
+
+ # If the maximum value in the table is 1, it can be a bool. (Being above
+ # a U8 is not currently handled
+ my $max_element = 0;
+ for my $i (0 .. $size - 1) {
+ for my $j (0 .. $size - 1) {
+ next if $max_element >= $table_ref->[$i][$j];
+ $max_element = $table_ref->[$i][$j];
+ }
+ }
+ die "Need wider table column width given '$max_element"
+ if length $max_element > $column_width;
+
+ my $table_type = ($max_element == 1)
+ ? 'bool'
+ : 'U8';
+
+ # If a name is longer than the width set aside for a column, its column
+ # needs to have increased spacing so that the name doesn't get truncated
+ # nor run into an adjacent column
+ my @spacers;
+
+ # If we are being compiled on a Unicode version earlier than that which
+ # this file was designed for, it may be that some of the property values
+ # aren't in the current release, and so would be undefined if we didn't
+ # define them ourselves. Earlier code has done this, making them
+ # lowercase characters of length one. We look to see if any exist, so
+ # that we can add an annotation to the output table
+ my $has_placeholder = 0;
+
+ for my $i (0 .. $size - 1) {
+ no warnings 'numeric';
+ $has_placeholder = 1 if $names_ref->[$i] =~ / ^ [[:lower:]] $ /ax;
+ $spacers[$i] = " " x (length($names_ref->[$i]) - $column_width);
+ }
+
+ print $out_fh "\nstatic const $table_type ${property}_table[$size][$size] = {\n";
+
+ # Calculate the column heading line
+ my $header_line = "/* "
+ . (" " x $max_hdr_len) # We let the row heading meld to
+ # the '*/' for those that are at
+ # the max
+ . " " x 3; # Space for '*/ '
+ # Now each column
+ for my $i (0 .. $size - 1) {
+ $header_line .= sprintf "%s%*s",
+ $spacers[$i],
+ $column_width + 1, # 1 for the ','
+ $names_ref->[$i];
+ }
+ $header_line .= " */\n";
+
+ # If we have annotations, output it now.
+ if ($has_placeholder || scalar %$abbreviations_ref) {
+ my $text = "";
+ foreach my $abbr (sort keys %$abbreviations_ref) {
+ $text .= "; " if $text;
+ $text .= "'$abbr' stands for '$abbreviations_ref->{$abbr}'";
+ }
+ if ($has_placeholder) {
+ $text .= "; other " if $text;
+ $text .= "lowercase names are placeholders for"
+ . " property values not defined until a later Unicode"
+ . " release, so are irrelevant in this one, as they are"
+ . " not assigned to any code points";
+ }
+
+ my $indent = " " x 3;
+ $text = $indent . "/* $text */";
+
+ # Wrap the text so that it is no wider than the table, which the
+ # header line gives.
+ my $output_width = length $header_line;
+ while (length $text > $output_width) {
+ my $cur_line = substr($text, 0, $output_width);
+
+ # Find the first blank back from the right end to wrap at.
+ for (my $i = $output_width -1; $i > 0; $i--) {
+ if (substr($text, $i, 1) eq " ") {
+ print $out_fh substr($text, 0, $i), "\n";
+
+ # Set so will look at just the remaining tail (which will
+ # be indented and have a '*' after the indent
+ $text = $indent . " * " . substr($text, $i + 1);
+ last;
+ }
+ }
+ }
+
+ # And any remaining
+ print $out_fh $text, "\n" if $text;
+ }
+
+ # We calculated the header line earlier just to get its width so that we
+ # could make sure the annotations fit into that.
+ print $out_fh $header_line;
+
+ # Now output the bulk of the table.
+ for my $i (0 .. $size - 1) {
+
+ # First the row heading.
+ printf $out_fh "/* %-*s*/ ", $max_hdr_len, $names_ref->[$i];
+ print $out_fh "{"; # Then the brace for this row
+
+ # Then each column
+ for my $j (0 .. $size -1) {
+ print $out_fh $spacers[$j];
+ printf $out_fh "%*d", $column_width, $table_ref->[$i][$j];
+ print $out_fh "," if $j < $size - 1;
+ }
+ print $out_fh " }";
+ print $out_fh "," if $i < $size - 1;
+ print $out_fh "\n";
+ }
+
+ print $out_fh "};\n";
+}
+
+sub output_GCB_table() {
+
+ # Create and output the pair table for use in determining Grapheme Cluster
+ # Breaks, given in http://www.unicode.org/reports/tr29/.
+ my %gcb_actions = (
+ GCB_NOBREAK => 0,
+ GCB_BREAKABLE => 1,
+ GCB_RI_then_RI => 2, # Rules 12 and 13
+ GCB_EX_then_EM => 3, # Rule 10
+ );
+
+ # The table is constructed in reverse order of the rules, to make the
+ # lower-numbered, higher priority ones override the later ones, as the
+ # algorithm stops at the earliest matching rule
+
+ my @gcb_table;
+ my $table_size = @gcb_short_enums;
+
+ # Otherwise, break everywhere.
+ # GB99 Any ÷ Any
+ for my $i (0 .. $table_size - 1) {
+ for my $j (0 .. $table_size - 1) {
+ $gcb_table[$i][$j] = 1;
+ }
+ }
+
+ # Do not break within emoji flag sequences. That is, do not break between
+ # regional indicator (RI) symbols if there is an odd number of RI
+ # characters before the break point. Must be resolved in runtime code.
+ #
+ # GB12 ^ (RI RI)* RI × RI
+ # GB13 [^RI] (RI RI)* RI × RI
+ $gcb_table[$gcb_enums{'Regional_Indicator'}]
+ [$gcb_enums{'Regional_Indicator'}] = $gcb_actions{GCB_RI_then_RI};
+
+ # Do not break within emoji modifier sequences or emoji zwj sequences.
+ # GB11 ZWJ × ( Glue_After_Zwj | E_Base_GAZ )
+ $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'Glue_After_Zwj'}] = 0;
+ $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'E_Base_GAZ'}] = 0;
+
+ # GB10 ( E_Base | E_Base_GAZ ) Extend* × E_Modifier
+ $gcb_table[$gcb_enums{'Extend'}][$gcb_enums{'E_Modifier'}]
+ = $gcb_actions{GCB_EX_then_EM};
+ $gcb_table[$gcb_enums{'E_Base'}][$gcb_enums{'E_Modifier'}] = 0;
+ $gcb_table[$gcb_enums{'E_Base_GAZ'}][$gcb_enums{'E_Modifier'}] = 0;
+
+ # Do not break before extending characters or ZWJ.
+ # Do not break before SpacingMarks, or after Prepend characters.
+ # GB9b Prepend ×
+ # GB9a × SpacingMark
+ # GB9 × ( Extend | ZWJ )
+ for my $i (0 .. @gcb_table - 1) {
+ $gcb_table[$gcb_enums{'Prepend'}][$i] = 0;
+ $gcb_table[$i][$gcb_enums{'SpacingMark'}] = 0;
+ $gcb_table[$i][$gcb_enums{'Extend'}] = 0;
+ $gcb_table[$i][$gcb_enums{'ZWJ'}] = 0;
+ }
+
+ # Do not break Hangul syllable sequences.
+ # GB8 ( LVT | T) × T
+ $gcb_table[$gcb_enums{'LVT'}][$gcb_enums{'T'}] = 0;
+ $gcb_table[$gcb_enums{'T'}][$gcb_enums{'T'}] = 0;
+
+ # GB7 ( LV | V ) × ( V | T )
+ $gcb_table[$gcb_enums{'LV'}][$gcb_enums{'V'}] = 0;
+ $gcb_table[$gcb_enums{'LV'}][$gcb_enums{'T'}] = 0;
+ $gcb_table[$gcb_enums{'V'}][$gcb_enums{'V'}] = 0;
+ $gcb_table[$gcb_enums{'V'}][$gcb_enums{'T'}] = 0;
+
+ # GB6 L × ( L | V | LV | LVT )
+ $gcb_table[$gcb_enums{'L'}][$gcb_enums{'L'}] = 0;
+ $gcb_table[$gcb_enums{'L'}][$gcb_enums{'V'}] = 0;
+ $gcb_table[$gcb_enums{'L'}][$gcb_enums{'LV'}] = 0;
+ $gcb_table[$gcb_enums{'L'}][$gcb_enums{'LVT'}] = 0;
+
+ # Do not break between a CR and LF. Otherwise, break before and after
+ # controls.
+ # GB5 ÷ ( Control | CR | LF )
+ # GB4 ( Control | CR | LF ) ÷
+ for my $i (0 .. @gcb_table - 1) {
+ $gcb_table[$i][$gcb_enums{'Control'}] = 1;
+ $gcb_table[$i][$gcb_enums{'CR'}] = 1;
+ $gcb_table[$i][$gcb_enums{'LF'}] = 1;
+ $gcb_table[$gcb_enums{'Control'}][$i] = 1;
+ $gcb_table[$gcb_enums{'CR'}][$i] = 1;
+ $gcb_table[$gcb_enums{'LF'}][$i] = 1;
+ }
+
+ # GB3 CR × LF
+ $gcb_table[$gcb_enums{'CR'}][$gcb_enums{'LF'}] = 0;
+
+ # Break at the start and end of text, unless the text is empty
+ # GB1 sot ÷
+ # GB2 ÷ eot
+ for my $i (0 .. @gcb_table - 1) {
+ $gcb_table[$i][$gcb_enums{'EDGE'}] = 1;
+ $gcb_table[$gcb_enums{'EDGE'}][$i] = 1;
+ }
+ $gcb_table[$gcb_enums{'EDGE'}][$gcb_enums{'EDGE'}] = 0;
+
+ output_table_common('GCB', \%gcb_actions,
+ \@gcb_table, \@gcb_short_enums, \%gcb_abbreviations);
+}
+
+sub output_LB_table() {
+
+ # Create and output the enums, #defines, and pair table for use in
+ # determining Line Breaks. This uses the default line break algorithm,
+ # given in http://www.unicode.org/reports/tr14/, but tailored by example 7
+ # in that page, as the Unicode-furnished tests assume that tailoring.
+
+ # The result is really just true or false. But we follow along with tr14,
+ # creating a rule which is false for something like X SP* X. That gets
+ # encoding 2. The rest of the actions are synthetic ones that indicate
+ # some context handling is required. These each are added to the
+ # underlying 0, 1, or 2, instead of replacing them, so that the underlying
+ # value can be retrieved. Actually only rules from 7 through 18 (which
+ # are the ones where space matter) are possible to have 2 added to them.
+ # The others below add just 0 or 1. It might be possible for one
+ # synthetic rule to be added to another, yielding a larger value. This
+ # doesn't happen in the Unicode 8.0 rule set, and as you can see from the
+ # names of the middle grouping below, it is impossible for that to occur
+ # for them because they all start with mutually exclusive classes. That
+ # the final rule can't be added to any of the others isn't obvious from
+ # its name, so it is assigned a power of 2 higher than the others can get
+ # to so any addition would preserve all data. (And the code will reach an
+ # assert(0) on debugging builds should this happen.)
+ my %lb_actions = (
+ LB_NOBREAK => 0,
+ LB_BREAKABLE => 1,
+ LB_NOBREAK_EVEN_WITH_SP_BETWEEN => 2,
+
+ LB_CM_ZWJ_foo => 3, # Rule 9
+ LB_SP_foo => 6, # Rule 18
+ LB_PR_or_PO_then_OP_or_HY => 9, # Rule 25
+ LB_SY_or_IS_then_various => 11, # Rule 25
+ LB_HY_or_BA_then_foo => 13, # Rule 21
+ LB_RI_then_RI => 15, # Rule 30a
+
+ LB_various_then_PO_or_PR => (1<<5), # Rule 25
+ );
+
+ # Construct the LB pair table. This is based on the rules in
+ # http://www.unicode.org/reports/tr14/, but modified as those rules are
+ # designed for someone taking a string of text and sequentially going
+ # through it to find the break opportunities, whereas, Perl requires
+ # determining if a given random spot is a break opportunity, without
+ # knowing all the entire string before it.
+ #
+ # The table is constructed in reverse order of the rules, to make the
+ # lower-numbered, higher priority ones override the later ones, as the
+ # algorithm stops at the earliest matching rule
+
+ my @lb_table;
+ my $table_size = @lb_short_enums;
+
+ # LB31. Break everywhere else
+ for my $i (0 .. $table_size - 1) {
+ for my $j (0 .. $table_size - 1) {
+ $lb_table[$i][$j] = $lb_actions{'LB_BREAKABLE'};
+ }
+ }
+
+ # LB30b Do not break between an emoji base and an emoji modifier.
+ # EB × EM
+ $lb_table[$lb_enums{'E_Base'}][$lb_enums{'E_Modifier'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB30a Break between two regional indicator symbols if and only if there
+ # are an even number of regional indicators preceding the position of the
+ # break.
+ # sot (RI RI)* RI × RI
+ # [^RI] (RI RI)* RI × RI
+ $lb_table[$lb_enums{'Regional_Indicator'}]
+ [$lb_enums{'Regional_Indicator'}] = $lb_actions{'LB_RI_then_RI'};
+
+ # LB30 Do not break between letters, numbers, or ordinary symbols and
+ # opening or closing parentheses.
+ # (AL | HL | NU) × OP
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Open_Punctuation'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Open_Punctuation'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Open_Punctuation'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # CP × (AL | HL | NU)
+ $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB29 Do not break between numeric punctuation and alphabetics (“e.g.”).
+ # IS × (AL | HL)
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB28 Do not break between alphabetics (“at”).
+ # (AL | HL) × (AL | HL)
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB27 Treat a Korean Syllable Block the same as ID.
+ # (JL | JV | JT | H2 | H3) × IN
+ $lb_table[$lb_enums{'JL'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JV'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JT'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H2'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H3'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # (JL | JV | JT | H2 | H3) × PO
+ $lb_table[$lb_enums{'JL'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JV'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JT'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H2'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H3'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # PR × (JL | JV | JT | H2 | H3)
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'JL'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'JV'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'JT'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'H2'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'H3'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB26 Do not break a Korean syllable.
+ # JL × (JL | JV | H2 | H3)
+ $lb_table[$lb_enums{'JL'}][$lb_enums{'JL'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JL'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JL'}][$lb_enums{'H2'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JL'}][$lb_enums{'H3'}] = $lb_actions{'LB_NOBREAK'};
+
+ # (JV | H2) × (JV | JT)
+ $lb_table[$lb_enums{'JV'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H2'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'JV'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H2'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
+
+ # (JT | H3) × JT
+ $lb_table[$lb_enums{'JT'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'H3'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
+
+ # LB25 Do not break between the following pairs of classes relevant to
+ # numbers, as tailored by example 7 in
+ # http://www.unicode.org/reports/tr14/#Examples
+ # We follow that tailoring because Unicode's test cases expect it
+ # (PR | PO) × ( OP | HY )? NU
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # Given that (OP | HY )? is optional, we have to test for it in code.
+ # We add in the action (instead of overriding) for this, so that in
+ # the code we can recover the underlying break value.
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Open_Punctuation'}]
+ += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
+ $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Open_Punctuation'}]
+ += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Hyphen'}]
+ += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
+ $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Hyphen'}]
+ += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
+
+ # ( OP | HY ) × NU
+ $lb_table[$lb_enums{'Open_Punctuation'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hyphen'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # NU (NU | SY | IS)* × (NU | SY | IS | CL | CP )
+ # which can be rewritten as:
+ # NU (SY | IS)* × (NU | SY | IS | CL | CP )
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Break_Symbols'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Infix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Close_Punctuation'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Close_Parenthesis'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # Like earlier where we have to test in code, we add in the action so
+ # that we can recover the underlying values. This is done in rules
+ # below, as well. The code assumes that we haven't added 2 actions.
+ # Shoul a later Unicode release break that assumption, then tests
+ # should start failing.
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Numeric'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Break_Symbols'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Infix_Numeric'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Close_Punctuation'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Close_Parenthesis'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Numeric'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Break_Symbols'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Infix_Numeric'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Close_Punctuation'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Close_Parenthesis'}]
+ += $lb_actions{'LB_SY_or_IS_then_various'};
+
+ # NU (NU | SY | IS)* (CL | CP)? × (PO | PR)
+ # which can be rewritten as:
+ # NU (SY | IS)* (CL | CP)? × (PO | PR)
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Prefix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Postfix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+ $lb_table[$lb_enums{'Close_Punctuation'}][$lb_enums{'Postfix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Postfix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Postfix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+
+ $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Prefix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+ $lb_table[$lb_enums{'Close_Punctuation'}][$lb_enums{'Prefix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+ $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Prefix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Prefix_Numeric'}]
+ += $lb_actions{'LB_various_then_PO_or_PR'};
+
+ # LB24 Do not break between numeric prefix/postfix and letters, or between
+ # letters and prefix/postfix.
+ # (PR | PO) × (AL | HL)
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # (AL | HL) × (PR | PO)
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Prefix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Prefix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB23a Do not break between numeric prefixes and ideographs, or between
+ # ideographs and numeric postfixes.
+ # PR × (ID | EB | EM)
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Ideographic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'E_Base'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'E_Modifier'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # (ID | EB | EM) × PO
+ $lb_table[$lb_enums{'Ideographic'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'E_Base'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'E_Modifier'}][$lb_enums{'Postfix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB23 Do not break between digits and letters
+ # (AL | HL) × NU
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Numeric'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # NU × (AL | HL)
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Alphabetic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB22 Do not break between two ellipses, or between letters, numbers or
+ # exclamations and ellipsis.
+ # (AL | HL) × IN
+ $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # Exclamation × IN
+ $lb_table[$lb_enums{'Exclamation'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # (ID | EB | EM) × IN
+ $lb_table[$lb_enums{'Ideographic'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'E_Base'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'E_Modifier'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # IN × IN
+ $lb_table[$lb_enums{'Inseparable'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # NU × IN
+ $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Inseparable'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB21b Don’t break between Solidus and Hebrew letters.
+ # SY × HL
+ $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Hebrew_Letter'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB21a Don't break after Hebrew + Hyphen.
+ # HL (HY | BA) ×
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'Hyphen'}][$i]
+ += $lb_actions{'LB_HY_or_BA_then_foo'};
+ $lb_table[$lb_enums{'Break_After'}][$i]
+ += $lb_actions{'LB_HY_or_BA_then_foo'};
+ }
+
+ # LB21 Do not break before hyphen-minus, other hyphens, fixed-width
+ # spaces, small kana, and other non-starters, or after acute accents.
+ # × BA
+ # × HY
+ # × NS
+ # BB ×
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'Break_After'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$i][$lb_enums{'Hyphen'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$i][$lb_enums{'Nonstarter'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Break_Before'}][$i] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB20 Break before and after unresolved CB.
+ # ÷ CB
+ # CB ÷
+ # Conditional breaks should be resolved external to the line breaking
+ # rules. However, the default action is to treat unresolved CB as breaking
+ # before and after.
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'Contingent_Break'}]
+ = $lb_actions{'LB_BREAKABLE'};
+ $lb_table[$lb_enums{'Contingent_Break'}][$i]
+ = $lb_actions{'LB_BREAKABLE'};
+ }
+
+ # LB19 Do not break before or after quotation marks, such as ‘ ” ’.
+ # × QU
+ # QU ×
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'Quotation'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'Quotation'}][$i] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB18 Break after spaces
+ # SP ÷
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'Space'}][$i] = $lb_actions{'LB_BREAKABLE'};
+ }
+
+ # LB17 Do not break within ‘——’, even with intervening spaces.
+ # B2 SP* × B2
+ $lb_table[$lb_enums{'Break_Both'}][$lb_enums{'Break_Both'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+
+ # LB16 Do not break between closing punctuation and a nonstarter even with
+ # intervening spaces.
+ # (CL | CP) SP* × NS
+ $lb_table[$lb_enums{'Close_Punctuation'}][$lb_enums{'Nonstarter'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Nonstarter'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+
+
+ # LB15 Do not break within ‘”[’, even with intervening spaces.
+ # QU SP* × OP
+ $lb_table[$lb_enums{'Quotation'}][$lb_enums{'Open_Punctuation'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+
+ # LB14 Do not break after ‘[’, even after spaces.
+ # OP SP* ×
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'Open_Punctuation'}][$i]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ }
+
+ # LB13 Do not break before ‘]’ or ‘!’ or ‘;’ or ‘/’, even after spaces, as
+ # tailored by example 7 in http://www.unicode.org/reports/tr14/#Examples
+ # [^NU] × CL
+ # [^NU] × CP
+ # × EX
+ # [^NU] × IS
+ # [^NU] × SY
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'Exclamation'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+
+ next if $i == $lb_enums{'Numeric'};
+
+ $lb_table[$i][$lb_enums{'Close_Punctuation'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ $lb_table[$i][$lb_enums{'Close_Parenthesis'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ $lb_table[$i][$lb_enums{'Infix_Numeric'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ $lb_table[$i][$lb_enums{'Break_Symbols'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ }
+
+ # LB12a Do not break before NBSP and related characters, except after
+ # spaces and hyphens.
+ # [^SP BA HY] × GL
+ for my $i (0 .. @lb_table - 1) {
+ next if $i == $lb_enums{'Space'}
+ || $i == $lb_enums{'Break_After'}
+ || $i == $lb_enums{'Hyphen'};
+
+ # We don't break, but if a property above has said don't break even
+ # with space between, don't override that (also in the next few rules)
+ next if $lb_table[$i][$lb_enums{'Glue'}]
+ == $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ $lb_table[$i][$lb_enums{'Glue'}] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB12 Do not break after NBSP and related characters.
+ # GL ×
+ for my $i (0 .. @lb_table - 1) {
+ next if $lb_table[$lb_enums{'Glue'}][$i]
+ == $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+ $lb_table[$lb_enums{'Glue'}][$i] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB11 Do not break before or after Word joiner and related characters.
+ # × WJ
+ # WJ ×
+ for my $i (0 .. @lb_table - 1) {
+ if ($lb_table[$i][$lb_enums{'Word_Joiner'}]
+ != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
+ {
+ $lb_table[$i][$lb_enums{'Word_Joiner'}] = $lb_actions{'LB_NOBREAK'};
+ }
+ if ($lb_table[$lb_enums{'Word_Joiner'}][$i]
+ != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
+ {
+ $lb_table[$lb_enums{'Word_Joiner'}][$i] = $lb_actions{'LB_NOBREAK'};
+ }
+ }
+
+ # Special case this here to avoid having to do a special case in the code,
+ # by making this the same as other things with a SP in front of them that
+ # don't break, we avoid an extra test
+ $lb_table[$lb_enums{'Space'}][$lb_enums{'Word_Joiner'}]
+ = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
+
+ # LB9 and LB10 are done in the same loop
+ #
+ # LB9 Do not break a combining character sequence; treat it as if it has
+ # the line breaking class of the base character in all of the
+ # higher-numbered rules. Treat ZWJ as if it were CM
+ # Treat X (CM|ZWJ)* as if it were X.
+ # where X is any line break class except BK, CR, LF, NL, SP, or ZW.
+
+ # LB10 Treat any remaining combining mark or ZWJ as AL. This catches the
+ # case where a CM or ZWJ is the first character on the line or follows SP,
+ # BK, CR, LF, NL, or ZW.
+ for my $i (0 .. @lb_table - 1) {
+
+ # When the CM or ZWJ is the first in the pair, we don't know without
+ # looking behind whether the CM or ZWJ is going to attach to an
+ # earlier character, or not. So have to figure this out at runtime in
+ # the code
+ $lb_table[$lb_enums{'Combining_Mark'}][$i]
+ = $lb_actions{'LB_CM_ZWJ_foo'};
+ $lb_table[$lb_enums{'ZWJ'}][$i] = $lb_actions{'LB_CM_ZWJ_foo'};
+
+ if ( $i == $lb_enums{'Mandatory_Break'}
+ || $i == $lb_enums{'EDGE'}
+ || $i == $lb_enums{'Carriage_Return'}
+ || $i == $lb_enums{'Line_Feed'}
+ || $i == $lb_enums{'Next_Line'}
+ || $i == $lb_enums{'Space'}
+ || $i == $lb_enums{'ZWSpace'})
+ {
+ # For these classes, a following CM doesn't combine, and should do
+ # whatever 'Alphabetic' would do.
+ $lb_table[$i][$lb_enums{'Combining_Mark'}]
+ = $lb_table[$i][$lb_enums{'Alphabetic'}];
+ $lb_table[$i][$lb_enums{'ZWJ'}]
+ = $lb_table[$i][$lb_enums{'Alphabetic'}];
+ }
+ else {
+ # For these classes, the CM or ZWJ combines, so doesn't break,
+ # inheriting the type of nobreak from the master character.
+ if ($lb_table[$i][$lb_enums{'Combining_Mark'}]
+ != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
+ {
+ $lb_table[$i][$lb_enums{'Combining_Mark'}]
+ = $lb_actions{'LB_NOBREAK'};
+ }
+ if ($lb_table[$i][$lb_enums{'ZWJ'}]
+ != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
+ {
+ $lb_table[$i][$lb_enums{'ZWJ'}]
+ = $lb_actions{'LB_NOBREAK'};
+ }
+ }
+ }
+
+ # LB8a Do not break between a zero width joiner and an ideograph, emoji
+ # base or emoji modifier. This rule prevents breaks within emoji joiner
+ # sequences.
+ # ZWJ × (ID | EB | EM)
+ $lb_table[$lb_enums{'ZWJ'}][$lb_enums{'Ideographic'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'ZWJ'}][$lb_enums{'E_Base'}]
+ = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$lb_enums{'ZWJ'}][$lb_enums{'E_Modifier'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB8 Break before any character following a zero-width space, even if one
+ # or more spaces intervene.
+ # ZW SP* ÷
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'ZWSpace'}][$i] = $lb_actions{'LB_BREAKABLE'};
+ }
+
+ # Because of LB8-10, we need to look at context for "SP x", and this must
+ # be done in the code. So override the existing rules for that, by adding
+ # a constant to get new rules that tell the code it needs to look at
+ # context. By adding this action instead of replacing the existing one,
+ # we can get back to the original rule if necessary.
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'Space'}][$i] += $lb_actions{'LB_SP_foo'};
+ }
+
+ # LB7 Do not break before spaces or zero width space.
+ # × SP
+ # × ZW
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'Space'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$i][$lb_enums{'ZWSpace'}] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB6 Do not break before hard line breaks.
+ # × ( BK | CR | LF | NL )
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'Mandatory_Break'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$i][$lb_enums{'Carriage_Return'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$i][$lb_enums{'Line_Feed'}] = $lb_actions{'LB_NOBREAK'};
+ $lb_table[$i][$lb_enums{'Next_Line'}] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB5 Treat CR followed by LF, as well as CR, LF, and NL as hard line breaks.
+ # CR × LF
+ # CR !
+ # LF !
+ # NL !
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'Carriage_Return'}][$i]
+ = $lb_actions{'LB_BREAKABLE'};
+ $lb_table[$lb_enums{'Line_Feed'}][$i] = $lb_actions{'LB_BREAKABLE'};
+ $lb_table[$lb_enums{'Next_Line'}][$i] = $lb_actions{'LB_BREAKABLE'};
+ }
+ $lb_table[$lb_enums{'Carriage_Return'}][$lb_enums{'Line_Feed'}]
+ = $lb_actions{'LB_NOBREAK'};
+
+ # LB4 Always break after hard line breaks.
+ # BK !
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$lb_enums{'Mandatory_Break'}][$i]
+ = $lb_actions{'LB_BREAKABLE'};
+ }
+
+ # LB3 Always break at the end of text.
+ # ! eot
+ # LB2 Never break at the start of text.
+ # sot ×
+ for my $i (0 .. @lb_table - 1) {
+ $lb_table[$i][$lb_enums{'EDGE'}] = $lb_actions{'LB_BREAKABLE'};
+ $lb_table[$lb_enums{'EDGE'}][$i] = $lb_actions{'LB_NOBREAK'};
+ }
+
+ # LB1 Assign a line breaking class to each code point of the input.
+ # Resolve AI, CB, CJ, SA, SG, and XX into other line breaking classes
+ # depending on criteria outside the scope of this algorithm.
+ #
+ # In the absence of such criteria all characters with a specific
+ # combination of original class and General_Category property value are
+ # resolved as follows:
+ # Original Resolved General_Category
+ # AI, SG, XX AL Any
+ # SA CM Only Mn or Mc
+ # SA AL Any except Mn and Mc
+ # CJ NS Any
+ #
+ # This is done in mktables, so we never see any of the remapped-from
+ # classes.
+
+ output_table_common('LB', \%lb_actions,
+ \@lb_table, \@lb_short_enums, \%lb_abbreviations);
+}
+
+sub output_WB_table() {
+
+ # Create and output the enums, #defines, and pair table for use in
+ # determining Word Breaks, given in http://www.unicode.org/reports/tr29/.
+
+ # This uses the same mechanism in the other bounds tables generated by
+ # this file. The actions that could override a 0 or 1 are added to those
+ # numbers; the actions that clearly don't depend on the underlying rule
+ # simply overwrite
+ my %wb_actions = (
+ WB_NOBREAK => 0,
+ WB_BREAKABLE => 1,
+ WB_hs_then_hs => 2,
+ WB_Ex_or_FO_or_ZWJ_then_foo => 3,
+ WB_DQ_then_HL => 4,
+ WB_HL_then_DQ => 6,
+ WB_LE_or_HL_then_MB_or_ML_or_SQ => 8,
+ WB_MB_or_ML_or_SQ_then_LE_or_HL => 10,
+ WB_MB_or_MN_or_SQ_then_NU => 12,
+ WB_NU_then_MB_or_MN_or_SQ => 14,
+ WB_RI_then_RI => 16,
+ );
+
+ # Construct the WB pair table.
+ # The table is constructed in reverse order of the rules, to make the
+ # lower-numbered, higher priority ones override the later ones, as the
+ # algorithm stops at the earliest matching rule
+
+ my @wb_table;
+ my $table_size = @wb_short_enums - 1; # -1 because we don't use UNKNOWN
+
+ # Otherwise, break everywhere (including around ideographs).
+ # WB99 Any ÷ Any
+ for my $i (0 .. $table_size - 1) {
+ for my $j (0 .. $table_size - 1) {
+ $wb_table[$i][$j] = $wb_actions{'WB_BREAKABLE'};
+ }
+ }
+
+ # Do not break within emoji flag sequences. That is, do not break between
+ # regional indicator (RI) symbols if there is an odd number of RI
+ # characters before the break point.
+ # WB16 [^RI] (RI RI)* RI × RI
+ # WB15 ^ (RI RI)* RI × RI
+ $wb_table[$wb_enums{'Regional_Indicator'}]
+ [$wb_enums{'Regional_Indicator'}] = $wb_actions{'WB_RI_then_RI'};
+
+ # Do not break within emoji modifier sequences.
+ # WB14 ( E_Base | EBG ) × E_Modifier
+ $wb_table[$wb_enums{'E_Base'}][$wb_enums{'E_Modifier'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'E_Base_GAZ'}][$wb_enums{'E_Modifier'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # Do not break from extenders.
+ # WB13b ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana)
+ $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'ALetter'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Hebrew_Letter'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Numeric'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Katakana'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # WB13a (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet)
+ # × # ExtendNumLet
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'ExtendNumLet'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'ExtendNumLet'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'ExtendNumLet'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Katakana'}][$wb_enums{'ExtendNumLet'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'ExtendNumLet'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # Do not break between Katakana.
+ # WB13 Katakana × Katakana
+ $wb_table[$wb_enums{'Katakana'}][$wb_enums{'Katakana'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # Do not break within sequences, such as “3.2” or “3,456.789”.
+ # WB12 Numeric × (MidNum | MidNumLet | Single_Quote) Numeric
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'MidNumLet'}]
+ += $wb_actions{'WB_NU_then_MB_or_MN_or_SQ'};
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'MidNum'}]
+ += $wb_actions{'WB_NU_then_MB_or_MN_or_SQ'};
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Single_Quote'}]
+ += $wb_actions{'WB_NU_then_MB_or_MN_or_SQ'};
+
+ # WB11 Numeric (MidNum | (MidNumLet | Single_Quote)) × Numeric
+ $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'Numeric'}]
+ += $wb_actions{'WB_MB_or_MN_or_SQ_then_NU'};
+ $wb_table[$wb_enums{'MidNum'}][$wb_enums{'Numeric'}]
+ += $wb_actions{'WB_MB_or_MN_or_SQ_then_NU'};
+ $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'Numeric'}]
+ += $wb_actions{'WB_MB_or_MN_or_SQ_then_NU'};
+
+ # Do not break within sequences of digits, or digits adjacent to letters
+ # (“3a”, or “A3”).
+ # WB10 Numeric × (ALetter | Hebrew_Letter)
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'ALetter'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Hebrew_Letter'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # WB9 (ALetter | Hebrew_Letter) × Numeric
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Numeric'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Numeric'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # WB8 Numeric × Numeric
+ $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Numeric'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # Do not break letters across certain punctuation.
+ # WB7c Hebrew_Letter Double_Quote × Hebrew_Letter
+ $wb_table[$wb_enums{'Double_Quote'}][$wb_enums{'Hebrew_Letter'}]
+ += $wb_actions{'WB_DQ_then_HL'};
+
+ # WB7b Hebrew_Letter × Double_Quote Hebrew_Letter
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Double_Quote'}]
+ += $wb_actions{'WB_HL_then_DQ'};
+
+ # WB7a Hebrew_Letter × Single_Quote
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Single_Quote'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # WB7 (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote)
+ # × (ALetter | Hebrew_Letter)
+ $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'ALetter'}]
+ += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+ $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'Hebrew_Letter'}]
+ += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+ $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'ALetter'}]
+ += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+ $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'Hebrew_Letter'}]
+ += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+ $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'ALetter'}]
+ += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+ $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'Hebrew_Letter'}]
+ += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+
+ # WB6 (ALetter | Hebrew_Letter) × (MidLetter | MidNumLet
+ # | Single_Quote) (ALetter | Hebrew_Letter)
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'MidNumLet'}]
+ += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'MidNumLet'}]
+ += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'MidLetter'}]
+ += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'MidLetter'}]
+ += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Single_Quote'}]
+ += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Single_Quote'}]
+ += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+
+ # Do not break between most letters.
+ # WB5 (ALetter | Hebrew_Letter) × (ALetter | Hebrew_Letter)
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'ALetter'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Hebrew_Letter'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'ALetter'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Hebrew_Letter'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # Ignore Format and Extend characters, except after sot, CR, LF, and
+ # Newline. This also has the effect of: Any × (Format | Extend | ZWJ)
+ # WB4 X (Extend | Format | ZWJ)* → X
+ for my $i (0 .. @wb_table - 1) {
+ $wb_table[$wb_enums{'Extend'}][$i]
+ = $wb_actions{'WB_Ex_or_FO_or_ZWJ_then_foo'};
+ $wb_table[$wb_enums{'Format'}][$i]
+ = $wb_actions{'WB_Ex_or_FO_or_ZWJ_then_foo'};
+ $wb_table[$wb_enums{'ZWJ'}][$i]
+ = $wb_actions{'WB_Ex_or_FO_or_ZWJ_then_foo'};
+ }
+ for my $i (0 .. @wb_table - 1) {
+ $wb_table[$i][$wb_enums{'Extend'}] = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$i][$wb_enums{'Format'}] = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$i][$wb_enums{'ZWJ'}] = $wb_actions{'WB_NOBREAK'};
+ }
+
+ # Implied is that these attach to the character before them, except for
+ # the characters that mark the end of a region of text. The rules below
+ # override the ones set up here, for all the characters that need
+ # overriding.
+ for my $i (0 .. @wb_table - 1) {
+ $wb_table[$i][$wb_enums{'Extend'}] = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$i][$wb_enums{'Format'}] = $wb_actions{'WB_NOBREAK'};
+ }
+
+ # Do not break within emoji zwj sequences.
+ # WB3c ZWJ × ( Glue_After_Zwj | EBG )
+ $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'Glue_After_Zwj'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'E_Base_GAZ'}]
+ = $wb_actions{'WB_NOBREAK'};
+
+ # Break before and after white space
+ # WB3b ÷ (Newline | CR | LF)
+ # WB3a (Newline | CR | LF) ÷
+ # et. al.
+ for my $i ('CR', 'LF', 'Newline', 'Perl_Tailored_HSpace') {
+ for my $j (0 .. @wb_table - 1) {
+ $wb_table[$j][$wb_enums{$i}] = $wb_actions{'WB_BREAKABLE'};
+ $wb_table[$wb_enums{$i}][$j] = $wb_actions{'WB_BREAKABLE'};
+ }
+ }
+
+ # But do not break within white space.
+ # WB3 CR × LF
+ # et.al.
+ for my $i ('CR', 'LF', 'Newline', 'Perl_Tailored_HSpace') {
+ for my $j ('CR', 'LF', 'Newline', 'Perl_Tailored_HSpace') {
+ $wb_table[$wb_enums{$i}][$wb_enums{$j}] = $wb_actions{'WB_NOBREAK'};
+ }
+ }
+
+ # And do not break horizontal space followed by Extend or Format or ZWJ
+ $wb_table[$wb_enums{'Perl_Tailored_HSpace'}][$wb_enums{'Extend'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Perl_Tailored_HSpace'}][$wb_enums{'Format'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Perl_Tailored_HSpace'}][$wb_enums{'ZWJ'}]
+ = $wb_actions{'WB_NOBREAK'};
+ $wb_table[$wb_enums{'Perl_Tailored_HSpace'}]
+ [$wb_enums{'Perl_Tailored_HSpace'}]
+ = $wb_actions{'WB_hs_then_hs'};
+
+ # Break at the start and end of text, unless the text is empty
+ # WB2 Any ÷ eot
+ # WB1 sot ÷ Any
+ for my $i (0 .. @wb_table - 1) {
+ $wb_table[$i][$wb_enums{'EDGE'}] = $wb_actions{'WB_BREAKABLE'};
+ $wb_table[$wb_enums{'EDGE'}][$i] = $wb_actions{'WB_BREAKABLE'};
+ }
+ $wb_table[$wb_enums{'EDGE'}][$wb_enums{'EDGE'}] = 0;
+
+ output_table_common('WB', \%wb_actions,
+ \@wb_table, \@wb_short_enums, \%wb_abbreviations);
}
output_invlist("Latin1", [ 0, 256 ]);
output_invlist("AboveLatin1", [ 256 ]);
-end_ifndef_ext_re;
+end_file_pound_if;
# We construct lists for all the POSIX and backslash sequence character
# classes in two forms:
for my $charset (get_supported_code_pages()) {
print $out_fh "\n" . get_conditional_compile_line_start($charset);
- my @a2n = @{get_a2n($charset)};
- for my $prop (qw(
- ASCII
- Cased
- VertSpace
- XPerlSpace
- XPosixAlnum
- XPosixAlpha
- XPosixBlank
- XPosixCntrl
- XPosixDigit
- XPosixGraph
- XPosixLower
- XPosixPrint
- XPosixPunct
- XPosixSpace
- XPosixUpper
- XPosixWord
- XPosixXDigit
- _Perl_Any_Folds
- &NonL1_Perl_Non_Final_Folds
- _Perl_Folds_To_Multi_Char
- &UpperLatin1
- _Perl_IDStart
- _Perl_IDCont
- )
+ @a2n = @{get_a2n($charset)};
+ no warnings 'qw';
+ # Ignore non-alpha in sort
+ for my $prop (sort { prop_name_for_cmp($a) cmp prop_name_for_cmp($b) } qw(
+ Assigned
+ ASCII
+ Cased
+ VertSpace
+ XPerlSpace
+ XPosixAlnum
+ XPosixAlpha
+ XPosixBlank
+ XPosixCntrl
+ XPosixDigit
+ XPosixGraph
+ XPosixLower
+ XPosixPrint
+ XPosixPunct
+ XPosixSpace
+ XPosixUpper
+ XPosixWord
+ XPosixXDigit
+ _Perl_Any_Folds
+ &NonL1_Perl_Non_Final_Folds
+ _Perl_Folds_To_Multi_Char
+ &UpperLatin1
+ _Perl_IDStart
+ _Perl_IDCont
+ _Perl_GCB,EDGE
+ _Perl_LB,EDGE
+ _Perl_SB,EDGE
+ _Perl_WB,EDGE,UNKNOWN
+ )
) {
# For the Latin1 properties, we change to use the eXtended version of the
# at 255, but does include the one at 256. We don't include the 256 one.
my $prop_name = $prop;
my $is_local_sub = $prop_name =~ s/^&//;
+ my $extra_enums = "";
+ $extra_enums = $1 if $prop_name =~ s/, ( .* ) //x;
my $lookup_prop = $prop_name;
my $l1_only = ($lookup_prop =~ s/^L1Posix/XPosix/
or $lookup_prop =~ s/^L1//);
my $nonl1_only = 0;
$nonl1_only = $lookup_prop =~ s/^NonL1// unless $l1_only;
+ ($lookup_prop, my $has_suffixes) = $lookup_prop =~ / (.*) ( , .* )? /x;
my @invlist;
+ my @invmap;
+ my $map_format;
+ my $map_default;
+ my $maps_to_code_point;
+ my $to_adjust;
if ($is_local_sub) {
@invlist = eval $lookup_prop;
+ die $@ if $@;
}
else {
@invlist = prop_invlist($lookup_prop, '_perl_core_internal_ok');
- }
- die "Could not find inversion list for '$lookup_prop'" unless @invlist;
- my @full_list;
- for (my $i = 0; $i < @invlist; $i += 2) {
- my $upper = ($i + 1) < @invlist
- ? $invlist[$i+1] - 1 # In range
- : $Unicode::UCD::MAX_CP; # To infinity. You may want
- # to stop much much earlier;
- # going this high may expose
- # perl deficiencies with very
- # large numbers.
- for my $j ($invlist[$i] .. $upper) {
- if ($j < 256) {
- push @full_list, $a2n[$j];
+ if (! @invlist) {
+
+ # If couldn't find a non-empty inversion list, see if it is
+ # instead an inversion map
+ my ($list_ref, $map_ref, $format, $default)
+ = prop_invmap($lookup_prop, '_perl_core_internal_ok');
+ if (! $list_ref) {
+ # An empty return here could mean an unknown property, or
+ # merely that the original inversion list is empty. Call
+ # in scalar context to differentiate
+ my $count = prop_invlist($lookup_prop,
+ '_perl_core_internal_ok');
+ die "Could not find inversion list for '$lookup_prop'"
+ unless defined $count;
}
else {
- push @full_list, $j;
+ @invlist = @$list_ref;
+ @invmap = @$map_ref;
+ $map_format = $format;
+ $map_default = $default;
+ $maps_to_code_point = $map_format =~ /x/;
+ $to_adjust = $map_format =~ /a/;
+ }
+ }
+ }
+
+
+ # Short-circuit an empty inversion list.
+ if (! @invlist) {
+ output_invlist($prop_name, \@invlist, $charset);
+ next;
+ }
+
+ # Re-order the Unicode code points to native ones for this platform.
+ # This is only needed for code points below 256, because native code
+ # points are only in that range. For inversion maps of properties
+ # where the mappings are adjusted (format =~ /a/), this reordering
+ # could mess up the adjustment pattern that was in the input, so that
+ # has to be dealt with.
+ #
+ # And inversion maps that map to code points need to eventually have
+ # all those code points remapped to native, and it's better to do that
+ # here, going through the whole list not just those below 256. This
+ # is because some inversion maps have adjustments (format =~ /a/)
+ # which may be affected by the reordering. This code needs to be done
+ # both for when we are translating the inversion lists for < 256, and
+ # for the inversion maps for everything. By doing both in this loop,
+ # we can share that code.
+ #
+ # So, we go through everything for an inversion map to code points;
+ # otherwise, we can skip any remapping at all if we are going to
+ # output only the above-Latin1 values, or if the range spans the whole
+ # of 0..256, as the remap will also include all of 0..256 (256 not
+ # 255 because a re-ordering could cause 256 to need to be in the same
+ # range as 255.)
+ if ((@invmap && $maps_to_code_point)
+ || (! $nonl1_only || ($invlist[0] < 256
+ && ! ($invlist[0] == 0 && $invlist[1] > 256))))
+ {
+
+ if (! @invmap) { # Straight inversion list
+ # Look at all the ranges that start before 257.
+ my @latin1_list;
+ while (@invlist) {
+ last if $invlist[0] > 256;
+ my $upper = @invlist > 1
+ ? $invlist[1] - 1 # In range
+
+ # To infinity. You may want to stop much much
+ # earlier; going this high may expose perl
+ # deficiencies with very large numbers.
+ : $Unicode::UCD::MAX_CP;
+ for my $j ($invlist[0] .. $upper) {
+ push @latin1_list, a2n($j);
+ }
+
+ shift @invlist; # Shift off the range that's in the list
+ shift @invlist; # Shift off the range not in the list
+ }
+
+ # Here @invlist contains all the ranges in the original that start
+ # at code points above 256, and @latin1_list contains all the
+ # native code points for ranges that start with a Unicode code
+ # point below 257. We sort the latter and convert it to inversion
+ # list format. Then simply prepend it to the list of the higher
+ # code points.
+ @latin1_list = sort { $a <=> $b } @latin1_list;
+ @latin1_list = mk_invlist_from_sorted_cp_list(\@latin1_list);
+ unshift @invlist, @latin1_list;
+ }
+ else { # Is an inversion map
+
+ # This is a similar procedure as plain inversion list, but has
+ # multiple buckets. A plain inversion list just has two
+ # buckets, 1) 'in' the list; and 2) 'not' in the list, and we
+ # pretty much can ignore the 2nd bucket, as it is completely
+ # defined by the 1st. But here, what we do is create buckets
+ # which contain the code points that map to each, translated
+ # to native and turned into an inversion list. Thus each
+ # bucket is an inversion list of native code points that map
+ # to it or don't map to it. We use these to create an
+ # inversion map for the whole property.
+
+ # As mentioned earlier, we use this procedure to not just
+ # remap the inversion list to native values, but also the maps
+ # of code points to native ones. In the latter case we have
+ # to look at the whole of the inversion map (or at least to
+ # above Unicode; as the maps of code points above that should
+ # all be to the default).
+ my $upper_limit = ($maps_to_code_point) ? 0x10FFFF : 256;
+
+ my %mapped_lists; # A hash whose keys are the buckets.
+ while (@invlist) {
+ last if $invlist[0] > $upper_limit;
+
+ # This shouldn't actually happen, as prop_invmap() returns
+ # an extra element at the end that is beyond $upper_limit
+ die "inversion map that extends to infinity is unimplemented" unless @invlist > 1;
+
+ my $bucket;
+
+ # A hash key can't be a ref (we are only expecting arrays
+ # of scalars here), so convert any such to a string that
+ # will be converted back later (using a vertical tab as
+ # the separator). Even if the mapping is to code points,
+ # we don't translate to native here because the code
+ # output_map() calls to output these arrays assumes the
+ # input is Unicode, not native.
+ if (ref $invmap[0]) {
+ $bucket = join "\cK", @{$invmap[0]};
+ }
+ elsif ($maps_to_code_point && $invmap[0] =~ $numeric_re) {
+
+ # Do convert to native for maps to single code points.
+ # There are some properties that have a few outlier
+ # maps that aren't code points, so the above test
+ # skips those.
+ $bucket = a2n($invmap[0]);
+ } else {
+ $bucket = $invmap[0];
+ }
+
+ # We now have the bucket that all code points in the range
+ # map to, though possibly they need to be adjusted. Go
+ # through the range and put each translated code point in
+ # it into its bucket.
+ my $base_map = $invmap[0];
+ for my $j ($invlist[0] .. $invlist[1] - 1) {
+ if ($to_adjust
+ # The 1st code point doesn't need adjusting
+ && $j > $invlist[0]
+
+ # Skip any non-numeric maps: these are outliers
+ # that aren't code points.
+ && $base_map =~ $numeric_re
+
+ # 'ne' because the default can be a string
+ && $base_map ne $map_default)
+ {
+ # We adjust, by incrementing each the bucket and
+ # the map. For code point maps, translate to
+ # native
+ $base_map++;
+ $bucket = ($maps_to_code_point)
+ ? a2n($base_map)
+ : $base_map;
+ }
+
+ # Add the native code point to the bucket for the
+ # current map
+ push @{$mapped_lists{$bucket}}, a2n($j);
+ } # End of loop through all code points in the range
+
+ # Get ready for the next range
+ shift @invlist;
+ shift @invmap;
+ } # End of loop through all ranges in the map.
+
+ # Here, @invlist and @invmap retain all the ranges from the
+ # originals that start with code points above $upper_limit.
+ # Each bucket in %mapped_lists contains all the code points
+ # that map to that bucket. If the bucket is for a map to a
+ # single code point is a single code point, the bucket has
+ # been converted to native. If something else (including
+ # multiple code points), no conversion is done.
+ #
+ # Now we recreate the inversion map into %xlated, but this
+ # time for the native character set.
+ my %xlated;
+ foreach my $bucket (keys %mapped_lists) {
+
+ # Sort and convert this bucket to an inversion list. The
+ # result will be that ranges that start with even-numbered
+ # indexes will be for code points that map to this bucket;
+ # odd ones map to some other bucket, and are discarded
+ # below.
+ @{$mapped_lists{$bucket}}
+ = sort{ $a <=> $b} @{$mapped_lists{$bucket}};
+ @{$mapped_lists{$bucket}}
+ = mk_invlist_from_sorted_cp_list(\@{$mapped_lists{$bucket}});
+
+ # Add each even-numbered range in the bucket to %xlated;
+ # so that the keys of %xlated become the range start code
+ # points, and the values are their corresponding maps.
+ while (@{$mapped_lists{$bucket}}) {
+ my $range_start = $mapped_lists{$bucket}->[0];
+ if ($bucket =~ /\cK/) {
+ @{$xlated{$range_start}} = split /\cK/, $bucket;
+ }
+ else {
+ $xlated{$range_start} = $bucket;
+ }
+ shift @{$mapped_lists{$bucket}}; # Discard odd ranges
+ shift @{$mapped_lists{$bucket}}; # Get ready for next
+ # iteration
+ }
+ } # End of loop through all the buckets.
+
+ # Here %xlated's keys are the range starts of all the code
+ # points in the inversion map. Construct an inversion list
+ # from them.
+ my @new_invlist = sort { $a <=> $b } keys %xlated;
+
+ # If the list is adjusted, we want to munge this list so that
+ # we only have one entry for where consecutive code points map
+ # to consecutive values. We just skip the subsequent entries
+ # where this is the case.
+ if ($to_adjust) {
+ my @temp;
+ for my $i (0 .. @new_invlist - 1) {
+ next if $i > 0
+ && $new_invlist[$i-1] + 1 == $new_invlist[$i]
+ && $xlated{$new_invlist[$i-1]} =~ $numeric_re
+ && $xlated{$new_invlist[$i]} =~ $numeric_re
+ && $xlated{$new_invlist[$i-1]} + 1 == $xlated{$new_invlist[$i]};
+ push @temp, $new_invlist[$i];
+ }
+ @new_invlist = @temp;
}
+
+ # The inversion map comes from %xlated's values. We can
+ # unshift each onto the front of the untouched portion, in
+ # reverse order of the portion we did process.
+ foreach my $start (reverse @new_invlist) {
+ unshift @invmap, $xlated{$start};
+ }
+
+ # Finally prepend the inversion list we have just constructed to the
+ # one that contains anything we didn't process.
+ unshift @invlist, @new_invlist;
}
}
- @full_list = sort { $a <=> $b } @full_list;
- @invlist = mk_invlist_from_cp_list(\@full_list);
+
+ # prop_invmap() returns an extra final entry, which we can now
+ # discard.
+ if (@invmap) {
+ pop @invlist;
+ pop @invmap;
+ }
if ($l1_only) {
+ die "Unimplemented to do a Latin-1 only inversion map" if @invmap;
for my $i (0 .. @invlist - 1 - 1) {
if ($invlist[$i] > 255) {
# Remove everything past this.
splice @invlist, $i;
+ splice @invmap, $i if @invmap;
last;
}
}
# Here, we have the first element in the array that indicates an
# element above Latin1. Get rid of all previous ones.
splice @invlist, 0, $i;
+ splice @invmap, 0, $i if @invmap;
# If this one's index is not divisible by 2, it means that this
# element is inverting away from being in the list, which means
- # all code points from 256 to this one are in this list.
- unshift @invlist, 256 if $i % 2 != 0;
+ # all code points from 256 to this one are in this list (or
+ # map to the default for inversion maps)
+ if ($i % 2 != 0) {
+ unshift @invlist, 256;
+ unshift @invmap, $map_default if @invmap;
+ }
$found_nonl1 = 1;
last;
}
}
output_invlist($prop_name, \@invlist, $charset);
+ output_invmap($prop_name, \@invmap, $lookup_prop, $map_format, $map_default, $extra_enums, $charset) if @invmap;
}
- end_ifndef_ext_re;
+ end_file_pound_if;
print $out_fh "\n" . get_conditional_compile_line_end();
}
-read_only_bottom_close_and_rename($out_fh, [$0])
+switch_pound_if('Boundary_pair_tables', 'PERL_IN_REGEXEC_C');
+
+output_GCB_table();
+output_LB_table();
+output_WB_table();
+
+end_file_pound_if;
+
+my $sources_list = "lib/unicore/mktables.lst";
+my @sources = ($0, qw(lib/unicore/mktables
+ lib/Unicode/UCD.pm
+ regen/charset_translations.pl
+ ));
+{
+ # Depend on mktables’ own sources. It’s a shorter list of files than
+ # those that Unicode::UCD uses.
+ if (! open my $mktables_list, '<', $sources_list) {
+
+ # This should force a rebuild once $sources_list exists
+ push @sources, $sources_list;
+ }
+ else {
+ while(<$mktables_list>) {
+ last if /===/;
+ chomp;
+ push @sources, "lib/unicore/$_" if /^[^#]/;
+ }
+ }
+}
+
+read_only_bottom_close_and_rename($out_fh, \@sources);
https://perl5.git.perl.org / perl5.git / blobdiff