X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/3f427fd9b3d9172856d6247beacfaedb3a38b17d..eeeafd9ca1534ea02405f35c1ce7ca8330ac6cd3:/regen/mk_invlists.pl diff --git a/regen/mk_invlists.pl b/regen/mk_invlists.pl index 46114bc..8304ee6 100644 --- a/regen/mk_invlists.pl +++ b/regen/mk_invlists.pl @@ -2,8 +2,14 @@ use 5.015; use strict; use warnings; -use Unicode::UCD "prop_invlist"; -require 'regen/regen_lib.pl'; +use Unicode::UCD qw(prop_aliases + prop_values + prop_value_aliases + prop_invlist + prop_invmap search_invlist + ); +require './regen/regen_lib.pl'; +require './regen/charset_translations.pl'; # This program outputs charclass_invlists.h, which contains various inversion # lists in the form of C arrays that are to be used as-is for inversion lists. @@ -15,62 +21,1701 @@ require 'regen/regen_lib.pl'; # in the headers is used to minimize the possibility of things getting # out-of-sync, or the wrong data structure being passed. Currently that # random number is: -my $VERSION_DATA_STRUCTURE_TYPE = 1064334010; + +# charclass_invlists.h now also has a partial implementation of inversion +# maps; enough to generate tables for the line break properties, such as GCB + +my $VERSION_DATA_STRUCTURE_TYPE = 148565664; + +# integer or float +my $numeric_re = qr/ ^ -? \d+ (:? \. \d+ )? $ /ax; + +# Matches valid C language enum names: begins with ASCII alphabetic, then any +# ASCII \w +my $enum_name_re = qr / ^ [[:alpha:]] \w* $ /ax; my $out_fh = open_new('charclass_invlists.h', '>', {style => '*', by => $0, from => "Unicode::UCD"}); +my $in_file_pound_if = 0; + +my $max_hdr_len = 3; # In headings, how wide a name is allowed? + print $out_fh "/* See the generating file for comments */\n\n"; -sub output_invlist ($$) { - my $name = shift; - my $invlist = shift; # Reference to inversion list array +# The symbols generated by this program are all currently defined only in a +# single dot c each. The code knows where most of them go, but this hash +# gives overrides for the exceptions to the typical place +my %exceptions_to_where_to_define = + ( NonL1_Perl_Non_Final_Folds => 'PERL_IN_REGCOMP_C', + AboveLatin1 => 'PERL_IN_REGCOMP_C', + Latin1 => 'PERL_IN_REGCOMP_C', + UpperLatin1 => 'PERL_IN_REGCOMP_C', + _Perl_Any_Folds => 'PERL_IN_REGCOMP_C', + _Perl_Folds_To_Multi_Char => 'PERL_IN_REGCOMP_C', + _Perl_IDCont => 'PERL_IN_UTF8_C', + _Perl_IDStart => 'PERL_IN_UTF8_C', + ); - # Output the inversion list $invlist using the name $name for it. - # It is output in the exact internal form for inversion lists. +my %gcb_enums; +my @gcb_short_enums; +my %gcb_abbreviations; +my %lb_enums; +my @lb_short_enums; +my %lb_abbreviations; +my %wb_enums; +my @wb_short_enums; +my %wb_abbreviations; - my $zero_or_one; # Is the last element of the header 0, or 1 ? +my @a2n; + +sub uniques { + # Returns non-duplicated input values. From "Perl Best Practices: + # Encapsulated Cleverness". p. 455 in first edition. + + my %seen; + return grep { ! $seen{$_}++ } @_; +} - # If the first element is 0, it goes in the header, instead of the body - if ($invlist->[0] == 0) { - shift @$invlist; +sub a2n($) { + my $cp = shift; - $zero_or_one = 0; + # Returns the input Unicode code point translated to native. - # Add a dummy 0 at the end so that the length is constant. inversion - # lists are always stored with enough room so that if they change from - # beginning with 0, they don't have to grow. - push @$invlist, 0; + return $cp if $cp !~ $numeric_re || $cp > 255; + return $a2n[$cp]; +} + +sub end_file_pound_if { + if ($in_file_pound_if) { + print $out_fh "\n#endif\t/* $in_file_pound_if */\n"; + $in_file_pound_if = 0; } - else { +} + +sub switch_pound_if ($$) { + my $name = shift; + my $new_pound_if = shift; + + # Switch to new #if given by the 2nd argument. If there is an override + # for this, it instead switches to that. The 1st argument is the + # static's name, used to look up the overrides + + if (exists $exceptions_to_where_to_define{$name}) { + $new_pound_if = $exceptions_to_where_to_define{$name}; + } + + # Exit current #if if the new one is different from the old + if ($in_file_pound_if + && $in_file_pound_if !~ /$new_pound_if/) + { + end_file_pound_if; + } + + # Enter new #if, if not already in it. + if (! $in_file_pound_if) { + $in_file_pound_if = "defined($new_pound_if)"; + print $out_fh "\n#if $in_file_pound_if\n"; + } +} + +sub output_invlist ($$;$) { + my $name = shift; + my $invlist = shift; # Reference to inversion list array + my $charset = shift // ""; # name of character set for comment + + die "No inversion list for $name" unless defined $invlist + && ref $invlist eq 'ARRAY'; + + # Output the inversion list $invlist using the name $name for it. + # It is output in the exact internal form for inversion lists. + + # Is the last element of the header 0, or 1 ? + my $zero_or_one = 0; + if (@$invlist && $invlist->[0] != 0) { + unshift @$invlist, 0; $zero_or_one = 1; } + my $count = @$invlist; - print $out_fh "\nUV ${name}_invlist[] = {\n"; + switch_pound_if ($name, 'PERL_IN_PERL_C'); - print $out_fh "\t", scalar @$invlist, ",\t/* Number of elements */\n"; - print $out_fh "\t0,\t/* Current iteration position */\n"; + print $out_fh "\nstatic const UV ${name}_invlist[] = {"; + print $out_fh " /* for $charset */" if $charset; + print $out_fh "\n"; + + print $out_fh "\t$count,\t/* Number of elements */\n"; print $out_fh "\t$VERSION_DATA_STRUCTURE_TYPE, /* Version and data structure type */\n"; print $out_fh "\t", $zero_or_one, - ",\t/* 0 if this is the first element of the list proper;", - "\n\t\t 1 if the next element is the first */\n"; + ",\t/* 0 if the list starts at 0;", + "\n\t\t 1 if it starts at the element beyond 0 */\n"; # The main body are the UVs passed in to this routine. Do the final # element separately - for my $i (0 .. @$invlist - 1 - 1) { - print $out_fh "\t$invlist->[$i],\n"; + for my $i (0 .. @$invlist - 1) { + printf $out_fh "\t0x%X", $invlist->[$i]; + print $out_fh "," if $i < @$invlist - 1; + print $out_fh "\n"; + } + + print $out_fh "};\n"; +} + +sub output_invmap ($$$$$$$) { + my $name = shift; + my $invmap = shift; # Reference to inversion map array + my $prop_name = shift; + my $input_format = shift; # The inversion map's format + my $default = shift; # The property value for code points who + # otherwise don't have a value specified. + my $extra_enums = shift; # comma-separated list of our additions to the + # property's standard possible values + my $charset = shift // ""; # name of character set for comment + + # Output the inversion map $invmap for property $prop_name, but use $name + # as the actual data structure's name. + + my $count = @$invmap; + + my $output_format; + my $declaration_type; + my %enums; + my $name_prefix; + + if ($input_format =~ / ^ s l? $ /x) { + $prop_name = (prop_aliases($prop_name))[1] // $prop_name =~ s/^_Perl_//r; # Get full name + my $short_name = (prop_aliases($prop_name))[0] // $prop_name; + my @input_enums; + + # Find all the possible input values. These become the enum names + # that comprise the inversion map. For inputs that don't have sub + # lists, we can just get the unique values. Otherwise, we have to + # expand the sublists first. + if ($input_format ne 'sl') { + @input_enums = sort(uniques(@$invmap)); + } + else { + foreach my $element (@$invmap) { + if (ref $element) { + push @input_enums, @$element; + } + else { + push @input_enums, $element; + } + } + @input_enums = sort(uniques(@input_enums)); + } + + # The internal enums come last, and in the order specified. + my @enums = @input_enums; + my @extras; + if ($extra_enums ne "") { + @extras = split /,/, $extra_enums; + + # Don't add if already there. + foreach my $this_extra (@extras) { + next if grep { $_ eq $this_extra } @enums; + + push @enums, $this_extra; + } + } + + # Assign a value to each element of the enum type we are creating. + # The default value always gets 0; the others are arbitrarily + # assigned. + my $enum_val = 0; + my $canonical_default = prop_value_aliases($prop_name, $default); + $default = $canonical_default if defined $canonical_default; + $enums{$default} = $enum_val++; + + for my $enum (@enums) { + $enums{$enum} = $enum_val++ unless exists $enums{$enum}; + } + + # Calculate the data for the special tables output for these properties. + if ($name =~ / ^ _Perl_ (?: GCB | LB | WB ) $ /x) { + + # The data includes the hashes %gcb_enums, %lb_enums, etc. + # Similarly we calculate column headings for the tables. + # + # We use string evals to allow the same code to work on + # all the tables + my $type = lc $prop_name; + + my $placeholder = "a"; + + # Skip if we've already done this code, which populated + # this hash + if (eval "! \%${type}_enums") { + + # For each enum in the type ... + foreach my $enum (sort keys %enums) { + my $value = $enums{$enum}; + my $short; + my $abbreviated_from; + + # Special case this wb property value to make the + # name more clear + if ($enum eq 'Perl_Tailored_HSpace') { + $short = 'hs'; + $abbreviated_from = $enum; + } + else { + + # Use the official short name, if found. + ($short) = prop_value_aliases($type, $enum); + + if (! defined $short) { + + # But if there is no official name, use the name + # that came from the data (if any). Otherwise, + # the name had to come from the extras list. + # There are two types of values in that list. + # + # First are those enums that are not part of the + # property, but are defined by this code. By + # convention these have all-caps names of at least + # 4 characters. We use the lowercased name for + # thse. + # + # Second are enums that are needed to get + # regexec.c to compile, but don't exist in all + # Unicode releases. To get here, we must be + # compiling an earlier Unicode release that + # doesn't have that enum, so just use a unique + # anonymous name for it. + if (grep { $_ eq $enum } @input_enums) { + $short = $enum + } + elsif ($enum !~ / ^ [A-Z]{4,} $ /x) { + $short = $placeholder++; + } + else { + $short = lc $enum; + } + } + } + + # If our short name is too long, or we already + # know that the name is an abbreviation, truncate + # to make sure it's short enough, and remember + # that we did this so we can later add a comment in the + # generated file + if ( $abbreviated_from + || length $short > $max_hdr_len) + { + $short = substr($short, 0, $max_hdr_len); + $abbreviated_from = $enum + unless $abbreviated_from; + # If the name we are to display conflicts, try + # another. + while (eval "exists + \$${type}_abbreviations{$short}") + { + die $@ if $@; + + # The increment operator on strings doesn't work + # on those containing an '_', so just use the + # final portion. + my @short = split '_', $short; + $short[-1]++; + $short = join "_", @short; + } + + eval "\$${type}_abbreviations{$short} = '$enum'"; + die $@ if $@; + } + + # Remember the mapping from the property value + # (enum) name to its value. + eval "\$${type}_enums{$enum} = $value"; + die $@ if $@; + + # Remember the inverse mapping to the short name + # so that we can properly label the generated + # table's rows and columns + eval "\$${type}_short_enums[$value] = '$short'"; + die $@ if $@; + } + } + } + + # Inversion map stuff is currently used only by regexec + switch_pound_if($name, 'PERL_IN_REGEXEC_C'); + + # The short names tend to be two lower case letters, but it looks + # better for those if they are upper. XXX + $short_name = uc($short_name) if length($short_name) < 3 + || substr($short_name, 0, 1) =~ /[[:lower:]]/; + $name_prefix = "${short_name}_"; + + # Currently unneeded + #print $out_fh "\n#define ${name_prefix}ENUM_COUNT ", scalar keys %enums, "\n"; + + if ($input_format eq 'sl') { + print $out_fh + "\n/* Negative enum values indicate the need to use an auxiliary" + . " table\n * consisting of the list of enums this one expands to." + . " The absolute\n * values of the negative enums are indices into" + . " a table of the auxiliary\n * tables' addresses */"; + } + + # Start the enum definition for this map + print $out_fh "\ntypedef enum {\n"; + my @enum_list; + foreach my $enum (keys %enums) { + $enum_list[$enums{$enum}] = $enum; + } + foreach my $i (0 .. @enum_list - 1) { + print $out_fh ",\n" if $i > 0; + + my $name = $enum_list[$i]; + print $out_fh "\t${name_prefix}$name = $i"; + } + + # For an 'sl' property, we need extra enums, because some of the + # elements are lists. Each such distinct list is placed in its own + # auxiliary map table. Here, we go through the inversion map, and for + # each distinct list found, create an enum value for it, numbered -1, + # -2, .... + my %multiples; + my $aux_table_prefix = "AUX_TABLE_"; + if ($input_format eq 'sl') { + foreach my $element (@$invmap) { + + # A regular scalar is not one of the lists we're looking for + # at this stage. + next unless ref $element; + + my $joined = join ",", sort @$element; + my $already_found = exists $multiples{$joined}; + + my $i; + if ($already_found) { # Use any existing one + $i = $multiples{$joined}; + } + else { # Otherwise increment to get a new table number + $i = keys(%multiples) + 1; + $multiples{$joined} = $i; + } + + # This changes the inversion map for this entry to not be the + # list + $element = "use_$aux_table_prefix$i"; + + # And add to the enum values + if (! $already_found) { + print $out_fh ",\n\t${name_prefix}$element = -$i"; + } + } + } + + print $out_fh "\n"; + $declaration_type = "${name_prefix}enum"; + print $out_fh "} $declaration_type;\n"; + # Finished with the enum defintion. + + $output_format = "${name_prefix}%s"; + + # If there are auxiliary tables, output them. + if (%multiples) { + + print $out_fh "\n#define HAS_${name_prefix}AUX_TABLES\n"; + + # Invert keys and values + my %inverted_mults; + while (my ($key, $value) = each %multiples) { + $inverted_mults{$value} = $key; + } + + # Output them in sorted order + my @sorted_table_list = sort { $a <=> $b } keys %inverted_mults; + + # Keep track of how big each aux table is + my @aux_counts; + + # Output each aux table. + foreach my $table_number (@sorted_table_list) { + my $table = $inverted_mults{$table_number}; + print $out_fh "\nstatic const $declaration_type $name_prefix$aux_table_prefix$table_number\[] = {\n"; + + # Earlier, we joined the elements of this table together with a comma + my @elements = split ",", $table; + + $aux_counts[$table_number] = scalar @elements; + for my $i (0 .. @elements - 1) { + print $out_fh ",\n" if $i > 0; + print $out_fh "\t${name_prefix}$elements[$i]"; + } + print $out_fh "\n};\n"; + } + + # Output the table that is indexed by the absolute value of the + # aux table enum and contains pointers to the tables output just + # above + print $out_fh "\nstatic const $declaration_type * const ${name_prefix}${aux_table_prefix}ptrs\[] = {\n"; + print $out_fh "\tNULL,\t/* Placeholder */\n"; + for my $i (1 .. @sorted_table_list) { + print $out_fh ",\n" if $i > 1; + print $out_fh "\t$name_prefix$aux_table_prefix$i"; + } + print $out_fh "\n};\n"; + + print $out_fh + "\n/* Parallel table to the above, giving the number of elements" + . " in each table\n * pointed to */\n"; + print $out_fh "static const U8 ${name_prefix}${aux_table_prefix}lengths\[] = {\n"; + print $out_fh "\t0,\t/* Placeholder */\n"; + for my $i (1 .. @sorted_table_list) { + print $out_fh ",\n" if $i > 1; + print $out_fh "\t$aux_counts[$i]\t/* $name_prefix$aux_table_prefix$i */"; + } + print $out_fh "\n};\n"; + } # End of outputting the auxiliary and associated tables + } + else { + die "'$input_format' invmap() format for '$prop_name' unimplemented"; + } + + die "No inversion map for $prop_name" unless defined $invmap + && ref $invmap eq 'ARRAY' + && $count; + + # Now output the inversion map proper + print $out_fh "\nstatic const $declaration_type ${name}_invmap[] = {"; + print $out_fh " /* for $charset */" if $charset; + print $out_fh "\n"; + + # The main body are the scalars passed in to this routine. + for my $i (0 .. $count - 1) { + my $element = $invmap->[$i]; + my $full_element_name = prop_value_aliases($prop_name, $element); + $element = $full_element_name if defined $full_element_name; + $element = $name_prefix . $element; + print $out_fh "\t$element"; + print $out_fh "," if $i < $count - 1; + print $out_fh "\n"; + } + print $out_fh "};\n"; +} + +sub mk_invlist_from_sorted_cp_list { + + # Returns an inversion list constructed from the sorted input array of + # code points + + my $list_ref = shift; + + return unless @$list_ref; + + # Initialize to just the first element + my @invlist = ( $list_ref->[0], $list_ref->[0] + 1); + + # For each succeeding element, if it extends the previous range, adjust + # up, otherwise add it. + for my $i (1 .. @$list_ref - 1) { + if ($invlist[-1] == $list_ref->[$i]) { + $invlist[-1]++; + } + else { + push @invlist, $list_ref->[$i], $list_ref->[$i] + 1; + } + } + return @invlist; +} + +# Read in the Case Folding rules, and construct arrays of code points for the +# properties we need. +my ($cp_ref, $folds_ref, $format) = prop_invmap("Case_Folding"); +die "Could not find inversion map for Case_Folding" unless defined $format; +die "Incorrect format '$format' for Case_Folding inversion map" + unless $format eq 'al' + || $format eq 'a'; +my @has_multi_char_fold; +my @is_non_final_fold; + +for my $i (0 .. @$folds_ref - 1) { + next unless ref $folds_ref->[$i]; # Skip single-char folds + push @has_multi_char_fold, $cp_ref->[$i]; + + # Add to the non-finals list each code point that is in a non-final + # position + for my $j (0 .. @{$folds_ref->[$i]} - 2) { + push @is_non_final_fold, $folds_ref->[$i][$j] + unless grep { $folds_ref->[$i][$j] == $_ } @is_non_final_fold; + } +} + +sub _Perl_Non_Final_Folds { + @is_non_final_fold = sort { $a <=> $b } @is_non_final_fold; + return mk_invlist_from_sorted_cp_list(\@is_non_final_fold); +} + +sub prop_name_for_cmp ($) { # Sort helper + my $name = shift; + + # Returns the input lowercased, with non-alphas removed, as well as + # everything starting with a comma + + $name =~ s/,.*//; + $name =~ s/[[:^alpha:]]//g; + return lc $name; +} + +sub UpperLatin1 { + return mk_invlist_from_sorted_cp_list([ 128 .. 255 ]); +} + +sub output_table_common { + + # Common subroutine to actually output the generated rules table. + + my ($property, + $table_value_defines_ref, + $table_ref, + $names_ref, + $abbreviations_ref) = @_; + my $size = @$table_ref; + + # Output the #define list, sorted by numeric value + if ($table_value_defines_ref) { + my $max_name_length = 0; + my @defines; + + # Put in order, and at the same time find the longest name + while (my ($enum, $value) = each %$table_value_defines_ref) { + $defines[$value] = $enum; + + my $length = length $enum; + $max_name_length = $length if $length > $max_name_length; + } + + print $out_fh "\n"; + + # Output, so that the values are vertically aligned in a column after + # the longest name + foreach my $i (0 .. @defines - 1) { + next unless defined $defines[$i]; + printf $out_fh "#define %-*s %2d\n", + $max_name_length, + $defines[$i], + $i; + } } - # The final element does not have a trailing comma, as C can't handle it. - print $out_fh "\t$invlist->[-1]\n"; + my $column_width = 2; # We currently allow 2 digits for the number + + # If the maximum value in the table is 1, it can be a bool. (Being above + # a U8 is not currently handled + my $max_element = 0; + for my $i (0 .. $size - 1) { + for my $j (0 .. $size - 1) { + next if $max_element >= $table_ref->[$i][$j]; + $max_element = $table_ref->[$i][$j]; + } + } + die "Need wider table column width given '$max_element" + if length $max_element > $column_width; + + my $table_type = ($max_element == 1) + ? 'bool' + : 'U8'; + + # If a name is longer than the width set aside for a column, its column + # needs to have increased spacing so that the name doesn't get truncated + # nor run into an adjacent column + my @spacers; + + # If we are being compiled on a Unicode version earlier than that which + # this file was designed for, it may be that some of the property values + # aren't in the current release, and so would be undefined if we didn't + # define them ourselves. Earlier code has done this, making them + # lowercase characters of length one. We look to see if any exist, so + # that we can add an annotation to the output table + my $has_placeholder = 0; + + for my $i (0 .. $size - 1) { + no warnings 'numeric'; + $has_placeholder = 1 if $names_ref->[$i] =~ / ^ [[:lower:]] $ /ax; + $spacers[$i] = " " x (length($names_ref->[$i]) - $column_width); + } + + print $out_fh "\nstatic const $table_type ${property}_table[$size][$size] = {\n"; + + # Calculate the column heading line + my $header_line = "/* " + . (" " x $max_hdr_len) # We let the row heading meld to + # the '*/' for those that are at + # the max + . " " x 3; # Space for '*/ ' + # Now each column + for my $i (0 .. $size - 1) { + $header_line .= sprintf "%s%*s", + $spacers[$i], + $column_width + 1, # 1 for the ',' + $names_ref->[$i]; + } + $header_line .= " */\n"; + + # If we have annotations, output it now. + if ($has_placeholder || scalar %$abbreviations_ref) { + my $text = ""; + foreach my $abbr (sort keys %$abbreviations_ref) { + $text .= "; " if $text; + $text .= "'$abbr' stands for '$abbreviations_ref->{$abbr}'"; + } + if ($has_placeholder) { + $text .= "; other " if $text; + $text .= "lowercase names are placeholders for" + . " property values not defined until a later Unicode" + . " release, so are irrelevant in this one, as they are" + . " not assigned to any code points"; + } + + my $indent = " " x 3; + $text = $indent . "/* $text */"; + + # Wrap the text so that it is no wider than the table, which the + # header line gives. + my $output_width = length $header_line; + while (length $text > $output_width) { + my $cur_line = substr($text, 0, $output_width); + + # Find the first blank back from the right end to wrap at. + for (my $i = $output_width -1; $i > 0; $i--) { + if (substr($text, $i, 1) eq " ") { + print $out_fh substr($text, 0, $i), "\n"; + + # Set so will look at just the remaining tail (which will + # be indented and have a '*' after the indent + $text = $indent . " * " . substr($text, $i + 1); + last; + } + } + } + + # And any remaining + print $out_fh $text, "\n" if $text; + } + + # We calculated the header line earlier just to get its width so that we + # could make sure the annotations fit into that. + print $out_fh $header_line; + + # Now output the bulk of the table. + for my $i (0 .. $size - 1) { + + # First the row heading. + printf $out_fh "/* %-*s*/ ", $max_hdr_len, $names_ref->[$i]; + print $out_fh "{"; # Then the brace for this row + + # Then each column + for my $j (0 .. $size -1) { + print $out_fh $spacers[$j]; + printf $out_fh "%*d", $column_width, $table_ref->[$i][$j]; + print $out_fh "," if $j < $size - 1; + } + print $out_fh " }"; + print $out_fh "," if $i < $size - 1; + print $out_fh "\n"; + } print $out_fh "};\n"; } +sub output_GCB_table() { + + # Create and output the pair table for use in determining Grapheme Cluster + # Breaks, given in http://www.unicode.org/reports/tr29/. + my %gcb_actions = ( + GCB_NOBREAK => 0, + GCB_BREAKABLE => 1, + GCB_RI_then_RI => 2, # Rules 12 and 13 + GCB_EX_then_EM => 3, # Rule 10 + ); + + # The table is constructed in reverse order of the rules, to make the + # lower-numbered, higher priority ones override the later ones, as the + # algorithm stops at the earliest matching rule + + my @gcb_table; + my $table_size = @gcb_short_enums; + + # Otherwise, break everywhere. + # GB99 Any ÷ Any + for my $i (0 .. $table_size - 1) { + for my $j (0 .. $table_size - 1) { + $gcb_table[$i][$j] = 1; + } + } + + # Do not break within emoji flag sequences. That is, do not break between + # regional indicator (RI) symbols if there is an odd number of RI + # characters before the break point. Must be resolved in runtime code. + # + # GB12 sot (RI RI)* RI × RI + # GB13 [^RI] (RI RI)* RI × RI + $gcb_table[$gcb_enums{'Regional_Indicator'}] + [$gcb_enums{'Regional_Indicator'}] = $gcb_actions{GCB_RI_then_RI}; + + # Do not break within emoji modifier sequences or emoji zwj sequences. + # GB11 ZWJ × ( Glue_After_Zwj | E_Base_GAZ ) + $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'Glue_After_Zwj'}] = 0; + $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'E_Base_GAZ'}] = 0; + + # GB10 ( E_Base | E_Base_GAZ ) Extend* × E_Modifier + $gcb_table[$gcb_enums{'Extend'}][$gcb_enums{'E_Modifier'}] + = $gcb_actions{GCB_EX_then_EM}; + $gcb_table[$gcb_enums{'E_Base'}][$gcb_enums{'E_Modifier'}] = 0; + $gcb_table[$gcb_enums{'E_Base_GAZ'}][$gcb_enums{'E_Modifier'}] = 0; + + # Do not break before extending characters or ZWJ. + # Do not break before SpacingMarks, or after Prepend characters. + # GB9b Prepend × + # GB9a × SpacingMark + # GB9 × ( Extend | ZWJ ) + for my $i (0 .. @gcb_table - 1) { + $gcb_table[$gcb_enums{'Prepend'}][$i] = 0; + $gcb_table[$i][$gcb_enums{'SpacingMark'}] = 0; + $gcb_table[$i][$gcb_enums{'Extend'}] = 0; + $gcb_table[$i][$gcb_enums{'ZWJ'}] = 0; + } + + # Do not break Hangul syllable sequences. + # GB8 ( LVT | T) × T + $gcb_table[$gcb_enums{'LVT'}][$gcb_enums{'T'}] = 0; + $gcb_table[$gcb_enums{'T'}][$gcb_enums{'T'}] = 0; + + # GB7 ( LV | V ) × ( V | T ) + $gcb_table[$gcb_enums{'LV'}][$gcb_enums{'V'}] = 0; + $gcb_table[$gcb_enums{'LV'}][$gcb_enums{'T'}] = 0; + $gcb_table[$gcb_enums{'V'}][$gcb_enums{'V'}] = 0; + $gcb_table[$gcb_enums{'V'}][$gcb_enums{'T'}] = 0; + + # GB6 L × ( L | V | LV | LVT ) + $gcb_table[$gcb_enums{'L'}][$gcb_enums{'L'}] = 0; + $gcb_table[$gcb_enums{'L'}][$gcb_enums{'V'}] = 0; + $gcb_table[$gcb_enums{'L'}][$gcb_enums{'LV'}] = 0; + $gcb_table[$gcb_enums{'L'}][$gcb_enums{'LVT'}] = 0; + + # Do not break between a CR and LF. Otherwise, break before and after + # controls. + # GB5 ÷ ( Control | CR | LF ) + # GB4 ( Control | CR | LF ) ÷ + for my $i (0 .. @gcb_table - 1) { + $gcb_table[$i][$gcb_enums{'Control'}] = 1; + $gcb_table[$i][$gcb_enums{'CR'}] = 1; + $gcb_table[$i][$gcb_enums{'LF'}] = 1; + $gcb_table[$gcb_enums{'Control'}][$i] = 1; + $gcb_table[$gcb_enums{'CR'}][$i] = 1; + $gcb_table[$gcb_enums{'LF'}][$i] = 1; + } + + # GB3 CR × LF + $gcb_table[$gcb_enums{'CR'}][$gcb_enums{'LF'}] = 0; + + # Break at the start and end of text, unless the text is empty + # GB1 sot ÷ + # GB2 ÷ eot + for my $i (0 .. @gcb_table - 1) { + $gcb_table[$i][$gcb_enums{'EDGE'}] = 1; + $gcb_table[$gcb_enums{'EDGE'}][$i] = 1; + } + $gcb_table[$gcb_enums{'EDGE'}][$gcb_enums{'EDGE'}] = 0; + + output_table_common('GCB', \%gcb_actions, + \@gcb_table, \@gcb_short_enums, \%gcb_abbreviations); +} + +sub output_LB_table() { + + # Create and output the enums, #defines, and pair table for use in + # determining Line Breaks. This uses the default line break algorithm, + # given in http://www.unicode.org/reports/tr14/, but tailored by example 7 + # in that page, as the Unicode-furnished tests assume that tailoring. + + # The result is really just true or false. But we follow along with tr14, + # creating a rule which is false for something like X SP* X. That gets + # encoding 2. The rest of the actions are synthetic ones that indicate + # some context handling is required. These each are added to the + # underlying 0, 1, or 2, instead of replacing them, so that the underlying + # value can be retrieved. Actually only rules from 7 through 18 (which + # are the ones where space matter) are possible to have 2 added to them. + # The others below add just 0 or 1. It might be possible for one + # synthetic rule to be added to another, yielding a larger value. This + # doesn't happen in the Unicode 8.0 rule set, and as you can see from the + # names of the middle grouping below, it is impossible for that to occur + # for them because they all start with mutually exclusive classes. That + # the final rule can't be added to any of the others isn't obvious from + # its name, so it is assigned a power of 2 higher than the others can get + # to so any addition would preserve all data. (And the code will reach an + # assert(0) on debugging builds should this happen.) + my %lb_actions = ( + LB_NOBREAK => 0, + LB_BREAKABLE => 1, + LB_NOBREAK_EVEN_WITH_SP_BETWEEN => 2, + + LB_CM_ZWJ_foo => 3, # Rule 9 + LB_SP_foo => 6, # Rule 18 + LB_PR_or_PO_then_OP_or_HY => 9, # Rule 25 + LB_SY_or_IS_then_various => 11, # Rule 25 + LB_HY_or_BA_then_foo => 13, # Rule 21 + LB_RI_then_RI => 15, # Rule 30a + + LB_various_then_PO_or_PR => (1<<5), # Rule 25 + ); + + # Construct the LB pair table. This is based on the rules in + # http://www.unicode.org/reports/tr14/, but modified as those rules are + # designed for someone taking a string of text and sequentially going + # through it to find the break opportunities, whereas, Perl requires + # determining if a given random spot is a break opportunity, without + # knowing all the entire string before it. + # + # The table is constructed in reverse order of the rules, to make the + # lower-numbered, higher priority ones override the later ones, as the + # algorithm stops at the earliest matching rule + + my @lb_table; + my $table_size = @lb_short_enums; + + # LB31. Break everywhere else + for my $i (0 .. $table_size - 1) { + for my $j (0 .. $table_size - 1) { + $lb_table[$i][$j] = $lb_actions{'LB_BREAKABLE'}; + } + } + + # LB30b Do not break between an emoji base and an emoji modifier. + # EB × EM + $lb_table[$lb_enums{'E_Base'}][$lb_enums{'E_Modifier'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB30a Break between two regional indicator symbols if and only if there + # are an even number of regional indicators preceding the position of the + # break. + # sot (RI RI)* RI × RI + # [^RI] (RI RI)* RI × RI + $lb_table[$lb_enums{'Regional_Indicator'}] + [$lb_enums{'Regional_Indicator'}] = $lb_actions{'LB_RI_then_RI'}; + + # LB30 Do not break between letters, numbers, or ordinary symbols and + # opening or closing parentheses. + # (AL | HL | NU) × OP + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Open_Punctuation'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Open_Punctuation'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Open_Punctuation'}] + = $lb_actions{'LB_NOBREAK'}; + + # CP × (AL | HL | NU) + $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB29 Do not break between numeric punctuation and alphabetics (“e.g.”). + # IS × (AL | HL) + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB28 Do not break between alphabetics (“at”). + # (AL | HL) × (AL | HL) + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB27 Treat a Korean Syllable Block the same as ID. + # (JL | JV | JT | H2 | H3) × IN + $lb_table[$lb_enums{'JL'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JV'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JT'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H2'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H3'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + + # (JL | JV | JT | H2 | H3) × PO + $lb_table[$lb_enums{'JL'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JV'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JT'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H2'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H3'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # PR × (JL | JV | JT | H2 | H3) + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'JL'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'JV'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'JT'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'H2'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'H3'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB26 Do not break a Korean syllable. + # JL × (JL | JV | H2 | H3) + $lb_table[$lb_enums{'JL'}][$lb_enums{'JL'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JL'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JL'}][$lb_enums{'H2'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JL'}][$lb_enums{'H3'}] = $lb_actions{'LB_NOBREAK'}; + + # (JV | H2) × (JV | JT) + $lb_table[$lb_enums{'JV'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H2'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'JV'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H2'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'}; + + # (JT | H3) × JT + $lb_table[$lb_enums{'JT'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'H3'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'}; + + # LB25 Do not break between the following pairs of classes relevant to + # numbers, as tailored by example 7 in + # http://www.unicode.org/reports/tr14/#Examples + # We follow that tailoring because Unicode's test cases expect it + # (PR | PO) × ( OP | HY )? NU + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # Given that (OP | HY )? is optional, we have to test for it in code. + # We add in the action (instead of overriding) for this, so that in + # the code we can recover the underlying break value. + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Open_Punctuation'}] + += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'}; + $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Open_Punctuation'}] + += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Hyphen'}] + += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'}; + $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Hyphen'}] + += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'}; + + # ( OP | HY ) × NU + $lb_table[$lb_enums{'Open_Punctuation'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hyphen'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # NU (NU | SY | IS)* × (NU | SY | IS | CL | CP ) + # which can be rewritten as: + # NU (SY | IS)* × (NU | SY | IS | CL | CP ) + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Break_Symbols'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Infix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Close_Punctuation'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Close_Parenthesis'}] + = $lb_actions{'LB_NOBREAK'}; + + # Like earlier where we have to test in code, we add in the action so + # that we can recover the underlying values. This is done in rules + # below, as well. The code assumes that we haven't added 2 actions. + # Shoul a later Unicode release break that assumption, then tests + # should start failing. + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Numeric'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Break_Symbols'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Infix_Numeric'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Close_Punctuation'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Close_Parenthesis'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Numeric'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Break_Symbols'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Infix_Numeric'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Close_Punctuation'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Close_Parenthesis'}] + += $lb_actions{'LB_SY_or_IS_then_various'}; + + # NU (NU | SY | IS)* (CL | CP)? × (PO | PR) + # which can be rewritten as: + # NU (SY | IS)* (CL | CP)? × (PO | PR) + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Prefix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Postfix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + $lb_table[$lb_enums{'Close_Punctuation'}][$lb_enums{'Postfix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Postfix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Postfix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + + $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Prefix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + $lb_table[$lb_enums{'Close_Punctuation'}][$lb_enums{'Prefix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + $lb_table[$lb_enums{'Infix_Numeric'}][$lb_enums{'Prefix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Prefix_Numeric'}] + += $lb_actions{'LB_various_then_PO_or_PR'}; + + # LB24 Do not break between numeric prefix/postfix and letters, or between + # letters and prefix/postfix. + # (PR | PO) × (AL | HL) + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Postfix_Numeric'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + + # (AL | HL) × (PR | PO) + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Prefix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Prefix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB23a Do not break between numeric prefixes and ideographs, or between + # ideographs and numeric postfixes. + # PR × (ID | EB | EM) + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'Ideographic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'E_Base'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Prefix_Numeric'}][$lb_enums{'E_Modifier'}] + = $lb_actions{'LB_NOBREAK'}; + + # (ID | EB | EM) × PO + $lb_table[$lb_enums{'Ideographic'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'E_Base'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'E_Modifier'}][$lb_enums{'Postfix_Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB23 Do not break between digits and letters + # (AL | HL) × NU + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Numeric'}] + = $lb_actions{'LB_NOBREAK'}; + + # NU × (AL | HL) + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Alphabetic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB22 Do not break between two ellipses, or between letters, numbers or + # exclamations and ellipsis. + # (AL | HL) × IN + $lb_table[$lb_enums{'Alphabetic'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Hebrew_Letter'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + + # Exclamation × IN + $lb_table[$lb_enums{'Exclamation'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + + # (ID | EB | EM) × IN + $lb_table[$lb_enums{'Ideographic'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'E_Base'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'E_Modifier'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + + # IN × IN + $lb_table[$lb_enums{'Inseparable'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + + # NU × IN + $lb_table[$lb_enums{'Numeric'}][$lb_enums{'Inseparable'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB21b Don’t break between Solidus and Hebrew letters. + # SY × HL + $lb_table[$lb_enums{'Break_Symbols'}][$lb_enums{'Hebrew_Letter'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB21a Don't break after Hebrew + Hyphen. + # HL (HY | BA) × + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'Hyphen'}][$i] + += $lb_actions{'LB_HY_or_BA_then_foo'}; + $lb_table[$lb_enums{'Break_After'}][$i] + += $lb_actions{'LB_HY_or_BA_then_foo'}; + } + + # LB21 Do not break before hyphen-minus, other hyphens, fixed-width + # spaces, small kana, and other non-starters, or after acute accents. + # × BA + # × HY + # × NS + # BB × + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'Break_After'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$i][$lb_enums{'Hyphen'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$i][$lb_enums{'Nonstarter'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Break_Before'}][$i] = $lb_actions{'LB_NOBREAK'}; + } + + # LB20 Break before and after unresolved CB. + # ÷ CB + # CB ÷ + # Conditional breaks should be resolved external to the line breaking + # rules. However, the default action is to treat unresolved CB as breaking + # before and after. + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'Contingent_Break'}] + = $lb_actions{'LB_BREAKABLE'}; + $lb_table[$lb_enums{'Contingent_Break'}][$i] + = $lb_actions{'LB_BREAKABLE'}; + } + + # LB19 Do not break before or after quotation marks, such as ‘ ” ’. + # × QU + # QU × + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'Quotation'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'Quotation'}][$i] = $lb_actions{'LB_NOBREAK'}; + } + + # LB18 Break after spaces + # SP ÷ + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'Space'}][$i] = $lb_actions{'LB_BREAKABLE'}; + } + + # LB17 Do not break within ‘——’, even with intervening spaces. + # B2 SP* × B2 + $lb_table[$lb_enums{'Break_Both'}][$lb_enums{'Break_Both'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + + # LB16 Do not break between closing punctuation and a nonstarter even with + # intervening spaces. + # (CL | CP) SP* × NS + $lb_table[$lb_enums{'Close_Punctuation'}][$lb_enums{'Nonstarter'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + $lb_table[$lb_enums{'Close_Parenthesis'}][$lb_enums{'Nonstarter'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + + + # LB15 Do not break within ‘”[’, even with intervening spaces. + # QU SP* × OP + $lb_table[$lb_enums{'Quotation'}][$lb_enums{'Open_Punctuation'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + + # LB14 Do not break after ‘[’, even after spaces. + # OP SP* × + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'Open_Punctuation'}][$i] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + } + + # LB13 Do not break before ‘]’ or ‘!’ or ‘;’ or ‘/’, even after spaces, as + # tailored by example 7 in http://www.unicode.org/reports/tr14/#Examples + # [^NU] × CL + # [^NU] × CP + # × EX + # [^NU] × IS + # [^NU] × SY + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'Exclamation'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + + next if $i == $lb_enums{'Numeric'}; + + $lb_table[$i][$lb_enums{'Close_Punctuation'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + $lb_table[$i][$lb_enums{'Close_Parenthesis'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + $lb_table[$i][$lb_enums{'Infix_Numeric'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + $lb_table[$i][$lb_enums{'Break_Symbols'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + } + + # LB12a Do not break before NBSP and related characters, except after + # spaces and hyphens. + # [^SP BA HY] × GL + for my $i (0 .. @lb_table - 1) { + next if $i == $lb_enums{'Space'} + || $i == $lb_enums{'Break_After'} + || $i == $lb_enums{'Hyphen'}; + + # We don't break, but if a property above has said don't break even + # with space between, don't override that (also in the next few rules) + next if $lb_table[$i][$lb_enums{'Glue'}] + == $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + $lb_table[$i][$lb_enums{'Glue'}] = $lb_actions{'LB_NOBREAK'}; + } + + # LB12 Do not break after NBSP and related characters. + # GL × + for my $i (0 .. @lb_table - 1) { + next if $lb_table[$lb_enums{'Glue'}][$i] + == $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + $lb_table[$lb_enums{'Glue'}][$i] = $lb_actions{'LB_NOBREAK'}; + } + + # LB11 Do not break before or after Word joiner and related characters. + # × WJ + # WJ × + for my $i (0 .. @lb_table - 1) { + if ($lb_table[$i][$lb_enums{'Word_Joiner'}] + != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}) + { + $lb_table[$i][$lb_enums{'Word_Joiner'}] = $lb_actions{'LB_NOBREAK'}; + } + if ($lb_table[$lb_enums{'Word_Joiner'}][$i] + != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}) + { + $lb_table[$lb_enums{'Word_Joiner'}][$i] = $lb_actions{'LB_NOBREAK'}; + } + } + + # Special case this here to avoid having to do a special case in the code, + # by making this the same as other things with a SP in front of them that + # don't break, we avoid an extra test + $lb_table[$lb_enums{'Space'}][$lb_enums{'Word_Joiner'}] + = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}; + + # LB9 and LB10 are done in the same loop + # + # LB9 Do not break a combining character sequence; treat it as if it has + # the line breaking class of the base character in all of the + # higher-numbered rules. Treat ZWJ as if it were CM + # Treat X (CM|ZWJ)* as if it were X. + # where X is any line break class except BK, CR, LF, NL, SP, or ZW. + + # LB10 Treat any remaining combining mark or ZWJ as AL. This catches the + # case where a CM or ZWJ is the first character on the line or follows SP, + # BK, CR, LF, NL, or ZW. + for my $i (0 .. @lb_table - 1) { + + # When the CM or ZWJ is the first in the pair, we don't know without + # looking behind whether the CM or ZWJ is going to attach to an + # earlier character, or not. So have to figure this out at runtime in + # the code + $lb_table[$lb_enums{'Combining_Mark'}][$i] + = $lb_actions{'LB_CM_ZWJ_foo'}; + $lb_table[$lb_enums{'ZWJ'}][$i] = $lb_actions{'LB_CM_ZWJ_foo'}; + + if ( $i == $lb_enums{'Mandatory_Break'} + || $i == $lb_enums{'EDGE'} + || $i == $lb_enums{'Carriage_Return'} + || $i == $lb_enums{'Line_Feed'} + || $i == $lb_enums{'Next_Line'} + || $i == $lb_enums{'Space'} + || $i == $lb_enums{'ZWSpace'}) + { + # For these classes, a following CM doesn't combine, and should do + # whatever 'Alphabetic' would do. + $lb_table[$i][$lb_enums{'Combining_Mark'}] + = $lb_table[$i][$lb_enums{'Alphabetic'}]; + $lb_table[$i][$lb_enums{'ZWJ'}] + = $lb_table[$i][$lb_enums{'Alphabetic'}]; + } + else { + # For these classes, the CM or ZWJ combines, so doesn't break, + # inheriting the type of nobreak from the master character. + if ($lb_table[$i][$lb_enums{'Combining_Mark'}] + != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}) + { + $lb_table[$i][$lb_enums{'Combining_Mark'}] + = $lb_actions{'LB_NOBREAK'}; + } + if ($lb_table[$i][$lb_enums{'ZWJ'}] + != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'}) + { + $lb_table[$i][$lb_enums{'ZWJ'}] + = $lb_actions{'LB_NOBREAK'}; + } + } + } + + # LB8a Do not break between a zero width joiner and an ideograph, emoji + # base or emoji modifier. This rule prevents breaks within emoji joiner + # sequences. + # ZWJ × (ID | EB | EM) + $lb_table[$lb_enums{'ZWJ'}][$lb_enums{'Ideographic'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'ZWJ'}][$lb_enums{'E_Base'}] + = $lb_actions{'LB_NOBREAK'}; + $lb_table[$lb_enums{'ZWJ'}][$lb_enums{'E_Modifier'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB8 Break before any character following a zero-width space, even if one + # or more spaces intervene. + # ZW SP* ÷ + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'ZWSpace'}][$i] = $lb_actions{'LB_BREAKABLE'}; + } + + # Because of LB8-10, we need to look at context for "SP x", and this must + # be done in the code. So override the existing rules for that, by adding + # a constant to get new rules that tell the code it needs to look at + # context. By adding this action instead of replacing the existing one, + # we can get back to the original rule if necessary. + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'Space'}][$i] += $lb_actions{'LB_SP_foo'}; + } + + # LB7 Do not break before spaces or zero width space. + # × SP + # × ZW + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'Space'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$i][$lb_enums{'ZWSpace'}] = $lb_actions{'LB_NOBREAK'}; + } + + # LB6 Do not break before hard line breaks. + # × ( BK | CR | LF | NL ) + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'Mandatory_Break'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$i][$lb_enums{'Carriage_Return'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$i][$lb_enums{'Line_Feed'}] = $lb_actions{'LB_NOBREAK'}; + $lb_table[$i][$lb_enums{'Next_Line'}] = $lb_actions{'LB_NOBREAK'}; + } + + # LB5 Treat CR followed by LF, as well as CR, LF, and NL as hard line breaks. + # CR × LF + # CR ! + # LF ! + # NL ! + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'Carriage_Return'}][$i] + = $lb_actions{'LB_BREAKABLE'}; + $lb_table[$lb_enums{'Line_Feed'}][$i] = $lb_actions{'LB_BREAKABLE'}; + $lb_table[$lb_enums{'Next_Line'}][$i] = $lb_actions{'LB_BREAKABLE'}; + } + $lb_table[$lb_enums{'Carriage_Return'}][$lb_enums{'Line_Feed'}] + = $lb_actions{'LB_NOBREAK'}; + + # LB4 Always break after hard line breaks. + # BK ! + for my $i (0 .. @lb_table - 1) { + $lb_table[$lb_enums{'Mandatory_Break'}][$i] + = $lb_actions{'LB_BREAKABLE'}; + } + + # LB3 Always break at the end of text. + # ! eot + # LB2 Never break at the start of text. + # sot × + for my $i (0 .. @lb_table - 1) { + $lb_table[$i][$lb_enums{'EDGE'}] = $lb_actions{'LB_BREAKABLE'}; + $lb_table[$lb_enums{'EDGE'}][$i] = $lb_actions{'LB_NOBREAK'}; + } + + # LB1 Assign a line breaking class to each code point of the input. + # Resolve AI, CB, CJ, SA, SG, and XX into other line breaking classes + # depending on criteria outside the scope of this algorithm. + # + # In the absence of such criteria all characters with a specific + # combination of original class and General_Category property value are + # resolved as follows: + # Original Resolved General_Category + # AI, SG, XX AL Any + # SA CM Only Mn or Mc + # SA AL Any except Mn and Mc + # CJ NS Any + # + # This is done in mktables, so we never see any of the remapped-from + # classes. + + output_table_common('LB', \%lb_actions, + \@lb_table, \@lb_short_enums, \%lb_abbreviations); +} + +sub output_WB_table() { + + # Create and output the enums, #defines, and pair table for use in + # determining Word Breaks, given in http://www.unicode.org/reports/tr29/. + + # This uses the same mechanism in the other bounds tables generated by + # this file. The actions that could override a 0 or 1 are added to those + # numbers; the actions that clearly don't depend on the underlying rule + # simply overwrite + my %wb_actions = ( + WB_NOBREAK => 0, + WB_BREAKABLE => 1, + WB_hs_then_hs => 2, + WB_Ex_or_FO_or_ZWJ_then_foo => 3, + WB_DQ_then_HL => 4, + WB_HL_then_DQ => 6, + WB_LE_or_HL_then_MB_or_ML_or_SQ => 8, + WB_MB_or_ML_or_SQ_then_LE_or_HL => 10, + WB_MB_or_MN_or_SQ_then_NU => 12, + WB_NU_then_MB_or_MN_or_SQ => 14, + WB_RI_then_RI => 16, + ); + + # Construct the WB pair table. + # The table is constructed in reverse order of the rules, to make the + # lower-numbered, higher priority ones override the later ones, as the + # algorithm stops at the earliest matching rule + + my @wb_table; + my $table_size = @wb_short_enums - 1; # -1 because we don't use UNKNOWN + die "UNKNOWN must be final WB enum" unless $wb_short_enums[-1] =~ /unk/i; + + # Otherwise, break everywhere (including around ideographs). + # WB99 Any ÷ Any + for my $i (0 .. $table_size - 1) { + for my $j (0 .. $table_size - 1) { + $wb_table[$i][$j] = $wb_actions{'WB_BREAKABLE'}; + } + } + + # Do not break within emoji flag sequences. That is, do not break between + # regional indicator (RI) symbols if there is an odd number of RI + # characters before the break point. + # WB16 [^RI] (RI RI)* RI × RI + # WB15 sot (RI RI)* RI × RI + $wb_table[$wb_enums{'Regional_Indicator'}] + [$wb_enums{'Regional_Indicator'}] = $wb_actions{'WB_RI_then_RI'}; + + # Do not break within emoji modifier sequences. + # WB14 ( E_Base | EBG ) × E_Modifier + $wb_table[$wb_enums{'E_Base'}][$wb_enums{'E_Modifier'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'E_Base_GAZ'}][$wb_enums{'E_Modifier'}] + = $wb_actions{'WB_NOBREAK'}; + + # Do not break from extenders. + # WB13b ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana) + $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'ALetter'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Hebrew_Letter'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Numeric'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Katakana'}] + = $wb_actions{'WB_NOBREAK'}; + + # WB13a (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet) + # × # ExtendNumLet + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'ExtendNumLet'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'ExtendNumLet'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'ExtendNumLet'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Katakana'}][$wb_enums{'ExtendNumLet'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'ExtendNumLet'}] + = $wb_actions{'WB_NOBREAK'}; + + # Do not break between Katakana. + # WB13 Katakana × Katakana + $wb_table[$wb_enums{'Katakana'}][$wb_enums{'Katakana'}] + = $wb_actions{'WB_NOBREAK'}; + + # Do not break within sequences, such as “3.2” or “3,456.789”. + # WB12 Numeric × (MidNum | MidNumLet | Single_Quote) Numeric + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'MidNumLet'}] + += $wb_actions{'WB_NU_then_MB_or_MN_or_SQ'}; + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'MidNum'}] + += $wb_actions{'WB_NU_then_MB_or_MN_or_SQ'}; + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Single_Quote'}] + += $wb_actions{'WB_NU_then_MB_or_MN_or_SQ'}; + + # WB11 Numeric (MidNum | (MidNumLet | Single_Quote)) × Numeric + $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'Numeric'}] + += $wb_actions{'WB_MB_or_MN_or_SQ_then_NU'}; + $wb_table[$wb_enums{'MidNum'}][$wb_enums{'Numeric'}] + += $wb_actions{'WB_MB_or_MN_or_SQ_then_NU'}; + $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'Numeric'}] + += $wb_actions{'WB_MB_or_MN_or_SQ_then_NU'}; + + # Do not break within sequences of digits, or digits adjacent to letters + # (“3a”, or “A3”). + # WB10 Numeric × (ALetter | Hebrew_Letter) + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'ALetter'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Hebrew_Letter'}] + = $wb_actions{'WB_NOBREAK'}; + + # WB9 (ALetter | Hebrew_Letter) × Numeric + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Numeric'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Numeric'}] + = $wb_actions{'WB_NOBREAK'}; + + # WB8 Numeric × Numeric + $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Numeric'}] + = $wb_actions{'WB_NOBREAK'}; + + # Do not break letters across certain punctuation. + # WB7c Hebrew_Letter Double_Quote × Hebrew_Letter + $wb_table[$wb_enums{'Double_Quote'}][$wb_enums{'Hebrew_Letter'}] + += $wb_actions{'WB_DQ_then_HL'}; + + # WB7b Hebrew_Letter × Double_Quote Hebrew_Letter + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Double_Quote'}] + += $wb_actions{'WB_HL_then_DQ'}; + + # WB7a Hebrew_Letter × Single_Quote + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Single_Quote'}] + = $wb_actions{'WB_NOBREAK'}; + + # WB7 (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote) + # × (ALetter | Hebrew_Letter) + $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'ALetter'}] + += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'}; + $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'Hebrew_Letter'}] + += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'}; + $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'ALetter'}] + += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'}; + $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'Hebrew_Letter'}] + += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'}; + $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'ALetter'}] + += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'}; + $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'Hebrew_Letter'}] + += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'}; + + # WB6 (ALetter | Hebrew_Letter) × (MidLetter | MidNumLet + # | Single_Quote) (ALetter | Hebrew_Letter) + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'MidNumLet'}] + += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'MidNumLet'}] + += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'}; + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'MidLetter'}] + += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'MidLetter'}] + += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'}; + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Single_Quote'}] + += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Single_Quote'}] + += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'}; + + # Do not break between most letters. + # WB5 (ALetter | Hebrew_Letter) × (ALetter | Hebrew_Letter) + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'ALetter'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Hebrew_Letter'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'ALetter'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Hebrew_Letter'}] + = $wb_actions{'WB_NOBREAK'}; + + # Ignore Format and Extend characters, except after sot, CR, LF, and + # Newline. This also has the effect of: Any × (Format | Extend | ZWJ) + # WB4 X (Extend | Format | ZWJ)* → X + for my $i (0 .. @wb_table - 1) { + $wb_table[$wb_enums{'Extend'}][$i] + = $wb_actions{'WB_Ex_or_FO_or_ZWJ_then_foo'}; + $wb_table[$wb_enums{'Format'}][$i] + = $wb_actions{'WB_Ex_or_FO_or_ZWJ_then_foo'}; + $wb_table[$wb_enums{'ZWJ'}][$i] + = $wb_actions{'WB_Ex_or_FO_or_ZWJ_then_foo'}; + } + for my $i (0 .. @wb_table - 1) { + $wb_table[$i][$wb_enums{'Extend'}] = $wb_actions{'WB_NOBREAK'}; + $wb_table[$i][$wb_enums{'Format'}] = $wb_actions{'WB_NOBREAK'}; + $wb_table[$i][$wb_enums{'ZWJ'}] = $wb_actions{'WB_NOBREAK'}; + } + + # Implied is that these attach to the character before them, except for + # the characters that mark the end of a region of text. The rules below + # override the ones set up here, for all the characters that need + # overriding. + for my $i (0 .. @wb_table - 1) { + $wb_table[$i][$wb_enums{'Extend'}] = $wb_actions{'WB_NOBREAK'}; + $wb_table[$i][$wb_enums{'Format'}] = $wb_actions{'WB_NOBREAK'}; + } + + # Do not break within emoji zwj sequences. + # WB3c ZWJ × ( Glue_After_Zwj | EBG ) + $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'Glue_After_Zwj'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'E_Base_GAZ'}] + = $wb_actions{'WB_NOBREAK'}; + + # Break before and after white space + # WB3b ÷ (Newline | CR | LF) + # WB3a (Newline | CR | LF) ÷ + # et. al. + for my $i ('CR', 'LF', 'Newline', 'Perl_Tailored_HSpace') { + for my $j (0 .. @wb_table - 1) { + $wb_table[$j][$wb_enums{$i}] = $wb_actions{'WB_BREAKABLE'}; + $wb_table[$wb_enums{$i}][$j] = $wb_actions{'WB_BREAKABLE'}; + } + } + + # But do not break within white space. + # WB3 CR × LF + # et.al. + for my $i ('CR', 'LF', 'Newline', 'Perl_Tailored_HSpace') { + for my $j ('CR', 'LF', 'Newline', 'Perl_Tailored_HSpace') { + $wb_table[$wb_enums{$i}][$wb_enums{$j}] = $wb_actions{'WB_NOBREAK'}; + } + } + + # And do not break horizontal space followed by Extend or Format or ZWJ + $wb_table[$wb_enums{'Perl_Tailored_HSpace'}][$wb_enums{'Extend'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Perl_Tailored_HSpace'}][$wb_enums{'Format'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Perl_Tailored_HSpace'}][$wb_enums{'ZWJ'}] + = $wb_actions{'WB_NOBREAK'}; + $wb_table[$wb_enums{'Perl_Tailored_HSpace'}] + [$wb_enums{'Perl_Tailored_HSpace'}] + = $wb_actions{'WB_hs_then_hs'}; + + # Break at the start and end of text, unless the text is empty + # WB2 Any ÷ eot + # WB1 sot ÷ Any + for my $i (0 .. @wb_table - 1) { + $wb_table[$i][$wb_enums{'EDGE'}] = $wb_actions{'WB_BREAKABLE'}; + $wb_table[$wb_enums{'EDGE'}][$i] = $wb_actions{'WB_BREAKABLE'}; + } + $wb_table[$wb_enums{'EDGE'}][$wb_enums{'EDGE'}] = 0; + + output_table_common('WB', \%wb_actions, + \@wb_table, \@wb_short_enums, \%wb_abbreviations); +} + output_invlist("Latin1", [ 0, 256 ]); output_invlist("AboveLatin1", [ 256 ]); +end_file_pound_if; + # We construct lists for all the POSIX and backslash sequence character # classes in two forms: # 1) ones which match only in the ASCII range @@ -90,62 +1735,439 @@ output_invlist("AboveLatin1", [ 256 ]); # In the list of properties below that get generated, the L1 prefix is a fake # property that means just the Latin1 range of the full property (whose name # has an X prefix instead of L1). +# +# An initial & means to use the subroutine from this file instead of an +# official inversion list. + +for my $charset (get_supported_code_pages()) { + print $out_fh "\n" . get_conditional_compile_line_start($charset); + + @a2n = @{get_a2n($charset)}; + # Below is the list of property names to generate. '&' means to use the + # subroutine to generate the inversion list instead of the generic code + # below. Some properties have a comma-separated list after the name, + # These are extra enums to add to those found in the Unicode tables. + no warnings 'qw'; + # Ignore non-alpha in sort + for my $prop (sort { prop_name_for_cmp($a) cmp prop_name_for_cmp($b) } qw( + Assigned + ASCII + Cased + VertSpace + XPerlSpace + XPosixAlnum + XPosixAlpha + XPosixBlank + XPosixCntrl + XPosixDigit + XPosixGraph + XPosixLower + XPosixPrint + XPosixPunct + XPosixSpace + XPosixUpper + XPosixWord + XPosixXDigit + _Perl_Any_Folds + &NonL1_Perl_Non_Final_Folds + _Perl_Folds_To_Multi_Char + &UpperLatin1 + _Perl_IDStart + _Perl_IDCont + _Perl_GCB,E_Base,E_Base_GAZ,E_Modifier,Glue_After_Zwj,LV,Prepend,Regional_Indicator,SpacingMark,ZWJ,EDGE + _Perl_LB,Close_Parenthesis,Hebrew_Letter,Next_Line,Regional_Indicator,ZWJ,Contingent_Break,E_Base,E_Modifier,H2,H3,JL,JT,JV,Word_Joiner,EDGE, + _Perl_SB,SContinue,CR,Extend,LF,EDGE + _Perl_WB,CR,Double_Quote,E_Base,E_Base_GAZ,E_Modifier,Extend,Glue_After_Zwj,Hebrew_Letter,LF,MidNumLet,Newline,Regional_Indicator,Single_Quote,ZWJ,EDGE,UNKNOWN + ) + # NOTE that the convention is that extra enum + # values come after the property name, separated by + # commas, with the enums that aren't ever defined + # by Unicode coming last, at least 4 all-uppercase + # characters. The others are enum names that are + # needed by perl, but aren't in all Unicode + # releases. + ) { + + # For the Latin1 properties, we change to use the eXtended version of the + # base property, then go through the result and get rid of everything not + # in Latin1 (above 255). Actually, we retain the element for the range + # that crosses the 255/256 boundary if it is one that matches the + # property. For example, in the Word property, there is a range of code + # points that start at U+00F8 and goes through U+02C1. Instead of + # artificially cutting that off at 256 because 256 is the first code point + # above Latin1, we let the range go to its natural ending. That gives us + # extra information with no added space taken. But if the range that + # crosses the boundary is one that doesn't match the property, we don't + # start a new range above 255, as that could be construed as going to + # infinity. For example, the Upper property doesn't include the character + # at 255, but does include the one at 256. We don't include the 256 one. + my $prop_name = $prop; + my $is_local_sub = $prop_name =~ s/^&//; + my $extra_enums = ""; + $extra_enums = $1 if $prop_name =~ s/, ( .* ) //x; + my $lookup_prop = $prop_name; + my $l1_only = ($lookup_prop =~ s/^L1Posix/XPosix/ + or $lookup_prop =~ s/^L1//); + my $nonl1_only = 0; + $nonl1_only = $lookup_prop =~ s/^NonL1// unless $l1_only; + ($lookup_prop, my $has_suffixes) = $lookup_prop =~ / (.*) ( , .* )? /x; + + my @invlist; + my @invmap; + my $map_format; + my $map_default; + my $maps_to_code_point; + my $to_adjust; + if ($is_local_sub) { + @invlist = eval $lookup_prop; + die $@ if $@; + } + else { + @invlist = prop_invlist($lookup_prop, '_perl_core_internal_ok'); + if (! @invlist) { + + # If couldn't find a non-empty inversion list, see if it is + # instead an inversion map + my ($list_ref, $map_ref, $format, $default) + = prop_invmap($lookup_prop, '_perl_core_internal_ok'); + if (! $list_ref) { + # An empty return here could mean an unknown property, or + # merely that the original inversion list is empty. Call + # in scalar context to differentiate + my $count = prop_invlist($lookup_prop, + '_perl_core_internal_ok'); + die "Could not find inversion list for '$lookup_prop'" + unless defined $count; + } + else { + @invlist = @$list_ref; + @invmap = @$map_ref; + $map_format = $format; + $map_default = $default; + $maps_to_code_point = $map_format =~ /x/; + $to_adjust = $map_format =~ /a/; + } + } + } -for my $prop (qw( - ASCII - HorizSpace - VertSpace - PerlSpace - XPerlSpace - PosixAlnum - L1PosixAlnum - PosixAlpha - L1PosixAlpha - PosixBlank - XPosixBlank - PosixCntrl - XPosixCntrl - PosixDigit - PosixGraph - L1PosixGraph - PosixLower - L1PosixLower - PosixPrint - L1PosixPrint - PosixPunct - L1PosixPunct - PosixSpace - XPosixSpace - PosixUpper - L1PosixUpper - PosixWord - L1PosixWord - PosixXDigit - XPosixXDigit - ) -) { - - # For the Latin1 properties, we change to use the eXtended version of the - # base property, then go through the result and get rid of everything not - # in Latin1 (above 255). Actually, we retain the element that crosses the - # 255/256 boundary. For example, in the Word property, there is a range - # of code points that start at U+00F8 and goes through U+02C1. Instead of - # artifically cutting that off at 256 because 256 is the first code point - # above Latin1, we let the range go to its natural ending. That gives us - # extra information with no added space taken. - my $lookup_prop = $prop =~ s/^L1/X/r; - my @invlist = prop_invlist($lookup_prop); - - if ($lookup_prop ne $prop) { - for my $i (0 .. @invlist - 1 - 1) { - if ($invlist[$i] > 255) { - splice @invlist, $i+1; + + # Short-circuit an empty inversion list. + if (! @invlist) { + output_invlist($prop_name, \@invlist, $charset); + next; + } + + # Re-order the Unicode code points to native ones for this platform. + # This is only needed for code points below 256, because native code + # points are only in that range. For inversion maps of properties + # where the mappings are adjusted (format =~ /a/), this reordering + # could mess up the adjustment pattern that was in the input, so that + # has to be dealt with. + # + # And inversion maps that map to code points need to eventually have + # all those code points remapped to native, and it's better to do that + # here, going through the whole list not just those below 256. This + # is because some inversion maps have adjustments (format =~ /a/) + # which may be affected by the reordering. This code needs to be done + # both for when we are translating the inversion lists for < 256, and + # for the inversion maps for everything. By doing both in this loop, + # we can share that code. + # + # So, we go through everything for an inversion map to code points; + # otherwise, we can skip any remapping at all if we are going to + # output only the above-Latin1 values, or if the range spans the whole + # of 0..256, as the remap will also include all of 0..256 (256 not + # 255 because a re-ordering could cause 256 to need to be in the same + # range as 255.) + if ((@invmap && $maps_to_code_point) + || (! $nonl1_only || ($invlist[0] < 256 + && ! ($invlist[0] == 0 && $invlist[1] > 256)))) + { + + if (! @invmap) { # Straight inversion list + # Look at all the ranges that start before 257. + my @latin1_list; + while (@invlist) { + last if $invlist[0] > 256; + my $upper = @invlist > 1 + ? $invlist[1] - 1 # In range + + # To infinity. You may want to stop much much + # earlier; going this high may expose perl + # deficiencies with very large numbers. + : $Unicode::UCD::MAX_CP; + for my $j ($invlist[0] .. $upper) { + push @latin1_list, a2n($j); + } + + shift @invlist; # Shift off the range that's in the list + shift @invlist; # Shift off the range not in the list + } + + # Here @invlist contains all the ranges in the original that start + # at code points above 256, and @latin1_list contains all the + # native code points for ranges that start with a Unicode code + # point below 257. We sort the latter and convert it to inversion + # list format. Then simply prepend it to the list of the higher + # code points. + @latin1_list = sort { $a <=> $b } @latin1_list; + @latin1_list = mk_invlist_from_sorted_cp_list(\@latin1_list); + unshift @invlist, @latin1_list; + } + else { # Is an inversion map + + # This is a similar procedure as plain inversion list, but has + # multiple buckets. A plain inversion list just has two + # buckets, 1) 'in' the list; and 2) 'not' in the list, and we + # pretty much can ignore the 2nd bucket, as it is completely + # defined by the 1st. But here, what we do is create buckets + # which contain the code points that map to each, translated + # to native and turned into an inversion list. Thus each + # bucket is an inversion list of native code points that map + # to it or don't map to it. We use these to create an + # inversion map for the whole property. + + # As mentioned earlier, we use this procedure to not just + # remap the inversion list to native values, but also the maps + # of code points to native ones. In the latter case we have + # to look at the whole of the inversion map (or at least to + # above Unicode; as the maps of code points above that should + # all be to the default). + my $upper_limit = ($maps_to_code_point) ? 0x10FFFF : 256; + + my %mapped_lists; # A hash whose keys are the buckets. + while (@invlist) { + last if $invlist[0] > $upper_limit; + + # This shouldn't actually happen, as prop_invmap() returns + # an extra element at the end that is beyond $upper_limit + die "inversion map that extends to infinity is unimplemented" unless @invlist > 1; + + my $bucket; + + # A hash key can't be a ref (we are only expecting arrays + # of scalars here), so convert any such to a string that + # will be converted back later (using a vertical tab as + # the separator). Even if the mapping is to code points, + # we don't translate to native here because the code + # output_invmap() calls to output these arrays assumes the + # input is Unicode, not native. + if (ref $invmap[0]) { + $bucket = join "\cK", @{$invmap[0]}; + } + elsif ($maps_to_code_point && $invmap[0] =~ $numeric_re) { + + # Do convert to native for maps to single code points. + # There are some properties that have a few outlier + # maps that aren't code points, so the above test + # skips those. + $bucket = a2n($invmap[0]); + } else { + $bucket = $invmap[0]; + } + + # We now have the bucket that all code points in the range + # map to, though possibly they need to be adjusted. Go + # through the range and put each translated code point in + # it into its bucket. + my $base_map = $invmap[0]; + for my $j ($invlist[0] .. $invlist[1] - 1) { + if ($to_adjust + # The 1st code point doesn't need adjusting + && $j > $invlist[0] + + # Skip any non-numeric maps: these are outliers + # that aren't code points. + && $base_map =~ $numeric_re + + # 'ne' because the default can be a string + && $base_map ne $map_default) + { + # We adjust, by incrementing each the bucket and + # the map. For code point maps, translate to + # native + $base_map++; + $bucket = ($maps_to_code_point) + ? a2n($base_map) + : $base_map; + } + + # Add the native code point to the bucket for the + # current map + push @{$mapped_lists{$bucket}}, a2n($j); + } # End of loop through all code points in the range + + # Get ready for the next range + shift @invlist; + shift @invmap; + } # End of loop through all ranges in the map. + + # Here, @invlist and @invmap retain all the ranges from the + # originals that start with code points above $upper_limit. + # Each bucket in %mapped_lists contains all the code points + # that map to that bucket. If the bucket is for a map to a + # single code point is a single code point, the bucket has + # been converted to native. If something else (including + # multiple code points), no conversion is done. + # + # Now we recreate the inversion map into %xlated, but this + # time for the native character set. + my %xlated; + foreach my $bucket (keys %mapped_lists) { + + # Sort and convert this bucket to an inversion list. The + # result will be that ranges that start with even-numbered + # indexes will be for code points that map to this bucket; + # odd ones map to some other bucket, and are discarded + # below. + @{$mapped_lists{$bucket}} + = sort{ $a <=> $b} @{$mapped_lists{$bucket}}; + @{$mapped_lists{$bucket}} + = mk_invlist_from_sorted_cp_list(\@{$mapped_lists{$bucket}}); + + # Add each even-numbered range in the bucket to %xlated; + # so that the keys of %xlated become the range start code + # points, and the values are their corresponding maps. + while (@{$mapped_lists{$bucket}}) { + my $range_start = $mapped_lists{$bucket}->[0]; + if ($bucket =~ /\cK/) { + @{$xlated{$range_start}} = split /\cK/, $bucket; + } + else { + $xlated{$range_start} = $bucket; + } + shift @{$mapped_lists{$bucket}}; # Discard odd ranges + shift @{$mapped_lists{$bucket}}; # Get ready for next + # iteration + } + } # End of loop through all the buckets. + + # Here %xlated's keys are the range starts of all the code + # points in the inversion map. Construct an inversion list + # from them. + my @new_invlist = sort { $a <=> $b } keys %xlated; + + # If the list is adjusted, we want to munge this list so that + # we only have one entry for where consecutive code points map + # to consecutive values. We just skip the subsequent entries + # where this is the case. + if ($to_adjust) { + my @temp; + for my $i (0 .. @new_invlist - 1) { + next if $i > 0 + && $new_invlist[$i-1] + 1 == $new_invlist[$i] + && $xlated{$new_invlist[$i-1]} =~ $numeric_re + && $xlated{$new_invlist[$i]} =~ $numeric_re + && $xlated{$new_invlist[$i-1]} + 1 == $xlated{$new_invlist[$i]}; + push @temp, $new_invlist[$i]; + } + @new_invlist = @temp; + } + + # The inversion map comes from %xlated's values. We can + # unshift each onto the front of the untouched portion, in + # reverse order of the portion we did process. + foreach my $start (reverse @new_invlist) { + unshift @invmap, $xlated{$start}; + } + + # Finally prepend the inversion list we have just constructed to the + # one that contains anything we didn't process. + unshift @invlist, @new_invlist; + } + } + + # prop_invmap() returns an extra final entry, which we can now + # discard. + if (@invmap) { + pop @invlist; + pop @invmap; + } + + if ($l1_only) { + die "Unimplemented to do a Latin-1 only inversion map" if @invmap; + for my $i (0 .. @invlist - 1 - 1) { + if ($invlist[$i] > 255) { + + # In an inversion list, even-numbered elements give the code + # points that begin ranges that match the property; + # odd-numbered give ones that begin ranges that don't match. + # If $i is odd, we are at the first code point above 255 that + # doesn't match, which means the range it is ending does + # match, and crosses the 255/256 boundary. We want to include + # this ending point, so increment $i, so the splice below + # includes it. Conversely, if $i is even, it is the first + # code point above 255 that matches, which means there was no + # matching range that crossed the boundary, and we don't want + # to include this code point, so splice before it. + $i++ if $i % 2 != 0; + + # Remove everything past this. + splice @invlist, $i; + splice @invmap, $i if @invmap; + last; + } + } + } + elsif ($nonl1_only) { + my $found_nonl1 = 0; + for my $i (0 .. @invlist - 1 - 1) { + next if $invlist[$i] < 256; + + # Here, we have the first element in the array that indicates an + # element above Latin1. Get rid of all previous ones. + splice @invlist, 0, $i; + splice @invmap, 0, $i if @invmap; + + # If this one's index is not divisible by 2, it means that this + # element is inverting away from being in the list, which means + # all code points from 256 to this one are in this list (or + # map to the default for inversion maps) + if ($i % 2 != 0) { + unshift @invlist, 256; + unshift @invmap, $map_default if @invmap; + } + $found_nonl1 = 1; last; } + die "No non-Latin1 code points in $lookup_prop" unless $found_nonl1; } + + output_invlist($prop_name, \@invlist, $charset); + output_invmap($prop_name, \@invmap, $lookup_prop, $map_format, $map_default, $extra_enums, $charset) if @invmap; } + end_file_pound_if; + print $out_fh "\n" . get_conditional_compile_line_end(); +} + +switch_pound_if('Boundary_pair_tables', 'PERL_IN_REGEXEC_C'); + +output_GCB_table(); +output_LB_table(); +output_WB_table(); - output_invlist($prop, \@invlist); +end_file_pound_if; + +my $sources_list = "lib/unicore/mktables.lst"; +my @sources = ($0, qw(lib/unicore/mktables + lib/Unicode/UCD.pm + regen/charset_translations.pl + )); +{ + # Depend on mktables’ own sources. It’s a shorter list of files than + # those that Unicode::UCD uses. + if (! open my $mktables_list, '<', $sources_list) { + + # This should force a rebuild once $sources_list exists + push @sources, $sources_list; + } + else { + while(<$mktables_list>) { + last if /===/; + chomp; + push @sources, "lib/unicore/$_" if /^[^#]/; + } + } } -read_only_bottom_close_and_rename($out_fh) +read_only_bottom_close_and_rename($out_fh, \@sources);