Allow for changing size of bracketed regex char class

This commit allows Perl to be compiled with a bitmap size that is larger
than 256.  This bitmap is used to directly look up whether a character
matches or not, without having to do a binary search or hash lookup.  It
might improve the performance for some installations that have a lot of
use of scripts that are above the Latin1 range.

diff --git a/embedvar.h b/embedvar.h
index 766880c..d481681 100644 (file)
--- a/embedvar.h
+++ b/embedvar.h
@@ -53,6 +53,7 @@
 #define PL_Dir                 (vTHX->IDir)
 #define PL_Env                 (vTHX->IEnv)
 #define PL_HasMultiCharFold    (vTHX->IHasMultiCharFold)
+#define PL_InBitmap            (vTHX->IInBitmap)
 #define PL_LIO                 (vTHX->ILIO)
 #define PL_Latin1              (vTHX->ILatin1)
 #define PL_Mem                 (vTHX->IMem)

diff --git a/intrpvar.h b/intrpvar.h
index 06194d9..57918b2 100644 (file)
--- a/intrpvar.h
+++ b/intrpvar.h
@@ -580,6 +580,7 @@ PERLVAR(I, numeric_radix_sv, SV *)  /* The radix separator if not '.' */
 PERLVAR(I, Latin1,     SV *)
 PERLVAR(I, UpperLatin1,        SV *)   /* Code points 128 - 255 */
 PERLVAR(I, AboveLatin1,        SV *)
+PERLVAR(I, InBitmap,   SV *)
 
 PERLVAR(I, NonL1NonFinalFold,   SV *)
 PERLVAR(I, HasMultiCharFold,   SV *)

diff --git a/perl.c b/perl.c
index b61e2ff..8f45273 100644 (file)
--- a/perl.c
+++ b/perl.c
@@ -1034,6 +1034,7 @@ perl_destruct(pTHXx)
     SvREFCNT_dec(PL_utf8_foldable);
     SvREFCNT_dec(PL_utf8_foldclosures);
     SvREFCNT_dec(PL_AboveLatin1);
+    SvREFCNT_dec(PL_InBitmap);
     SvREFCNT_dec(PL_UpperLatin1);
     SvREFCNT_dec(PL_Latin1);
     SvREFCNT_dec(PL_NonL1NonFinalFold);
@@ -1047,6 +1048,7 @@ perl_destruct(pTHXx)
     PL_utf8_idcont     = NULL;
     PL_utf8_foldclosures = NULL;
     PL_AboveLatin1       = NULL;
+    PL_InBitmap          = NULL;
     PL_HasMultiCharFold  = NULL;
     PL_Latin1            = NULL;
     PL_NonL1NonFinalFold = NULL;

diff --git a/regcomp.c b/regcomp.c
index afe6c39..3f12e97 100644 (file)
--- a/regcomp.c
+++ b/regcomp.c
@@ -1062,7 +1062,7 @@ S_get_ANYOF_cp_list_for_ssc(pTHX_ const RExC_state_t *pRExC_state,
 
     /* Similarly for these */
     if (ANYOF_FLAGS(node) & ANYOF_MATCHES_ALL_ABOVE_BITMAP) {
-        invlist = _add_range_to_invlist(invlist, 256, UV_MAX);
+        _invlist_union_complement_2nd(invlist, PL_InBitmap, &invlist);
     }
 
     if (ANYOF_FLAGS(node) & ANYOF_INVERT) {
@@ -6276,6 +6276,13 @@ Perl_re_op_compile(pTHX_ SV ** const patternp, int pat_count,
         PL_utf8_foldable = _new_invlist_C_array(_Perl_Any_Folds_invlist);
         PL_HasMultiCharFold =
                        _new_invlist_C_array(_Perl_Folds_To_Multi_Char_invlist);
+
+        /* This is calculated here, because the Perl program that generates the
+         * static global ones doesn't currently have access to
+         * NUM_ANYOF_CODE_POINTS */
+       PL_InBitmap = _new_invlist(2);
+       PL_InBitmap = _add_range_to_invlist(PL_InBitmap, 0,
+                                                    NUM_ANYOF_CODE_POINTS - 1);
     }
 #endif
 
@@ -12455,10 +12462,10 @@ S_populate_ANYOF_from_invlist(pTHX_ regnode *node, SV** invlist_ptr)
            UV high;
            int i;
 
-            if (end == UV_MAX && start <= 256) {
+            if (end == UV_MAX && start <= NUM_ANYOF_CODE_POINTS) {
                 ANYOF_FLAGS(node) |= ANYOF_MATCHES_ALL_ABOVE_BITMAP;
             }
-            else if (end >= 256) {
+            else if (end >= NUM_ANYOF_CODE_POINTS) {
                 ANYOF_FLAGS(node) |= ANYOF_HAS_UTF8_NONBITMAP_MATCHES;
             }
 
@@ -12485,10 +12492,10 @@ S_populate_ANYOF_from_invlist(pTHX_ regnode *node, SV** invlist_ptr)
          * *invlist_ptr; similarly for code points above the bitmap if we have
          * a flag to match all of them anyways */
        if (change_invlist) {
-           _invlist_subtract(*invlist_ptr, PL_Latin1, invlist_ptr);
+           _invlist_subtract(*invlist_ptr, PL_InBitmap, invlist_ptr);
        }
         if (ANYOF_FLAGS(node) & ANYOF_MATCHES_ALL_ABOVE_BITMAP) {
-           _invlist_intersection(*invlist_ptr, PL_Latin1, invlist_ptr);
+           _invlist_intersection(*invlist_ptr, PL_InBitmap, invlist_ptr);
        }
 
        /* If have completely emptied it, remove it completely */
@@ -16967,8 +16974,7 @@ S_put_charclass_bitmap_innards(pTHX_ SV *sv, char *bitmap, SV** bitmap_invlist)
 
         /* Add everything remaining to the list, so when we invert it just
          * below, it will be excluded */
-        *invlist_ptr = _add_range_to_invlist(*invlist_ptr,
-                                             NUM_ANYOF_CODE_POINTS, UV_MAX);
+        _invlist_union_complement_2nd(*invlist_ptr, PL_InBitmap, invlist_ptr);
         _invlist_invert(*invlist_ptr);
     }
 

diff --git a/regcomp.h b/regcomp.h
index f1f55eb..9b11ce5 100644 (file)
--- a/regcomp.h
+++ b/regcomp.h
@@ -184,7 +184,20 @@ struct regnode_2 {
     U16 arg2;
 };
 
-#define NUM_ANYOF_CODE_POINTS   256
+/* This give the number of code points that can be in the bitmap of an ANYOF
+ * node.  The shift number must currently be one of: 8..12.  It can't be less
+ * than 8 (256) because some code relies on it being at least that.  Above 12
+ * (4096), and you start running into warnings that some data structure widths
+ * have been exceeded, though the test suite as of this writing still passes
+ * for up through 16, which is as high as anyone would ever want to go,
+ * encompassing all of the Unicode BMP, and thus including all the economically
+ * important world scripts.  At 12 most of them are: including Arabic,
+ * Cyrillic, Greek, Hebrew, Indian subcontinent, Latin, and Thai; but not Han,
+ * Japanese, nor Korean.  (The regarglen structure in regnodes.h is a U8, and
+ * the trie types TRIEC and AHOCORASICKC are larger than U8 for shift values
+ * below above 12.)  Be sure to benchmark before changing, as larger sizes do
+ * significantly slow down the test suite */
+#define NUM_ANYOF_CODE_POINTS   (1 << 8)
 
 #define ANYOF_BITMAP_SIZE      (NUM_ANYOF_CODE_POINTS / 8)   /* 8 bits/Byte */
 

diff --git a/regexec.c b/regexec.c
index 6a8c197..52ff312 100644 (file)
--- a/regexec.c
+++ b/regexec.c
@@ -7686,11 +7686,14 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const
        }
        else if (flags & ANYOF_LOCALE_FLAGS) {
            if ((flags & ANYOF_LOC_FOLD)
+                && c < 256
                && ANYOF_BITMAP_TEST(n, PL_fold_locale[c]))
             {
                 match = TRUE;
             }
-            else if (ANYOF_POSIXL_TEST_ANY_SET(n)) {
+            else if (ANYOF_POSIXL_TEST_ANY_SET(n)
+                     && c < 256
+            ) {
 
                 /* The data structure is arranged so bits 0, 2, 4, ... are set
                  * if the class includes the Posix character class given by

diff --git a/sv.c b/sv.c
index 65aa456..78086b4 100644 (file)
--- a/sv.c
+++ b/sv.c
@@ -14444,6 +14444,7 @@ perl_clone_using(PerlInterpreter *proto_perl, UV flags,
     PL_Latin1          = sv_dup_inc(proto_perl->ILatin1, param);
     PL_UpperLatin1     = sv_dup_inc(proto_perl->IUpperLatin1, param);
     PL_AboveLatin1     = sv_dup_inc(proto_perl->IAboveLatin1, param);
+    PL_InBitmap         = sv_dup_inc(proto_perl->IInBitmap, param);
 
     PL_NonL1NonFinalFold = sv_dup_inc(proto_perl->INonL1NonFinalFold, param);
     PL_HasMultiCharFold = sv_dup_inc(proto_perl->IHasMultiCharFold, param);