regcomp.c: Fix \N{} multi-char fold buffer boundary bug

An earlier commit in this topic branch fixed the bug (for non-\N{})
cases where a multi-character fold could try to span two EXACTFish
nodes, where they are split because the first one would otherwise
contain too long a string.

This commit extends that fix to include characters entered via \N{...}.
It does this by causing \N handling to be split, so that if the \N
resolves to a single code point, it goes through the normal processing,
so that it no longer bypasses the code that was added in the earlier
commit.

diff --git a/regcomp.c b/regcomp.c
index 311fae6..17e7579 100644 (file)
--- a/regcomp.c
+++ b/regcomp.c
@@ -9611,12 +9611,15 @@ S_regpiece(pTHX_ RExC_state_t *pRExC_state, I32 *flagp, U32 depth)
    consisting of a just a single code point; <*valuep> is set to that value
    if the input is such.
 
-   If <node_p> is non-null it signifies that the caller can accept any legal
-   sequence.  <*node_p> is set as follows:
+   If <node_p> is non-null it signifies that the caller can accept any other
+   legal sequence (i.e., one that isn't just a single code point).  <*node_p>
+   is set as follows:
     1) \N means not-a-NL: points to a newly created REG_ANY node;
     2) \N{}:              points to a new NOTHING node;
     3) otherwise:         points to a new EXACT node containing the resolved
                           string.
+   Note that FALSE is returned for single code point sequences if <valuep> is
+   null.
  */
 
 STATIC bool
@@ -9635,7 +9638,7 @@ S_grok_bslash_N(pTHX_ RExC_state_t *pRExC_state, regnode** node_p, UV *valuep, I
 
     GET_RE_DEBUG_FLAGS;
 
-    assert(node_p || valuep);
+    assert(cBOOL(node_p) ^ cBOOL(valuep));  /* Exactly one should be set */
 
     /* The [^\n] meaning of \N ignores spaces and comments under the /x
      * modifier.  The other meaning does not */
@@ -9713,7 +9716,7 @@ S_grok_bslash_N(pTHX_ RExC_state_t *pRExC_state, regnode** node_p, UV *valuep, I
      * point, and is terminated by the brace */
     has_multiple_chars = (endchar < endbrace);
 
-    if (valuep && ! node_p && (! has_multiple_chars || in_char_class)) {
+    if (valuep && (! has_multiple_chars || in_char_class)) {
        /* We only pay attention to the first char of
         multichar strings being returned in char classes. I kinda wonder
        if this makes sense as it does change the behaviour
@@ -9749,7 +9752,7 @@ S_grok_bslash_N(pTHX_ RExC_state_t *pRExC_state, regnode** node_p, UV *valuep, I
         }
         RExC_parse = endbrace + 1;
     }
-    else if (! node_p) {
+    else if (! node_p || ! has_multiple_chars) {
 
         /* Here, the input is legal, but not according to the caller's
          * options.  We fail without advancing the parse, so that the
@@ -10259,12 +10262,21 @@ tryagain:
            }
            break;
         case 'N': 
-            /* Handle \N and \N{NAME} here and not below because it can be
-            multicharacter. join_exact() will join them up later on. 
-            Also this makes sure that things like /\N{BLAH}+/ and 
-            \N{BLAH} being multi char Just Happen. dmq*/
+            /* Handle \N and \N{NAME} with multiple code points here and not
+             * below because it can be multicharacter. join_exact() will join
+             * them up later on.  Also this makes sure that things like
+             * /\N{BLAH}+/ and \N{BLAH} being multi char Just Happen. dmq.
+             * The options to the grok function call causes it to fail if the
+             * sequence is just a single code point.  We then go treat it as
+             * just another character in the current EXACT node, and hence it
+             * gets uniform treatment with all the other characters.  The
+             * special treatment for quantifiers is not needed for such single
+             * character sequences */
             ++RExC_parse;
-            grok_bslash_N(pRExC_state, &ret, NULL, flagp, depth, FALSE);
+            if (! grok_bslash_N(pRExC_state, &ret, NULL, flagp, depth, FALSE)) {
+                RExC_parse--;
+                goto defchar;
+            }
             break;
        case 'k':    /* Handle \k<NAME> and \k'NAME' */
        parse_named_seq:
@@ -10410,7 +10422,7 @@ tryagain:
 
        defchar: {
            register STRLEN len = 0;
-           register UV ender;
+           UV ender;
            register char *p;
            char *s;
 #define MAX_NODE_STRING_SIZE 127
@@ -10492,7 +10504,6 @@ tryagain:
                    case 'g': case 'G':   /* generic-backref, pos assertion */
                    case 'h': case 'H':   /* HORIZWS */
                    case 'k': case 'K':   /* named backref, keep marker */
-                   case 'N':             /* named char sequence */
                    case 'p': case 'P':   /* Unicode property */
                              case 'R':   /* LNBREAK */
                    case 's': case 'S':   /* space class */
@@ -10510,6 +10521,19 @@ tryagain:
                        ender = '\n';
                        p++;
                        break;
+                   case 'N': /* Handle a single-code point named character. */
+                        /* The options cause it to fail if a multiple code
+                         * point sequence.  Handle those in the switch() above
+                         * */
+                        RExC_parse = p + 1;
+                        if (! grok_bslash_N(pRExC_state, NULL, &ender,
+                                            flagp, depth, FALSE))
+                        {
+                            RExC_parse = p = oldp;
+                            goto loopdone;
+                        }
+                        p = RExC_parse;
+                        break;
                    case 'r':
                        ender = '\r';
                        p++;

diff --git a/t/re/pat_advanced.t b/t/re/pat_advanced.t
index a3540fd..95f904f 100644 (file)
--- a/t/re/pat_advanced.t
+++ b/t/re/pat_advanced.t
@@ -2083,6 +2083,27 @@ EOP
         }
         ok(! $failed, "Matched multi-char fold across EXACTFish node boundaries; if failed, was at count $failed");
 
+        $failed = 0;
+        for my $repeat (1 .. 300) {
+            my $string = $single x $repeat;
+            my $lhs = $string . "\N{LATIN SMALL LIGATURE FFI}";
+            if ($lhs !~ m/${string}ff\N{LATIN SMALL LETTER I}/i) {
+                $failed = $repeat;
+                last;
+            }
+        }
+        ok(! $failed, "Matched multi-char fold across EXACTFish node boundaries; if failed, was at count $failed");
+
+        $failed = 0;
+        for my $repeat (1 .. 300) {
+            my $string = $single x $repeat;
+            my $lhs = $string . "\N{LATIN SMALL LIGATURE FFL}";
+            if ($lhs !~ m/${string}ff\N{U+6c}/i) {
+                $failed = $repeat;
+                last;
+            }
+        }
+        ok(! $failed, "Matched multi-char fold across EXACTFish node boundaries; if failed, was at count $failed");
     }
 
     #

diff --git a/t/re/re_tests b/t/re/re_tests
index 9fa374e..3fdaf80 100644 (file)
--- a/t/re/re_tests
+++ b/t/re/re_tests
@@ -1443,7 +1443,7 @@ abc\N     abc\n   n
 # figures it out.
 \N{U+} -       c       -       Invalid hexadecimal number
 [\N{U+}]       -       c       -       Invalid hexadecimal number
-\N{U+4AG3}     -       c       -       Illegal hexadecimal digit
+\N{U+4AG3}     -       c       -       Invalid hexadecimal number
 [\N{U+4AG3}]   -       c       -       Invalid hexadecimal number
 abc\N{def      -       c       -       \\N{NAME} must be resolved by the lexer
 
@@ -1457,7 +1457,7 @@ abc\N{def -       c       -       \\N{NAME} must be resolved by the lexer
 
 # Verify works in single quotish context; regex compiler delivers slightly different msg
 # \N{U+BEEF.BEAD} succeeds here, because can't completely hide it from the outside.
-\N{U+0xBEEF}   -       c       -       Illegal hexadecimal digit
+\N{U+0xBEEF}   -       c       -       Invalid hexadecimal number
 \c`    -       c       -       \"\\c`\" is more clearly written simply as \"\\ \"
 \c1    -       c       -       \"\\c1\" is more clearly written simply as \"q\"
 \cA    \001    y       $&      \1