toke.c: Remove soon-to-be invalid t assumption

The code in toke.c assumes that the UTF8 expansion of the string
"\x{foo}" takes no more bytes than the original input text, which
includes the 4 bytes of overhead "\x{}".  Similarly for "\o{}".  The
functions that convert to the code point actually now assert for this.
The next commit will make this assumption definitely invalid on EBCDIC
platforms.  Remove the assertions, and actually handle the case
properly.  The other places that call the conversion functions do not
make this assumption, so there is no harm in removing them from there.

Since we believe that this can't happen except on EBCDIC, we
could #ifdef this code and use just an assert on non-EBCDIC.  But it's
easier to maintain if #ifdef's are minimized.  Parsing is not a
time-critical operation, like being in an inner loop, and the extra test
gives a branch prediction hint to the compiler.

diff --git a/dquote.c b/dquote.c
index 42864d4..895f17d 100644 (file)
--- a/dquote.c
+++ b/dquote.c
@@ -158,10 +158,6 @@ Perl_grok_bslash_o(pTHX_ char **s, UV *uv, const char** error_msg,
     /* Return past the '}' */
     *s = e + 1;
 
-    /* guarantee replacing "\o{...}" with utf8 bytes fits within
-     * existing space */
-    assert(UVCHR_SKIP(*uv) < *s - start);
-
     return TRUE;
 }
 

diff --git a/dquote_inline.h b/dquote_inline.h
index 02c4f1d..050b14f 100644 (file)
--- a/dquote_inline.h
+++ b/dquote_inline.h
@@ -101,7 +101,7 @@ S_grok_bslash_x(pTHX_ char **s, UV *uv, const char** error_msg,
             }
             return FALSE;
         }
-       goto ok;
+       return TRUE;
     }
 
     e = strchr(*s, '}');
@@ -128,7 +128,7 @@ S_grok_bslash_x(pTHX_ char **s, UV *uv, const char** error_msg,
         }
         *s = e + 1;
         *uv = 0;
-        goto ok;
+        return TRUE;
     }
 
     flags |= PERL_SCAN_ALLOW_UNDERSCORES;
@@ -150,10 +150,6 @@ S_grok_bslash_x(pTHX_ char **s, UV *uv, const char** error_msg,
     /* Return past the '}' */
     *s = e + 1;
 
-  ok:
-    /* guarantee replacing "\x{...}" with utf8 bytes fits within
-     * existing space */
-    assert(UVCHR_SKIP(*uv) < *s - start);
     return TRUE;
 }
 

diff --git a/toke.c b/toke.c
index 6d6975c..f7d4e53 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -3357,10 +3357,7 @@ S_scan_const(pTHX_ char *start)
                }
 
              NUM_ESCAPE_INSERT:
-               /* Insert oct or hex escaped character.  There will always be
-                * enough room in sv since such escapes will be longer than any
-                * UTF-8 sequence they can end up as, except if they force us
-                * to recode the rest of the string into utf8 */
+               /* Insert oct or hex escaped character. */
                
                /* Here uv is the ordinal of the next character being added */
                if (UVCHR_IS_INVARIANT(uv)) {
@@ -3388,6 +3385,20 @@ S_scan_const(pTHX_ char *start)
                     }
 
                     if (has_utf8) {
+                       /* Usually, there will already be enough room in 'sv'
+                        * since such escapes are likely longer than any UTF-8
+                        * sequence they can end up as.  This isn't the case on
+                        * EBCDIC where \x{40000000} contains 12 bytes, and the
+                        * UTF-8 for it contains 14.  And, we have to allow for
+                        * a trailing NUL.  It probably can't happen on ASCII
+                        * platforms, but be safe */
+                        const STRLEN needed = d - SvPVX(sv) + UVCHR_SKIP(uv)
+                                            + 1;
+                        if (UNLIKELY(needed > SvLEN(sv))) {
+                            SvCUR_set(sv, d - SvPVX_const(sv));
+                            d = sv_grow(sv, needed) + SvCUR(sv);
+                        }
+
                        d = (char*)uvchr_to_utf8((U8*)d, uv);
                        if (PL_lex_inwhat == OP_TRANS
                             && PL_sublex_info.sub_op)