This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Version bumps
[perl5.git] / toke.c
diff --git a/toke.c b/toke.c
index 113632e..7b5c465 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -133,7 +133,7 @@ static const char ident_too_long[] = "Identifier too long";
 #ifdef USE_UTF8_SCRIPTS
 #   define UTF (!IN_BYTES)
 #else
-#   define UTF ((PL_linestr && DO_UTF8(PL_linestr)) || (PL_hints & HINT_UTF8))
+#   define UTF ((PL_linestr && DO_UTF8(PL_linestr)) || ( !(PL_parser->lex_flags & LEX_IGNORE_UTF8_HINTS) && (PL_hints & HINT_UTF8)))
 #endif
 
 /* The maximum number of characters preceding the unrecognized one to display */
@@ -224,6 +224,7 @@ static const char* const lex_state_names[] = {
  * LOOPX        : loop exiting command (goto, last, dump, etc)
  * FTST         : file test operator
  * FUN0         : zero-argument function
+ * FUN0OP       : zero-argument function, with its op created in this file
  * FUN1         : not used, except for not, which isn't a UNIOP
  * BOop         : bitwise or or xor
  * BAop         : bitwise and
@@ -254,6 +255,7 @@ static const char* const lex_state_names[] = {
 #define LOOPX(f) return (pl_yylval.ival=f, PL_expect=XTERM, PL_bufptr=s, REPORT((int)LOOPEX))
 #define FTST(f)  return (pl_yylval.ival=f, PL_expect=XTERMORDORDOR, PL_bufptr=s, REPORT((int)UNIOP))
 #define FUN0(f)  return (pl_yylval.ival=f, PL_expect=XOPERATOR, PL_bufptr=s, REPORT((int)FUNC0))
+#define FUN0OP(f)  return (pl_yylval.opval=f, CLINE, PL_expect=XOPERATOR, PL_bufptr=s, REPORT((int)FUNC0OP))
 #define FUN1(f)  return (pl_yylval.ival=f, PL_expect=XOPERATOR, PL_bufptr=s, REPORT((int)FUNC1))
 #define BOop(f)  return ao((pl_yylval.ival=f, PL_expect=XTERM, PL_bufptr=s, REPORT((int)BITOROP)))
 #define BAop(f)  return ao((pl_yylval.ival=f, PL_expect=XTERM, PL_bufptr=s, REPORT((int)BITANDOP)))
@@ -346,6 +348,7 @@ static struct debug_tokens {
     { FORMAT,          TOKENTYPE_NONE,         "FORMAT" },
     { FUNC,            TOKENTYPE_OPNUM,        "FUNC" },
     { FUNC0,           TOKENTYPE_OPNUM,        "FUNC0" },
+    { FUNC0OP,         TOKENTYPE_OPVAL,        "FUNC0OP" },
     { FUNC0SUB,                TOKENTYPE_OPVAL,        "FUNC0SUB" },
     { FUNC1,           TOKENTYPE_OPNUM,        "FUNC1" },
     { FUNCMETH,                TOKENTYPE_OPVAL,        "FUNCMETH" },
@@ -666,20 +669,22 @@ from which code will be read to be parsed.  If both are non-null, the
 code in I<line> comes first and must consist of complete lines of input,
 and I<rsfp> supplies the remainder of the source.
 
-The I<flags> parameter is reserved for future use, and must always
-be zero.
+The I<flags> parameter is reserved for future use.  Currently it is only
+used by perl internally, so extensions should always pass zero.
 
 =cut
 */
 
+/* LEX_START_SAME_FILTER indicates that this is not a new file, so it
+   can share filters with the current parser. */
+
 void
 Perl_lex_start(pTHX_ SV *line, PerlIO *rsfp, U32 flags)
 {
     dVAR;
     const char *s = NULL;
-    STRLEN len;
     yy_parser *parser, *oparser;
-    if (flags)
+    if (flags && flags & ~LEX_START_FLAGS)
        Perl_croak(aTHX_ "Lexing code internal error (%s)", "lex_start");
 
     /* create and initialise a parser */
@@ -708,24 +713,29 @@ Perl_lex_start(pTHX_ SV *line, PerlIO *rsfp, U32 flags)
     parser->lex_state = LEX_NORMAL;
     parser->expect = XSTATE;
     parser->rsfp = rsfp;
-    parser->rsfp_filters = newAV();
+    parser->rsfp_filters =
+      !(flags & LEX_START_SAME_FILTER) || !oparser
+        ? NULL
+        : MUTABLE_AV(SvREFCNT_inc(
+            oparser->rsfp_filters
+             ? oparser->rsfp_filters
+             : (oparser->rsfp_filters = newAV())
+          ));
 
     Newx(parser->lex_brackstack, 120, char);
     Newx(parser->lex_casestack, 12, char);
     *parser->lex_casestack = '\0';
 
     if (line) {
+       STRLEN len;
        s = SvPV_const(line, len);
+       parser->linestr = flags & LEX_START_COPIED
+                           ? SvREFCNT_inc_simple_NN(line)
+                           : newSVpvn_flags(s, len, SvUTF8(line));
+       if (!len || s[len-1] != ';')
+           sv_catpvs(parser->linestr, "\n;");
     } else {
-       len = 0;
-    }
-
-    if (!len) {
        parser->linestr = newSVpvs("\n;");
-    } else {
-       parser->linestr = newSVpvn_flags(s, len, SvUTF8(line));
-       if (s[len-1] != ';')
-           sv_catpvs(parser->linestr, "\n;");
     }
     parser->oldoldbufptr =
        parser->oldbufptr =
@@ -733,8 +743,9 @@ Perl_lex_start(pTHX_ SV *line, PerlIO *rsfp, U32 flags)
        parser->linestart = SvPVX(parser->linestr);
     parser->bufend = parser->bufptr + SvCUR(parser->linestr);
     parser->last_lop = parser->last_uni = NULL;
+    parser->lex_flags = flags & (LEX_IGNORE_UTF8_HINTS|LEX_EVALBYTES);
 
-    parser->in_pod = 0;
+    parser->in_pod = parser->filtered = 0;
 }
 
 
@@ -1252,7 +1263,7 @@ Perl_lex_next_chunk(pTHX_ U32 flags)
     }
     if (flags & LEX_FAKE_EOF) {
        goto eof;
-    } else if (!PL_parser->rsfp) {
+    } else if (!PL_parser->rsfp && !PL_parser->filtered) {
        got_some = 0;
     } else if (filter_gets(linestr, old_bufend_pos)) {
        got_some = 1;
@@ -1269,7 +1280,7 @@ Perl_lex_next_chunk(pTHX_ U32 flags)
        else if (PL_parser->rsfp)
            (void)PerlIO_close(PL_parser->rsfp);
        PL_parser->rsfp = NULL;
-       PL_parser->in_pod = 0;
+       PL_parser->in_pod = PL_parser->filtered = 0;
 #ifdef PERL_MAD
        if (PL_madskills && !PL_in_eval && (PL_minus_p || PL_minus_n))
            PL_faketokens = 1;
@@ -1407,7 +1418,10 @@ Perl_lex_read_unichar(pTHX_ U32 flags)
     if (c != -1) {
        if (c == '\n')
            CopLINE_inc(PL_curcop);
-       PL_parser->bufptr += UTF8SKIP(PL_parser->bufptr);
+       if (UTF)
+           PL_parser->bufptr += UTF8SKIP(PL_parser->bufptr);
+       else
+           ++(PL_parser->bufptr);
     }
     return c;
 }
@@ -1511,6 +1525,7 @@ S_incline(pTHX_ const char *s)
     const char *t;
     const char *n;
     const char *e;
+    line_t line_num;
 
     PERL_ARGS_ASSERT_INCLINE;
 
@@ -1554,9 +1569,10 @@ S_incline(pTHX_ const char *s)
     if (*e != '\n' && *e != '\0')
        return;         /* false alarm */
 
+    line_num = atoi(n)-1;
+
     if (t - s > 0) {
        const STRLEN len = t - s;
-#ifndef USE_ITHREADS
        SV *const temp_sv = CopFILESV(PL_curcop);
        const char *cf;
        STRLEN tmplen;
@@ -1569,7 +1585,7 @@ S_incline(pTHX_ const char *s)
            tmplen = 0;
        }
 
-       if (tmplen > 7 && strnEQ(cf, "(eval ", 6)) {
+       if (!PL_rsfp && !PL_parser->filtered) {
            /* must copy *{"::_<(eval N)[oldfilename:L]"}
             * to *{"::_<newfilename"} */
            /* However, the long form of evals is only turned on by the
@@ -1611,19 +1627,35 @@ S_incline(pTHX_ const char *s)
                    gv_init(gv2, PL_defstash, tmpbuf2, tmplen2, FALSE);
                    /* adjust ${"::_<newfilename"} to store the new file name */
                    GvSV(gv2) = newSVpvn(tmpbuf2 + 2, tmplen2 - 2);
-                   GvHV(gv2) = MUTABLE_HV(SvREFCNT_inc(GvHV(*gvp)));
-                   GvAV(gv2) = MUTABLE_AV(SvREFCNT_inc(GvAV(*gvp)));
+                   /* The line number may differ. If that is the case,
+                      alias the saved lines that are in the array.
+                      Otherwise alias the whole array. */
+                   if (CopLINE(PL_curcop) == line_num) {
+                       GvHV(gv2) = MUTABLE_HV(SvREFCNT_inc(GvHV(*gvp)));
+                       GvAV(gv2) = MUTABLE_AV(SvREFCNT_inc(GvAV(*gvp)));
+                   }
+                   else if (GvAV(*gvp)) {
+                       AV * const av = GvAV(*gvp);
+                       const I32 start = CopLINE(PL_curcop)+1;
+                       I32 items = AvFILLp(av) - start;
+                       if (items > 0) {
+                           AV * const av2 = GvAVn(gv2);
+                           SV **svp = AvARRAY(av) + start;
+                           I32 l = (I32)line_num+1;
+                           while (items--)
+                               av_store(av2, l++, SvREFCNT_inc(*svp++));
+                       }
+                   }
                }
 
                if (tmpbuf2 != smallbuf) Safefree(tmpbuf2);
            }
            if (tmpbuf != smallbuf) Safefree(tmpbuf);
        }
-#endif
        CopFILE_free(PL_curcop);
        CopFILE_setn(PL_curcop, s, len);
     }
-    CopLINE_set(PL_curcop, atoi(n)-1);
+    CopLINE_set(PL_curcop, line_num);
 }
 
 #ifdef PERL_MAD
@@ -2062,7 +2094,8 @@ S_force_ident(pTHX_ register const char *s, int kind)
 
     if (*s) {
        const STRLEN len = strlen(s);
-       OP* const o = (OP*)newSVOP(OP_CONST, 0, newSVpvn(s, len));
+       OP* const o = (OP*)newSVOP(OP_CONST, 0, newSVpvn_flags(s, len,
+                                                                UTF ? SVf_UTF8 : 0));
        start_force(PL_curforce);
        NEXTVAL_NEXTTOKE.opval = o;
        force_next(WORD);
@@ -2072,8 +2105,8 @@ S_force_ident(pTHX_ register const char *s, int kind)
               warnings if the symbol must be introduced in an eval.
               GSAR 96-10-12 */
            gv_fetchpvn_flags(s, len,
-                             PL_in_eval ? (GV_ADDMULTI | GV_ADDINEVAL)
-                             : GV_ADD,
+                             (PL_in_eval ? (GV_ADDMULTI | GV_ADDINEVAL)
+                             : GV_ADD) | ( UTF ? SVf_UTF8 : 0 ),
                              kind == '$' ? SVt_PV :
                              kind == '@' ? SVt_PVAV :
                              kind == '%' ? SVt_PVHV :
@@ -3122,12 +3155,22 @@ S_scan_const(pTHX_ char *start)
 
                    if (PL_lex_inpat) {
 
-                       /* Pass through to the regex compiler unchanged.  The
-                        * reason we evaluated the number above is to make sure
-                        * there wasn't a syntax error. */
+                       /* On non-EBCDIC platforms, pass through to the regex
+                        * compiler unchanged.  The reason we evaluated the
+                        * number above is to make sure there wasn't a syntax
+                        * error.  But on EBCDIC we convert to native so
+                        * downstream code can continue to assume it's native
+                        */
                        s -= 5;     /* Include the '\N{U+' */
+#ifdef EBCDIC
+                       d += my_snprintf(d, e - s + 1 + 1,  /* includes the }
+                                                              and the \0 */
+                                   "\\N{U+%X}",
+                                   (unsigned int) UNI_TO_NATIVE(uv));
+#else
                        Copy(s, d, e - s + 1, char);    /* 1 = include the } */
                        d += e - s + 1;
+#endif
                    }
                    else {  /* Not a pattern: convert the hex to string */
 
@@ -3221,10 +3264,13 @@ S_scan_const(pTHX_ char *start)
                            }
 
                            /* Convert first code point to hex, including the
-                            * boiler plate before it */
+                            * boiler plate before it.  For all these, we
+                            * convert to native format so that downstream code
+                            * can continue to assume the input is native */
                            output_length =
                                my_snprintf(hex_string, sizeof(hex_string),
-                                           "\\N{U+%X", (unsigned int) uv);
+                                           "\\N{U+%X",
+                                           (unsigned int) UNI_TO_NATIVE(uv));
 
                            /* Make sure there is enough space to hold it */
                            d = off + SvGROW(sv, off
@@ -3249,7 +3295,8 @@ S_scan_const(pTHX_ char *start)
 
                                output_length =
                                    my_snprintf(hex_string, sizeof(hex_string),
-                                               ".%X", (unsigned int) uv);
+                                           ".%X",
+                                           (unsigned int) UNI_TO_NATIVE(uv));
 
                                d = off + SvGROW(sv, off
                                                     + output_length
@@ -3585,7 +3632,8 @@ S_intuit_more(pTHX_ register char *s)
                    int len;
                    scan_ident(s, send, tmpbuf, sizeof tmpbuf, FALSE);
                    len = (int)strlen(tmpbuf);
-                   if (len > 1 && gv_fetchpvn_flags(tmpbuf, len, 0, SVt_PV))
+                   if (len > 1 && gv_fetchpvn_flags(tmpbuf, len,
+                                                    UTF ? SVf_UTF8 : 0, SVt_PV))
                        weight -= 100;
                    else
                        weight -= 10;
@@ -3692,7 +3740,7 @@ S_intuit_method(pTHX_ char *start, GV *gv, CV *cv)
            return 0;
        if (cv) {
            if (SvPOK(cv)) {
-               const char *proto = SvPVX_const(cv);
+               const char *proto = CvPROTO(cv);
                if (proto) {
                    if (*proto == ';')
                        proto++;
@@ -3733,11 +3781,11 @@ S_intuit_method(pTHX_ char *start, GV *gv, CV *cv)
 #endif
            goto bare_package;
        }
-       indirgv = gv_fetchpvn_flags(tmpbuf, len, 0, SVt_PVCV);
+       indirgv = gv_fetchpvn_flags(tmpbuf, len, ( UTF ? SVf_UTF8 : 0 ), SVt_PVCV);
        if (indirgv && GvCVu(indirgv))
            return 0;
        /* filehandle or package name makes it a method */
-       if (!gv || GvIO(indirgv) || gv_stashpvn(tmpbuf, len, 0)) {
+       if (!gv || GvIO(indirgv) || gv_stashpvn(tmpbuf, len, UTF ? SVf_UTF8 : 0)) {
 #ifdef PERL_MAD
            soff = s - SvPVX(PL_linestr);
 #endif
@@ -3750,7 +3798,8 @@ S_intuit_method(pTHX_ char *start, GV *gv, CV *cv)
                                                  S_newSV_maybe_utf8(aTHX_ tmpbuf, len));
            NEXTVAL_NEXTTOKE.opval->op_private = OPpCONST_BARE;
            if (PL_madskills)
-               curmad('X', newSVpvn(start,SvPVX(PL_linestr) + soff - start));
+               curmad('X', newSVpvn_flags(start,SvPVX(PL_linestr) + soff - start,
+                                                            ( UTF ? SVf_UTF8 : 0 )));
            PL_expect = XTERM;
            force_next(WORD);
            PL_bufptr = s;
@@ -3790,6 +3839,9 @@ Perl_filter_add(pTHX_ filter_t funcp, SV *datasv)
     if (!PL_parser)
        return NULL;
 
+    if (PL_parser->lex_flags & LEX_IGNORE_UTF8_HINTS)
+       Perl_croak(aTHX_ "Source filters apply only to byte streams");
+
     if (!PL_rsfp_filters)
        PL_rsfp_filters = newAV();
     if (!datasv)
@@ -3802,6 +3854,45 @@ Perl_filter_add(pTHX_ filter_t funcp, SV *datasv)
                          SvPV_nolen(datasv)));
     av_unshift(PL_rsfp_filters, 1);
     av_store(PL_rsfp_filters, 0, datasv) ;
+    if (
+       !PL_parser->filtered
+     && PL_parser->lex_flags & LEX_EVALBYTES
+     && PL_bufptr < PL_bufend
+    ) {
+       const char *s = PL_bufptr;
+       while (s < PL_bufend) {
+           if (*s == '\n') {
+               SV *linestr = PL_parser->linestr;
+               char *buf = SvPVX(linestr);
+               STRLEN const bufptr_pos = PL_parser->bufptr - buf;
+               STRLEN const oldbufptr_pos = PL_parser->oldbufptr - buf;
+               STRLEN const oldoldbufptr_pos=PL_parser->oldoldbufptr-buf;
+               STRLEN const linestart_pos = PL_parser->linestart - buf;
+               STRLEN const last_uni_pos =
+                   PL_parser->last_uni ? PL_parser->last_uni - buf : 0;
+               STRLEN const last_lop_pos =
+                   PL_parser->last_lop ? PL_parser->last_lop - buf : 0;
+               av_push(PL_rsfp_filters, linestr);
+               PL_parser->linestr = 
+                   newSVpvn(SvPVX(linestr), ++s-SvPVX(linestr));
+               buf = SvPVX(PL_parser->linestr);
+               PL_parser->bufend = buf + SvCUR(PL_parser->linestr);
+               PL_parser->bufptr = buf + bufptr_pos;
+               PL_parser->oldbufptr = buf + oldbufptr_pos;
+               PL_parser->oldoldbufptr = buf + oldoldbufptr_pos;
+               PL_parser->linestart = buf + linestart_pos;
+               if (PL_parser->last_uni)
+                   PL_parser->last_uni = buf + last_uni_pos;
+               if (PL_parser->last_lop)
+                   PL_parser->last_lop = buf + last_lop_pos;
+               SvLEN(linestr) = SvCUR(linestr);
+               SvCUR(linestr) = s-SvPVX(linestr);
+               PL_parser->filtered = 1;
+               break;
+           }
+           s++;
+       }
+    }
     return(datasv);
 }
 
@@ -3844,7 +3935,7 @@ Perl_filter_read(pTHX_ int idx, SV *buf_sv, int maxlen)
     /* This API is bad. It should have been using unsigned int for maxlen.
        Not sure if we want to change the API, but if not we should sanity
        check the value here.  */
-    const unsigned int correct_length
+    unsigned int correct_length
        = maxlen < 0 ?
 #ifdef PERL_MICRO
        0x7FFFFFFF
@@ -3896,6 +3987,31 @@ Perl_filter_read(pTHX_ int idx, SV *buf_sv, int maxlen)
                              idx));
        return FILTER_READ(idx+1, buf_sv, correct_length); /* recurse */
     }
+    if (SvTYPE(datasv) != SVt_PVIO) {
+       if (correct_length) {
+           /* Want a block */
+           const STRLEN remainder = SvLEN(datasv) - SvCUR(datasv);
+           if (!remainder) return 0; /* eof */
+           if (correct_length > remainder) correct_length = remainder;
+           sv_catpvn(buf_sv, SvEND(datasv), correct_length);
+           SvCUR_set(datasv, SvCUR(datasv) + correct_length);
+       } else {
+           /* Want a line */
+           const char *s = SvEND(datasv);
+           const char *send = SvPVX(datasv) + SvLEN(datasv);
+           while (s < send) {
+               if (*s == '\n') {
+                   s++;
+                   break;
+               }
+               s++;
+           }
+           if (s == send) return 0; /* eof */
+           sv_catpvn(buf_sv, SvEND(datasv), s-SvEND(datasv));
+           SvCUR_set(datasv, s-SvPVX(datasv));
+       }
+       return SvCUR(buf_sv);
+    }
     /* Get function pointer hidden within datasv       */
     funcp = DPTR2FPTR(filter_t, IoANY(datasv));
     DEBUG_P(PerlIO_printf(Perl_debug_log,
@@ -3944,20 +4060,20 @@ S_find_in_my_stash(pTHX_ const char *pkgname, STRLEN len)
 
     if (len > 2 &&
         (pkgname[len - 2] == ':' && pkgname[len - 1] == ':') &&
-        (gv = gv_fetchpvn_flags(pkgname, len, 0, SVt_PVHV)))
+        (gv = gv_fetchpvn_flags(pkgname, len, ( UTF ? SVf_UTF8 : 0 ), SVt_PVHV)))
     {
         return GvHV(gv);                       /* Foo:: */
     }
 
     /* use constant CLASS => 'MyClass' */
-    gv = gv_fetchpvn_flags(pkgname, len, 0, SVt_PVCV);
+    gv = gv_fetchpvn_flags(pkgname, len, UTF ? SVf_UTF8 : 0, SVt_PVCV);
     if (gv && GvCV(gv)) {
        SV * const sv = cv_const_sv(GvCV(gv));
        if (sv)
             pkgname = SvPV_const(sv, len);
     }
 
-    return gv_stashpvn(pkgname, len, 0);
+    return gv_stashpvn(pkgname, len, UTF ? SVf_UTF8 : 0);
 }
 
 /*
@@ -4048,7 +4164,7 @@ Perl_madlex(pTHX)
        }
 
        /* put off final whitespace till peg */
-       if (optype == ';' && !PL_rsfp) {
+       if (optype == ';' && !PL_rsfp && !PL_parser->filtered) {
            PL_nextwhite = PL_thiswhite;
            PL_thiswhite = 0;
        }
@@ -4328,7 +4444,7 @@ Perl_yylex(pTHX)
                    if (PL_lex_brackets > 100)
                        Renew(PL_lex_brackstack, PL_lex_brackets + 10, char);
                    PL_lex_brackstack[PL_lex_brackets++] =
-                       (next_type >> 16) & 0xff;
+                       (char) ((next_type >> 16) & 0xff);
                }
                if (next_type & (2<<24))
                    PL_lex_allbrackets++;
@@ -4632,7 +4748,7 @@ Perl_yylex(pTHX)
        if (PL_madskills)
            PL_faketokens = 0;
 #endif
-       if (!PL_rsfp) {
+       if (!PL_rsfp && (!PL_parser->filtered || s+1 < PL_bufend)) {
            PL_last_uni = 0;
            PL_last_lop = 0;
            if (PL_lex_brackets &&
@@ -4753,7 +4869,13 @@ Perl_yylex(pTHX)
                      *(U8*)s == 0xEF ||
                      *(U8*)s >= 0xFE ||
                      s[1] == 0)) {
-               bof = PerlIO_tell(PL_rsfp) == (Off_t)SvCUR(PL_linestr);
+               Off_t offset = (IV)PerlIO_tell(PL_rsfp);
+               bof = (offset == (Off_t)SvCUR(PL_linestr));
+#if defined(PERLIO_USING_CRLF) && defined(PERL_TEXTMODE_SCRIPTS)
+               /* offset may include swallowed CR */
+               if (!bof)
+                   bof = (offset == (Off_t)SvCUR(PL_linestr)+1);
+#endif
                if (bof) {
                    PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
                    s = swallow_bom((U8*)s);
@@ -4773,7 +4895,7 @@ Perl_yylex(pTHX)
                    PL_parser->in_pod = 0;
                }
            }
-           if (PL_rsfp)
+           if (PL_rsfp || PL_parser->filtered)
                incline(s);
        } while (PL_parser->in_pod);
        PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = s;
@@ -4999,15 +5121,17 @@ Perl_yylex(pTHX)
        if (PL_madskills)
            PL_faketokens = 0;
 #endif
-       if (PL_lex_state != LEX_NORMAL || (PL_in_eval && !PL_rsfp)) {
-           if (*s == '#' && s == PL_linestart && PL_in_eval && !PL_rsfp) {
+       if (PL_lex_state != LEX_NORMAL ||
+            (PL_in_eval && !PL_rsfp && !PL_parser->filtered)) {
+           if (*s == '#' && s == PL_linestart && PL_in_eval
+            && !PL_rsfp && !PL_parser->filtered) {
                /* handle eval qq[#line 1 "foo"\n ...] */
                CopLINE_dec(PL_curcop);
                incline(s);
            }
            if (PL_madskills && !PL_lex_formbrack && !PL_in_eval) {
                s = SKIPSPACE0(s);
-               if (!PL_in_eval || PL_rsfp)
+               if (!PL_in_eval || PL_rsfp || PL_parser->filtered)
                    incline(s);
            }
            else {
@@ -5328,7 +5452,7 @@ Perl_yylex(pTHX)
                        break;
                    }
                }
-               sv = newSVpvn(s, len);
+               sv = newSVpvn_flags(s, len, UTF ? SVf_UTF8 : 0);
                if (*d == '(') {
                    d = scan_str(d,TRUE,TRUE);
                    if (!d) {
@@ -5784,7 +5908,7 @@ Perl_yylex(pTHX)
            if (PL_expect == XSTATE && isALPHA(tmp) &&
                (s == PL_linestart+1 || s[-2] == '\n') )
                {
-                   if (PL_in_eval && !PL_rsfp) {
+                   if (PL_in_eval && !PL_rsfp && !PL_parser->filtered) {
                        d = PL_bufend;
                        while (s < d) {
                            if (*s++ == '\n') {
@@ -5975,14 +6099,6 @@ Perl_yylex(pTHX)
            PREREF('$');
        }
 
-       /* This kludge not intended to be bulletproof. */
-       if (PL_tokenbuf[1] == '[' && !PL_tokenbuf[2]) {
-           pl_yylval.opval = newSVOP(OP_CONST, 0,
-                                  newSViv(CopARYBASE_get(&PL_compiling)));
-           pl_yylval.opval->op_private = OPpCONST_ARYBASE;
-           TERM(THING);
-       }
-
        d = s;
        {
            const char tmp = *s;
@@ -6298,7 +6414,8 @@ Perl_yylex(pTHX)
            else if (!isALPHA(*start) && (PL_expect == XTERM
                        || PL_expect == XREF || PL_expect == XSTATE
                        || PL_expect == XTERMORDORDOR)) {
-               GV *const gv = gv_fetchpvn_flags(s, start - s, 0, SVt_PVCV);
+               GV *const gv = gv_fetchpvn_flags(s, start - s,
+                                                    UTF ? SVf_UTF8 : 0, SVt_PVCV);
                if (!gv) {
                    s = scan_num(s, &pl_yylval);
                    TERM(THING);
@@ -6417,7 +6534,8 @@ Perl_yylex(pTHX)
            GV *hgv = NULL;     /* hidden (loser) */
            if (PL_expect != XOPERATOR && (*s != ':' || s[1] != ':')) {
                CV *cv;
-               if ((gv = gv_fetchpvn_flags(PL_tokenbuf, len, 0, SVt_PVCV)) &&
+               if ((gv = gv_fetchpvn_flags(PL_tokenbuf, len,
+                                            UTF ? SVf_UTF8 : 0, SVt_PVCV)) &&
                    (cv = GvCVu(gv)))
                {
                    if (GvIMPORTED_CV(gv))
@@ -6426,7 +6544,8 @@ Perl_yylex(pTHX)
                        hgv = gv;
                }
                if (!ogv &&
-                   (gvp = (GV**)hv_fetch(PL_globalstash,PL_tokenbuf,len,FALSE)) &&
+                   (gvp = (GV**)hv_fetch(PL_globalstash, PL_tokenbuf,
+                                            UTF ? -(I32)len : (I32)len, FALSE)) &&
                    (gv = *gvp) && isGV_with_GP(gv) &&
                    GvCVu(gv) && GvIMPORTED_CV(gv))
                {
@@ -6515,7 +6634,7 @@ Perl_yylex(pTHX)
                    PL_tokenbuf[len - 2] == ':' && PL_tokenbuf[len - 1] == ':')
                {
                    if (ckWARN(WARN_BAREWORD)
-                       && ! gv_fetchpvn_flags(PL_tokenbuf, len, 0, SVt_PVHV))
+                       && ! gv_fetchpvn_flags(PL_tokenbuf, len, UTF ? SVf_UTF8 : 0, SVt_PVHV))
                        Perl_warner(aTHX_ packWARN(WARN_BAREWORD),
                            "Bareword \"%s\" refers to nonexistent package",
                             PL_tokenbuf);
@@ -6531,7 +6650,8 @@ Perl_yylex(pTHX)
                           constants that might already be there into full
                           blown PVGVs with attached PVCV.  */
                        gv = gv_fetchpvn_flags(PL_tokenbuf, len,
-                                              GV_NOADD_NOINIT, SVt_PVCV);
+                                              GV_NOADD_NOINIT | ( UTF ? SVf_UTF8 : 0 ),
+                                              SVt_PVCV);
                    }
                    len = 0;
                }
@@ -6565,7 +6685,7 @@ Perl_yylex(pTHX)
                    goto safe_bareword;
 
                {
-                   OP *const_op = newSVOP(OP_CONST, 0, SvREFCNT_inc(sv));
+                   OP *const_op = newSVOP(OP_CONST, 0, SvREFCNT_inc_NN(sv));
                    const_op->op_private = OPpCONST_BARE;
                    rv2cv_op = newCVREF(0, const_op);
                }
@@ -6725,8 +6845,8 @@ Perl_yylex(pTHX)
 #endif
                        SvPOK(cv))
                    {
-                       STRLEN protolen;
-                       const char *proto = SvPV_const(MUTABLE_SV(cv), protolen);
+                       STRLEN protolen = CvPROTOLEN(cv);
+                       const char *proto = CvPROTO(cv);
                        if (!protolen)
                            TERM(FUNC0SUB);
                        while (*proto == ';')
@@ -6803,7 +6923,8 @@ Perl_yylex(pTHX)
                        }
                    }
                    if (probable_sub) {
-                       gv = gv_fetchpv(PL_tokenbuf, GV_ADD, SVt_PVCV);
+                       gv = gv_fetchpv(PL_tokenbuf, GV_ADD | ( UTF ? SVf_UTF8 : 0 ),
+                                        SVt_PVCV);
                        op_free(pl_yylval.opval);
                        pl_yylval.opval = rv2cv_op;
                        pl_yylval.opval->op_private |= OPpENTERSUB_NOPAREN;
@@ -6857,7 +6978,7 @@ Perl_yylex(pTHX)
                            d = PL_tokenbuf;
                            while (isLOWER(*d))
                                d++;
-                           if (!*d && !gv_stashpv(PL_tokenbuf, 0))
+                           if (!*d && !gv_stashpv(PL_tokenbuf, UTF ? SVf_UTF8 : 0))
                                Perl_warner(aTHX_ packWARN(WARN_RESERVED), PL_warn_reserved,
                                       PL_tokenbuf);
                        }
@@ -6878,31 +6999,43 @@ Perl_yylex(pTHX)
            }
 
        case KEY___FILE__:
-           pl_yylval.opval = (OP*)newSVOP(OP_CONST, 0,
-                                       newSVpv(CopFILE(PL_curcop),0));
-           TERM(THING);
+           FUN0OP(
+               (OP*)newSVOP(OP_CONST, 0, newSVpv(CopFILE(PL_curcop),0))
+           );
 
        case KEY___LINE__:
-            pl_yylval.opval = (OP*)newSVOP(OP_CONST, 0,
-                                    Perl_newSVpvf(aTHX_ "%"IVdf, (IV)CopLINE(PL_curcop)));
-           TERM(THING);
+           FUN0OP(
+               (OP*)newSVOP(OP_CONST, 0,
+                   Perl_newSVpvf(aTHX_ "%"IVdf, (IV)CopLINE(PL_curcop)))
+           );
 
        case KEY___PACKAGE__:
-           pl_yylval.opval = (OP*)newSVOP(OP_CONST, 0,
+           FUN0OP(
+               (OP*)newSVOP(OP_CONST, 0,
                                        (PL_curstash
                                         ? newSVhek(HvNAME_HEK(PL_curstash))
-                                        : &PL_sv_undef));
-           TERM(THING);
+                                        : &PL_sv_undef))
+           );
 
        case KEY___DATA__:
        case KEY___END__: {
            GV *gv;
            if (PL_rsfp && (!PL_in_eval || PL_tokenbuf[2] == 'D')) {
                const char *pname = "main";
+               STRLEN plen = 4;
+               U32 putf8 = 0;
                if (PL_tokenbuf[2] == 'D')
-                   pname = HvNAME_get(PL_curstash ? PL_curstash : PL_defstash);
-               gv = gv_fetchpv(Perl_form(aTHX_ "%s::DATA", pname), GV_ADD,
-                               SVt_PVIO);
+               {
+                   HV * const stash =
+                       PL_curstash ? PL_curstash : PL_defstash;
+                   pname = HvNAME_get(stash);
+                   plen  = HvNAMELEN (stash);
+                   if(HvNAMEUTF8(stash)) putf8 = SVf_UTF8;
+               }
+               gv = gv_fetchpvn_flags(
+                       Perl_form(aTHX_ "%*s::DATA", (int)plen, pname),
+                       plen+6, GV_ADD|putf8, SVt_PVIO
+               );
                GvMULTI_on(gv);
                if (!GvIO(gv))
                    GvIOp(gv) = newIO();
@@ -6936,12 +7069,6 @@ Perl_yylex(pTHX)
 #else
                    if (PerlLIO_setmode(PerlIO_fileno(PL_rsfp), O_TEXT) != -1) {
 #endif /* NETWARE */
-#ifdef PERLIO_IS_STDIO /* really? */
-#  if defined(__BORLANDC__)
-                       /* XXX see note in do_binmode() */
-                       ((FILE*)PL_rsfp)->flags &= ~_F_BIN;
-#  endif
-#endif
                        if (loc > 0)
                            PerlIO_seek(PL_rsfp, loc, 0);
                    }
@@ -6992,6 +7119,9 @@ Perl_yylex(pTHX)
            goto fake_eof;
        }
 
+       case KEY___SUB__:
+           FUN0(OP_RUNCV);
+
        case KEY_AUTOLOAD:
        case KEY_DESTROY:
        case KEY_BEGIN:
@@ -7010,11 +7140,12 @@ Perl_yylex(pTHX)
                s += 2;
                d = s;
                s = scan_word(s, PL_tokenbuf, sizeof PL_tokenbuf, FALSE, &len);
-               if (!(tmp = keyword(PL_tokenbuf, len, 0)))
+               if (!(tmp = keyword(PL_tokenbuf, len, 1)))
                    Perl_croak(aTHX_ "CORE::%s is not a keyword", PL_tokenbuf);
                if (tmp < 0)
                    tmp = -tmp;
-               else if (tmp == KEY_require || tmp == KEY_do)
+               else if (tmp == KEY_require || tmp == KEY_do
+                     || tmp == KEY_glob)
                    /* that's a way to remember we saw "CORE::" */
                    orig_keyword = tmp;
                goto reserved_word;
@@ -7054,12 +7185,6 @@ Perl_yylex(pTHX)
            UNI(OP_CHOP);
 
        case KEY_continue:
-           /* When 'use switch' is in effect, continue has a dual
-              life as a control operator. */
-           {
-               if (!FEATURE_IS_ENABLED("switch"))
-                   PREBLOCK(CONTINUE);
-               else {
                    /* We have to disambiguate the two senses of
                      "continue". If the next token is a '{' then
                      treat it as the start of a continue block;
@@ -7070,8 +7195,6 @@ Perl_yylex(pTHX)
            PREBLOCK(CONTINUE);
                    else
                        FUN0(OP_CONTINUE);
-               }
-           }
 
        case KEY_chdir:
            /* may use HOME */
@@ -7194,6 +7317,10 @@ Perl_yylex(pTHX)
                UNIBRACK(OP_ENTEREVAL);
            }
 
+       case KEY_evalbytes:
+           PL_expect = XTERM;
+           UNIBRACK(-OP_ENTEREVAL);
+
        case KEY_eof:
            UNI(OP_EOF);
 
@@ -7374,7 +7501,10 @@ Perl_yylex(pTHX)
            OPERATOR(GIVEN);
 
        case KEY_glob:
-           LOP(OP_GLOB,XTERM);
+           LOP(
+            orig_keyword==KEY_glob ? (orig_keyword=0, -OP_GLOB) : OP_GLOB,
+            XTERM
+           );
 
        case KEY_hex:
            UNI(OP_HEX);
@@ -7608,7 +7738,8 @@ Perl_yylex(pTHX)
                missingterm(NULL);
            PL_expect = XOPERATOR;
            if (SvCUR(PL_lex_stuff)) {
-               int warned = 0;
+               int warned_comma = !ckWARN(WARN_QW);
+               int warned_comment = warned_comma;
                d = SvPV_force(PL_lex_stuff, len);
                while (len) {
                    for (; isSPACE(*d) && len; --len, ++d)
@@ -7616,17 +7747,17 @@ Perl_yylex(pTHX)
                    if (len) {
                        SV *sv;
                        const char *b = d;
-                       if (!warned && ckWARN(WARN_QW)) {
+                       if (!warned_comma || !warned_comment) {
                            for (; !isSPACE(*d) && len; --len, ++d) {
-                               if (*d == ',') {
+                               if (!warned_comma && *d == ',') {
                                    Perl_warner(aTHX_ packWARN(WARN_QW),
                                        "Possible attempt to separate words with commas");
-                                   ++warned;
+                                   ++warned_comma;
                                }
-                               else if (*d == '#') {
+                               else if (!warned_comment && *d == '#') {
                                    Perl_warner(aTHX_ packWARN(WARN_QW),
                                        "Possible attempt to put comments in qw() list");
-                                   ++warned;
+                                   ++warned_comment;
                                }
                            }
                        }
@@ -7685,7 +7816,8 @@ Perl_yylex(pTHX)
                *PL_tokenbuf = '\0';
                s = force_word(s,WORD,TRUE,TRUE,FALSE);
                if (isIDFIRST_lazy_if(PL_tokenbuf,UTF))
-                   gv_stashpvn(PL_tokenbuf, strlen(PL_tokenbuf), GV_ADD);
+                   gv_stashpvn(PL_tokenbuf, strlen(PL_tokenbuf),
+                                GV_ADD | (UTF ? SVf_UTF8 : 0));
                else if (*s == '<')
                    yyerror("<> should be quotes");
            }
@@ -7890,7 +8022,7 @@ Perl_yylex(pTHX)
                SV *tmpwhite = 0;
 
                char *tstart = SvPVX(PL_linestr) + PL_realtokenstart;
-               SV *subtoken = newSVpvn(tstart, s - tstart);
+               SV *subtoken = newSVpvn_flags(tstart, s - tstart, SvUTF8(PL_linestr));
                PL_thistoken = 0;
 
                d = s;
@@ -7913,7 +8045,7 @@ Perl_yylex(pTHX)
                    d = scan_word(s, tmpbuf, sizeof tmpbuf, TRUE, &len);
 #ifdef PERL_MAD
                    if (PL_madskills)
-                       nametoke = newSVpvn(s, d - s);
+                       nametoke = newSVpvn_flags(s, d - s, SvUTF8(PL_linestr));
 #endif
                    if (memchr(tmpbuf, ':', len))
                        sv_setpvn(PL_subname, tmpbuf, len);
@@ -7922,6 +8054,8 @@ Perl_yylex(pTHX)
                        sv_catpvs(PL_subname,"::");
                        sv_catpvn(PL_subname,tmpbuf,len);
                    }
+                    if (SvUTF8(PL_linestr))
+                        SvUTF8_on(PL_subname);
                    have_name = TRUE;
 
 #ifdef PERL_MAD
@@ -7971,21 +8105,22 @@ Perl_yylex(pTHX)
                    bool underscore = FALSE;
                    bool seen_underscore = FALSE;
                    const bool warnillegalproto = ckWARN(WARN_ILLEGALPROTO);
+                    STRLEN tmplen;
 
                    s = scan_str(s,!!PL_madskills,FALSE);
                    if (!s)
                        Perl_croak(aTHX_ "Prototype not terminated");
                    /* strip spaces and check for bad characters */
-                   d = SvPVX(PL_lex_stuff);
+                   d = SvPV(PL_lex_stuff, tmplen);
                    tmp = 0;
-                   for (p = d; *p; ++p) {
+                   for (p = d; tmplen; tmplen--, ++p) {
                        if (!isSPACE(*p)) {
-                           d[tmp++] = *p;
+                            d[tmp++] = *p;
 
                            if (warnillegalproto) {
                                if (must_be_last)
                                    proto_after_greedy_proto = TRUE;
-                               if (!strchr("$@%*;[]&\\_+", *p)) {
+                               if (!strchr("$@%*;[]&\\_+", *p) || *p == '\0') {
                                    bad_proto = TRUE;
                                }
                                else {
@@ -8013,17 +8148,22 @@ Perl_yylex(pTHX)
                            }
                        }
                    }
-                   d[tmp] = '\0';
+                    d[tmp] = '\0';
                    if (proto_after_greedy_proto)
                        Perl_warner(aTHX_ packWARN(WARN_ILLEGALPROTO),
                                    "Prototype after '%c' for %"SVf" : %s",
                                    greedy_proto, SVfARG(PL_subname), d);
-                   if (bad_proto)
+                   if (bad_proto) {
+                        SV *dsv = newSVpvs_flags("", SVs_TEMP);
                        Perl_warner(aTHX_ packWARN(WARN_ILLEGALPROTO),
                                    "Illegal character %sin prototype for %"SVf" : %s",
                                    seen_underscore ? "after '_' " : "",
-                                   SVfARG(PL_subname), d);
-                   SvCUR_set(PL_lex_stuff, tmp);
+                                   SVfARG(PL_subname),
+                                    sv_uni_display(dsv,
+                                         newSVpvn_flags(d, tmp, SVs_TEMP | SvUTF8(PL_lex_stuff)),
+                                         tmp, UNI_DISPLAY_ISPRINT));
+                    }
+                    SvCUR_set(PL_lex_stuff, tmp);
                    have_proto = TRUE;
 
 #ifdef PERL_MAD
@@ -8277,7 +8417,7 @@ S_pending_ident(pTHX)
                 yyerror(Perl_form(aTHX_ "No package name allowed for "
                                   "variable %s in \"our\"",
                                   PL_tokenbuf));
-            tmp = allocmy(PL_tokenbuf, tokenbuf_len, 0);
+            tmp = allocmy(PL_tokenbuf, tokenbuf_len, UTF ? SVf_UTF8 : 0);
         }
         else {
             if (has_colon)
@@ -8285,7 +8425,8 @@ S_pending_ident(pTHX)
                            PL_in_my == KEY_my ? "my" : "state", PL_tokenbuf));
 
             pl_yylval.opval = newOP(OP_PADANY, 0);
-            pl_yylval.opval->op_targ = allocmy(PL_tokenbuf, tokenbuf_len, 0);
+            pl_yylval.opval->op_targ = allocmy(PL_tokenbuf, tokenbuf_len,
+                                                        UTF ? SVf_UTF8 : 0);
             return PRIVATEREF;
         }
     }
@@ -8304,7 +8445,8 @@ S_pending_ident(pTHX)
 
     if (!has_colon) {
        if (!PL_in_my)
-           tmp = pad_findmy(PL_tokenbuf, tokenbuf_len, 0);
+           tmp = pad_findmy_pvn(PL_tokenbuf, tokenbuf_len,
+                                    UTF ? SVf_UTF8 : 0);
         if (tmp != NOT_IN_PAD) {
             /* might be an "our" variable" */
             if (PAD_COMPNAME_FLAGS_isOUR(tmp)) {
@@ -8313,7 +8455,7 @@ S_pending_ident(pTHX)
                HEK * const stashname = HvNAME_HEK(stash);
                SV *  const sym = newSVhek(stashname);
                 sv_catpvs(sym, "::");
-                sv_catpvn(sym, PL_tokenbuf+1, tokenbuf_len - 1);
+                sv_catpvn_flags(sym, PL_tokenbuf+1, tokenbuf_len - 1, (UTF ? SV_CATUTF8 : SV_CATBYTES ));
                 pl_yylval.opval = (OP*)newSVOP(OP_CONST, 0, sym);
                 pl_yylval.opval->op_private = OPpCONST_ENTERED;
                 gv_fetchsv(sym,
@@ -8357,8 +8499,8 @@ S_pending_ident(pTHX)
     */
     if (ckWARN(WARN_AMBIGUOUS) &&
        pit == '@' && PL_lex_state != LEX_NORMAL && !PL_lex_brackets) {
-        GV *const gv = gv_fetchpvn_flags(PL_tokenbuf + 1, tokenbuf_len - 1, 0,
-                                        SVt_PVAV);
+        GV *const gv = gv_fetchpvn_flags(PL_tokenbuf + 1, tokenbuf_len - 1,
+                                        ( UTF ? SVf_UTF8 : 0 ), SVt_PVAV);
         if ((!gv || ((PL_tokenbuf[0] == '@') ? !GvAV(gv) : !GvHV(gv)))
                /* DO NOT warn for @- and @+ */
                && !( PL_tokenbuf[2] == '\0' &&
@@ -8373,11 +8515,13 @@ S_pending_ident(pTHX)
     }
 
     /* build ops for a bareword */
-    pl_yylval.opval = (OP*)newSVOP(OP_CONST, 0, newSVpvn(PL_tokenbuf + 1,
-                                                     tokenbuf_len - 1));
+    pl_yylval.opval = (OP*)newSVOP(OP_CONST, 0, newSVpvn_flags(PL_tokenbuf + 1,
+                                                     tokenbuf_len - 1,
+                                                      UTF ? SVf_UTF8 : 0 ));
     pl_yylval.opval->op_private = OPpCONST_ENTERED;
     gv_fetchpvn_flags(PL_tokenbuf+1, tokenbuf_len - 1,
-                    PL_in_eval ? (GV_ADDMULTI | GV_ADDINEVAL) : GV_ADD,
+                    (PL_in_eval ? (GV_ADDMULTI | GV_ADDINEVAL) : GV_ADD)
+                     | ( UTF ? SVf_UTF8 : 0 ),
                     ((PL_tokenbuf[0] == '$') ? SVt_PV
                      : (PL_tokenbuf[0] == '@') ? SVt_PVAV
                      : SVt_PVHV));
@@ -8429,7 +8573,7 @@ S_checkcomma(pTHX_ const char *s, const char *name, const char *what)
            if (keyword(w, s - w, 0))
                return;
 
-           gv = gv_fetchpvn_flags(w, s - w, 0, SVt_PVCV);
+           gv = gv_fetchpvn_flags(w, s - w, ( UTF ? SVf_UTF8 : 0 ), SVt_PVCV);
            if (gv && GvCVu(gv))
                return;
            Perl_croak(aTHX_ "No comma allowed after %s", what);
@@ -8658,9 +8802,19 @@ S_scan_ident(pTHX_ register char *s, register const char *send, char *dest, STRL
     }
     else if (ck_uni)
        check_uni();
-    if (s < send)
-       *d = *s++;
-    d[1] = '\0';
+    if (s < send) {
+        if (UTF) {
+            const STRLEN skip = UTF8SKIP(s);
+            STRLEN i;
+            d[skip] = '\0';
+            for ( i = 0; i < skip; i++ )
+                d[i] = *s++;
+        }
+        else {
+            *d = *s++;
+            d[1] = '\0';
+        }
+    }
     if (*d == '^' && *s && isCONTROLVAR(*s)) {
        *d = toCTRL(*s);
        s++;
@@ -8676,7 +8830,7 @@ S_scan_ident(pTHX_ register char *s, register const char *send, char *dest, STRL
            }
        }
        if (isIDFIRST_lazy_if(d,UTF)) {
-           d++;
+           d += UTF8SKIP(d);
            if (UTF) {
                char *end = s;
                while ((end < send && isALNUM_lazy_if(end,UTF)) || *end == ':') {
@@ -8702,6 +8856,7 @@ S_scan_ident(pTHX_ register char *s, register const char *send, char *dest, STRL
                    const char * const brack =
                        (const char *)
                        ((*s == '[') ? "[...]" : "{...}");
+   /* diag_listed_as: Ambiguous use of %c{%s[...]} resolved to %c%s[...] */
                    Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                        "Ambiguous use of %c{%s%s} resolved to %c%s%s",
                        funny, dest, brack, funny, dest, brack);
@@ -8754,17 +8909,138 @@ S_scan_ident(pTHX_ register char *s, register const char *send, char *dest, STRL
     return s;
 }
 
-static U32
-S_pmflag(U32 pmfl, const char ch) {
-    switch (ch) {
-       CASE_STD_PMMOD_FLAGS_PARSE_SET(&pmfl);
-    case GLOBAL_PAT_MOD:      pmfl |= PMf_GLOBAL; break;
-    case CONTINUE_PAT_MOD:    pmfl |= PMf_CONTINUE; break;
-    case ONCE_PAT_MOD:        pmfl |= PMf_KEEP; break;
-    case KEEPCOPY_PAT_MOD:    pmfl |= RXf_PMf_KEEPCOPY; break;
-    case NONDESTRUCT_PAT_MOD: pmfl |= PMf_NONDESTRUCT; break;
-    }
-    return pmfl;
+static bool
+S_pmflag(pTHX_ const char* const valid_flags, U32 * pmfl, char** s, char* charset) {
+
+    /* Adds, subtracts to/from 'pmfl' based on regex modifier flags found in
+     * the parse starting at 's', based on the subset that are valid in this
+     * context input to this routine in 'valid_flags'. Advances s.  Returns
+     * TRUE if the input was a valid flag, so the next char may be as well;
+     * otherwise FALSE. 'charset' should point to a NUL upon first call on the
+     * current regex.  This routine will set it to any charset modifier found.
+     * The caller shouldn't change it.  This way, another charset modifier
+     * encountered in the parse can be detected as an error, as we have decided
+     * allow only one */
+
+    const char c = **s;
+
+    if (! strchr(valid_flags, c)) {
+        if (isALNUM(c)) {
+           goto deprecate;
+        }
+        return FALSE;
+    }
+
+    switch (c) {
+
+        CASE_STD_PMMOD_FLAGS_PARSE_SET(pmfl);
+        case GLOBAL_PAT_MOD:      *pmfl |= PMf_GLOBAL; break;
+        case CONTINUE_PAT_MOD:    *pmfl |= PMf_CONTINUE; break;
+        case ONCE_PAT_MOD:        *pmfl |= PMf_KEEP; break;
+        case KEEPCOPY_PAT_MOD:    *pmfl |= RXf_PMf_KEEPCOPY; break;
+        case NONDESTRUCT_PAT_MOD: *pmfl |= PMf_NONDESTRUCT; break;
+       case LOCALE_PAT_MOD:
+
+           /* In 5.14, qr//lt is legal but deprecated; the 't' means they
+            * can't be regex modifiers.
+            * In 5.14, s///le is legal and ambiguous.  Try to disambiguate as
+            * much as easily done.  s///lei, for example, has to mean regex
+            * modifiers if it's not an error (as does any word character
+            * following the 'e').  Otherwise, we resolve to the backwards-
+            * compatible, but less likely 's/// le ...', i.e. as meaning
+            * less-than-or-equal.  The reason it's not likely is that s//
+            * returns a number for code in the field (/r returns a string, but
+            * that wasn't added until the 5.13 series), and so '<=' should be
+            * used for comparing, not 'le'. */
+           if (*((*s) + 1) == 't') {
+               goto deprecate;
+           }
+           else if (*((*s) + 1) == 'e' && ! isALNUM(*((*s) + 2))) {
+
+               /* 'e' is valid only for substitutes, s///e.  If it is not
+                * valid in the current context, then 'm//le' must mean the
+                * comparison operator, so use the regular deprecation message.
+                */
+               if (! strchr(valid_flags, 'e')) {
+                   goto deprecate;
+               }
+               Perl_ck_warner_d(aTHX_ packWARN(WARN_AMBIGUOUS),
+                   "Ambiguous use of 's//le...' resolved as 's// le...'; Rewrite as 's//el' if you meant 'use locale rules and evaluate rhs as an expression'.  In Perl 5.16, it will be resolved the other way");
+               return FALSE;
+           }
+           if (*charset) {
+               goto multiple_charsets;
+           }
+           set_regex_charset(pmfl, REGEX_LOCALE_CHARSET);
+           *charset = c;
+           break;
+       case UNICODE_PAT_MOD:
+           /* In 5.14, qr//unless and qr//until are legal but deprecated; the
+            * 'n' means they can't be regex modifiers */
+           if (*((*s) + 1) == 'n') {
+               goto deprecate;
+           }
+           if (*charset) {
+               goto multiple_charsets;
+           }
+           set_regex_charset(pmfl, REGEX_UNICODE_CHARSET);
+           *charset = c;
+           break;
+       case ASCII_RESTRICT_PAT_MOD:
+           /* In 5.14, qr//and is legal but deprecated; the 'n' means they
+            * can't be regex modifiers */
+           if (*((*s) + 1) == 'n') {
+               goto deprecate;
+           }
+
+           if (! *charset) {
+               set_regex_charset(pmfl, REGEX_ASCII_RESTRICTED_CHARSET);
+           }
+           else {
+
+               /* Error if previous modifier wasn't an 'a', but if it was, see
+                * if, and accept, a second occurrence (only) */
+               if (*charset != 'a'
+                   || get_regex_charset(*pmfl)
+                       != REGEX_ASCII_RESTRICTED_CHARSET)
+               {
+                       goto multiple_charsets;
+               }
+               set_regex_charset(pmfl, REGEX_ASCII_MORE_RESTRICTED_CHARSET);
+           }
+           *charset = c;
+           break;
+       case DEPENDS_PAT_MOD:
+           if (*charset) {
+               goto multiple_charsets;
+           }
+           set_regex_charset(pmfl, REGEX_DEPENDS_CHARSET);
+           *charset = c;
+           break;
+    }
+
+    (*s)++;
+    return TRUE;
+
+    deprecate:
+       Perl_ck_warner_d(aTHX_ packWARN(WARN_SYNTAX),
+           "Having no space between pattern and following word is deprecated");
+        return FALSE;
+
+    multiple_charsets:
+       if (*charset != c) {
+           yyerror(Perl_form(aTHX_ "Regexp modifiers \"/%c\" and \"/%c\" are mutually exclusive", *charset, c));
+       }
+       else if (c == 'a') {
+           yyerror("Regexp modifier \"/a\" may appear a maximum of twice");
+       }
+       else {
+           yyerror(Perl_form(aTHX_ "Regexp modifier \"/%c\" may not appear twice", c));
+       }
+
+       /* Pretend that it worked, so will continue processing before dieing */
+       (*s)++;
+       return TRUE;
 }
 
 STATIC char *
@@ -8775,6 +9051,7 @@ S_scan_pat(pTHX_ char *start, I32 type)
     char *s = scan_str(start,!!PL_madskills,FALSE);
     const char * const valid_flags =
        (const char *)((type == OP_QR) ? QR_PAT_MODS : M_PAT_MODS);
+    char charset = '\0';    /* character set modifier */
 #ifdef PERL_MAD
     char *modstart;
 #endif
@@ -8816,14 +9093,7 @@ S_scan_pat(pTHX_ char *start, I32 type)
 #ifdef PERL_MAD
     modstart = s;
 #endif
-    while (*s && strchr(valid_flags, *s))
-       pm->op_pmflags = S_pmflag(pm->op_pmflags, *s++);
-
-    if (isALNUM(*s)) {
-       Perl_ck_warner_d(aTHX_ packWARN(WARN_SYNTAX),
-           "Having no space between pattern and following word is deprecated");
-
-    }
+    while (*s && S_pmflag(aTHX_ valid_flags, &(pm->op_pmflags), &s, &charset)) {};
 #ifdef PERL_MAD
     if (PL_madskills && modstart != s) {
        SV* tmptoken = newSVpvn(modstart, s - modstart);
@@ -8846,10 +9116,11 @@ STATIC char *
 S_scan_subst(pTHX_ char *start)
 {
     dVAR;
-    register char *s;
+    char *s;
     register PMOP *pm;
     I32 first_start;
     I32 es = 0;
+    char charset = '\0';    /* character set modifier */
 #ifdef PERL_MAD
     char *modstart;
 #endif
@@ -8902,14 +9173,8 @@ S_scan_subst(pTHX_ char *start)
            s++;
            es++;
        }
-       else if (strchr(S_PAT_MODS, *s))
-           pm->op_pmflags = S_pmflag(pm->op_pmflags, *s++);
-       else {
-           if (isALNUM(*s)) {
-               Perl_ck_warner_d(aTHX_ packWARN(WARN_SYNTAX),
-                   "Having no space between pattern and following word is deprecated");
-
-           }
+       else if (! S_pmflag(aTHX_ S_PAT_MODS, &(pm->op_pmflags), &s, &charset))
+       {
            break;
        }
     }
@@ -9063,7 +9328,8 @@ S_scan_heredoc(pTHX_ register char *s)
     register char *d;
     register char *e;
     char *peek;
-    const int outer = (PL_rsfp && !(PL_lex_inwhat == OP_SCALAR));
+    const int outer = (PL_rsfp || PL_parser->filtered)
+                  && !(PL_lex_inwhat == OP_SCALAR);
 #ifdef PERL_MAD
     I32 stuffstart = s - SvPVX(PL_linestr);
     char *tstart;
@@ -9187,7 +9453,8 @@ S_scan_heredoc(pTHX_ register char *s)
     PL_multi_start = CopLINE(PL_curcop);
     PL_multi_open = PL_multi_close = '<';
     term = *PL_tokenbuf;
-    if (PL_lex_inwhat == OP_SUBST && PL_in_eval && !PL_rsfp) {
+    if (PL_lex_inwhat == OP_SUBST && PL_in_eval && !PL_rsfp
+     && !PL_parser->filtered) {
        char * const bufptr = PL_sublex_info.super_bufptr;
        char * const bufend = PL_sublex_info.super_bufend;
        char * const olds = s - SvCUR(herewas);
@@ -9287,6 +9554,7 @@ S_scan_heredoc(pTHX_ register char *s)
        if (*s == term && memEQ(s,PL_tokenbuf,len)) {
            STRLEN off = PL_bufend - 1 - SvPVX_const(PL_linestr);
            *(SvPVX(PL_linestr) + off ) = ' ';
+           lex_grow_linestr(SvCUR(PL_linestr) + SvCUR(herewas) + 1);
            sv_catsv(PL_linestr,herewas);
            PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
            s = SvPVX(PL_linestr) + off; /* In case PV of PL_linestr moved. */
@@ -9369,7 +9637,7 @@ S_scan_inputsymbol(pTHX_ char *start)
 
     /* allow <Pkg'VALUE> or <Pkg::VALUE> */
     while (*d && (isALNUM_lazy_if(d,UTF) || *d == '\'' || *d == ':'))
-       d++;
+       d += UTF ? UTF8SKIP(d) : 1;
 
     /* If we've tried to read what we allow filehandles to look like, and
        there's still text left, then it must be a glob() and not a getline.
@@ -9412,7 +9680,7 @@ S_scan_inputsymbol(pTHX_ char *start)
            /* try to find it in the pad for this block, otherwise find
               add symbol table ops
            */
-           const PADOFFSET tmp = pad_findmy(d, len, 0);
+           const PADOFFSET tmp = pad_findmy_pvn(d, len, UTF ? SVf_UTF8 : 0);
            if (tmp != NOT_IN_PAD) {
                if (PAD_COMPNAME_FLAGS_isOUR(tmp)) {
                    HV * const stash = PAD_COMPNAME_OURSTASH(tmp);
@@ -9440,7 +9708,7 @@ intro_sym:
                gv = gv_fetchpv(d,
                                (PL_in_eval
                                 ? (GV_ADDMULTI | GV_ADDINEVAL)
-                                : GV_ADDMULTI),
+                                : GV_ADDMULTI) | ( UTF ? SVf_UTF8 : 0 ),
                                SVt_PV);
                PL_lex_op = readline_overriden
                    ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
@@ -9460,7 +9728,7 @@ intro_sym:
        /* If it's none of the above, it must be a literal filehandle
           (<Foo::BAR> or <FOO>) so build a simple readline OP */
        else {
-           GV * const gv = gv_fetchpv(d, GV_ADD, SVt_PVIO);
+           GV * const gv = gv_fetchpv(d, GV_ADD | ( UTF ? SVf_UTF8 : 0 ), SVt_PVIO);
            PL_lex_op = readline_overriden
                ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
                        op_append_elem(OP_LIST,
@@ -9609,7 +9877,7 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
                char * const svlast = SvEND(sv) - 1;
 
                for (; s < ns; s++) {
-                   if (*s == '\n' && !PL_rsfp)
+                   if (*s == '\n' && !PL_rsfp && !PL_parser->filtered)
                        CopLINE_inc(PL_curcop);
                }
                if (!found)
@@ -9676,7 +9944,7 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
        if (PL_multi_open == PL_multi_close) {
            for (; s < PL_bufend; s++,to++) {
                /* embedded newlines increment the current line number */
-               if (*s == '\n' && !PL_rsfp)
+               if (*s == '\n' && !PL_rsfp && !PL_parser->filtered)
                    CopLINE_inc(PL_curcop);
                /* handle quoted delimiters */
                if (*s == '\\' && s+1 < PL_bufend && term != '\\') {
@@ -9708,7 +9976,7 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
            /* read until we run out of string, or we find the terminator */
            for (; s < PL_bufend; s++,to++) {
                /* embedded newlines increment the line count */
-               if (*s == '\n' && !PL_rsfp)
+               if (*s == '\n' && !PL_rsfp && !PL_parser->filtered)
                    CopLINE_inc(PL_curcop);
                /* backslashes can escape the open or closing characters */
                if (*s == '\\' && s+1 < PL_bufend) {
@@ -10268,7 +10536,7 @@ S_scan_formline(pTHX_ register char *s)
                break;
             }
        }
-       if (PL_in_eval && !PL_rsfp) {
+       if (PL_in_eval && !PL_rsfp && !PL_parser->filtered) {
            eol = (char *) memchr(s,'\n',PL_bufend-s);
            if (!eol++)
                eol = PL_bufend;
@@ -10299,7 +10567,7 @@ S_scan_formline(pTHX_ register char *s)
              break;
        }
        s = (char*)eol;
-       if (PL_rsfp) {
+       if (PL_rsfp || PL_parser->filtered) {
            bool got_some;
 #ifdef PERL_MAD
            if (PL_madskills) {