This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
regen/mph.pl: Add comments
[perl5.git] / toke.c
diff --git a/toke.c b/toke.c
index 4f2aa57..3806b55 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -39,6 +39,7 @@ Individual members of C<PL_parser> have their own documentation.
 #define PERL_IN_TOKE_C
 #include "perl.h"
 #include "dquote_inline.h"
+#include "invlist_inline.h"
 
 #define new_constant(a,b,c,d,e,f,g)    \
        S_new_constant(aTHX_ a,b,STR_WITH_LEN(c),d,e,f, g)
@@ -310,6 +311,7 @@ static struct debug_tokens {
     { ANDAND,          TOKENTYPE_NONE,         "ANDAND" },
     { ANDOP,           TOKENTYPE_NONE,         "ANDOP" },
     { ANONSUB,         TOKENTYPE_IVAL,         "ANONSUB" },
+    { ANON_SIGSUB,     TOKENTYPE_IVAL,         "ANON_SIGSUB" },
     { ARROW,           TOKENTYPE_NONE,         "ARROW" },
     { ASSIGNOP,                TOKENTYPE_OPNUM,        "ASSIGNOP" },
     { BITANDOP,                TOKENTYPE_OPNUM,        "BITANDOP" },
@@ -367,6 +369,7 @@ static struct debug_tokens {
     { RELOP,           TOKENTYPE_OPNUM,        "RELOP" },
     { REQUIRE,         TOKENTYPE_NONE,         "REQUIRE" },
     { SHIFTOP,         TOKENTYPE_OPNUM,        "SHIFTOP" },
+    { SIGSUB,          TOKENTYPE_NONE,         "SIGSUB" },
     { SUB,             TOKENTYPE_NONE,         "SUB" },
     { THING,           TOKENTYPE_OPVAL,        "THING" },
     { UMINUS,          TOKENTYPE_NONE,         "UMINUS" },
@@ -2681,14 +2684,11 @@ S_get_and_check_backslash_N_name(pTHX_ const char* s, const char* const e)
             s += 2;
         }
         else {
-            if (! PL_utf8_charname_begin) {
-                U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST;
-                PL_utf8_charname_begin = _core_swash_init("utf8",
-                                                        "_Perl_Charname_Begin",
-                                                        &PL_sv_undef,
-                                                        1, 0, NULL, &flags);
-            }
-            if (! swash_fetch(PL_utf8_charname_begin, (U8 *) s, TRUE)) {
+            if (! _invlist_contains_cp(PL_utf8_charname_begin,
+                                       utf8_to_uvchr_buf((U8 *) s,
+                                                         (U8 *) e,
+                                                         NULL)))
+            {
                 goto bad_charname;
             }
             s += UTF8SKIP(s);
@@ -2712,14 +2712,11 @@ S_get_and_check_backslash_N_name(pTHX_ const char* s, const char* const e)
                 s += 2;
             }
             else {
-                if (! PL_utf8_charname_continue) {
-                    U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST;
-                    PL_utf8_charname_continue = _core_swash_init("utf8",
-                                                "_Perl_Charname_Continue",
-                                                &PL_sv_undef,
-                                                1, 0, NULL, &flags);
-                }
-                if (! swash_fetch(PL_utf8_charname_continue, (U8 *) s, TRUE)) {
+                if (! _invlist_contains_cp(PL_utf8_charname_continue,
+                                           utf8_to_uvchr_buf((U8 *) s,
+                                                             (U8 *) e,
+                                                             NULL)))
+                {
                     goto bad_charname;
                 }
                 s += UTF8SKIP(s);
@@ -5942,9 +5939,17 @@ Perl_yylex(pTHX)
        case XATTRTERM:
            PL_expect = XTERMBLOCK;
         grabattrs:
+            /* NB: as well as parsing normal attributes, we also end up
+             * here if there is something looking like attributes
+             * following a signature (which is illegal, but used to be
+             * legal in 5.20..5.26). If the latter, we still parse the
+             * attributes so that error messages(s) are less confusing,
+             * but ignore them (parser->sig_seen).
+             */
            s = skipspace(s);
            attrs = NULL;
             while (isIDFIRST_lazy_if_safe(s, PL_bufend, UTF)) {
+                bool sig = PL_parser->sig_seen;
                I32 tmp;
                SV *sv;
                d = scan_word(s, PL_tokenbuf, sizeof PL_tokenbuf, FALSE, &len);
@@ -5987,23 +5992,27 @@ Perl_yylex(pTHX)
                       the CVf_BUILTIN_ATTRS define in cv.h! */
                    if (!PL_in_my && memEQs(SvPVX(sv), len, "lvalue")) {
                        sv_free(sv);
-                       CvLVALUE_on(PL_compcv);
+                       if (!sig)
+                            CvLVALUE_on(PL_compcv);
                    }
                    else if (!PL_in_my && memEQs(SvPVX(sv), len, "method")) {
                        sv_free(sv);
-                       CvMETHOD_on(PL_compcv);
+                       if (!sig)
+                            CvMETHOD_on(PL_compcv);
                    }
                    else if (!PL_in_my && memEQs(SvPVX(sv), len, "const"))
                    {
                        sv_free(sv);
-                       Perl_ck_warner_d(aTHX_
-                           packWARN(WARN_EXPERIMENTAL__CONST_ATTR),
-                          ":const is experimental"
-                       );
-                       CvANONCONST_on(PL_compcv);
-                       if (!CvANON(PL_compcv))
-                           yyerror(":const is not permitted on named "
-                                   "subroutines");
+                        if (!sig) {
+                            Perl_ck_warner_d(aTHX_
+                                packWARN(WARN_EXPERIMENTAL__CONST_ATTR),
+                               ":const is experimental"
+                            );
+                            CvANONCONST_on(PL_compcv);
+                            if (!CvANON(PL_compcv))
+                                yyerror(":const is not permitted on named "
+                                        "subroutines");
+                        }
                    }
                    /* After we've set the flags, it could be argued that
                       we don't need to do the attributes.pm-based setting
@@ -6056,6 +6065,14 @@ Perl_yylex(pTHX)
                }
            }
        got_attrs:
+            if (PL_parser->sig_seen) {
+                /* see comment about about sig_seen and parser error
+                 * handling */
+                if (attrs)
+                    op_free(attrs);
+                Perl_croak(aTHX_ "Subroutine attributes must come "
+                                 "before the signature");
+                }
            if (attrs) {
                NEXTVAL_NEXTTOKE.opval = attrs;
                force_next(THING);
@@ -8658,22 +8675,24 @@ Perl_yylex(pTHX)
          really_sub:
            {
                char * const tmpbuf = PL_tokenbuf + 1;
-               expectation attrful;
                bool have_name, have_proto;
                const int key = tmp;
                 SV *format_name = NULL;
+                bool is_sigsub = FEATURE_SIGNATURES_IS_ENABLED;
 
                 SSize_t off = s-SvPVX(PL_linestr);
                s = skipspace(s);
                 d = SvPVX(PL_linestr)+off;
 
+                SAVEBOOL(PL_parser->sig_seen);
+                PL_parser->sig_seen = FALSE;
+
                 if (   isIDFIRST_lazy_if_safe(s, PL_bufend, UTF)
                     || *s == '\''
                     || (*s == ':' && s[1] == ':'))
                {
 
-                   PL_expect = XBLOCK;
-                   attrful = XATTRBLOCK;
+                   PL_expect = XATTRBLOCK;
                    d = scan_word(s, tmpbuf, sizeof PL_tokenbuf - 1, TRUE,
                                  &len);
                     if (key == KEY_format)
@@ -8704,8 +8723,7 @@ Perl_yylex(pTHX)
                        Perl_croak(aTHX_
                                  "Missing name in \"%s\"", PL_bufptr);
                    }
-                   PL_expect = XTERMBLOCK;
-                   attrful = XATTRTERM;
+                   PL_expect = XATTRTERM;
                    sv_setpvs(PL_subname,"?");
                    have_name = FALSE;
                }
@@ -8721,7 +8739,7 @@ Perl_yylex(pTHX)
                }
 
                /* Look for a prototype */
-               if (*s == '(' && !FEATURE_SIGNATURES_IS_ENABLED) {
+               if (*s == '(' && !is_sigsub) {
                    s = scan_str(s,FALSE,FALSE,FALSE,NULL);
                    COPLINE_SET_FROM_MULTI_END;
                    if (!s)
@@ -8735,9 +8753,9 @@ Perl_yylex(pTHX)
                else
                    have_proto = FALSE;
 
-               if (*s == ':' && s[1] != ':')
-                   PL_expect = attrful;
-               else if ((*s != '{' && *s != '(') && key != KEY_format) {
+               if (  !(*s == ':' && s[1] != ':')
+                    && (*s != '{' && *s != '(') && key != KEY_format)
+                {
                     assert(key == KEY_sub || key == KEY_AUTOLOAD ||
                            key == KEY_DESTROY || key == KEY_BEGIN ||
                            key == KEY_UNITCHECK || key == KEY_CHECK ||
@@ -8761,10 +8779,16 @@ Perl_yylex(pTHX)
                        sv_setpvs(PL_subname, "__ANON__");
                    else
                        sv_setpvs(PL_subname, "__ANON__::__ANON__");
-                   TOKEN(ANONSUB);
+                    if (is_sigsub)
+                        TOKEN(ANON_SIGSUB);
+                    else
+                        TOKEN(ANONSUB);
                }
                force_ident_maybe_lex('&');
-               TOKEN(SUB);
+                if (is_sigsub)
+                    TOKEN(SIGSUB);
+                else
+                    TOKEN(SUB);
            }
 
        case KEY_system:
@@ -9028,7 +9052,7 @@ S_pending_ident(pTHX)
                HEK * const stashname = HvNAME_HEK(stash);
                SV *  const sym = newSVhek(stashname);
                 sv_catpvs(sym, "::");
-                sv_catpvn_flags(sym, PL_tokenbuf+1, tokenbuf_len - 1, (UTF ? SV_CATUTF8 : SV_CATBYTES ));
+                sv_catpvn_flags(sym, PL_tokenbuf+1, tokenbuf_len > 0 ? tokenbuf_len - 1 : 0, (UTF ? SV_CATUTF8 : SV_CATBYTES ));
                 pl_yylval.opval = newSVOP(OP_CONST, 0, sym);
                 pl_yylval.opval->op_private = OPpCONST_ENTERED;
                 if (pit != '&')
@@ -9056,7 +9080,7 @@ S_pending_ident(pTHX)
         && PL_lex_state != LEX_NORMAL
         && !PL_lex_brackets)
     {
-        GV *const gv = gv_fetchpvn_flags(PL_tokenbuf + 1, tokenbuf_len - 1,
+        GV *const gv = gv_fetchpvn_flags(PL_tokenbuf + 1, tokenbuf_len > 0 ? tokenbuf_len - 1 : 0,
                                          ( UTF ? SVf_UTF8 : 0 ) | GV_ADDMG,
                                          SVt_PVAV);
         if ((!gv || ((PL_tokenbuf[0] == '@') ? !GvAV(gv) : !GvHV(gv)))
@@ -9073,11 +9097,11 @@ S_pending_ident(pTHX)
     /* build ops for a bareword */
     pl_yylval.opval = newSVOP(OP_CONST, 0,
                                   newSVpvn_flags(PL_tokenbuf + 1,
-                                                     tokenbuf_len - 1,
+                                                      tokenbuf_len > 0 ? tokenbuf_len - 1 : 0,
                                                       UTF ? SVf_UTF8 : 0 ));
     pl_yylval.opval->op_private = OPpCONST_ENTERED;
     if (pit != '&')
-       gv_fetchpvn_flags(PL_tokenbuf+1, tokenbuf_len - 1,
+        gv_fetchpvn_flags(PL_tokenbuf+1, tokenbuf_len > 0 ? tokenbuf_len - 1 : 0,
                     (PL_in_eval ? GV_ADDMULTI : GV_ADD)
                      | ( UTF ? SVf_UTF8 : 0 ),
                     ((PL_tokenbuf[0] == '$') ? SVt_PV
@@ -9855,7 +9879,7 @@ S_scan_subst(pTHX_ char *start)
          * the NVX field indicates how many src code lines the replacement
          * spreads over */
         sv_upgrade(PL_parser->lex_sub_repl, SVt_PVNV);
-        ((XPVNV*)SvANY(PL_parser->lex_sub_repl))->xnv_u.xnv_lines = 0;
+        ((XPVNV*)SvANY(PL_parser->lex_sub_repl))->xnv_u.xnv_lines = linediff;
         ((XPVIV*)SvANY(PL_parser->lex_sub_repl))->xiv_u.xivu_eval_seen =
                                                                     cBOOL(es);
     }
@@ -10589,14 +10613,11 @@ S_scan_str(pTHX_ char *start, int keep_bracketed_quoted, int keep_delims, int re
     const char * opening_delims = "([{<";
     const char * closing_delims = ")]}>";
 
+    /* The only non-UTF character that isn't a stand alone grapheme is
+     * white-space, hence can't be a delimiter. */
     const char * non_grapheme_msg = "Use of unassigned code point or"
                                     " non-standalone grapheme for a delimiter"
-                                    " will be a fatal error starting in Perl"
-                                    " 5.30";
-    /* The only non-UTF character that isn't a stand alone grapheme is
-     * white-space, hence can't be a delimiter.  So can skip for non-UTF-8 */
-    bool check_grapheme = UTF && ckWARN_d(WARN_DEPRECATED);
-
+                                    " is not allowed";
     PERL_ARGS_ASSERT_SCAN_STR;
 
     /* skip space before the delimiter */
@@ -10615,26 +10636,12 @@ S_scan_str(pTHX_ char *start, int keep_bracketed_quoted, int keep_delims, int re
     }
     else {
        termcode = utf8_to_uvchr_buf((U8*)s, (U8*)PL_bufend, &termlen);
-        if (check_grapheme) {
-            if (   UNLIKELY(UNICODE_IS_SUPER(termcode))
-                || UNLIKELY(UNICODE_IS_NONCHAR(termcode)))
-            {
-                /* These are considered graphemes, and since the ending
-                 * delimiter will be the same, we don't have to check the other
-                 * end */
-                check_grapheme = FALSE;
-            }
-            else if (UNLIKELY(! _is_grapheme((U8 *) start,
-                                             (U8 *) s,
-                                             (U8 *) PL_bufend,
-                                             termcode)))
-            {
-                Perl_warner(aTHX_ packWARN(WARN_DEPRECATED), "%s", non_grapheme_msg);
-
-                /* Don't have to check the other end, as have already warned at
-                 * this one */
-                check_grapheme = FALSE;
-            }
+        if (UTF && UNLIKELY(! _is_grapheme((U8 *) start,
+                                           (U8 *) s,
+                                           (U8 *) PL_bufend,
+                                                  termcode)))
+        {
+            yyerror(non_grapheme_msg);
         }
 
        Copy(s, termstr, termlen, U8);
@@ -10700,14 +10707,13 @@ S_scan_str(pTHX_ char *start, int keep_bracketed_quoted, int keep_delims, int re
                     if (   s + termlen <= PL_bufend
                         && memEQ(s + 1, (char*)termstr + 1, termlen - 1))
                     {
-                        if (   check_grapheme
+                        if (   UTF
                             && UNLIKELY(! _is_grapheme((U8 *) start,
-                                                              (U8 *) s,
-                                                              (U8 *) PL_bufend,
+                                                       (U8 *) s,
+                                                       (U8 *) PL_bufend,
                                                               termcode)))
                         {
-                            Perl_warner(aTHX_ packWARN(WARN_DEPRECATED),
-                                        "%s", non_grapheme_msg);
+                            yyerror(non_grapheme_msg);
                         }
                        break;
                     }