This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Clearing up to_utf8_case() continues: this time use
[perl5.git] / toke.c
diff --git a/toke.c b/toke.c
index e6d7abc..a9a2821 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -1,6 +1,6 @@
 /*    toke.c
  *
- *    Copyright (c) 1991-2001, Larry Wall
+ *    Copyright (c) 1991-2002, Larry Wall
  *
  *    You may distribute under the terms of either the GNU General Public
  *    License or the Artistic License, as specified in the README file.
@@ -514,11 +514,7 @@ S_incline(pTHX_ char *s)
     ch = *t;
     *t = '\0';
     if (t - s > 0) {
-#ifdef USE_ITHREADS
-       Safefree(CopFILE(PL_curcop));
-#else
-       SvREFCNT_dec(CopFILEGV(PL_curcop));
-#endif
+       CopFILE_free(PL_curcop);
        CopFILE_set(PL_curcop, s);
     }
     *t = ch;
@@ -634,6 +630,8 @@ S_skipspace(pTHX_ register char *s)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setpvn(sv,PL_bufptr,PL_bufend-PL_bufptr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop),(I32)CopLINE(PL_curcop),sv);
        }
     }
@@ -1426,7 +1424,9 @@ S_scan_const(pTHX_ char *start)
                /* FALL THROUGH */
            default:
                {
-                   if (ckWARN(WARN_MISC) && isALNUM(*s))
+                   if (ckWARN(WARN_MISC) &&
+                       isALNUM(*s) && 
+                       *s != '_')
                        Perl_warner(aTHX_ WARN_MISC,
                               "Unrecognized escape \\%c passed through",
                               *s);
@@ -1537,7 +1537,7 @@ S_scan_const(pTHX_ char *start)
                }
                continue;
 
-           /* \N{latin small letter a} is a named character */
+           /* \N{LATIN SMALL LETTER A} is a named character */
            case 'N':
                ++s;
                if (*s == '{') {
@@ -1557,6 +1557,26 @@ S_scan_const(pTHX_ char *start)
                    if (has_utf8)
                        sv_utf8_upgrade(res);
                    str = SvPV(res,len);
+#ifdef EBCDIC_NEVER_MIND
+                   /* charnames uses pack U and that has been
+                    * recently changed to do the below uni->native
+                    * mapping, so this would be redundant (and wrong,
+                    * the code point would be doubly converted).
+                    * But leave this in just in case the pack U change
+                    * gets revoked, but the semantics is still
+                    * desireable for charnames. --jhi */
+                   {
+                        UV uv = utf8_to_uvchr((U8*)str, 0);
+
+                        if (uv < 0x100) {
+                             U8 tmpbuf[UTF8_MAXLEN+1], *d;
+
+                             d = uvchr_to_utf8(tmpbuf, UNI_TO_NATIVE(uv));
+                             sv_setpvn(res, (char *)tmpbuf, d - tmpbuf);
+                             str = SvPV(res, len);
+                        }
+                   }
+#endif
                    if (!has_utf8 && SvUTF8(res)) {
                        char *ostart = SvPVX(sv);
                        SvCUR_set(sv, d - ostart);
@@ -1568,7 +1588,7 @@ S_scan_const(pTHX_ char *start)
                        d = SvPVX(sv) + SvCUR(sv);
                        has_utf8 = TRUE;
                    }
-                   if (len > e - s + 4) {
+                   if (len > e - s + 4) { /* I _guess_ 4 is \N{} --jhi */
                        char *odest = SvPVX(sv);
 
                        SvGROW(sv, (SvLEN(sv) + len - (e - s + 4)));
@@ -1653,6 +1673,10 @@ S_scan_const(pTHX_ char *start)
       Perl_croak(aTHX_ "panic: constant overflowed allocated space");
 
     SvPOK_on(sv);
+    if (PL_encoding && !has_utf8) {
+        Perl_sv_recode_to_utf8(aTHX_ sv, PL_encoding);
+        has_utf8 = TRUE;
+    }
     if (has_utf8) {
        SvUTF8_on(sv);
        if (PL_lex_inwhat == OP_TRANS && PL_sublex_info.sub_op) {
@@ -1974,7 +1998,7 @@ Perl_filter_add(pTHX_ filter_t funcp, SV *datasv)
     IoANY(datasv) = (void *)funcp; /* stash funcp into spare field */
     IoFLAGS(datasv) |= IOf_FAKE_DIRP;
     DEBUG_P(PerlIO_printf(Perl_debug_log, "filter_add func %p (%s)\n",
-                         funcp, SvPV_nolen(datasv)));
+                         (void*)funcp, SvPV_nolen(datasv)));
     av_unshift(PL_rsfp_filters, 1);
     av_store(PL_rsfp_filters, 0, datasv) ;
     return(datasv);
@@ -1986,7 +2010,7 @@ void
 Perl_filter_del(pTHX_ filter_t funcp)
 {
     SV *datasv;
-    DEBUG_P(PerlIO_printf(Perl_debug_log, "filter_del func %p", funcp));
+    DEBUG_P(PerlIO_printf(Perl_debug_log, "filter_del func %p", (void*)funcp));
     if (!PL_rsfp_filters || AvFILLp(PL_rsfp_filters)<0)
        return;
     /* if filter is on top of stack (usual case) just pop it off */
@@ -2056,7 +2080,7 @@ Perl_filter_read(pTHX_ int idx, SV *buf_sv, int maxlen)
     funcp = (filter_t)IoANY(datasv);
     DEBUG_P(PerlIO_printf(Perl_debug_log,
                          "filter_read %d: via function %p (%s)\n",
-                         idx, funcp, SvPV_nolen(datasv)));
+                         idx, (void*)funcp, SvPV_nolen(datasv)));
     /* Call function. The function is expected to      */
     /* call "FILTER_READ(idx+1, buf_sv)" first.                */
     /* Return: <0:error, =0:eof, >0:not eof            */
@@ -2178,7 +2202,7 @@ Perl_yylex(pTHX)
     bool bof = FALSE;
 
     /* check if there's an identifier for us to look at */
-    if (PL_pending_ident) 
+    if (PL_pending_ident)
         return S_pending_ident(aTHX);
 
     /* no identifier pending identification */
@@ -2436,7 +2460,7 @@ Perl_yylex(pTHX)
                    if (PL_minus_F) {
                        if (strchr("/'\"", *PL_splitstr)
                              && strchr(PL_splitstr + 1, *PL_splitstr))
-                           Perl_sv_catpvf(aTHX_ PL_linestr, "@F=split(%s);", PL_splitstr);
+                           Perl_sv_catpvf(aTHX_ PL_linestr, "our @F=split(%s);", PL_splitstr);
                        else {
                            char delim;
                            s = "'~#\200\1'"; /* surely one char is unused...*/
@@ -2465,6 +2489,8 @@ Perl_yylex(pTHX)
 
                sv_upgrade(sv, SVt_PVMG);
                sv_setsv(sv,PL_linestr);
+                (void)SvIOK_on(sv);
+                SvIVX(sv) = 0;
                av_store(CopFILEAV(PL_curcop),(I32)CopLINE(PL_curcop),sv);
            }
            goto retry;
@@ -2548,6 +2574,8 @@ Perl_yylex(pTHX)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setsv(sv,PL_linestr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop),(I32)CopLINE(PL_curcop),sv);
        }
        PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
@@ -2588,7 +2616,7 @@ Perl_yylex(pTHX)
                     * at least, set argv[0] to the basename of the Perl
                     * interpreter. So, having found "#!", we'll set it right.
                     */
-                   SV *x = GvSV(gv_fetchpv("\030", TRUE, SVt_PV));
+                   SV *x = GvSV(gv_fetchpv("\030", TRUE, SVt_PV)); /* $^X */
                    assert(SvPOK(x) || SvGMAGICAL(x));
                    if (sv_eq(x, CopFILESV(PL_curcop))) {
                        sv_setpvn(x, ipath, ipathend - ipath);
@@ -2678,6 +2706,7 @@ Perl_yylex(pTHX)
                    while (SPACE_OR_TAB(*d)) d++;
 
                    if (*d++ == '-') {
+                       bool switches_done = PL_doswitches;
                        do {
                            if (*d == 'M' || *d == 'm') {
                                char *m = d;
@@ -2701,6 +2730,14 @@ Perl_yylex(pTHX)
                                (void)gv_fetchfile(PL_origfilename);
                            goto retry;
                        }
+                       if (PL_doswitches && !switches_done) {
+                           int argc = PL_origargc;
+                           char **argv = PL_origargv;
+                           do {
+                               argc--,argv++;
+                           } while (argc && argv[0][0] == '-' && argv[0][1]);
+                           init_argv_symbols(argc,argv);
+                       }
                    }
                }
            }
@@ -3794,6 +3831,10 @@ Perl_yylex(pTHX)
            }
            else {                      /* no override */
                tmp = -tmp;
+               if (tmp == KEY_dump && ckWARN(WARN_MISC)) {
+                   Perl_warner(aTHX_ WARN_MISC,
+                           "dump() better written as CORE::dump()");
+               }
                gv = Nullgv;
                gvp = 0;
                if (ckWARN(WARN_AMBIGUOUS) && hgv
@@ -4885,7 +4926,7 @@ Perl_yylex(pTHX)
                char tmpbuf[sizeof PL_tokenbuf];
                SSize_t tboffset = 0;
                expectation attrful;
-               bool have_name, have_proto;
+               bool have_name, have_proto, bad_proto;
                int key = tmp;
 
                s = skipspace(s);
@@ -4933,14 +4974,22 @@ Perl_yylex(pTHX)
                    s = scan_str(s,FALSE,FALSE);
                    if (!s)
                        Perl_croak(aTHX_ "Prototype not terminated");
-                   /* strip spaces */
+                   /* strip spaces and check for bad characters */
                    d = SvPVX(PL_lex_stuff);
                    tmp = 0;
+                   bad_proto = FALSE;
                    for (p = d; *p; ++p) {
-                       if (!isSPACE(*p))
+                       if (!isSPACE(*p)) {
                            d[tmp++] = *p;
+                           if (!strchr("$@%*;[]&\\", *p))
+                               bad_proto = TRUE;
+                       }
                    }
                    d[tmp] = '\0';
+                   if (bad_proto && ckWARN(WARN_SYNTAX))
+                       Perl_warner(aTHX_ WARN_SYNTAX,
+                                   "Illegal character in prototype for %s : %s",
+                                   SvPVX(PL_subname), d);
                    SvCUR(PL_lex_stuff) = tmp;
                    have_proto = TRUE;
 
@@ -5202,7 +5251,7 @@ S_pending_ident(pTHX)
                 gv_fetchpv(SvPVX(sym),
                     (PL_in_eval
                         ? (GV_ADDMULTI | GV_ADDINEVAL)
-                        : TRUE
+                        : GV_ADDMULTI
                     ),
                     ((PL_tokenbuf[0] == '$') ? SVt_PV
                      : (PL_tokenbuf[0] == '@') ? SVt_PVAV
@@ -6529,6 +6578,8 @@ S_scan_heredoc(pTHX_ register char *s)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setsv(sv,PL_linestr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop), (I32)CopLINE(PL_curcop),sv);
        }
        if (*s == term && memEQ(s,PL_tokenbuf,len)) {
@@ -6628,6 +6679,9 @@ S_scan_inputsymbol(pTHX_ char *start)
        return s;
     }
     else {
+       bool readline_overriden = FALSE;
+       GV *gv_readline = Nullgv;
+       GV **gvp;
        /* we're in a filehandle read situation */
        d = PL_tokenbuf;
 
@@ -6635,6 +6689,15 @@ S_scan_inputsymbol(pTHX_ char *start)
        if (!len)
            (void)strcpy(d,"ARGV");
 
+       /* Check whether readline() is overriden */
+       if (((gv_readline = gv_fetchpv("readline", FALSE, SVt_PVCV))
+               && GvCVu(gv_readline) && GvIMPORTED_CV(gv_readline))
+               ||
+               ((gvp = (GV**)hv_fetch(PL_globalstash, "readline", 8, FALSE))
+               && (gv_readline = *gvp) != (GV*)&PL_sv_undef
+               && GvCVu(gv_readline) && GvIMPORTED_CV(gv_readline)))
+           readline_overriden = TRUE;
+
        /* if <$fh>, create the ops to turn the variable into a
           filehandle
        */
@@ -6656,7 +6719,11 @@ S_scan_inputsymbol(pTHX_ char *start)
                else {
                    OP *o = newOP(OP_PADSV, 0);
                    o->op_targ = tmp;
-                   PL_lex_op = (OP*)newUNOP(OP_READLINE, 0, o);
+                   PL_lex_op = readline_overriden
+                       ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
+                               append_elem(OP_LIST, o,
+                                   newCVREF(0, newGVOP(OP_GV,0,gv_readline))))
+                       : (OP*)newUNOP(OP_READLINE, 0, o);
                }
            }
            else {
@@ -6666,11 +6733,16 @@ intro_sym:
                gv = gv_fetchpv(d,
                                (PL_in_eval
                                 ? (GV_ADDMULTI | GV_ADDINEVAL)
-                                : TRUE),
+                                : GV_ADDMULTI),
                                SVt_PV);
-               PL_lex_op = (OP*)newUNOP(OP_READLINE, 0,
-                                           newUNOP(OP_RV2SV, 0,
-                                               newGVOP(OP_GV, 0, gv)));
+               PL_lex_op = readline_overriden
+                   ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
+                           append_elem(OP_LIST,
+                               newUNOP(OP_RV2SV, 0, newGVOP(OP_GV, 0, gv)),
+                               newCVREF(0, newGVOP(OP_GV, 0, gv_readline))))
+                   : (OP*)newUNOP(OP_READLINE, 0,
+                           newUNOP(OP_RV2SV, 0,
+                               newGVOP(OP_GV, 0, gv)));
            }
            PL_lex_op->op_flags |= OPf_SPECIAL;
            /* we created the ops in PL_lex_op, so make yylval.ival a null op */
@@ -6681,7 +6753,12 @@ intro_sym:
           (<Foo::BAR> or <FOO>) so build a simple readline OP */
        else {
            GV *gv = gv_fetchpv(d,TRUE, SVt_PVIO);
-           PL_lex_op = (OP*)newUNOP(OP_READLINE, 0, newGVOP(OP_GV, 0, gv));
+           PL_lex_op = readline_overriden
+               ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
+                       append_elem(OP_LIST,
+                           newGVOP(OP_GV, 0, gv),
+                           newCVREF(0, newGVOP(OP_GV, 0, gv_readline))))
+               : (OP*)newUNOP(OP_READLINE, 0, newGVOP(OP_GV, 0, gv));
            yylval.ival = OP_NULL;
        }
     }
@@ -6879,6 +6956,8 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setsv(sv,PL_linestr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop), (I32)CopLINE(PL_curcop), sv);
        }
 
@@ -7199,7 +7278,7 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
            }
            if (*s == '.' && isDIGIT(s[1])) {
                /* oops, it's really a v-string, but without the "v" */
-               s = start - 1;
+               s = start;
                goto vstring;
            }
        }
@@ -7293,58 +7372,8 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
     /* if it starts with a v, it could be a v-string */
     case 'v':
 vstring:
-       {
-           char *pos = s;
-           pos++;
-           while (isDIGIT(*pos) || *pos == '_')
-               pos++;
-           if (!isALPHA(*pos)) {
-               UV rev;
-               U8 tmpbuf[UTF8_MAXLEN+1];
-               U8 *tmpend;
-               s++;                            /* get past 'v' */
-
-               sv = NEWSV(92,5);
-               sv_setpvn(sv, "", 0);
-
-               for (;;) {
-                   if (*s == '0' && isDIGIT(s[1]))
-                       yyerror("Octal number in vector unsupported");
-                   rev = 0;
-                   {
-                       /* this is atoi() that tolerates underscores */
-                       char *end = pos;
-                       UV mult = 1;
-                       while (--end >= s) {
-                           UV orev;
-                           if (*end == '_')
-                               continue;
-                           orev = rev;
-                           rev += (*end - '0') * mult;
-                           mult *= 10;
-                           if (orev > rev && ckWARN_d(WARN_OVERFLOW))
-                               Perl_warner(aTHX_ WARN_OVERFLOW,
-                                           "Integer overflow in decimal number");
-                       }
-                   }
-                   /* Append native character for the rev point */
-                   tmpend = uvchr_to_utf8(tmpbuf, rev);
-                   sv_catpvn(sv, (const char*)tmpbuf, tmpend - tmpbuf);
-                   if (!UNI_IS_INVARIANT(NATIVE_TO_UNI(rev)))
-                       SvUTF8_on(sv);
-                   if (*pos == '.' && isDIGIT(pos[1]))
-                       s = ++pos;
-                   else {
-                       s = pos;
-                       break;
-                   }
-                   while (isDIGIT(*pos) || *pos == '_')
-                       pos++;
-               }
-               SvPOK_on(sv);
-               SvREADONLY_on(sv);
-           }
-       }
+               sv = NEWSV(92,5); /* preallocate storage space */
+               s = new_vstring(s,sv);
        break;
     }
 
@@ -7572,7 +7601,7 @@ Perl_yyerror(pTHX_ char *s)
     }
     msg = sv_2mortal(newSVpv(s, 0));
     Perl_sv_catpvf(aTHX_ msg, " at %s line %"IVdf", ",
-                  CopFILE(PL_curcop), (IV)CopLINE(PL_curcop));
+        OutCopFILE(PL_curcop), (IV)CopLINE(PL_curcop));
     if (context)
        Perl_sv_catpvf(aTHX_ msg, "near \"%.*s\"\n", contlen, context);
     else
@@ -7590,10 +7619,10 @@ Perl_yyerror(pTHX_ char *s)
     if (PL_error_count >= 10) {
        if (PL_in_eval && SvCUR(ERRSV))
            Perl_croak(aTHX_ "%"SVf"%s has too many errors.\n",
-                      ERRSV, CopFILE(PL_curcop));
+            ERRSV, OutCopFILE(PL_curcop));
        else
            Perl_croak(aTHX_ "%s has too many errors.\n",
-                      CopFILE(PL_curcop));
+            OutCopFILE(PL_curcop));
     }
     PL_in_my = 0;
     PL_in_my_stash = Nullhv;
@@ -7734,3 +7763,4 @@ utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen)
     return count;
 }
 #endif
+