X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/ad64d0ecd555e97c5a216efca1ec5a96b7fd0b34..6b46370c614272cc427575562cb4f6c5af6e4aef:/regexec.c diff --git a/regexec.c b/regexec.c index 0c4329d..17a0dc6 100644 --- a/regexec.c +++ b/regexec.c @@ -2,7 +2,11 @@ */ /* - * "One Ring to rule them all, One Ring to find them..." + * One Ring to rule them all, One Ring to find them + & + * [p.v of _The Lord of the Rings_, opening poem] + * [p.50 of _The Lord of the Rings_, I/iii: "The Shadow of the Past"] + * [p.254 of _The Lord of the Rings_, II/ii: "The Council of Elrond"] */ /* This file contains functions for executing a regular expression. See @@ -113,12 +117,130 @@ #define HOP3(pos,off,lim) (PL_reg_match_utf8 ? reghop3((U8*)(pos), off, (U8*)(lim)) : (U8*)(pos + off)) #define HOP3c(pos,off,lim) ((char*)HOP3(pos,off,lim)) +/* these are unrolled below in the CCC_TRY_XXX defined */ #define LOAD_UTF8_CHARCLASS(class,str) STMT_START { \ if (!CAT2(PL_utf8_,class)) { bool ok; ENTER; save_re_context(); ok=CAT2(is_utf8_,class)((const U8*)str); assert(ok); LEAVE; } } STMT_END + +/* Doesn't do an assert to verify that is correct */ +#define LOAD_UTF8_CHARCLASS_NO_CHECK(class) STMT_START { \ + if (!CAT2(PL_utf8_,class)) { bool ok; ENTER; save_re_context(); ok=CAT2(is_utf8_,class)((const U8*)" "); LEAVE; } } STMT_END + #define LOAD_UTF8_CHARCLASS_ALNUM() LOAD_UTF8_CHARCLASS(alnum,"a") #define LOAD_UTF8_CHARCLASS_DIGIT() LOAD_UTF8_CHARCLASS(digit,"0") #define LOAD_UTF8_CHARCLASS_SPACE() LOAD_UTF8_CHARCLASS(space," ") -#define LOAD_UTF8_CHARCLASS_MARK() LOAD_UTF8_CHARCLASS(mark, "\xcd\x86") + +#define LOAD_UTF8_CHARCLASS_GCB() /* Grapheme cluster boundaries */ \ + LOAD_UTF8_CHARCLASS(X_begin, " "); \ + LOAD_UTF8_CHARCLASS(X_non_hangul, "A"); \ + /* These are utf8 constants, and not utf-ebcdic constants, so the \ + * assert should likely and hopefully fail on an EBCDIC machine */ \ + LOAD_UTF8_CHARCLASS(X_extend, "\xcc\x80"); /* U+0300 */ \ + \ + /* No asserts are done for these, in case called on an early \ + * Unicode version in which they map to nothing */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_prepend);/* U+0E40 "\xe0\xb9\x80" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_L); /* U+1100 "\xe1\x84\x80" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_LV); /* U+AC00 "\xea\xb0\x80" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_LVT); /* U+AC01 "\xea\xb0\x81" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_LV_LVT_V);/* U+AC01 "\xea\xb0\x81" */\ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_T); /* U+11A8 "\xe1\x86\xa8" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_V) /* U+1160 "\xe1\x85\xa0" */ + +/* + We dont use PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS as the direct test + so that it is possible to override the option here without having to + rebuild the entire core. as we are required to do if we change regcomp.h + which is where PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS is defined. +*/ +#if PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS +#define BROKEN_UNICODE_CHARCLASS_MAPPINGS +#endif + +#ifdef BROKEN_UNICODE_CHARCLASS_MAPPINGS +#define LOAD_UTF8_CHARCLASS_PERL_WORD() LOAD_UTF8_CHARCLASS_ALNUM() +#define LOAD_UTF8_CHARCLASS_PERL_SPACE() LOAD_UTF8_CHARCLASS_SPACE() +#define LOAD_UTF8_CHARCLASS_POSIX_DIGIT() LOAD_UTF8_CHARCLASS_DIGIT() +#define RE_utf8_perl_word PL_utf8_alnum +#define RE_utf8_perl_space PL_utf8_space +#define RE_utf8_posix_digit PL_utf8_digit +#define perl_word alnum +#define perl_space space +#define posix_digit digit +#else +#define LOAD_UTF8_CHARCLASS_PERL_WORD() LOAD_UTF8_CHARCLASS(perl_word,"a") +#define LOAD_UTF8_CHARCLASS_PERL_SPACE() LOAD_UTF8_CHARCLASS(perl_space," ") +#define LOAD_UTF8_CHARCLASS_POSIX_DIGIT() LOAD_UTF8_CHARCLASS(posix_digit,"0") +#define RE_utf8_perl_word PL_utf8_perl_word +#define RE_utf8_perl_space PL_utf8_perl_space +#define RE_utf8_posix_digit PL_utf8_posix_digit +#endif + + +#define CCC_TRY_AFF(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC,LCFUNC) \ + case NAMEL: \ + PL_reg_flags |= RF_tainted; \ + /* FALL THROUGH */ \ + case NAME: \ + if (!nextchr) \ + sayNO; \ + if (do_utf8 && UTF8_IS_CONTINUED(nextchr)) { \ + if (!CAT2(PL_utf8_,CLASS)) { \ + bool ok; \ + ENTER; \ + save_re_context(); \ + ok=CAT2(is_utf8_,CLASS)((const U8*)STR); \ + assert(ok); \ + LEAVE; \ + } \ + if (!(OP(scan) == NAME \ + ? (bool)swash_fetch(CAT2(PL_utf8_,CLASS), (U8*)locinput, do_utf8) \ + : LCFUNC_utf8((U8*)locinput))) \ + { \ + sayNO; \ + } \ + locinput += PL_utf8skip[nextchr]; \ + nextchr = UCHARAT(locinput); \ + break; \ + } \ + if (!(OP(scan) == NAME ? FUNC(nextchr) : LCFUNC(nextchr))) \ + sayNO; \ + nextchr = UCHARAT(++locinput); \ + break + +#define CCC_TRY_NEG(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC,LCFUNC) \ + case NAMEL: \ + PL_reg_flags |= RF_tainted; \ + /* FALL THROUGH */ \ + case NAME : \ + if (!nextchr && locinput >= PL_regeol) \ + sayNO; \ + if (do_utf8 && UTF8_IS_CONTINUED(nextchr)) { \ + if (!CAT2(PL_utf8_,CLASS)) { \ + bool ok; \ + ENTER; \ + save_re_context(); \ + ok=CAT2(is_utf8_,CLASS)((const U8*)STR); \ + assert(ok); \ + LEAVE; \ + } \ + if ((OP(scan) == NAME \ + ? (bool)swash_fetch(CAT2(PL_utf8_,CLASS), (U8*)locinput, do_utf8) \ + : LCFUNC_utf8((U8*)locinput))) \ + { \ + sayNO; \ + } \ + locinput += PL_utf8skip[nextchr]; \ + nextchr = UCHARAT(locinput); \ + break; \ + } \ + if ((OP(scan) == NAME ? FUNC(nextchr) : LCFUNC(nextchr))) \ + sayNO; \ + nextchr = UCHARAT(++locinput); \ + break + + + + /* TODO: Combine JUMPABLE and HAS_TEXT to cache OP(rn) */ @@ -859,9 +981,9 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, { /* If flags & SOMETHING - do not do it many times on the same match */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "... Disabling check substring...\n")); + /* XXX Does the destruction order has to change with do_utf8? */ SvREFCNT_dec(do_utf8 ? prog->check_utf8 : prog->check_substr); - if (do_utf8 ? prog->check_substr : prog->check_utf8) - SvREFCNT_dec(do_utf8 ? prog->check_substr : prog->check_utf8); + SvREFCNT_dec(do_utf8 ? prog->check_substr : prog->check_utf8); prog->check_substr = prog->check_utf8 = NULL; /* disable */ prog->float_substr = prog->float_utf8 = NULL; /* clear */ check = NULL; /* abort */ @@ -1006,12 +1128,12 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ switch (trie_type) { \ case trie_utf8_fold: \ if ( foldlen>0 ) { \ - uvc = utf8n_to_uvuni( uscan, UTF8_MAXLEN, &len, uniflags ); \ + uvc = utf8n_to_uvuni( uscan, UTF8_MAXLEN, &len, uniflags ); \ foldlen -= len; \ uscan += len; \ len=0; \ } else { \ - uvc = utf8n_to_uvuni( (U8*)uc, UTF8_MAXLEN, &len, uniflags ); \ + uvc = utf8n_to_uvuni( (U8*)uc, UTF8_MAXLEN, &len, uniflags ); \ uvc = to_uni_fold( uvc, foldbuf, &foldlen ); \ foldlen -= UNISKIP( uvc ); \ uscan = foldbuf + UNISKIP( uvc ); \ @@ -1037,7 +1159,6 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ uvc = (UV)*uc; \ len = 1; \ } \ - \ if (uvc < 256) { \ charid = trie->charmap[ uvc ]; \ } \ @@ -1418,8 +1539,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, break; case ALNUM: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_ALNUM(), - swash_fetch(PL_utf8_alnum, (U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_PERL_WORD(), + swash_fetch(RE_utf8_perl_word, (U8*)s, do_utf8), isALNUM(*s) ); case ALNUML: @@ -1429,8 +1550,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case NALNUM: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_ALNUM(), - !swash_fetch(PL_utf8_alnum, (U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_PERL_WORD(), + !swash_fetch(RE_utf8_perl_word, (U8*)s, do_utf8), !isALNUM(*s) ); case NALNUML: @@ -1440,8 +1561,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case SPACE: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_SPACE(), - *s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_PERL_SPACE(), + *s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, do_utf8), isSPACE(*s) ); case SPACEL: @@ -1451,8 +1572,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case NSPACE: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_SPACE(), - !(*s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, do_utf8)), + LOAD_UTF8_CHARCLASS_PERL_SPACE(), + !(*s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, do_utf8)), !isSPACE(*s) ); case NSPACEL: @@ -1462,8 +1583,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case DIGIT: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_DIGIT(), - swash_fetch(PL_utf8_digit,(U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_POSIX_DIGIT(), + swash_fetch(RE_utf8_posix_digit,(U8*)s, do_utf8), isDIGIT(*s) ); case DIGITL: @@ -1473,8 +1594,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case NDIGIT: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_DIGIT(), - !swash_fetch(PL_utf8_digit,(U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_POSIX_DIGIT(), + !swash_fetch(RE_utf8_posix_digit,(U8*)s, do_utf8), !isDIGIT(*s) ); case NDIGITL: @@ -1726,28 +1847,6 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, return s; } -static void -S_swap_match_buff (pTHX_ regexp *prog) -{ - regexp_paren_pair *t; - - PERL_ARGS_ASSERT_SWAP_MATCH_BUFF; - - if (!prog->swap) { - /* We have to be careful. If the previous successful match - was from this regex we don't want a subsequent paritally - successful match to clobber the old results. - So when we detect this possibility we add a swap buffer - to the re, and switch the buffer each match. If we fail - we switch it back, otherwise we leave it swapped. - */ - Newxz(prog->swap, (prog->nparens + 1), regexp_paren_pair); - } - t = prog->swap; - prog->swap = prog->offs; - prog->offs = t; -} - /* - regexec_flags - match a regexp against a string @@ -1777,7 +1876,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre I32 multiline; RXi_GET_DECL(prog,progi); regmatch_info reginfo; /* create some info to pass to regtry etc */ - bool swap_on_fail = 0; + regexp_paren_pair *swap = NULL; GET_RE_DEBUG_FLAGS_DECL; PERL_ARGS_ASSERT_REGEXEC_FLAGS; @@ -1835,29 +1934,50 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre if (prog->extflags & RXf_GPOS_SEEN) { /* Need to set reginfo->ganch */ MAGIC *mg; - - if (flags & REXEC_IGNOREPOS) /* Means: check only at start */ + if (flags & REXEC_IGNOREPOS){ /* Means: check only at start */ reginfo.ganch = startpos + prog->gofs; - else if (sv && SvTYPE(sv) >= SVt_PVMG + DEBUG_GPOS_r(PerlIO_printf(Perl_debug_log, + "GPOS IGNOREPOS: reginfo.ganch = startpos + %"UVxf"\n",(UV)prog->gofs)); + } else if (sv && SvTYPE(sv) >= SVt_PVMG && SvMAGIC(sv) && (mg = mg_find(sv, PERL_MAGIC_regex_global)) && mg->mg_len >= 0) { reginfo.ganch = strbeg + mg->mg_len; /* Defined pos() */ + DEBUG_GPOS_r(PerlIO_printf(Perl_debug_log, + "GPOS MAGIC: reginfo.ganch = strbeg + %"IVdf"\n",(IV)mg->mg_len)); + if (prog->extflags & RXf_ANCH_GPOS) { if (s > reginfo.ganch) goto phooey; s = reginfo.ganch - prog->gofs; + DEBUG_GPOS_r(PerlIO_printf(Perl_debug_log, + "GPOS ANCH_GPOS: s = ganch - %"UVxf"\n",(UV)prog->gofs)); + if (s < strbeg) + goto phooey; } } else if (data) { reginfo.ganch = strbeg + PTR2UV(data); - } else /* pos() not defined */ + DEBUG_GPOS_r(PerlIO_printf(Perl_debug_log, + "GPOS DATA: reginfo.ganch= strbeg + %"UVxf"\n",PTR2UV(data))); + + } else { /* pos() not defined */ reginfo.ganch = strbeg; + DEBUG_GPOS_r(PerlIO_printf(Perl_debug_log, + "GPOS: reginfo.ganch = strbeg\n")); + } } if (PL_curpm && (PM_GETRE(PL_curpm) == rx)) { - swap_on_fail = 1; - swap_match_buff(prog); /* do we need a save destructor here for - eval dies? */ + /* We have to be careful. If the previous successful match + was from this regex we don't want a subsequent partially + successful match to clobber the old results. + So when we detect this possibility we add a swap buffer + to the re, and switch the buffer each match. If we fail + we switch it back, otherwise we leave it swapped. + */ + swap = prog->offs; + /* do we need a save destructor here for eval dies? */ + Newxz(prog->offs, (prog->nparens + 1), regexp_paren_pair); } if (!(flags & REXEC_CHECKED) && (prog->check_substr != NULL || prog->check_utf8 != NULL)) { re_scream_pos_data d; @@ -1922,7 +2042,8 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre is bogus -- we set it above, when prog->extflags & RXf_GPOS_SEEN and we only enter this block when the same bit is set. */ char *tmp_s = reginfo.ganch - prog->gofs; - if (regtry(®info, &tmp_s)) + + if (tmp_s >= strbeg && regtry(®info, &tmp_s)) goto got_it; goto phooey; } @@ -2156,6 +2277,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre goto phooey; got_it: + Safefree(swap); RX_MATCH_TAINTED_set(rx, PL_reg_flags & RF_tainted); if (PL_reg_eval_set) @@ -2201,10 +2323,12 @@ phooey: PL_colors[4], PL_colors[5])); if (PL_reg_eval_set) restore_pos(aTHX_ prog); - if (swap_on_fail) + if (swap) { /* we failed :-( roll it back */ - swap_match_buff(prog); - + Safefree(prog->offs); + prog->offs = swap; + } + return 0; } @@ -2246,7 +2370,7 @@ S_regtry(pTHX_ regmatch_info *reginfo, char **startpos) /* Make $_ available to executed code. */ if (reginfo->sv != DEFSV) { SAVE_DEFSV; - DEFSV = reginfo->sv; + DEFSV_set(reginfo->sv); } if (!(SvTYPE(reginfo->sv) >= SVt_PVMG && SvMAGIC(reginfo->sv) @@ -2586,7 +2710,7 @@ S_debug_start_match(pTHX_ const REGEXP *prog, const bool do_utf8, reginitcolors(); { RE_PV_QUOTED_DECL(s0, utf8_pat, PERL_DEBUG_PAD_ZERO(0), - RX_PRECOMP(prog), RX_PRELEN(prog), 60); + RX_PRECOMP_const(prog), RX_PRELEN(prog), 60); RE_PV_QUOTED_DECL(s1, do_utf8, PERL_DEBUG_PAD_ZERO(1), start, end - start, 60); @@ -2833,6 +2957,11 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) state_num = OP(scan); reenter_switch: + + assert(PL_reglastparen == &rex->lastparen); + assert(PL_reglastcloseparen == &rex->lastcloseparen); + assert(PL_regoffs == rex->offs); + switch (state_num) { case BOL: if (locinput == PL_bostr) @@ -2999,7 +3128,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) if ( got_wordnum ) { if ( ! ST.accepted ) { ENTER; - /* SAVETMPS; */ /* XXX is this necessary? dmq */ + SAVETMPS; /* XXX is this necessary? dmq */ bufflen = TRIE_INITAL_ACCEPT_BUFFLEN; sv_accept_buff=newSV(bufflen * sizeof(reg_trie_accepted) - 1); @@ -3218,6 +3347,9 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) } /* NOTREACHED */ case TRIE_next: + /* we dont want to throw this away, see bug 57042*/ + if (oreplsv != GvSV(PL_replgv)) + sv_setsv(oreplsv, GvSV(PL_replgv)); FREETMPS; LEAVE; sayYES; @@ -3325,85 +3457,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) nextchr = UCHARAT(locinput); break; } - case ANYOF: - if (do_utf8) { - STRLEN inclasslen = PL_regeol - locinput; - - if (!reginclass(rex, scan, (U8*)locinput, &inclasslen, do_utf8)) - goto anyof_fail; - if (locinput >= PL_regeol) - sayNO; - locinput += inclasslen ? inclasslen : UTF8SKIP(locinput); - nextchr = UCHARAT(locinput); - break; - } - else { - if (nextchr < 0) - nextchr = UCHARAT(locinput); - if (!REGINCLASS(rex, scan, (U8*)locinput)) - goto anyof_fail; - if (!nextchr && locinput >= PL_regeol) - sayNO; - nextchr = UCHARAT(++locinput); - break; - } - anyof_fail: - /* If we might have the case of the German sharp s - * in a casefolding Unicode character class. */ - - if (ANYOF_FOLD_SHARP_S(scan, locinput, PL_regeol)) { - locinput += SHARP_S_SKIP; - nextchr = UCHARAT(locinput); - } - else - sayNO; - break; - case ALNUML: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case ALNUM: - if (!nextchr) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_ALNUM(); - if (!(OP(scan) == ALNUM - ? (bool)swash_fetch(PL_utf8_alnum, (U8*)locinput, do_utf8) - : isALNUM_LC_utf8((U8*)locinput))) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; - } - if (!(OP(scan) == ALNUM - ? isALNUM(nextchr) : isALNUM_LC(nextchr))) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case NALNUML: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case NALNUM: - if (!nextchr && locinput >= PL_regeol) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_ALNUM(); - if (OP(scan) == NALNUM - ? (bool)swash_fetch(PL_utf8_alnum, (U8*)locinput, do_utf8) - : isALNUM_LC_utf8((U8*)locinput)) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; - } - if (OP(scan) == NALNUM - ? isALNUM(nextchr) : isALNUM_LC(nextchr)) - sayNO; - nextchr = UCHARAT(++locinput); - break; case BOUNDL: case NBOUNDL: PL_reg_flags |= RF_tainted; @@ -3416,7 +3469,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) ln = '\n'; else { const U8 * const r = reghop3((U8*)locinput, -1, (U8*)PL_bostr); - + ln = utf8n_to_uvchr(r, UTF8SKIP(r), 0, uniflags); } if (OP(scan) == BOUND || OP(scan) == NBOUND) { @@ -3445,122 +3498,259 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) OP(scan) == BOUNDL)) sayNO; break; - case SPACEL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case SPACE: - if (!nextchr) - sayNO; + case ANYOF: if (do_utf8) { - if (UTF8_IS_CONTINUED(nextchr)) { - LOAD_UTF8_CHARCLASS_SPACE(); - if (!(OP(scan) == SPACE - ? (bool)swash_fetch(PL_utf8_space, (U8*)locinput, do_utf8) - : isSPACE_LC_utf8((U8*)locinput))) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; - } - if (!(OP(scan) == SPACE - ? isSPACE(nextchr) : isSPACE_LC(nextchr))) + STRLEN inclasslen = PL_regeol - locinput; + + if (!reginclass(rex, scan, (U8*)locinput, &inclasslen, do_utf8)) + goto anyof_fail; + if (locinput >= PL_regeol) sayNO; - nextchr = UCHARAT(++locinput); + locinput += inclasslen ? inclasslen : UTF8SKIP(locinput); + nextchr = UCHARAT(locinput); + break; } else { - if (!(OP(scan) == SPACE - ? isSPACE(nextchr) : isSPACE_LC(nextchr))) + if (nextchr < 0) + nextchr = UCHARAT(locinput); + if (!REGINCLASS(rex, scan, (U8*)locinput)) + goto anyof_fail; + if (!nextchr && locinput >= PL_regeol) sayNO; nextchr = UCHARAT(++locinput); - } - break; - case NSPACEL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case NSPACE: - if (!nextchr && locinput >= PL_regeol) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_SPACE(); - if (OP(scan) == NSPACE - ? (bool)swash_fetch(PL_utf8_space, (U8*)locinput, do_utf8) - : isSPACE_LC_utf8((U8*)locinput)) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); break; } - if (OP(scan) == NSPACE - ? isSPACE(nextchr) : isSPACE_LC(nextchr)) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case DIGITL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case DIGIT: - if (!nextchr) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_DIGIT(); - if (!(OP(scan) == DIGIT - ? (bool)swash_fetch(PL_utf8_digit, (U8*)locinput, do_utf8) - : isDIGIT_LC_utf8((U8*)locinput))) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; + anyof_fail: + /* If we might have the case of the German sharp s + * in a casefolding Unicode character class. */ + + if (ANYOF_FOLD_SHARP_S(scan, locinput, PL_regeol)) { + locinput += SHARP_S_SKIP; + nextchr = UCHARAT(locinput); } - if (!(OP(scan) == DIGIT - ? isDIGIT(nextchr) : isDIGIT_LC(nextchr))) - sayNO; - nextchr = UCHARAT(++locinput); + else + sayNO; break; - case NDIGITL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case NDIGIT: - if (!nextchr && locinput >= PL_regeol) + /* Special char classes - The defines start on line 129 or so */ + CCC_TRY_AFF( ALNUM, ALNUML, perl_word, "a", isALNUM_LC_utf8, isALNUM, isALNUM_LC); + CCC_TRY_NEG(NALNUM, NALNUML, perl_word, "a", isALNUM_LC_utf8, isALNUM, isALNUM_LC); + + CCC_TRY_AFF( SPACE, SPACEL, perl_space, " ", isSPACE_LC_utf8, isSPACE, isSPACE_LC); + CCC_TRY_NEG(NSPACE, NSPACEL, perl_space, " ", isSPACE_LC_utf8, isSPACE, isSPACE_LC); + + CCC_TRY_AFF( DIGIT, DIGITL, posix_digit, "0", isDIGIT_LC_utf8, isDIGIT, isDIGIT_LC); + CCC_TRY_NEG(NDIGIT, NDIGITL, posix_digit, "0", isDIGIT_LC_utf8, isDIGIT, isDIGIT_LC); + + case CLUMP: /* Match \X: logical Unicode character. This is defined as + a Unicode extended Grapheme Cluster */ + /* From http://www.unicode.org/reports/tr29 (5.2 version). An + extended Grapheme Cluster is: + + CR LF + | Prepend* Begin Extend* + | . + + Begin is (Hangul-syllable | ! Control) + Extend is (Grapheme_Extend | Spacing_Mark) + Control is [ GCB_Control CR LF ] + + The discussion below shows how the code for CLUMP is derived + from this regex. Note that most of these concepts are from + property values of the Grapheme Cluster Boundary (GCB) property. + No code point can have multiple property values for a given + property. Thus a code point in Prepend can't be in Control, but + it must be in !Control. This is why Control above includes + GCB_Control plus CR plus LF. The latter two are used in the GCB + property separately, and so can't be in GCB_Control, even though + they logically are controls. Control is not the same as gc=cc, + but includes format and other characters as well. + + The Unicode definition of Hangul-syllable is: + L+ + | (L* ( ( V | LV ) V* | LVT ) T*) + | T+ + ) + Each of these is a value for the GCB property, and hence must be + disjoint, so the order they are tested is immaterial, so the + above can safely be changed to + T+ + | L+ + | (L* ( LVT | ( V | LV ) V*) T*) + + The last two terms can be combined like this: + L* ( L + | (( LVT | ( V | LV ) V*) T*)) + + And refactored into this: + L* (L | LVT T* | V V* T* | LV V* T*) + + That means that if we have seen any L's at all we can quit + there, but if the next character is a LVT, a V or and LV we + should keep going. + + There is a subtlety with Prepend* which showed up in testing. + Note that the Begin, and only the Begin is required in: + | Prepend* Begin Extend* + Also, Begin contains '! Control'. A Prepend must be a '! + Control', which means it must be a Begin. What it comes down to + is that if we match Prepend* and then find no suitable Begin + afterwards, that if we backtrack the last Prepend, that one will + be a suitable Begin. + */ + + if (locinput >= PL_regeol) sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_DIGIT(); - if (OP(scan) == NDIGIT - ? (bool)swash_fetch(PL_utf8_digit, (U8*)locinput, do_utf8) - : isDIGIT_LC_utf8((U8*)locinput)) - { - sayNO; + if (! do_utf8) { + + /* Match either CR LF or '.', as all the other possibilities + * require utf8 */ + locinput++; /* Match the . or CR */ + if (nextchr == '\r' + && locinput < PL_regeol + && UCHARAT(locinput) == '\n') locinput++; + } + else { + + /* Utf8: See if is ( CR LF ); already know that locinput < + * PL_regeol, so locinput+1 is in bounds */ + if (nextchr == '\r' && UCHARAT(locinput + 1) == '\n') { + locinput += 2; } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; + else { + /* In case have to backtrack to beginning, then match '.' */ + char *starting = locinput; + + /* In case have to backtrack the last prepend */ + char *previous_prepend = 0; + + LOAD_UTF8_CHARCLASS_GCB(); + + /* Match (prepend)* */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_prepend, + (U8*)locinput, do_utf8)) + { + previous_prepend = locinput; + locinput += UTF8SKIP(locinput); + } + + /* As noted above, if we matched a prepend character, but + * the next thing won't match, back off the last prepend we + * matched, as it is guaranteed to match the begin */ + if (previous_prepend + && (locinput >= PL_regeol + || ! swash_fetch(PL_utf8_X_begin, + (U8*)locinput, do_utf8))) + { + locinput = previous_prepend; + } + + /* Note that here we know PL_regeol > locinput, as we + * tested that upon input to this switch case, and if we + * moved locinput forward, we tested the result just above + * and it either passed, or we backed off so that it will + * now pass */ + if (! swash_fetch(PL_utf8_X_begin, (U8*)locinput, do_utf8)) { + + /* Here did not match the required 'Begin' in the + * second term. So just match the very first + * character, the '.' of the final term of the regex */ + locinput = starting + UTF8SKIP(starting); + } else { + + /* Here is the beginning of a character that can have + * an extender. It is either a hangul syllable, or a + * non-control */ + if (swash_fetch(PL_utf8_X_non_hangul, + (U8*)locinput, do_utf8)) + { + + /* Here not a Hangul syllable, must be a + * ('! * Control') */ + locinput += UTF8SKIP(locinput); + } else { + + /* Here is a Hangul syllable. It can be composed + * of several individual characters. One + * possibility is T+ */ + if (swash_fetch(PL_utf8_X_T, + (U8*)locinput, do_utf8)) + { + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_T, + (U8*)locinput, do_utf8)) + { + locinput += UTF8SKIP(locinput); + } + } else { + + /* Here, not T+, but is a Hangul. That means + * it is one of the others: L, LV, LVT or V, + * and matches: + * L* (L | LVT T* | V V* T* | LV V* T*) */ + + /* Match L* */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_L, + (U8*)locinput, do_utf8)) + { + locinput += UTF8SKIP(locinput); + } + + /* Here, have exhausted L*. If the next + * character is not an LV, LVT nor V, it means + * we had to have at least one L, so matches L+ + * in the original equation, we have a complete + * hangul syllable. Are done. */ + + if (locinput < PL_regeol + && swash_fetch(PL_utf8_X_LV_LVT_V, + (U8*)locinput, do_utf8)) + { + + /* Otherwise keep going. Must be LV, LVT + * or V. See if LVT */ + if (swash_fetch(PL_utf8_X_LVT, + (U8*)locinput, do_utf8)) + { + locinput += UTF8SKIP(locinput); + } else { + + /* Must be V or LV. Take it, then + * match V* */ + locinput += UTF8SKIP(locinput); + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_V, + (U8*)locinput, do_utf8)) + { + locinput += UTF8SKIP(locinput); + } + } + + /* And any of LV, LVT, or V can be followed + * by T* */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_T, + (U8*)locinput, + do_utf8)) + { + locinput += UTF8SKIP(locinput); + } + } + } + } + + /* Match any extender */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_extend, + (U8*)locinput, do_utf8)) + { + locinput += UTF8SKIP(locinput); + } + } + } + if (locinput > PL_regeol) sayNO; } - if (OP(scan) == NDIGIT - ? isDIGIT(nextchr) : isDIGIT_LC(nextchr)) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case CLUMP: - if (locinput >= PL_regeol) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_MARK(); - if (swash_fetch(PL_utf8_mark,(U8*)locinput, do_utf8)) - sayNO; - locinput += PL_utf8skip[nextchr]; - while (locinput < PL_regeol && - swash_fetch(PL_utf8_mark,(U8*)locinput, do_utf8)) - locinput += UTF8SKIP(locinput); - if (locinput > PL_regeol) - sayNO; - } - else - locinput++; nextchr = UCHARAT(locinput); break; @@ -3700,6 +3890,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) OP_4tree * const oop = PL_op; COP * const ocurcop = PL_curcop; PAD *old_comppad; + char *saved_regeol = PL_regeol; n = ARG(scan); PL_op = (OP_4tree*)rexi->data->data[n]; @@ -3725,6 +3916,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) PL_op = oop; PAD_RESTORE_LOCAL(old_comppad); PL_curcop = ocurcop; + PL_regeol = saved_regeol; if (!logical) { /* /(?{...})/ */ sv_setsv(save_scalar(PL_replgv), ret); @@ -3772,7 +3964,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) assert(rx); } if (rx) { - rx = reg_temp_copy(rx); + rx = reg_temp_copy(NULL, rx); } else { U32 pm_flags = 0; @@ -3878,9 +4070,12 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) regcpblow(ST.cp); cur_eval = ST.prev_eval; cur_curlyx = ST.prev_curlyx; - + + /* rex was changed so update the pointer in PL_reglastparen and PL_reglastcloseparen */ PL_reglastparen = &rex->lastparen; PL_reglastcloseparen = &rex->lastcloseparen; + /* also update PL_regoffs */ + PL_regoffs = rex->offs; /* XXXX This is too dramatic a measure... */ PL_reg_maxiter = 0; @@ -3896,6 +4091,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) SETREX(rex_sv,ST.prev_rex); rex = (struct regexp *)SvANY(rex_sv); rexi = RXi_GET(rex); + /* rex was changed so update the pointer in PL_reglastparen and PL_reglastcloseparen */ PL_reglastparen = &rex->lastparen; PL_reglastcloseparen = &rex->lastcloseparen; @@ -4391,7 +4587,7 @@ NULL case CURLYM: /* /A{m,n}B/ where A is fixed-length */ /* This is an optimisation of CURLYX that enables us to push - * only a single backtracking state, no matter now many matches + * only a single backtracking state, no matter how many matches * there are in {m,n}. It relies on the pattern being constant * length, with no parens to influence future backrefs */ @@ -4458,8 +4654,11 @@ NULL cur_eval->u.eval.close_paren == (U32)ST.me->flags) goto fake_end; - if ( ST.count < (ST.minmod ? ARG1(ST.me) : ARG2(ST.me)) ) - goto curlym_do_A; /* try to match another A */ + { + I32 max = (ST.minmod ? ARG1(ST.me) : ARG2(ST.me)); + if ( max == REG_INFTY || ST.count < max ) + goto curlym_do_A; /* try to match another A */ + } goto curlym_do_B; /* try to match B */ case CURLYM_A_fail: /* just failed to match an A */ @@ -4551,7 +4750,8 @@ NULL case CURLYM_B_fail: /* just failed to match a B */ REGCP_UNWIND(ST.cp); if (ST.minmod) { - if (ST.count == ARG2(ST.me) /* max */) + I32 max = ARG2(ST.me); + if (max != REG_INFTY && ST.count == max) sayNO; goto curlym_do_A; /* try to match a further A */ } @@ -4898,6 +5098,11 @@ NULL cur_curlyx = cur_eval->u.eval.prev_curlyx; ReREFCNT_inc(rex_sv); st->u.eval.cp = regcppush(0); /* Save *all* the positions. */ + + /* rex was changed so update the pointer in PL_reglastparen and PL_reglastcloseparen */ + PL_reglastparen = &rex->lastparen; + PL_reglastcloseparen = &rex->lastcloseparen; + REGCP_SET(st->u.eval.lastcp); PL_reginput = locinput; @@ -4979,6 +5184,8 @@ NULL do_ifmatch: ST.me = scan; ST.logical = logical; + logical = 0; /* XXX: reset state of logical once it has been saved into ST */ + /* execute body of (?...A) */ PUSH_YES_STATE_GOTO(IFMATCH_A, NEXTOPER(NEXTOPER(scan))); /* NOTREACHED */ @@ -5741,8 +5948,10 @@ S_reginclass(pTHX_ const regexp *prog, register const regnode *n, register const if (do_utf8 && !UTF8_IS_INVARIANT(c)) { c = utf8n_to_uvchr(p, UTF8_MAXBYTES, &len, - (UTF8_ALLOW_DEFAULT & UTF8_ALLOW_ANYUV) | UTF8_CHECK_ONLY); - /* see [perl #37836] for UTF8_ALLOW_ANYUV */ + (UTF8_ALLOW_DEFAULT & UTF8_ALLOW_ANYUV) + | UTF8_ALLOW_FFFF | UTF8_CHECK_ONLY); + /* see [perl #37836] for UTF8_ALLOW_ANYUV; [perl #38293] for + * UTF8_ALLOW_FFFF */ if (len == (STRLEN)-1) Perl_croak(aTHX_ "Malformed UTF-8 character (fatal)"); } @@ -5762,7 +5971,14 @@ S_reginclass(pTHX_ const regexp *prog, register const regnode *n, register const SV * const sw = regclass_swash(prog, n, TRUE, 0, (SV**)&av); if (sw) { - if (swash_fetch(sw, p, do_utf8)) + U8 * utf8_p; + if (do_utf8) { + utf8_p = (U8 *) p; + } else { + STRLEN len = 1; + utf8_p = bytes_to_utf8(p, &len); + } + if (swash_fetch(sw, utf8_p, 1)) match = TRUE; else if (flags & ANYOF_FOLD) { if (!match && lenp && av) { @@ -5771,8 +5987,7 @@ S_reginclass(pTHX_ const regexp *prog, register const regnode *n, register const SV* const sv = *av_fetch(av, i, FALSE); STRLEN len; const char * const s = SvPV_const(sv, len); - - if (len <= plen && memEQ(s, (char*)p, len)) { + if (len <= plen && memEQ(s, (char*)utf8_p, len)) { *lenp = len; match = TRUE; break; @@ -5781,13 +5996,16 @@ S_reginclass(pTHX_ const regexp *prog, register const regnode *n, register const } if (!match) { U8 tmpbuf[UTF8_MAXBYTES_CASE+1]; - STRLEN tmplen; - to_utf8_fold(p, tmpbuf, &tmplen); - if (swash_fetch(sw, tmpbuf, do_utf8)) + STRLEN tmplen; + to_utf8_fold(utf8_p, tmpbuf, &tmplen); + if (swash_fetch(sw, tmpbuf, 1)) match = TRUE; } } + + /* If we allocated a string above, free it */ + if (! do_utf8) Safefree(utf8_p); } } if (match && lenp && *lenp == 0)