X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/c269f40313641284eb670ea37200c7fcb6caec05..9ac792f4c8d84f4e7dc8fa8d1eb254f0389b3b38:/regexec.c diff --git a/regexec.c b/regexec.c index a47bfc7..e6d3fa4 100644 --- a/regexec.c +++ b/regexec.c @@ -80,7 +80,7 @@ # include "regcomp.h" #endif -#define RF_tainted 1 /* tainted information used? */ +#define RF_tainted 1 /* tainted information used? e.g. locale */ #define RF_warned 2 /* warned about big count? */ #define RF_utf8 8 /* Pattern contains multibyte chars? */ @@ -94,9 +94,9 @@ #define STATIC static #endif -/* Valid for non-utf8 strings only: avoids the reginclass call if there are no - * complications: i.e., if everything matchable is straight forward in the - * bitmap */ +/* Valid for non-utf8 strings, non-ANYOFV nodes only: avoids the reginclass + * call if there are no complications: i.e., if everything matchable is + * straight forward in the bitmap */ #define REGINCLASS(prog,p,c) (ANYOF_FLAGS(p) ? reginclass(prog,p,c,0,0) \ : ANYOF_BITMAP_TEST(p,*(c))) @@ -123,11 +123,19 @@ /* these are unrolled below in the CCC_TRY_XXX defined */ #define LOAD_UTF8_CHARCLASS(class,str) STMT_START { \ - if (!CAT2(PL_utf8_,class)) { bool ok; ENTER; save_re_context(); ok=CAT2(is_utf8_,class)((const U8*)str); assert(ok); LEAVE; } } STMT_END + if (!CAT2(PL_utf8_,class)) { \ + bool ok; \ + ENTER; save_re_context(); \ + ok=CAT2(is_utf8_,class)((const U8*)str); \ + assert(ok); assert(CAT2(PL_utf8_,class)); LEAVE; } } STMT_END /* Doesn't do an assert to verify that is correct */ #define LOAD_UTF8_CHARCLASS_NO_CHECK(class) STMT_START { \ - if (!CAT2(PL_utf8_,class)) { bool throw_away; ENTER; save_re_context(); throw_away = CAT2(is_utf8_,class)((const U8*)" "); LEAVE; } } STMT_END + if (!CAT2(PL_utf8_,class)) { \ + bool throw_away __attribute__unused__; \ + ENTER; save_re_context(); \ + throw_away = CAT2(is_utf8_,class)((const U8*)" "); \ + LEAVE; } } STMT_END #define LOAD_UTF8_CHARCLASS_ALNUM() LOAD_UTF8_CHARCLASS(alnum,"a") #define LOAD_UTF8_CHARCLASS_DIGIT() LOAD_UTF8_CHARCLASS(digit,"0") @@ -150,35 +158,7 @@ LOAD_UTF8_CHARCLASS_NO_CHECK(X_T); /* U+11A8 "\xe1\x86\xa8" */ \ LOAD_UTF8_CHARCLASS_NO_CHECK(X_V) /* U+1160 "\xe1\x85\xa0" */ -/* - We dont use PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS as the direct test - so that it is possible to override the option here without having to - rebuild the entire core. as we are required to do if we change regcomp.h - which is where PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS is defined. -*/ -#if PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS -#define BROKEN_UNICODE_CHARCLASS_MAPPINGS -#endif - -#ifdef BROKEN_UNICODE_CHARCLASS_MAPPINGS -#define LOAD_UTF8_CHARCLASS_PERL_WORD() LOAD_UTF8_CHARCLASS_ALNUM() -#define LOAD_UTF8_CHARCLASS_PERL_SPACE() LOAD_UTF8_CHARCLASS_SPACE() -#define LOAD_UTF8_CHARCLASS_POSIX_DIGIT() LOAD_UTF8_CHARCLASS_DIGIT() -#define RE_utf8_perl_word PL_utf8_alnum -#define RE_utf8_perl_space PL_utf8_space -#define RE_utf8_posix_digit PL_utf8_digit -#define perl_word alnum -#define perl_space space -#define posix_digit digit -#else -#define LOAD_UTF8_CHARCLASS_PERL_WORD() LOAD_UTF8_CHARCLASS(perl_word,"a") -#define LOAD_UTF8_CHARCLASS_PERL_SPACE() LOAD_UTF8_CHARCLASS(perl_space," ") -#define LOAD_UTF8_CHARCLASS_POSIX_DIGIT() LOAD_UTF8_CHARCLASS(posix_digit,"0") -#define RE_utf8_perl_word PL_utf8_perl_word -#define RE_utf8_perl_space PL_utf8_perl_space -#define RE_utf8_posix_digit PL_utf8_posix_digit -#endif - +#define PLACEHOLDER /* Something for the preprocessor to grab onto */ /* The actual code for CCC_TRY, which uses several variables from the routine * it's callable from. It is designed to be the bulk of a case statement. @@ -219,14 +199,15 @@ * This is because that code fails when the test succeeds, so we want to have * the test fail so that the code succeeds. The swash is stored in a * predictable PL_ place */ -#define _CCC_TRY_NONLOCALE(NAME, NNAME, FUNC, CLASS, STR) \ +#define _CCC_TRY_NONLOCALE(NAME, NNAME, FUNC, \ + CLASS, STR) \ case NAME: \ _CCC_TRY_CODE( !, FUNC, \ cBOOL(swash_fetch(CAT2(PL_utf8_,CLASS), \ (U8*)locinput, TRUE)), \ CLASS, STR) \ case NNAME: \ - _CCC_TRY_CODE( , FUNC, \ + _CCC_TRY_CODE( PLACEHOLDER , FUNC, \ cBOOL(swash_fetch(CAT2(PL_utf8_,CLASS), \ (U8*)locinput, TRUE)), \ CLASS, STR) \ @@ -243,13 +224,34 @@ * irrelevant here */ #define CCC_TRY(NAME, NNAME, FUNC, \ NAMEL, NNAMEL, LCFUNC, LCFUNC_utf8, \ + NAMEA, NNAMEA, FUNCA, \ CLASS, STR) \ case NAMEL: \ PL_reg_flags |= RF_tainted; \ _CCC_TRY_CODE( !, LCFUNC, LCFUNC_utf8((U8*)locinput), CLASS, STR) \ case NNAMEL: \ PL_reg_flags |= RF_tainted; \ - _CCC_TRY_CODE( , LCFUNC, LCFUNC_utf8((U8*)locinput), CLASS, STR) \ + _CCC_TRY_CODE( PLACEHOLDER, LCFUNC, LCFUNC_utf8((U8*)locinput), \ + CLASS, STR) \ + case NAMEA: \ + if (locinput >= PL_regeol || ! FUNCA(nextchr)) { \ + sayNO; \ + } \ + /* Matched a utf8-invariant, so don't have to worry about utf8 */ \ + nextchr = UCHARAT(++locinput); \ + break; \ + case NNAMEA: \ + if (locinput >= PL_regeol || FUNCA(nextchr)) { \ + sayNO; \ + } \ + if (utf8_target) { \ + locinput += PL_utf8skip[nextchr]; \ + nextchr = UCHARAT(locinput); \ + } \ + else { \ + nextchr = UCHARAT(++locinput); \ + } \ + break; \ /* Generate the non-locale cases */ \ _CCC_TRY_NONLOCALE(NAME, NNAME, FUNC, CLASS, STR) @@ -258,8 +260,12 @@ #define CCC_TRY_U(NAME, NNAME, FUNC, \ NAMEL, NNAMEL, LCFUNC, LCFUNC_utf8, \ NAMEU, NNAMEU, FUNCU, \ + NAMEA, NNAMEA, FUNCA, \ CLASS, STR) \ - CCC_TRY(NAME, NNAME, FUNC, NAMEL, NNAMEL, LCFUNC, LCFUNC_utf8, CLASS, STR) \ + CCC_TRY(NAME, NNAME, FUNC, \ + NAMEL, NNAMEL, LCFUNC, LCFUNC_utf8, \ + NAMEA, NNAMEA, FUNCA, \ + CLASS, STR) \ _CCC_TRY_NONLOCALE(NAMEU, NNAMEU, FUNCU, CLASS, STR) /* TODO: Combine JUMPABLE and HAS_TEXT to cache OP(rn) */ @@ -291,13 +297,13 @@ /* Currently these are only used when PL_regkind[OP(rn)] == EXACT so we don't need this definition. */ #define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==REF || OP(rn)==NREF ) -#define IS_TEXTF(rn) ( (OP(rn)==EXACTFU || OP(rn)==EXACTF) || OP(rn)==REFF || OP(rn)==NREFF ) +#define IS_TEXTF(rn) ( (OP(rn)==EXACTFU || OP(rn)==EXACTFA || OP(rn)==EXACTF) || OP(rn)==REFF || OP(rn)==NREFF ) #define IS_TEXTFL(rn) ( OP(rn)==EXACTFL || OP(rn)==REFFL || OP(rn)==NREFFL ) #else /* ... so we use this as its faster. */ #define IS_TEXT(rn) ( OP(rn)==EXACT ) -#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU ) +#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn) == EXACTFA) #define IS_TEXTF(rn) ( OP(rn)==EXACTF ) #define IS_TEXTFL(rn) ( OP(rn)==EXACTFL ) @@ -686,11 +692,26 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, (IV)prog->check_end_shift); }); - if (flags & REXEC_SCREAM) { + if ((flags & REXEC_SCREAM) && SvSCREAM(sv)) { I32 p = -1; /* Internal iterator of scream. */ I32 * const pp = data ? data->scream_pos : &p; + const MAGIC *mg; + bool found = FALSE; - if (PL_screamfirst[BmRARE(check)] >= 0 + assert(SvMAGICAL(sv)); + mg = mg_find(sv, PERL_MAGIC_study); + assert(mg); + + if (mg->mg_private == 1) { + found = ((U8 *)mg->mg_ptr)[BmRARE(check)] != (U8)~0; + } else if (mg->mg_private == 2) { + found = ((U16 *)mg->mg_ptr)[BmRARE(check)] != (U16)~0; + } else { + assert (mg->mg_private == 4); + found = ((U32 *)mg->mg_ptr)[BmRARE(check)] != (U32)~0; + } + + if (found || ( BmRARE(check) == '\n' && (BmPREVIOUS(check) == SvCUR(check) - 1) && SvTAIL(check) )) @@ -1049,7 +1070,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, even for \b or \B. But (minlen? 1 : 0) below assumes that regstclass does not come from lookahead... */ /* If regstclass takes bytelength more than 1: If charlength==1, OK. - This leaves EXACTF, EXACTFU only, which are dealt with in find_byclass(). */ + This leaves EXACTF-ish only, which are dealt with in find_byclass(). */ const U8* const str = (U8*)STRING(progi->regstclass); const int cl_l = (PL_regkind[OP(progi->regstclass)] == EXACT ? CHR_DIST(str+STR_LEN(progi->regstclass), str) @@ -1213,43 +1234,11 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ } \ } STMT_END -#define REXEC_FBC_EXACTISH_CHECK(CoNd) \ -{ \ - char *my_strend= (char *)strend; \ - if ( (CoNd) \ - && (ln == len || \ - foldEQ_utf8(s, &my_strend, 0, utf8_target, \ - m, NULL, ln, cBOOL(UTF_PATTERN))) \ - && (!reginfo || regtry(reginfo, &s)) ) \ - goto got_it; \ - else { \ - U8 foldbuf[UTF8_MAXBYTES_CASE+1]; \ - uvchr_to_utf8(tmpbuf, c); \ - f = to_utf8_fold(tmpbuf, foldbuf, &foldlen); \ - if ( f != c \ - && (f == c1 || f == c2) \ - && (ln == len || \ - foldEQ_utf8(s, &my_strend, 0, utf8_target,\ - m, NULL, ln, cBOOL(UTF_PATTERN)))\ - && (!reginfo || regtry(reginfo, &s)) ) \ - goto got_it; \ - } \ -} \ -s += len - #define REXEC_FBC_EXACTISH_SCAN(CoNd) \ STMT_START { \ - re_fold_t folder; \ - switch (OP(c)) { \ - case EXACTFU: folder = foldEQ_latin1; break; \ - case EXACTFL: folder = foldEQ_locale; break; \ - case EXACTF: folder = foldEQ; break; \ - default: \ - Perl_croak(aTHX_ "panic: Unexpected op %u", OP(c)); \ - } \ while (s <= e) { \ if ( (CoNd) \ - && (ln == 1 || folder(s, m, ln)) \ + && (ln == 1 || folder(s, pat_string, ln)) \ && (!reginfo || regtry(reginfo, &s)) ) \ goto got_it; \ s++; \ @@ -1306,8 +1295,7 @@ if ((!reginfo || regtry(reginfo, &s))) \ } \ else { \ REXEC_FBC_CLASS_SCAN(CoNd); \ - } \ - break + } #define REXEC_FBC_CSCAN_PRELOAD(UtFpReLoAd,CoNdUtF8,CoNd) \ if (utf8_target) { \ @@ -1316,8 +1304,7 @@ if ((!reginfo || regtry(reginfo, &s))) \ } \ else { \ REXEC_FBC_CLASS_SCAN(CoNd); \ - } \ - break + } #define REXEC_FBC_CSCAN_TAINT(CoNdUtF8,CoNd) \ PL_reg_flags |= RF_tainted; \ @@ -1326,12 +1313,91 @@ if ((!reginfo || regtry(reginfo, &s))) \ } \ else { \ REXEC_FBC_CLASS_SCAN(CoNd); \ - } \ - break + } #define DUMP_EXEC_POS(li,s,doutf8) \ dump_exec_pos(li,s,(PL_regeol),(PL_bostr),(PL_reg_starttry),doutf8) + +#define UTF8_NOLOAD(TEST_NON_UTF8, IF_SUCCESS, IF_FAIL) \ + tmp = (s != PL_bostr) ? UCHARAT(s - 1) : '\n'; \ + tmp = TEST_NON_UTF8(tmp); \ + REXEC_FBC_UTF8_SCAN( \ + if (tmp == ! TEST_NON_UTF8((U8) *s)) { \ + tmp = !tmp; \ + IF_SUCCESS; \ + } \ + else { \ + IF_FAIL; \ + } \ + ); \ + +#define UTF8_LOAD(TeSt1_UtF8, TeSt2_UtF8, IF_SUCCESS, IF_FAIL) \ + if (s == PL_bostr) { \ + tmp = '\n'; \ + } \ + else { \ + U8 * const r = reghop3((U8*)s, -1, (U8*)PL_bostr); \ + tmp = utf8n_to_uvchr(r, UTF8SKIP(r), 0, UTF8_ALLOW_DEFAULT); \ + } \ + tmp = TeSt1_UtF8; \ + LOAD_UTF8_CHARCLASS_ALNUM(); \ + REXEC_FBC_UTF8_SCAN( \ + if (tmp == ! (TeSt2_UtF8)) { \ + tmp = !tmp; \ + IF_SUCCESS; \ + } \ + else { \ + IF_FAIL; \ + } \ + ); \ + +/* The only difference between the BOUND and NBOUND cases is that + * REXEC_FBC_TRYIT is called when matched in BOUND, and when non-matched in + * NBOUND. This is accomplished by passing it in either the if or else clause, + * with the other one being empty */ +#define FBC_BOUND(TEST_NON_UTF8, TEST1_UTF8, TEST2_UTF8) \ + FBC_BOUND_COMMON(UTF8_LOAD(TEST1_UTF8, TEST2_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER), TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER) + +#define FBC_BOUND_NOLOAD(TEST_NON_UTF8, TEST1_UTF8, TEST2_UTF8) \ + FBC_BOUND_COMMON(UTF8_NOLOAD(TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER), TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER) + +#define FBC_NBOUND(TEST_NON_UTF8, TEST1_UTF8, TEST2_UTF8) \ + FBC_BOUND_COMMON(UTF8_LOAD(TEST1_UTF8, TEST2_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT), TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT) + +#define FBC_NBOUND_NOLOAD(TEST_NON_UTF8, TEST1_UTF8, TEST2_UTF8) \ + FBC_BOUND_COMMON(UTF8_NOLOAD(TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT), TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT) + + +/* Common to the BOUND and NBOUND cases. Unfortunately the UTF8 tests need to + * be passed in completely with the variable name being tested, which isn't + * such a clean interface, but this is easier to read than it was before. We + * are looking for the boundary (or non-boundary between a word and non-word + * character. The utf8 and non-utf8 cases have the same logic, but the details + * must be different. Find the "wordness" of the character just prior to this + * one, and compare it with the wordness of this one. If they differ, we have + * a boundary. At the beginning of the string, pretend that the previous + * character was a new-line */ +#define FBC_BOUND_COMMON(UTF8_CODE, TEST_NON_UTF8, IF_SUCCESS, IF_FAIL) \ + if (utf8_target) { \ + UTF8_CODE \ + } \ + else { /* Not utf8 */ \ + tmp = (s != PL_bostr) ? UCHARAT(s - 1) : '\n'; \ + tmp = TEST_NON_UTF8(tmp); \ + REXEC_FBC_SCAN( \ + if (tmp == ! TEST_NON_UTF8((U8) *s)) { \ + tmp = !tmp; \ + IF_SUCCESS; \ + } \ + else { \ + IF_FAIL; \ + } \ + ); \ + } \ + if ((!prog->minlen && tmp) && (!reginfo || regtry(reginfo, &s))) \ + goto got_it; + /* We know what class REx starts with. Try to find this position... */ /* if reginfo is NULL, its a dryrun */ /* annoyingly all the vars in this routine have different names from their counterparts @@ -1343,15 +1409,19 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, { dVAR; const I32 doevery = (prog->intflags & PREGf_SKIP) == 0; - char *m; + char *pat_string; /* The pattern's exactish string */ + char *pat_end; /* ptr to end char of pat_string */ + re_fold_t folder; /* Function for computing non-utf8 folds */ + const U8 *fold_array; /* array for folding ords < 256 */ STRLEN ln; STRLEN lnc; register STRLEN uskip; - unsigned int c1; - unsigned int c2; + U8 c1; + U8 c2; char *e; register I32 tmp = 1; /* Scratch variable? */ register const bool utf8_target = PL_reg_match_utf8; + UV utf8_fold_flags = 0; RXi_GET_DECL(prog,progi); PERL_ARGS_ASSERT_FIND_BYCLASS; @@ -1361,29 +1431,12 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, case ANYOFV: case ANYOF: if (utf8_target || OP(c) == ANYOFV) { - REXEC_FBC_UTF8_CLASS_SCAN((ANYOF_FLAGS(c) & ANYOF_NONBITMAP) || - !UTF8_IS_INVARIANT((U8)s[0]) ? - reginclass(prog, c, (U8*)s, 0, utf8_target) : - REGINCLASS(prog, c, (U8*)s)); + STRLEN inclasslen = strend - s; + REXEC_FBC_UTF8_CLASS_SCAN( + reginclass(prog, c, (U8*)s, &inclasslen, utf8_target)); } else { - while (s < strend) { - STRLEN skip = 1; - - if (REGINCLASS(prog, c, (U8*)s) || - (ANYOF_FOLD_SHARP_S(c, s, strend) && - /* The assignment of 2 is intentional: - * for the folded sharp s, the skip is 2. */ - (skip = SHARP_S_SKIP))) { - if (tmp && (!reginfo || regtry(reginfo, &s))) - goto got_it; - else - tmp = doevery; - } - else - tmp = 1; - s += skip; - } + REXEC_FBC_CLASS_SCAN(REGINCLASS(prog, c, (U8*)s)); } break; case CANY: @@ -1394,337 +1447,316 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, tmp = doevery; ); break; + + case EXACTFA: + if (UTF_PATTERN || utf8_target) { + utf8_fold_flags = FOLDEQ_UTF8_NOMIX_ASCII; + goto do_exactf_utf8; + } + fold_array = PL_fold_latin1; /* Latin1 folds are not affected by */ + folder = foldEQ_latin1; /* /a, except the sharp s one which */ + goto do_exactf_non_utf8; /* isn't dealt with by these */ + case EXACTFU: - case EXACTF: - m = STRING(c); - ln = STR_LEN(c); /* length to match in octets/bytes */ - lnc = (I32) ln; /* length to match in characters */ - if (UTF_PATTERN) { - STRLEN ulen1, ulen2; - U8 *sm = (U8 *) m; - U8 tmpbuf1[UTF8_MAXBYTES_CASE+1]; - U8 tmpbuf2[UTF8_MAXBYTES_CASE+1]; - /* used by commented-out code below */ - /*const U32 uniflags = UTF8_ALLOW_DEFAULT;*/ - - /* XXX: Since the node will be case folded at compile - time this logic is a little odd, although im not - sure that its actually wrong. --dmq */ - - c1 = to_utf8_lower((U8*)m, tmpbuf1, &ulen1); - c2 = to_utf8_upper((U8*)m, tmpbuf2, &ulen2); - - /* XXX: This is kinda strange. to_utf8_XYZ returns the - codepoint of the first character in the converted - form, yet originally we did the extra step. - No tests fail by commenting this code out however - so Ive left it out. -- dmq. - - c1 = utf8n_to_uvchr(tmpbuf1, UTF8_MAXBYTES_CASE, - 0, uniflags); - c2 = utf8n_to_uvchr(tmpbuf2, UTF8_MAXBYTES_CASE, - 0, uniflags); - */ - - lnc = 0; - while (sm < ((U8 *) m + ln)) { - lnc++; - sm += UTF8SKIP(sm); - } + if (UTF_PATTERN || utf8_target) { + utf8_fold_flags = 0; + goto do_exactf_utf8; } - else { - c1 = *(U8*)m; - if (utf8_target || OP(c) == EXACTFU) { - - /* Micro sign folds to GREEK SMALL LETTER MU; - LATIN_SMALL_LETTER_SHARP_S folds to 'ss', and this sets - c2 to the first 's' of the pair, and the code below will - look for others */ - c2 = (c1 == MICRO_SIGN) - ? GREEK_SMALL_LETTER_MU - : (c1 == LATIN_SMALL_LETTER_SHARP_S) - ? 's' - : PL_fold_latin1[c1]; - } else c2 = PL_fold[c1]; + + /* Any 'ss' in the pattern should have been replaced by regcomp, + * so we don't have to worry here about this single special case + * in the Latin1 range */ + fold_array = PL_fold_latin1; + folder = foldEQ_latin1; + goto do_exactf_non_utf8; + + case EXACTF: + if (UTF_PATTERN || utf8_target) { + utf8_fold_flags = 0; + goto do_exactf_utf8; } - goto do_exactf; + fold_array = PL_fold; + folder = foldEQ; + goto do_exactf_non_utf8; + case EXACTFL: - m = STRING(c); - ln = STR_LEN(c); - lnc = (I32) ln; - c1 = *(U8*)m; - c2 = PL_fold_locale[c1]; - do_exactf: - e = HOP3c(strend, -((I32)lnc), s); + if (UTF_PATTERN || utf8_target) { + utf8_fold_flags = FOLDEQ_UTF8_LOCALE; + goto do_exactf_utf8; + } + fold_array = PL_fold_locale; + folder = foldEQ_locale; - if (!reginfo && e < s) - e = s; /* Due to minlen logic of intuit() */ + /* FALL THROUGH */ - /* The idea in the EXACTF* cases is to first find the - * first character of the EXACTF* node and then, if - * necessary, case-insensitively compare the full - * text of the node. The c1 and c2 are the first - * characters (though in Unicode it gets a bit - * more complicated because there are more cases - * than just upper and lower: one needs to use - * the so-called folding case for case-insensitive - * matching (called "loose matching" in Unicode). - * foldEQ_utf8() will do just that. */ + do_exactf_non_utf8: /* Neither pattern nor string are UTF8 */ - if (utf8_target || UTF_PATTERN) { - UV c, f; - U8 tmpbuf [UTF8_MAXBYTES+1]; - STRLEN len = 1; - STRLEN foldlen; - const U32 uniflags = UTF8_ALLOW_DEFAULT; - if (c1 == c2) { - /* Upper and lower of 1st char are equal - - * probably not a "letter". */ - while (s <= e) { - if (utf8_target) { - c = utf8n_to_uvchr((U8*)s, UTF8_MAXBYTES, &len, - uniflags); - } else { - c = *((U8*)s); - } - REXEC_FBC_EXACTISH_CHECK(c == c1); - } - } - else { - while (s <= e) { - if (utf8_target) { - c = utf8n_to_uvchr((U8*)s, UTF8_MAXBYTES, &len, - uniflags); - } else { - c = *((U8*)s); - } + /* The idea in the non-utf8 EXACTF* cases is to first find the + * first character of the EXACTF* node and then, if necessary, + * case-insensitively compare the full text of the node. c1 is the + * first character. c2 is its fold. This logic will not work for + * Unicode semantics and the german sharp ss, which hence should + * not be compiled into a node that gets here. */ + pat_string = STRING(c); + ln = STR_LEN(c); /* length to match in octets/bytes */ - /* Handle some of the three Greek sigmas cases. - * Note that not all the possible combinations - * are handled here: some of them are handled - * by the standard folding rules, and some of - * them (the character class or ANYOF cases) - * are handled during compiletime in - * regexec.c:S_regclass(). */ - if (c == (UV)UNICODE_GREEK_CAPITAL_LETTER_SIGMA || - c == (UV)UNICODE_GREEK_SMALL_LETTER_FINAL_SIGMA) - c = (UV)UNICODE_GREEK_SMALL_LETTER_SIGMA; - - REXEC_FBC_EXACTISH_CHECK(c == c1 || c == c2); - } - } + e = HOP3c(strend, -((I32)ln), s); + + if (!reginfo && e < s) { + e = s; /* Due to minlen logic of intuit() */ + } + + c1 = *pat_string; + c2 = fold_array[c1]; + if (c1 == c2) { /* If char and fold are the same */ + REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1); } else { - /* Neither pattern nor string are UTF8 */ - if (c1 == c2) - REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1); - else - REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1 || *(U8*)s == c2); + REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1 || *(U8*)s == c2); } break; - case BOUNDL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case BOUND: - if (utf8_target) { - if (s == PL_bostr) - tmp = '\n'; - else { - U8 * const r = reghop3((U8*)s, -1, (U8*)PL_bostr); - tmp = utf8n_to_uvchr(r, UTF8SKIP(r), 0, UTF8_ALLOW_DEFAULT); - } - tmp = ((OP(c) == BOUND ? - isALNUM_uni(tmp) : isALNUM_LC_uvchr(UNI_TO_NATIVE(tmp))) != 0); - LOAD_UTF8_CHARCLASS_ALNUM(); - REXEC_FBC_UTF8_SCAN( - if (tmp == !(OP(c) == BOUND ? - cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target)) : - isALNUM_LC_utf8((U8*)s))) - { - tmp = !tmp; - REXEC_FBC_TRYIT; - } - ); + + do_exactf_utf8: + + /* If one of the operands is in utf8, we can't use the simpler + * folding above, due to the fact that many different characters + * can have the same fold, or portion of a fold, or different- + * length fold */ + pat_string = STRING(c); + ln = STR_LEN(c); /* length to match in octets/bytes */ + pat_end = pat_string + ln; + lnc = (UTF_PATTERN) /* length to match in characters */ + ? utf8_length((U8 *) pat_string, (U8 *) pat_end) + : ln; + + e = HOP3c(strend, -((I32)lnc), s); + + if (!reginfo && e < s) { + e = s; /* Due to minlen logic of intuit() */ } - else { /* Not utf8 */ - tmp = (s != PL_bostr) ? UCHARAT(s - 1) : '\n'; - tmp = cBOOL((OP(c) == BOUNDL) - ? isALNUM_LC(tmp) - : (isWORDCHAR_L1(tmp) - && (isASCII(tmp) || (FLAGS(c) == REGEX_UNICODE_CHARSET)))); - REXEC_FBC_SCAN( - if (tmp == - !((OP(c) == BOUNDL) - ? isALNUM_LC(*s) - : (isWORDCHAR_L1((U8) *s) - && (isASCII((U8) *s) || (FLAGS(c) == REGEX_UNICODE_CHARSET))))) - { - tmp = !tmp; - REXEC_FBC_TRYIT; + + while (s <= e) { + char *my_strend= (char *)strend; + if (foldEQ_utf8_flags(s, &my_strend, 0, utf8_target, + pat_string, NULL, ln, cBOOL(UTF_PATTERN), utf8_fold_flags) + && (!reginfo || regtry(reginfo, &s)) ) + { + goto got_it; } - ); + s += UTF8SKIP(s); } - if ((!prog->minlen && tmp) && (!reginfo || regtry(reginfo, &s))) - goto got_it; + break; + case BOUNDL: + PL_reg_flags |= RF_tainted; + FBC_BOUND(isALNUM_LC, + isALNUM_LC_uvchr(UNI_TO_NATIVE(tmp)), + isALNUM_LC_utf8((U8*)s)); break; case NBOUNDL: PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ + FBC_NBOUND(isALNUM_LC, + isALNUM_LC_uvchr(UNI_TO_NATIVE(tmp)), + isALNUM_LC_utf8((U8*)s)); + break; + case BOUND: + FBC_BOUND(isWORDCHAR, + isALNUM_uni(tmp), + cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target))); + break; + case BOUNDA: + FBC_BOUND_NOLOAD(isWORDCHAR_A, + isWORDCHAR_A(tmp), + isWORDCHAR_A((U8*)s)); + break; case NBOUND: - if (utf8_target) { - if (s == PL_bostr) - tmp = '\n'; - else { - U8 * const r = reghop3((U8*)s, -1, (U8*)PL_bostr); - tmp = utf8n_to_uvchr(r, UTF8SKIP(r), 0, UTF8_ALLOW_DEFAULT); - } - tmp = ((OP(c) == NBOUND ? - isALNUM_uni(tmp) : isALNUM_LC_uvchr(UNI_TO_NATIVE(tmp))) != 0); - LOAD_UTF8_CHARCLASS_ALNUM(); - REXEC_FBC_UTF8_SCAN( - if (tmp == !(OP(c) == NBOUND ? - cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target)) : - isALNUM_LC_utf8((U8*)s))) - tmp = !tmp; - else REXEC_FBC_TRYIT; - ); - } - else { - tmp = (s != PL_bostr) ? UCHARAT(s - 1) : '\n'; - tmp = cBOOL((OP(c) == NBOUNDL) - ? isALNUM_LC(tmp) - : (isWORDCHAR_L1(tmp) - && (isASCII(tmp) || (FLAGS(c) == REGEX_UNICODE_CHARSET)))); - REXEC_FBC_SCAN( - if (tmp == ! cBOOL( - (OP(c) == NBOUNDL) - ? isALNUM_LC(*s) - : (isWORDCHAR_L1((U8) *s) - && (isASCII((U8) *s) || (FLAGS(c) == REGEX_UNICODE_CHARSET))))) - { - tmp = !tmp; - } - else REXEC_FBC_TRYIT; - ); - } - if ((!prog->minlen && !tmp) && (!reginfo || regtry(reginfo, &s))) - goto got_it; + FBC_NBOUND(isWORDCHAR, + isALNUM_uni(tmp), + cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target))); + break; + case NBOUNDA: + FBC_NBOUND_NOLOAD(isWORDCHAR_A, + isWORDCHAR_A(tmp), + isWORDCHAR_A((U8*)s)); + break; + case BOUNDU: + FBC_BOUND(isWORDCHAR_L1, + isALNUM_uni(tmp), + cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target))); + break; + case NBOUNDU: + FBC_NBOUND(isWORDCHAR_L1, + isALNUM_uni(tmp), + cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target))); break; case ALNUML: REXEC_FBC_CSCAN_TAINT( isALNUM_LC_utf8((U8*)s), isALNUM_LC(*s) ); + break; case ALNUMU: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_WORD(), - swash_fetch(RE_utf8_perl_word,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_ALNUM(), + swash_fetch(PL_utf8_alnum,(U8*)s, utf8_target), isWORDCHAR_L1((U8) *s) ); + break; case ALNUM: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_WORD(), - swash_fetch(RE_utf8_perl_word,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_ALNUM(), + swash_fetch(PL_utf8_alnum,(U8*)s, utf8_target), isWORDCHAR((U8) *s) ); + break; + case ALNUMA: + /* Don't need to worry about utf8, as it can match only a single + * byte invariant character */ + REXEC_FBC_CLASS_SCAN( isWORDCHAR_A(*s)); + break; case NALNUMU: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_WORD(), - swash_fetch(RE_utf8_perl_word,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_ALNUM(), + !swash_fetch(PL_utf8_alnum,(U8*)s, utf8_target), ! isWORDCHAR_L1((U8) *s) ); + break; case NALNUM: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_WORD(), - !swash_fetch(RE_utf8_perl_word, (U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_ALNUM(), + !swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target), ! isALNUM(*s) ); + break; + case NALNUMA: + REXEC_FBC_CSCAN( + !isWORDCHAR_A(*s), + !isWORDCHAR_A(*s) + ); + break; case NALNUML: REXEC_FBC_CSCAN_TAINT( !isALNUM_LC_utf8((U8*)s), !isALNUM_LC(*s) ); + break; case SPACEU: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_SPACE(), - *s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_SPACE(), + *s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, utf8_target), isSPACE_L1((U8) *s) ); + break; case SPACE: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_SPACE(), - *s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_SPACE(), + *s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, utf8_target), isSPACE((U8) *s) ); + break; + case SPACEA: + /* Don't need to worry about utf8, as it can match only a single + * byte invariant character */ + REXEC_FBC_CLASS_SCAN( isSPACE_A(*s)); + break; case SPACEL: REXEC_FBC_CSCAN_TAINT( isSPACE_LC_utf8((U8*)s), isSPACE_LC(*s) ); + break; case NSPACEU: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_SPACE(), - !( *s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, utf8_target)), + LOAD_UTF8_CHARCLASS_SPACE(), + !( *s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, utf8_target)), ! isSPACE_L1((U8) *s) ); + break; case NSPACE: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_PERL_SPACE(), - !(*s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, utf8_target)), + LOAD_UTF8_CHARCLASS_SPACE(), + !(*s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, utf8_target)), ! isSPACE((U8) *s) ); + break; + case NSPACEA: + REXEC_FBC_CSCAN( + !isSPACE_A(*s), + !isSPACE_A(*s) + ); + break; case NSPACEL: REXEC_FBC_CSCAN_TAINT( !isSPACE_LC_utf8((U8*)s), !isSPACE_LC(*s) ); + break; case DIGIT: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_POSIX_DIGIT(), - swash_fetch(RE_utf8_posix_digit,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_DIGIT(), + swash_fetch(PL_utf8_digit,(U8*)s, utf8_target), isDIGIT(*s) ); + break; + case DIGITA: + /* Don't need to worry about utf8, as it can match only a single + * byte invariant character */ + REXEC_FBC_CLASS_SCAN( isDIGIT_A(*s)); + break; case DIGITL: REXEC_FBC_CSCAN_TAINT( isDIGIT_LC_utf8((U8*)s), isDIGIT_LC(*s) ); + break; case NDIGIT: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_POSIX_DIGIT(), - !swash_fetch(RE_utf8_posix_digit,(U8*)s, utf8_target), + LOAD_UTF8_CHARCLASS_DIGIT(), + !swash_fetch(PL_utf8_digit,(U8*)s, utf8_target), !isDIGIT(*s) ); + break; + case NDIGITA: + REXEC_FBC_CSCAN( + !isDIGIT_A(*s), + !isDIGIT_A(*s) + ); + break; case NDIGITL: REXEC_FBC_CSCAN_TAINT( !isDIGIT_LC_utf8((U8*)s), !isDIGIT_LC(*s) ); + break; case LNBREAK: REXEC_FBC_CSCAN( is_LNBREAK_utf8(s), is_LNBREAK_latin1(s) ); + break; case VERTWS: REXEC_FBC_CSCAN( is_VERTWS_utf8(s), is_VERTWS_latin1(s) ); + break; case NVERTWS: REXEC_FBC_CSCAN( !is_VERTWS_utf8(s), !is_VERTWS_latin1(s) ); + break; case HORIZWS: REXEC_FBC_CSCAN( is_HORIZWS_utf8(s), is_HORIZWS_latin1(s) ); + break; case NHORIZWS: REXEC_FBC_CSCAN( !is_HORIZWS_utf8(s), !is_HORIZWS_latin1(s) ); + break; case AHOCORASICKC: case AHOCORASICK: { @@ -2272,7 +2304,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre dontbother = end_shift; strend = HOPc(strend, -dontbother); while ( (s <= last) && - ((flags & REXEC_SCREAM) + ((flags & REXEC_SCREAM) && SvSCREAM(sv) ? (s = screaminstr(sv, must, HOP3c(s, back_min, (back_min<0 ? strbeg : strend)) - strbeg, end_shift, &scream_pos, 0)) : (s = fbm_instr((unsigned char*)HOP3(s, back_min, (back_min<0 ? strbeg : strend)), @@ -2351,7 +2383,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre utf8_target ? to_utf8_substr(prog) : to_byte_substr(prog); float_real = utf8_target ? prog->float_utf8 : prog->float_substr; - if (flags & REXEC_SCREAM) { + if ((flags & REXEC_SCREAM) && SvSCREAM(sv)) { last = screaminstr(sv, float_real, s - strbeg, end_shift, &scream_pos, 1); /* last one */ if (!last) @@ -2545,7 +2577,7 @@ S_regtry(pTHX_ regmatch_info *reginfo, char **startpos) /* This is safe against NULLs: */ ReREFCNT_dec(PM_GETRE(PL_reg_curpm)); /* PM_reg_curpm owns a reference to this regexp. */ - ReREFCNT_inc(rx); + (void)ReREFCNT_inc(rx); #endif PM_SETRE(PL_reg_curpm, rx); PL_reg_oldcurpm = PL_curpm; @@ -3580,20 +3612,30 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) re_fold_t folder; const U8 * fold_array; const char * s; + U32 fold_utf8_flags; PL_reg_flags |= RF_tainted; folder = foldEQ_locale; fold_array = PL_fold_locale; + fold_utf8_flags = FOLDEQ_UTF8_LOCALE; goto do_exactf; case EXACTFU: folder = foldEQ_latin1; fold_array = PL_fold_latin1; + fold_utf8_flags = 0; + goto do_exactf; + + case EXACTFA: + folder = foldEQ_latin1; + fold_array = PL_fold_latin1; + fold_utf8_flags = FOLDEQ_UTF8_NOMIX_ASCII; goto do_exactf; case EXACTF: folder = foldEQ; fold_array = PL_fold; + fold_utf8_flags = 0; do_exactf: s = STRING(scan); @@ -3604,30 +3646,17 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) const char * const l = locinput; char *e = PL_regeol; - if (! foldEQ_utf8(s, 0, ln, cBOOL(UTF_PATTERN), - l, &e, 0, utf8_target)) { - /* One more case for the sharp s: - * pack("U0U*", 0xDF) =~ /ss/i, - * the 0xC3 0x9F are the UTF-8 - * byte sequence for the U+00DF. */ - - if (!(utf8_target && - toLOWER(s[0]) == 's' && - ln >= 2 && - toLOWER(s[1]) == 's' && - (U8)l[0] == 0xC3 && - e - l >= 2 && - (U8)l[1] == 0x9F)) - sayNO; + if (! foldEQ_utf8_flags(s, 0, ln, cBOOL(UTF_PATTERN), + l, &e, 0, utf8_target, fold_utf8_flags)) + { + sayNO; } locinput = e; nextchr = UCHARAT(locinput); break; } - /* Neither the target and the pattern are utf8. */ - - /* Inline the first character, for speed. */ + /* Neither the target nor the pattern are utf8 */ if (UCHARAT(s) != nextchr && UCHARAT(s) != fold_array[nextchr]) { @@ -3641,14 +3670,25 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) nextchr = UCHARAT(locinput); break; } + + /* XXX Could improve efficiency by separating these all out using a + * macro or in-line function. At that point regcomp.c would no longer + * have to set the FLAGS fields of these */ case BOUNDL: case NBOUNDL: PL_reg_flags |= RF_tainted; /* FALL THROUGH */ case BOUND: + case BOUNDU: + case BOUNDA: case NBOUND: + case NBOUNDU: + case NBOUNDA: /* was last char in word? */ - if (utf8_target) { + if (utf8_target + && FLAGS(scan) != REGEX_ASCII_RESTRICTED_CHARSET + && FLAGS(scan) != REGEX_ASCII_MORE_RESTRICTED_CHARSET) + { if (locinput == PL_bostr) ln = '\n'; else { @@ -3656,7 +3696,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) ln = utf8n_to_uvchr(r, UTF8SKIP(r), 0, uniflags); } - if (OP(scan) == BOUND || OP(scan) == NBOUND) { + if (FLAGS(scan) != REGEX_LOCALE_CHARSET) { ln = isALNUM_uni(ln); LOAD_UTF8_CHARCLASS_ALNUM(); n = swash_fetch(PL_utf8_alnum, (U8*)locinput, utf8_target); @@ -3667,26 +3707,46 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) } } else { + + /* Here the string isn't utf8, or is utf8 and only ascii + * characters are to match \w. In the latter case looking at + * the byte just prior to the current one may be just the final + * byte of a multi-byte character. This is ok. There are two + * cases: + * 1) it is a single byte character, and then the test is doing + * just what it's supposed to. + * 2) it is a multi-byte character, in which case the final + * byte is never mistakable for ASCII, and so the test + * will say it is not a word character, which is the + * correct answer. */ ln = (locinput != PL_bostr) ? UCHARAT(locinput - 1) : '\n'; - if (FLAGS(scan) == REGEX_UNICODE_CHARSET) { - - /* Here, can't be BOUNDL or NBOUNDL because they never set - * the flags to REGEX_UNICODE_CHARSET */ - ln = isWORDCHAR_L1(ln); - n = isWORDCHAR_L1(nextchr); - } - else if (OP(scan) == BOUND || OP(scan) == NBOUND) { - ln = isALNUM(ln); - n = isALNUM(nextchr); - } - else { - ln = isALNUM_LC(ln); - n = isALNUM_LC(nextchr); + switch (FLAGS(scan)) { + case REGEX_UNICODE_CHARSET: + ln = isWORDCHAR_L1(ln); + n = isWORDCHAR_L1(nextchr); + break; + case REGEX_LOCALE_CHARSET: + ln = isALNUM_LC(ln); + n = isALNUM_LC(nextchr); + break; + case REGEX_DEPENDS_CHARSET: + ln = isALNUM(ln); + n = isALNUM(nextchr); + break; + case REGEX_ASCII_RESTRICTED_CHARSET: + case REGEX_ASCII_MORE_RESTRICTED_CHARSET: + ln = isWORDCHAR_A(ln); + n = isWORDCHAR_A(nextchr); + break; + default: + Perl_croak(aTHX_ "panic: Unexpected FLAGS %u in op %u", FLAGS(scan), OP(scan)); + break; } } - if (((!ln) == (!n)) == (OP(scan) == BOUND || - OP(scan) == BOUNDL)) + /* Note requires that all BOUNDs be lower than all NBOUNDs in + * regcomp.sym */ + if (((!ln) == (!n)) == (OP(scan) < NBOUND)) sayNO; break; case ANYOFV: @@ -3717,16 +3777,19 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) CCC_TRY_U(ALNUM, NALNUM, isWORDCHAR, ALNUML, NALNUML, isALNUM_LC, isALNUM_LC_utf8, ALNUMU, NALNUMU, isWORDCHAR_L1, - perl_word, "a"); + ALNUMA, NALNUMA, isWORDCHAR_A, + alnum, "a"); CCC_TRY_U(SPACE, NSPACE, isSPACE, SPACEL, NSPACEL, isSPACE_LC, isSPACE_LC_utf8, SPACEU, NSPACEU, isSPACE_L1, - perl_space, " "); + SPACEA, NSPACEA, isSPACE_A, + space, " "); CCC_TRY(DIGIT, NDIGIT, isDIGIT, DIGITL, NDIGITL, isDIGIT_LC, isDIGIT_LC_utf8, - posix_digit, "0"); + DIGITA, NDIGITA, isDIGIT_A, + digit, "0"); case CLUMP: /* Match \X: logical Unicode character. This is defined as a Unicode extended Grapheme Cluster */ @@ -3772,17 +3835,17 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) L* (L | LVT T* | V V* T* | LV V* T*) That means that if we have seen any L's at all we can quit - there, but if the next character is a LVT, a V or and LV we + there, but if the next character is an LVT, a V, or an LV we should keep going. There is a subtlety with Prepend* which showed up in testing. Note that the Begin, and only the Begin is required in: | Prepend* Begin Extend* - Also, Begin contains '! Control'. A Prepend must be a '! - Control', which means it must be a Begin. What it comes down to - is that if we match Prepend* and then find no suitable Begin - afterwards, that if we backtrack the last Prepend, that one will - be a suitable Begin. + Also, Begin contains '! Control'. A Prepend must be a + '! Control', which means it must also be a Begin. What it + comes down to is that if we match Prepend* and then find no + suitable Begin afterwards, that if we backtrack the last + Prepend, that one will be a suitable Begin. */ if (locinput >= PL_regeol) @@ -3792,7 +3855,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) /* Match either CR LF or '.', as all the other possibilities * require utf8 */ locinput++; /* Match the . or CR */ - if (nextchr == '\r' + if (nextchr == '\r' /* And if it was CR, and the next is LF, + match the LF */ && locinput < PL_regeol && UCHARAT(locinput) == '\n') locinput++; } @@ -3946,34 +4010,47 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) named buffers just convert to the equivalent numbered and pretend they were called as the corresponding numbered buffer op. */ - /* don't initialize these, it makes C++ unhappy */ + /* don't initialize these in the declaration, it makes C++ + unhappy */ char *s; char type; re_fold_t folder; const U8 *fold_array; + UV utf8_fold_flags; PL_reg_flags |= RF_tainted; folder = foldEQ_locale; fold_array = PL_fold_locale; type = REFFL; + utf8_fold_flags = FOLDEQ_UTF8_LOCALE; + goto do_nref; + + case NREFFA: + folder = foldEQ_latin1; + fold_array = PL_fold_latin1; + type = REFFA; + utf8_fold_flags = FOLDEQ_UTF8_NOMIX_ASCII; goto do_nref; case NREFFU: folder = foldEQ_latin1; fold_array = PL_fold_latin1; type = REFFU; + utf8_fold_flags = 0; goto do_nref; case NREFF: folder = foldEQ; fold_array = PL_fold; type = REFF; + utf8_fold_flags = 0; goto do_nref; case NREF: type = REF; folder = NULL; fold_array = NULL; + utf8_fold_flags = 0; do_nref: /* For the named back references, find the corresponding buffer @@ -3989,21 +4066,31 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) PL_reg_flags |= RF_tainted; folder = foldEQ_locale; fold_array = PL_fold_locale; + utf8_fold_flags = FOLDEQ_UTF8_LOCALE; + goto do_ref; + + case REFFA: + folder = foldEQ_latin1; + fold_array = PL_fold_latin1; + utf8_fold_flags = FOLDEQ_UTF8_NOMIX_ASCII; goto do_ref; case REFFU: folder = foldEQ_latin1; fold_array = PL_fold_latin1; + utf8_fold_flags = 0; goto do_ref; case REFF: folder = foldEQ; fold_array = PL_fold; + utf8_fold_flags = 0; goto do_ref; case REF: folder = NULL; fold_array = NULL; + utf8_fold_flags = 0; do_ref: type = OP(scan); @@ -4019,10 +4106,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) s = PL_bostr + ln; if (type != REF /* REF can do byte comparison */ - && (utf8_target - || (type == REFFU - && (*s == (char) LATIN_SMALL_LETTER_SHARP_S - || *locinput == (char) LATIN_SMALL_LETTER_SHARP_S)))) + && (utf8_target || type == REFFU)) { /* XXX handle REFFL better */ char * limit = PL_regeol; @@ -4031,8 +4115,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) * not going off the end given by PL_regeol, and returns in * limit upon success, how much of the current input was * matched */ - if (! foldEQ_utf8(s, NULL, PL_regoffs[n].end - ln, utf8_target, - locinput, &limit, 0, utf8_target)) + if (! foldEQ_utf8_flags(s, NULL, PL_regoffs[n].end - ln, utf8_target, + locinput, &limit, 0, utf8_target, utf8_fold_flags)) { sayNO; } @@ -4134,6 +4218,12 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) PL_op = (OP_4tree*)rexi->data->data[n]; DEBUG_STATE_r( PerlIO_printf(Perl_debug_log, " re_eval 0x%"UVxf"\n", PTR2UV(PL_op)) ); + /* wrap the call in two SAVECOMPPADs. This ensures that + * when the save stack is eventually unwound, all the + * accumulated SAVEt_CLEARSV's will be processed with + * interspersed SAVEt_COMPPAD's to ensure that lexicals + * are cleared in the right pad */ + SAVECOMPPAD(); PAD_SAVE_LOCAL(old_comppad, (PAD*)rexi->data->data[n + 2]); PL_regoffs[0].end = PL_reg_magic->mg_len = locinput - PL_bostr; @@ -4154,6 +4244,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) Copy(&saved_state, &PL_reg_state, 1, struct re_save_state); PL_op = oop; + SAVECOMPPAD(); PAD_RESTORE_LOCAL(old_comppad); PL_curcop = ocurcop; PL_regeol = saved_regeol; @@ -4580,7 +4671,10 @@ NULL /* First just match a string of min A's. */ if (n < min) { + ST.cp = regcppush(cur_curlyx->u.curlyx.parenfloor); cur_curlyx->u.curlyx.lastloc = locinput; + REGCP_SET(ST.lastcp); + PUSH_STATE_GOTO(WHILEM_A_pre, A); /* NOTREACHED */ } @@ -4686,10 +4780,10 @@ NULL /* NOTREACHED */ case WHILEM_A_min_fail: /* just failed to match A in a minimal match */ - REGCP_UNWIND(ST.lastcp); - regcppop(rex); /* FALL THROUGH */ case WHILEM_A_pre_fail: /* just failed to match even minimal A */ + REGCP_UNWIND(ST.lastcp); + regcppop(rex); cur_curlyx->u.curlyx.lastloc = ST.save_lastloc; cur_curlyx->u.curlyx.count--; CACHEsayNO; @@ -4935,6 +5029,7 @@ NULL ST.c1 = (U8)*STRING(text_node); switch (OP(text_node)) { case EXACTF: ST.c2 = PL_fold[ST.c1]; break; + case EXACTFA: case EXACTFU: ST.c2 = PL_fold_latin1[ST.c1]; break; case EXACTFL: ST.c2 = PL_fold_locale[ST.c1]; break; default: ST.c2 = ST.c1; @@ -5088,6 +5183,7 @@ NULL ST.c1 = *s; switch (OP(text_node)) { case EXACTF: ST.c2 = PL_fold[ST.c1]; break; + case EXACTFA: case EXACTFU: ST.c2 = PL_fold_latin1[ST.c1]; break; case EXACTFL: ST.c2 = PL_fold_locale[ST.c1]; break; default: ST.c2 = ST.c1; break; @@ -5339,7 +5435,7 @@ NULL rex = (struct regexp *)SvANY(rex_sv); rexi = RXi_GET(rex); cur_curlyx = cur_eval->u.eval.prev_curlyx; - ReREFCNT_inc(rex_sv); + (void)ReREFCNT_inc(rex_sv); st->u.eval.cp = regcppush(0); /* Save *all* the positions. */ /* rex was changed so update the pointer in PL_reglastparen and PL_reglastcloseparen */ @@ -5587,6 +5683,8 @@ NULL #define CASE_CLASS(nAmE) \ case nAmE: \ + if (locinput >= PL_regeol) \ + sayNO; \ if ((n=is_##nAmE(locinput,utf8_target))) { \ locinput += n; \ nextchr = UCHARAT(locinput); \ @@ -5594,6 +5692,8 @@ NULL sayNO; \ break; \ case N##nAmE: \ + if (locinput >= PL_regeol) \ + sayNO; \ if ((n=is_##nAmE(locinput,utf8_target))) { \ sayNO; \ } else { \ @@ -5808,6 +5908,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) register char *loceol = PL_regeol; register I32 hardcount = 0; register bool utf8_target = PL_reg_match_utf8; + UV utf8_flags; #ifndef DEBUGGING PERL_UNUSED_ARG(depth); #endif @@ -5883,28 +5984,31 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } } break; + case EXACTFA: + utf8_flags = FOLDEQ_UTF8_NOMIX_ASCII; + goto do_exactf; + case EXACTFL: PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ + utf8_flags = FOLDEQ_UTF8_LOCALE; + goto do_exactf; + case EXACTF: case EXACTFU: + utf8_flags = 0; /* The comments for the EXACT case above apply as well to these fold * ones */ + do_exactf: c = (U8)*STRING(p); assert(! UTF_PATTERN || UNI_IS_INVARIANT(c)); if (utf8_target) { /* Use full Unicode fold matching */ - - /* For the EXACTFL case, It doesn't really make sense to compare - * locale and utf8, but it is best we can do. The documents warn - * against mixing them */ - char *tmpeol = loceol; while (hardcount < max - && foldEQ_utf8(scan, &tmpeol, 0, utf8_target, - STRING(p), NULL, 1, cBOOL(UTF_PATTERN))) + && foldEQ_utf8_flags(scan, &tmpeol, 0, utf8_target, + STRING(p), NULL, 1, cBOOL(UTF_PATTERN), utf8_flags)) { scan = tmpeol; tmpeol = loceol; @@ -5930,6 +6034,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) * fold matching. */ switch (OP(p)) { case EXACTF: folded = PL_fold[c]; break; + case EXACTFA: case EXACTFU: folded = PL_fold_latin1[c]; break; case EXACTFL: folded = PL_fold_locale[c]; break; default: Perl_croak(aTHX_ "panic: Unexpected op %u", OP(p)); @@ -5943,11 +6048,15 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) break; case ANYOFV: case ANYOF: - if (utf8_target) { + if (utf8_target || OP(p) == ANYOFV) { + STRLEN inclasslen; loceol = PL_regeol; - while (hardcount < max && scan < loceol && - reginclass(prog, p, (U8*)scan, 0, utf8_target)) { - scan += UTF8SKIP(scan); + inclasslen = loceol - scan; + while (hardcount < max + && ((inclasslen = loceol - scan) > 0) + && reginclass(prog, p, (U8*)scan, &inclasslen, utf8_target)) + { + scan += inclasslen; hardcount++; } } else { @@ -5979,6 +6088,11 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) scan++; } break; + case ALNUMA: + while (scan < loceol && isWORDCHAR_A((U8) *scan)) { + scan++; + } + break; case ALNUML: PL_reg_flags |= RF_tainted; if (utf8_target) { @@ -6019,6 +6133,18 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) scan++; } break; + case NALNUMA: + if (utf8_target) { + while (scan < loceol && ! isWORDCHAR_A((U8) *scan)) { + scan += UTF8SKIP(scan); + } + } + else { + while (scan < loceol && ! isWORDCHAR_A((U8) *scan)) { + scan++; + } + } + break; case NALNUML: PL_reg_flags |= RF_tainted; if (utf8_target) { @@ -6063,6 +6189,11 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) scan++; } break; + case SPACEA: + while (scan < loceol && isSPACE_A((U8) *scan)) { + scan++; + } + break; case SPACEL: PL_reg_flags |= RF_tainted; if (utf8_target) { @@ -6107,6 +6238,18 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) scan++; } break; + case NSPACEA: + if (utf8_target) { + while (scan < loceol && ! isSPACE_A((U8) *scan)) { + scan += UTF8SKIP(scan); + } + } + else { + while (scan < loceol && ! isSPACE_A((U8) *scan)) { + scan++; + } + } + break; case NSPACEL: PL_reg_flags |= RF_tainted; if (utf8_target) { @@ -6135,6 +6278,11 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) scan++; } break; + case DIGITA: + while (scan < loceol && isDIGIT_A((U8) *scan)) { + scan++; + } + break; case DIGITL: PL_reg_flags |= RF_tainted; if (utf8_target) { @@ -6162,6 +6310,19 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) while (scan < loceol && !isDIGIT(*scan)) scan++; } + break; + case NDIGITA: + if (utf8_target) { + while (scan < loceol && ! isDIGIT_A((U8) *scan)) { + scan += UTF8SKIP(scan); + } + } + else { + while (scan < loceol && ! isDIGIT_A((U8) *scan)) { + scan++; + } + } + break; case NDIGITL: PL_reg_flags |= RF_tainted; if (utf8_target) { @@ -6289,6 +6450,8 @@ Perl_regclass_swash(pTHX_ const regexp *prog, register const regnode* node, bool PERL_ARGS_ASSERT_REGCLASS_SWASH; + assert(ANYOF_NONBITMAP(node)); + if (data && data->count) { const U32 n = ARG(node); @@ -6442,20 +6605,24 @@ S_reginclass(pTHX_ const regexp * const prog, register const regnode * const n, } /* If the bitmap didn't (or couldn't) match, and something outside the - * bitmap could match, try that */ + * bitmap could match, try that. Locale nodes specifiy completely the + * behavior of code points in the bit map (otherwise, a utf8 target would + * cause them to be treated as Unicode and not locale), except in + * the very unlikely event when this node is a synthetic start class, which + * could be a combination of locale and non-locale nodes. So allow locale + * to match for the synthetic start class, which will give a false + * positive that will be resolved when the match is done again as not part + * of the synthetic start class */ if (!match) { - if (utf8_target && (flags & ANYOF_UNICODE_ALL)) { - if (c >= 256 - || ((flags & ANYOF_LOC_NONBITMAP_FOLD) /* Latin1 1 that has a - non-Latin1 fold - should match */ - && _HAS_NONLATIN1_FOLD_CLOSURE_ONLY_FOR_USE_BY_REGCOMP_DOT_C_AND_REGEXEC_DOT_C(c))) - { - match = TRUE; - } + if (utf8_target && (flags & ANYOF_UNICODE_ALL) && c >= 256) { + match = TRUE; /* Everything above 255 matches */ } - if (!match && ((flags & ANYOF_NONBITMAP_NON_UTF8) - || (utf8_target && flags & ANYOF_UTF8))) + else if (ANYOF_NONBITMAP(n) + && ((flags & ANYOF_NONBITMAP_NON_UTF8) + || (utf8_target + && (c >=256 + || (! (flags & ANYOF_LOCALE)) + || (flags & ANYOF_IS_SYNTHETIC))))) { AV *av; SV * const sw = regclass_swash(prog, n, TRUE, 0, (SV**)&av); @@ -6488,31 +6655,19 @@ S_reginclass(pTHX_ const regexp * const prog, register const regnode * const n, else if (flags & ANYOF_LOC_NONBITMAP_FOLD) { /* Here, we need to test if the fold of the target string - * matches. In the case of a multi-char fold that is - * caught by regcomp.c, it has stored all such folds into - * 'av'; we linearly check to see if any match the target - * string (folded). We know that the originals were each - * one character, but we don't currently know how many - * characters/bytes each folded to, except we do know that - * there are small limits imposed by Unicode. XXX A - * performance enhancement would be to have regcomp.c store - * the max number of chars/bytes that are in an av entry, - * as, say the 0th element. Even better would be to have a - * hash of the few characters that can start a multi-char - * fold to the max number of chars of those folds. - * - * Further down, if there isn't a - * match in the av, we will check if there is another - * fold-type match. For that, we also need the fold, but - * only the first character. No sense in folding it twice, - * so we do it here, even if there isn't any multi-char - * fold, so we always fold at least the first character. - * If the node is a straight ANYOF node, or there is only - * one character available in the string, or if there isn't - * any av, that's all we have to fold. In the case of a - * multi-char fold, we do have guarantees in Unicode that - * it can only expand up to so many characters and so many - * bytes. We keep track so don't exceed either. + * matches. The non-multi char folds have all been moved to + * the compilation phase, and the multi-char folds have + * been stored by regcomp into 'av'; we linearly check to + * see if any match the target string (folded). We know + * that the originals were each one character, but we don't + * currently know how many characters/bytes each folded to, + * except we do know that there are small limits imposed by + * Unicode. XXX A performance enhancement would be to have + * regcomp.c store the max number of chars/bytes that are + * in an av entry, as, say the 0th element. Even better + * would be to have a hash of the few characters that can + * start a multi-char fold to the max number of chars of + * those folds. * * If there is a match, we will need to advance (if lenp is * specified) the match pointer in the target string. But @@ -6522,28 +6677,34 @@ S_reginclass(pTHX_ const regexp * const prog, register const regnode * const n, * create a map so that we know how many bytes in the * source to advance given that we have matched a certain * number of bytes in the fold. This map is stored in - * 'map_fold_len_back'. The first character in the fold - * has array element 1 contain the number of bytes in the - * source that folded to it; the 2nd is the cumulative - * number to match it; ... */ - U8 map_fold_len_back[UTF8_MAX_FOLD_CHAR_EXPAND] = { 0 }; + * 'map_fold_len_back'. Let n mean the number of bytes in + * the fold of the first character that we are folding. + * Then map_fold_len_back[n] is set to the number of bytes + * in that first character. Similarly let m be the + * corresponding number for the second character to be + * folded. Then map_fold_len_back[n+m] is set to the + * number of bytes occupied by the first two source + * characters. ... */ + U8 map_fold_len_back[UTF8_MAXBYTES_CASE+1] = { 0 }; U8 folded[UTF8_MAXBYTES_CASE+1]; STRLEN foldlen = 0; /* num bytes in fold of 1st char */ - STRLEN foldlen_for_av; /* num bytes in fold of all chars */ + STRLEN total_foldlen = 0; /* num bytes in fold of all + chars */ if (OP(n) == ANYOF || maxlen == 1 || ! lenp || ! av) { /* Here, only need to fold the first char of the target - * string */ + * string. It the source wasn't utf8, is 1 byte long */ to_utf8_fold(utf8_p, folded, &foldlen); - foldlen_for_av = foldlen; - map_fold_len_back[1] = UTF8SKIP(utf8_p); + total_foldlen = foldlen; + map_fold_len_back[foldlen] = (utf8_target) + ? UTF8SKIP(utf8_p) + : 1; } else { /* Here, need to fold more than the first char. Do so * up to the limits */ - UV which_char = 0; U8* source_ptr = utf8_p; /* The source for the fold is the regex target string */ @@ -6551,8 +6712,10 @@ S_reginclass(pTHX_ const regexp * const prog, register const regnode * const n, U8* e = utf8_p + maxlen; /* Can't go beyond last available byte in the target string */ - while (which_char < UTF8_MAX_FOLD_CHAR_EXPAND - && source_ptr < e) + U8 i; + for (i = 0; + i < UTF8_MAX_FOLD_CHAR_EXPAND && source_ptr < e; + i++) { /* Fold the next character */ @@ -6570,123 +6733,57 @@ S_reginclass(pTHX_ const regexp * const prog, register const regnode * const n, break; } - /* Save the first character's folded length, in - * case we have to use it later */ - if (! foldlen) { - foldlen = this_char_foldlen; - } - - /* Here, add the fold of this character */ + /* Add the fold of this character */ Copy(this_char_folded, folded_ptr, this_char_foldlen, U8); - which_char++; - map_fold_len_back[which_char] = - map_fold_len_back[which_char - 1] - + UTF8SKIP(source_ptr); - folded_ptr += this_char_foldlen; source_ptr += UTF8SKIP(source_ptr); + folded_ptr += this_char_foldlen; + total_foldlen = folded_ptr - folded; + + /* Create map from the number of bytes in the fold + * back to the number of bytes in the source. If + * the source isn't utf8, the byte count is just + * the number of characters so far */ + map_fold_len_back[total_foldlen] + = (utf8_target) + ? source_ptr - utf8_p + : i + 1; } *folded_ptr = '\0'; - foldlen_for_av = folded_ptr - folded; } /* Do the linear search to see if the fold is in the list - * of multi-char folds. (Useless to look if won't be able - * to store that it is a multi-char fold in *lenp) */ - if (lenp && av) { + * of multi-char folds. */ + if (av) { I32 i; for (i = 0; i <= av_len(av); i++) { SV* const sv = *av_fetch(av, i, FALSE); STRLEN len; const char * const s = SvPV_const(sv, len); - if (len <= foldlen_for_av && memEQ(s, - (char*)folded, - len)) + + if (len <= total_foldlen + && memEQ(s, (char*)folded, len) + + /* If 0, means matched a partial char. See + * [perl #90536] */ + && map_fold_len_back[len]) { /* Advance the target string ptr to account for * this fold, but have to translate from the * folded length to the corresponding source - * length. The array is indexed by how many - * characters in the match */ - *lenp = map_fold_len_back[ - utf8_length(folded, folded + len)]; + * length. */ + if (lenp) { + *lenp = map_fold_len_back[len]; + } match = TRUE; break; } } } - if (!match) { /* See if the folded version matches */ - SV** listp; - - /* Consider "k" =~ /[K]/i. The line above would have - * just folded the 'k' to itself, and that isn't going - * to match 'K'. So we look through the closure of - * everything that folds to 'k'. That will find the - * 'K'. Initialize the list, if necessary */ - if (! PL_utf8_foldclosures) { - - /* If the folds haven't been read in, call a fold - * function to force that */ - if (! PL_utf8_tofold) { - U8 dummy[UTF8_MAXBYTES+1]; - STRLEN dummy_len; - to_utf8_fold((U8*) "A", dummy, &dummy_len); - } - PL_utf8_foldclosures = - _swash_inversion_hash(PL_utf8_tofold); - } - - /* The data structure is a hash with the keys every - * character that is folded to, like 'k', and the - * values each an array of everything that folds to its - * key. e.g. [ 'k', 'K', KELVIN_SIGN ] */ - if ((listp = hv_fetch(PL_utf8_foldclosures, - (char *) folded, foldlen, FALSE))) - { - AV* list = (AV*) *listp; - IV i; - for (i = 0; i <= av_len(list); i++) { - SV** try_p = av_fetch(list, i, FALSE); - char* try_c; - if (try_p == NULL) { - Perl_croak(aTHX_ "panic: invalid PL_utf8_foldclosures structure"); - } - /* Don't have to worry about embedded nulls - * since NULL isn't folded or foldable */ - try_c = SvPVX(*try_p); - - /* The fold in a few cases of an above Latin1 - * char is in the Latin1 range, and hence may - * be in the bitmap */ - if (UTF8_IS_INVARIANT(*try_c) - && ANYOF_BITMAP_TEST(n, - UNI_TO_NATIVE(*try_c))) - { - match = TRUE; - break; - } - else if - (UTF8_IS_DOWNGRADEABLE_START(*try_c) - && ANYOF_BITMAP_TEST(n, UNI_TO_NATIVE( - TWO_BYTE_UTF8_TO_UNI(try_c[0], - try_c[1])))) - { - /* Since the fold comes from internally - * generated data, we can safely assume it - * is valid utf8 in the test above */ - match = TRUE; - break; - } else if (swash_fetch(sw, (U8*) try_c, TRUE)) { - match = TRUE; - break; - } - } - } - } } /* If we allocated a string above, free it */ @@ -6820,16 +6917,16 @@ S_to_utf8_substr(pTHX_ register regexp *prog) prog->substrs->data[i].utf8_substr = sv; sv_utf8_upgrade(sv); if (SvVALID(prog->substrs->data[i].substr)) { - const U8 flags = BmFLAGS(prog->substrs->data[i].substr); - if (flags & FBMcf_TAIL) { + if (SvTAIL(prog->substrs->data[i].substr)) { /* Trim the trailing \n that fbm_compile added last time. */ SvCUR_set(sv, SvCUR(sv) - 1); /* Whilst this makes the SV technically "invalid" (as its buffer is no longer followed by "\0") when fbm_compile() adds the "\n" back, a "\0" is restored. */ - } - fbm_compile(sv, flags); + fbm_compile(sv, FBMcf_TAIL); + } else + fbm_compile(sv, 0); } if (prog->substrs->data[i].substr == prog->check_substr) prog->check_utf8 = sv; @@ -6851,15 +6948,14 @@ S_to_byte_substr(pTHX_ register regexp *prog) SV* sv = newSVsv(prog->substrs->data[i].utf8_substr); if (sv_utf8_downgrade(sv, TRUE)) { if (SvVALID(prog->substrs->data[i].utf8_substr)) { - const U8 flags - = BmFLAGS(prog->substrs->data[i].utf8_substr); - if (flags & FBMcf_TAIL) { + if (SvTAIL(prog->substrs->data[i].utf8_substr)) { /* Trim the trailing \n that fbm_compile added last time. */ SvCUR_set(sv, SvCUR(sv) - 1); - } - fbm_compile(sv, flags); - } + fbm_compile(sv, FBMcf_TAIL); + } else + fbm_compile(sv, 0); + } } else { SvREFCNT_dec(sv); sv = &PL_sv_undef;