X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/28e464fba839d02f376952199261fc8b7d58a0d8..972dd5923c5c2d608cbd9b6dd1203e87acb97937:/regexec.c diff --git a/regexec.c b/regexec.c index 83d216b..d95b27a 100644 --- a/regexec.c +++ b/regexec.c @@ -85,7 +85,7 @@ #define RF_utf8 8 /* Pattern contains multibyte chars? */ -#define UTF ((PL_reg_flags & RF_utf8) != 0) +#define UTF_PATTERN ((PL_reg_flags & RF_utf8) != 0) #define RS_init 1 /* eval environment created */ #define RS_set 2 /* replsv value is set */ @@ -94,13 +94,17 @@ #define STATIC static #endif -#define REGINCLASS(prog,p,c) (ANYOF_FLAGS(p) ? reginclass(prog,p,c,0,0) : ANYOF_BITMAP_TEST(p,*(c))) +/* Valid for non-utf8 strings only: avoids the reginclass call if there are no + * complications: i.e., if everything matchable is straight forward in the + * bitmap */ +#define REGINCLASS(prog,p,c) (ANYOF_FLAGS(p) ? reginclass(prog,p,c,0,0) \ + : ANYOF_BITMAP_TEST(p,*(c))) /* * Forwards. */ -#define CHR_SVLEN(sv) (do_utf8 ? sv_len_utf8(sv) : SvCUR(sv)) +#define CHR_SVLEN(sv) (utf8_target ? sv_len_utf8(sv) : SvCUR(sv)) #define CHR_DIST(a,b) (PL_reg_match_utf8 ? utf8_distance(a,b) : a - b) #define HOPc(pos,off) \ @@ -117,24 +121,172 @@ #define HOP3(pos,off,lim) (PL_reg_match_utf8 ? reghop3((U8*)(pos), off, (U8*)(lim)) : (U8*)(pos + off)) #define HOP3c(pos,off,lim) ((char*)HOP3(pos,off,lim)) +/* these are unrolled below in the CCC_TRY_XXX defined */ #define LOAD_UTF8_CHARCLASS(class,str) STMT_START { \ if (!CAT2(PL_utf8_,class)) { bool ok; ENTER; save_re_context(); ok=CAT2(is_utf8_,class)((const U8*)str); assert(ok); LEAVE; } } STMT_END + +/* Doesn't do an assert to verify that is correct */ +#define LOAD_UTF8_CHARCLASS_NO_CHECK(class) STMT_START { \ + if (!CAT2(PL_utf8_,class)) { bool ok; ENTER; save_re_context(); ok=CAT2(is_utf8_,class)((const U8*)" "); LEAVE; } } STMT_END + #define LOAD_UTF8_CHARCLASS_ALNUM() LOAD_UTF8_CHARCLASS(alnum,"a") #define LOAD_UTF8_CHARCLASS_DIGIT() LOAD_UTF8_CHARCLASS(digit,"0") #define LOAD_UTF8_CHARCLASS_SPACE() LOAD_UTF8_CHARCLASS(space," ") -#define LOAD_UTF8_CHARCLASS_MARK() LOAD_UTF8_CHARCLASS(mark, "\xcd\x86") + +#define LOAD_UTF8_CHARCLASS_GCB() /* Grapheme cluster boundaries */ \ + LOAD_UTF8_CHARCLASS(X_begin, " "); \ + LOAD_UTF8_CHARCLASS(X_non_hangul, "A"); \ + /* These are utf8 constants, and not utf-ebcdic constants, so the \ + * assert should likely and hopefully fail on an EBCDIC machine */ \ + LOAD_UTF8_CHARCLASS(X_extend, "\xcc\x80"); /* U+0300 */ \ + \ + /* No asserts are done for these, in case called on an early \ + * Unicode version in which they map to nothing */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_prepend);/* U+0E40 "\xe0\xb9\x80" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_L); /* U+1100 "\xe1\x84\x80" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_LV); /* U+AC00 "\xea\xb0\x80" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_LVT); /* U+AC01 "\xea\xb0\x81" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_LV_LVT_V);/* U+AC01 "\xea\xb0\x81" */\ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_T); /* U+11A8 "\xe1\x86\xa8" */ \ + LOAD_UTF8_CHARCLASS_NO_CHECK(X_V) /* U+1160 "\xe1\x85\xa0" */ + +/* + We dont use PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS as the direct test + so that it is possible to override the option here without having to + rebuild the entire core. as we are required to do if we change regcomp.h + which is where PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS is defined. +*/ +#if PERL_LEGACY_UNICODE_CHARCLASS_MAPPINGS +#define BROKEN_UNICODE_CHARCLASS_MAPPINGS +#endif + +#ifdef BROKEN_UNICODE_CHARCLASS_MAPPINGS +#define LOAD_UTF8_CHARCLASS_PERL_WORD() LOAD_UTF8_CHARCLASS_ALNUM() +#define LOAD_UTF8_CHARCLASS_PERL_SPACE() LOAD_UTF8_CHARCLASS_SPACE() +#define LOAD_UTF8_CHARCLASS_POSIX_DIGIT() LOAD_UTF8_CHARCLASS_DIGIT() +#define RE_utf8_perl_word PL_utf8_alnum +#define RE_utf8_perl_space PL_utf8_space +#define RE_utf8_posix_digit PL_utf8_digit +#define perl_word alnum +#define perl_space space +#define posix_digit digit +#else +#define LOAD_UTF8_CHARCLASS_PERL_WORD() LOAD_UTF8_CHARCLASS(perl_word,"a") +#define LOAD_UTF8_CHARCLASS_PERL_SPACE() LOAD_UTF8_CHARCLASS(perl_space," ") +#define LOAD_UTF8_CHARCLASS_POSIX_DIGIT() LOAD_UTF8_CHARCLASS(posix_digit,"0") +#define RE_utf8_perl_word PL_utf8_perl_word +#define RE_utf8_perl_space PL_utf8_perl_space +#define RE_utf8_posix_digit PL_utf8_posix_digit +#endif + + +#define _CCC_TRY_AFF_COMMON(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC) \ + case NAMEL: \ + PL_reg_flags |= RF_tainted; \ + /* FALL THROUGH */ \ + case NAME: \ + if (!nextchr) \ + sayNO; \ + if (utf8_target && UTF8_IS_CONTINUED(nextchr)) { \ + if (!CAT2(PL_utf8_,CLASS)) { \ + bool ok; \ + ENTER; \ + save_re_context(); \ + ok=CAT2(is_utf8_,CLASS)((const U8*)STR); \ + assert(ok); \ + LEAVE; \ + } \ + if (!(OP(scan) == NAME \ + ? cBOOL(swash_fetch(CAT2(PL_utf8_,CLASS), (U8*)locinput, utf8_target)) \ + : LCFUNC_utf8((U8*)locinput))) \ + { \ + sayNO; \ + } \ + locinput += PL_utf8skip[nextchr]; \ + nextchr = UCHARAT(locinput); \ + break; \ + } \ + /* Drops through to the macro that calls this one */ + +#define CCC_TRY_AFF(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC,LCFUNC) \ + _CCC_TRY_AFF_COMMON(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC) \ + if (!(OP(scan) == NAME ? FUNC(nextchr) : LCFUNC(nextchr))) \ + sayNO; \ + nextchr = UCHARAT(++locinput); \ + break + +/* Almost identical to the above, but has a case for a node that matches chars + * between 128 and 255 using Unicode (latin1) semantics. */ +#define CCC_TRY_AFF_U(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNCU,LCFUNC) \ + _CCC_TRY_AFF_COMMON(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC) \ + if (!(OP(scan) == NAMEL ? LCFUNC(nextchr) : (FUNCU(nextchr) && (isASCII(nextchr) || (FLAGS(scan) & USE_UNI))))) \ + sayNO; \ + nextchr = UCHARAT(++locinput); \ + break + +#define _CCC_TRY_NEG_COMMON(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC) \ + case NAMEL: \ + PL_reg_flags |= RF_tainted; \ + /* FALL THROUGH */ \ + case NAME : \ + if (!nextchr && locinput >= PL_regeol) \ + sayNO; \ + if (utf8_target && UTF8_IS_CONTINUED(nextchr)) { \ + if (!CAT2(PL_utf8_,CLASS)) { \ + bool ok; \ + ENTER; \ + save_re_context(); \ + ok=CAT2(is_utf8_,CLASS)((const U8*)STR); \ + assert(ok); \ + LEAVE; \ + } \ + if ((OP(scan) == NAME \ + ? cBOOL(swash_fetch(CAT2(PL_utf8_,CLASS), (U8*)locinput, utf8_target)) \ + : LCFUNC_utf8((U8*)locinput))) \ + { \ + sayNO; \ + } \ + locinput += PL_utf8skip[nextchr]; \ + nextchr = UCHARAT(locinput); \ + break; \ + } + +#define CCC_TRY_NEG(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC,LCFUNC) \ + _CCC_TRY_NEG_COMMON(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNC) \ + if ((OP(scan) == NAME ? FUNC(nextchr) : LCFUNC(nextchr))) \ + sayNO; \ + nextchr = UCHARAT(++locinput); \ + break + + +#define CCC_TRY_NEG_U(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNCU,LCFUNC) \ + _CCC_TRY_NEG_COMMON(NAME,NAMEL,CLASS,STR,LCFUNC_utf8,FUNCU) \ + if ((OP(scan) == NAMEL ? LCFUNC(nextchr) : (FUNCU(nextchr) && (isASCII(nextchr) || (FLAGS(scan) & USE_UNI))))) \ + sayNO; \ + nextchr = UCHARAT(++locinput); \ + break + + /* TODO: Combine JUMPABLE and HAS_TEXT to cache OP(rn) */ /* for use after a quantifier and before an EXACT-like node -- japhy */ -/* it would be nice to rework regcomp.sym to generate this stuff. sigh */ +/* it would be nice to rework regcomp.sym to generate this stuff. sigh + * + * NOTE that *nothing* that affects backtracking should be in here, specifically + * VERBS must NOT be included. JUMPABLE is used to determine if we can ignore a + * node that is in between two EXACT like nodes when ascertaining what the required + * "follow" character is. This should probably be moved to regex compile time + * although it may be done at run time beause of the REF possibility - more + * investigation required. -- demerphq +*/ #define JUMPABLE(rn) ( \ OP(rn) == OPEN || \ (OP(rn) == CLOSE && (!cur_eval || cur_eval->u.eval.close_paren != ARG(rn))) || \ OP(rn) == EVAL || \ OP(rn) == SUSPEND || OP(rn) == IFMATCH || \ OP(rn) == PLUS || OP(rn) == MINMOD || \ - OP(rn) == KEEPS || (PL_regkind[OP(rn)] == VERB) || \ + OP(rn) == KEEPS || \ (PL_regkind[OP(rn)] == CURLY && ARG1(rn) > 0) \ ) #define IS_EXACT(rn) (PL_regkind[OP(rn)] == EXACT) @@ -176,21 +328,32 @@ static void restore_pos(pTHX_ void *arg); +#define REGCP_PAREN_ELEMS 4 +#define REGCP_OTHER_ELEMS 5 +#define REGCP_FRAME_ELEMS 1 +/* REGCP_FRAME_ELEMS are not part of the REGCP_OTHER_ELEMS and + * are needed for the regexp context stack bookkeeping. */ + STATIC CHECKPOINT S_regcppush(pTHX_ I32 parenfloor) { dVAR; const int retval = PL_savestack_ix; -#define REGCP_PAREN_ELEMS 4 const int paren_elems_to_push = (PL_regsize - parenfloor) * REGCP_PAREN_ELEMS; + const UV total_elems = paren_elems_to_push + REGCP_OTHER_ELEMS; + const UV elems_shifted = total_elems << SAVE_TIGHT_SHIFT; int p; GET_RE_DEBUG_FLAGS_DECL; if (paren_elems_to_push < 0) Perl_croak(aTHX_ "panic: paren_elems_to_push < 0"); -#define REGCP_OTHER_ELEMS 7 - SSGROW(paren_elems_to_push + REGCP_OTHER_ELEMS); + if ((elems_shifted >> SAVE_TIGHT_SHIFT) != total_elems) + Perl_croak(aTHX_ "panic: paren_elems_to_push offset %"UVuf + " out of range (%lu-%ld)", + total_elems, (unsigned long)PL_regsize, (long)parenfloor); + + SSGROW(total_elems + REGCP_FRAME_ELEMS); for (p = PL_regsize; p > parenfloor; p--) { /* REGCP_PARENS_ELEMS are pushed per pairs of parentheses. */ @@ -211,11 +374,7 @@ S_regcppush(pTHX_ I32 parenfloor) SSPUSHINT(*PL_reglastparen); SSPUSHINT(*PL_reglastcloseparen); SSPUSHPTR(PL_reginput); -#define REGCP_FRAME_ELEMS 2 -/* REGCP_FRAME_ELEMS are part of the REGCP_OTHER_ELEMS and - * are needed for the regexp context stack bookkeeping. */ - SSPUSHINT(paren_elems_to_push + REGCP_OTHER_ELEMS - REGCP_FRAME_ELEMS); - SSPUSHINT(SAVEt_REGCONTEXT); /* Magic cookie. */ + SSPUSHUV(SAVEt_REGCONTEXT | elems_shifted); /* Magic cookie. */ return retval; } @@ -240,26 +399,25 @@ STATIC char * S_regcppop(pTHX_ const regexp *rex) { dVAR; - U32 i; + UV i; char *input; GET_RE_DEBUG_FLAGS_DECL; PERL_ARGS_ASSERT_REGCPPOP; /* Pop REGCP_OTHER_ELEMS before the parentheses loop starts. */ - i = SSPOPINT; - assert(i == SAVEt_REGCONTEXT); /* Check that the magic cookie is there. */ - i = SSPOPINT; /* Parentheses elements to pop. */ + i = SSPOPUV; + assert((i & SAVE_MASK) == SAVEt_REGCONTEXT); /* Check that the magic cookie is there. */ + i >>= SAVE_TIGHT_SHIFT; /* Parentheses elements to pop. */ input = (char *) SSPOPPTR; *PL_reglastcloseparen = SSPOPINT; *PL_reglastparen = SSPOPINT; PL_regsize = SSPOPINT; PL_regoffs=(regexp_paren_pair *) SSPOPPTR; - + i -= REGCP_OTHER_ELEMS; /* Now restore the parentheses context. */ - for (i -= (REGCP_OTHER_ELEMS - REGCP_FRAME_ELEMS); - i > 0; i -= REGCP_PAREN_ELEMS) { + for ( ; i > 0; i -= REGCP_PAREN_ELEMS) { I32 tmps; U32 paren = (U32)SSPOPINT; PL_reg_start_tmp[paren] = (char *) SSPOPPTR; @@ -391,7 +549,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, register SV *check; char *strbeg; char *t; - const bool do_utf8 = (sv && SvUTF8(sv)) ? 1 : 0; /* if no sv we have to assume bytes */ + const bool utf8_target = (sv && SvUTF8(sv)) ? 1 : 0; /* if no sv we have to assume bytes */ I32 ml_anch; register char *other_last = NULL; /* other substr checked before this */ char *check_at = NULL; /* check substr found at this pos */ @@ -404,13 +562,13 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, PERL_ARGS_ASSERT_RE_INTUIT_START; - RX_MATCH_UTF8_set(rx,do_utf8); + RX_MATCH_UTF8_set(rx,utf8_target); if (RX_UTF8(rx)) { PL_reg_flags |= RF_utf8; } DEBUG_EXECUTE_r( - debug_start_match(rx, do_utf8, strpos, strend, + debug_start_match(rx, utf8_target, strpos, strend, sv ? "Guessing start of match in sv for" : "Guessing start of match in string for"); ); @@ -424,7 +582,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, strbeg = (sv && SvPOK(sv)) ? strend - SvCUR(sv) : strpos; PL_regeol = strend; - if (do_utf8) { + if (utf8_target) { if (!prog->check_utf8 && prog->check_substr) to_utf8_substr(prog); check = prog->check_utf8; @@ -576,11 +734,11 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, unshift s. */ DEBUG_EXECUTE_r({ - RE_PV_QUOTED_DECL(quoted, do_utf8, PERL_DEBUG_PAD_ZERO(0), + RE_PV_QUOTED_DECL(quoted, utf8_target, PERL_DEBUG_PAD_ZERO(0), SvPVX_const(check), RE_SV_DUMPLEN(check), 30); PerlIO_printf(Perl_debug_log, "%s %s substr %s%s%s", (s ? "Found" : "Did not find"), - (check == (do_utf8 ? prog->anchored_utf8 : prog->anchored_substr) + (check == (utf8_target ? prog->anchored_utf8 : prog->anchored_substr) ? "anchored" : "floating"), quoted, RE_SV_TAIL(check), @@ -611,14 +769,14 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, Probably it is right to do no SCREAM here... */ - if (do_utf8 ? (prog->float_utf8 && prog->anchored_utf8) + if (utf8_target ? (prog->float_utf8 && prog->anchored_utf8) : (prog->float_substr && prog->anchored_substr)) { /* Take into account the "other" substring. */ /* XXXX May be hopelessly wrong for UTF... */ if (!other_last) other_last = strpos; - if (check == (do_utf8 ? prog->float_utf8 : prog->float_substr)) { + if (check == (utf8_target ? prog->float_utf8 : prog->float_substr)) { do_other_anchored: { char * const last = HOP3c(s, -start_shift, strbeg); @@ -628,7 +786,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, t = s - prog->check_offset_max; if (s - strpos > prog->check_offset_max /* signed-corrected t > strpos */ - && (!do_utf8 + && (!utf8_target || ((t = (char*)reghopmaybe3((U8*)s, -(prog->check_offset_max), (U8*)strpos)) && t > strpos))) NOOP; @@ -647,7 +805,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, */ /* On end-of-str: see comment below. */ - must = do_utf8 ? prog->anchored_utf8 : prog->anchored_substr; + must = utf8_target ? prog->anchored_utf8 : prog->anchored_substr; if (must == &PL_sv_undef) { s = (char*)NULL; DEBUG_r(must = prog->anchored_utf8); /* for debug */ @@ -661,7 +819,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, multiline ? FBMrf_MULTILINE : 0 ); DEBUG_EXECUTE_r({ - RE_PV_QUOTED_DECL(quoted, do_utf8, PERL_DEBUG_PAD_ZERO(0), + RE_PV_QUOTED_DECL(quoted, utf8_target, PERL_DEBUG_PAD_ZERO(0), SvPVX_const(must), RE_SV_DUMPLEN(must), 30); PerlIO_printf(Perl_debug_log, "%s anchored substr %s%s", (s ? "Found" : "Contradicts"), @@ -708,7 +866,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, if (s < other_last) s = other_last; /* XXXX It is not documented what units *_offsets are in. Assume bytes. */ - must = do_utf8 ? prog->float_utf8 : prog->float_substr; + must = utf8_target ? prog->float_utf8 : prog->float_substr; /* fbm_instr() takes into account exact value of end-of-str if the check is SvTAIL(ed). Since false positives are OK, and end-of-str is not later than strend we are OK. */ @@ -722,7 +880,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, - (SvTAIL(must)!=0), must, multiline ? FBMrf_MULTILINE : 0); DEBUG_EXECUTE_r({ - RE_PV_QUOTED_DECL(quoted, do_utf8, PERL_DEBUG_PAD_ZERO(0), + RE_PV_QUOTED_DECL(quoted, utf8_target, PERL_DEBUG_PAD_ZERO(0), SvPVX_const(must), RE_SV_DUMPLEN(must), 30); PerlIO_printf(Perl_debug_log, "%s floating substr %s%s", (s ? "Found" : "Contradicts"), @@ -769,7 +927,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, ); if (s - strpos > prog->check_offset_max /* signed-corrected t > strpos */ - && (!do_utf8 + && (!utf8_target || ((t = (char*)reghopmaybe3((U8*)s, -prog->check_offset_max, (U8*) ((prog->check_offset_max<0) ? strend : strpos))) && t > strpos))) { @@ -787,7 +945,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, while (t < strend - prog->minlen) { if (*t == '\n') { if (t < check_at - prog->check_offset_min) { - if (do_utf8 ? prog->anchored_utf8 : prog->anchored_substr) { + if (utf8_target ? prog->anchored_utf8 : prog->anchored_substr) { /* Since we moved from the found position, we definitely contradict the found anchored substr. Due to the above check we do not @@ -827,7 +985,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, } s = t; set_useful: - ++BmUSEFUL(do_utf8 ? prog->check_utf8 : prog->check_substr); /* hooray/5 */ + ++BmUSEFUL(utf8_target ? prog->check_utf8 : prog->check_substr); /* hooray/5 */ } else { /* The found string does not prohibit matching at strpos, @@ -851,7 +1009,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, ); success_at_start: if (!(prog->intflags & PREGf_NAUGHTY) /* XXXX If strpos moved? */ - && (do_utf8 ? ( + && (utf8_target ? ( prog->check_utf8 /* Could be deleted already */ && --BmUSEFUL(prog->check_utf8) < 0 && (prog->check_utf8 == prog->float_utf8) @@ -863,17 +1021,22 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, { /* If flags & SOMETHING - do not do it many times on the same match */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "... Disabling check substring...\n")); - SvREFCNT_dec(do_utf8 ? prog->check_utf8 : prog->check_substr); - if (do_utf8 ? prog->check_substr : prog->check_utf8) - SvREFCNT_dec(do_utf8 ? prog->check_substr : prog->check_utf8); + /* XXX Does the destruction order has to change with utf8_target? */ + SvREFCNT_dec(utf8_target ? prog->check_utf8 : prog->check_substr); + SvREFCNT_dec(utf8_target ? prog->check_substr : prog->check_utf8); prog->check_substr = prog->check_utf8 = NULL; /* disable */ prog->float_substr = prog->float_utf8 = NULL; /* clear */ check = NULL; /* abort */ s = strpos; + /* XXXX If the check string was an implicit check MBOL, then we need to unset the relevent flag + see http://bugs.activestate.com/show_bug.cgi?id=87173 */ + if (prog->intflags & PREGf_IMPLICIT) + prog->extflags &= ~RXf_ANCH_MBOL; /* XXXX This is a remnant of the old implementation. It looks wasteful, since now INTUIT can use many other heuristics. */ prog->extflags &= ~RXf_USE_INTUIT; + /* XXXX What other flags might need to be cleared in this branch? */ } else s = strpos; @@ -924,7 +1087,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, goto fail; /* Contradict one of substrings */ if (prog->anchored_substr || prog->anchored_utf8) { - if ((do_utf8 ? prog->anchored_utf8 : prog->anchored_substr) == check) { + if ((utf8_target ? prog->anchored_utf8 : prog->anchored_substr) == check) { DEBUG_EXECUTE_r( what = "anchored" ); hop_and_restart: s = HOP3c(t, 1, strend); @@ -964,7 +1127,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, PL_colors[0], PL_colors[1], (long)(t - i_strpos)) ); goto try_at_offset; } - if (!(do_utf8 ? prog->float_utf8 : prog->float_substr)) /* Could have been deleted */ + if (!(utf8_target ? prog->float_utf8 : prog->float_substr)) /* Could have been deleted */ goto fail; /* Check is floating subtring. */ retry_floating_check: @@ -992,7 +1155,7 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, fail_finish: /* Substring not found */ if (prog->check_substr || prog->check_utf8) /* could be removed already */ - BmUSEFUL(do_utf8 ? prog->check_utf8 : prog->check_substr) += 5; /* hooray */ + BmUSEFUL(utf8_target ? prog->check_utf8 : prog->check_substr) += 5; /* hooray */ fail: DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "%sMatch rejected by optimizer%s\n", PL_colors[4], PL_colors[5])); @@ -1002,21 +1165,20 @@ Perl_re_intuit_start(pTHX_ REGEXP * const rx, SV *sv, char *strpos, #define DECL_TRIE_TYPE(scan) \ const enum { trie_plain, trie_utf8, trie_utf8_fold, trie_latin_utf8_fold } \ trie_type = (scan->flags != EXACT) \ - ? (do_utf8 ? trie_utf8_fold : (UTF ? trie_latin_utf8_fold : trie_plain)) \ - : (do_utf8 ? trie_utf8 : trie_plain) + ? (utf8_target ? trie_utf8_fold : (UTF_PATTERN ? trie_latin_utf8_fold : trie_plain)) \ + : (utf8_target ? trie_utf8 : trie_plain) #define REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc, uscan, len, \ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ - UV uvc_unfolded = 0; \ switch (trie_type) { \ case trie_utf8_fold: \ if ( foldlen>0 ) { \ - uvc_unfolded = uvc = utf8n_to_uvuni( uscan, UTF8_MAXLEN, &len, uniflags ); \ + uvc = utf8n_to_uvuni( uscan, UTF8_MAXLEN, &len, uniflags ); \ foldlen -= len; \ uscan += len; \ len=0; \ } else { \ - uvc_unfolded = uvc = utf8n_to_uvuni( (U8*)uc, UTF8_MAXLEN, &len, uniflags ); \ + uvc = utf8n_to_uvuni( (U8*)uc, UTF8_MAXLEN, &len, uniflags ); \ uvc = to_uni_fold( uvc, foldbuf, &foldlen ); \ foldlen -= UNISKIP( uvc ); \ uscan = foldbuf + UNISKIP( uvc ); \ @@ -1042,7 +1204,6 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ uvc = (UV)*uc; \ len = 1; \ } \ - \ if (uvc < 256) { \ charid = trie->charmap[ uvc ]; \ } \ @@ -1055,9 +1216,6 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ charid = (U16)SvIV(*svpp); \ } \ } \ - if (!charid && trie_type == trie_utf8_fold && !UTF) { \ - charid = trie->charmap[uvc_unfolded]; \ - } \ } STMT_END #define REXEC_FBC_EXACTISH_CHECK(CoNd) \ @@ -1065,8 +1223,8 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ char *my_strend= (char *)strend; \ if ( (CoNd) \ && (ln == len || \ - !ibcmp_utf8(s, &my_strend, 0, do_utf8, \ - m, NULL, ln, (bool)UTF)) \ + foldEQ_utf8(s, &my_strend, 0, utf8_target, \ + m, NULL, ln, cBOOL(UTF_PATTERN))) \ && (!reginfo || regtry(reginfo, &s)) ) \ goto got_it; \ else { \ @@ -1076,8 +1234,8 @@ uvc, charid, foldlen, foldbuf, uniflags) STMT_START { \ if ( f != c \ && (f == c1 || f == c2) \ && (ln == len || \ - !ibcmp_utf8(s, &my_strend, 0, do_utf8,\ - m, NULL, ln, (bool)UTF)) \ + foldEQ_utf8(s, &my_strend, 0, utf8_target,\ + m, NULL, ln, cBOOL(UTF_PATTERN)))\ && (!reginfo || regtry(reginfo, &s)) ) \ goto got_it; \ } \ @@ -1088,9 +1246,9 @@ s += len STMT_START { \ while (s <= e) { \ if ( (CoNd) \ - && (ln == 1 || !(OP(c) == EXACTF \ - ? ibcmp(s, m, ln) \ - : ibcmp_locale(s, m, ln))) \ + && (ln == 1 || (OP(c) == EXACTF \ + ? foldEQ(s, m, ln) \ + : foldEQ_locale(s, m, ln))) \ && (!reginfo || regtry(reginfo, &s)) ) \ goto got_it; \ s++; \ @@ -1142,7 +1300,7 @@ if ((!reginfo || regtry(reginfo, &s))) \ goto got_it #define REXEC_FBC_CSCAN(CoNdUtF8,CoNd) \ - if (do_utf8) { \ + if (utf8_target) { \ REXEC_FBC_UTF8_CLASS_SCAN(CoNdUtF8); \ } \ else { \ @@ -1151,7 +1309,7 @@ if ((!reginfo || regtry(reginfo, &s))) \ break #define REXEC_FBC_CSCAN_PRELOAD(UtFpReLoAd,CoNdUtF8,CoNd) \ - if (do_utf8) { \ + if (utf8_target) { \ UtFpReLoAd; \ REXEC_FBC_UTF8_CLASS_SCAN(CoNdUtF8); \ } \ @@ -1162,7 +1320,7 @@ if ((!reginfo || regtry(reginfo, &s))) \ #define REXEC_FBC_CSCAN_TAINT(CoNdUtF8,CoNd) \ PL_reg_flags |= RF_tainted; \ - if (do_utf8) { \ + if (utf8_target) { \ REXEC_FBC_UTF8_CLASS_SCAN(CoNdUtF8); \ } \ else { \ @@ -1192,7 +1350,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, unsigned int c2; char *e; register I32 tmp = 1; /* Scratch variable? */ - register const bool do_utf8 = PL_reg_match_utf8; + register const bool utf8_target = PL_reg_match_utf8; RXi_GET_DECL(prog,progi); PERL_ARGS_ASSERT_FIND_BYCLASS; @@ -1200,10 +1358,10 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* We know what class it must start with. */ switch (OP(c)) { case ANYOF: - if (do_utf8) { + if (utf8_target) { REXEC_FBC_UTF8_CLASS_SCAN((ANYOF_FLAGS(c) & ANYOF_UNICODE) || !UTF8_IS_INVARIANT((U8)s[0]) ? - reginclass(prog, c, (U8*)s, 0, do_utf8) : + reginclass(prog, c, (U8*)s, 0, utf8_target) : REGINCLASS(prog, c, (U8*)s)); } else { @@ -1238,7 +1396,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, m = STRING(c); ln = STR_LEN(c); /* length to match in octets/bytes */ lnc = (I32) ln; /* length to match in characters */ - if (UTF) { + if (UTF_PATTERN) { STRLEN ulen1, ulen2; U8 *sm = (U8 *) m; U8 tmpbuf1[UTF8_MAXBYTES_CASE+1]; @@ -1297,9 +1455,9 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, * than just upper and lower: one needs to use * the so-called folding case for case-insensitive * matching (called "loose matching" in Unicode). - * ibcmp_utf8() will do just that. */ + * foldEQ_utf8() will do just that. */ - if (do_utf8 || UTF) { + if (utf8_target || UTF_PATTERN) { UV c, f; U8 tmpbuf [UTF8_MAXBYTES+1]; STRLEN len = 1; @@ -1309,7 +1467,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* Upper and lower of 1st char are equal - * probably not a "letter". */ while (s <= e) { - if (do_utf8) { + if (utf8_target) { c = utf8n_to_uvchr((U8*)s, UTF8_MAXBYTES, &len, uniflags); } else { @@ -1320,7 +1478,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } else { while (s <= e) { - if (do_utf8) { + if (utf8_target) { c = utf8n_to_uvchr((U8*)s, UTF8_MAXBYTES, &len, uniflags); } else { @@ -1354,7 +1512,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, PL_reg_flags |= RF_tainted; /* FALL THROUGH */ case BOUND: - if (do_utf8) { + if (utf8_target) { if (s == PL_bostr) tmp = '\n'; else { @@ -1366,7 +1524,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, LOAD_UTF8_CHARCLASS_ALNUM(); REXEC_FBC_UTF8_SCAN( if (tmp == !(OP(c) == BOUND ? - (bool)swash_fetch(PL_utf8_alnum, (U8*)s, do_utf8) : + cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target)) : isALNUM_LC_utf8((U8*)s))) { tmp = !tmp; @@ -1374,12 +1532,19 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } ); } - else { + else { /* Not utf8 */ tmp = (s != PL_bostr) ? UCHARAT(s - 1) : '\n'; - tmp = ((OP(c) == BOUND ? isALNUM(tmp) : isALNUM_LC(tmp)) != 0); + tmp = cBOOL((OP(c) == BOUNDL) + ? isALNUM_LC(tmp) + : (isWORDCHAR_L1(tmp) + && (isASCII(tmp) || (FLAGS(c) & USE_UNI)))); REXEC_FBC_SCAN( if (tmp == - !(OP(c) == BOUND ? isALNUM(*s) : isALNUM_LC(*s))) { + !((OP(c) == BOUNDL) + ? isALNUM_LC(*s) + : (isWORDCHAR_L1((U8) *s) + && (isASCII((U8) *s) || (FLAGS(c) & USE_UNI))))) + { tmp = !tmp; REXEC_FBC_TRYIT; } @@ -1392,7 +1557,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, PL_reg_flags |= RF_tainted; /* FALL THROUGH */ case NBOUND: - if (do_utf8) { + if (utf8_target) { if (s == PL_bostr) tmp = '\n'; else { @@ -1404,7 +1569,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, LOAD_UTF8_CHARCLASS_ALNUM(); REXEC_FBC_UTF8_SCAN( if (tmp == !(OP(c) == NBOUND ? - (bool)swash_fetch(PL_utf8_alnum, (U8*)s, do_utf8) : + cBOOL(swash_fetch(PL_utf8_alnum, (U8*)s, utf8_target)) : isALNUM_LC_utf8((U8*)s))) tmp = !tmp; else REXEC_FBC_TRYIT; @@ -1412,12 +1577,19 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } else { tmp = (s != PL_bostr) ? UCHARAT(s - 1) : '\n'; - tmp = ((OP(c) == NBOUND ? - isALNUM(tmp) : isALNUM_LC(tmp)) != 0); + tmp = cBOOL((OP(c) == NBOUNDL) + ? isALNUM_LC(tmp) + : (isWORDCHAR_L1(tmp) + && (isASCII(tmp) || (FLAGS(c) & USE_UNI)))); REXEC_FBC_SCAN( - if (tmp == - !(OP(c) == NBOUND ? isALNUM(*s) : isALNUM_LC(*s))) + if (tmp == ! cBOOL( + (OP(c) == NBOUNDL) + ? isALNUM_LC(*s) + : (isWORDCHAR_L1((U8) *s) + && (isASCII((U8) *s) || (FLAGS(c) & USE_UNI))))) + { tmp = !tmp; + } else REXEC_FBC_TRYIT; ); } @@ -1426,9 +1598,9 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, break; case ALNUM: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_ALNUM(), - swash_fetch(PL_utf8_alnum, (U8*)s, do_utf8), - isALNUM(*s) + LOAD_UTF8_CHARCLASS_PERL_WORD(), + swash_fetch(RE_utf8_perl_word, (U8*)s, utf8_target), + (FLAGS(c) & USE_UNI) ? isWORDCHAR_L1((U8) *s) : isALNUM(*s) ); case ALNUML: REXEC_FBC_CSCAN_TAINT( @@ -1437,9 +1609,9 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case NALNUM: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_ALNUM(), - !swash_fetch(PL_utf8_alnum, (U8*)s, do_utf8), - !isALNUM(*s) + LOAD_UTF8_CHARCLASS_PERL_WORD(), + !swash_fetch(RE_utf8_perl_word, (U8*)s, utf8_target), + ! ((FLAGS(c) & USE_UNI) ? isWORDCHAR_L1((U8) *s) : isALNUM(*s)) ); case NALNUML: REXEC_FBC_CSCAN_TAINT( @@ -1448,9 +1620,9 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case SPACE: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_SPACE(), - *s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, do_utf8), - isSPACE(*s) + LOAD_UTF8_CHARCLASS_PERL_SPACE(), + *s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, utf8_target), + isSPACE_L1((U8) *s) && (isASCII((U8) *s) || (FLAGS(c) & USE_UNI)) ); case SPACEL: REXEC_FBC_CSCAN_TAINT( @@ -1459,9 +1631,9 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case NSPACE: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_SPACE(), - !(*s == ' ' || swash_fetch(PL_utf8_space,(U8*)s, do_utf8)), - !isSPACE(*s) + LOAD_UTF8_CHARCLASS_PERL_SPACE(), + !(*s == ' ' || swash_fetch(RE_utf8_perl_space,(U8*)s, utf8_target)), + !(isSPACE_L1((U8) *s) && (isASCII((U8) *s) || (FLAGS(c) & USE_UNI))) ); case NSPACEL: REXEC_FBC_CSCAN_TAINT( @@ -1470,8 +1642,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case DIGIT: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_DIGIT(), - swash_fetch(PL_utf8_digit,(U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_POSIX_DIGIT(), + swash_fetch(RE_utf8_posix_digit,(U8*)s, utf8_target), isDIGIT(*s) ); case DIGITL: @@ -1481,8 +1653,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, ); case NDIGIT: REXEC_FBC_CSCAN_PRELOAD( - LOAD_UTF8_CHARCLASS_DIGIT(), - !swash_fetch(PL_utf8_digit,(U8*)s, do_utf8), + LOAD_UTF8_CHARCLASS_POSIX_DIGIT(), + !swash_fetch(RE_utf8_posix_digit,(U8*)s, utf8_target), !isDIGIT(*s) ); case NDIGITL: @@ -1604,21 +1776,27 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, DEBUG_TRIE_EXECUTE_r( if ( uc <= (U8*)last_start && !BITMAP_TEST(bitmap,*uc) ) { dump_exec_pos( (char *)uc, c, strend, real_start, - (char *)uc, do_utf8 ); + (char *)uc, utf8_target ); PerlIO_printf( Perl_debug_log, " Scanning for legal start char...\n"); } - ); - while ( uc <= (U8*)last_start && !BITMAP_TEST(bitmap,*uc) ) { - uc++; - } + ); + if (utf8_target) { + while ( uc <= (U8*)last_start && !BITMAP_TEST(bitmap,*uc) ) { + uc += UTF8SKIP(uc); + } + } else { + while ( uc <= (U8*)last_start && !BITMAP_TEST(bitmap,*uc) ) { + uc++; + } + } s= (char *)uc; } if (uc >(U8*)last_start) break; } if ( word ) { - U8 *lpos= points[ (pointpos - trie->wordlen[word-1] ) % maxlen ]; + U8 *lpos= points[ (pointpos - trie->wordinfo[word].len) % maxlen ]; if (!leftmost || lpos < leftmost) { DEBUG_r(accepted_word=word); leftmost= lpos; @@ -1632,7 +1810,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, foldbuf, uniflags); DEBUG_TRIE_EXECUTE_r({ dump_exec_pos( (char *)uc, c, strend, real_start, - s, do_utf8 ); + s, utf8_target ); PerlIO_printf(Perl_debug_log, " Charid:%3u CP:%4"UVxf" ", charid, uvc); @@ -1647,7 +1825,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, DEBUG_TRIE_EXECUTE_r({ if (failed) dump_exec_pos( (char *)uc, c, strend, real_start, - s, do_utf8 ); + s, utf8_target ); PerlIO_printf( Perl_debug_log, "%sState: %4"UVxf", word=%"UVxf, failed ? " Fail transition to " : "", @@ -1655,14 +1833,13 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, }); if ( base ) { U32 tmp; + I32 offset; if (charid && - (base + charid > trie->uniquecharcount ) - && (base + charid - 1 - trie->uniquecharcount - < trie->lasttrans) - && trie->trans[base + charid - 1 - - trie->uniquecharcount].check == state - && (tmp=trie->trans[base + charid - 1 - - trie->uniquecharcount ].next)) + ( ((offset = base + charid + - 1 - trie->uniquecharcount)) >= 0) + && ((U32)offset < trie->lasttrans) + && trie->trans[offset].check == state + && (tmp=trie->trans[offset].next)) { DEBUG_TRIE_EXECUTE_r( PerlIO_printf( Perl_debug_log," - legal\n")); @@ -1692,7 +1869,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } } if ( aho->states[ state ].wordnum ) { - U8 *lpos = points[ (pointpos - trie->wordlen[aho->states[ state ].wordnum-1]) % maxlen ]; + U8 *lpos = points[ (pointpos - trie->wordinfo[aho->states[ state ].wordnum].len) % maxlen ]; if (!leftmost || lpos < leftmost) { DEBUG_r(accepted_word=aho->states[ state ].wordnum); leftmost = lpos; @@ -1759,7 +1936,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre I32 end_shift = 0; /* Same for the end. */ /* CC */ I32 scream_pos = -1; /* Internal iterator of scream. */ char *scream_olds = NULL; - const bool do_utf8 = (bool)DO_UTF8(sv); + const bool utf8_target = cBOOL(DO_UTF8(sv)); I32 multiline; RXi_GET_DECL(prog,progi); regmatch_info reginfo; /* create some info to pass to regtry etc */ @@ -1778,9 +1955,9 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre multiline = prog->extflags & RXf_PMf_MULTILINE; reginfo.prog = rx; /* Yes, sorry that this is confusing. */ - RX_MATCH_UTF8_set(rx, do_utf8); + RX_MATCH_UTF8_set(rx, utf8_target); DEBUG_EXECUTE_r( - debug_start_match(rx, do_utf8, startpos, strend, + debug_start_match(rx, utf8_target, startpos, strend, "Matching"); ); @@ -1895,33 +2072,68 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre end = HOP3c(strend, -dontbother, strbeg) - 1; /* for multiline we only have to try after newlines */ if (prog->check_substr || prog->check_utf8) { - if (s == startpos) - goto after_try; - while (1) { - if (regtry(®info, &s)) - goto got_it; - after_try: - if (s > end) - goto phooey; - if (prog->extflags & RXf_USE_INTUIT) { - s = re_intuit_start(rx, sv, s + 1, strend, flags, NULL); - if (!s) - goto phooey; - } - else - s++; - } - } else { - if (s > startpos) + /* because of the goto we can not easily reuse the macros for bifurcating the + unicode/non-unicode match modes here like we do elsewhere - demerphq */ + if (utf8_target) { + if (s == startpos) + goto after_try_utf8; + while (1) { + if (regtry(®info, &s)) { + goto got_it; + } + after_try_utf8: + if (s > end) { + goto phooey; + } + if (prog->extflags & RXf_USE_INTUIT) { + s = re_intuit_start(rx, sv, s + UTF8SKIP(s), strend, flags, NULL); + if (!s) { + goto phooey; + } + } + else { + s += UTF8SKIP(s); + } + } + } /* end search for check string in unicode */ + else { + if (s == startpos) { + goto after_try_latin; + } + while (1) { + if (regtry(®info, &s)) { + goto got_it; + } + after_try_latin: + if (s > end) { + goto phooey; + } + if (prog->extflags & RXf_USE_INTUIT) { + s = re_intuit_start(rx, sv, s + 1, strend, flags, NULL); + if (!s) { + goto phooey; + } + } + else { + s++; + } + } + } /* end search for check string in latin*/ + } /* end search for check string */ + else { /* search for newline */ + if (s > startpos) { + /*XXX: The s-- is almost definitely wrong here under unicode - demeprhq*/ s--; + } + /* We can use a more efficient search as newlines are the same in unicode as they are in latin */ while (s < end) { if (*s++ == '\n') { /* don't need PL_utf8skip here */ if (regtry(®info, &s)) goto got_it; } - } - } - } + } + } /* end search for newline */ + } /* end anchored/multiline check string search */ goto phooey; } else if (RXf_GPOS_CHECK == (prog->extflags & RXf_GPOS_CHECK)) { @@ -1938,16 +2150,16 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre /* Messy cases: unanchored match. */ if ((prog->anchored_substr || prog->anchored_utf8) && prog->intflags & PREGf_SKIP) { /* we have /x+whatever/ */ - /* it must be a one character string (XXXX Except UTF?) */ + /* it must be a one character string (XXXX Except UTF_PATTERN?) */ char ch; #ifdef DEBUGGING int did_match = 0; #endif - if (!(do_utf8 ? prog->anchored_utf8 : prog->anchored_substr)) - do_utf8 ? to_utf8_substr(prog) : to_byte_substr(prog); - ch = SvPVX_const(do_utf8 ? prog->anchored_utf8 : prog->anchored_substr)[0]; + if (!(utf8_target ? prog->anchored_utf8 : prog->anchored_substr)) + utf8_target ? to_utf8_substr(prog) : to_byte_substr(prog); + ch = SvPVX_const(utf8_target ? prog->anchored_utf8 : prog->anchored_substr)[0]; - if (do_utf8) { + if (utf8_target) { REXEC_FBC_SCAN( if (*s == ch) { DEBUG_EXECUTE_r( did_match = 1 ); @@ -1987,14 +2199,14 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre int did_match = 0; #endif if (prog->anchored_substr || prog->anchored_utf8) { - if (!(do_utf8 ? prog->anchored_utf8 : prog->anchored_substr)) - do_utf8 ? to_utf8_substr(prog) : to_byte_substr(prog); - must = do_utf8 ? prog->anchored_utf8 : prog->anchored_substr; + if (!(utf8_target ? prog->anchored_utf8 : prog->anchored_substr)) + utf8_target ? to_utf8_substr(prog) : to_byte_substr(prog); + must = utf8_target ? prog->anchored_utf8 : prog->anchored_substr; back_max = back_min = prog->anchored_offset; } else { - if (!(do_utf8 ? prog->float_utf8 : prog->float_substr)) - do_utf8 ? to_utf8_substr(prog) : to_byte_substr(prog); - must = do_utf8 ? prog->float_utf8 : prog->float_substr; + if (!(utf8_target ? prog->float_utf8 : prog->float_substr)) + utf8_target ? to_utf8_substr(prog) : to_byte_substr(prog); + must = utf8_target ? prog->float_utf8 : prog->float_substr; back_max = prog->float_max_offset; back_min = prog->float_min_offset; } @@ -2042,7 +2254,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre last1 = HOPc(s, -back_min); s = t; } - if (do_utf8) { + if (utf8_target) { while (s <= last1) { if (regtry(®info, &s)) goto got_it; @@ -2058,7 +2270,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre } } DEBUG_EXECUTE_r(if (!did_match) { - RE_PV_QUOTED_DECL(quoted, do_utf8, PERL_DEBUG_PAD_ZERO(0), + RE_PV_QUOTED_DECL(quoted, utf8_target, PERL_DEBUG_PAD_ZERO(0), SvPVX_const(must), RE_SV_DUMPLEN(must), 30); PerlIO_printf(Perl_debug_log, "Did not find %s substr %s%s...\n", ((must == prog->anchored_substr || must == prog->anchored_utf8) @@ -2078,10 +2290,10 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre SV * const prop = sv_newmortal(); regprop(prog, prop, c); { - RE_PV_QUOTED_DECL(quoted,do_utf8,PERL_DEBUG_PAD_ZERO(1), + RE_PV_QUOTED_DECL(quoted,utf8_target,PERL_DEBUG_PAD_ZERO(1), s,strend-s,60); PerlIO_printf(Perl_debug_log, - "Matching stclass %.*s against %s (%d chars)\n", + "Matching stclass %.*s against %s (%d bytes)\n", (int)SvCUR(prop), SvPVX_const(prop), quoted, (int)(strend - s)); } @@ -2097,9 +2309,9 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre char *last; SV* float_real; - if (!(do_utf8 ? prog->float_utf8 : prog->float_substr)) - do_utf8 ? to_utf8_substr(prog) : to_byte_substr(prog); - float_real = do_utf8 ? prog->float_utf8 : prog->float_substr; + if (!(utf8_target ? prog->float_utf8 : prog->float_substr)) + utf8_target ? to_utf8_substr(prog) : to_byte_substr(prog); + float_real = utf8_target ? prog->float_utf8 : prog->float_substr; if (flags & REXEC_SCREAM) { last = screaminstr(sv, float_real, s - strbeg, @@ -2143,7 +2355,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, register char *stre dontbother = minlen - 1; strend -= dontbother; /* this one's always in bytes! */ /* We don't know much -- general case. */ - if (do_utf8) { + if (utf8_target) { for (;;) { if (regtry(®info, &s)) goto got_it; @@ -2387,9 +2599,6 @@ S_regtry(pTHX_ regmatch_info *reginfo, char **startpos) #define REPORT_CODE_OFF 32 -/* Make sure there is a test for this +1 options in re_tests */ -#define TRIE_INITAL_ACCEPT_BUFFLEN 4; - #define CHRTEST_UNINIT -1001 /* c1/c2 haven't been calculated yet */ #define CHRTEST_VOID -1000 /* the c1/c2 "next char" test should be skipped */ @@ -2568,7 +2777,7 @@ regmatch(), slabs allocated since entry are freed. #define DEBUG_STATE_pp(pp) \ DEBUG_STATE_r({ \ - DUMP_EXEC_POS(locinput, scan, do_utf8); \ + DUMP_EXEC_POS(locinput, scan, utf8_target); \ PerlIO_printf(Perl_debug_log, \ " %*s"pp" %s%s%s%s%s\n", \ depth*2, "", \ @@ -2586,7 +2795,7 @@ regmatch(), slabs allocated since entry are freed. #ifdef DEBUGGING STATIC void -S_debug_start_match(pTHX_ const REGEXP *prog, const bool do_utf8, +S_debug_start_match(pTHX_ const REGEXP *prog, const bool utf8_target, const char *start, const char *end, const char *blurb) { const bool utf8_pat = RX_UTF8(prog) ? 1 : 0; @@ -2599,18 +2808,18 @@ S_debug_start_match(pTHX_ const REGEXP *prog, const bool do_utf8, RE_PV_QUOTED_DECL(s0, utf8_pat, PERL_DEBUG_PAD_ZERO(0), RX_PRECOMP_const(prog), RX_PRELEN(prog), 60); - RE_PV_QUOTED_DECL(s1, do_utf8, PERL_DEBUG_PAD_ZERO(1), + RE_PV_QUOTED_DECL(s1, utf8_target, PERL_DEBUG_PAD_ZERO(1), start, end - start, 60); PerlIO_printf(Perl_debug_log, "%s%s REx%s %s against %s\n", PL_colors[4], blurb, PL_colors[5], s0, s1); - if (do_utf8||utf8_pat) + if (utf8_target||utf8_pat) PerlIO_printf(Perl_debug_log, "UTF-8 %s%s%s...\n", utf8_pat ? "pattern" : "", - utf8_pat && do_utf8 ? " and " : "", - do_utf8 ? "string" : "" + utf8_pat && utf8_target ? " and " : "", + utf8_target ? "string" : "" ); } } @@ -2621,7 +2830,7 @@ S_dump_exec_pos(pTHX_ const char *locinput, const char *loc_regeol, const char *loc_bostr, const char *loc_reg_starttry, - const bool do_utf8) + const bool utf8_target) { const int docolor = *PL_colors[0] || *PL_colors[2] || *PL_colors[4]; const int taill = (docolor ? 10 : 7); /* 3 chars for "> <" */ @@ -2638,20 +2847,20 @@ S_dump_exec_pos(pTHX_ const char *locinput, PERL_ARGS_ASSERT_DUMP_EXEC_POS; - while (do_utf8 && UTF8_IS_CONTINUATION(*(U8*)(locinput - pref_len))) + while (utf8_target && UTF8_IS_CONTINUATION(*(U8*)(locinput - pref_len))) pref_len++; pref0_len = pref_len - (locinput - loc_reg_starttry); if (l + pref_len < (5 + taill) && l < loc_regeol - locinput) l = ( loc_regeol - locinput > (5 + taill) - pref_len ? (5 + taill) - pref_len : loc_regeol - locinput); - while (do_utf8 && UTF8_IS_CONTINUATION(*(U8*)(locinput + l))) + while (utf8_target && UTF8_IS_CONTINUATION(*(U8*)(locinput + l))) l--; if (pref0_len < 0) pref0_len = 0; if (pref0_len > pref_len) pref0_len = pref_len; { - const int is_uni = (do_utf8 && OP(scan) != CANY) ? 1 : 0; + const int is_uni = (utf8_target && OP(scan) != CANY) ? 1 : 0; RE_PV_COLOR_DECL(s0,len0,is_uni,PERL_DEBUG_PAD(0), (locinput - pref_len),pref0_len, 60, 4, 5); @@ -2738,7 +2947,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) dMY_CXT; #endif dVAR; - register const bool do_utf8 = PL_reg_match_utf8; + register const bool utf8_target = PL_reg_match_utf8; const U32 uniflags = UTF8_ALLOW_DEFAULT; REGEXP *rex_sv = reginfo->prog; regexp *rex = (struct regexp *)SvANY(rex_sv); @@ -2827,7 +3036,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) DEBUG_EXECUTE_r( { SV * const prop = sv_newmortal(); regnode *rnext=regnext(scan); - DUMP_EXEC_POS( locinput, scan, do_utf8 ); + DUMP_EXEC_POS( locinput, scan, utf8_target ); regprop(rex, prop, scan); PerlIO_printf(Perl_debug_log, @@ -2905,7 +3114,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) case SANY: if (!nextchr && locinput >= PL_regeol) sayNO; - if (do_utf8) { + if (utf8_target) { locinput += PL_utf8skip[nextchr]; if (locinput > PL_regeol) sayNO; @@ -2922,7 +3131,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) case REG_ANY: if ((!nextchr && locinput >= PL_regeol) || nextchr == '\n') sayNO; - if (do_utf8) { + if (utf8_target) { locinput += PL_utf8skip[nextchr]; if (locinput > PL_regeol) sayNO; @@ -2938,7 +3147,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) /* In this case the charclass data is available inline so we can fail fast without a lot of extra overhead. */ - if (scan->flags == EXACT || !do_utf8) { + if (scan->flags == EXACT || !utf8_target) { if(!ANYOF_BITMAP_TEST(scan, *locinput)) { DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, @@ -2951,6 +3160,50 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) } /* FALL THROUGH */ case TRIE: + /* the basic plan of execution of the trie is: + * At the beginning, run though all the states, and + * find the longest-matching word. Also remember the position + * of the shortest matching word. For example, this pattern: + * 1 2 3 4 5 + * ab|a|x|abcd|abc + * when matched against the string "abcde", will generate + * accept states for all words except 3, with the longest + * matching word being 4, and the shortest being 1 (with + * the position being after char 1 of the string). + * + * Then for each matching word, in word order (i.e. 1,2,4,5), + * we run the remainder of the pattern; on each try setting + * the current position to the character following the word, + * returning to try the next word on failure. + * + * We avoid having to build a list of words at runtime by + * using a compile-time structure, wordinfo[].prev, which + * gives, for each word, the previous accepting word (if any). + * In the case above it would contain the mappings 1->2, 2->0, + * 3->0, 4->5, 5->1. We can use this table to generate, from + * the longest word (4 above), a list of all words, by + * following the list of prev pointers; this gives us the + * unordered list 4,5,1,2. Then given the current word we have + * just tried, we can go through the list and find the + * next-biggest word to try (so if we just failed on word 2, + * the next in the list is 4). + * + * Since at runtime we don't record the matching position in + * the string for each word, we have to work that out for + * each word we're about to process. The wordinfo table holds + * the character length of each word; given that we recorded + * at the start: the position of the shortest word and its + * length in chars, we just need to move the pointer the + * difference between the two char lengths. Depending on + * Unicode status and folding, that's cheap or expensive. + * + * This algorithm is optimised for the case where are only a + * small number of accept states, i.e. 0,1, or maybe 2. + * With lots of accepts states, and having to try all of them, + * it becomes quadratic on number of accept states to find all + * the next words. + */ + { /* what type of TRIE am I? (utf8 makes this contextual) */ DECL_TRIE_TYPE(scan); @@ -2970,7 +3223,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) "%*s %smatched empty string...%s\n", REPORT_CODE_OFF+depth*2, "", PL_colors[4], PL_colors[5]) ); - break; + if (!trie->jump) + break; } else { DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, @@ -2987,85 +3241,71 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) STRLEN len = 0; STRLEN foldlen = 0; U8 *uscan = (U8*)NULL; - STRLEN bufflen=0; - SV *sv_accept_buff = NULL; U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ]; + U32 charcount = 0; /* how many input chars we have matched */ + U32 accepted = 0; /* have we seen any accepting states? */ - ST.accepted = 0; /* how many accepting states we have seen */ ST.B = next; ST.jump = trie->jump; ST.me = scan; - /* - traverse the TRIE keeping track of all accepting states - we transition through until we get to a failing node. - */ + ST.firstpos = NULL; + ST.longfold = FALSE; /* char longer if folded => it's harder */ + ST.nextword = 0; + + /* fully traverse the TRIE; note the position of the + shortest accept state and the wordnum of the longest + accept state */ while ( state && uc <= (U8*)PL_regeol ) { U32 base = trie->states[ state ].trans.base; UV uvc = 0; - U16 charid; - /* We use charid to hold the wordnum as we don't use it - for charid until after we have done the wordnum logic. - We define an alias just so that the wordnum logic reads - more naturally. */ - -#define got_wordnum charid - got_wordnum = trie->states[ state ].wordnum; - - if ( got_wordnum ) { - if ( ! ST.accepted ) { - ENTER; - SAVETMPS; /* XXX is this necessary? dmq */ - bufflen = TRIE_INITAL_ACCEPT_BUFFLEN; - sv_accept_buff=newSV(bufflen * - sizeof(reg_trie_accepted) - 1); - SvCUR_set(sv_accept_buff, 0); - SvPOK_on(sv_accept_buff); - sv_2mortal(sv_accept_buff); - SAVETMPS; - ST.accept_buff = - (reg_trie_accepted*)SvPV_nolen(sv_accept_buff ); - } - do { - if (ST.accepted >= bufflen) { - bufflen *= 2; - ST.accept_buff =(reg_trie_accepted*) - SvGROW(sv_accept_buff, - bufflen * sizeof(reg_trie_accepted)); + U16 charid = 0; + U16 wordnum; + wordnum = trie->states[ state ].wordnum; + + if (wordnum) { /* it's an accept state */ + if (!accepted) { + accepted = 1; + /* record first match position */ + if (ST.longfold) { + ST.firstpos = (U8*)locinput; + ST.firstchars = 0; } - SvCUR_set(sv_accept_buff,SvCUR(sv_accept_buff) - + sizeof(reg_trie_accepted)); - - - ST.accept_buff[ST.accepted].wordnum = got_wordnum; - ST.accept_buff[ST.accepted].endpos = uc; - ++ST.accepted; - } while (trie->nextword && (got_wordnum= trie->nextword[got_wordnum])); + else { + ST.firstpos = uc; + ST.firstchars = charcount; + } + } + if (!ST.nextword || wordnum < ST.nextword) + ST.nextword = wordnum; + ST.topword = wordnum; } -#undef got_wordnum DEBUG_TRIE_EXECUTE_r({ - DUMP_EXEC_POS( (char *)uc, scan, do_utf8 ); + DUMP_EXEC_POS( (char *)uc, scan, utf8_target ); PerlIO_printf( Perl_debug_log, - "%*s %sState: %4"UVxf" Accepted: %4"UVxf" ", + "%*s %sState: %4"UVxf" Accepted: %c ", 2+depth * 2, "", PL_colors[4], - (UV)state, (UV)ST.accepted ); + (UV)state, (accepted ? 'Y' : 'N')); }); + /* read a char and goto next state */ if ( base ) { + I32 offset; REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc, uscan, len, uvc, charid, foldlen, foldbuf, uniflags); - + charcount++; + if (foldlen>0) + ST.longfold = TRUE; if (charid && - (base + charid > trie->uniquecharcount ) - && (base + charid - 1 - trie->uniquecharcount - < trie->lasttrans) - && trie->trans[base + charid - 1 - - trie->uniquecharcount].check == state) + ( ((offset = + base + charid - 1 - trie->uniquecharcount)) >= 0) + + && ((U32)offset < trie->lasttrans) + && trie->trans[offset].check == state) { - state = trie->trans[base + charid - 1 - - trie->uniquecharcount ].next; + state = trie->trans[offset].next; } else { state = 0; @@ -3082,77 +3322,38 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) charid, uvc, (UV)state, PL_colors[5] ); ); } - if (!ST.accepted ) + if (!accepted) sayNO; + /* calculate total number of accept states */ + { + U16 w = ST.topword; + accepted = 0; + while (w) { + w = trie->wordinfo[w].prev; + accepted++; + } + ST.accepted = accepted; + } + DEBUG_EXECUTE_r( PerlIO_printf( Perl_debug_log, "%*s %sgot %"IVdf" possible matches%s\n", REPORT_CODE_OFF + depth * 2, "", PL_colors[4], (IV)ST.accepted, PL_colors[5] ); ); + goto trie_first_try; /* jump into the fail handler */ }} - goto trie_first_try; /* jump into the fail handler */ /* NOTREACHED */ - case TRIE_next_fail: /* we failed - try next alterative */ + + case TRIE_next_fail: /* we failed - try next alternative */ if ( ST.jump) { REGCP_UNWIND(ST.cp); for (n = *PL_reglastparen; n > ST.lastparen; n--) PL_regoffs[n].end = -1; *PL_reglastparen = n; } - trie_first_try: - if (do_cutgroup) { - do_cutgroup = 0; - no_final = 0; - } - - if ( ST.jump) { - ST.lastparen = *PL_reglastparen; - REGCP_SET(ST.cp); - } - if ( ST.accepted == 1 ) { - /* only one choice left - just continue */ - DEBUG_EXECUTE_r({ - AV *const trie_words - = MUTABLE_AV(rexi->data->data[ARG(ST.me)+TRIE_WORDS_OFFSET]); - SV ** const tmp = av_fetch( trie_words, - ST.accept_buff[ 0 ].wordnum-1, 0 ); - SV *sv= tmp ? sv_newmortal() : NULL; - - PerlIO_printf( Perl_debug_log, - "%*s %sonly one match left: #%d <%s>%s\n", - REPORT_CODE_OFF+depth*2, "", PL_colors[4], - ST.accept_buff[ 0 ].wordnum, - tmp ? pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), 0, - PL_colors[0], PL_colors[1], - (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) - ) - : "not compiled under -Dr", - PL_colors[5] ); - }); - PL_reginput = (char *)ST.accept_buff[ 0 ].endpos; - /* in this case we free tmps/leave before we call regmatch - as we wont be using accept_buff again. */ - - locinput = PL_reginput; - nextchr = UCHARAT(locinput); - if ( !ST.jump || !ST.jump[ST.accept_buff[0].wordnum]) - scan = ST.B; - else - scan = ST.me + ST.jump[ST.accept_buff[0].wordnum]; - if (!has_cutgroup) { - FREETMPS; - LEAVE; - } else { - ST.accepted--; - PUSH_YES_STATE_GOTO(TRIE_next, scan); - } - - continue; /* execute rest of RE */ - } - - if ( !ST.accepted-- ) { + if (!--ST.accepted) { DEBUG_EXECUTE_r({ PerlIO_printf( Perl_debug_log, "%*s %sTRIE failed...%s\n", @@ -3160,97 +3361,140 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) PL_colors[4], PL_colors[5] ); }); - FREETMPS; - LEAVE; sayNO_SILENT; - /*NOTREACHED*/ - } + } + { + /* Find next-highest word to process. Note that this code + * is O(N^2) per trie run (O(N) per branch), so keep tight */ + register U16 min = 0; + register U16 word; + register U16 const nextword = ST.nextword; + register reg_trie_wordinfo * const wordinfo + = ((reg_trie_data*)rexi->data->data[ARG(ST.me)])->wordinfo; + for (word=ST.topword; word; word=wordinfo[word].prev) { + if (word > nextword && (!min || word < min)) + min = word; + } + ST.nextword = min; + } - /* - There are at least two accepting states left. Presumably - the number of accepting states is going to be low, - typically two. So we simply scan through to find the one - with lowest wordnum. Once we find it, we swap the last - state into its place and decrement the size. We then try to - match the rest of the pattern at the point where the word - ends. If we succeed, control just continues along the - regex; if we fail we return here to try the next accepting - state - */ + trie_first_try: + if (do_cutgroup) { + do_cutgroup = 0; + no_final = 0; + } - { - U32 best = 0; - U32 cur; - for( cur = 1 ; cur <= ST.accepted ; cur++ ) { - DEBUG_TRIE_EXECUTE_r( - PerlIO_printf( Perl_debug_log, - "%*s %sgot %"IVdf" (%d) as best, looking at %"IVdf" (%d)%s\n", - REPORT_CODE_OFF + depth * 2, "", PL_colors[4], - (IV)best, ST.accept_buff[ best ].wordnum, (IV)cur, - ST.accept_buff[ cur ].wordnum, PL_colors[5] ); - ); + if ( ST.jump) { + ST.lastparen = *PL_reglastparen; + REGCP_SET(ST.cp); + } - if (ST.accept_buff[cur].wordnum < - ST.accept_buff[best].wordnum) - best = cur; + /* find start char of end of current word */ + { + U32 chars; /* how many chars to skip */ + U8 *uc = ST.firstpos; + reg_trie_data * const trie + = (reg_trie_data*)rexi->data->data[ARG(ST.me)]; + + assert((trie->wordinfo[ST.nextword].len - trie->prefixlen) + >= ST.firstchars); + chars = (trie->wordinfo[ST.nextword].len - trie->prefixlen) + - ST.firstchars; + + if (ST.longfold) { + /* the hard option - fold each char in turn and find + * its folded length (which may be different */ + U8 foldbuf[UTF8_MAXBYTES_CASE + 1]; + STRLEN foldlen; + STRLEN len; + UV uvc; + U8 *uscan; + + while (chars) { + if (utf8_target) { + uvc = utf8n_to_uvuni((U8*)uc, UTF8_MAXLEN, &len, + uniflags); + uc += len; + } + else { + uvc = *uc; + uc++; + } + uvc = to_uni_fold(uvc, foldbuf, &foldlen); + uscan = foldbuf; + while (foldlen) { + if (!--chars) + break; + uvc = utf8n_to_uvuni(uscan, UTF8_MAXLEN, &len, + uniflags); + uscan += len; + foldlen -= len; + } + } + } + else { + if (utf8_target) + while (chars--) + uc += UTF8SKIP(uc); + else + uc += chars; } + PL_reginput = (char *)uc; + } - DEBUG_EXECUTE_r({ - AV *const trie_words - = MUTABLE_AV(rexi->data->data[ARG(ST.me)+TRIE_WORDS_OFFSET]); - SV ** const tmp = av_fetch( trie_words, - ST.accept_buff[ best ].wordnum - 1, 0 ); - regnode *nextop=(!ST.jump || !ST.jump[ST.accept_buff[best].wordnum]) ? - ST.B : - ST.me + ST.jump[ST.accept_buff[best].wordnum]; - SV *sv= tmp ? sv_newmortal() : NULL; - - PerlIO_printf( Perl_debug_log, - "%*s %strying alternation #%d <%s> at node #%d %s\n", - REPORT_CODE_OFF+depth*2, "", PL_colors[4], - ST.accept_buff[best].wordnum, - tmp ? pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), 0, - PL_colors[0], PL_colors[1], - (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) - ) : "not compiled under -Dr", - REG_NODE_NUM(nextop), - PL_colors[5] ); - }); + scan = (ST.jump && ST.jump[ST.nextword]) + ? ST.me + ST.jump[ST.nextword] + : ST.B; - if ( best 1 || has_cutgroup) { + PUSH_STATE_GOTO(TRIE_next, scan); + /* NOTREACHED */ } + /* only one choice left - just continue */ + DEBUG_EXECUTE_r({ + AV *const trie_words + = MUTABLE_AV(rexi->data->data[ARG(ST.me)+TRIE_WORDS_OFFSET]); + SV ** const tmp = av_fetch( trie_words, + ST.nextword-1, 0 ); + SV *sv= tmp ? sv_newmortal() : NULL; + + PerlIO_printf( Perl_debug_log, + "%*s %sonly one match left, short-circuiting: #%d <%s>%s\n", + REPORT_CODE_OFF+depth*2, "", PL_colors[4], + ST.nextword, + tmp ? pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), 0, + PL_colors[0], PL_colors[1], + (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) + ) + : "not compiled under -Dr", + PL_colors[5] ); + }); + + locinput = PL_reginput; + nextchr = UCHARAT(locinput); + continue; /* execute rest of RE */ /* NOTREACHED */ - case TRIE_next: - /* we dont want to throw this away, see bug 57042*/ - if (oreplsv != GvSV(PL_replgv)) - sv_setsv(oreplsv, GvSV(PL_replgv)); - FREETMPS; - LEAVE; - sayYES; #undef ST case EXACT: { char *s = STRING(scan); ln = STR_LEN(scan); - if (do_utf8 != UTF) { + if (utf8_target != UTF_PATTERN) { /* The target and the pattern have differing utf8ness. */ char *l = locinput; const char * const e = s + ln; - if (do_utf8) { + if (utf8_target) { /* The target is utf8, the pattern is not utf8. */ while (s < e) { STRLEN ulen; @@ -3301,19 +3545,19 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) char * const s = STRING(scan); ln = STR_LEN(scan); - if (do_utf8 || UTF) { + if (utf8_target || UTF_PATTERN) { /* Either target or the pattern are utf8. */ const char * const l = locinput; char *e = PL_regeol; - if (ibcmp_utf8(s, 0, ln, (bool)UTF, - l, &e, 0, do_utf8)) { + if (! foldEQ_utf8(s, 0, ln, cBOOL(UTF_PATTERN), + l, &e, 0, utf8_target)) { /* One more case for the sharp s: * pack("U0U*", 0xDF) =~ /ss/i, * the 0xC3 0x9F are the UTF-8 * byte sequence for the U+00DF. */ - if (!(do_utf8 && + if (!(utf8_target && toLOWER(s[0]) == 's' && ln >= 2 && toLOWER(s[1]) == 's' && @@ -3337,92 +3581,13 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) if (PL_regeol - locinput < ln) sayNO; if (ln > 1 && (OP(scan) == EXACTF - ? ibcmp(s, locinput, ln) - : ibcmp_locale(s, locinput, ln))) + ? ! foldEQ(s, locinput, ln) + : ! foldEQ_locale(s, locinput, ln))) sayNO; locinput += ln; nextchr = UCHARAT(locinput); break; } - case ANYOF: - if (do_utf8) { - STRLEN inclasslen = PL_regeol - locinput; - - if (!reginclass(rex, scan, (U8*)locinput, &inclasslen, do_utf8)) - goto anyof_fail; - if (locinput >= PL_regeol) - sayNO; - locinput += inclasslen ? inclasslen : UTF8SKIP(locinput); - nextchr = UCHARAT(locinput); - break; - } - else { - if (nextchr < 0) - nextchr = UCHARAT(locinput); - if (!REGINCLASS(rex, scan, (U8*)locinput)) - goto anyof_fail; - if (!nextchr && locinput >= PL_regeol) - sayNO; - nextchr = UCHARAT(++locinput); - break; - } - anyof_fail: - /* If we might have the case of the German sharp s - * in a casefolding Unicode character class. */ - - if (ANYOF_FOLD_SHARP_S(scan, locinput, PL_regeol)) { - locinput += SHARP_S_SKIP; - nextchr = UCHARAT(locinput); - } - else - sayNO; - break; - case ALNUML: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case ALNUM: - if (!nextchr) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_ALNUM(); - if (!(OP(scan) == ALNUM - ? (bool)swash_fetch(PL_utf8_alnum, (U8*)locinput, do_utf8) - : isALNUM_LC_utf8((U8*)locinput))) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; - } - if (!(OP(scan) == ALNUM - ? isALNUM(nextchr) : isALNUM_LC(nextchr))) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case NALNUML: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case NALNUM: - if (!nextchr && locinput >= PL_regeol) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_ALNUM(); - if (OP(scan) == NALNUM - ? (bool)swash_fetch(PL_utf8_alnum, (U8*)locinput, do_utf8) - : isALNUM_LC_utf8((U8*)locinput)) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; - } - if (OP(scan) == NALNUM - ? isALNUM(nextchr) : isALNUM_LC(nextchr)) - sayNO; - nextchr = UCHARAT(++locinput); - break; case BOUNDL: case NBOUNDL: PL_reg_flags |= RF_tainted; @@ -3430,18 +3595,18 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) case BOUND: case NBOUND: /* was last char in word? */ - if (do_utf8) { + if (utf8_target) { if (locinput == PL_bostr) ln = '\n'; else { const U8 * const r = reghop3((U8*)locinput, -1, (U8*)PL_bostr); - + ln = utf8n_to_uvchr(r, UTF8SKIP(r), 0, uniflags); } if (OP(scan) == BOUND || OP(scan) == NBOUND) { ln = isALNUM_uni(ln); LOAD_UTF8_CHARCLASS_ALNUM(); - n = swash_fetch(PL_utf8_alnum, (U8*)locinput, do_utf8); + n = swash_fetch(PL_utf8_alnum, (U8*)locinput, utf8_target); } else { ln = isALNUM_LC_uvchr(UNI_TO_NATIVE(ln)); @@ -3451,7 +3616,14 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) else { ln = (locinput != PL_bostr) ? UCHARAT(locinput - 1) : '\n'; - if (OP(scan) == BOUND || OP(scan) == NBOUND) { + if (FLAGS(scan) & USE_UNI) { + + /* Here, can't be BOUNDL or NBOUNDL because they never set + * the flags to USE_UNI */ + ln = isWORDCHAR_L1(ln); + n = isWORDCHAR_L1(nextchr); + } + else if (OP(scan) == BOUND || OP(scan) == NBOUND) { ln = isALNUM(ln); n = isALNUM(nextchr); } @@ -3464,116 +3636,259 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) OP(scan) == BOUNDL)) sayNO; break; - case SPACEL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case SPACE: - if (!nextchr) - sayNO; - if (do_utf8) { - if (UTF8_IS_CONTINUED(nextchr)) { - LOAD_UTF8_CHARCLASS_SPACE(); - if (!(OP(scan) == SPACE - ? (bool)swash_fetch(PL_utf8_space, (U8*)locinput, do_utf8) - : isSPACE_LC_utf8((U8*)locinput))) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; - } - } - if (!(OP(scan) == SPACE - ? isSPACE(nextchr) : isSPACE_LC(nextchr))) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case NSPACEL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case NSPACE: - if (!nextchr && locinput >= PL_regeol) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_SPACE(); - if (OP(scan) == NSPACE - ? (bool)swash_fetch(PL_utf8_space, (U8*)locinput, do_utf8) - : isSPACE_LC_utf8((U8*)locinput)) - { + case ANYOF: + if (utf8_target) { + STRLEN inclasslen = PL_regeol - locinput; + if (locinput >= PL_regeol) sayNO; - } - locinput += PL_utf8skip[nextchr]; + + if (!reginclass(rex, scan, (U8*)locinput, &inclasslen, utf8_target)) + goto anyof_fail; + locinput += inclasslen; nextchr = UCHARAT(locinput); break; } - if (OP(scan) == NSPACE - ? isSPACE(nextchr) : isSPACE_LC(nextchr)) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case DIGITL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case DIGIT: - if (!nextchr) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_DIGIT(); - if (!(OP(scan) == DIGIT - ? (bool)swash_fetch(PL_utf8_digit, (U8*)locinput, do_utf8) - : isDIGIT_LC_utf8((U8*)locinput))) - { + else { + if (nextchr < 0) + nextchr = UCHARAT(locinput); + if (!nextchr && locinput >= PL_regeol) sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); + if (!REGINCLASS(rex, scan, (U8*)locinput)) + goto anyof_fail; + nextchr = UCHARAT(++locinput); break; } - if (!(OP(scan) == DIGIT - ? isDIGIT(nextchr) : isDIGIT_LC(nextchr))) - sayNO; - nextchr = UCHARAT(++locinput); - break; - case NDIGITL: - PL_reg_flags |= RF_tainted; - /* FALL THROUGH */ - case NDIGIT: - if (!nextchr && locinput >= PL_regeol) - sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_DIGIT(); - if (OP(scan) == NDIGIT - ? (bool)swash_fetch(PL_utf8_digit, (U8*)locinput, do_utf8) - : isDIGIT_LC_utf8((U8*)locinput)) - { - sayNO; - } - locinput += PL_utf8skip[nextchr]; - nextchr = UCHARAT(locinput); - break; + anyof_fail: + /* If we might have the case of the German sharp s + * in a casefolding Unicode character class. */ + + if (ANYOF_FOLD_SHARP_S(scan, locinput, PL_regeol)) { + locinput += SHARP_S_SKIP; + nextchr = UCHARAT(locinput); } - if (OP(scan) == NDIGIT - ? isDIGIT(nextchr) : isDIGIT_LC(nextchr)) - sayNO; - nextchr = UCHARAT(++locinput); + else + sayNO; break; - case CLUMP: + /* Special char classes - The defines start on line 129 or so */ + CCC_TRY_AFF_U( ALNUM, ALNUML, perl_word, "a", isALNUM_LC_utf8, isWORDCHAR_L1, isALNUM_LC); + CCC_TRY_NEG_U(NALNUM, NALNUML, perl_word, "a", isALNUM_LC_utf8, isWORDCHAR_L1, isALNUM_LC); + + CCC_TRY_AFF_U( SPACE, SPACEL, perl_space, " ", isSPACE_LC_utf8, isSPACE_L1, isSPACE_LC); + CCC_TRY_NEG_U(NSPACE, NSPACEL, perl_space, " ", isSPACE_LC_utf8, isSPACE_L1, isSPACE_LC); + + CCC_TRY_AFF( DIGIT, DIGITL, posix_digit, "0", isDIGIT_LC_utf8, isDIGIT, isDIGIT_LC); + CCC_TRY_NEG(NDIGIT, NDIGITL, posix_digit, "0", isDIGIT_LC_utf8, isDIGIT, isDIGIT_LC); + + case CLUMP: /* Match \X: logical Unicode character. This is defined as + a Unicode extended Grapheme Cluster */ + /* From http://www.unicode.org/reports/tr29 (5.2 version). An + extended Grapheme Cluster is: + + CR LF + | Prepend* Begin Extend* + | . + + Begin is (Hangul-syllable | ! Control) + Extend is (Grapheme_Extend | Spacing_Mark) + Control is [ GCB_Control CR LF ] + + The discussion below shows how the code for CLUMP is derived + from this regex. Note that most of these concepts are from + property values of the Grapheme Cluster Boundary (GCB) property. + No code point can have multiple property values for a given + property. Thus a code point in Prepend can't be in Control, but + it must be in !Control. This is why Control above includes + GCB_Control plus CR plus LF. The latter two are used in the GCB + property separately, and so can't be in GCB_Control, even though + they logically are controls. Control is not the same as gc=cc, + but includes format and other characters as well. + + The Unicode definition of Hangul-syllable is: + L+ + | (L* ( ( V | LV ) V* | LVT ) T*) + | T+ + ) + Each of these is a value for the GCB property, and hence must be + disjoint, so the order they are tested is immaterial, so the + above can safely be changed to + T+ + | L+ + | (L* ( LVT | ( V | LV ) V*) T*) + + The last two terms can be combined like this: + L* ( L + | (( LVT | ( V | LV ) V*) T*)) + + And refactored into this: + L* (L | LVT T* | V V* T* | LV V* T*) + + That means that if we have seen any L's at all we can quit + there, but if the next character is a LVT, a V or and LV we + should keep going. + + There is a subtlety with Prepend* which showed up in testing. + Note that the Begin, and only the Begin is required in: + | Prepend* Begin Extend* + Also, Begin contains '! Control'. A Prepend must be a '! + Control', which means it must be a Begin. What it comes down to + is that if we match Prepend* and then find no suitable Begin + afterwards, that if we backtrack the last Prepend, that one will + be a suitable Begin. + */ + if (locinput >= PL_regeol) sayNO; - if (do_utf8) { - LOAD_UTF8_CHARCLASS_MARK(); - if (swash_fetch(PL_utf8_mark,(U8*)locinput, do_utf8)) - sayNO; - locinput += PL_utf8skip[nextchr]; - while (locinput < PL_regeol && - swash_fetch(PL_utf8_mark,(U8*)locinput, do_utf8)) - locinput += UTF8SKIP(locinput); - if (locinput > PL_regeol) - sayNO; - } - else - locinput++; + if (! utf8_target) { + + /* Match either CR LF or '.', as all the other possibilities + * require utf8 */ + locinput++; /* Match the . or CR */ + if (nextchr == '\r' + && locinput < PL_regeol + && UCHARAT(locinput) == '\n') locinput++; + } + else { + + /* Utf8: See if is ( CR LF ); already know that locinput < + * PL_regeol, so locinput+1 is in bounds */ + if (nextchr == '\r' && UCHARAT(locinput + 1) == '\n') { + locinput += 2; + } + else { + /* In case have to backtrack to beginning, then match '.' */ + char *starting = locinput; + + /* In case have to backtrack the last prepend */ + char *previous_prepend = 0; + + LOAD_UTF8_CHARCLASS_GCB(); + + /* Match (prepend)* */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_prepend, + (U8*)locinput, utf8_target)) + { + previous_prepend = locinput; + locinput += UTF8SKIP(locinput); + } + + /* As noted above, if we matched a prepend character, but + * the next thing won't match, back off the last prepend we + * matched, as it is guaranteed to match the begin */ + if (previous_prepend + && (locinput >= PL_regeol + || ! swash_fetch(PL_utf8_X_begin, + (U8*)locinput, utf8_target))) + { + locinput = previous_prepend; + } + + /* Note that here we know PL_regeol > locinput, as we + * tested that upon input to this switch case, and if we + * moved locinput forward, we tested the result just above + * and it either passed, or we backed off so that it will + * now pass */ + if (! swash_fetch(PL_utf8_X_begin, (U8*)locinput, utf8_target)) { + + /* Here did not match the required 'Begin' in the + * second term. So just match the very first + * character, the '.' of the final term of the regex */ + locinput = starting + UTF8SKIP(starting); + } else { + + /* Here is the beginning of a character that can have + * an extender. It is either a hangul syllable, or a + * non-control */ + if (swash_fetch(PL_utf8_X_non_hangul, + (U8*)locinput, utf8_target)) + { + + /* Here not a Hangul syllable, must be a + * ('! * Control') */ + locinput += UTF8SKIP(locinput); + } else { + + /* Here is a Hangul syllable. It can be composed + * of several individual characters. One + * possibility is T+ */ + if (swash_fetch(PL_utf8_X_T, + (U8*)locinput, utf8_target)) + { + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_T, + (U8*)locinput, utf8_target)) + { + locinput += UTF8SKIP(locinput); + } + } else { + + /* Here, not T+, but is a Hangul. That means + * it is one of the others: L, LV, LVT or V, + * and matches: + * L* (L | LVT T* | V V* T* | LV V* T*) */ + + /* Match L* */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_L, + (U8*)locinput, utf8_target)) + { + locinput += UTF8SKIP(locinput); + } + + /* Here, have exhausted L*. If the next + * character is not an LV, LVT nor V, it means + * we had to have at least one L, so matches L+ + * in the original equation, we have a complete + * hangul syllable. Are done. */ + + if (locinput < PL_regeol + && swash_fetch(PL_utf8_X_LV_LVT_V, + (U8*)locinput, utf8_target)) + { + + /* Otherwise keep going. Must be LV, LVT + * or V. See if LVT */ + if (swash_fetch(PL_utf8_X_LVT, + (U8*)locinput, utf8_target)) + { + locinput += UTF8SKIP(locinput); + } else { + + /* Must be V or LV. Take it, then + * match V* */ + locinput += UTF8SKIP(locinput); + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_V, + (U8*)locinput, utf8_target)) + { + locinput += UTF8SKIP(locinput); + } + } + + /* And any of LV, LVT, or V can be followed + * by T* */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_T, + (U8*)locinput, + utf8_target)) + { + locinput += UTF8SKIP(locinput); + } + } + } + } + + /* Match any extender */ + while (locinput < PL_regeol + && swash_fetch(PL_utf8_X_extend, + (U8*)locinput, utf8_target)) + { + locinput += UTF8SKIP(locinput); + } + } + } + if (locinput > PL_regeol) sayNO; + } nextchr = UCHARAT(locinput); break; @@ -3611,7 +3926,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) break; s = PL_bostr + ln; - if (do_utf8 && type != REF) { /* REF can do byte comparison */ + if (utf8_target && type != REF) { /* REF can do byte comparison */ char *l = locinput; const char *e = PL_bostr + PL_regoffs[n].end; /* @@ -3652,8 +3967,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) if (ln > 1 && (type == REF ? memNE(s, locinput, ln) : (type == REFF - ? ibcmp(s, locinput, ln) - : ibcmp_locale(s, locinput, ln)))) + ? ! foldEQ(s, locinput, ln) + : ! foldEQ_locale(s, locinput, ln)))) sayNO; locinput += ln; nextchr = UCHARAT(locinput); @@ -3714,7 +4029,24 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) COP * const ocurcop = PL_curcop; PAD *old_comppad; char *saved_regeol = PL_regeol; - + struct re_save_state saved_state; + + /* To not corrupt the existing regex state while executing the + * eval we would normally put it on the save stack, like with + * save_re_context. However, re-evals have a weird scoping so we + * can't just add ENTER/LEAVE here. With that, things like + * + * (?{$a=2})(a(?{local$a=$a+1}))*aak*c(?{$b=$a}) + * + * would break, as they expect the localisation to be unwound + * only when the re-engine backtracks through the bit that + * localised it. + * + * What we do instead is just saving the state in a local c + * variable. + */ + Copy(&PL_reg_state, &saved_state, 1, struct re_save_state); + n = ARG(scan); PL_op = (OP_4tree*)rexi->data->data[n]; DEBUG_STATE_r( PerlIO_printf(Perl_debug_log, @@ -3736,6 +4068,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) PUTBACK; } + Copy(&saved_state, &PL_reg_state, 1, struct re_save_state); + PL_op = oop; PAD_RESTORE_LOCAL(old_comppad); PL_curcop = ocurcop; @@ -3787,7 +4121,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) assert(rx); } if (rx) { - rx = reg_temp_copy(rx); + rx = reg_temp_copy(NULL, rx); } else { U32 pm_flags = 0; @@ -3824,7 +4158,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) re->sublen = rex->sublen; rei = RXi_GET(re); DEBUG_EXECUTE_r( - debug_start_match(re_sv, do_utf8, locinput, PL_regeol, + debug_start_match(re_sv, utf8_target, locinput, PL_regeol, "Matching embedded"); ); startpoint = rei->program + 1; @@ -3878,7 +4212,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) /* NOTREACHED */ } /* logical is 1, /(?(?{...})X|Y)/ */ - sw = (bool)SvTRUE(ret); + sw = cBOOL(SvTRUE(ret)); logical = 0; break; } @@ -3979,11 +4313,11 @@ S_regmatch(pTHX_ regmatch_info *reginfo, regnode *prog) /*NOTREACHED*/ case GROUPP: n = ARG(scan); /* which paren pair */ - sw = (bool)(*PL_reglastparen >= n && PL_regoffs[n].end != -1); + sw = cBOOL(*PL_reglastparen >= n && PL_regoffs[n].end != -1); break; case NGROUPP: /* reg_check_named_buff_matched returns 0 for no match */ - sw = (bool)(0 < reg_check_named_buff_matched(rex,scan)); + sw = cBOOL(0 < reg_check_named_buff_matched(rex,scan)); break; case INSUBP: n = ARG(scan); @@ -4112,9 +4446,7 @@ NULL /* these fields contain the state of the current curly. * they are accessed by subsequent WHILEMs */ ST.parenfloor = parenfloor; - ST.min = ARG1(scan); - ST.max = ARG2(scan); - ST.A = NEXTOPER(scan) + EXTRA_STEP_2ARGS; + ST.me = scan; ST.B = next; ST.minmod = minmod; minmod = 0; @@ -4145,6 +4477,10 @@ NULL { /* see the discussion above about CURLYX/WHILEM */ I32 n; + int min = ARG1(cur_curlyx->u.curlyx.me); + int max = ARG2(cur_curlyx->u.curlyx.me); + regnode *A = NEXTOPER(cur_curlyx->u.curlyx.me) + EXTRA_STEP_2ARGS; + assert(cur_curlyx); /* keep Coverity happy */ n = ++cur_curlyx->u.curlyx.count; /* how many A's matched */ ST.save_lastloc = cur_curlyx->u.curlyx.lastloc; @@ -4154,17 +4490,15 @@ NULL PL_reginput = locinput; DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - "%*s whilem: matched %ld out of %ld..%ld\n", - REPORT_CODE_OFF+depth*2, "", (long)n, - (long)cur_curlyx->u.curlyx.min, - (long)cur_curlyx->u.curlyx.max) + "%*s whilem: matched %ld out of %d..%d\n", + REPORT_CODE_OFF+depth*2, "", (long)n, min, max) ); /* First just match a string of min A's. */ - if (n < cur_curlyx->u.curlyx.min) { + if (n < min) { cur_curlyx->u.curlyx.lastloc = locinput; - PUSH_STATE_GOTO(WHILEM_A_pre, cur_curlyx->u.curlyx.A); + PUSH_STATE_GOTO(WHILEM_A_pre, A); /* NOTREACHED */ } @@ -4244,11 +4578,11 @@ NULL /* Prefer A over B for maximal matching. */ - if (n < cur_curlyx->u.curlyx.max) { /* More greed allowed? */ + if (n < max) { /* More greed allowed? */ ST.cp = regcppush(cur_curlyx->u.curlyx.parenfloor); cur_curlyx->u.curlyx.lastloc = locinput; REGCP_SET(ST.lastcp); - PUSH_STATE_GOTO(WHILEM_A_max, cur_curlyx->u.curlyx.A); + PUSH_STATE_GOTO(WHILEM_A_max, A); /* NOTREACHED */ } goto do_whilem_B_max; @@ -4308,7 +4642,7 @@ NULL REGCP_UNWIND(ST.lastcp); regcppop(rex); - if (cur_curlyx->u.curlyx.count >= cur_curlyx->u.curlyx.max) { + if (cur_curlyx->u.curlyx.count >= /*max*/ARG2(cur_curlyx->u.curlyx.me)) { /* Maximum greed exceeded */ if (cur_curlyx->u.curlyx.count >= REG_INFTY && ckWARN(WARN_REGEXP) @@ -4332,7 +4666,8 @@ NULL cur_curlyx->u.curlyx.lastloc = locinput; ST.cp = regcppush(cur_curlyx->u.curlyx.parenfloor); REGCP_SET(ST.lastcp); - PUSH_STATE_GOTO(WHILEM_A_min, ST.save_curlyx->u.curlyx.A); + PUSH_STATE_GOTO(WHILEM_A_min, + /*A*/ NEXTOPER(ST.save_curlyx->u.curlyx.me) + EXTRA_STEP_2ARGS); /* NOTREACHED */ #undef ST @@ -4666,14 +5001,14 @@ NULL if this changes back then the macro for IS_TEXT and friends need to change. */ - if (!UTF) { + if (!UTF_PATTERN) { ST.c2 = ST.c1 = *s; if (IS_TEXTF(text_node)) ST.c2 = PL_fold[ST.c1]; else if (IS_TEXTFL(text_node)) ST.c2 = PL_fold_locale[ST.c1]; } - else { /* UTF */ + else { /* UTF_PATTERN */ if (IS_TEXTF(text_node)) { STRLEN ulen1, ulen2; U8 tmpbuf1[UTF8_MAXBYTES_CASE+1]; @@ -4725,11 +5060,11 @@ NULL * string that could possibly match */ if (ST.max == REG_INFTY) { ST.maxpos = PL_regeol - 1; - if (do_utf8) + if (utf8_target) while (UTF8_IS_CONTINUATION(*(U8*)ST.maxpos)) ST.maxpos--; } - else if (do_utf8) { + else if (utf8_target) { int m = ST.max - ST.min; for (ST.maxpos = locinput; m >0 && ST.maxpos + UTF8SKIP(ST.maxpos) <= PL_regeol; m--) @@ -4775,7 +5110,7 @@ NULL REGCP_UNWIND(ST.cp); /* Couldn't or didn't -- move forward. */ ST.oldloc = locinput; - if (do_utf8) + if (utf8_target) locinput += UTF8SKIP(locinput); else locinput++; @@ -4784,7 +5119,7 @@ NULL /* find the next place where 'B' could work, then call B */ { int n; - if (do_utf8) { + if (utf8_target) { n = (ST.oldloc == locinput) ? 0 : 1; if (ST.c1 == ST.c2) { STRLEN len; @@ -4880,7 +5215,7 @@ NULL { UV c = 0; if (ST.c1 != CHRTEST_VOID) - c = do_utf8 ? utf8n_to_uvchr((U8*)PL_reginput, + c = utf8_target ? utf8n_to_uvchr((U8*)PL_reginput, UTF8_MAXBYTES, 0, uniflags) : (UV) UCHARAT(PL_reginput); /* If it could work, try it. */ @@ -4990,7 +5325,7 @@ NULL /* trivial fail */ if (logical) { logical = 0; - sw = 1 - (bool)ST.wanted; + sw = 1 - cBOOL(ST.wanted); } else if (ST.wanted) sayNO; @@ -5019,7 +5354,7 @@ NULL case IFMATCH_A: /* body of (?...A) succeeded */ if (ST.logical) { - sw = (bool)ST.wanted; + sw = cBOOL(ST.wanted); } else if (!ST.wanted) sayNO; @@ -5138,9 +5473,9 @@ NULL #undef ST case FOLDCHAR: n = ARG(scan); - if ( n == (U32)what_len_TRICKYFOLD(locinput,do_utf8,ln) ) { + if ( n == (U32)what_len_TRICKYFOLD(locinput,utf8_target,ln) ) { locinput += ln; - } else if ( 0xDF == n && !do_utf8 && !UTF ) { + } else if ( 0xDF == n && !utf8_target && !UTF_PATTERN ) { sayNO; } else { U8 folded[UTF8_MAXBYTES_CASE+1]; @@ -5149,8 +5484,8 @@ NULL char *e = PL_regeol; to_uni_fold(n, folded, &foldlen); - if (ibcmp_utf8((const char*) folded, 0, foldlen, 1, - l, &e, 0, do_utf8)) { + if (! foldEQ_utf8((const char*) folded, 0, foldlen, 1, + l, &e, 0, utf8_target)) { sayNO; } locinput = e; @@ -5158,7 +5493,7 @@ NULL nextchr = UCHARAT(locinput); break; case LNBREAK: - if ((n=is_LNBREAK(locinput,do_utf8))) { + if ((n=is_LNBREAK(locinput,utf8_target))) { locinput += n; nextchr = UCHARAT(locinput); } else @@ -5167,14 +5502,14 @@ NULL #define CASE_CLASS(nAmE) \ case nAmE: \ - if ((n=is_##nAmE(locinput,do_utf8))) { \ + if ((n=is_##nAmE(locinput,utf8_target))) { \ locinput += n; \ nextchr = UCHARAT(locinput); \ } else \ sayNO; \ break; \ case N##nAmE: \ - if ((n=is_##nAmE(locinput,do_utf8))) { \ + if ((n=is_##nAmE(locinput,utf8_target))) { \ sayNO; \ } else { \ locinput += UTF8SKIP(locinput); \ @@ -5387,7 +5722,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) register I32 c; register char *loceol = PL_regeol; register I32 hardcount = 0; - register bool do_utf8 = PL_reg_match_utf8; + register bool utf8_target = PL_reg_match_utf8; #ifndef DEBUGGING PERL_UNUSED_ARG(depth); #endif @@ -5401,7 +5736,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) loceol = scan + max; switch (OP(p)) { case REG_ANY: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (scan < loceol && hardcount < max && *scan != '\n') { scan += UTF8SKIP(scan); @@ -5413,7 +5748,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case SANY: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (scan < loceol && hardcount < max) { scan += UTF8SKIP(scan); @@ -5426,29 +5761,112 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) case CANY: scan = loceol; break; - case EXACT: /* length of string is 1 */ - c = (U8)*STRING(p); - while (scan < loceol && UCHARAT(scan) == c) - scan++; - break; - case EXACTF: /* length of string is 1 */ - c = (U8)*STRING(p); - while (scan < loceol && - (UCHARAT(scan) == c || UCHARAT(scan) == PL_fold[c])) - scan++; - break; - case EXACTFL: /* length of string is 1 */ + case EXACTFL: PL_reg_flags |= RF_tainted; + /* FALL THROUGH */ + case EXACT: + case EXACTF: + /* To get here, EXACTish nodes must have *byte* length == 1. That means + * they match only characters in the string that can be expressed as a + * single byte. For non-utf8 strings, that means a simple match. For + * utf8 strings, the character matched must be an invariant, or + * downgradable to a single byte. The pattern's utf8ness is + * irrelevant, as it must be a single byte, so either it isn't utf8, or + * if it is it's an invariant */ + c = (U8)*STRING(p); - while (scan < loceol && - (UCHARAT(scan) == c || UCHARAT(scan) == PL_fold_locale[c])) - scan++; + assert(! UTF_PATTERN || UNI_IS_INVARIANT(c)); + + if ((! utf8_target) || UNI_IS_INVARIANT(c)) { + + /* Here, the string isn't utf8, or the character in the EXACT + * node is the same in utf8 as not, so can just do equality. + * Each matching char must be 1 byte long */ + switch (OP(p)) { + case EXACT: + while (scan < loceol && UCHARAT(scan) == c) { + scan++; + } + break; + case EXACTF: + while (scan < loceol && + (UCHARAT(scan) == c || UCHARAT(scan) == PL_fold[c])) + { + scan++; + } + break; + case EXACTFL: + while (scan < loceol && + (UCHARAT(scan) == c || UCHARAT(scan) == PL_fold_locale[c])) + { + scan++; + } + break; + default: + Perl_croak(aTHX_ "panic: Unexpected op %u", OP(p)); + } + } + else { + + /* Here, the string is utf8, and the pattern char is different + * in utf8 than not. */ + + switch (OP(p)) { + case EXACT: + { + /* Fastest to find the two utf8 bytes that represent c, and + * then look for those in sequence in the utf8 string */ + U8 high = UTF8_TWO_BYTE_HI(c); + U8 low = UTF8_TWO_BYTE_LO(c); + loceol = PL_regeol; + + while (hardcount < max + && scan + 1 < loceol + && UCHARAT(scan) == high + && UCHARAT(scan + 1) == low) + { + scan += 2; + hardcount++; + } + } + break; + case EXACTFL: /* Doesn't really make sense, but is best we can + do. The documents warn against mixing locale + and utf8 */ + case EXACTF: + { /* utf8 string, so use utf8 foldEQ */ + char *tmpeol = loceol; + while (hardcount < max + && foldEQ_utf8(scan, &tmpeol, 0, utf8_target, + STRING(p), NULL, 1, UTF_PATTERN)) + { + scan = tmpeol; + tmpeol = loceol; + hardcount++; + } + + /* XXX Note that the above handles properly the German + * sharp ss in the pattern matching ss in the string. But + * it doesn't handle properly cases where the string + * contains say 'LIGATURE ff' and the pattern is 'f+'. + * This would require, say, a new function or revised + * interface to foldEQ_utf8(), in which the maximum number + * of characters to match could be passed and it would + * return how many actually did. This is just one of many + * cases where multi-char folds don't work properly, and so + * the fix is being deferred */ + } + break; + default: + Perl_croak(aTHX_ "panic: Unexpected op %u", OP(p)); + } + } break; case ANYOF: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && - reginclass(prog, p, (U8*)scan, 0, do_utf8)) { + reginclass(prog, p, (U8*)scan, 0, utf8_target)) { scan += UTF8SKIP(scan); hardcount++; } @@ -5458,22 +5876,28 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case ALNUM: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; LOAD_UTF8_CHARCLASS_ALNUM(); while (hardcount < max && scan < loceol && - swash_fetch(PL_utf8_alnum, (U8*)scan, do_utf8)) { + swash_fetch(PL_utf8_alnum, (U8*)scan, utf8_target)) + { scan += UTF8SKIP(scan); hardcount++; } + } else if (FLAGS(p) & USE_UNI) { + while (scan < loceol && isWORDCHAR_L1((U8) *scan)) { + scan++; + } } else { - while (scan < loceol && isALNUM(*scan)) - scan++; + while (scan < loceol && isALNUM((U8) *scan)) { + scan++; + } } break; case ALNUML: PL_reg_flags |= RF_tainted; - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && isALNUM_LC_utf8((U8*)scan)) { @@ -5486,22 +5910,28 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case NALNUM: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; LOAD_UTF8_CHARCLASS_ALNUM(); while (hardcount < max && scan < loceol && - !swash_fetch(PL_utf8_alnum, (U8*)scan, do_utf8)) { + !swash_fetch(PL_utf8_alnum, (U8*)scan, utf8_target)) + { scan += UTF8SKIP(scan); hardcount++; } + } else if (FLAGS(p) & USE_UNI) { + while (scan < loceol && ! isWORDCHAR_L1((U8) *scan)) { + scan++; + } } else { - while (scan < loceol && !isALNUM(*scan)) - scan++; + while (scan < loceol && ! isALNUM((U8) *scan)) { + scan++; + } } break; case NALNUML: PL_reg_flags |= RF_tainted; - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && !isALNUM_LC_utf8((U8*)scan)) { @@ -5514,23 +5944,28 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case SPACE: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; LOAD_UTF8_CHARCLASS_SPACE(); while (hardcount < max && scan < loceol && (*scan == ' ' || - swash_fetch(PL_utf8_space,(U8*)scan, do_utf8))) { + swash_fetch(PL_utf8_space,(U8*)scan, utf8_target))) + { scan += UTF8SKIP(scan); hardcount++; } + } else if (FLAGS(p) & USE_UNI) { + while (scan < loceol && isSPACE_L1((U8) *scan)) { + scan++; + } } else { - while (scan < loceol && isSPACE(*scan)) - scan++; + while (scan < loceol && isSPACE((U8) *scan)) + scan++; } break; case SPACEL: PL_reg_flags |= RF_tainted; - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && (*scan == ' ' || isSPACE_LC_utf8((U8*)scan))) { @@ -5543,23 +5978,29 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case NSPACE: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; LOAD_UTF8_CHARCLASS_SPACE(); while (hardcount < max && scan < loceol && !(*scan == ' ' || - swash_fetch(PL_utf8_space,(U8*)scan, do_utf8))) { + swash_fetch(PL_utf8_space,(U8*)scan, utf8_target))) + { scan += UTF8SKIP(scan); hardcount++; } + } else if (FLAGS(p) & USE_UNI) { + while (scan < loceol && ! isSPACE_L1((U8) *scan)) { + scan++; + } } else { - while (scan < loceol && !isSPACE(*scan)) - scan++; + while (scan < loceol && ! isSPACE((U8) *scan)) { + scan++; + } } break; case NSPACEL: PL_reg_flags |= RF_tainted; - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && !(*scan == ' ' || isSPACE_LC_utf8((U8*)scan))) { @@ -5572,11 +6013,11 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case DIGIT: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; LOAD_UTF8_CHARCLASS_DIGIT(); while (hardcount < max && scan < loceol && - swash_fetch(PL_utf8_digit, (U8*)scan, do_utf8)) { + swash_fetch(PL_utf8_digit, (U8*)scan, utf8_target)) { scan += UTF8SKIP(scan); hardcount++; } @@ -5586,11 +6027,11 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case NDIGIT: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; LOAD_UTF8_CHARCLASS_DIGIT(); while (hardcount < max && scan < loceol && - !swash_fetch(PL_utf8_digit, (U8*)scan, do_utf8)) { + !swash_fetch(PL_utf8_digit, (U8*)scan, utf8_target)) { scan += UTF8SKIP(scan); hardcount++; } @@ -5599,7 +6040,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) scan++; } case LNBREAK: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && (c=is_LNBREAK_utf8(scan))) { scan += c; @@ -5618,7 +6059,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case HORIZWS: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && (c=is_HORIZWS_utf8(scan))) { scan += c; @@ -5630,7 +6071,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case NHORIZWS: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && !is_HORIZWS_utf8(scan)) { scan += UTF8SKIP(scan); @@ -5643,7 +6084,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case VERTWS: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && (c=is_VERTWS_utf8(scan))) { scan += c; @@ -5656,7 +6097,7 @@ S_regrepeat(pTHX_ const regexp *prog, const regnode *p, I32 max, int depth) } break; case NVERTWS: - if (do_utf8) { + if (utf8_target) { loceol = PL_regeol; while (hardcount < max && scan < loceol && !is_VERTWS_utf8(scan)) { scan += UTF8SKIP(scan); @@ -5750,89 +6191,60 @@ Perl_regclass_swash(pTHX_ const regexp *prog, register const regnode* node, bool /* - reginclass - determine if a character falls into a character class - The n is the ANYOF regnode, the p is the target string, lenp - is pointer to the maximum length of how far to go in the p - (if the lenp is zero, UTF8SKIP(p) is used), - do_utf8 tells whether the target string is in UTF-8. + n is the ANYOF regnode + p is the target string + lenp is pointer to the maximum number of bytes of how far to go in p + (This is assumed wthout checking to always be at least the current + character's size) + utf8_target tells whether p is in UTF-8. + + Returns true if matched; false otherwise. If lenp is not NULL, on return + from a successful match, the value it points to will be updated to how many + bytes in p were matched. If there was no match, the value is undefined, + possibly changed from the input. */ STATIC bool -S_reginclass(pTHX_ const regexp *prog, register const regnode *n, register const U8* p, STRLEN* lenp, register bool do_utf8) +S_reginclass(pTHX_ const regexp * const prog, register const regnode * const n, register const U8* const p, STRLEN* lenp, register const bool utf8_target) { dVAR; const char flags = ANYOF_FLAGS(n); bool match = FALSE; UV c = *p; - STRLEN len = 0; - STRLEN plen; + STRLEN c_len = 0; + STRLEN maxlen; PERL_ARGS_ASSERT_REGINCLASS; - if (do_utf8 && !UTF8_IS_INVARIANT(c)) { - c = utf8n_to_uvchr(p, UTF8_MAXBYTES, &len, - (UTF8_ALLOW_DEFAULT & UTF8_ALLOW_ANYUV) | UTF8_CHECK_ONLY); - /* see [perl #37836] for UTF8_ALLOW_ANYUV */ - if (len == (STRLEN)-1) + /* If c is not already the code point, get it */ + if (utf8_target && !UTF8_IS_INVARIANT(c)) { + c = utf8n_to_uvchr(p, UTF8_MAXBYTES, &c_len, + (UTF8_ALLOW_DEFAULT & UTF8_ALLOW_ANYUV) + | UTF8_ALLOW_FFFF | UTF8_CHECK_ONLY); + /* see [perl #37836] for UTF8_ALLOW_ANYUV; [perl #38293] for + * UTF8_ALLOW_FFFF */ + if (c_len == (STRLEN)-1) Perl_croak(aTHX_ "Malformed UTF-8 character (fatal)"); } + else { + c_len = 1; + } - plen = lenp ? *lenp : UNISKIP(NATIVE_TO_UNI(c)); - if (do_utf8 || (flags & ANYOF_UNICODE)) { - if (lenp) - *lenp = 0; - if (do_utf8 && !ANYOF_RUNTIME(n)) { - if (len != (STRLEN)-1 && c < 256 && ANYOF_BITMAP_TEST(n, c)) - match = TRUE; - } - if (!match && do_utf8 && (flags & ANYOF_UNICODE_ALL) && c >= 256) - match = TRUE; - if (!match) { - AV *av; - SV * const sw = regclass_swash(prog, n, TRUE, 0, (SV**)&av); - - if (sw) { - U8 * utf8_p; - if (do_utf8) { - utf8_p = (U8 *) p; - } else { - STRLEN len = 1; - utf8_p = bytes_to_utf8(p, &len); - } - if (swash_fetch(sw, utf8_p, 1)) - match = TRUE; - else if (flags & ANYOF_FOLD) { - if (!match && lenp && av) { - I32 i; - for (i = 0; i <= av_len(av); i++) { - SV* const sv = *av_fetch(av, i, FALSE); - STRLEN len; - const char * const s = SvPV_const(sv, len); - if (len <= plen && memEQ(s, (char*)utf8_p, len)) { - *lenp = len; - match = TRUE; - break; - } - } - } - if (!match) { - U8 tmpbuf[UTF8_MAXBYTES_CASE+1]; - - STRLEN tmplen; - to_utf8_fold(utf8_p, tmpbuf, &tmplen); - if (swash_fetch(sw, tmpbuf, 1)) - match = TRUE; - } - } + /* Use passed in max length, or one character if none passed in or less + * than one character. And assume will match just one character. This is + * overwritten later if matched more. */ + if (lenp) { + maxlen = (*lenp > c_len) ? *lenp : c_len; + *lenp = c_len; - /* If we allocated a string above, free it */ - if (! do_utf8) Safefree(utf8_p); - } - } - if (match && lenp && *lenp == 0) - *lenp = UNISKIP(NATIVE_TO_UNI(c)); } - if (!match && c < 256) { + else { + maxlen = c_len; + } + + /* If this character is potentially in the bitmap, check it */ + if (c < 256) { if (ANYOF_BITMAP_TEST(n, c)) match = TRUE; else if (flags & ANYOF_FOLD) { @@ -5888,6 +6300,56 @@ S_reginclass(pTHX_ const regexp *prog, register const regnode *n, register const } } + /* If the bitmap didn't (or couldn't) match, and something outside the + * bitmap could match, try that */ + if (!match && (utf8_target || (flags & ANYOF_UNICODE))) { + if (utf8_target && (flags & ANYOF_UNICODE_ALL) && c >= 256) { + match = TRUE; + } + else { + AV *av; + SV * const sw = regclass_swash(prog, n, TRUE, 0, (SV**)&av); + + if (sw) { + U8 * utf8_p; + if (utf8_target) { + utf8_p = (U8 *) p; + } else { + STRLEN len = 1; + utf8_p = bytes_to_utf8(p, &len); + } + if (swash_fetch(sw, utf8_p, 1)) + match = TRUE; + else if (flags & ANYOF_FOLD) { + if (!match && lenp && av) { + I32 i; + for (i = 0; i <= av_len(av); i++) { + SV* const sv = *av_fetch(av, i, FALSE); + STRLEN len; + const char * const s = SvPV_const(sv, len); + if (len <= maxlen && memEQ(s, (char*)utf8_p, len)) { + *lenp = len; + match = TRUE; + break; + } + } + } + if (!match) { + U8 tmpbuf[UTF8_MAXBYTES_CASE+1]; + + STRLEN tmplen; + to_utf8_fold(utf8_p, tmpbuf, &tmplen); + if (swash_fetch(sw, tmpbuf, 1)) + match = TRUE; + } + } + + /* If we allocated a string above, free it */ + if (! utf8_target) Safefree(utf8_p); + } + } + } + return (flags & ANYOF_INVERT) ? !match : match; }