X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/e7fd4aa18abbfe0099d4947060c99ca85f42f764..ab65fd1a46259a8e942fd65724ad269ebda72919:/regexec.c diff --git a/regexec.c b/regexec.c index 10343a0..a3ce034 100644 --- a/regexec.c +++ b/regexec.c @@ -2,8 +2,8 @@ */ /* - * One Ring to rule them all, One Ring to find them - & + * One Ring to rule them all, One Ring to find them + * * [p.v of _The Lord of the Rings_, opening poem] * [p.50 of _The Lord of the Rings_, I/iii: "The Shadow of the Past"] * [p.254 of _The Lord of the Rings_, II/ii: "The Council of Elrond"] @@ -80,9 +80,15 @@ # include "regcomp.h" #endif -#include "inline_invlist.c" +#include "invlist_inline.h" #include "unicode_constants.h" +#define B_ON_NON_UTF8_LOCALE_IS_WRONG \ + "Use of \\b{} or \\B{} for non-UTF-8 locale is wrong. Assuming a UTF-8 locale" + +static const char utf8_locale_required[] = + "Use of (?[ ]) for non-UTF-8 locale is wrong. Assuming a UTF-8 locale"; + #ifdef DEBUGGING /* At least one required character in the target string is expressible only in * UTF-8. */ @@ -189,19 +195,7 @@ static const char* const non_utf8_target_but_utf8_required PL_utf8_swash_ptrs[_CC_WORDCHAR], \ "", \ PL_XPosix_ptrs[_CC_WORDCHAR], \ - LATIN_CAPITAL_LETTER_SHARP_S_UTF8); - -#define LOAD_UTF8_CHARCLASS_GCB() /* Grapheme cluster boundaries */ \ - STMT_START { \ - LOAD_UTF8_CHARCLASS_DEBUG_TEST(PL_utf8_X_regular_begin, \ - "_X_regular_begin", \ - NULL, \ - LATIN_CAPITAL_LETTER_SHARP_S_UTF8); \ - LOAD_UTF8_CHARCLASS_DEBUG_TEST(PL_utf8_X_extend, \ - "_X_extend", \ - NULL, \ - COMBINING_GRAVE_ACCENT_UTF8); \ - } STMT_END + LATIN_SMALL_LIGATURE_LONG_S_T_UTF8); #define PLACEHOLDER /* Something for the preprocessor to grab onto */ /* TODO: Combine JUMPABLE and HAS_TEXT to cache OP(rn) */ @@ -231,15 +225,15 @@ static const char* const non_utf8_target_but_utf8_required #if 0 /* Currently these are only used when PL_regkind[OP(rn)] == EXACT so - we don't need this definition. */ + we don't need this definition. XXX These are now out-of-sync*/ #define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==REF || OP(rn)==NREF ) #define IS_TEXTF(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn)==EXACTFA || OP(rn)==EXACTFA_NO_TRIE || OP(rn)==EXACTF || OP(rn)==REFF || OP(rn)==NREFF ) #define IS_TEXTFL(rn) ( OP(rn)==EXACTFL || OP(rn)==REFFL || OP(rn)==NREFFL ) #else /* ... so we use this as its faster. */ -#define IS_TEXT(rn) ( OP(rn)==EXACT ) -#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFA || OP(rn) == EXACTFA_NO_TRIE) +#define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==EXACTL ) +#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFLU8 || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFA || OP(rn) == EXACTFA_NO_TRIE) #define IS_TEXTF(rn) ( OP(rn)==EXACTF ) #define IS_TEXTFL(rn) ( OP(rn)==EXACTFL ) @@ -262,16 +256,6 @@ static const char* const non_utf8_target_but_utf8_required } \ } STMT_END -/* These constants are for finding GCB=LV and GCB=LVT in the CLUMP regnode. - * These are for the pre-composed Hangul syllables, which are all in a - * contiguous block and arranged there in such a way so as to facilitate - * alorithmic determination of their characteristics. As such, they don't need - * a swash, but can be determined by simple arithmetic. Almost all are - * GCB=LVT, but every 28th one is a GCB=LV */ -#define SBASE 0xAC00 /* Start of block */ -#define SCount 11172 /* Length of block */ -#define TCount 28 - #define SLAB_FIRST(s) (&(s)->states[0]) #define SLAB_LAST(s) (&(s)->states[PERL_REGMATCH_SLAB_SLOTS-1]) @@ -475,7 +459,6 @@ S_isFOO_lc(pTHX_ const U8 classnum, const U8 character) case _CC_ENUM_GRAPH: return isGRAPH_LC(character); case _CC_ENUM_LOWER: return isLOWER_LC(character); case _CC_ENUM_PRINT: return isPRINT_LC(character); - case _CC_ENUM_PSXSPC: return isPSXSPC_LC(character); case _CC_ENUM_PUNCT: return isPUNCT_LC(character); case _CC_ENUM_SPACE: return isSPACE_LC(character); case _CC_ENUM_UPPER: return isUPPER_LC(character); @@ -509,9 +492,11 @@ S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character) } else if (UTF8_IS_DOWNGRADEABLE_START(*character)) { return isFOO_lc(classnum, - TWO_BYTE_UTF8_TO_NATIVE(*character, *(character + 1))); + EIGHT_BIT_UTF8_TO_NATIVE(*character, *(character + 1))); } + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(character, character + UTF8SKIP(character)); + if (classnum < _FIRST_NON_SWASH_CC) { /* Initialize the swash unless done already */ @@ -530,9 +515,7 @@ S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character) } switch ((_char_class_number) classnum) { - case _CC_ENUM_SPACE: - case _CC_ENUM_PSXSPC: return is_XPERLSPACE_high(character); - + case _CC_ENUM_SPACE: return is_XPERLSPACE_high(character); case _CC_ENUM_BLANK: return is_HORIZWS_high(character); case _CC_ENUM_XDIGIT: return is_XDIGIT_high(character); case _CC_ENUM_VERTSPACE: return is_VERTWS_high(character); @@ -671,7 +654,7 @@ Perl_re_intuit_start(pTHX_ "Intuit: trying to determine minimum start position...\n")); /* for now, assume that all substr offsets are positive. If at some point - * in the future someone wants to do clever things with look-behind and + * in the future someone wants to do clever things with lookbehind and * -ve offsets, they'll need to fix up any code in this function * which uses these offsets. See the thread beginning * <20140113145929.GF27210@iabyn.com> @@ -752,7 +735,7 @@ Perl_re_intuit_start(pTHX_ /* ml_anch: check after \n? * - * A note about IMPLICIT: on an un-anchored pattern beginning + * A note about PREGf_IMPLICIT: on an un-anchored pattern beginning * with /.*.../, these flags will have been added by the * compiler: * /.*abc/, /.*abc/m: PREGf_IMPLICIT | PREGf_ANCH_MBOL @@ -790,9 +773,7 @@ Perl_re_intuit_start(pTHX_ * caller will have set strpos=pos()-4; we look for the substr * at position pos()-4+1, which lines up with the "a" */ - if (prog->check_offset_min == prog->check_offset_max - && !(prog->intflags & PREGf_CANY_SEEN)) - { + if (prog->check_offset_min == prog->check_offset_max) { /* Substring at constant offset from beg-of-str... */ SSize_t slen = SvCUR(check); char *s = HOP3c(strpos, prog->check_offset_min, strend); @@ -876,24 +857,17 @@ Perl_re_intuit_start(pTHX_ " At restart: rx_origin=%"IVdf" Check offset min: %"IVdf " Start shift: %"IVdf" End shift %"IVdf " Real end Shift: %"IVdf"\n", - (IV)(rx_origin - strpos), + (IV)(rx_origin - strbeg), (IV)prog->check_offset_min, (IV)start_shift, (IV)end_shift, (IV)prog->check_end_shift); }); - if (prog->intflags & PREGf_CANY_SEEN) { - start_point= (U8*)(rx_origin + start_shift); - end_point= (U8*)(strend - end_shift); - if (start_point > end_point) - goto fail_finish; - } else { - end_point = HOP3(strend, -end_shift, strbeg); - start_point = HOPMAYBE3(rx_origin, start_shift, end_point); - if (!start_point) - goto fail_finish; - } + end_point = HOP3(strend, -end_shift, strbeg); + start_point = HOPMAYBE3(rx_origin, start_shift, end_point); + if (!start_point) + goto fail_finish; /* If the regex is absolutely anchored to either the start of the @@ -924,16 +898,16 @@ Perl_re_intuit_start(pTHX_ } } - DEBUG_OPTIMISE_MORE_r({ - PerlIO_printf(Perl_debug_log, " fbm_instr len=%d str=<%.*s>\n", - (int)(end_point - start_point), - (int)(end_point - start_point) > 20 ? 20 : (int)(end_point - start_point), - start_point); - }); - check_at = fbm_instr( start_point, end_point, check, multiline ? FBMrf_MULTILINE : 0); + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " doing 'check' fbm scan, [%"IVdf"..%"IVdf"] gave %"IVdf"\n", + (IV)((char*)start_point - strbeg), + (IV)((char*)end_point - strbeg), + (IV)(check_at ? check_at - strbeg : -1) + )); + /* Update the count-of-usability, remove useless subpatterns, unshift s. */ @@ -951,9 +925,6 @@ Perl_re_intuit_start(pTHX_ if (!check_at) goto fail_finish; - /* Finish the diagnostic message */ - DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "%ld...\n", (long)(check_at - strpos)) ); - /* set rx_origin to the minimum position where the regex could start * matching, given the constraint of the just-matched check substring. * But don't set it lower than previously. @@ -961,6 +932,12 @@ Perl_re_intuit_start(pTHX_ if (check_at - rx_origin > prog->check_offset_max) rx_origin = HOP3c(check_at, -prog->check_offset_max, rx_origin); + /* Finish the diagnostic message */ + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + "%ld (rx_origin now %"IVdf")...\n", + (long)(check_at - strbeg), + (IV)(rx_origin - strbeg) + )); } @@ -1064,12 +1041,34 @@ Perl_re_intuit_start(pTHX_ must = utf8_target ? other->utf8_substr : other->substr; assert(SvPOK(must)); - s = fbm_instr( - (unsigned char*)s, - (unsigned char*)last + SvCUR(must) - (SvTAIL(must)!=0), - must, - multiline ? FBMrf_MULTILINE : 0 - ); + { + char *from = s; + char *to = last + SvCUR(must) - (SvTAIL(must)!=0); + + if (from > to) { + s = NULL; + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " skipping 'other' fbm scan: %"IVdf" > %"IVdf"\n", + (IV)(from - strbeg), + (IV)(to - strbeg) + )); + } + else { + s = fbm_instr( + (unsigned char*)from, + (unsigned char*)to, + must, + multiline ? FBMrf_MULTILINE : 0 + ); + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " doing 'other' fbm scan, [%"IVdf"..%"IVdf"] gave %"IVdf"\n", + (IV)(from - strbeg), + (IV)(to - strbeg), + (IV)(s ? s - strbeg : -1) + )); + } + } + DEBUG_EXECUTE_r({ RE_PV_QUOTED_DECL(quoted, utf8_target, PERL_DEBUG_PAD_ZERO(0), SvPVX_const(must), RE_SV_DUMPLEN(must), 30); @@ -1085,29 +1084,27 @@ Perl_re_intuit_start(pTHX_ * find it before there, we never will */ if (last >= last1) { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - ", giving up...\n")); + "; giving up...\n")); goto fail_finish; } /* try to find the check substr again at a later * position. Maybe next time we'll find the "other" substr * in range too */ - DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - ", trying %s at offset %ld...\n", - (other_ix ? "floating" : "anchored"), - (long)(HOP3c(check_at, 1, strend) - strpos))); - other_last = HOP3c(last, 1, strend) /* highest failure */; rx_origin = other_ix /* i.e. if other-is-float */ ? HOP3c(rx_origin, 1, strend) : HOP4c(last, 1 - other->min_offset, strbeg, strend); + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + "; about to retry %s at offset %ld (rx_origin now %"IVdf")...\n", + (other_ix ? "floating" : "anchored"), + (long)(HOP3c(check_at, 1, strend) - strbeg), + (IV)(rx_origin - strbeg) + )); goto restart; } else { - DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, " at offset %ld...\n", - (long)(s - strpos))); - if (other_ix) { /* if (other-is-float) */ /* other_last is set to s, not s+1, since its possible for * a floating substr to fail first time, then succeed @@ -1123,6 +1120,12 @@ Perl_re_intuit_start(pTHX_ rx_origin = HOP3c(s, -other->min_offset, strbeg); other_last = HOP3c(s, 1, strend); } + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " at offset %ld (rx_origin now %"IVdf")...\n", + (long)(s - strbeg), + (IV)(rx_origin - strbeg) + )); + } } else { @@ -1130,13 +1133,13 @@ Perl_re_intuit_start(pTHX_ PerlIO_printf(Perl_debug_log, " Check-only match: offset min:%"IVdf" max:%"IVdf " check_at:%"IVdf" rx_origin:%"IVdf" rx_origin-check_at:%"IVdf - " strend-strpos:%"IVdf"\n", + " strend:%"IVdf"\n", (IV)prog->check_offset_min, (IV)prog->check_offset_max, - (IV)(check_at-strpos), - (IV)(rx_origin-strpos), + (IV)(check_at-strbeg), + (IV)(rx_origin-strbeg), (IV)(rx_origin-check_at), - (IV)(strend-strpos) + (IV)(strend-strbeg) ) ); } @@ -1157,7 +1160,7 @@ Perl_re_intuit_start(pTHX_ * scanning ahead for the next \n or the next substr is debatable. * On the one hand you'd expect rare substrings to appear less * often than \n's. On the other hand, searching for \n means - * we're effectively flipping been check_substr and "\n" on each + * we're effectively flipping between check_substr and "\n" on each * iteration as the current "rarest" string candidate, which * means for example that we'll quickly reject the whole string if * hasn't got a \n, rather than trying every substr position @@ -1186,8 +1189,8 @@ Perl_re_intuit_start(pTHX_ * check was anchored (and thus has no wiggle room), * or check was float and rx_origin is above the float range */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - " Found /%s^%s/m, restarting lookup for check-string at offset %ld...\n", - PL_colors[0], PL_colors[1], (long)(rx_origin - strpos))); + " Found /%s^%s/m, about to restart lookup for check-string with rx_origin %ld...\n", + PL_colors[0], PL_colors[1], (long)(rx_origin - strbeg))); goto restart; } @@ -1202,18 +1205,19 @@ Perl_re_intuit_start(pTHX_ * didn't contradict, so just retry the anchored "other" * substr */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - " Found /%s^%s/m at offset %ld, rescanning for anchored from offset %ld...\n", + " Found /%s^%s/m, rescanning for anchored from offset %"IVdf" (rx_origin now %"IVdf")...\n", PL_colors[0], PL_colors[1], - (long)(rx_origin - strpos), - (long)(rx_origin - strpos + prog->anchored_offset))); + (IV)(rx_origin - strbeg + prog->anchored_offset), + (IV)(rx_origin - strbeg) + )); goto do_other_substr; } /* success: we don't contradict the found floating substring * (and there's no anchored substr). */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - " Found /%s^%s/m at offset %ld...\n", - PL_colors[0], PL_colors[1], (long)(rx_origin - strpos))); + " Found /%s^%s/m with rx_origin %ld...\n", + PL_colors[0], PL_colors[1], (long)(rx_origin - strbeg))); } else { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, @@ -1305,11 +1309,15 @@ Perl_re_intuit_start(pTHX_ * The condition above is in bytes rather than * chars for efficiency. It's conservative, in * that it errs on the side of doing 'goto - * do_other_substr', where a more accurate - * char-based calculation will be done */ + * do_other_substr'. In this case, at worst, + * an extra anchored search may get done, but in + * practice the extra fbm_instr() is likely to + * get skipped anyway. */ DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - " Looking for anchored substr starting at offset %ld...\n", - (long)(other_last - strpos)) ); + " about to retry anchored at offset %ld (rx_origin now %"IVdf")...\n", + (long)(other_last - strbeg), + (IV)(rx_origin - strbeg) + )); goto do_other_substr; } } @@ -1327,9 +1335,9 @@ Perl_re_intuit_start(pTHX_ * search for the next \n if any, its safe here */ rx_origin++; DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - " Looking for /%s^%s/m starting at offset %ld...\n", + " about to look for /%s^%s/m starting at rx_origin %ld...\n", PL_colors[0], PL_colors[1], - (long)(rx_origin - strpos)) ); + (long)(rx_origin - strbeg)) ); goto postprocess_substr_matches; } @@ -1355,9 +1363,11 @@ Perl_re_intuit_start(pTHX_ goto fail; } DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - " Looking for %s substr starting at offset %ld...\n", + " about to look for %s substr starting at offset %ld (rx_origin now %"IVdf")...\n", (prog->substrs->check_ix ? "floating" : "anchored"), - (long)(rx_origin + start_shift - strpos)) ); + (long)(rx_origin + start_shift - strbeg), + (IV)(rx_origin - strbeg) + )); goto restart; } @@ -1366,7 +1376,7 @@ Perl_re_intuit_start(pTHX_ if (rx_origin != s) { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, " By STCLASS: moving %ld --> %ld\n", - (long)(rx_origin - strpos), (long)(s - strpos)) + (long)(rx_origin - strbeg), (long)(s - strbeg)) ); } else { @@ -1418,7 +1428,7 @@ Perl_re_intuit_start(pTHX_ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "Intuit: %sSuccessfully guessed:%s match at offset %ld\n", - PL_colors[4], PL_colors[5], (long)(rx_origin - strpos)) ); + PL_colors[4], PL_colors[5], (long)(rx_origin - strbeg)) ); return rx_origin; @@ -1434,26 +1444,38 @@ Perl_re_intuit_start(pTHX_ #define DECL_TRIE_TYPE(scan) \ const enum { trie_plain, trie_utf8, trie_utf8_fold, trie_latin_utf8_fold, \ - trie_utf8_exactfa_fold, trie_latin_utf8_exactfa_fold } \ + trie_utf8_exactfa_fold, trie_latin_utf8_exactfa_fold, \ + trie_utf8l, trie_flu8 } \ trie_type = ((scan->flags == EXACT) \ ? (utf8_target ? trie_utf8 : trie_plain) \ - : (scan->flags == EXACTFA) \ - ? (utf8_target \ - ? trie_utf8_exactfa_fold \ - : trie_latin_utf8_exactfa_fold) \ - : (utf8_target \ - ? trie_utf8_fold \ - : trie_latin_utf8_fold)) + : (scan->flags == EXACTL) \ + ? (utf8_target ? trie_utf8l : trie_plain) \ + : (scan->flags == EXACTFA) \ + ? (utf8_target \ + ? trie_utf8_exactfa_fold \ + : trie_latin_utf8_exactfa_fold) \ + : (scan->flags == EXACTFLU8 \ + ? trie_flu8 \ + : (utf8_target \ + ? trie_utf8_fold \ + : trie_latin_utf8_fold))) #define REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc, uscan, len, uvc, charid, foldlen, foldbuf, uniflags) \ STMT_START { \ STRLEN skiplen; \ U8 flags = FOLD_FLAGS_FULL; \ switch (trie_type) { \ + case trie_flu8: \ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; \ + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*uc)) { \ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(uc, uc + UTF8SKIP(uc)); \ + } \ + goto do_trie_utf8_fold; \ case trie_utf8_exactfa_fold: \ flags |= FOLD_FLAGS_NOMIX_ASCII; \ - /* FALLTHROUGH */ \ + /* FALLTHROUGH */ \ case trie_utf8_fold: \ + do_trie_utf8_fold: \ if ( foldlen>0 ) { \ uvc = utf8n_to_uvchr( (const U8*) uscan, UTF8_MAXLEN, &len, uniflags ); \ foldlen -= len; \ @@ -1462,14 +1484,14 @@ STMT_START { } else { \ uvc = _to_utf8_fold_flags( (const U8*) uc, foldbuf, &foldlen, flags); \ len = UTF8SKIP(uc); \ - skiplen = UNISKIP( uvc ); \ + skiplen = UVCHR_SKIP( uvc ); \ foldlen -= skiplen; \ uscan = foldbuf + skiplen; \ } \ break; \ case trie_latin_utf8_exactfa_fold: \ flags |= FOLD_FLAGS_NOMIX_ASCII; \ - /* FALLTHROUGH */ \ + /* FALLTHROUGH */ \ case trie_latin_utf8_fold: \ if ( foldlen>0 ) { \ uvc = utf8n_to_uvchr( (const U8*) uscan, UTF8_MAXLEN, &len, uniflags ); \ @@ -1479,11 +1501,17 @@ STMT_START { } else { \ len = 1; \ uvc = _to_fold_latin1( (U8) *uc, foldbuf, &foldlen, flags); \ - skiplen = UNISKIP( uvc ); \ + skiplen = UVCHR_SKIP( uvc ); \ foldlen -= skiplen; \ uscan = foldbuf + skiplen; \ } \ break; \ + case trie_utf8l: \ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; \ + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*uc)) { \ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(uc, uc + UTF8SKIP(uc)); \ + } \ + /* FALLTHROUGH */ \ case trie_utf8: \ uvc = utf8n_to_uvchr( (const U8*) uc, UTF8_MAXLEN, &len, uniflags ); \ break; \ @@ -1693,7 +1721,7 @@ REXEC_FBC_SCAN( /* Loops while (s < strend) */ \ FBC_UTF8(TEST_UV, TEST_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER), \ TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER) -#define FBC_BOUND_A(TEST_NON_UTF8, TEST_UV, TEST_UTF8) \ +#define FBC_BOUND_A(TEST_NON_UTF8) \ FBC_BOUND_COMMON( \ FBC_UTF8_A(TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER), \ TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER) @@ -1703,11 +1731,89 @@ REXEC_FBC_SCAN( /* Loops while (s < strend) */ \ FBC_UTF8(TEST_UV, TEST_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT), \ TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT) -#define FBC_NBOUND_A(TEST_NON_UTF8, TEST_UV, TEST_UTF8) \ +#define FBC_NBOUND_A(TEST_NON_UTF8) \ FBC_BOUND_COMMON( \ FBC_UTF8_A(TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT), \ TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT) +#ifdef DEBUGGING +static IV +S_get_break_val_cp_checked(SV* const invlist, const UV cp_in) { + IV cp_out = Perl__invlist_search(invlist, cp_in); + assert(cp_out >= 0); + return cp_out; +} +# define _generic_GET_BREAK_VAL_CP_CHECKED(invlist, invmap, cp) \ + invmap[S_get_break_val_cp_checked(invlist, cp)] +#else +# define _generic_GET_BREAK_VAL_CP_CHECKED(invlist, invmap, cp) \ + invmap[_invlist_search(invlist, cp)] +#endif + +/* Takes a pointer to an inversion list, a pointer to its corresponding + * inversion map, and a code point, and returns the code point's value + * according to the two arrays. It assumes that all code points have a value. + * This is used as the base macro for macros for particular properties */ +#define _generic_GET_BREAK_VAL_CP(invlist, invmap, cp) \ + _generic_GET_BREAK_VAL_CP_CHECKED(invlist, invmap, cp) + +/* Same as above, but takes begin, end ptrs to a UTF-8 encoded string instead + * of a code point, returning the value for the first code point in the string. + * And it takes the particular macro name that finds the desired value given a + * code point. Merely convert the UTF-8 to code point and call the cp macro */ +#define _generic_GET_BREAK_VAL_UTF8(cp_macro, pos, strend) \ + (__ASSERT_(pos < strend) \ + /* Note assumes is valid UTF-8 */ \ + (cp_macro(utf8_to_uvchr_buf((pos), (strend), NULL)))) + +/* Returns the GCB value for the input code point */ +#define getGCB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_GCB_invlist, \ + _Perl_GCB_invmap, \ + (cp)) + +/* Returns the GCB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getGCB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getGCB_VAL_CP, pos, strend) + +/* Returns the LB value for the input code point */ +#define getLB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_LB_invlist, \ + _Perl_LB_invmap, \ + (cp)) + +/* Returns the LB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getLB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getLB_VAL_CP, pos, strend) + + +/* Returns the SB value for the input code point */ +#define getSB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_SB_invlist, \ + _Perl_SB_invmap, \ + (cp)) + +/* Returns the SB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getSB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getSB_VAL_CP, pos, strend) + +/* Returns the WB value for the input code point */ +#define getWB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_WB_invlist, \ + _Perl_WB_invmap, \ + (cp)) + +/* Returns the WB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getWB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getWB_VAL_CP, pos, strend) /* We know what class REx starts with. Try to find this position... */ /* if reginfo->intuit, its a dryrun */ @@ -1743,6 +1849,15 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* We know what class it must start with. */ switch (OP(c)) { + case ANYOFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (ANYOFL_UTF8_LOCALE_REQD(FLAGS(c)) && ! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), utf8_locale_required); + } + + /* FALLTHROUGH */ + case ANYOFD: case ANYOF: if (utf8_target) { REXEC_FBC_UTF8_CLASS_SCAN( @@ -1752,14 +1867,6 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, REXEC_FBC_CLASS_SCAN(REGINCLASS(prog, c, (U8*)s)); } break; - case CANY: - REXEC_FBC_SCAN( - if (tmp && (reginfo->intuit || regtry(reginfo, &s))) - goto got_it; - else - tmp = doevery; - ); - break; case EXACTFA_NO_TRIE: /* This node only generated for non-utf8 patterns */ assert(! is_utf8_pat); @@ -1784,6 +1891,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, goto do_exactf_non_utf8; case EXACTFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; if (is_utf8_pat || utf8_target || IN_UTF8_CTYPE_LOCALE) { utf8_fold_flags = FOLDEQ_LOCALE; goto do_exactf_utf8; @@ -1798,6 +1906,15 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } goto do_exactf_utf8; + case EXACTFLU8: + if (! utf8_target) { /* All code points in this node require + UTF-8 to express. */ + break; + } + utf8_fold_flags = FOLDEQ_LOCALE | FOLDEQ_S2_ALREADY_FOLDED + | FOLDEQ_S2_FOLDS_SANE; + goto do_exactf_utf8; + case EXACTFU: if (is_utf8_pat || utf8_target) { utf8_fold_flags = is_utf8_pat ? FOLDEQ_S2_ALREADY_FOLDED : 0; @@ -1812,7 +1929,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* FALLTHROUGH */ - do_exactf_non_utf8: /* Neither pattern nor string are UTF8, and there + do_exactf_non_utf8: /* Neither pattern nor string are UTF8, and there are no glitches with fold-length differences between the target string and pattern */ @@ -1846,8 +1963,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } break; - do_exactf_utf8: - { + do_exactf_utf8: + { unsigned expansion; /* If one of the operands is in utf8, we can't use the simpler folding @@ -1903,29 +2020,315 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } case BOUNDL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (FLAGS(c) != TRADITIONAL_BOUND) { + if (! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + B_ON_NON_UTF8_LOCALE_IS_WRONG); + } + goto do_boundu; + } + FBC_BOUND(isWORDCHAR_LC, isWORDCHAR_LC_uvchr, isWORDCHAR_LC_utf8); break; + case NBOUNDL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (FLAGS(c) != TRADITIONAL_BOUND) { + if (! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + B_ON_NON_UTF8_LOCALE_IS_WRONG); + } + goto do_nboundu; + } + FBC_NBOUND(isWORDCHAR_LC, isWORDCHAR_LC_uvchr, isWORDCHAR_LC_utf8); break; - case BOUND: + + case BOUND: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + FBC_BOUND(isWORDCHAR, isWORDCHAR_uni, isWORDCHAR_utf8); break; - case BOUNDA: - FBC_BOUND_A(isWORDCHAR_A, isWORDCHAR_A, isWORDCHAR_A); + + case BOUNDA: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + + FBC_BOUND_A(isWORDCHAR_A); break; - case NBOUND: + + case NBOUND: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + FBC_NBOUND(isWORDCHAR, isWORDCHAR_uni, isWORDCHAR_utf8); break; - case NBOUNDA: - FBC_NBOUND_A(isWORDCHAR_A, isWORDCHAR_A, isWORDCHAR_A); - break; - case BOUNDU: - FBC_BOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + + case NBOUNDA: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + + FBC_NBOUND_A(isWORDCHAR_A); break; + case NBOUNDU: - FBC_NBOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + if ((bound_type) FLAGS(c) == TRADITIONAL_BOUND) { + FBC_NBOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + break; + } + + do_nboundu: + + to_complement = 1; + /* FALLTHROUGH */ + + case BOUNDU: + do_boundu: + switch((bound_type) FLAGS(c)) { + case TRADITIONAL_BOUND: + FBC_BOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + break; + case GCB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) + { + goto got_it; + } + + /* Didn't match. Try at the next position (if there is one) */ + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + GCB_enum before = getGCB_VAL_UTF8( + reghop3((U8*)s, -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + GCB_enum after = getGCB_VAL_UTF8((U8*) s, + (U8*) reginfo->strend); + if ( (to_complement ^ isGCB(before, after)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. Everything is a GCB except between CR and + LF */ + while (s < strend) { + if ((to_complement ^ ( UCHARAT(s - 1) != '\r' + || UCHARAT(s) != '\n')) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + s++; + } + } + + /* And, since this is a bound, it can match after the final + * character in the string */ + if ((reginfo->intuit || regtry(reginfo, &s))) { + goto got_it; + } + break; + + case LB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + LB_enum before = getLB_VAL_UTF8(reghop3((U8*)s, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + LB_enum after = getLB_VAL_UTF8((U8*) s, (U8*) reginfo->strend); + if (to_complement ^ isLB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. */ + LB_enum before = getLB_VAL_CP((U8) *(s -1)); + while (s < strend) { + LB_enum after = getLB_VAL_CP((U8) *s); + if (to_complement ^ isLB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s++; + } + } + + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + + break; + + case SB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + SB_enum before = getSB_VAL_UTF8(reghop3((U8*)s, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + SB_enum after = getSB_VAL_UTF8((U8*) s, + (U8*) reginfo->strend); + if ((to_complement ^ isSB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. */ + SB_enum before = getSB_VAL_CP((U8) *(s -1)); + while (s < strend) { + SB_enum after = getSB_VAL_CP((U8) *s); + if ((to_complement ^ isSB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s++; + } + } + + /* Here are at the final position in the target string. The SB + * value is always true here, so matches, depending on other + * constraints */ + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + + break; + + case WB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + /* We are at a boundary between char_sub_0 and char_sub_1. + * We also keep track of the value for char_sub_-1 as we + * loop through the line. Context may be needed to make a + * determination, and if so, this can save having to + * recalculate it */ + WB_enum previous = WB_UNKNOWN; + WB_enum before = getWB_VAL_UTF8( + reghop3((U8*)s, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + WB_enum after = getWB_VAL_UTF8((U8*) s, + (U8*) reginfo->strend); + if ((to_complement ^ isWB(previous, + before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + previous = before; + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. */ + WB_enum previous = WB_UNKNOWN; + WB_enum before = getWB_VAL_CP((U8) *(s -1)); + while (s < strend) { + WB_enum after = getWB_VAL_CP((U8) *s); + if ((to_complement ^ isWB(previous, + before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + previous = before; + before = after; + s++; + } + } + + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + } break; + case LNBREAK: REXEC_FBC_CSCAN(is_LNBREAK_utf8_safe(s, strend), is_LNBREAK_latin1_safe(s, strend) @@ -1940,6 +2343,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* FALLTHROUGH */ case POSIXL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; REXEC_FBC_CSCAN(to_complement ^ cBOOL(isFOO_utf8_lc(FLAGS(c), (U8 *) s)), to_complement ^ cBOOL(isFOO_lc(FLAGS(c), *s))); break; @@ -1985,7 +2389,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } else { - posix_utf8: + posix_utf8: classnum = (_char_class_number) FLAGS(c); if (classnum < _FIRST_NON_SWASH_CC) { while (s < strend) { @@ -2002,7 +2406,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, classnum))) || (UTF8_IS_DOWNGRADEABLE_START(*s) && to_complement ^ cBOOL( - _generic_isCC(TWO_BYTE_UTF8_TO_NATIVE(*s, + _generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*s, *(s + 1)), classnum)))) { @@ -2020,11 +2424,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } else switch (classnum) { /* These classes are implemented as macros */ - case _CC_ENUM_SPACE: /* XXX would require separate code if we - revert the change of \v matching this */ - /* FALLTHROUGH */ - - case _CC_ENUM_PSXSPC: + case _CC_ENUM_SPACE: REXEC_FBC_UTF8_CLASS_SCAN( to_complement ^ cBOOL(isSPACE_utf8(s))); break; @@ -2366,7 +2766,7 @@ S_reg_set_capture_string(pTHX_ REGEXP * const rx, U32 n = 0; max = -1; /* calculate the right-most part of the string covered - * by a capture. Due to look-ahead, this may be to + * by a capture. Due to lookahead, this may be to * the right of $&, so we have to scan all captures */ while (n <= prog->lastparen) { if (prog->offs[n].end > max) @@ -2387,7 +2787,7 @@ S_reg_set_capture_string(pTHX_ REGEXP * const rx, U32 n = 0; min = max; /* calculate the left-most part of the string covered - * by a capture. Due to look-behind, this may be to + * by a capture. Due to lookbehind, this may be to * the left of $&, so we have to scan all captures */ while (min && n <= prog->lastparen) { if ( prog->offs[n].start != -1 @@ -2492,7 +2892,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, PERL_UNUSED_ARG(data); /* Be paranoid... */ - if (prog == NULL || stringarg == NULL) { + if (prog == NULL) { Perl_croak(aTHX_ "NULL regexp parameter"); } @@ -2503,6 +2903,11 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, startpos = stringarg; + /* set these early as they may be used by the HOP macros below */ + reginfo->strbeg = strbeg; + reginfo->strend = strend; + reginfo->is_utf8_target = cBOOL(utf8_target); + if (prog->intflags & PREGf_GPOS_SEEN) { MAGIC *mg; @@ -2511,7 +2916,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, reginfo->ganch = (flags & REXEC_IGNOREPOS) ? stringarg /* use start pos rather than pos() */ - : (sv && (mg = mg_find_mglob(sv)) && mg->mg_len >= 0) + : ((mg = mg_find_mglob(sv)) && mg->mg_len >= 0) /* Defined pos(): */ ? strbeg + MgBYTEPOS(mg, sv, strbeg, strend-strbeg) : strbeg; /* pos() not defined; use start of string */ @@ -2530,20 +2935,23 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, */ if (prog->intflags & PREGf_ANCH_GPOS) { - startpos = reginfo->ganch - prog->gofs; - if (startpos < - ((flags & REXEC_FAIL_ON_UNDERFLOW) ? stringarg : strbeg)) - { - DEBUG_r(PerlIO_printf(Perl_debug_log, - "fail: ganch-gofs before earliest possible start\n")); - return 0; + if (prog->gofs) { + startpos = HOPBACKc(reginfo->ganch, prog->gofs); + if (!startpos || + ((flags & REXEC_FAIL_ON_UNDERFLOW) && startpos < stringarg)) + { + DEBUG_r(PerlIO_printf(Perl_debug_log, + "fail: ganch-gofs before earliest possible start\n")); + return 0; + } } + else + startpos = reginfo->ganch; } else if (prog->gofs) { - if (startpos - prog->gofs < strbeg) + startpos = HOPBACKc(startpos, prog->gofs); + if (!startpos) startpos = strbeg; - else - startpos -= prog->gofs; } else if (prog->intflags & PREGf_GPOS_FLOAT) startpos = strbeg; @@ -2626,13 +3034,10 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, reginfo->prog = rx; /* Yes, sorry that this is confusing. */ reginfo->intuit = 0; - reginfo->is_utf8_target = cBOOL(utf8_target); reginfo->is_utf8_pat = cBOOL(RX_UTF8(rx)); reginfo->warned = FALSE; - reginfo->strbeg = strbeg; reginfo->sv = sv; reginfo->poscache_maxiter = 0; /* not yet started a countdown */ - reginfo->strend = strend; /* see how far we have to get to not match where we matched before */ reginfo->till = stringarg + minend; @@ -2719,93 +3124,59 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, )); } - /* Simplest case: anchored match need be tried only once. */ - /* [unless only anchor is MBOL - implying multiline is set] */ + /* Simplest case: anchored match need be tried only once, or with + * MBOL, only at the beginning of each line. + * + * Note that /.*.../ sets PREGf_IMPLICIT|MBOL, while /.*.../s sets + * PREGf_IMPLICIT|SBOL. The idea is that with /.*.../s, if it doesn't + * match at the start of the string then it won't match anywhere else + * either; while with /.*.../, if it doesn't match at the beginning, + * the earliest it could match is at the start of the next line */ + if (prog->intflags & (PREGf_ANCH & ~PREGf_ANCH_GPOS)) { - if (s == startpos && regtry(reginfo, &s)) + char *end; + + if (regtry(reginfo, &s)) goto got_it; - else if (multiline || (prog->intflags & (PREGf_IMPLICIT | PREGf_ANCH_MBOL))) /* XXXX SBOL? */ - { - char *end; - - if (minlen) - dontbother = minlen - 1; - end = HOP3c(strend, -dontbother, strbeg) - 1; - /* for multiline we only have to try after newlines */ - if (prog->check_substr || prog->check_utf8) { - /* because of the goto we can not easily reuse the macros for bifurcating the - unicode/non-unicode match modes here like we do elsewhere - demerphq */ - if (utf8_target) { - if (s == startpos) - goto after_try_utf8; - while (1) { - if (regtry(reginfo, &s)) { - goto got_it; - } - after_try_utf8: - if (s > end) { - goto phooey; - } - if (prog->extflags & RXf_USE_INTUIT) { - s = re_intuit_start(rx, sv, strbeg, - s + UTF8SKIP(s), strend, flags, NULL); - if (!s) { - goto phooey; - } - } - else { - s += UTF8SKIP(s); - } - } - } /* end search for check string in unicode */ - else { - if (s == startpos) { - goto after_try_latin; - } - while (1) { - if (regtry(reginfo, &s)) { - goto got_it; - } - after_try_latin: - if (s > end) { - goto phooey; - } - if (prog->extflags & RXf_USE_INTUIT) { - s = re_intuit_start(rx, sv, strbeg, - s + 1, strend, flags, NULL); - if (!s) { - goto phooey; - } - } - else { - s++; - } - } - } /* end search for check string in latin*/ - } /* end search for check string */ - else { /* search for newline */ - if (s > startpos) { - /*XXX: The s-- is almost definitely wrong here under unicode - demeprhq*/ - s--; - } - /* We can use a more efficient search as newlines are the same in unicode as they are in latin */ - while (s <= end) { /* note it could be possible to match at the end of the string */ - if (*s++ == '\n') { /* don't need PL_utf8skip here */ - if (regtry(reginfo, &s)) - goto got_it; - } - } - } /* end search for newline */ - } /* end anchored/multiline check string search */ - goto phooey; - } else if (prog->intflags & PREGf_ANCH_GPOS) + + if (!(prog->intflags & PREGf_ANCH_MBOL)) + goto phooey; + + /* didn't match at start, try at other newline positions */ + + if (minlen) + dontbother = minlen - 1; + end = HOP3c(strend, -dontbother, strbeg) - 1; + + /* skip to next newline */ + + while (s <= end) { /* note it could be possible to match at the end of the string */ + /* NB: newlines are the same in unicode as they are in latin */ + if (*s++ != '\n') + continue; + if (prog->check_substr || prog->check_utf8) { + /* note that with PREGf_IMPLICIT, intuit can only fail + * or return the start position, so it's of limited utility. + * Nevertheless, I made the decision that the potential for + * quick fail was still worth it - DAPM */ + s = re_intuit_start(rx, sv, strbeg, s, strend, flags, NULL); + if (!s) + goto phooey; + } + if (regtry(reginfo, &s)) + goto got_it; + } + goto phooey; + } /* end anchored search */ + + if (prog->intflags & PREGf_ANCH_GPOS) { /* PREGf_ANCH_GPOS should never be true if PREGf_GPOS_SEEN is not true */ assert(prog->intflags & PREGf_GPOS_SEEN); /* For anchored \G, the only position it can match from is * (ganch-gofs); we already set startpos to this above; if intuit * moved us on from there, we can't possibly succeed */ - assert(startpos == reginfo->ganch - prog->gofs); + assert(startpos == HOPBACKc(reginfo->ganch, prog->gofs)); if (s == startpos && regtry(reginfo, &s)) goto got_it; goto phooey; @@ -2969,7 +3340,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, if (minlen) { const OPCODE op = OP(progi->regstclass); /* don't bother with what can't match */ - if (PL_regkind[op] != EXACT && op != CANY && PL_regkind[op] != TRIE) + if (PL_regkind[op] != EXACT && PL_regkind[op] != TRIE) strend = HOPc(strend, -(minlen - 1)); } DEBUG_EXECUTE_r({ @@ -3111,7 +3482,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, /* Failure. */ goto phooey; -got_it: + got_it: /* s/// doesn't like it if $& is earlier than where we asked it to * start searching (which can happen on something like /.\G/) */ if ( (flags & REXEC_FAIL_ON_UNDERFLOW) @@ -3151,7 +3522,7 @@ got_it: return 1; -phooey: + phooey: DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "%sMatch failed%s\n", PL_colors[4], PL_colors[5])); @@ -3189,7 +3560,7 @@ phooey: /* - regtry - try match at specific point */ -STATIC I32 /* 0 failure, 1 success */ +STATIC bool /* 0 failure, 1 success */ S_regtry(pTHX_ regmatch_info *reginfo, char **startposp) { CHECKPOINT lastcp; @@ -3525,7 +3896,7 @@ S_dump_exec_pos(pTHX_ const char *locinput, if (pref0_len > pref_len) pref0_len = pref_len; { - const int is_uni = (utf8_target && OP(scan) != CANY) ? 1 : 0; + const int is_uni = utf8_target ? 1 : 0; RE_PV_COLOR_DECL(s0,len0,is_uni,PERL_DEBUG_PAD(0), (locinput - pref_len),pref0_len, 60, 4, 5); @@ -3652,7 +4023,7 @@ S_setup_EXACTISH_ST_c1_c2(pTHX_ const regnode * const text_node, int *c1p, U8 *pat = (U8*)STRING(text_node); U8 folded[UTF8_MAX_FOLD_CHAR_EXPAND * UTF8_MAXBYTES_CASE + 1] = { '\0' }; - if (OP(text_node) == EXACT) { + if (OP(text_node) == EXACT || OP(text_node) == EXACTL) { /* In an exact node, only one thing can be matched, that first * character. If both the pat and the target are UTF-8, we can just @@ -3846,54 +4217,901 @@ S_setup_EXACTISH_ST_c1_c2(pTHX_ const regnode * const text_node, int *c1p, c2 = PL_fold_latin1[c1]; break; - default: - Perl_croak(aTHX_ "panic: Unexpected op %u", OP(text_node)); - NOT_REACHED; /* NOTREACHED */ - } + default: + Perl_croak(aTHX_ "panic: Unexpected op %u", OP(text_node)); + NOT_REACHED; /* NOTREACHED */ + } + } + } + } + + /* Here have figured things out. Set up the returns */ + if (use_chrtest_void) { + *c2p = *c1p = CHRTEST_VOID; + } + else if (utf8_target) { + if (! utf8_has_been_setup) { /* Don't have the utf8; must get it */ + uvchr_to_utf8(c1_utf8, c1); + uvchr_to_utf8(c2_utf8, c2); + } + + /* Invariants are stored in both the utf8 and byte outputs; Use + * negative numbers otherwise for the byte ones. Make sure that the + * byte ones are the same iff the utf8 ones are the same */ + *c1p = (UTF8_IS_INVARIANT(*c1_utf8)) ? *c1_utf8 : CHRTEST_NOT_A_CP_1; + *c2p = (UTF8_IS_INVARIANT(*c2_utf8)) + ? *c2_utf8 + : (c1 == c2) + ? CHRTEST_NOT_A_CP_1 + : CHRTEST_NOT_A_CP_2; + } + else if (c1 > 255) { + if (c2 > 255) { /* both possibilities are above what a non-utf8 string + can represent */ + return FALSE; + } + + *c1p = *c2p = c2; /* c2 is the only representable value */ + } + else { /* c1 is representable; see about c2 */ + *c1p = c1; + *c2p = (c2 < 256) ? c2 : c1; + } + + return TRUE; +} + +PERL_STATIC_INLINE bool +S_isGCB(const GCB_enum before, const GCB_enum after) +{ + /* returns a boolean indicating if there is a Grapheme Cluster Boundary + * between the inputs. See http://www.unicode.org/reports/tr29/ */ + + return GCB_table[before][after]; +} + +/* Combining marks attach to most classes that precede them, but this defines + * the exceptions (from TR14) */ +#define LB_CM_ATTACHES_TO(prev) ( ! ( prev == LB_EDGE \ + || prev == LB_Mandatory_Break \ + || prev == LB_Carriage_Return \ + || prev == LB_Line_Feed \ + || prev == LB_Next_Line \ + || prev == LB_Space \ + || prev == LB_ZWSpace)) + +STATIC bool +S_isLB(pTHX_ LB_enum before, + LB_enum after, + const U8 * const strbeg, + const U8 * const curpos, + const U8 * const strend, + const bool utf8_target) +{ + U8 * temp_pos = (U8 *) curpos; + LB_enum prev = before; + + /* Is the boundary between 'before' and 'after' line-breakable? + * Most of this is just a table lookup of a generated table from Unicode + * rules. But some rules require context to decide, and so have to be + * implemented in code */ + + PERL_ARGS_ASSERT_ISLB; + + /* Rule numbers in the comments below are as of Unicode 8.0 */ + + redo: + before = prev; + switch (LB_table[before][after]) { + case LB_BREAKABLE: + return TRUE; + + case LB_NOBREAK: + case LB_NOBREAK_EVEN_WITH_SP_BETWEEN: + return FALSE; + + case LB_SP_foo + LB_BREAKABLE: + case LB_SP_foo + LB_NOBREAK: + case LB_SP_foo + LB_NOBREAK_EVEN_WITH_SP_BETWEEN: + + /* When we have something following a SP, we have to look at the + * context in order to know what to do. + * + * SP SP should not reach here because LB7: Do not break before + * spaces. (For two spaces in a row there is nothing that + * overrides that) */ + assert(after != LB_Space); + + /* Here we have a space followed by a non-space. Mostly this is a + * case of LB18: "Break after spaces". But there are complications + * as the handling of spaces is somewhat tricky. They are in a + * number of rules, which have to be applied in priority order, but + * something earlier in the string can cause a rule to be skipped + * and a lower priority rule invoked. A prime example is LB7 which + * says don't break before a space. But rule LB8 (lower priority) + * says that the first break opportunity after a ZW is after any + * span of spaces immediately after it. If a ZW comes before a SP + * in the input, rule LB8 applies, and not LB7. Other such rules + * involve combining marks which are rules 9 and 10, but they may + * override higher priority rules if they come earlier in the + * string. Since we're doing random access into the middle of the + * string, we have to look for rules that should get applied based + * on both string position and priority. Combining marks do not + * attach to either ZW nor SP, so we don't have to consider them + * until later. + * + * To check for LB8, we have to find the first non-space character + * before this span of spaces */ + do { + prev = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (prev == LB_Space); + + /* LB8 Break before any character following a zero-width space, + * even if one or more spaces intervene. + * ZW SP* ÷ + * So if we have a ZW just before this span, and to get here this + * is the final space in the span. */ + if (prev == LB_ZWSpace) { + return TRUE; + } + + /* Here, not ZW SP+. There are several rules that have higher + * priority than LB18 and can be resolved now, as they don't depend + * on anything earlier in the string (except ZW, which we have + * already handled). One of these rules is LB11 Do not break + * before Word joiner, but we have specially encoded that in the + * lookup table so it is caught by the single test below which + * catches the other ones. */ + if (LB_table[LB_Space][after] - LB_SP_foo + == LB_NOBREAK_EVEN_WITH_SP_BETWEEN) + { + return FALSE; + } + + /* If we get here, we have to XXX consider combining marks. */ + if (prev == LB_Combining_Mark) { + + /* What happens with these depends on the character they + * follow. */ + do { + prev = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (prev == LB_Combining_Mark); + + /* Most times these attach to and inherit the characteristics + * of that character, but not always, and when not, they are to + * be treated as AL by rule LB10. */ + if (! LB_CM_ATTACHES_TO(prev)) { + prev = LB_Alphabetic; + } + } + + /* Here, we have the character preceding the span of spaces all set + * up. We follow LB18: "Break after spaces" unless the table shows + * that is overriden */ + return LB_table[prev][after] != LB_NOBREAK_EVEN_WITH_SP_BETWEEN; + + case LB_CM_foo: + + /* We don't know how to treat the CM except by looking at the first + * non-CM character preceding it */ + do { + prev = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (prev == LB_Combining_Mark); + + /* Here, 'prev' is that first earlier non-CM character. If the CM + * attatches to it, then it inherits the behavior of 'prev'. If it + * doesn't attach, it is to be treated as an AL */ + if (! LB_CM_ATTACHES_TO(prev)) { + prev = LB_Alphabetic; + } + + goto redo; + + case LB_HY_or_BA_then_foo + LB_BREAKABLE: + case LB_HY_or_BA_then_foo + LB_NOBREAK: + + /* LB21a Don't break after Hebrew + Hyphen. + * HL (HY | BA) × */ + + if (backup_one_LB(strbeg, &temp_pos, utf8_target) + == LB_Hebrew_Letter) + { + return FALSE; + } + + return LB_table[prev][after] - LB_HY_or_BA_then_foo == LB_BREAKABLE; + + case LB_PR_or_PO_then_OP_or_HY + LB_BREAKABLE: + case LB_PR_or_PO_then_OP_or_HY + LB_NOBREAK: + + /* LB25a (PR | PO) × ( OP | HY )? NU */ + if (advance_one_LB(&temp_pos, strend, utf8_target) == LB_Numeric) { + return FALSE; + } + + return LB_table[prev][after] - LB_PR_or_PO_then_OP_or_HY + == LB_BREAKABLE; + + case LB_SY_or_IS_then_various + LB_BREAKABLE: + case LB_SY_or_IS_then_various + LB_NOBREAK: + { + /* LB25d NU (SY | IS)* × (NU | SY | IS | CL | CP ) */ + + LB_enum temp = prev; + do { + temp = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (temp == LB_Break_Symbols || temp == LB_Infix_Numeric); + if (temp == LB_Numeric) { + return FALSE; + } + + return LB_table[prev][after] - LB_SY_or_IS_then_various + == LB_BREAKABLE; + } + + case LB_various_then_PO_or_PR + LB_BREAKABLE: + case LB_various_then_PO_or_PR + LB_NOBREAK: + { + /* LB25e NU (SY | IS)* (CL | CP)? × (PO | PR) */ + + LB_enum temp = prev; + if (temp == LB_Close_Punctuation || temp == LB_Close_Parenthesis) + { + temp = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (temp == LB_Break_Symbols || temp == LB_Infix_Numeric) { + temp = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + if (temp == LB_Numeric) { + return FALSE; + } + return LB_various_then_PO_or_PR; + } + + default: + break; + } + +#ifdef DEBUGGING + PerlIO_printf(Perl_error_log, "Unhandled LB pair: LB_table[%d, %d] = %d\n", + before, after, LB_table[before][after]); + assert(0); +#endif + return TRUE; +} + +STATIC LB_enum +S_advance_one_LB(pTHX_ U8 ** curpos, const U8 * const strend, const bool utf8_target) +{ + LB_enum lb; + + PERL_ARGS_ASSERT_ADVANCE_ONE_LB; + + if (*curpos >= strend) { + return LB_EDGE; + } + + if (utf8_target) { + *curpos += UTF8SKIP(*curpos); + if (*curpos >= strend) { + return LB_EDGE; + } + lb = getLB_VAL_UTF8(*curpos, strend); + } + else { + (*curpos)++; + if (*curpos >= strend) { + return LB_EDGE; + } + lb = getLB_VAL_CP(**curpos); + } + + return lb; +} + +STATIC LB_enum +S_backup_one_LB(pTHX_ const U8 * const strbeg, U8 ** curpos, const bool utf8_target) +{ + LB_enum lb; + + PERL_ARGS_ASSERT_BACKUP_ONE_LB; + + if (*curpos < strbeg) { + return LB_EDGE; + } + + if (utf8_target) { + U8 * prev_char_pos = reghopmaybe3(*curpos, -1, strbeg); + U8 * prev_prev_char_pos; + + if (! prev_char_pos) { + return LB_EDGE; + } + + if ((prev_prev_char_pos = reghopmaybe3((U8 *) prev_char_pos, -1, strbeg))) { + lb = getLB_VAL_UTF8(prev_prev_char_pos, prev_char_pos); + *curpos = prev_char_pos; + prev_char_pos = prev_prev_char_pos; + } + else { + *curpos = (U8 *) strbeg; + return LB_EDGE; + } + } + else { + if (*curpos - 2 < strbeg) { + *curpos = (U8 *) strbeg; + return LB_EDGE; + } + (*curpos)--; + lb = getLB_VAL_CP(*(*curpos - 1)); + } + + return lb; +} + +STATIC bool +S_isSB(pTHX_ SB_enum before, + SB_enum after, + const U8 * const strbeg, + const U8 * const curpos, + const U8 * const strend, + const bool utf8_target) +{ + /* returns a boolean indicating if there is a Sentence Boundary Break + * between the inputs. See http://www.unicode.org/reports/tr29/ */ + + U8 * lpos = (U8 *) curpos; + bool has_para_sep = FALSE; + bool has_sp = FALSE; + + PERL_ARGS_ASSERT_ISSB; + + /* Break at the start and end of text. + SB1. sot ÷ + SB2. ÷ eot + But unstated in Unicode is don't break if the text is empty */ + if (before == SB_EDGE || after == SB_EDGE) { + return before != after; + } + + /* SB 3: Do not break within CRLF. */ + if (before == SB_CR && after == SB_LF) { + return FALSE; + } + + /* Break after paragraph separators. CR and LF are considered + * so because Unicode views text as like word processing text where there + * are no newlines except between paragraphs, and the word processor takes + * care of wrapping without there being hard line-breaks in the text *./ + SB4. Sep | CR | LF ÷ */ + if (before == SB_Sep || before == SB_CR || before == SB_LF) { + return TRUE; + } + + /* Ignore Format and Extend characters, except after sot, Sep, CR, or LF. + * (See Section 6.2, Replacing Ignore Rules.) + SB5. X (Extend | Format)* → X */ + if (after == SB_Extend || after == SB_Format) { + + /* Implied is that the these characters attach to everything + * immediately prior to them except for those separator-type + * characters. And the rules earlier have already handled the case + * when one of those immediately precedes the extend char */ + return FALSE; + } + + if (before == SB_Extend || before == SB_Format) { + U8 * temp_pos = lpos; + const SB_enum backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + if ( backup != SB_EDGE + && backup != SB_Sep + && backup != SB_CR + && backup != SB_LF) + { + before = backup; + lpos = temp_pos; + } + + /* Here, both 'before' and 'backup' are these types; implied is that we + * don't break between them */ + if (backup == SB_Extend || backup == SB_Format) { + return FALSE; + } + } + + /* Do not break after ambiguous terminators like period, if they are + * immediately followed by a number or lowercase letter, if they are + * between uppercase letters, if the first following letter (optionally + * after certain punctuation) is lowercase, or if they are followed by + * "continuation" punctuation such as comma, colon, or semicolon. For + * example, a period may be an abbreviation or numeric period, and thus may + * not mark the end of a sentence. + + * SB6. ATerm × Numeric */ + if (before == SB_ATerm && after == SB_Numeric) { + return FALSE; + } + + /* SB7. (Upper | Lower) ATerm × Upper */ + if (before == SB_ATerm && after == SB_Upper) { + U8 * temp_pos = lpos; + SB_enum backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + if (backup == SB_Upper || backup == SB_Lower) { + return FALSE; + } + } + + /* The remaining rules that aren't the final one, all require an STerm or + * an ATerm after having backed up over some Close* Sp*, and in one case an + * optional Paragraph separator, although one rule doesn't have any Sp's in it. + * So do that backup now, setting flags if either Sp or a paragraph + * separator are found */ + + if (before == SB_Sep || before == SB_CR || before == SB_LF) { + has_para_sep = TRUE; + before = backup_one_SB(strbeg, &lpos, utf8_target); + } + + if (before == SB_Sp) { + has_sp = TRUE; + do { + before = backup_one_SB(strbeg, &lpos, utf8_target); + } + while (before == SB_Sp); + } + + while (before == SB_Close) { + before = backup_one_SB(strbeg, &lpos, utf8_target); + } + + /* The next few rules apply only when the backed-up-to is an ATerm, and in + * most cases an STerm */ + if (before == SB_STerm || before == SB_ATerm) { + + /* So, here the lhs matches + * (STerm | ATerm) Close* Sp* (Sep | CR | LF)? + * and we have set flags if we found an Sp, or the optional Sep,CR,LF. + * The rules that apply here are: + * + * SB8 ATerm Close* Sp* × ( ¬(OLetter | Upper | Lower | Sep | CR + | LF | STerm | ATerm) )* Lower + SB8a (STerm | ATerm) Close* Sp* × (SContinue | STerm | ATerm) + SB9 (STerm | ATerm) Close* × (Close | Sp | Sep | CR | LF) + SB10 (STerm | ATerm) Close* Sp* × (Sp | Sep | CR | LF) + SB11 (STerm | ATerm) Close* Sp* (Sep | CR | LF)? ÷ + */ + + /* And all but SB11 forbid having seen a paragraph separator */ + if (! has_para_sep) { + if (before == SB_ATerm) { /* SB8 */ + U8 * rpos = (U8 *) curpos; + SB_enum later = after; + + while ( later != SB_OLetter + && later != SB_Upper + && later != SB_Lower + && later != SB_Sep + && later != SB_CR + && later != SB_LF + && later != SB_STerm + && later != SB_ATerm + && later != SB_EDGE) + { + later = advance_one_SB(&rpos, strend, utf8_target); + } + if (later == SB_Lower) { + return FALSE; + } + } + + if ( after == SB_SContinue /* SB8a */ + || after == SB_STerm + || after == SB_ATerm) + { + return FALSE; + } + + if (! has_sp) { /* SB9 applies only if there was no Sp* */ + if ( after == SB_Close + || after == SB_Sp + || after == SB_Sep + || after == SB_CR + || after == SB_LF) + { + return FALSE; + } + } + + /* SB10. This and SB9 could probably be combined some way, but khw + * has decided to follow the Unicode rule book precisely for + * simplified maintenance */ + if ( after == SB_Sp + || after == SB_Sep + || after == SB_CR + || after == SB_LF) + { + return FALSE; + } + } + + /* SB11. */ + return TRUE; + } + + /* Otherwise, do not break. + SB12. Any × Any */ + + return FALSE; +} + +STATIC SB_enum +S_advance_one_SB(pTHX_ U8 ** curpos, const U8 * const strend, const bool utf8_target) +{ + SB_enum sb; + + PERL_ARGS_ASSERT_ADVANCE_ONE_SB; + + if (*curpos >= strend) { + return SB_EDGE; + } + + if (utf8_target) { + do { + *curpos += UTF8SKIP(*curpos); + if (*curpos >= strend) { + return SB_EDGE; + } + sb = getSB_VAL_UTF8(*curpos, strend); + } while (sb == SB_Extend || sb == SB_Format); + } + else { + do { + (*curpos)++; + if (*curpos >= strend) { + return SB_EDGE; + } + sb = getSB_VAL_CP(**curpos); + } while (sb == SB_Extend || sb == SB_Format); + } + + return sb; +} + +STATIC SB_enum +S_backup_one_SB(pTHX_ const U8 * const strbeg, U8 ** curpos, const bool utf8_target) +{ + SB_enum sb; + + PERL_ARGS_ASSERT_BACKUP_ONE_SB; + + if (*curpos < strbeg) { + return SB_EDGE; + } + + if (utf8_target) { + U8 * prev_char_pos = reghopmaybe3(*curpos, -1, strbeg); + if (! prev_char_pos) { + return SB_EDGE; + } + + /* Back up over Extend and Format. curpos is always just to the right + * of the characater whose value we are getting */ + do { + U8 * prev_prev_char_pos; + if ((prev_prev_char_pos = reghopmaybe3((U8 *) prev_char_pos, -1, + strbeg))) + { + sb = getSB_VAL_UTF8(prev_prev_char_pos, prev_char_pos); + *curpos = prev_char_pos; + prev_char_pos = prev_prev_char_pos; + } + else { + *curpos = (U8 *) strbeg; + return SB_EDGE; + } + } while (sb == SB_Extend || sb == SB_Format); + } + else { + do { + if (*curpos - 2 < strbeg) { + *curpos = (U8 *) strbeg; + return SB_EDGE; + } + (*curpos)--; + sb = getSB_VAL_CP(*(*curpos - 1)); + } while (sb == SB_Extend || sb == SB_Format); + } + + return sb; +} + +STATIC bool +S_isWB(pTHX_ WB_enum previous, + WB_enum before, + WB_enum after, + const U8 * const strbeg, + const U8 * const curpos, + const U8 * const strend, + const bool utf8_target) +{ + /* Return a boolean as to if the boundary between 'before' and 'after' is + * a Unicode word break, using their published algorithm, but tailored for + * Perl by treating spans of white space as one unit. Context may be + * needed to make this determination. If the value for the character + * before 'before' is known, it is passed as 'previous'; otherwise that + * should be set to WB_UNKNOWN. The other input parameters give the + * boundaries and current position in the matching of the string. That + * is, 'curpos' marks the position where the character whose wb value is + * 'after' begins. See http://www.unicode.org/reports/tr29/ */ + + U8 * before_pos = (U8 *) curpos; + U8 * after_pos = (U8 *) curpos; + WB_enum prev = before; + WB_enum next; + + PERL_ARGS_ASSERT_ISWB; + + /* Rule numbers in the comments below are as of Unicode 8.0 */ + + redo: + before = prev; + switch (WB_table[before][after]) { + case WB_BREAKABLE: + return TRUE; + + case WB_NOBREAK: + return FALSE; + + case WB_hs_then_hs: /* 2 horizontal spaces in a row */ + next = advance_one_WB(&after_pos, strend, utf8_target, + FALSE /* Don't skip Extend nor Format */ ); + /* A space immediately preceeding an Extend or Format is attached + * to by them, and hence gets separated from previous spaces. + * Otherwise don't break between horizontal white space */ + return next == WB_Extend || next == WB_Format; + + /* WB4 Ignore Format and Extend characters, except when they appear at + * the beginning of a region of text. This code currently isn't + * general purpose, but it works as the rules are currently and likely + * to be laid out. The reason it works is that when 'they appear at + * the beginning of a region of text', the rule is to break before + * them, just like any other character. Therefore, the default rule + * applies and we don't have to look in more depth. Should this ever + * change, we would have to have 2 'case' statements, like in the + * rules below, and backup a single character (not spacing over the + * extend ones) and then see if that is one of the region-end + * characters and go from there */ + case WB_Ex_or_FO_then_foo: + prev = backup_one_WB(&previous, strbeg, &before_pos, utf8_target); + goto redo; + + case WB_DQ_then_HL + WB_BREAKABLE: + case WB_DQ_then_HL + WB_NOBREAK: + + /* WB7c Hebrew_Letter Double_Quote × Hebrew_Letter */ + + if (backup_one_WB(&previous, strbeg, &before_pos, utf8_target) + == WB_Hebrew_Letter) + { + return FALSE; + } + + return WB_table[before][after] - WB_DQ_then_HL == WB_BREAKABLE; + + case WB_HL_then_DQ + WB_BREAKABLE: + case WB_HL_then_DQ + WB_NOBREAK: + + /* WB7b Hebrew_Letter × Double_Quote Hebrew_Letter */ + + if (advance_one_WB(&after_pos, strend, utf8_target, + TRUE /* Do skip Extend and Format */ ) + == WB_Hebrew_Letter) + { + return FALSE; + } + + return WB_table[before][after] - WB_HL_then_DQ == WB_BREAKABLE; + + case WB_LE_or_HL_then_MB_or_ML_or_SQ + WB_NOBREAK: + case WB_LE_or_HL_then_MB_or_ML_or_SQ + WB_BREAKABLE: + + /* WB6 (ALetter | Hebrew_Letter) × (MidLetter | MidNumLet + * | Single_Quote) (ALetter | Hebrew_Letter) */ + + next = advance_one_WB(&after_pos, strend, utf8_target, + TRUE /* Do skip Extend and Format */ ); + + if (next == WB_ALetter || next == WB_Hebrew_Letter) + { + return FALSE; + } + + return WB_table[before][after] + - WB_LE_or_HL_then_MB_or_ML_or_SQ == WB_BREAKABLE; + + case WB_MB_or_ML_or_SQ_then_LE_or_HL + WB_NOBREAK: + case WB_MB_or_ML_or_SQ_then_LE_or_HL + WB_BREAKABLE: + + /* WB7 (ALetter | Hebrew_Letter) (MidLetter | MidNumLet + * | Single_Quote) × (ALetter | Hebrew_Letter) */ + + prev = backup_one_WB(&previous, strbeg, &before_pos, utf8_target); + if (prev == WB_ALetter || prev == WB_Hebrew_Letter) + { + return FALSE; + } + + return WB_table[before][after] + - WB_MB_or_ML_or_SQ_then_LE_or_HL == WB_BREAKABLE; + + case WB_MB_or_MN_or_SQ_then_NU + WB_NOBREAK: + case WB_MB_or_MN_or_SQ_then_NU + WB_BREAKABLE: + + /* WB11 Numeric (MidNum | (MidNumLet | Single_Quote)) × Numeric + * */ + + if (backup_one_WB(&previous, strbeg, &before_pos, utf8_target) + == WB_Numeric) + { + return FALSE; + } + + return WB_table[before][after] + - WB_MB_or_MN_or_SQ_then_NU == WB_BREAKABLE; + + case WB_NU_then_MB_or_MN_or_SQ + WB_NOBREAK: + case WB_NU_then_MB_or_MN_or_SQ + WB_BREAKABLE: + + /* WB12 Numeric × (MidNum | MidNumLet | Single_Quote) Numeric */ + + if (advance_one_WB(&after_pos, strend, utf8_target, + TRUE /* Do skip Extend and Format */ ) + == WB_Numeric) + { + return FALSE; + } + + return WB_table[before][after] + - WB_NU_then_MB_or_MN_or_SQ == WB_BREAKABLE; + + default: + break; + } + +#ifdef DEBUGGING + PerlIO_printf(Perl_error_log, "Unhandled WB pair: WB_table[%d, %d] = %d\n", + before, after, WB_table[before][after]); + assert(0); +#endif + return TRUE; +} + +STATIC WB_enum +S_advance_one_WB(pTHX_ U8 ** curpos, + const U8 * const strend, + const bool utf8_target, + const bool skip_Extend_Format) +{ + WB_enum wb; + + PERL_ARGS_ASSERT_ADVANCE_ONE_WB; + + if (*curpos >= strend) { + return WB_EDGE; + } + + if (utf8_target) { + + /* Advance over Extend and Format */ + do { + *curpos += UTF8SKIP(*curpos); + if (*curpos >= strend) { + return WB_EDGE; + } + wb = getWB_VAL_UTF8(*curpos, strend); + } while ( skip_Extend_Format + && (wb == WB_Extend || wb == WB_Format)); + } + else { + do { + (*curpos)++; + if (*curpos >= strend) { + return WB_EDGE; + } + wb = getWB_VAL_CP(**curpos); + } while ( skip_Extend_Format + && (wb == WB_Extend || wb == WB_Format)); + } + + return wb; +} + +STATIC WB_enum +S_backup_one_WB(pTHX_ WB_enum * previous, const U8 * const strbeg, U8 ** curpos, const bool utf8_target) +{ + WB_enum wb; + + PERL_ARGS_ASSERT_BACKUP_ONE_WB; + + /* If we know what the previous character's break value is, don't have + * to look it up */ + if (*previous != WB_UNKNOWN) { + wb = *previous; + + /* But we need to move backwards by one */ + if (utf8_target) { + *curpos = reghopmaybe3(*curpos, -1, strbeg); + if (! *curpos) { + *previous = WB_EDGE; + *curpos = (U8 *) strbeg; + } + else { + *previous = WB_UNKNOWN; } } - } + else { + (*curpos)--; + *previous = (*curpos <= strbeg) ? WB_EDGE : WB_UNKNOWN; + } - /* Here have figured things out. Set up the returns */ - if (use_chrtest_void) { - *c2p = *c1p = CHRTEST_VOID; - } - else if (utf8_target) { - if (! utf8_has_been_setup) { /* Don't have the utf8; must get it */ - uvchr_to_utf8(c1_utf8, c1); - uvchr_to_utf8(c2_utf8, c2); + /* And we always back up over these two types */ + if (wb != WB_Extend && wb != WB_Format) { + return wb; } + } - /* Invariants are stored in both the utf8 and byte outputs; Use - * negative numbers otherwise for the byte ones. Make sure that the - * byte ones are the same iff the utf8 ones are the same */ - *c1p = (UTF8_IS_INVARIANT(*c1_utf8)) ? *c1_utf8 : CHRTEST_NOT_A_CP_1; - *c2p = (UTF8_IS_INVARIANT(*c2_utf8)) - ? *c2_utf8 - : (c1 == c2) - ? CHRTEST_NOT_A_CP_1 - : CHRTEST_NOT_A_CP_2; + if (*curpos < strbeg) { + return WB_EDGE; } - else if (c1 > 255) { - if (c2 > 255) { /* both possibilities are above what a non-utf8 string - can represent */ - return FALSE; - } - *c1p = *c2p = c2; /* c2 is the only representable value */ + if (utf8_target) { + U8 * prev_char_pos = reghopmaybe3(*curpos, -1, strbeg); + if (! prev_char_pos) { + return WB_EDGE; + } + + /* Back up over Extend and Format. curpos is always just to the right + * of the characater whose value we are getting */ + do { + U8 * prev_prev_char_pos; + if ((prev_prev_char_pos = reghopmaybe3((U8 *) prev_char_pos, + -1, + strbeg))) + { + wb = getWB_VAL_UTF8(prev_prev_char_pos, prev_char_pos); + *curpos = prev_char_pos; + prev_char_pos = prev_prev_char_pos; + } + else { + *curpos = (U8 *) strbeg; + return WB_EDGE; + } + } while (wb == WB_Extend || wb == WB_Format); } - else { /* c1 is representable; see about c2 */ - *c1p = c1; - *c2p = (c2 < 256) ? c2 : c1; + else { + do { + if (*curpos - 2 < strbeg) { + *curpos = (U8 *) strbeg; + return WB_EDGE; + } + (*curpos)--; + wb = getWB_VAL_CP(*(*curpos - 1)); + } while (wb == WB_Extend || wb == WB_Format); } - return TRUE; + return wb; } /* returns -1 on failure, $+[0] on success */ STATIC SSize_t S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) { + #if PERL_VERSION < 9 && !defined(PERL_CORE) dMY_CXT; #endif @@ -3954,7 +5172,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) */ PAD* last_pad = NULL; dMULTICALL; - I32 gimme = G_SCALAR; + U8 gimme = G_SCALAR; CV *caller_cv = NULL; /* who called us */ CV *last_pushed_cv = NULL; /* most recently called (?{}) CV */ CHECKPOINT runops_cp; /* savestack position before executing EVAL */ @@ -3962,6 +5180,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) int to_complement; /* Invert the result? */ _char_class_number classnum; bool is_utf8_pat = reginfo->is_utf8_pat; + bool match = FALSE; + #ifdef DEBUGGING GET_RE_DEBUG_FLAGS_DECL; @@ -3972,11 +5192,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* shut up 'may be used uninitialized' compiler warnings for dMULTICALL */ multicall_oldcatch = 0; - multicall_cv = NULL; - cx = NULL; PERL_UNUSED_VAR(multicall_cop); - PERL_UNUSED_VAR(newsp); - PERL_ARGS_ASSERT_REGMATCH; @@ -4041,14 +5257,14 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) rex->offs[0].start = locinput - reginfo->strbeg; PUSH_STATE_GOTO(KEEPS_next, next, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case KEEPS_next_fail: /* rollback the start point change */ rex->offs[0].start = st->u.keeper.val; sayNO_SILENT; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case MEOL: /* /..$/m */ if (!NEXTCHR_IS_EOS && nextchr != '\n') @@ -4072,12 +5288,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) sayNO; goto increment_locinput; - case CANY: /* \C */ - if (NEXTCHR_IS_EOS) - sayNO; - locinput++; - break; - case REG_ANY: /* /./ */ if ((NEXTCHR_IS_EOS) || nextchr == '\n') sayNO; @@ -4098,7 +5308,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) ); sayNO_SILENT; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } /* FALLTHROUGH */ case TRIE: /* (ab|cd) */ @@ -4156,6 +5366,19 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) HV * widecharmap = MUTABLE_HV(rexi->data->data[ ARG( scan ) + 1 ]); U32 state = trie->startstate; + if (scan->flags == EXACTL || scan->flags == EXACTFLU8) { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (utf8_target + && UTF8_IS_ABOVE_LATIN1(nextchr) + && scan->flags == EXACTL) + { + /* We only output for EXACTL, as we let the folder + * output this message for EXACTFLU8 to avoid + * duplication */ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(locinput, + reginfo->strend); + } + } if ( trie->bitmap && (NEXTCHR_IS_EOS || !TRIE_BITMAP_TEST(trie, nextchr))) { @@ -4286,7 +5509,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) goto trie_first_try; /* jump into the fail handler */ }} /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case TRIE_next_fail: /* we failed - try next alternative */ { @@ -4401,14 +5624,14 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) if (ST.accepted > 1 || has_cutgroup) { PUSH_STATE_GOTO(TRIE_next, scan, (char*)uc); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } /* only one choice left - just continue */ DEBUG_EXECUTE_r({ AV *const trie_words = MUTABLE_AV(rexi->data->data[ARG(ST.me)+TRIE_WORDS_OFFSET]); - SV ** const tmp = av_fetch( trie_words, - ST.nextword-1, 0 ); + SV ** const tmp = trie_words + ? av_fetch(trie_words, ST.nextword - 1, 0) : NULL; SV *sv= tmp ? sv_newmortal() : NULL; PerlIO_printf( Perl_debug_log, @@ -4429,6 +5652,19 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } #undef ST + case EXACTL: /* /abc/l */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + /* Complete checking would involve going through every character + * matched by the string to see if any is above latin1. But the + * comparision otherwise might very well be a fast assembly + * language routine, and I (khw) don't think slowing things down + * just to check for this warning is worth it. So this just checks + * the first character */ + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*locinput)) { + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(locinput, reginfo->strend); + } + /* FALLTHROUGH */ case EXACT: { /* /abc/ */ char *s = STRING(scan); ln = STR_LEN(scan); @@ -4460,7 +5696,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) l++; } else { - if (TWO_BYTE_UTF8_TO_NATIVE(*l, *(l+1)) != * (U8*) s) + if (EIGHT_BIT_UTF8_TO_NATIVE(*l, *(l+1)) != * (U8*) s) { sayNO; } @@ -4484,7 +5720,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) s++; } else { - if (TWO_BYTE_UTF8_TO_NATIVE(*s, *(s+1)) != * (U8*) l) + if (EIGHT_BIT_UTF8_TO_NATIVE(*s, *(s+1)) != * (U8*) l) { sayNO; } @@ -4515,11 +5751,24 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) const char * s; U32 fold_utf8_flags; + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; folder = foldEQ_locale; fold_array = PL_fold_locale; fold_utf8_flags = FOLDEQ_LOCALE; goto do_exactf; + case EXACTFLU8: /* /abc/il; but all 'abc' are above 255, so + is effectively /u; hence to match, target + must be UTF-8. */ + if (! utf8_target) { + sayNO; + } + fold_utf8_flags = FOLDEQ_LOCALE | FOLDEQ_S1_ALREADY_FOLDED + | FOLDEQ_S1_FOLDS_SANE; + folder = foldEQ_latin1; + fold_array = PL_fold_latin1; + goto do_exactf; + case EXACTFU_SS: /* /\x{df}/iu */ case EXACTFU: /* /abc/iu */ folder = foldEQ_latin1; @@ -4582,93 +5831,287 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) break; } - /* XXX Could improve efficiency by separating these all out using a - * macro or in-line function. At that point regcomp.c would no longer - * have to set the FLAGS fields of these */ - case BOUNDL: /* /\b/l */ case NBOUNDL: /* /\B/l */ + to_complement = 1; + /* FALLTHROUGH */ + + case BOUNDL: /* /\b/l */ + { + bool b1, b2; + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (FLAGS(scan) != TRADITIONAL_BOUND) { + if (! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + B_ON_NON_UTF8_LOCALE_IS_WRONG); + } + goto boundu; + } + + if (utf8_target) { + if (locinput == reginfo->strbeg) + b1 = isWORDCHAR_LC('\n'); + else { + b1 = isWORDCHAR_LC_utf8(reghop3((U8*)locinput, -1, + (U8*)(reginfo->strbeg))); + } + b2 = (NEXTCHR_IS_EOS) + ? isWORDCHAR_LC('\n') + : isWORDCHAR_LC_utf8((U8*)locinput); + } + else { /* Here the string isn't utf8 */ + b1 = (locinput == reginfo->strbeg) + ? isWORDCHAR_LC('\n') + : isWORDCHAR_LC(UCHARAT(locinput - 1)); + b2 = (NEXTCHR_IS_EOS) + ? isWORDCHAR_LC('\n') + : isWORDCHAR_LC(nextchr); + } + if (to_complement ^ (b1 == b2)) { + sayNO; + } + break; + } + + case NBOUND: /* /\B/ */ + to_complement = 1; + /* FALLTHROUGH */ + case BOUND: /* /\b/ */ - case BOUNDU: /* /\b/u */ + if (utf8_target) { + goto bound_utf8; + } + goto bound_ascii_match_only; + + case NBOUNDA: /* /\B/a */ + to_complement = 1; + /* FALLTHROUGH */ + case BOUNDA: /* /\b/a */ - case NBOUND: /* /\B/ */ + { + bool b1, b2; + + bound_ascii_match_only: + /* Here the string isn't utf8, or is utf8 and only ascii characters + * are to match \w. In the latter case looking at the byte just + * prior to the current one may be just the final byte of a + * multi-byte character. This is ok. There are two cases: + * 1) it is a single byte character, and then the test is doing + * just what it's supposed to. + * 2) it is a multi-byte character, in which case the final byte is + * never mistakable for ASCII, and so the test will say it is + * not a word character, which is the correct answer. */ + b1 = (locinput == reginfo->strbeg) + ? isWORDCHAR_A('\n') + : isWORDCHAR_A(UCHARAT(locinput - 1)); + b2 = (NEXTCHR_IS_EOS) + ? isWORDCHAR_A('\n') + : isWORDCHAR_A(nextchr); + if (to_complement ^ (b1 == b2)) { + sayNO; + } + break; + } + case NBOUNDU: /* /\B/u */ - case NBOUNDA: /* /\B/a */ - /* was last char in word? */ - if (utf8_target - && FLAGS(scan) != REGEX_ASCII_RESTRICTED_CHARSET - && FLAGS(scan) != REGEX_ASCII_MORE_RESTRICTED_CHARSET) - { - if (locinput == reginfo->strbeg) - ln = '\n'; - else { - const U8 * const r = - reghop3((U8*)locinput, -1, (U8*)(reginfo->strbeg)); + to_complement = 1; + /* FALLTHROUGH */ - ln = utf8n_to_uvchr(r, (U8*) reginfo->strend - r, - 0, uniflags); - } - if (FLAGS(scan) != REGEX_LOCALE_CHARSET) { - ln = isWORDCHAR_uni(ln); - if (NEXTCHR_IS_EOS) - n = 0; - else { - LOAD_UTF8_CHARCLASS_ALNUM(); - n = swash_fetch(PL_utf8_swash_ptrs[_CC_WORDCHAR], (U8*)locinput, - utf8_target); + case BOUNDU: /* /\b/u */ + + boundu: + if (UNLIKELY(reginfo->strbeg >= reginfo->strend)) { + match = FALSE; + } + else if (utf8_target) { + bound_utf8: + switch((bound_type) FLAGS(scan)) { + case TRADITIONAL_BOUND: + { + bool b1, b2; + b1 = (locinput == reginfo->strbeg) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_utf8(reghop3((U8*)locinput, -1, + (U8*)(reginfo->strbeg))); + b2 = (NEXTCHR_IS_EOS) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_utf8((U8*)locinput); + match = cBOOL(b1 != b2); + break; } - } - else { - ln = isWORDCHAR_LC_uvchr(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_LC_utf8((U8*)locinput); - } + case GCB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; /* GCB always matches at begin and + end */ + } + else { + /* Find the gcb values of previous and current + * chars, then see if is a break point */ + match = isGCB(getGCB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getGCB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend)); + } + break; + + case LB_BOUND: + if (locinput == reginfo->strbeg) { + match = FALSE; + } + else if (NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isLB(getLB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getLB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case SB_BOUND: /* Always matches at begin and end */ + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isSB(getSB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getSB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case WB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isWB(WB_UNKNOWN, + getWB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getWB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + } } - else { + else { /* Not utf8 target */ + switch((bound_type) FLAGS(scan)) { + case TRADITIONAL_BOUND: + { + bool b1, b2; + b1 = (locinput == reginfo->strbeg) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_L1(UCHARAT(locinput - 1)); + b2 = (NEXTCHR_IS_EOS) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_L1(nextchr); + match = cBOOL(b1 != b2); + break; + } - /* Here the string isn't utf8, or is utf8 and only ascii - * characters are to match \w. In the latter case looking at - * the byte just prior to the current one may be just the final - * byte of a multi-byte character. This is ok. There are two - * cases: - * 1) it is a single byte character, and then the test is doing - * just what it's supposed to. - * 2) it is a multi-byte character, in which case the final - * byte is never mistakable for ASCII, and so the test - * will say it is not a word character, which is the - * correct answer. */ - ln = (locinput != reginfo->strbeg) ? - UCHARAT(locinput - 1) : '\n'; - switch (FLAGS(scan)) { - case REGEX_UNICODE_CHARSET: - ln = isWORDCHAR_L1(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_L1(nextchr); - break; - case REGEX_LOCALE_CHARSET: - ln = isWORDCHAR_LC(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_LC(nextchr); - break; - case REGEX_DEPENDS_CHARSET: - ln = isWORDCHAR(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR(nextchr); - break; - case REGEX_ASCII_RESTRICTED_CHARSET: - case REGEX_ASCII_MORE_RESTRICTED_CHARSET: - ln = isWORDCHAR_A(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_A(nextchr); - break; - default: - Perl_croak(aTHX_ "panic: Unexpected FLAGS %u in op %u", FLAGS(scan), OP(scan)); - } + case GCB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; /* GCB always matches at begin and + end */ + } + else { /* Only CR-LF combo isn't a GCB in 0-255 + range */ + match = UCHARAT(locinput - 1) != '\r' + || UCHARAT(locinput) != '\n'; + } + break; + + case LB_BOUND: + if (locinput == reginfo->strbeg) { + match = FALSE; + } + else if (NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isLB(getLB_VAL_CP(UCHARAT(locinput -1)), + getLB_VAL_CP(UCHARAT(locinput)), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case SB_BOUND: /* Always matches at begin and end */ + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isSB(getSB_VAL_CP(UCHARAT(locinput -1)), + getSB_VAL_CP(UCHARAT(locinput)), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case WB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isWB(WB_UNKNOWN, + getWB_VAL_CP(UCHARAT(locinput -1)), + getWB_VAL_CP(UCHARAT(locinput)), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + } } - /* Note requires that all BOUNDs be lower than all NBOUNDs in - * regcomp.sym */ - if (((!ln) == (!n)) == (OP(scan) < NBOUND)) - sayNO; + + if (to_complement ^ ! match) { + sayNO; + } break; - case ANYOF: /* /[abc]/ */ + case ANYOFL: /* /[abc]/l */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (ANYOFL_UTF8_LOCALE_REQD(FLAGS(scan)) && ! IN_UTF8_CTYPE_LOCALE) + { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), utf8_locale_required); + } + /* FALLTHROUGH */ + case ANYOFD: /* /[abc]/d */ + case ANYOF: /* /[abc]/ */ if (NEXTCHR_IS_EOS) sayNO; - if (utf8_target) { + if (utf8_target && ! UTF8_IS_INVARIANT(*locinput)) { if (!reginclass(rex, scan, (U8*)locinput, (U8*)reginfo->strend, utf8_target)) sayNO; @@ -4689,6 +6132,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* FALLTHROUGH */ case POSIXL: /* \w or [:punct:] etc. under /l */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; if (NEXTCHR_IS_EOS) sayNO; @@ -4702,14 +6146,15 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else if (UTF8_IS_DOWNGRADEABLE_START(nextchr)) { if (! (to_complement ^ cBOOL(isFOO_lc(FLAGS(scan), - (U8) TWO_BYTE_UTF8_TO_NATIVE(nextchr, - *(locinput + 1)))))) + EIGHT_BIT_UTF8_TO_NATIVE(nextchr, + *(locinput + 1)))))) { sayNO; } } else { /* Here, must be an above Latin-1 code point */ - goto utf8_posix_not_eos; + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(locinput, reginfo->strend); + goto utf8_posix_above_latin1; } /* Here, must be utf8 */ @@ -4768,7 +6213,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) if (NEXTCHR_IS_EOS) { sayNO; } - utf8_posix_not_eos: /* Use _generic_isCC() for characters within Latin1. (Note that * UTF8_IS_INVARIANT works even on non-UTF-8 strings, or else @@ -4783,7 +6227,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else if (UTF8_IS_DOWNGRADEABLE_START(nextchr)) { if (! (to_complement - ^ cBOOL(_generic_isCC(TWO_BYTE_UTF8_TO_NATIVE(nextchr, + ^ cBOOL(_generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(nextchr, *(locinput + 1)), FLAGS(scan))))) { @@ -4792,6 +6236,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) locinput += 2; } else { /* Handle above Latin-1 code points */ + utf8_posix_above_latin1: classnum = (_char_class_number) FLAGS(scan); if (classnum < _FIRST_NON_SWASH_CC) { @@ -4814,10 +6259,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else { /* Here, uses macros to find above Latin-1 code points */ switch (classnum) { - case _CC_ENUM_SPACE: /* XXX would require separate - code if we revert the change - of \v matching this */ - case _CC_ENUM_PSXSPC: + case _CC_ENUM_SPACE: if (! (to_complement ^ cBOOL(is_XPERLSPACE_high(locinput)))) { @@ -4859,38 +6301,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) case CLUMP: /* Match \X: logical Unicode character. This is defined as a Unicode extended Grapheme Cluster */ - /* From http://www.unicode.org/reports/tr29 (5.2 version). An - extended Grapheme Cluster is: - - CR LF - | Prepend* Begin Extend* - | . - - Begin is: ( Special_Begin | ! Control ) - Special_Begin is: ( Regional-Indicator+ | Hangul-syllable ) - Extend is: ( Grapheme_Extend | Spacing_Mark ) - Control is: [ GCB_Control | CR | LF ] - Hangul-syllable is: ( T+ | ( L* ( L | ( LVT | ( V | LV ) V* ) T* ) )) - - If we create a 'Regular_Begin' = Begin - Special_Begin, then - we can rewrite - - Begin is ( Regular_Begin + Special Begin ) - - It turns out that 98.4% of all Unicode code points match - Regular_Begin. Doing it this way eliminates a table match in - the previous implementation for almost all Unicode code points. - - There is a subtlety with Prepend* which showed up in testing. - Note that the Begin, and only the Begin is required in: - | Prepend* Begin Extend* - Also, Begin contains '! Control'. A Prepend must be a - '! Control', which means it must also be a Begin. What it - comes down to is that if we match Prepend* and then find no - suitable Begin afterwards, that if we backtrack the last - Prepend, that one will be a suitable Begin. - */ - if (NEXTCHR_IS_EOS) sayNO; if (! utf8_target) { @@ -4908,147 +6318,27 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else { - /* Utf8: See if is ( CR LF ); already know that locinput < - * reginfo->strend, so locinput+1 is in bounds */ - if ( nextchr == '\r' && locinput+1 < reginfo->strend - && UCHARAT(locinput + 1) == '\n') - { - locinput += 2; - } - else { - STRLEN len; - - /* In case have to backtrack to beginning, then match '.' */ - char *starting = locinput; - - /* In case have to backtrack the last prepend */ - char *previous_prepend = NULL; - - LOAD_UTF8_CHARCLASS_GCB(); - - /* Match (prepend)* */ - while (locinput < reginfo->strend - && (len = is_GCB_Prepend_utf8(locinput))) - { - previous_prepend = locinput; - locinput += len; - } - - /* As noted above, if we matched a prepend character, but - * the next thing won't match, back off the last prepend we - * matched, as it is guaranteed to match the begin */ - if (previous_prepend - && (locinput >= reginfo->strend - || (! swash_fetch(PL_utf8_X_regular_begin, - (U8*)locinput, utf8_target) - && ! is_GCB_SPECIAL_BEGIN_START_utf8(locinput))) - ) - { - locinput = previous_prepend; - } + /* Get the gcb type for the current character */ + GCB_enum prev_gcb = getGCB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend); - /* Note that here we know reginfo->strend > locinput, as we - * tested that upon input to this switch case, and if we - * moved locinput forward, we tested the result just above - * and it either passed, or we backed off so that it will - * now pass */ - if (swash_fetch(PL_utf8_X_regular_begin, - (U8*)locinput, utf8_target)) { - locinput += UTF8SKIP(locinput); + /* Then scan through the input until we get to the first + * character whose type is supposed to be a gcb with the + * current character. (There is always a break at the + * end-of-input) */ + locinput += UTF8SKIP(locinput); + while (locinput < reginfo->strend) { + GCB_enum cur_gcb = getGCB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend); + if (isGCB(prev_gcb, cur_gcb)) { + break; } - else if (! is_GCB_SPECIAL_BEGIN_START_utf8(locinput)) { - - /* Here did not match the required 'Begin' in the - * second term. So just match the very first - * character, the '.' of the final term of the regex */ - locinput = starting + UTF8SKIP(starting); - goto exit_utf8; - } else { - - /* Here is a special begin. It can be composed of - * several individual characters. One possibility is - * RI+ */ - if ((len = is_GCB_RI_utf8(locinput))) { - locinput += len; - while (locinput < reginfo->strend - && (len = is_GCB_RI_utf8(locinput))) - { - locinput += len; - } - } else if ((len = is_GCB_T_utf8(locinput))) { - /* Another possibility is T+ */ - locinput += len; - while (locinput < reginfo->strend - && (len = is_GCB_T_utf8(locinput))) - { - locinput += len; - } - } else { - - /* Here, neither RI+ nor T+; must be some other - * Hangul. That means it is one of the others: L, - * LV, LVT or V, and matches: - * L* (L | LVT T* | V * V* T* | LV V* T*) */ - - /* Match L* */ - while (locinput < reginfo->strend - && (len = is_GCB_L_utf8(locinput))) - { - locinput += len; - } - - /* Here, have exhausted L*. If the next character - * is not an LV, LVT nor V, it means we had to have - * at least one L, so matches L+ in the original - * equation, we have a complete hangul syllable. - * Are done. */ - if (locinput < reginfo->strend - && is_GCB_LV_LVT_V_utf8(locinput)) - { - /* Otherwise keep going. Must be LV, LVT or V. - * See if LVT, by first ruling out V, then LV */ - if (! is_GCB_V_utf8(locinput) - /* All but every TCount one is LV */ - && (valid_utf8_to_uvchr((U8 *) locinput, - NULL) - - SBASE) - % TCount != 0) - { - locinput += UTF8SKIP(locinput); - } else { - - /* Must be V or LV. Take it, then match - * V* */ - locinput += UTF8SKIP(locinput); - while (locinput < reginfo->strend - && (len = is_GCB_V_utf8(locinput))) - { - locinput += len; - } - } + prev_gcb = cur_gcb; + locinput += UTF8SKIP(locinput); + } - /* And any of LV, LVT, or V can be followed - * by T* */ - while (locinput < reginfo->strend - && (len = is_GCB_T_utf8(locinput))) - { - locinput += len; - } - } - } - } - /* Match any extender */ - while (locinput < reginfo->strend - && swash_fetch(PL_utf8_X_extend, - (U8*)locinput, utf8_target)) - { - locinput += UTF8SKIP(locinput); - } - } - exit_utf8: - if (locinput > reginfo->strend) sayNO; } break; @@ -5065,6 +6355,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) const U8 *fold_array; UV utf8_fold_flags; + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; folder = foldEQ_locale; fold_array = PL_fold_locale; type = REFFL; @@ -5109,6 +6400,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) goto do_nref_ref_common; case REFFL: /* /\1/il */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; folder = foldEQ_locale; fold_array = PL_fold_locale; utf8_fold_flags = FOLDEQ_LOCALE; @@ -5289,6 +6581,13 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) U8 flags = (CXp_SUB_RE | ((newcv == caller_cv) ? CXp_SUB_RE_FAKE : 0)); if (last_pushed_cv) { + /* PUSH/POP_MULTICALL save and restore the + * caller's PL_comppad; if we call multiple subs + * using the same CX block, we have to save and + * unwind the varying PL_comppad's ourselves, + * especially restoring the right PL_comppad on + * backtrack - so save it on the save stack */ + SAVECOMPPAD(); CHANGE_MULTICALL_FLAGS(newcv, flags); } else { @@ -5300,7 +6599,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* these assignments are just to silence compiler * warnings */ multicall_cop = NULL; - newsp = NULL; } last_pad = PL_comppad; @@ -5352,7 +6650,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* we don't use MULTICALL here as we want to call the * first op of the block of interest, rather than the - * first op of the sub */ + * first op of the sub. Also, we don't want to free + * the savestack frame */ before = (IV)(SP-PL_stack_base); PL_op = nop; CALLRUNOPS(aTHX); /* Scalar context. */ @@ -5445,7 +6744,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) assert(!(scan->flags & ~RXf_PMf_COMPILETIME)); re_sv = rex->engine->op_comp(aTHX_ &ret, 1, NULL, rex->engine, NULL, NULL, - /* copy /msix etc to inner pattern */ + /* copy /msixn etc to inner pattern */ ARG2L(scan), pm_flags); @@ -5482,7 +6781,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) maxopenparen = 0; /* run the pattern returned from (??{...}) */ - eval_recurse_doit: /* Share code with GOSUB below this line + eval_recurse_doit: /* Share code with GOSUB below this line * At this point we expect the stack context to be * set up correctly */ @@ -5512,7 +6811,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* now continue from first node in postoned RE */ PUSH_YES_STATE_GOTO(EVAL_AB, startpoint, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } case EVAL_AB: /* cleanup after a successful (??{A})B */ @@ -5600,7 +6899,9 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) break; case ACCEPT: /* (*ACCEPT) */ - if (ARG(scan)){ + if (scan->flags) + sv_yes_mark = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); + if (ARG2L(scan)){ regnode *cursor; for (cursor=scan; cursor && OP(cursor)!=END; @@ -5772,21 +7073,21 @@ NULL PUSH_YES_STATE_GOTO(CURLYX_end, PREVOPER(next), locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } case CURLYX_end: /* just finished matching all of A*B */ cur_curlyx = ST.prev_curlyx; sayYES; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CURLYX_end_fail: /* just failed to match all of A*B */ regcpblow(ST.cp); cur_curlyx = ST.prev_curlyx; sayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ #undef ST @@ -5825,7 +7126,7 @@ NULL PUSH_STATE_GOTO(WHILEM_A_pre, A, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } /* If degenerate A matches "", assume A done. */ @@ -5938,7 +7239,7 @@ NULL PUSH_YES_STATE_GOTO(WHILEM_B_min, ST.save_curlyx->u.curlyx.B, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } /* Prefer A over B for maximal matching. */ @@ -5950,19 +7251,19 @@ NULL REGCP_SET(ST.lastcp); PUSH_STATE_GOTO(WHILEM_A_max, A, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } goto do_whilem_B_max; } /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case WHILEM_B_min: /* just matched B in a minimal match */ case WHILEM_B_max: /* just matched B in a maximal match */ cur_curlyx = ST.save_curlyx; sayYES; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case WHILEM_B_max_fail: /* just failed to match B in a maximal match */ cur_curlyx = ST.save_curlyx; @@ -5970,7 +7271,7 @@ NULL cur_curlyx->u.curlyx.count--; CACHEsayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case WHILEM_A_min_fail: /* just failed to match A in a minimal match */ /* FALLTHROUGH */ @@ -5981,7 +7282,7 @@ NULL cur_curlyx->u.curlyx.count--; CACHEsayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case WHILEM_A_max_fail: /* just failed to match A in a maximal match */ REGCP_UNWIND(ST.lastcp); @@ -6008,7 +7309,7 @@ NULL PUSH_YES_STATE_GOTO(WHILEM_B_max, ST.save_curlyx->u.curlyx.B, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case WHILEM_B_min_fail: /* just failed to match B in a minimal match */ cur_curlyx = ST.save_curlyx; @@ -6043,7 +7344,7 @@ NULL /*A*/ NEXTOPER(ST.save_curlyx->u.curlyx.me) + EXTRA_STEP_2ARGS, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ #undef ST #define ST st->u.branch @@ -6069,14 +7370,15 @@ NULL PUSH_STATE_GOTO(BRANCH_next, scan, locinput); } /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CUTGROUP: /* /(*THEN)/ */ - sv_yes_mark = st->u.mark.mark_name = scan->flags ? NULL : - MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); + sv_yes_mark = st->u.mark.mark_name = scan->flags + ? MUTABLE_SV(rexi->data->data[ ARG( scan ) ]) + : NULL; PUSH_STATE_GOTO(CUTGROUP_next, next, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CUTGROUP_next_fail: do_cutgroup = 1; @@ -6085,12 +7387,12 @@ NULL sv_commit = st->u.mark.mark_name; sayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case BRANCH_next: sayYES; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case BRANCH_next_fail: /* that branch failed; try the next, if any */ if (do_cutgroup) { @@ -6157,7 +7459,7 @@ NULL curlym_do_A: /* execute the A in /A{m,n}B/ */ PUSH_YES_STATE_GOTO(CURLYM_A, ST.A, locinput); /* match A */ /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CURLYM_A: /* we've just matched an A */ ST.count++; @@ -6294,7 +7596,7 @@ NULL PUSH_STATE_GOTO(CURLYM_B, ST.B, locinput); /* match B */ /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CURLYM_B_fail: /* just failed to match a B */ REGCP_UNWIND(ST.cp); @@ -6473,7 +7775,7 @@ NULL goto curly_try_B_max; } /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CURLY_B_min_known_fail: /* failed to find B in a non-greedy match where c1,c2 valid */ @@ -6549,7 +7851,7 @@ NULL PUSH_STATE_GOTO(CURLY_B_min_known, ST.B, locinput); } /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case CURLY_B_min_fail: /* failed to find B in a non-greedy match where c1,c2 invalid */ @@ -6582,9 +7884,9 @@ NULL } sayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ - curly_try_B_max: + curly_try_B_max: /* a successful greedy match: now try to match B */ if (cur_eval && cur_eval->u.eval.close_paren && cur_eval->u.eval.close_paren == (U32)ST.paren) { @@ -6613,7 +7915,7 @@ NULL CURLY_SETPAREN(ST.paren, ST.count); PUSH_STATE_GOTO(CURLY_B_max, ST.B, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } } /* FALLTHROUGH */ @@ -6634,7 +7936,7 @@ NULL #undef ST case END: /* last op of main pattern */ - fake_end: + fake_end: if (cur_eval) { /* we've just finished A in /(??{A})B/; now continue with B */ @@ -6733,7 +8035,7 @@ NULL /* execute body of (?...A) */ PUSH_YES_STATE_GOTO(IFMATCH_A, NEXTOPER(NEXTOPER(scan)), newstart); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } case IFMATCH_A_fail: /* body of (?...A) failed */ @@ -6770,20 +8072,32 @@ NULL /* FALLTHROUGH */ case PRUNE: /* (*PRUNE) */ - if (!scan->flags) + if (scan->flags) sv_yes_mark = sv_commit = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); PUSH_STATE_GOTO(COMMIT_next, next, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case COMMIT_next_fail: no_final = 1; /* FALLTHROUGH */ + sayNO; + NOT_REACHED; /* NOTREACHED */ case OPFAIL: /* (*FAIL) */ - sayNO; + if (scan->flags) + sv_commit = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); + if (logical) { + /* deal with (?(?!)X|Y) properly, + * make sure we trigger the no branch + * of the trailing IFTHEN structure*/ + sw= 0; + break; + } else { + sayNO; + } /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ #define ST st->u.mark case MARKPOINT: /* (*MARK:foo) */ @@ -6794,13 +8108,13 @@ NULL ST.mark_loc = locinput; PUSH_YES_STATE_GOTO(MARKPOINT_next, next, locinput); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case MARKPOINT_next: mark_state = ST.prev_mark; sayYES; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case MARKPOINT_next_fail: if (popmark && sv_eq(ST.mark_name,popmark)) @@ -6822,10 +8136,10 @@ NULL mark_state->u.mark.mark_name : NULL; sayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ case SKIP: /* (*SKIP) */ - if (scan->flags) { + if (!scan->flags) { /* (*SKIP) : if we fail we cut here*/ ST.mark_name = NULL; ST.mark_loc = locinput; @@ -6868,7 +8182,7 @@ NULL no_final = 1; sayNO; /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ #undef ST case LNBREAK: /* \R */ @@ -6885,7 +8199,7 @@ NULL /* this is a point to jump to in order to increment * locinput by one character */ - increment_locinput: + increment_locinput: assert(!NEXTCHR_IS_EOS); if (utf8_target) { locinput += PL_utf8skip[nextchr]; @@ -6956,9 +8270,9 @@ NULL Perl_croak(aTHX_ "corrupted regexp pointers"); /* NOTREACHED */ sayNO; - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ -yes: + yes: if (yes_state) { /* we have successfully completed a subexpression, but we must now * pop to the state marked by yes_state and continue from there */ @@ -7019,7 +8333,7 @@ yes: result = 1; goto final_exit; -no: + no: DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, "%*s %sfailed...%s\n", @@ -7027,7 +8341,7 @@ no: PL_colors[4], PL_colors[5]) ); -no_silent: + no_silent: if (no_final) { if (yes_state) { goto yes; @@ -7108,7 +8422,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, char *loceol = reginfo->strend; /* local version */ I32 hardcount = 0; /* How many matches so far */ bool utf8_target = reginfo->is_utf8_target; - int to_complement = 0; /* Invert the result? */ + unsigned int to_complement = 0; /* Invert the result? */ UV utf8_flags; _char_class_number classnum; #ifndef DEBUGGING @@ -7168,16 +8482,12 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, else scan = loceol; break; - case CANY: /* Move forward bytes, unless goes off end */ - if (utf8_target && loceol - scan > max) { - - /* hadn't been adjusted in the UTF-8 case */ - scan += max; + case EXACTL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*scan)) { + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(scan, loceol); } - else { - scan = loceol; - } - break; + /* FALLTHROUGH */ case EXACT: assert(STR_LEN(p) == reginfo->is_utf8_pat ? UTF8SKIP(STRING(p)) : 1); @@ -7216,7 +8526,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, /* Target isn't utf8; convert the character in the UTF-8 * pattern to non-UTF8, and do a simple loop */ - c = TWO_BYTE_UTF8_TO_NATIVE(c, *(STRING(p) + 1)); + c = EIGHT_BIT_UTF8_TO_NATIVE(c, *(STRING(p) + 1)); while (scan < loceol && UCHARAT(scan) == c) { scan++; } @@ -7251,6 +8561,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, goto do_exactf; case EXACTFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; utf8_flags = FOLDEQ_LOCALE; goto do_exactf; @@ -7259,11 +8570,19 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, utf8_flags = 0; goto do_exactf; + case EXACTFLU8: + if (! utf8_target) { + break; + } + utf8_flags = FOLDEQ_LOCALE | FOLDEQ_S2_ALREADY_FOLDED + | FOLDEQ_S2_FOLDS_SANE; + goto do_exactf; + case EXACTFU_SS: case EXACTFU: utf8_flags = reginfo->is_utf8_pat ? FOLDEQ_S2_ALREADY_FOLDED : 0; - do_exactf: { + do_exactf: { int c1, c2; U8 c1_utf8[UTF8_MAXBYTES+1], c2_utf8[UTF8_MAXBYTES+1]; @@ -7322,6 +8641,14 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } break; } + case ANYOFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (ANYOFL_UTF8_LOCALE_REQD(FLAGS(p)) && ! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), utf8_locale_required); + } + /* FALLTHROUGH */ + case ANYOFD: case ANYOF: if (utf8_target) { while (hardcount < max @@ -7344,6 +8671,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, /* FALLTHROUGH */ case POSIXL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; if (! utf8_target) { while (scan < loceol && to_complement ^ cBOOL(isFOO_lc(FLAGS(p), *scan))) @@ -7420,7 +8748,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } } else { - utf8_posix: + utf8_posix: classnum = (_char_class_number) FLAGS(p); if (classnum < _FIRST_NON_SWASH_CC) { @@ -7442,7 +8770,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } else if (UTF8_IS_DOWNGRADEABLE_START(*scan)) { if (! (to_complement - ^ cBOOL(_generic_isCC(TWO_BYTE_UTF8_TO_NATIVE(*scan, + ^ cBOOL(_generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*scan, *(scan + 1)), classnum)))) { @@ -7463,11 +8791,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, * code is written for making the loops as tight as possible. * It could be refactored to save space instead */ switch (classnum) { - case _CC_ENUM_SPACE: /* XXX would require separate code - if we revert the change of \v - matching this */ - /* FALLTHROUGH */ - case _CC_ENUM_PSXSPC: + case _CC_ENUM_SPACE: while (hardcount < max && scan < loceol && (to_complement ^ cBOOL(isSPACE_utf8(scan)))) @@ -7563,16 +8887,18 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } break; + case BOUNDL: + case NBOUNDL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + /* FALLTHROUGH */ case BOUND: case BOUNDA: - case BOUNDL: case BOUNDU: case EOS: case GPOS: case KEEPS: case NBOUND: case NBOUNDA: - case NBOUNDL: case NBOUNDU: case OPFAIL: case SBOL: @@ -7583,7 +8909,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, default: Perl_croak(aTHX_ "panic: regrepeat() called with unrecognized node type %d='%s'", OP(p), PL_reg_name[OP(p)]); /* NOTREACHED */ - NOT_REACHED; + NOT_REACHED; /* NOTREACHED */ } @@ -7631,7 +8957,7 @@ Perl_regclass_swash(pTHX_ const regexp *prog, const regnode* node, bool doinit, /* - reginclass - determine if a character falls into a character class - n is the ANYOF regnode + n is the ANYOF-type regnode p is the target string p_end points to one byte beyond the end of the target string utf8_target tells whether p is in UTF-8. @@ -7665,20 +8991,25 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const * UTF8_ALLOW_FFFF */ if (c_len == (STRLEN)-1) Perl_croak(aTHX_ "Malformed UTF-8 character (fatal)"); + if (c > 255 && OP(n) == ANYOFL && ! ANYOFL_UTF8_LOCALE_REQD(flags)) { + _CHECK_AND_OUTPUT_WIDE_LOCALE_CP_MSG(c); + } } /* If this character is potentially in the bitmap, check it */ if (c < NUM_ANYOF_CODE_POINTS) { if (ANYOF_BITMAP_TEST(n, c)) match = TRUE; - else if ((flags & ANYOF_MATCHES_ALL_NON_UTF8_NON_ASCII) + else if ((flags + & ANYOF_SHARED_d_MATCHES_ALL_NON_UTF8_NON_ASCII_non_d_WARN_SUPER) + && OP(n) == ANYOFD && ! utf8_target && ! isASCII(c)) { match = TRUE; } else if (flags & ANYOF_LOCALE_FLAGS) { - if ((flags & ANYOF_LOC_FOLD) + if ((flags & ANYOFL_FOLD) && c < 256 && ANYOF_BITMAP_TEST(n, PL_fold_locale[c])) { @@ -7744,11 +9075,27 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const { match = TRUE; /* Everything above the bitmap matches */ } - else if ((flags & ANYOF_HAS_NONBITMAP_NON_UTF8_MATCHES) - || (utf8_target && (flags & ANYOF_HAS_UTF8_NONBITMAP_MATCHES)) - || ((flags & ANYOF_LOC_FOLD) - && IN_UTF8_CTYPE_LOCALE - && ARG(n) != ANYOF_ONLY_HAS_BITMAP)) + /* Here doesn't match everything above the bitmap. If there is + * some information available beyond the bitmap, we may find a + * match in it. If so, this is most likely because the code point + * is outside the bitmap range. But rarely, it could be because of + * some other reason. If so, various flags are set to indicate + * this possibility. On ANYOFD nodes, there may be matches that + * happen only when the target string is UTF-8; or for other node + * types, because runtime lookup is needed, regardless of the + * UTF-8ness of the target string. Finally, under /il, there may + * be some matches only possible if the locale is a UTF-8 one. */ + else if ( ARG(n) != ANYOF_ONLY_HAS_BITMAP + && ( c >= NUM_ANYOF_CODE_POINTS + || ( (flags & ANYOF_SHARED_d_UPPER_LATIN1_UTF8_STRING_MATCHES_non_d_RUNTIME_USER_PROP) + && ( UNLIKELY(OP(n) != ANYOFD) + || (utf8_target && ! isASCII_uni(c) +# if NUM_ANYOF_CODE_POINTS > 256 + && c < 256 +# endif + ))) + || ( ANYOFL_SOME_FOLDS_ONLY_IN_UTF8_LOCALE(flags) + && IN_UTF8_CTYPE_LOCALE))) { SV* only_utf8_locale = NULL; SV * const sw = _get_regclass_nonbitmap_data(prog, n, TRUE, 0, @@ -7774,7 +9121,9 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const } if (UNICODE_IS_SUPER(c) - && (flags & ANYOF_WARN_SUPER) + && (flags + & ANYOF_SHARED_d_MATCHES_ALL_NON_UTF8_NON_ASCII_non_d_WARN_SUPER) + && OP(n) != ANYOFD && ckWARN_d(WARN_NON_UNICODE)) { Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), @@ -7813,6 +9162,9 @@ S_reghop3(U8 *s, SSize_t off, const U8* lim) if (UTF8_IS_CONTINUED(*s)) { while (s > lim && UTF8_IS_CONTINUATION(*s)) s--; + if (! UTF8_IS_START(*s)) { + Perl_croak_nocontext("Malformed UTF-8 character (fatal)"); + } } /* XXX could check well-formedness here */ } @@ -7837,6 +9189,9 @@ S_reghop4(U8 *s, SSize_t off, const U8* llim, const U8* rlim) if (UTF8_IS_CONTINUED(*s)) { while (s > llim && UTF8_IS_CONTINUATION(*s)) s--; + if (! UTF8_IS_START(*s)) { + Perl_croak_nocontext("Malformed UTF-8 character (fatal)"); + } } /* XXX could check well-formedness here */ } @@ -7866,6 +9221,9 @@ S_reghopmaybe3(U8* s, SSize_t off, const U8* lim) if (UTF8_IS_CONTINUED(*s)) { while (s > lim && UTF8_IS_CONTINUATION(*s)) s--; + if (! UTF8_IS_START(*s)) { + Perl_croak_nocontext("Malformed UTF-8 character (fatal)"); + } } /* XXX could check well-formedness here */ } @@ -8087,11 +9445,5 @@ S_to_byte_substr(pTHX_ regexp *prog) } /* - * Local variables: - * c-indentation-style: bsd - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * * ex: set ts=8 sts=4 sw=4 et: */