X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/ef9bc83298763750079a8f1ff37b18b092bdfb33..113ebcf5c66556023782d22f6bacda6584cada8b:/regexec.c diff --git a/regexec.c b/regexec.c index b6d163e..9aec6c3 100644 --- a/regexec.c +++ b/regexec.c @@ -2,8 +2,8 @@ */ /* - * One Ring to rule them all, One Ring to find them - & + * One Ring to rule them all, One Ring to find them + * * [p.v of _The Lord of the Rings_, opening poem] * [p.50 of _The Lord of the Rings_, I/iii: "The Shadow of the Past"] * [p.254 of _The Lord of the Rings_, II/ii: "The Council of Elrond"] @@ -80,9 +80,15 @@ # include "regcomp.h" #endif -#include "inline_invlist.c" +#include "invlist_inline.h" #include "unicode_constants.h" +#define B_ON_NON_UTF8_LOCALE_IS_WRONG \ + "Use of \\b{} or \\B{} for non-UTF-8 locale is wrong. Assuming a UTF-8 locale" + +static const char utf8_locale_required[] = + "Use of (?[ ]) for non-UTF-8 locale is wrong. Assuming a UTF-8 locale"; + #ifdef DEBUGGING /* At least one required character in the target string is expressible only in * UTF-8. */ @@ -189,19 +195,7 @@ static const char* const non_utf8_target_but_utf8_required PL_utf8_swash_ptrs[_CC_WORDCHAR], \ "", \ PL_XPosix_ptrs[_CC_WORDCHAR], \ - LATIN_CAPITAL_LETTER_SHARP_S_UTF8); - -#define LOAD_UTF8_CHARCLASS_GCB() /* Grapheme cluster boundaries */ \ - STMT_START { \ - LOAD_UTF8_CHARCLASS_DEBUG_TEST(PL_utf8_X_regular_begin, \ - "_X_regular_begin", \ - NULL, \ - LATIN_CAPITAL_LETTER_SHARP_S_UTF8); \ - LOAD_UTF8_CHARCLASS_DEBUG_TEST(PL_utf8_X_extend, \ - "_X_extend", \ - NULL, \ - COMBINING_GRAVE_ACCENT_UTF8); \ - } STMT_END + LATIN_SMALL_LIGATURE_LONG_S_T_UTF8); #define PLACEHOLDER /* Something for the preprocessor to grab onto */ /* TODO: Combine JUMPABLE and HAS_TEXT to cache OP(rn) */ @@ -231,15 +225,15 @@ static const char* const non_utf8_target_but_utf8_required #if 0 /* Currently these are only used when PL_regkind[OP(rn)] == EXACT so - we don't need this definition. */ + we don't need this definition. XXX These are now out-of-sync*/ #define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==REF || OP(rn)==NREF ) #define IS_TEXTF(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn)==EXACTFA || OP(rn)==EXACTFA_NO_TRIE || OP(rn)==EXACTF || OP(rn)==REFF || OP(rn)==NREFF ) #define IS_TEXTFL(rn) ( OP(rn)==EXACTFL || OP(rn)==REFFL || OP(rn)==NREFFL ) #else /* ... so we use this as its faster. */ -#define IS_TEXT(rn) ( OP(rn)==EXACT ) -#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFA || OP(rn) == EXACTFA_NO_TRIE) +#define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==EXACTL ) +#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFLU8 || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFA || OP(rn) == EXACTFA_NO_TRIE) #define IS_TEXTF(rn) ( OP(rn)==EXACTF ) #define IS_TEXTFL(rn) ( OP(rn)==EXACTFL ) @@ -262,16 +256,6 @@ static const char* const non_utf8_target_but_utf8_required } \ } STMT_END -/* These constants are for finding GCB=LV and GCB=LVT in the CLUMP regnode. - * These are for the pre-composed Hangul syllables, which are all in a - * contiguous block and arranged there in such a way so as to facilitate - * alorithmic determination of their characteristics. As such, they don't need - * a swash, but can be determined by simple arithmetic. Almost all are - * GCB=LVT, but every 28th one is a GCB=LV */ -#define SBASE 0xAC00 /* Start of block */ -#define SCount 11172 /* Length of block */ -#define TCount 28 - #define SLAB_FIRST(s) (&(s)->states[0]) #define SLAB_LAST(s) (&(s)->states[PERL_REGMATCH_SLAB_SLOTS-1]) @@ -475,7 +459,6 @@ S_isFOO_lc(pTHX_ const U8 classnum, const U8 character) case _CC_ENUM_GRAPH: return isGRAPH_LC(character); case _CC_ENUM_LOWER: return isLOWER_LC(character); case _CC_ENUM_PRINT: return isPRINT_LC(character); - case _CC_ENUM_PSXSPC: return isPSXSPC_LC(character); case _CC_ENUM_PUNCT: return isPUNCT_LC(character); case _CC_ENUM_SPACE: return isSPACE_LC(character); case _CC_ENUM_UPPER: return isUPPER_LC(character); @@ -485,7 +468,7 @@ S_isFOO_lc(pTHX_ const U8 classnum, const U8 character) Perl_croak(aTHX_ "panic: isFOO_lc() has an unexpected character class '%d'", classnum); } - assert(0); /* NOTREACHED */ + NOT_REACHED; /* NOTREACHED */ return FALSE; } @@ -498,7 +481,7 @@ S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character) * '_char_class_number'. * * This just calls isFOO_lc on the code point for the character if it is in - * the range 0-255. Outside that range, all characters avoid Unicode + * the range 0-255. Outside that range, all characters use Unicode * rules, ignoring any locale. So use the Unicode function if this class * requires a swash, and use the Unicode macro otherwise. */ @@ -509,9 +492,11 @@ S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character) } else if (UTF8_IS_DOWNGRADEABLE_START(*character)) { return isFOO_lc(classnum, - TWO_BYTE_UTF8_TO_NATIVE(*character, *(character + 1))); + EIGHT_BIT_UTF8_TO_NATIVE(*character, *(character + 1))); } + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(character, character + UTF8SKIP(character)); + if (classnum < _FIRST_NON_SWASH_CC) { /* Initialize the swash unless done already */ @@ -530,9 +515,7 @@ S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character) } switch ((_char_class_number) classnum) { - case _CC_ENUM_SPACE: - case _CC_ENUM_PSXSPC: return is_XPERLSPACE_high(character); - + case _CC_ENUM_SPACE: return is_XPERLSPACE_high(character); case _CC_ENUM_BLANK: return is_HORIZWS_high(character); case _CC_ENUM_XDIGIT: return is_XDIGIT_high(character); case _CC_ENUM_VERTSPACE: return is_VERTWS_high(character); @@ -671,7 +654,7 @@ Perl_re_intuit_start(pTHX_ "Intuit: trying to determine minimum start position...\n")); /* for now, assume that all substr offsets are positive. If at some point - * in the future someone wants to do clever things with look-behind and + * in the future someone wants to do clever things with lookbehind and * -ve offsets, they'll need to fix up any code in this function * which uses these offsets. See the thread beginning * <20140113145929.GF27210@iabyn.com> @@ -703,6 +686,7 @@ Perl_re_intuit_start(pTHX_ goto fail; } + RX_MATCH_UTF8_set(rx,utf8_target); reginfo->is_utf8_target = cBOOL(utf8_target); reginfo->info_aux = NULL; reginfo->strbeg = strbeg; @@ -751,7 +735,7 @@ Perl_re_intuit_start(pTHX_ /* ml_anch: check after \n? * - * A note about IMPLICIT: on an un-anchored pattern beginning + * A note about PREGf_IMPLICIT: on an un-anchored pattern beginning * with /.*.../, these flags will have been added by the * compiler: * /.*abc/, /.*abc/m: PREGf_IMPLICIT | PREGf_ANCH_MBOL @@ -772,7 +756,7 @@ Perl_re_intuit_start(pTHX_ * be too fiddly (e.g. REXEC_IGNOREPOS). */ if ( strpos != strbeg - && (prog->intflags & (PREGf_ANCH_BOL|PREGf_ANCH_SBOL))) + && (prog->intflags & PREGf_ANCH_SBOL)) { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, " Not at start...\n")); @@ -789,9 +773,7 @@ Perl_re_intuit_start(pTHX_ * caller will have set strpos=pos()-4; we look for the substr * at position pos()-4+1, which lines up with the "a" */ - if (prog->check_offset_min == prog->check_offset_max - && !(prog->intflags & PREGf_CANY_SEEN)) - { + if (prog->check_offset_min == prog->check_offset_max) { /* Substring at constant offset from beg-of-str... */ SSize_t slen = SvCUR(check); char *s = HOP3c(strpos, prog->check_offset_min, strend); @@ -875,28 +857,21 @@ Perl_re_intuit_start(pTHX_ " At restart: rx_origin=%"IVdf" Check offset min: %"IVdf " Start shift: %"IVdf" End shift %"IVdf " Real end Shift: %"IVdf"\n", - (IV)(rx_origin - strpos), + (IV)(rx_origin - strbeg), (IV)prog->check_offset_min, (IV)start_shift, (IV)end_shift, (IV)prog->check_end_shift); }); - if (prog->intflags & PREGf_CANY_SEEN) { - start_point= (U8*)(rx_origin + start_shift); - end_point= (U8*)(strend - end_shift); - if (start_point > end_point) - goto fail_finish; - } else { - end_point = HOP3(strend, -end_shift, strbeg); - start_point = HOPMAYBE3(rx_origin, start_shift, end_point); - if (!start_point) - goto fail_finish; - } + end_point = HOP3(strend, -end_shift, strbeg); + start_point = HOPMAYBE3(rx_origin, start_shift, end_point); + if (!start_point) + goto fail_finish; /* If the regex is absolutely anchored to either the start of the - * string (BOL,SBOL) or to pos() (ANCH_GPOS), then + * string (SBOL) or to pos() (ANCH_GPOS), then * check_offset_max represents an upper bound on the string where * the substr could start. For the ANCH_GPOS case, we assume that * the caller of intuit will have already set strpos to @@ -923,16 +898,16 @@ Perl_re_intuit_start(pTHX_ } } - DEBUG_OPTIMISE_MORE_r({ - PerlIO_printf(Perl_debug_log, " fbm_instr len=%d str=<%.*s>\n", - (int)(end_point - start_point), - (int)(end_point - start_point) > 20 ? 20 : (int)(end_point - start_point), - start_point); - }); - check_at = fbm_instr( start_point, end_point, check, multiline ? FBMrf_MULTILINE : 0); + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " doing 'check' fbm scan, [%"IVdf"..%"IVdf"] gave %"IVdf"\n", + (IV)((char*)start_point - strbeg), + (IV)((char*)end_point - strbeg), + (IV)(check_at ? check_at - strbeg : -1) + )); + /* Update the count-of-usability, remove useless subpatterns, unshift s. */ @@ -950,9 +925,6 @@ Perl_re_intuit_start(pTHX_ if (!check_at) goto fail_finish; - /* Finish the diagnostic message */ - DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "%ld...\n", (long)(check_at - strpos)) ); - /* set rx_origin to the minimum position where the regex could start * matching, given the constraint of the just-matched check substring. * But don't set it lower than previously. @@ -960,6 +932,12 @@ Perl_re_intuit_start(pTHX_ if (check_at - rx_origin > prog->check_offset_max) rx_origin = HOP3c(check_at, -prog->check_offset_max, rx_origin); + /* Finish the diagnostic message */ + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + "%ld (rx_origin now %"IVdf")...\n", + (long)(check_at - strbeg), + (IV)(rx_origin - strbeg) + )); } @@ -1063,12 +1041,34 @@ Perl_re_intuit_start(pTHX_ must = utf8_target ? other->utf8_substr : other->substr; assert(SvPOK(must)); - s = fbm_instr( - (unsigned char*)s, - (unsigned char*)last + SvCUR(must) - (SvTAIL(must)!=0), - must, - multiline ? FBMrf_MULTILINE : 0 - ); + { + char *from = s; + char *to = last + SvCUR(must) - (SvTAIL(must)!=0); + + if (from > to) { + s = NULL; + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " skipping 'other' fbm scan: %"IVdf" > %"IVdf"\n", + (IV)(from - strbeg), + (IV)(to - strbeg) + )); + } + else { + s = fbm_instr( + (unsigned char*)from, + (unsigned char*)to, + must, + multiline ? FBMrf_MULTILINE : 0 + ); + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " doing 'other' fbm scan, [%"IVdf"..%"IVdf"] gave %"IVdf"\n", + (IV)(from - strbeg), + (IV)(to - strbeg), + (IV)(s ? s - strbeg : -1) + )); + } + } + DEBUG_EXECUTE_r({ RE_PV_QUOTED_DECL(quoted, utf8_target, PERL_DEBUG_PAD_ZERO(0), SvPVX_const(must), RE_SV_DUMPLEN(must), 30); @@ -1084,29 +1084,27 @@ Perl_re_intuit_start(pTHX_ * find it before there, we never will */ if (last >= last1) { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - ", giving up...\n")); + "; giving up...\n")); goto fail_finish; } /* try to find the check substr again at a later * position. Maybe next time we'll find the "other" substr * in range too */ - DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - ", trying %s at offset %ld...\n", - (other_ix ? "floating" : "anchored"), - (long)(HOP3c(check_at, 1, strend) - strpos))); - other_last = HOP3c(last, 1, strend) /* highest failure */; rx_origin = other_ix /* i.e. if other-is-float */ ? HOP3c(rx_origin, 1, strend) : HOP4c(last, 1 - other->min_offset, strbeg, strend); + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + "; about to retry %s at offset %ld (rx_origin now %"IVdf")...\n", + (other_ix ? "floating" : "anchored"), + (long)(HOP3c(check_at, 1, strend) - strbeg), + (IV)(rx_origin - strbeg) + )); goto restart; } else { - DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, " at offset %ld...\n", - (long)(s - strpos))); - if (other_ix) { /* if (other-is-float) */ /* other_last is set to s, not s+1, since its possible for * a floating substr to fail first time, then succeed @@ -1122,6 +1120,12 @@ Perl_re_intuit_start(pTHX_ rx_origin = HOP3c(s, -other->min_offset, strbeg); other_last = HOP3c(s, 1, strend); } + DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, + " at offset %ld (rx_origin now %"IVdf")...\n", + (long)(s - strbeg), + (IV)(rx_origin - strbeg) + )); + } } else { @@ -1129,13 +1133,13 @@ Perl_re_intuit_start(pTHX_ PerlIO_printf(Perl_debug_log, " Check-only match: offset min:%"IVdf" max:%"IVdf " check_at:%"IVdf" rx_origin:%"IVdf" rx_origin-check_at:%"IVdf - " strend-strpos:%"IVdf"\n", + " strend:%"IVdf"\n", (IV)prog->check_offset_min, (IV)prog->check_offset_max, - (IV)(check_at-strpos), - (IV)(rx_origin-strpos), + (IV)(check_at-strbeg), + (IV)(rx_origin-strbeg), (IV)(rx_origin-check_at), - (IV)(strend-strpos) + (IV)(strend-strbeg) ) ); } @@ -1156,7 +1160,7 @@ Perl_re_intuit_start(pTHX_ * scanning ahead for the next \n or the next substr is debatable. * On the one hand you'd expect rare substrings to appear less * often than \n's. On the other hand, searching for \n means - * we're effectively flipping been check_substr and "\n" on each + * we're effectively flipping between check_substr and "\n" on each * iteration as the current "rarest" string candidate, which * means for example that we'll quickly reject the whole string if * hasn't got a \n, rather than trying every substr position @@ -1185,8 +1189,8 @@ Perl_re_intuit_start(pTHX_ * check was anchored (and thus has no wiggle room), * or check was float and rx_origin is above the float range */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - " Found /%s^%s/m, restarting lookup for check-string at offset %ld...\n", - PL_colors[0], PL_colors[1], (long)(rx_origin - strpos))); + " Found /%s^%s/m, about to restart lookup for check-string with rx_origin %ld...\n", + PL_colors[0], PL_colors[1], (long)(rx_origin - strbeg))); goto restart; } @@ -1201,18 +1205,19 @@ Perl_re_intuit_start(pTHX_ * didn't contradict, so just retry the anchored "other" * substr */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - " Found /%s^%s/m at offset %ld, rescanning for anchored from offset %ld...\n", + " Found /%s^%s/m, rescanning for anchored from offset %"IVdf" (rx_origin now %"IVdf")...\n", PL_colors[0], PL_colors[1], - (long)(rx_origin - strpos), - (long)(rx_origin - strpos + prog->anchored_offset))); + (IV)(rx_origin - strbeg + prog->anchored_offset), + (IV)(rx_origin - strbeg) + )); goto do_other_substr; } /* success: we don't contradict the found floating substring * (and there's no anchored substr). */ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, - " Found /%s^%s/m at offset %ld...\n", - PL_colors[0], PL_colors[1], (long)(rx_origin - strpos))); + " Found /%s^%s/m with rx_origin %ld...\n", + PL_colors[0], PL_colors[1], (long)(rx_origin - strbeg))); } else { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, @@ -1304,11 +1309,15 @@ Perl_re_intuit_start(pTHX_ * The condition above is in bytes rather than * chars for efficiency. It's conservative, in * that it errs on the side of doing 'goto - * do_other_substr', where a more accurate - * char-based calculation will be done */ + * do_other_substr'. In this case, at worst, + * an extra anchored search may get done, but in + * practice the extra fbm_instr() is likely to + * get skipped anyway. */ DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - " Looking for anchored substr starting at offset %ld...\n", - (long)(other_last - strpos)) ); + " about to retry anchored at offset %ld (rx_origin now %"IVdf")...\n", + (long)(other_last - strbeg), + (IV)(rx_origin - strbeg) + )); goto do_other_substr; } } @@ -1326,9 +1335,9 @@ Perl_re_intuit_start(pTHX_ * search for the next \n if any, its safe here */ rx_origin++; DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - " Looking for /%s^%s/m starting at offset %ld...\n", + " about to look for /%s^%s/m starting at rx_origin %ld...\n", PL_colors[0], PL_colors[1], - (long)(rx_origin - strpos)) ); + (long)(rx_origin - strbeg)) ); goto postprocess_substr_matches; } @@ -1354,9 +1363,11 @@ Perl_re_intuit_start(pTHX_ goto fail; } DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - " Looking for %s substr starting at offset %ld...\n", + " about to look for %s substr starting at offset %ld (rx_origin now %"IVdf")...\n", (prog->substrs->check_ix ? "floating" : "anchored"), - (long)(rx_origin + start_shift - strpos)) ); + (long)(rx_origin + start_shift - strbeg), + (IV)(rx_origin - strbeg) + )); goto restart; } @@ -1365,7 +1376,7 @@ Perl_re_intuit_start(pTHX_ if (rx_origin != s) { DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, " By STCLASS: moving %ld --> %ld\n", - (long)(rx_origin - strpos), (long)(s - strpos)) + (long)(rx_origin - strbeg), (long)(s - strbeg)) ); } else { @@ -1417,7 +1428,7 @@ Perl_re_intuit_start(pTHX_ DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "Intuit: %sSuccessfully guessed:%s match at offset %ld\n", - PL_colors[4], PL_colors[5], (long)(rx_origin - strpos)) ); + PL_colors[4], PL_colors[5], (long)(rx_origin - strbeg)) ); return rx_origin; @@ -1432,23 +1443,39 @@ Perl_re_intuit_start(pTHX_ #define DECL_TRIE_TYPE(scan) \ - const enum { trie_plain, trie_utf8, trie_utf8_fold, trie_latin_utf8_fold, \ - trie_utf8_exactfa_fold, trie_latin_utf8_exactfa_fold } \ - trie_type = ((scan->flags == EXACT) \ - ? (utf8_target ? trie_utf8 : trie_plain) \ - : (scan->flags == EXACTFA) \ - ? (utf8_target ? trie_utf8_exactfa_fold : trie_latin_utf8_exactfa_fold) \ - : (utf8_target ? trie_utf8_fold : trie_latin_utf8_fold)) + const enum { trie_plain, trie_utf8, trie_utf8_fold, trie_latin_utf8_fold, \ + trie_utf8_exactfa_fold, trie_latin_utf8_exactfa_fold, \ + trie_utf8l, trie_flu8 } \ + trie_type = ((scan->flags == EXACT) \ + ? (utf8_target ? trie_utf8 : trie_plain) \ + : (scan->flags == EXACTL) \ + ? (utf8_target ? trie_utf8l : trie_plain) \ + : (scan->flags == EXACTFA) \ + ? (utf8_target \ + ? trie_utf8_exactfa_fold \ + : trie_latin_utf8_exactfa_fold) \ + : (scan->flags == EXACTFLU8 \ + ? trie_flu8 \ + : (utf8_target \ + ? trie_utf8_fold \ + : trie_latin_utf8_fold))) #define REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc, uscan, len, uvc, charid, foldlen, foldbuf, uniflags) \ STMT_START { \ STRLEN skiplen; \ U8 flags = FOLD_FLAGS_FULL; \ switch (trie_type) { \ + case trie_flu8: \ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; \ + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*uc)) { \ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(uc, uc + UTF8SKIP(uc)); \ + } \ + goto do_trie_utf8_fold; \ case trie_utf8_exactfa_fold: \ flags |= FOLD_FLAGS_NOMIX_ASCII; \ - /* FALLTHROUGH */ \ + /* FALLTHROUGH */ \ case trie_utf8_fold: \ + do_trie_utf8_fold: \ if ( foldlen>0 ) { \ uvc = utf8n_to_uvchr( (const U8*) uscan, UTF8_MAXLEN, &len, uniflags ); \ foldlen -= len; \ @@ -1457,14 +1484,14 @@ STMT_START { } else { \ uvc = _to_utf8_fold_flags( (const U8*) uc, foldbuf, &foldlen, flags); \ len = UTF8SKIP(uc); \ - skiplen = UNISKIP( uvc ); \ + skiplen = UVCHR_SKIP( uvc ); \ foldlen -= skiplen; \ uscan = foldbuf + skiplen; \ } \ break; \ case trie_latin_utf8_exactfa_fold: \ flags |= FOLD_FLAGS_NOMIX_ASCII; \ - /* FALLTHROUGH */ \ + /* FALLTHROUGH */ \ case trie_latin_utf8_fold: \ if ( foldlen>0 ) { \ uvc = utf8n_to_uvchr( (const U8*) uscan, UTF8_MAXLEN, &len, uniflags ); \ @@ -1474,11 +1501,17 @@ STMT_START { } else { \ len = 1; \ uvc = _to_fold_latin1( (U8) *uc, foldbuf, &foldlen, flags); \ - skiplen = UNISKIP( uvc ); \ + skiplen = UVCHR_SKIP( uvc ); \ foldlen -= skiplen; \ uscan = foldbuf + skiplen; \ } \ break; \ + case trie_utf8l: \ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; \ + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*uc)) { \ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(uc, uc + UTF8SKIP(uc)); \ + } \ + /* FALLTHROUGH */ \ case trie_utf8: \ uvc = utf8n_to_uvchr( (const U8*) uc, UTF8_MAXLEN, &len, uniflags ); \ break; \ @@ -1688,7 +1721,7 @@ REXEC_FBC_SCAN( /* Loops while (s < strend) */ \ FBC_UTF8(TEST_UV, TEST_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER), \ TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER) -#define FBC_BOUND_A(TEST_NON_UTF8, TEST_UV, TEST_UTF8) \ +#define FBC_BOUND_A(TEST_NON_UTF8) \ FBC_BOUND_COMMON( \ FBC_UTF8_A(TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER), \ TEST_NON_UTF8, REXEC_FBC_TRYIT, PLACEHOLDER) @@ -1698,11 +1731,75 @@ REXEC_FBC_SCAN( /* Loops while (s < strend) */ \ FBC_UTF8(TEST_UV, TEST_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT), \ TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT) -#define FBC_NBOUND_A(TEST_NON_UTF8, TEST_UV, TEST_UTF8) \ +#define FBC_NBOUND_A(TEST_NON_UTF8) \ FBC_BOUND_COMMON( \ FBC_UTF8_A(TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT), \ TEST_NON_UTF8, PLACEHOLDER, REXEC_FBC_TRYIT) +/* Takes a pointer to an inversion list, a pointer to its corresponding + * inversion map, and a code point, and returns the code point's value + * according to the two arrays. It assumes that all code points have a value. + * This is used as the base macro for macros for particular properties */ +#define _generic_GET_BREAK_VAL_CP(invlist, invmap, cp) \ + invmap[_invlist_search(invlist, cp)] + +/* Same as above, but takes begin, end ptrs to a UTF-8 encoded string instead + * of a code point, returning the value for the first code point in the string. + * And it takes the particular macro name that finds the desired value given a + * code point. Merely convert the UTF-8 to code point and call the cp macro */ +#define _generic_GET_BREAK_VAL_UTF8(cp_macro, pos, strend) \ + (__ASSERT_(pos < strend) \ + /* Note assumes is valid UTF-8 */ \ + (cp_macro(utf8_to_uvchr_buf((pos), (strend), NULL)))) + +/* Returns the GCB value for the input code point */ +#define getGCB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_GCB_invlist, \ + _Perl_GCB_invmap, \ + (cp)) + +/* Returns the GCB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getGCB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getGCB_VAL_CP, pos, strend) + +/* Returns the LB value for the input code point */ +#define getLB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_LB_invlist, \ + _Perl_LB_invmap, \ + (cp)) + +/* Returns the LB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getLB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getLB_VAL_CP, pos, strend) + + +/* Returns the SB value for the input code point */ +#define getSB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_SB_invlist, \ + _Perl_SB_invmap, \ + (cp)) + +/* Returns the SB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getSB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getSB_VAL_CP, pos, strend) + +/* Returns the WB value for the input code point */ +#define getWB_VAL_CP(cp) \ + _generic_GET_BREAK_VAL_CP( \ + PL_WB_invlist, \ + _Perl_WB_invmap, \ + (cp)) + +/* Returns the WB value for the first code point in the UTF-8 encoded string + * bounded by pos and strend */ +#define getWB_VAL_UTF8(pos, strend) \ + _generic_GET_BREAK_VAL_UTF8(getWB_VAL_CP, pos, strend) /* We know what class REx starts with. Try to find this position... */ /* if reginfo->intuit, its a dryrun */ @@ -1738,6 +1835,15 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* We know what class it must start with. */ switch (OP(c)) { + case ANYOFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (ANYOFL_UTF8_LOCALE_REQD(FLAGS(c)) && ! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), utf8_locale_required); + } + + /* FALLTHROUGH */ + case ANYOFD: case ANYOF: if (utf8_target) { REXEC_FBC_UTF8_CLASS_SCAN( @@ -1747,14 +1853,6 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, REXEC_FBC_CLASS_SCAN(REGINCLASS(prog, c, (U8*)s)); } break; - case CANY: - REXEC_FBC_SCAN( - if (tmp && (reginfo->intuit || regtry(reginfo, &s))) - goto got_it; - else - tmp = doevery; - ); - break; case EXACTFA_NO_TRIE: /* This node only generated for non-utf8 patterns */ assert(! is_utf8_pat); @@ -1779,6 +1877,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, goto do_exactf_non_utf8; case EXACTFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; if (is_utf8_pat || utf8_target || IN_UTF8_CTYPE_LOCALE) { utf8_fold_flags = FOLDEQ_LOCALE; goto do_exactf_utf8; @@ -1793,6 +1892,15 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } goto do_exactf_utf8; + case EXACTFLU8: + if (! utf8_target) { /* All code points in this node require + UTF-8 to express. */ + break; + } + utf8_fold_flags = FOLDEQ_LOCALE | FOLDEQ_S2_ALREADY_FOLDED + | FOLDEQ_S2_FOLDS_SANE; + goto do_exactf_utf8; + case EXACTFU: if (is_utf8_pat || utf8_target) { utf8_fold_flags = is_utf8_pat ? FOLDEQ_S2_ALREADY_FOLDED : 0; @@ -1807,7 +1915,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* FALLTHROUGH */ - do_exactf_non_utf8: /* Neither pattern nor string are UTF8, and there + do_exactf_non_utf8: /* Neither pattern nor string are UTF8, and there are no glitches with fold-length differences between the target string and pattern */ @@ -1841,8 +1949,8 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } break; - do_exactf_utf8: - { + do_exactf_utf8: + { unsigned expansion; /* If one of the operands is in utf8, we can't use the simpler folding @@ -1898,29 +2006,315 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } case BOUNDL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (FLAGS(c) != TRADITIONAL_BOUND) { + if (! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + B_ON_NON_UTF8_LOCALE_IS_WRONG); + } + goto do_boundu; + } + FBC_BOUND(isWORDCHAR_LC, isWORDCHAR_LC_uvchr, isWORDCHAR_LC_utf8); break; + case NBOUNDL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (FLAGS(c) != TRADITIONAL_BOUND) { + if (! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + B_ON_NON_UTF8_LOCALE_IS_WRONG); + } + goto do_nboundu; + } + FBC_NBOUND(isWORDCHAR_LC, isWORDCHAR_LC_uvchr, isWORDCHAR_LC_utf8); break; - case BOUND: + + case BOUND: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + FBC_BOUND(isWORDCHAR, isWORDCHAR_uni, isWORDCHAR_utf8); break; - case BOUNDA: - FBC_BOUND_A(isWORDCHAR_A, isWORDCHAR_A, isWORDCHAR_A); + + case BOUNDA: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + + FBC_BOUND_A(isWORDCHAR_A); break; - case NBOUND: + + case NBOUND: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + FBC_NBOUND(isWORDCHAR, isWORDCHAR_uni, isWORDCHAR_utf8); break; - case NBOUNDA: - FBC_NBOUND_A(isWORDCHAR_A, isWORDCHAR_A, isWORDCHAR_A); - break; - case BOUNDU: - FBC_BOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + + case NBOUNDA: /* regcomp.c makes sure that this only has the traditional \b + meaning */ + assert(FLAGS(c) == TRADITIONAL_BOUND); + + FBC_NBOUND_A(isWORDCHAR_A); break; + case NBOUNDU: - FBC_NBOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + if ((bound_type) FLAGS(c) == TRADITIONAL_BOUND) { + FBC_NBOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + break; + } + + do_nboundu: + + to_complement = 1; + /* FALLTHROUGH */ + + case BOUNDU: + do_boundu: + switch((bound_type) FLAGS(c)) { + case TRADITIONAL_BOUND: + FBC_BOUND(isWORDCHAR_L1, isWORDCHAR_uni, isWORDCHAR_utf8); + break; + case GCB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) + { + goto got_it; + } + + /* Didn't match. Try at the next position (if there is one) */ + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + GCB_enum before = getGCB_VAL_UTF8( + reghop3((U8*)s, -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + GCB_enum after = getGCB_VAL_UTF8((U8*) s, + (U8*) reginfo->strend); + if ( (to_complement ^ isGCB(before, after)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. Everything is a GCB except between CR and + LF */ + while (s < strend) { + if ((to_complement ^ ( UCHARAT(s - 1) != '\r' + || UCHARAT(s) != '\n')) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + s++; + } + } + + /* And, since this is a bound, it can match after the final + * character in the string */ + if ((reginfo->intuit || regtry(reginfo, &s))) { + goto got_it; + } + break; + + case LB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + LB_enum before = getLB_VAL_UTF8(reghop3((U8*)s, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + LB_enum after = getLB_VAL_UTF8((U8*) s, (U8*) reginfo->strend); + if (to_complement ^ isLB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. */ + LB_enum before = getLB_VAL_CP((U8) *(s -1)); + while (s < strend) { + LB_enum after = getLB_VAL_CP((U8) *s); + if (to_complement ^ isLB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s++; + } + } + + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + + break; + + case SB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + SB_enum before = getSB_VAL_UTF8(reghop3((U8*)s, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + SB_enum after = getSB_VAL_UTF8((U8*) s, + (U8*) reginfo->strend); + if ((to_complement ^ isSB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. */ + SB_enum before = getSB_VAL_CP((U8) *(s -1)); + while (s < strend) { + SB_enum after = getSB_VAL_CP((U8) *s); + if ((to_complement ^ isSB(before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + before = after; + s++; + } + } + + /* Here are at the final position in the target string. The SB + * value is always true here, so matches, depending on other + * constraints */ + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + + break; + + case WB_BOUND: + if (s == reginfo->strbeg) { + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + s += (utf8_target) ? UTF8SKIP(s) : 1; + if (UNLIKELY(s >= reginfo->strend)) { + break; + } + } + + if (utf8_target) { + /* We are at a boundary between char_sub_0 and char_sub_1. + * We also keep track of the value for char_sub_-1 as we + * loop through the line. Context may be needed to make a + * determination, and if so, this can save having to + * recalculate it */ + WB_enum previous = WB_UNKNOWN; + WB_enum before = getWB_VAL_UTF8( + reghop3((U8*)s, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend); + while (s < strend) { + WB_enum after = getWB_VAL_UTF8((U8*) s, + (U8*) reginfo->strend); + if ((to_complement ^ isWB(previous, + before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + previous = before; + before = after; + s += UTF8SKIP(s); + } + } + else { /* Not utf8. */ + WB_enum previous = WB_UNKNOWN; + WB_enum before = getWB_VAL_CP((U8) *(s -1)); + while (s < strend) { + WB_enum after = getWB_VAL_CP((U8) *s); + if ((to_complement ^ isWB(previous, + before, + after, + (U8*) reginfo->strbeg, + (U8*) s, + (U8*) reginfo->strend, + utf8_target)) + && (reginfo->intuit || regtry(reginfo, &s))) + { + goto got_it; + } + previous = before; + before = after; + s++; + } + } + + if (reginfo->intuit || regtry(reginfo, &s)) { + goto got_it; + } + } break; + case LNBREAK: REXEC_FBC_CSCAN(is_LNBREAK_utf8_safe(s, strend), is_LNBREAK_latin1_safe(s, strend) @@ -1935,6 +2329,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, /* FALLTHROUGH */ case POSIXL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; REXEC_FBC_CSCAN(to_complement ^ cBOOL(isFOO_utf8_lc(FLAGS(c), (U8 *) s)), to_complement ^ cBOOL(isFOO_lc(FLAGS(c), *s))); break; @@ -1980,7 +2375,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } else { - posix_utf8: + posix_utf8: classnum = (_char_class_number) FLAGS(c); if (classnum < _FIRST_NON_SWASH_CC) { while (s < strend) { @@ -1997,7 +2392,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, classnum))) || (UTF8_IS_DOWNGRADEABLE_START(*s) && to_complement ^ cBOOL( - _generic_isCC(TWO_BYTE_UTF8_TO_NATIVE(*s, + _generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*s, *(s + 1)), classnum)))) { @@ -2015,11 +2410,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, } else switch (classnum) { /* These classes are implemented as macros */ - case _CC_ENUM_SPACE: /* XXX would require separate code if we - revert the change of \v matching this */ - /* FALLTHROUGH */ - - case _CC_ENUM_PSXSPC: + case _CC_ENUM_SPACE: REXEC_FBC_UTF8_CLASS_SCAN( to_complement ^ cBOOL(isSPACE_utf8(s))); break; @@ -2046,7 +2437,7 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, default: Perl_croak(aTHX_ "panic: find_byclass() node %d='%s' has an unexpected character class '%d'", OP(c), PL_reg_name[OP(c)], classnum); - assert(0); /* NOTREACHED */ + NOT_REACHED; /* NOTREACHED */ } } break; @@ -2361,7 +2752,7 @@ S_reg_set_capture_string(pTHX_ REGEXP * const rx, U32 n = 0; max = -1; /* calculate the right-most part of the string covered - * by a capture. Due to look-ahead, this may be to + * by a capture. Due to lookahead, this may be to * the right of $&, so we have to scan all captures */ while (n <= prog->lastparen) { if (prog->offs[n].end > max) @@ -2382,7 +2773,7 @@ S_reg_set_capture_string(pTHX_ REGEXP * const rx, U32 n = 0; min = max; /* calculate the left-most part of the string covered - * by a capture. Due to look-behind, this may be to + * by a capture. Due to lookbehind, this may be to * the left of $&, so we have to scan all captures */ while (min && n <= prog->lastparen) { if ( prog->offs[n].start != -1 @@ -2487,7 +2878,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, PERL_UNUSED_ARG(data); /* Be paranoid... */ - if (prog == NULL || stringarg == NULL) { + if (prog == NULL) { Perl_croak(aTHX_ "NULL regexp parameter"); } @@ -2498,6 +2889,11 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, startpos = stringarg; + /* set these early as they may be used by the HOP macros below */ + reginfo->strbeg = strbeg; + reginfo->strend = strend; + reginfo->is_utf8_target = cBOOL(utf8_target); + if (prog->intflags & PREGf_GPOS_SEEN) { MAGIC *mg; @@ -2506,7 +2902,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, reginfo->ganch = (flags & REXEC_IGNOREPOS) ? stringarg /* use start pos rather than pos() */ - : (sv && (mg = mg_find_mglob(sv)) && mg->mg_len >= 0) + : ((mg = mg_find_mglob(sv)) && mg->mg_len >= 0) /* Defined pos(): */ ? strbeg + MgBYTEPOS(mg, sv, strbeg, strend-strbeg) : strbeg; /* pos() not defined; use start of string */ @@ -2525,20 +2921,23 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, */ if (prog->intflags & PREGf_ANCH_GPOS) { - startpos = reginfo->ganch - prog->gofs; - if (startpos < - ((flags & REXEC_FAIL_ON_UNDERFLOW) ? stringarg : strbeg)) - { - DEBUG_r(PerlIO_printf(Perl_debug_log, - "fail: ganch-gofs before earliest possible start\n")); - return 0; + if (prog->gofs) { + startpos = HOPBACKc(reginfo->ganch, prog->gofs); + if (!startpos || + ((flags & REXEC_FAIL_ON_UNDERFLOW) && startpos < stringarg)) + { + DEBUG_r(PerlIO_printf(Perl_debug_log, + "fail: ganch-gofs before earliest possible start\n")); + return 0; + } } + else + startpos = reginfo->ganch; } else if (prog->gofs) { - if (startpos - prog->gofs < strbeg) + startpos = HOPBACKc(startpos, prog->gofs); + if (!startpos) startpos = strbeg; - else - startpos -= prog->gofs; } else if (prog->intflags & PREGf_GPOS_FLOAT) startpos = strbeg; @@ -2617,16 +3016,14 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, } RX_MATCH_TAINTED_off(rx); + RX_MATCH_UTF8_set(rx, utf8_target); reginfo->prog = rx; /* Yes, sorry that this is confusing. */ reginfo->intuit = 0; - reginfo->is_utf8_target = cBOOL(utf8_target); reginfo->is_utf8_pat = cBOOL(RX_UTF8(rx)); reginfo->warned = FALSE; - reginfo->strbeg = strbeg; reginfo->sv = sv; reginfo->poscache_maxiter = 0; /* not yet started a countdown */ - reginfo->strend = strend; /* see how far we have to get to not match where we matched before */ reginfo->till = stringarg + minend; @@ -2637,7 +3034,6 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, magic belonging to this SV. Not newSVsv, either, as it does not COW. */ - assert(!IS_PADGV(sv)); reginfo->sv = newSV(0); SvSetSV_nosteal(reginfo->sv, sv); SAVEFREESV(reginfo->sv); @@ -2714,93 +3110,59 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, )); } - /* Simplest case: anchored match need be tried only once. */ - /* [unless only anchor is BOL and multiline is set] */ + /* Simplest case: anchored match need be tried only once, or with + * MBOL, only at the beginning of each line. + * + * Note that /.*.../ sets PREGf_IMPLICIT|MBOL, while /.*.../s sets + * PREGf_IMPLICIT|SBOL. The idea is that with /.*.../s, if it doesn't + * match at the start of the string then it won't match anywhere else + * either; while with /.*.../, if it doesn't match at the beginning, + * the earliest it could match is at the start of the next line */ + if (prog->intflags & (PREGf_ANCH & ~PREGf_ANCH_GPOS)) { - if (s == startpos && regtry(reginfo, &s)) + char *end; + + if (regtry(reginfo, &s)) goto got_it; - else if (multiline || (prog->intflags & (PREGf_IMPLICIT | PREGf_ANCH_MBOL))) /* XXXX SBOL? */ - { - char *end; - - if (minlen) - dontbother = minlen - 1; - end = HOP3c(strend, -dontbother, strbeg) - 1; - /* for multiline we only have to try after newlines */ - if (prog->check_substr || prog->check_utf8) { - /* because of the goto we can not easily reuse the macros for bifurcating the - unicode/non-unicode match modes here like we do elsewhere - demerphq */ - if (utf8_target) { - if (s == startpos) - goto after_try_utf8; - while (1) { - if (regtry(reginfo, &s)) { - goto got_it; - } - after_try_utf8: - if (s > end) { - goto phooey; - } - if (prog->extflags & RXf_USE_INTUIT) { - s = re_intuit_start(rx, sv, strbeg, - s + UTF8SKIP(s), strend, flags, NULL); - if (!s) { - goto phooey; - } - } - else { - s += UTF8SKIP(s); - } - } - } /* end search for check string in unicode */ - else { - if (s == startpos) { - goto after_try_latin; - } - while (1) { - if (regtry(reginfo, &s)) { - goto got_it; - } - after_try_latin: - if (s > end) { - goto phooey; - } - if (prog->extflags & RXf_USE_INTUIT) { - s = re_intuit_start(rx, sv, strbeg, - s + 1, strend, flags, NULL); - if (!s) { - goto phooey; - } - } - else { - s++; - } - } - } /* end search for check string in latin*/ - } /* end search for check string */ - else { /* search for newline */ - if (s > startpos) { - /*XXX: The s-- is almost definitely wrong here under unicode - demeprhq*/ - s--; - } - /* We can use a more efficient search as newlines are the same in unicode as they are in latin */ - while (s <= end) { /* note it could be possible to match at the end of the string */ - if (*s++ == '\n') { /* don't need PL_utf8skip here */ - if (regtry(reginfo, &s)) - goto got_it; - } - } - } /* end search for newline */ - } /* end anchored/multiline check string search */ - goto phooey; - } else if (prog->intflags & PREGf_ANCH_GPOS) + + if (!(prog->intflags & PREGf_ANCH_MBOL)) + goto phooey; + + /* didn't match at start, try at other newline positions */ + + if (minlen) + dontbother = minlen - 1; + end = HOP3c(strend, -dontbother, strbeg) - 1; + + /* skip to next newline */ + + while (s <= end) { /* note it could be possible to match at the end of the string */ + /* NB: newlines are the same in unicode as they are in latin */ + if (*s++ != '\n') + continue; + if (prog->check_substr || prog->check_utf8) { + /* note that with PREGf_IMPLICIT, intuit can only fail + * or return the start position, so it's of limited utility. + * Nevertheless, I made the decision that the potential for + * quick fail was still worth it - DAPM */ + s = re_intuit_start(rx, sv, strbeg, s, strend, flags, NULL); + if (!s) + goto phooey; + } + if (regtry(reginfo, &s)) + goto got_it; + } + goto phooey; + } /* end anchored search */ + + if (prog->intflags & PREGf_ANCH_GPOS) { /* PREGf_ANCH_GPOS should never be true if PREGf_GPOS_SEEN is not true */ assert(prog->intflags & PREGf_GPOS_SEEN); /* For anchored \G, the only position it can match from is * (ganch-gofs); we already set startpos to this above; if intuit * moved us on from there, we can't possibly succeed */ - assert(startpos == reginfo->ganch - prog->gofs); + assert(startpos == HOPBACKc(reginfo->ganch, prog->gofs)); if (s == startpos && regtry(reginfo, &s)) goto got_it; goto phooey; @@ -2964,12 +3326,12 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, if (minlen) { const OPCODE op = OP(progi->regstclass); /* don't bother with what can't match */ - if (PL_regkind[op] != EXACT && op != CANY && PL_regkind[op] != TRIE) + if (PL_regkind[op] != EXACT && PL_regkind[op] != TRIE) strend = HOPc(strend, -(minlen - 1)); } DEBUG_EXECUTE_r({ SV * const prop = sv_newmortal(); - regprop(prog, prop, c, reginfo); + regprop(prog, prop, c, reginfo, NULL); { RE_PV_QUOTED_DECL(quoted,utf8_target,PERL_DEBUG_PAD_ZERO(1), s,strend-s,60); @@ -3075,7 +3437,8 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, * and replaced it with this one. Yves */ DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, - "String does not contain required substring, cannot match.\n" + "%sString does not contain required substring, cannot match.%s\n", + PL_colors[4], PL_colors[5] )); goto phooey; } @@ -3105,7 +3468,7 @@ Perl_regexec_flags(pTHX_ REGEXP * const rx, char *stringarg, char *strend, /* Failure. */ goto phooey; -got_it: + got_it: /* s/// doesn't like it if $& is earlier than where we asked it to * start searching (which can happen on something like /.\G/) */ if ( (flags & REXEC_FAIL_ON_UNDERFLOW) @@ -3137,8 +3500,6 @@ got_it: if (RXp_PAREN_NAMES(prog)) (void)hv_iterinit(RXp_PAREN_NAMES(prog)); - RX_MATCH_UTF8_set(rx, utf8_target); - /* make sure $`, $&, $', and $digit will work later */ if ( !(flags & REXEC_NOT_FIRST) ) S_reg_set_capture_string(aTHX_ rx, @@ -3147,7 +3508,7 @@ got_it: return 1; -phooey: + phooey: DEBUG_EXECUTE_r(PerlIO_printf(Perl_debug_log, "%sMatch failed%s\n", PL_colors[4], PL_colors[5])); @@ -3185,7 +3546,7 @@ phooey: /* - regtry - try match at specific point */ -STATIC I32 /* 0 failure, 1 success */ +STATIC bool /* 0 failure, 1 success */ S_regtry(pTHX_ regmatch_info *reginfo, char **startposp) { CHECKPOINT lastcp; @@ -3521,7 +3882,7 @@ S_dump_exec_pos(pTHX_ const char *locinput, if (pref0_len > pref_len) pref0_len = pref_len; { - const int is_uni = (utf8_target && OP(scan) != CANY) ? 1 : 0; + const int is_uni = utf8_target ? 1 : 0; RE_PV_COLOR_DECL(s0,len0,is_uni,PERL_DEBUG_PAD(0), (locinput - pref_len),pref0_len, 60, 4, 5); @@ -3648,7 +4009,7 @@ S_setup_EXACTISH_ST_c1_c2(pTHX_ const regnode * const text_node, int *c1p, U8 *pat = (U8*)STRING(text_node); U8 folded[UTF8_MAX_FOLD_CHAR_EXPAND * UTF8_MAXBYTES_CASE + 1] = { '\0' }; - if (OP(text_node) == EXACT) { + if (OP(text_node) == EXACT || OP(text_node) == EXACTL) { /* In an exact node, only one thing can be matched, that first * character. If both the pat and the target are UTF-8, we can just @@ -3842,48 +4203,914 @@ S_setup_EXACTISH_ST_c1_c2(pTHX_ const regnode * const text_node, int *c1p, c2 = PL_fold_latin1[c1]; break; - default: - Perl_croak(aTHX_ "panic: Unexpected op %u", OP(text_node)); - assert(0); /* NOTREACHED */ - } + default: + Perl_croak(aTHX_ "panic: Unexpected op %u", OP(text_node)); + NOT_REACHED; /* NOTREACHED */ + } + } + } + } + + /* Here have figured things out. Set up the returns */ + if (use_chrtest_void) { + *c2p = *c1p = CHRTEST_VOID; + } + else if (utf8_target) { + if (! utf8_has_been_setup) { /* Don't have the utf8; must get it */ + uvchr_to_utf8(c1_utf8, c1); + uvchr_to_utf8(c2_utf8, c2); + } + + /* Invariants are stored in both the utf8 and byte outputs; Use + * negative numbers otherwise for the byte ones. Make sure that the + * byte ones are the same iff the utf8 ones are the same */ + *c1p = (UTF8_IS_INVARIANT(*c1_utf8)) ? *c1_utf8 : CHRTEST_NOT_A_CP_1; + *c2p = (UTF8_IS_INVARIANT(*c2_utf8)) + ? *c2_utf8 + : (c1 == c2) + ? CHRTEST_NOT_A_CP_1 + : CHRTEST_NOT_A_CP_2; + } + else if (c1 > 255) { + if (c2 > 255) { /* both possibilities are above what a non-utf8 string + can represent */ + return FALSE; + } + + *c1p = *c2p = c2; /* c2 is the only representable value */ + } + else { /* c1 is representable; see about c2 */ + *c1p = c1; + *c2p = (c2 < 256) ? c2 : c1; + } + + return TRUE; +} + +PERL_STATIC_INLINE bool +S_isGCB(const GCB_enum before, const GCB_enum after) +{ + /* returns a boolean indicating if there is a Grapheme Cluster Boundary + * between the inputs. See http://www.unicode.org/reports/tr29/ */ + + return GCB_table[before][after]; +} + +/* Combining marks attach to most classes that precede them, but this defines + * the exceptions (from TR14) */ +#define LB_CM_ATTACHES_TO(prev) ( ! ( prev == LB_EDGE \ + || prev == LB_Mandatory_Break \ + || prev == LB_Carriage_Return \ + || prev == LB_Line_Feed \ + || prev == LB_Next_Line \ + || prev == LB_Space \ + || prev == LB_ZWSpace)) + +STATIC bool +S_isLB(pTHX_ LB_enum before, + LB_enum after, + const U8 * const strbeg, + const U8 * const curpos, + const U8 * const strend, + const bool utf8_target) +{ + U8 * temp_pos = (U8 *) curpos; + LB_enum prev = before; + + /* Is the boundary between 'before' and 'after' line-breakable? + * Most of this is just a table lookup of a generated table from Unicode + * rules. But some rules require context to decide, and so have to be + * implemented in code */ + + PERL_ARGS_ASSERT_ISLB; + + /* Rule numbers in the comments below are as of Unicode 8.0 */ + + redo: + before = prev; + switch (LB_table[before][after]) { + case LB_BREAKABLE: + return TRUE; + + case LB_NOBREAK: + case LB_NOBREAK_EVEN_WITH_SP_BETWEEN: + return FALSE; + + case LB_SP_foo + LB_BREAKABLE: + case LB_SP_foo + LB_NOBREAK: + case LB_SP_foo + LB_NOBREAK_EVEN_WITH_SP_BETWEEN: + + /* When we have something following a SP, we have to look at the + * context in order to know what to do. + * + * SP SP should not reach here because LB7: Do not break before + * spaces. (For two spaces in a row there is nothing that + * overrides that) */ + assert(after != LB_Space); + + /* Here we have a space followed by a non-space. Mostly this is a + * case of LB18: "Break after spaces". But there are complications + * as the handling of spaces is somewhat tricky. They are in a + * number of rules, which have to be applied in priority order, but + * something earlier in the string can cause a rule to be skipped + * and a lower priority rule invoked. A prime example is LB7 which + * says don't break before a space. But rule LB8 (lower priority) + * says that the first break opportunity after a ZW is after any + * span of spaces immediately after it. If a ZW comes before a SP + * in the input, rule LB8 applies, and not LB7. Other such rules + * involve combining marks which are rules 9 and 10, but they may + * override higher priority rules if they come earlier in the + * string. Since we're doing random access into the middle of the + * string, we have to look for rules that should get applied based + * on both string position and priority. Combining marks do not + * attach to either ZW nor SP, so we don't have to consider them + * until later. + * + * To check for LB8, we have to find the first non-space character + * before this span of spaces */ + do { + prev = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (prev == LB_Space); + + /* LB8 Break before any character following a zero-width space, + * even if one or more spaces intervene. + * ZW SP* ÷ + * So if we have a ZW just before this span, and to get here this + * is the final space in the span. */ + if (prev == LB_ZWSpace) { + return TRUE; + } + + /* Here, not ZW SP+. There are several rules that have higher + * priority than LB18 and can be resolved now, as they don't depend + * on anything earlier in the string (except ZW, which we have + * already handled). One of these rules is LB11 Do not break + * before Word joiner, but we have specially encoded that in the + * lookup table so it is caught by the single test below which + * catches the other ones. */ + if (LB_table[LB_Space][after] - LB_SP_foo + == LB_NOBREAK_EVEN_WITH_SP_BETWEEN) + { + return FALSE; + } + + /* If we get here, we have to XXX consider combining marks. */ + if (prev == LB_Combining_Mark) { + + /* What happens with these depends on the character they + * follow. */ + do { + prev = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (prev == LB_Combining_Mark); + + /* Most times these attach to and inherit the characteristics + * of that character, but not always, and when not, they are to + * be treated as AL by rule LB10. */ + if (! LB_CM_ATTACHES_TO(prev)) { + prev = LB_Alphabetic; + } + } + + /* Here, we have the character preceding the span of spaces all set + * up. We follow LB18: "Break after spaces" unless the table shows + * that is overriden */ + return LB_table[prev][after] != LB_NOBREAK_EVEN_WITH_SP_BETWEEN; + + case LB_CM_foo: + + /* We don't know how to treat the CM except by looking at the first + * non-CM character preceding it */ + do { + prev = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (prev == LB_Combining_Mark); + + /* Here, 'prev' is that first earlier non-CM character. If the CM + * attatches to it, then it inherits the behavior of 'prev'. If it + * doesn't attach, it is to be treated as an AL */ + if (! LB_CM_ATTACHES_TO(prev)) { + prev = LB_Alphabetic; + } + + goto redo; + + case LB_HY_or_BA_then_foo + LB_BREAKABLE: + case LB_HY_or_BA_then_foo + LB_NOBREAK: + + /* LB21a Don't break after Hebrew + Hyphen. + * HL (HY | BA) × */ + + if (backup_one_LB(strbeg, &temp_pos, utf8_target) + == LB_Hebrew_Letter) + { + return FALSE; + } + + return LB_table[prev][after] - LB_HY_or_BA_then_foo == LB_BREAKABLE; + + case LB_PR_or_PO_then_OP_or_HY + LB_BREAKABLE: + case LB_PR_or_PO_then_OP_or_HY + LB_NOBREAK: + + /* LB25a (PR | PO) × ( OP | HY )? NU */ + if (advance_one_LB(&temp_pos, strend, utf8_target) == LB_Numeric) { + return FALSE; + } + + return LB_table[prev][after] - LB_PR_or_PO_then_OP_or_HY + == LB_BREAKABLE; + + case LB_SY_or_IS_then_various + LB_BREAKABLE: + case LB_SY_or_IS_then_various + LB_NOBREAK: + { + /* LB25d NU (SY | IS)* × (NU | SY | IS | CL | CP ) */ + + LB_enum temp = prev; + do { + temp = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (temp == LB_Break_Symbols || temp == LB_Infix_Numeric); + if (temp == LB_Numeric) { + return FALSE; + } + + return LB_table[prev][after] - LB_SY_or_IS_then_various + == LB_BREAKABLE; + } + + case LB_various_then_PO_or_PR + LB_BREAKABLE: + case LB_various_then_PO_or_PR + LB_NOBREAK: + { + /* LB25e NU (SY | IS)* (CL | CP)? × (PO | PR) */ + + LB_enum temp = prev; + if (temp == LB_Close_Punctuation || temp == LB_Close_Parenthesis) + { + temp = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + while (temp == LB_Break_Symbols || temp == LB_Infix_Numeric) { + temp = backup_one_LB(strbeg, &temp_pos, utf8_target); + } + if (temp == LB_Numeric) { + return FALSE; + } + return LB_various_then_PO_or_PR; + } + + default: + break; + } + +#ifdef DEBUGGING + PerlIO_printf(Perl_error_log, "Unhandled LB pair: LB_table[%d, %d] = %d\n", + before, after, LB_table[before][after]); + assert(0); +#endif + return TRUE; +} + +STATIC LB_enum +S_advance_one_LB(pTHX_ U8 ** curpos, const U8 * const strend, const bool utf8_target) +{ + LB_enum lb; + + PERL_ARGS_ASSERT_ADVANCE_ONE_LB; + + if (*curpos >= strend) { + return LB_EDGE; + } + + if (utf8_target) { + *curpos += UTF8SKIP(*curpos); + if (*curpos >= strend) { + return LB_EDGE; + } + lb = getLB_VAL_UTF8(*curpos, strend); + } + else { + (*curpos)++; + if (*curpos >= strend) { + return LB_EDGE; + } + lb = getLB_VAL_CP(**curpos); + } + + return lb; +} + +STATIC LB_enum +S_backup_one_LB(pTHX_ const U8 * const strbeg, U8 ** curpos, const bool utf8_target) +{ + LB_enum lb; + + PERL_ARGS_ASSERT_BACKUP_ONE_LB; + + if (*curpos < strbeg) { + return LB_EDGE; + } + + if (utf8_target) { + U8 * prev_char_pos = reghopmaybe3(*curpos, -1, strbeg); + U8 * prev_prev_char_pos; + + if (! prev_char_pos) { + return LB_EDGE; + } + + if ((prev_prev_char_pos = reghopmaybe3((U8 *) prev_char_pos, -1, strbeg))) { + lb = getLB_VAL_UTF8(prev_prev_char_pos, prev_char_pos); + *curpos = prev_char_pos; + prev_char_pos = prev_prev_char_pos; + } + else { + *curpos = (U8 *) strbeg; + return LB_EDGE; + } + } + else { + if (*curpos - 2 < strbeg) { + *curpos = (U8 *) strbeg; + return LB_EDGE; + } + (*curpos)--; + lb = getLB_VAL_CP(*(*curpos - 1)); + } + + return lb; +} + +/* This creates a single number by combining two, with 'before' being like the + * 10's digit, but this isn't necessarily base 10; it is base however many + * elements of the enum there are */ +#define SBcase(before, after) ((SB_ENUM_COUNT * before) + after) + +STATIC bool +S_isSB(pTHX_ SB_enum before, + SB_enum after, + const U8 * const strbeg, + const U8 * const curpos, + const U8 * const strend, + const bool utf8_target) +{ + /* returns a boolean indicating if there is a Sentence Boundary Break + * between the inputs. See http://www.unicode.org/reports/tr29/ */ + + U8 * lpos = (U8 *) curpos; + U8 * temp_pos; + SB_enum backup; + + PERL_ARGS_ASSERT_ISSB; + + /* Break at the start and end of text. + SB1. sot ÷ + SB2. ÷ eot */ + if (before == SB_EDGE || after == SB_EDGE) { + return TRUE; + } + + /* SB 3: Do not break within CRLF. */ + if (before == SB_CR && after == SB_LF) { + return FALSE; + } + + /* Break after paragraph separators. (though why CR and LF are considered + * so is beyond me (khw) + SB4. Sep | CR | LF ÷ */ + if (before == SB_Sep || before == SB_CR || before == SB_LF) { + return TRUE; + } + + /* Ignore Format and Extend characters, except after sot, Sep, CR, or LF. + * (See Section 6.2, Replacing Ignore Rules.) + SB5. X (Extend | Format)* → X */ + if (after == SB_Extend || after == SB_Format) { + return FALSE; + } + + if (before == SB_Extend || before == SB_Format) { + before = backup_one_SB(strbeg, &lpos, utf8_target); + } + + /* Do not break after ambiguous terminators like period, if they are + * immediately followed by a number or lowercase letter, if they are + * between uppercase letters, if the first following letter (optionally + * after certain punctuation) is lowercase, or if they are followed by + * "continuation" punctuation such as comma, colon, or semicolon. For + * example, a period may be an abbreviation or numeric period, and thus may + * not mark the end of a sentence. + + * SB6. ATerm × Numeric */ + if (before == SB_ATerm && after == SB_Numeric) { + return FALSE; + } + + /* SB7. (Upper | Lower) ATerm × Upper */ + if (before == SB_ATerm && after == SB_Upper) { + temp_pos = lpos; + backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + if (backup == SB_Upper || backup == SB_Lower) { + return FALSE; + } + } + + /* SB8a. (STerm | ATerm) Close* Sp* × (SContinue | STerm | ATerm) + * SB10. (STerm | ATerm) Close* Sp* × ( Sp | Sep | CR | LF ) */ + backup = before; + temp_pos = lpos; + while (backup == SB_Sp) { + backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + } + while (backup == SB_Close) { + backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + } + if ((backup == SB_STerm || backup == SB_ATerm) + && ( after == SB_SContinue + || after == SB_STerm + || after == SB_ATerm + || after == SB_Sp + || after == SB_Sep + || after == SB_CR + || after == SB_LF)) + { + return FALSE; + } + + /* SB8. ATerm Close* Sp* × ( ¬(OLetter | Upper | Lower | Sep | CR | LF | + * STerm | ATerm) )* Lower */ + if (backup == SB_ATerm) { + U8 * rpos = (U8 *) curpos; + SB_enum later = after; + + while ( later != SB_OLetter + && later != SB_Upper + && later != SB_Lower + && later != SB_Sep + && later != SB_CR + && later != SB_LF + && later != SB_STerm + && later != SB_ATerm + && later != SB_EDGE) + { + later = advance_one_SB(&rpos, strend, utf8_target); + } + if (later == SB_Lower) { + return FALSE; + } + } + + /* Break after sentence terminators, but include closing punctuation, + * trailing spaces, and a paragraph separator (if present). [See note + * below.] + * SB9. ( STerm | ATerm ) Close* × ( Close | Sp | Sep | CR | LF ) */ + backup = before; + temp_pos = lpos; + while (backup == SB_Close) { + backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + } + if ((backup == SB_STerm || backup == SB_ATerm) + && ( after == SB_Close + || after == SB_Sp + || after == SB_Sep + || after == SB_CR + || after == SB_LF)) + { + return FALSE; + } + + + /* SB11. ( STerm | ATerm ) Close* Sp* ( Sep | CR | LF )? ÷ */ + temp_pos = lpos; + backup = backup_one_SB(strbeg, &temp_pos, utf8_target); + if ( backup == SB_Sep + || backup == SB_CR + || backup == SB_LF) + { + lpos = temp_pos; + } + else { + backup = before; + } + while (backup == SB_Sp) { + backup = backup_one_SB(strbeg, &lpos, utf8_target); + } + while (backup == SB_Close) { + backup = backup_one_SB(strbeg, &lpos, utf8_target); + } + if (backup == SB_STerm || backup == SB_ATerm) { + return TRUE; + } + + /* Otherwise, do not break. + SB12. Any × Any */ + + return FALSE; +} + +STATIC SB_enum +S_advance_one_SB(pTHX_ U8 ** curpos, const U8 * const strend, const bool utf8_target) +{ + SB_enum sb; + + PERL_ARGS_ASSERT_ADVANCE_ONE_SB; + + if (*curpos >= strend) { + return SB_EDGE; + } + + if (utf8_target) { + do { + *curpos += UTF8SKIP(*curpos); + if (*curpos >= strend) { + return SB_EDGE; + } + sb = getSB_VAL_UTF8(*curpos, strend); + } while (sb == SB_Extend || sb == SB_Format); + } + else { + do { + (*curpos)++; + if (*curpos >= strend) { + return SB_EDGE; + } + sb = getSB_VAL_CP(**curpos); + } while (sb == SB_Extend || sb == SB_Format); + } + + return sb; +} + +STATIC SB_enum +S_backup_one_SB(pTHX_ const U8 * const strbeg, U8 ** curpos, const bool utf8_target) +{ + SB_enum sb; + + PERL_ARGS_ASSERT_BACKUP_ONE_SB; + + if (*curpos < strbeg) { + return SB_EDGE; + } + + if (utf8_target) { + U8 * prev_char_pos = reghopmaybe3(*curpos, -1, strbeg); + if (! prev_char_pos) { + return SB_EDGE; + } + + /* Back up over Extend and Format. curpos is always just to the right + * of the characater whose value we are getting */ + do { + U8 * prev_prev_char_pos; + if ((prev_prev_char_pos = reghopmaybe3((U8 *) prev_char_pos, -1, + strbeg))) + { + sb = getSB_VAL_UTF8(prev_prev_char_pos, prev_char_pos); + *curpos = prev_char_pos; + prev_char_pos = prev_prev_char_pos; + } + else { + *curpos = (U8 *) strbeg; + return SB_EDGE; + } + } while (sb == SB_Extend || sb == SB_Format); + } + else { + do { + if (*curpos - 2 < strbeg) { + *curpos = (U8 *) strbeg; + return SB_EDGE; + } + (*curpos)--; + sb = getSB_VAL_CP(*(*curpos - 1)); + } while (sb == SB_Extend || sb == SB_Format); + } + + return sb; +} + +#define WBcase(before, after) ((WB_ENUM_COUNT * before) + after) + +STATIC bool +S_isWB(pTHX_ WB_enum previous, + WB_enum before, + WB_enum after, + const U8 * const strbeg, + const U8 * const curpos, + const U8 * const strend, + const bool utf8_target) +{ + /* Return a boolean as to if the boundary between 'before' and 'after' is + * a Unicode word break, using their published algorithm, but tailored for + * Perl by treating spans of white space as one unit. Context may be + * needed to make this determination. If the value for the character + * before 'before' is known, it is passed as 'previous'; otherwise that + * should be set to WB_UNKNOWN. The other input parameters give the + * boundaries and current position in the matching of the string. That + * is, 'curpos' marks the position where the character whose wb value is + * 'after' begins. See http://www.unicode.org/reports/tr29/ */ + + U8 * before_pos = (U8 *) curpos; + U8 * after_pos = (U8 *) curpos; + + PERL_ARGS_ASSERT_ISWB; + + /* WB1 and WB2: Break at the start and end of text. */ + if (before == WB_EDGE || after == WB_EDGE) { + return TRUE; + } + + /* WB 3 is: "Do not break within CRLF." Perl extends this so that all + * white space sequences ending in a vertical space are treated as one + * unit. */ + + if (after == WB_CR || after == WB_LF || after == WB_Newline) { + if (before == WB_CR || before == WB_LF || before == WB_Newline + || before == WB_Perl_Tailored_HSpace) + { + return FALSE; + } + + /* WB 3a: Otherwise break before Newlines (including CR and LF) */ + return TRUE; + } + + /* Here, we know that 'after' is not a vertical space character, but + * 'before' could be. WB 3b is: "Otherwise break after Newlines (including + * CR and LF)." Perl changes that to not break-up spans of white space, + * except when horizontal space is followed by an Extend or Format + * character. These apply just to the final white space character in the + * span, so it is broken away from the rest. (If the Extend or Format + * character follows a vertical space character, it is treated as beginning + * a line, and doesn't modify the preceeding character.) */ + if ( before == WB_CR || before == WB_LF || before == WB_Newline + || before == WB_Perl_Tailored_HSpace) + { + if (after == WB_Perl_Tailored_HSpace) { + U8 * temp_pos = (U8 *) curpos; + const WB_enum next + = advance_one_WB(&temp_pos, strend, utf8_target, + FALSE /* Don't skip Extend nor Format */ ); + return next == WB_Extend || next == WB_Format; + } + else if (before != WB_Perl_Tailored_HSpace) { + + /* Here, 'before' must be one of the vertical space characters, and + * after is not any type of white-space. Follow WB 3b. */ + return TRUE; + } + + /* Here, 'before' is horizontal space, and 'after' is not any kind of + * space. Normal rules apply */ + } + + /* Ignore Format and Extend characters, except when they appear at the + * beginning of a region of text. + * WB4. X (Extend | Format)* → X. */ + + if (after == WB_Extend || after == WB_Format) { + return FALSE; + } + + if (before == WB_Extend || before == WB_Format) { + before = backup_one_WB(&previous, strbeg, &before_pos, utf8_target); + } + + switch (WBcase(before, after)) { + /* Otherwise, break everywhere (including around ideographs). + WB14. Any ÷ Any */ + default: + return TRUE; + + /* Do not break between most letters. + WB5. (ALetter | Hebrew_Letter) × (ALetter | Hebrew_Letter) */ + case WBcase(WB_ALetter, WB_ALetter): + case WBcase(WB_ALetter, WB_Hebrew_Letter): + case WBcase(WB_Hebrew_Letter, WB_ALetter): + case WBcase(WB_Hebrew_Letter, WB_Hebrew_Letter): + return FALSE; + + /* Do not break letters across certain punctuation. + WB6. (ALetter | Hebrew_Letter) + × (MidLetter | MidNumLet | Single_Quote) (ALetter + | Hebrew_Letter) */ + case WBcase(WB_ALetter, WB_MidLetter): + case WBcase(WB_ALetter, WB_MidNumLet): + case WBcase(WB_ALetter, WB_Single_Quote): + case WBcase(WB_Hebrew_Letter, WB_MidLetter): + case WBcase(WB_Hebrew_Letter, WB_MidNumLet): + /*case WBcase(WB_Hebrew_Letter, WB_Single_Quote):*/ + after = advance_one_WB(&after_pos, strend, utf8_target, + TRUE /* Do skip Extend and Format */ ); + return after != WB_ALetter && after != WB_Hebrew_Letter; + + /* WB7. (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | + * Single_Quote) × (ALetter | Hebrew_Letter) */ + case WBcase(WB_MidLetter, WB_ALetter): + case WBcase(WB_MidLetter, WB_Hebrew_Letter): + case WBcase(WB_MidNumLet, WB_ALetter): + case WBcase(WB_MidNumLet, WB_Hebrew_Letter): + case WBcase(WB_Single_Quote, WB_ALetter): + case WBcase(WB_Single_Quote, WB_Hebrew_Letter): + before + = backup_one_WB(&previous, strbeg, &before_pos, utf8_target); + return before != WB_ALetter && before != WB_Hebrew_Letter; + + /* WB7a. Hebrew_Letter × Single_Quote */ + case WBcase(WB_Hebrew_Letter, WB_Single_Quote): + return FALSE; + + /* WB7b. Hebrew_Letter × Double_Quote Hebrew_Letter */ + case WBcase(WB_Hebrew_Letter, WB_Double_Quote): + return advance_one_WB(&after_pos, strend, utf8_target, + TRUE /* Do skip Extend and Format */ ) + != WB_Hebrew_Letter; + + /* WB7c. Hebrew_Letter Double_Quote × Hebrew_Letter */ + case WBcase(WB_Double_Quote, WB_Hebrew_Letter): + return backup_one_WB(&previous, strbeg, &before_pos, utf8_target) + != WB_Hebrew_Letter; + + /* Do not break within sequences of digits, or digits adjacent to + * letters (“3a”, or “A3”). + WB8. Numeric × Numeric */ + case WBcase(WB_Numeric, WB_Numeric): + return FALSE; + + /* WB9. (ALetter | Hebrew_Letter) × Numeric */ + case WBcase(WB_ALetter, WB_Numeric): + case WBcase(WB_Hebrew_Letter, WB_Numeric): + return FALSE; + + /* WB10. Numeric × (ALetter | Hebrew_Letter) */ + case WBcase(WB_Numeric, WB_ALetter): + case WBcase(WB_Numeric, WB_Hebrew_Letter): + return FALSE; + + /* Do not break within sequences, such as “3.2” or “3,456.789”. + WB11. Numeric (MidNum | MidNumLet | Single_Quote) × Numeric + */ + case WBcase(WB_MidNum, WB_Numeric): + case WBcase(WB_MidNumLet, WB_Numeric): + case WBcase(WB_Single_Quote, WB_Numeric): + return backup_one_WB(&previous, strbeg, &before_pos, utf8_target) + != WB_Numeric; + + /* WB12. Numeric × (MidNum | MidNumLet | Single_Quote) Numeric + * */ + case WBcase(WB_Numeric, WB_MidNum): + case WBcase(WB_Numeric, WB_MidNumLet): + case WBcase(WB_Numeric, WB_Single_Quote): + return advance_one_WB(&after_pos, strend, utf8_target, + TRUE /* Do skip Extend and Format */ ) + != WB_Numeric; + + /* Do not break between Katakana. + WB13. Katakana × Katakana */ + case WBcase(WB_Katakana, WB_Katakana): + return FALSE; + + /* Do not break from extenders. + WB13a. (ALetter | Hebrew_Letter | Numeric | Katakana | + ExtendNumLet) × ExtendNumLet */ + case WBcase(WB_ALetter, WB_ExtendNumLet): + case WBcase(WB_Hebrew_Letter, WB_ExtendNumLet): + case WBcase(WB_Numeric, WB_ExtendNumLet): + case WBcase(WB_Katakana, WB_ExtendNumLet): + case WBcase(WB_ExtendNumLet, WB_ExtendNumLet): + return FALSE; + + /* WB13b. ExtendNumLet × (ALetter | Hebrew_Letter | Numeric + * | Katakana) */ + case WBcase(WB_ExtendNumLet, WB_ALetter): + case WBcase(WB_ExtendNumLet, WB_Hebrew_Letter): + case WBcase(WB_ExtendNumLet, WB_Numeric): + case WBcase(WB_ExtendNumLet, WB_Katakana): + return FALSE; + + /* Do not break between regional indicator symbols. + WB13c. Regional_Indicator × Regional_Indicator */ + case WBcase(WB_Regional_Indicator, WB_Regional_Indicator): + return FALSE; + + } + + NOT_REACHED; /* NOTREACHED */ +} + +STATIC WB_enum +S_advance_one_WB(pTHX_ U8 ** curpos, + const U8 * const strend, + const bool utf8_target, + const bool skip_Extend_Format) +{ + WB_enum wb; + + PERL_ARGS_ASSERT_ADVANCE_ONE_WB; + + if (*curpos >= strend) { + return WB_EDGE; + } + + if (utf8_target) { + + /* Advance over Extend and Format */ + do { + *curpos += UTF8SKIP(*curpos); + if (*curpos >= strend) { + return WB_EDGE; + } + wb = getWB_VAL_UTF8(*curpos, strend); + } while ( skip_Extend_Format + && (wb == WB_Extend || wb == WB_Format)); + } + else { + do { + (*curpos)++; + if (*curpos >= strend) { + return WB_EDGE; + } + wb = getWB_VAL_CP(**curpos); + } while ( skip_Extend_Format + && (wb == WB_Extend || wb == WB_Format)); + } + + return wb; +} + +STATIC WB_enum +S_backup_one_WB(pTHX_ WB_enum * previous, const U8 * const strbeg, U8 ** curpos, const bool utf8_target) +{ + WB_enum wb; + + PERL_ARGS_ASSERT_BACKUP_ONE_WB; + + /* If we know what the previous character's break value is, don't have + * to look it up */ + if (*previous != WB_UNKNOWN) { + wb = *previous; + + /* But we need to move backwards by one */ + if (utf8_target) { + *curpos = reghopmaybe3(*curpos, -1, strbeg); + if (! *curpos) { + *previous = WB_EDGE; + *curpos = (U8 *) strbeg; + } + else { + *previous = WB_UNKNOWN; } } - } + else { + (*curpos)--; + *previous = (*curpos <= strbeg) ? WB_EDGE : WB_UNKNOWN; + } - /* Here have figured things out. Set up the returns */ - if (use_chrtest_void) { - *c2p = *c1p = CHRTEST_VOID; - } - else if (utf8_target) { - if (! utf8_has_been_setup) { /* Don't have the utf8; must get it */ - uvchr_to_utf8(c1_utf8, c1); - uvchr_to_utf8(c2_utf8, c2); + /* And we always back up over these two types */ + if (wb != WB_Extend && wb != WB_Format) { + return wb; } + } - /* Invariants are stored in both the utf8 and byte outputs; Use - * negative numbers otherwise for the byte ones. Make sure that the - * byte ones are the same iff the utf8 ones are the same */ - *c1p = (UTF8_IS_INVARIANT(*c1_utf8)) ? *c1_utf8 : CHRTEST_NOT_A_CP_1; - *c2p = (UTF8_IS_INVARIANT(*c2_utf8)) - ? *c2_utf8 - : (c1 == c2) - ? CHRTEST_NOT_A_CP_1 - : CHRTEST_NOT_A_CP_2; + if (*curpos < strbeg) { + return WB_EDGE; } - else if (c1 > 255) { - if (c2 > 255) { /* both possibilities are above what a non-utf8 string - can represent */ - return FALSE; - } - *c1p = *c2p = c2; /* c2 is the only representable value */ + if (utf8_target) { + U8 * prev_char_pos = reghopmaybe3(*curpos, -1, strbeg); + if (! prev_char_pos) { + return WB_EDGE; + } + + /* Back up over Extend and Format. curpos is always just to the right + * of the characater whose value we are getting */ + do { + U8 * prev_prev_char_pos; + if ((prev_prev_char_pos = reghopmaybe3((U8 *) prev_char_pos, + -1, + strbeg))) + { + wb = getWB_VAL_UTF8(prev_prev_char_pos, prev_char_pos); + *curpos = prev_char_pos; + prev_char_pos = prev_prev_char_pos; + } + else { + *curpos = (U8 *) strbeg; + return WB_EDGE; + } + } while (wb == WB_Extend || wb == WB_Format); } - else { /* c1 is representable; see about c2 */ - *c1p = c1; - *c2p = (c2 < 256) ? c2 : c1; + else { + do { + if (*curpos - 2 < strbeg) { + *curpos = (U8 *) strbeg; + return WB_EDGE; + } + (*curpos)--; + wb = getWB_VAL_CP(*(*curpos - 1)); + } while (wb == WB_Extend || wb == WB_Format); } - return TRUE; + return wb; } /* returns -1 on failure, $+[0] on success */ @@ -3958,6 +5185,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) int to_complement; /* Invert the result? */ _char_class_number classnum; bool is_utf8_pat = reginfo->is_utf8_pat; + bool match = FALSE; + #ifdef DEBUGGING GET_RE_DEBUG_FLAGS_DECL; @@ -3991,7 +5220,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) SV * const prop = sv_newmortal(); regnode *rnext=regnext(scan); DUMP_EXEC_POS( locinput, scan, utf8_target ); - regprop(rex, prop, scan, reginfo); + regprop(rex, prop, scan, reginfo, NULL); PerlIO_printf(Perl_debug_log, "%3"IVdf":%*s%s(%"IVdf")\n", @@ -4013,8 +5242,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) assert(nextchr < 256 && (nextchr >= 0 || nextchr == NEXTCHR_EOS)); switch (state_num) { - case BOL: /* /^../ */ - case SBOL: /* /^../s */ + case SBOL: /* /^../ and /\A../ */ if (locinput == reginfo->strbeg) break; sayNO; @@ -4038,23 +5266,21 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) rex->offs[0].start = locinput - reginfo->strbeg; PUSH_STATE_GOTO(KEEPS_next, next, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case KEEPS_next_fail: /* rollback the start point change */ rex->offs[0].start = st->u.keeper.val; sayNO_SILENT; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case MEOL: /* /..$/m */ if (!NEXTCHR_IS_EOS && nextchr != '\n') sayNO; break; - case EOL: /* /..$/ */ - /* FALLTHROUGH */ - case SEOL: /* /..$/s */ + case SEOL: /* /..$/ */ if (!NEXTCHR_IS_EOS && nextchr != '\n') sayNO; if (reginfo->strend - locinput > 1) @@ -4071,12 +5297,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) sayNO; goto increment_locinput; - case CANY: /* \C */ - if (NEXTCHR_IS_EOS) - sayNO; - locinput++; - break; - case REG_ANY: /* /./ */ if ((NEXTCHR_IS_EOS) || nextchr == '\n') sayNO; @@ -4097,7 +5317,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) ); sayNO_SILENT; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } /* FALLTHROUGH */ case TRIE: /* (ab|cd) */ @@ -4155,6 +5375,19 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) HV * widecharmap = MUTABLE_HV(rexi->data->data[ ARG( scan ) + 1 ]); U32 state = trie->startstate; + if (scan->flags == EXACTL || scan->flags == EXACTFLU8) { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (utf8_target + && UTF8_IS_ABOVE_LATIN1(nextchr) + && scan->flags == EXACTL) + { + /* We only output for EXACTL, as we let the folder + * output this message for EXACTFLU8 to avoid + * duplication */ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(locinput, + reginfo->strend); + } + } if ( trie->bitmap && (NEXTCHR_IS_EOS || !TRIE_BITMAP_TEST(trie, nextchr))) { @@ -4285,7 +5518,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) goto trie_first_try; /* jump into the fail handler */ }} /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case TRIE_next_fail: /* we failed - try next alternative */ { @@ -4400,14 +5633,14 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) if (ST.accepted > 1 || has_cutgroup) { PUSH_STATE_GOTO(TRIE_next, scan, (char*)uc); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } /* only one choice left - just continue */ DEBUG_EXECUTE_r({ AV *const trie_words = MUTABLE_AV(rexi->data->data[ARG(ST.me)+TRIE_WORDS_OFFSET]); - SV ** const tmp = av_fetch( trie_words, - ST.nextword-1, 0 ); + SV ** const tmp = trie_words + ? av_fetch(trie_words, ST.nextword - 1, 0) : NULL; SV *sv= tmp ? sv_newmortal() : NULL; PerlIO_printf( Perl_debug_log, @@ -4425,10 +5658,22 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) locinput = (char*)uc; continue; /* execute rest of RE */ /* NOTREACHED */ - assert(0); } #undef ST + case EXACTL: /* /abc/l */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + /* Complete checking would involve going through every character + * matched by the string to see if any is above latin1. But the + * comparision otherwise might very well be a fast assembly + * language routine, and I (khw) don't think slowing things down + * just to check for this warning is worth it. So this just checks + * the first character */ + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*locinput)) { + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(locinput, reginfo->strend); + } + /* FALLTHROUGH */ case EXACT: { /* /abc/ */ char *s = STRING(scan); ln = STR_LEN(scan); @@ -4460,7 +5705,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) l++; } else { - if (TWO_BYTE_UTF8_TO_NATIVE(*l, *(l+1)) != * (U8*) s) + if (EIGHT_BIT_UTF8_TO_NATIVE(*l, *(l+1)) != * (U8*) s) { sayNO; } @@ -4484,7 +5729,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) s++; } else { - if (TWO_BYTE_UTF8_TO_NATIVE(*s, *(s+1)) != * (U8*) l) + if (EIGHT_BIT_UTF8_TO_NATIVE(*s, *(s+1)) != * (U8*) l) { sayNO; } @@ -4515,11 +5760,24 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) const char * s; U32 fold_utf8_flags; + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; folder = foldEQ_locale; fold_array = PL_fold_locale; fold_utf8_flags = FOLDEQ_LOCALE; goto do_exactf; + case EXACTFLU8: /* /abc/il; but all 'abc' are above 255, so + is effectively /u; hence to match, target + must be UTF-8. */ + if (! utf8_target) { + sayNO; + } + fold_utf8_flags = FOLDEQ_LOCALE | FOLDEQ_S1_ALREADY_FOLDED + | FOLDEQ_S1_FOLDS_SANE; + folder = foldEQ_latin1; + fold_array = PL_fold_latin1; + goto do_exactf; + case EXACTFU_SS: /* /\x{df}/iu */ case EXACTFU: /* /abc/iu */ folder = foldEQ_latin1; @@ -4582,93 +5840,287 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) break; } - /* XXX Could improve efficiency by separating these all out using a - * macro or in-line function. At that point regcomp.c would no longer - * have to set the FLAGS fields of these */ - case BOUNDL: /* /\b/l */ case NBOUNDL: /* /\B/l */ + to_complement = 1; + /* FALLTHROUGH */ + + case BOUNDL: /* /\b/l */ + { + bool b1, b2; + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (FLAGS(scan) != TRADITIONAL_BOUND) { + if (! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + B_ON_NON_UTF8_LOCALE_IS_WRONG); + } + goto boundu; + } + + if (utf8_target) { + if (locinput == reginfo->strbeg) + b1 = isWORDCHAR_LC('\n'); + else { + b1 = isWORDCHAR_LC_utf8(reghop3((U8*)locinput, -1, + (U8*)(reginfo->strbeg))); + } + b2 = (NEXTCHR_IS_EOS) + ? isWORDCHAR_LC('\n') + : isWORDCHAR_LC_utf8((U8*)locinput); + } + else { /* Here the string isn't utf8 */ + b1 = (locinput == reginfo->strbeg) + ? isWORDCHAR_LC('\n') + : isWORDCHAR_LC(UCHARAT(locinput - 1)); + b2 = (NEXTCHR_IS_EOS) + ? isWORDCHAR_LC('\n') + : isWORDCHAR_LC(nextchr); + } + if (to_complement ^ (b1 == b2)) { + sayNO; + } + break; + } + + case NBOUND: /* /\B/ */ + to_complement = 1; + /* FALLTHROUGH */ + case BOUND: /* /\b/ */ - case BOUNDU: /* /\b/u */ + if (utf8_target) { + goto bound_utf8; + } + goto bound_ascii_match_only; + + case NBOUNDA: /* /\B/a */ + to_complement = 1; + /* FALLTHROUGH */ + case BOUNDA: /* /\b/a */ - case NBOUND: /* /\B/ */ + { + bool b1, b2; + + bound_ascii_match_only: + /* Here the string isn't utf8, or is utf8 and only ascii characters + * are to match \w. In the latter case looking at the byte just + * prior to the current one may be just the final byte of a + * multi-byte character. This is ok. There are two cases: + * 1) it is a single byte character, and then the test is doing + * just what it's supposed to. + * 2) it is a multi-byte character, in which case the final byte is + * never mistakable for ASCII, and so the test will say it is + * not a word character, which is the correct answer. */ + b1 = (locinput == reginfo->strbeg) + ? isWORDCHAR_A('\n') + : isWORDCHAR_A(UCHARAT(locinput - 1)); + b2 = (NEXTCHR_IS_EOS) + ? isWORDCHAR_A('\n') + : isWORDCHAR_A(nextchr); + if (to_complement ^ (b1 == b2)) { + sayNO; + } + break; + } + case NBOUNDU: /* /\B/u */ - case NBOUNDA: /* /\B/a */ - /* was last char in word? */ - if (utf8_target - && FLAGS(scan) != REGEX_ASCII_RESTRICTED_CHARSET - && FLAGS(scan) != REGEX_ASCII_MORE_RESTRICTED_CHARSET) - { - if (locinput == reginfo->strbeg) - ln = '\n'; - else { - const U8 * const r = - reghop3((U8*)locinput, -1, (U8*)(reginfo->strbeg)); + to_complement = 1; + /* FALLTHROUGH */ - ln = utf8n_to_uvchr(r, (U8*) reginfo->strend - r, - 0, uniflags); - } - if (FLAGS(scan) != REGEX_LOCALE_CHARSET) { - ln = isWORDCHAR_uni(ln); - if (NEXTCHR_IS_EOS) - n = 0; - else { - LOAD_UTF8_CHARCLASS_ALNUM(); - n = swash_fetch(PL_utf8_swash_ptrs[_CC_WORDCHAR], (U8*)locinput, - utf8_target); + case BOUNDU: /* /\b/u */ + + boundu: + if (UNLIKELY(reginfo->strbeg >= reginfo->strend)) { + match = FALSE; + } + else if (utf8_target) { + bound_utf8: + switch((bound_type) FLAGS(scan)) { + case TRADITIONAL_BOUND: + { + bool b1, b2; + b1 = (locinput == reginfo->strbeg) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_utf8(reghop3((U8*)locinput, -1, + (U8*)(reginfo->strbeg))); + b2 = (NEXTCHR_IS_EOS) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_utf8((U8*)locinput); + match = cBOOL(b1 != b2); + break; } - } - else { - ln = isWORDCHAR_LC_uvchr(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_LC_utf8((U8*)locinput); - } + case GCB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; /* GCB always matches at begin and + end */ + } + else { + /* Find the gcb values of previous and current + * chars, then see if is a break point */ + match = isGCB(getGCB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getGCB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend)); + } + break; + + case LB_BOUND: + if (locinput == reginfo->strbeg) { + match = FALSE; + } + else if (NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isLB(getLB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getLB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case SB_BOUND: /* Always matches at begin and end */ + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isSB(getSB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getSB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case WB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isWB(WB_UNKNOWN, + getWB_VAL_UTF8( + reghop3((U8*)locinput, + -1, + (U8*)(reginfo->strbeg)), + (U8*) reginfo->strend), + getWB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + } } - else { + else { /* Not utf8 target */ + switch((bound_type) FLAGS(scan)) { + case TRADITIONAL_BOUND: + { + bool b1, b2; + b1 = (locinput == reginfo->strbeg) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_L1(UCHARAT(locinput - 1)); + b2 = (NEXTCHR_IS_EOS) + ? 0 /* isWORDCHAR_L1('\n') */ + : isWORDCHAR_L1(nextchr); + match = cBOOL(b1 != b2); + break; + } - /* Here the string isn't utf8, or is utf8 and only ascii - * characters are to match \w. In the latter case looking at - * the byte just prior to the current one may be just the final - * byte of a multi-byte character. This is ok. There are two - * cases: - * 1) it is a single byte character, and then the test is doing - * just what it's supposed to. - * 2) it is a multi-byte character, in which case the final - * byte is never mistakable for ASCII, and so the test - * will say it is not a word character, which is the - * correct answer. */ - ln = (locinput != reginfo->strbeg) ? - UCHARAT(locinput - 1) : '\n'; - switch (FLAGS(scan)) { - case REGEX_UNICODE_CHARSET: - ln = isWORDCHAR_L1(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_L1(nextchr); - break; - case REGEX_LOCALE_CHARSET: - ln = isWORDCHAR_LC(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_LC(nextchr); - break; - case REGEX_DEPENDS_CHARSET: - ln = isWORDCHAR(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR(nextchr); - break; - case REGEX_ASCII_RESTRICTED_CHARSET: - case REGEX_ASCII_MORE_RESTRICTED_CHARSET: - ln = isWORDCHAR_A(ln); - n = NEXTCHR_IS_EOS ? 0 : isWORDCHAR_A(nextchr); - break; - default: - Perl_croak(aTHX_ "panic: Unexpected FLAGS %u in op %u", FLAGS(scan), OP(scan)); - } + case GCB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; /* GCB always matches at begin and + end */ + } + else { /* Only CR-LF combo isn't a GCB in 0-255 + range */ + match = UCHARAT(locinput - 1) != '\r' + || UCHARAT(locinput) != '\n'; + } + break; + + case LB_BOUND: + if (locinput == reginfo->strbeg) { + match = FALSE; + } + else if (NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isLB(getLB_VAL_CP(UCHARAT(locinput -1)), + getLB_VAL_CP(UCHARAT(locinput)), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case SB_BOUND: /* Always matches at begin and end */ + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isSB(getSB_VAL_CP(UCHARAT(locinput -1)), + getSB_VAL_CP(UCHARAT(locinput)), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + + case WB_BOUND: + if (locinput == reginfo->strbeg || NEXTCHR_IS_EOS) { + match = TRUE; + } + else { + match = isWB(WB_UNKNOWN, + getWB_VAL_CP(UCHARAT(locinput -1)), + getWB_VAL_CP(UCHARAT(locinput)), + (U8*) reginfo->strbeg, + (U8*) locinput, + (U8*) reginfo->strend, + utf8_target); + } + break; + } } - /* Note requires that all BOUNDs be lower than all NBOUNDs in - * regcomp.sym */ - if (((!ln) == (!n)) == (OP(scan) < NBOUND)) - sayNO; + + if (to_complement ^ ! match) { + sayNO; + } break; - case ANYOF: /* /[abc]/ */ + case ANYOFL: /* /[abc]/l */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (ANYOFL_UTF8_LOCALE_REQD(FLAGS(scan)) && ! IN_UTF8_CTYPE_LOCALE) + { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), utf8_locale_required); + } + /* FALLTHROUGH */ + case ANYOFD: /* /[abc]/d */ + case ANYOF: /* /[abc]/ */ if (NEXTCHR_IS_EOS) sayNO; - if (utf8_target) { + if (utf8_target && ! UTF8_IS_INVARIANT(*locinput)) { if (!reginclass(rex, scan, (U8*)locinput, (U8*)reginfo->strend, utf8_target)) sayNO; @@ -4689,6 +6141,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* FALLTHROUGH */ case POSIXL: /* \w or [:punct:] etc. under /l */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; if (NEXTCHR_IS_EOS) sayNO; @@ -4702,14 +6155,15 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else if (UTF8_IS_DOWNGRADEABLE_START(nextchr)) { if (! (to_complement ^ cBOOL(isFOO_lc(FLAGS(scan), - (U8) TWO_BYTE_UTF8_TO_NATIVE(nextchr, - *(locinput + 1)))))) + EIGHT_BIT_UTF8_TO_NATIVE(nextchr, + *(locinput + 1)))))) { sayNO; } } else { /* Here, must be an above Latin-1 code point */ - goto utf8_posix_not_eos; + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(locinput, reginfo->strend); + goto utf8_posix_above_latin1; } /* Here, must be utf8 */ @@ -4768,7 +6222,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) if (NEXTCHR_IS_EOS) { sayNO; } - utf8_posix_not_eos: /* Use _generic_isCC() for characters within Latin1. (Note that * UTF8_IS_INVARIANT works even on non-UTF-8 strings, or else @@ -4783,7 +6236,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else if (UTF8_IS_DOWNGRADEABLE_START(nextchr)) { if (! (to_complement - ^ cBOOL(_generic_isCC(TWO_BYTE_UTF8_TO_NATIVE(nextchr, + ^ cBOOL(_generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(nextchr, *(locinput + 1)), FLAGS(scan))))) { @@ -4792,6 +6245,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) locinput += 2; } else { /* Handle above Latin-1 code points */ + utf8_posix_above_latin1: classnum = (_char_class_number) FLAGS(scan); if (classnum < _FIRST_NON_SWASH_CC) { @@ -4814,10 +6268,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else { /* Here, uses macros to find above Latin-1 code points */ switch (classnum) { - case _CC_ENUM_SPACE: /* XXX would require separate - code if we revert the change - of \v matching this */ - case _CC_ENUM_PSXSPC: + case _CC_ENUM_SPACE: if (! (to_complement ^ cBOOL(is_XPERLSPACE_high(locinput)))) { @@ -4859,38 +6310,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) case CLUMP: /* Match \X: logical Unicode character. This is defined as a Unicode extended Grapheme Cluster */ - /* From http://www.unicode.org/reports/tr29 (5.2 version). An - extended Grapheme Cluster is: - - CR LF - | Prepend* Begin Extend* - | . - - Begin is: ( Special_Begin | ! Control ) - Special_Begin is: ( Regional-Indicator+ | Hangul-syllable ) - Extend is: ( Grapheme_Extend | Spacing_Mark ) - Control is: [ GCB_Control | CR | LF ] - Hangul-syllable is: ( T+ | ( L* ( L | ( LVT | ( V | LV ) V* ) T* ) )) - - If we create a 'Regular_Begin' = Begin - Special_Begin, then - we can rewrite - - Begin is ( Regular_Begin + Special Begin ) - - It turns out that 98.4% of all Unicode code points match - Regular_Begin. Doing it this way eliminates a table match in - the previous implementation for almost all Unicode code points. - - There is a subtlety with Prepend* which showed up in testing. - Note that the Begin, and only the Begin is required in: - | Prepend* Begin Extend* - Also, Begin contains '! Control'. A Prepend must be a - '! Control', which means it must also be a Begin. What it - comes down to is that if we match Prepend* and then find no - suitable Begin afterwards, that if we backtrack the last - Prepend, that one will be a suitable Begin. - */ - if (NEXTCHR_IS_EOS) sayNO; if (! utf8_target) { @@ -4908,147 +6327,27 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) } else { - /* Utf8: See if is ( CR LF ); already know that locinput < - * reginfo->strend, so locinput+1 is in bounds */ - if ( nextchr == '\r' && locinput+1 < reginfo->strend - && UCHARAT(locinput + 1) == '\n') - { - locinput += 2; - } - else { - STRLEN len; - - /* In case have to backtrack to beginning, then match '.' */ - char *starting = locinput; - - /* In case have to backtrack the last prepend */ - char *previous_prepend = NULL; - - LOAD_UTF8_CHARCLASS_GCB(); - - /* Match (prepend)* */ - while (locinput < reginfo->strend - && (len = is_GCB_Prepend_utf8(locinput))) - { - previous_prepend = locinput; - locinput += len; - } - - /* As noted above, if we matched a prepend character, but - * the next thing won't match, back off the last prepend we - * matched, as it is guaranteed to match the begin */ - if (previous_prepend - && (locinput >= reginfo->strend - || (! swash_fetch(PL_utf8_X_regular_begin, - (U8*)locinput, utf8_target) - && ! is_GCB_SPECIAL_BEGIN_START_utf8(locinput))) - ) - { - locinput = previous_prepend; - } + /* Get the gcb type for the current character */ + GCB_enum prev_gcb = getGCB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend); - /* Note that here we know reginfo->strend > locinput, as we - * tested that upon input to this switch case, and if we - * moved locinput forward, we tested the result just above - * and it either passed, or we backed off so that it will - * now pass */ - if (swash_fetch(PL_utf8_X_regular_begin, - (U8*)locinput, utf8_target)) { - locinput += UTF8SKIP(locinput); + /* Then scan through the input until we get to the first + * character whose type is supposed to be a gcb with the + * current character. (There is always a break at the + * end-of-input) */ + locinput += UTF8SKIP(locinput); + while (locinput < reginfo->strend) { + GCB_enum cur_gcb = getGCB_VAL_UTF8((U8*) locinput, + (U8*) reginfo->strend); + if (isGCB(prev_gcb, cur_gcb)) { + break; } - else if (! is_GCB_SPECIAL_BEGIN_START_utf8(locinput)) { - - /* Here did not match the required 'Begin' in the - * second term. So just match the very first - * character, the '.' of the final term of the regex */ - locinput = starting + UTF8SKIP(starting); - goto exit_utf8; - } else { - - /* Here is a special begin. It can be composed of - * several individual characters. One possibility is - * RI+ */ - if ((len = is_GCB_RI_utf8(locinput))) { - locinput += len; - while (locinput < reginfo->strend - && (len = is_GCB_RI_utf8(locinput))) - { - locinput += len; - } - } else if ((len = is_GCB_T_utf8(locinput))) { - /* Another possibility is T+ */ - locinput += len; - while (locinput < reginfo->strend - && (len = is_GCB_T_utf8(locinput))) - { - locinput += len; - } - } else { - - /* Here, neither RI+ nor T+; must be some other - * Hangul. That means it is one of the others: L, - * LV, LVT or V, and matches: - * L* (L | LVT T* | V * V* T* | LV V* T*) */ - - /* Match L* */ - while (locinput < reginfo->strend - && (len = is_GCB_L_utf8(locinput))) - { - locinput += len; - } - /* Here, have exhausted L*. If the next character - * is not an LV, LVT nor V, it means we had to have - * at least one L, so matches L+ in the original - * equation, we have a complete hangul syllable. - * Are done. */ - - if (locinput < reginfo->strend - && is_GCB_LV_LVT_V_utf8(locinput)) - { - /* Otherwise keep going. Must be LV, LVT or V. - * See if LVT, by first ruling out V, then LV */ - if (! is_GCB_V_utf8(locinput) - /* All but every TCount one is LV */ - && (valid_utf8_to_uvchr((U8 *) locinput, - NULL) - - SBASE) - % TCount != 0) - { - locinput += UTF8SKIP(locinput); - } else { - - /* Must be V or LV. Take it, then match - * V* */ - locinput += UTF8SKIP(locinput); - while (locinput < reginfo->strend - && (len = is_GCB_V_utf8(locinput))) - { - locinput += len; - } - } + prev_gcb = cur_gcb; + locinput += UTF8SKIP(locinput); + } - /* And any of LV, LVT, or V can be followed - * by T* */ - while (locinput < reginfo->strend - && (len = is_GCB_T_utf8(locinput))) - { - locinput += len; - } - } - } - } - /* Match any extender */ - while (locinput < reginfo->strend - && swash_fetch(PL_utf8_X_extend, - (U8*)locinput, utf8_target)) - { - locinput += UTF8SKIP(locinput); - } - } - exit_utf8: - if (locinput > reginfo->strend) sayNO; } break; @@ -5065,6 +6364,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) const U8 *fold_array; UV utf8_fold_flags; + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; folder = foldEQ_locale; fold_array = PL_fold_locale; type = REFFL; @@ -5109,6 +6409,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) goto do_nref_ref_common; case REFFL: /* /\1/il */ + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; folder = foldEQ_locale; fold_array = PL_fold_locale; utf8_fold_flags = FOLDEQ_LOCALE; @@ -5192,9 +6493,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) case TAIL: /* placeholder while compiling (A|B|C) */ break; - case BACK: /* ??? doesn't appear to be used ??? */ - break; - #undef ST #define ST st->u.eval { @@ -5233,9 +6531,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* and then jump to the code we share with EVAL */ goto eval_recurse_doit; - /* NOTREACHED */ - assert(0); case EVAL: /* /(?{A})B/ /(??{A})B/ and /(?(?{A})X|Y)B/ */ if (cur_eval && cur_eval->locinput==locinput) { @@ -5294,6 +6590,13 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) U8 flags = (CXp_SUB_RE | ((newcv == caller_cv) ? CXp_SUB_RE_FAKE : 0)); if (last_pushed_cv) { + /* PUSH/POP_MULTICALL save and restore the + * caller's PL_comppad; if we call multiple subs + * using the same CX block, we have to save and + * unwind the varying PL_comppad's ourselves, + * especially restoring the right PL_comppad on + * backtrack - so save it on the save stack */ + SAVECOMPPAD(); CHANGE_MULTICALL_FLAGS(newcv, flags); } else { @@ -5324,7 +6627,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) assert(o->op_targ == OP_LEAVE); o = cUNOPo->op_first; assert(o->op_type == OP_ENTER); - o = OP_SIBLING(o); + o = OpSIBLING(o); } if (o->op_type != OP_STUB) { @@ -5450,8 +6753,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) assert(!(scan->flags & ~RXf_PMf_COMPILETIME)); re_sv = rex->engine->op_comp(aTHX_ &ret, 1, NULL, rex->engine, NULL, NULL, - /* copy /msix etc to inner pattern */ - scan->flags, + /* copy /msixn etc to inner pattern */ + ARG2L(scan), pm_flags); if (!(SvFLAGS(ret) @@ -5487,7 +6790,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) maxopenparen = 0; /* run the pattern returned from (??{...}) */ - eval_recurse_doit: /* Share code with GOSUB below this line + eval_recurse_doit: /* Share code with GOSUB below this line * At this point we expect the stack context to be * set up correctly */ @@ -5517,7 +6820,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) /* now continue from first node in postoned RE */ PUSH_YES_STATE_GOTO(EVAL_AB, startpoint, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } case EVAL_AB: /* cleanup after a successful (??{A})B */ @@ -5605,7 +6908,9 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) break; case ACCEPT: /* (*ACCEPT) */ - if (ARG(scan)){ + if (scan->flags) + sv_yes_mark = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); + if (ARG2L(scan)){ regnode *cursor; for (cursor=scan; cursor && OP(cursor)!=END; @@ -5777,21 +7082,21 @@ NULL PUSH_YES_STATE_GOTO(CURLYX_end, PREVOPER(next), locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } case CURLYX_end: /* just finished matching all of A*B */ cur_curlyx = ST.prev_curlyx; sayYES; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CURLYX_end_fail: /* just failed to match all of A*B */ regcpblow(ST.cp); cur_curlyx = ST.prev_curlyx; sayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ #undef ST @@ -5830,7 +7135,7 @@ NULL PUSH_STATE_GOTO(WHILEM_A_pre, A, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } /* If degenerate A matches "", assume A done. */ @@ -5943,7 +7248,7 @@ NULL PUSH_YES_STATE_GOTO(WHILEM_B_min, ST.save_curlyx->u.curlyx.B, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } /* Prefer A over B for maximal matching. */ @@ -5955,19 +7260,19 @@ NULL REGCP_SET(ST.lastcp); PUSH_STATE_GOTO(WHILEM_A_max, A, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } goto do_whilem_B_max; } /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case WHILEM_B_min: /* just matched B in a minimal match */ case WHILEM_B_max: /* just matched B in a maximal match */ cur_curlyx = ST.save_curlyx; sayYES; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case WHILEM_B_max_fail: /* just failed to match B in a maximal match */ cur_curlyx = ST.save_curlyx; @@ -5975,7 +7280,7 @@ NULL cur_curlyx->u.curlyx.count--; CACHEsayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case WHILEM_A_min_fail: /* just failed to match A in a minimal match */ /* FALLTHROUGH */ @@ -5986,7 +7291,7 @@ NULL cur_curlyx->u.curlyx.count--; CACHEsayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case WHILEM_A_max_fail: /* just failed to match A in a maximal match */ REGCP_UNWIND(ST.lastcp); @@ -6013,7 +7318,7 @@ NULL PUSH_YES_STATE_GOTO(WHILEM_B_max, ST.save_curlyx->u.curlyx.B, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case WHILEM_B_min_fail: /* just failed to match B in a minimal match */ cur_curlyx = ST.save_curlyx; @@ -6048,7 +7353,7 @@ NULL /*A*/ NEXTOPER(ST.save_curlyx->u.curlyx.me) + EXTRA_STEP_2ARGS, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ #undef ST #define ST st->u.branch @@ -6074,14 +7379,15 @@ NULL PUSH_STATE_GOTO(BRANCH_next, scan, locinput); } /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CUTGROUP: /* /(*THEN)/ */ - sv_yes_mark = st->u.mark.mark_name = scan->flags ? NULL : - MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); + sv_yes_mark = st->u.mark.mark_name = scan->flags + ? MUTABLE_SV(rexi->data->data[ ARG( scan ) ]) + : NULL; PUSH_STATE_GOTO(CUTGROUP_next, next, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CUTGROUP_next_fail: do_cutgroup = 1; @@ -6090,12 +7396,12 @@ NULL sv_commit = st->u.mark.mark_name; sayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case BRANCH_next: sayYES; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case BRANCH_next_fail: /* that branch failed; try the next, if any */ if (do_cutgroup) { @@ -6118,7 +7424,6 @@ NULL } continue; /* execute next BRANCH[J] op */ /* NOTREACHED */ - assert(0); case MINMOD: /* next op will be non-greedy, e.g. A*? */ minmod = 1; @@ -6163,7 +7468,7 @@ NULL curlym_do_A: /* execute the A in /A{m,n}B/ */ PUSH_YES_STATE_GOTO(CURLYM_A, ST.A, locinput); /* match A */ /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CURLYM_A: /* we've just matched an A */ ST.count++; @@ -6300,7 +7605,7 @@ NULL PUSH_STATE_GOTO(CURLYM_B, ST.B, locinput); /* match B */ /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CURLYM_B_fail: /* just failed to match a B */ REGCP_UNWIND(ST.cp); @@ -6479,7 +7784,7 @@ NULL goto curly_try_B_max; } /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CURLY_B_min_known_fail: /* failed to find B in a non-greedy match where c1,c2 valid */ @@ -6555,7 +7860,7 @@ NULL PUSH_STATE_GOTO(CURLY_B_min_known, ST.B, locinput); } /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case CURLY_B_min_fail: /* failed to find B in a non-greedy match where c1,c2 invalid */ @@ -6588,9 +7893,9 @@ NULL } sayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ - curly_try_B_max: + curly_try_B_max: /* a successful greedy match: now try to match B */ if (cur_eval && cur_eval->u.eval.close_paren && cur_eval->u.eval.close_paren == (U32)ST.paren) { @@ -6619,7 +7924,7 @@ NULL CURLY_SETPAREN(ST.paren, ST.count); PUSH_STATE_GOTO(CURLY_B_max, ST.B, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } } /* FALLTHROUGH */ @@ -6640,7 +7945,7 @@ NULL #undef ST case END: /* last op of main pattern */ - fake_end: + fake_end: if (cur_eval) { /* we've just finished A in /(??{A})B/; now continue with B */ @@ -6739,7 +8044,7 @@ NULL /* execute body of (?...A) */ PUSH_YES_STATE_GOTO(IFMATCH_A, NEXTOPER(NEXTOPER(scan)), newstart); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } case IFMATCH_A_fail: /* body of (?...A) failed */ @@ -6776,20 +8081,32 @@ NULL /* FALLTHROUGH */ case PRUNE: /* (*PRUNE) */ - if (!scan->flags) + if (scan->flags) sv_yes_mark = sv_commit = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); PUSH_STATE_GOTO(COMMIT_next, next, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case COMMIT_next_fail: no_final = 1; /* FALLTHROUGH */ + sayNO; + NOT_REACHED; /* NOTREACHED */ case OPFAIL: /* (*FAIL) */ - sayNO; + if (scan->flags) + sv_commit = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]); + if (logical) { + /* deal with (?(?!)X|Y) properly, + * make sure we trigger the no branch + * of the trailing IFTHEN structure*/ + sw= 0; + break; + } else { + sayNO; + } /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ #define ST st->u.mark case MARKPOINT: /* (*MARK:foo) */ @@ -6800,13 +8117,13 @@ NULL ST.mark_loc = locinput; PUSH_YES_STATE_GOTO(MARKPOINT_next, next, locinput); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case MARKPOINT_next: mark_state = ST.prev_mark; sayYES; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case MARKPOINT_next_fail: if (popmark && sv_eq(ST.mark_name,popmark)) @@ -6828,10 +8145,10 @@ NULL mark_state->u.mark.mark_name : NULL; sayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ case SKIP: /* (*SKIP) */ - if (scan->flags) { + if (!scan->flags) { /* (*SKIP) : if we fail we cut here*/ ST.mark_name = NULL; ST.mark_loc = locinput; @@ -6874,7 +8191,7 @@ NULL no_final = 1; sayNO; /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ #undef ST case LNBREAK: /* \R */ @@ -6891,7 +8208,7 @@ NULL /* this is a point to jump to in order to increment * locinput by one character */ - increment_locinput: + increment_locinput: assert(!NEXTCHR_IS_EOS); if (utf8_target) { locinput += PL_utf8skip[nextchr]; @@ -6909,7 +8226,6 @@ NULL scan = next; /* prepare to execute the next op and ... */ continue; /* ... jump back to the top, reusing st */ /* NOTREACHED */ - assert(0); push_yes_state: /* push a state that backtracks on success */ @@ -6953,7 +8269,6 @@ NULL st = newst; continue; /* NOTREACHED */ - assert(0); } } @@ -6964,8 +8279,9 @@ NULL Perl_croak(aTHX_ "corrupted regexp pointers"); /* NOTREACHED */ sayNO; + NOT_REACHED; /* NOTREACHED */ -yes: + yes: if (yes_state) { /* we have successfully completed a subexpression, but we must now * pop to the state marked by yes_state and continue from there */ @@ -7026,7 +8342,7 @@ yes: result = 1; goto final_exit; -no: + no: DEBUG_EXECUTE_r( PerlIO_printf(Perl_debug_log, "%*s %sfailed...%s\n", @@ -7034,7 +8350,7 @@ no: PL_colors[4], PL_colors[5]) ); -no_silent: + no_silent: if (no_final) { if (yes_state) { goto yes; @@ -7115,7 +8431,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, char *loceol = reginfo->strend; /* local version */ I32 hardcount = 0; /* How many matches so far */ bool utf8_target = reginfo->is_utf8_target; - int to_complement = 0; /* Invert the result? */ + unsigned int to_complement = 0; /* Invert the result? */ UV utf8_flags; _char_class_number classnum; #ifndef DEBUGGING @@ -7175,16 +8491,12 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, else scan = loceol; break; - case CANY: /* Move forward bytes, unless goes off end */ - if (utf8_target && loceol - scan > max) { - - /* hadn't been adjusted in the UTF-8 case */ - scan += max; - } - else { - scan = loceol; + case EXACTL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + if (utf8_target && UTF8_IS_ABOVE_LATIN1(*scan)) { + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(scan, loceol); } - break; + /* FALLTHROUGH */ case EXACT: assert(STR_LEN(p) == reginfo->is_utf8_pat ? UTF8SKIP(STRING(p)) : 1); @@ -7223,7 +8535,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, /* Target isn't utf8; convert the character in the UTF-8 * pattern to non-UTF8, and do a simple loop */ - c = TWO_BYTE_UTF8_TO_NATIVE(c, *(STRING(p) + 1)); + c = EIGHT_BIT_UTF8_TO_NATIVE(c, *(STRING(p) + 1)); while (scan < loceol && UCHARAT(scan) == c) { scan++; } @@ -7258,6 +8570,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, goto do_exactf; case EXACTFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; utf8_flags = FOLDEQ_LOCALE; goto do_exactf; @@ -7266,11 +8579,19 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, utf8_flags = 0; goto do_exactf; + case EXACTFLU8: + if (! utf8_target) { + break; + } + utf8_flags = FOLDEQ_LOCALE | FOLDEQ_S2_ALREADY_FOLDED + | FOLDEQ_S2_FOLDS_SANE; + goto do_exactf; + case EXACTFU_SS: case EXACTFU: utf8_flags = reginfo->is_utf8_pat ? FOLDEQ_S2_ALREADY_FOLDED : 0; - do_exactf: { + do_exactf: { int c1, c2; U8 c1_utf8[UTF8_MAXBYTES+1], c2_utf8[UTF8_MAXBYTES+1]; @@ -7329,6 +8650,14 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } break; } + case ANYOFL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + + if (ANYOFL_UTF8_LOCALE_REQD(FLAGS(p)) && ! IN_UTF8_CTYPE_LOCALE) { + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), utf8_locale_required); + } + /* FALLTHROUGH */ + case ANYOFD: case ANYOF: if (utf8_target) { while (hardcount < max @@ -7351,6 +8680,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, /* FALLTHROUGH */ case POSIXL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; if (! utf8_target) { while (scan < loceol && to_complement ^ cBOOL(isFOO_lc(FLAGS(p), *scan))) @@ -7427,7 +8757,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } } else { - utf8_posix: + utf8_posix: classnum = (_char_class_number) FLAGS(p); if (classnum < _FIRST_NON_SWASH_CC) { @@ -7449,7 +8779,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } else if (UTF8_IS_DOWNGRADEABLE_START(*scan)) { if (! (to_complement - ^ cBOOL(_generic_isCC(TWO_BYTE_UTF8_TO_NATIVE(*scan, + ^ cBOOL(_generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*scan, *(scan + 1)), classnum)))) { @@ -7470,11 +8800,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, * code is written for making the loops as tight as possible. * It could be refactored to save space instead */ switch (classnum) { - case _CC_ENUM_SPACE: /* XXX would require separate code - if we revert the change of \v - matching this */ - /* FALLTHROUGH */ - case _CC_ENUM_PSXSPC: + case _CC_ENUM_SPACE: while (hardcount < max && scan < loceol && (to_complement ^ cBOOL(isSPACE_utf8(scan)))) @@ -7570,16 +8896,18 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, } break; + case BOUNDL: + case NBOUNDL: + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + /* FALLTHROUGH */ case BOUND: case BOUNDA: - case BOUNDL: case BOUNDU: case EOS: case GPOS: case KEEPS: case NBOUND: case NBOUNDA: - case NBOUNDL: case NBOUNDU: case OPFAIL: case SBOL: @@ -7590,7 +8918,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, default: Perl_croak(aTHX_ "panic: regrepeat() called with unrecognized node type %d='%s'", OP(p), PL_reg_name[OP(p)]); /* NOTREACHED */ - assert(0); + NOT_REACHED; /* NOTREACHED */ } @@ -7604,7 +8932,7 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, GET_RE_DEBUG_FLAGS_DECL; DEBUG_EXECUTE_r({ SV * const prop = sv_newmortal(); - regprop(prog, prop, p, reginfo); + regprop(prog, prop, p, reginfo, NULL); PerlIO_printf(Perl_debug_log, "%*s %s can match %"IVdf" times out of %"IVdf"...\n", REPORT_CODE_OFF + depth*2, "", SvPVX_const(prop),(IV)c,(IV)max); @@ -7638,7 +8966,7 @@ Perl_regclass_swash(pTHX_ const regexp *prog, const regnode* node, bool doinit, /* - reginclass - determine if a character falls into a character class - n is the ANYOF regnode + n is the ANYOF-type regnode p is the target string p_end points to one byte beyond the end of the target string utf8_target tells whether p is in UTF-8. @@ -7672,25 +9000,33 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const * UTF8_ALLOW_FFFF */ if (c_len == (STRLEN)-1) Perl_croak(aTHX_ "Malformed UTF-8 character (fatal)"); + if (c > 255 && OP(n) == ANYOFL && ! ANYOFL_UTF8_LOCALE_REQD(flags)) { + _CHECK_AND_OUTPUT_WIDE_LOCALE_CP_MSG(c); + } } /* If this character is potentially in the bitmap, check it */ if (c < NUM_ANYOF_CODE_POINTS) { if (ANYOF_BITMAP_TEST(n, c)) match = TRUE; - else if (flags & ANYOF_NON_UTF8_NON_ASCII_ALL - && ! utf8_target - && ! isASCII(c)) + else if ((flags + & ANYOF_SHARED_d_MATCHES_ALL_NON_UTF8_NON_ASCII_non_d_WARN_SUPER) + && OP(n) == ANYOFD + && ! utf8_target + && ! isASCII(c)) { match = TRUE; } else if (flags & ANYOF_LOCALE_FLAGS) { - if (flags & ANYOF_LOC_FOLD) { - if (ANYOF_BITMAP_TEST(n, PL_fold_locale[c])) { - match = TRUE; - } + if ((flags & ANYOFL_FOLD) + && c < 256 + && ANYOF_BITMAP_TEST(n, PL_fold_locale[c])) + { + match = TRUE; } - if (! match && ANYOF_POSIXL_TEST_ANY_SET(n)) { + else if (ANYOF_POSIXL_TEST_ANY_SET(n) + && c < 256 + ) { /* The data structure is arranged so bits 0, 2, 4, ... are set * if the class includes the Posix character class given by @@ -7743,14 +9079,32 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const /* If the bitmap didn't (or couldn't) match, and something outside the * bitmap could match, try that. */ if (!match) { - if (c >= 256 && (flags & ANYOF_ABOVE_LATIN1_ALL)) { - match = TRUE; /* Everything above 255 matches */ + if (c >= NUM_ANYOF_CODE_POINTS + && (flags & ANYOF_MATCHES_ALL_ABOVE_BITMAP)) + { + match = TRUE; /* Everything above the bitmap matches */ } - else if ((flags & ANYOF_NONBITMAP_NON_UTF8) - || (utf8_target && (flags & ANYOF_UTF8)) - || ((flags & ANYOF_LOC_FOLD) - && IN_UTF8_CTYPE_LOCALE - && ARG(n) != ANYOF_NONBITMAP_EMPTY)) + /* Here doesn't match everything above the bitmap. If there is + * some information available beyond the bitmap, we may find a + * match in it. If so, this is most likely because the code point + * is outside the bitmap range. But rarely, it could be because of + * some other reason. If so, various flags are set to indicate + * this possibility. On ANYOFD nodes, there may be matches that + * happen only when the target string is UTF-8; or for other node + * types, because runtime lookup is needed, regardless of the + * UTF-8ness of the target string. Finally, under /il, there may + * be some matches only possible if the locale is a UTF-8 one. */ + else if ( ARG(n) != ANYOF_ONLY_HAS_BITMAP + && ( c >= NUM_ANYOF_CODE_POINTS + || ( (flags & ANYOF_SHARED_d_UPPER_LATIN1_UTF8_STRING_MATCHES_non_d_RUNTIME_USER_PROP) + && ( UNLIKELY(OP(n) != ANYOFD) + || (utf8_target && ! isASCII_uni(c) +# if NUM_ANYOF_CODE_POINTS > 256 + && c < 256 +# endif + ))) + || ( ANYOFL_SOME_FOLDS_ONLY_IN_UTF8_LOCALE(flags) + && IN_UTF8_CTYPE_LOCALE))) { SV* only_utf8_locale = NULL; SV * const sw = _get_regclass_nonbitmap_data(prog, n, TRUE, 0, @@ -7776,7 +9130,9 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const } if (UNICODE_IS_SUPER(c) - && (flags & ANYOF_WARN_SUPER) + && (flags + & ANYOF_SHARED_d_MATCHES_ALL_NON_UTF8_NON_ASCII_non_d_WARN_SUPER) + && OP(n) != ANYOFD && ckWARN_d(WARN_NON_UNICODE)) { Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), @@ -7815,6 +9171,9 @@ S_reghop3(U8 *s, SSize_t off, const U8* lim) if (UTF8_IS_CONTINUED(*s)) { while (s > lim && UTF8_IS_CONTINUATION(*s)) s--; + if (! UTF8_IS_START(*s)) { + Perl_croak_nocontext("Malformed UTF-8 character (fatal)"); + } } /* XXX could check well-formedness here */ } @@ -7839,6 +9198,9 @@ S_reghop4(U8 *s, SSize_t off, const U8* llim, const U8* rlim) if (UTF8_IS_CONTINUED(*s)) { while (s > llim && UTF8_IS_CONTINUATION(*s)) s--; + if (! UTF8_IS_START(*s)) { + Perl_croak_nocontext("Malformed UTF-8 character (fatal)"); + } } /* XXX could check well-formedness here */ } @@ -7868,6 +9230,9 @@ S_reghopmaybe3(U8* s, SSize_t off, const U8* lim) if (UTF8_IS_CONTINUED(*s)) { while (s > lim && UTF8_IS_CONTINUATION(*s)) s--; + if (! UTF8_IS_START(*s)) { + Perl_croak_nocontext("Malformed UTF-8 character (fatal)"); + } } /* XXX could check well-formedness here */ } @@ -8089,11 +9454,5 @@ S_to_byte_substr(pTHX_ regexp *prog) } /* - * Local variables: - * c-indentation-style: bsd - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * * ex: set ts=8 sts=4 sw=4 et: */