locinput = (p); \
SET_nextchr
-
-#define LOAD_UTF8_CHARCLASS(swash_ptr, property_name, invlist) STMT_START { \
- if (!swash_ptr) { \
- U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST; \
- swash_ptr = _core_swash_init("utf8", property_name, &PL_sv_undef, \
- 1, 0, invlist, &flags); \
- assert(swash_ptr); \
- } \
- } STMT_END
-
-/* If in debug mode, we test that a known character properly matches */
-#ifdef DEBUGGING
-# define LOAD_UTF8_CHARCLASS_DEBUG_TEST(swash_ptr, \
- property_name, \
- invlist, \
- utf8_char_in_property) \
- LOAD_UTF8_CHARCLASS(swash_ptr, property_name, invlist); \
- assert(swash_fetch(swash_ptr, (U8 *) utf8_char_in_property, TRUE));
-#else
-# define LOAD_UTF8_CHARCLASS_DEBUG_TEST(swash_ptr, \
- property_name, \
- invlist, \
- utf8_char_in_property) \
- LOAD_UTF8_CHARCLASS(swash_ptr, property_name, invlist)
-#endif
-
-#define LOAD_UTF8_CHARCLASS_ALNUM() LOAD_UTF8_CHARCLASS_DEBUG_TEST( \
- PL_utf8_swash_ptrs[_CC_WORDCHAR], \
- "", \
- PL_XPosix_ptrs[_CC_WORDCHAR], \
- LATIN_SMALL_LIGATURE_LONG_S_T_UTF8);
-
#define PLACEHOLDER /* Something for the preprocessor to grab onto */
/* TODO: Combine JUMPABLE and HAS_TEXT to cache OP(rn) */
/* Currently these are only used when PL_regkind[OP(rn)] == EXACT so
we don't need this definition. XXX These are now out-of-sync*/
#define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==REF || OP(rn)==NREF )
-#define IS_TEXTF(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn)==EXACTFA || OP(rn)==EXACTFA_NO_TRIE || OP(rn)==EXACTF || OP(rn)==REFF || OP(rn)==NREFF )
+#define IS_TEXTF(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn)==EXACTFAA || OP(rn)==EXACTFAA_NO_TRIE || OP(rn)==EXACTF || OP(rn)==REFF || OP(rn)==NREFF )
#define IS_TEXTFL(rn) ( OP(rn)==EXACTFL || OP(rn)==REFFL || OP(rn)==NREFFL )
#else
/* ... so we use this as its faster. */
#define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==EXACTL )
-#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFLU8 || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFA || OP(rn) == EXACTFA_NO_TRIE)
+#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFLU8 || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFAA || OP(rn) == EXACTFAA_NO_TRIE)
#define IS_TEXTF(rn) ( OP(rn)==EXACTF )
#define IS_TEXTFL(rn) ( OP(rn)==EXACTFL )
); \
regcpblow(cp)
+/* set the start and end positions of capture ix */
+#define CLOSE_CAPTURE(ix, s, e) \
+ rex->offs[ix].start = s; \
+ rex->offs[ix].end = e; \
+ if (ix > rex->lastparen) \
+ rex->lastparen = ix; \
+ rex->lastcloseparen = ix; \
+ DEBUG_BUFFERS_r(Perl_re_exec_indentf( aTHX_ \
+ "CLOSE: rex=0x%" UVxf " offs=0x%" UVxf ": \\%" UVuf ": set %" IVdf "..%" IVdf " max: %" UVuf "\n", \
+ depth, \
+ PTR2UV(rex), \
+ PTR2UV(rex->offs), \
+ (UV)ix, \
+ (IV)rex->offs[ix].start, \
+ (IV)rex->offs[ix].end, \
+ (UV)rex->lastparen \
+ ))
+
#define UNWIND_PAREN(lp, lcp) \
+ DEBUG_BUFFERS_r(Perl_re_exec_indentf( aTHX_ \
+ "UNWIND_PAREN: rex=0x%" UVxf " offs=0x%" UVxf ": invalidate (%" UVuf "..%" UVuf "] set lcp: %" UVuf "\n", \
+ depth, \
+ PTR2UV(rex), \
+ PTR2UV(rex->offs), \
+ (UV)(lp), \
+ (UV)(rex->lastparen), \
+ (UV)(lcp) \
+ )); \
for (n = rex->lastparen; n > lp; n--) \
rex->offs[n].end = -1; \
rex->lastparen = n; \
#endif
STATIC bool
-S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character)
+S_isFOO_utf8_lc(pTHX_ const U8 classnum, const U8* character, const U8* e)
{
/* Returns a boolean as to whether or not the (well-formed) UTF-8-encoded
* 'character' is a member of the Posix character class given by 'classnum'
EIGHT_BIT_UTF8_TO_NATIVE(*character, *(character + 1)));
}
- _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(character, character + UTF8SKIP(character));
-
- if (classnum < _FIRST_NON_SWASH_CC) {
-
- /* Initialize the swash unless done already */
- if (! PL_utf8_swash_ptrs[classnum]) {
- U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST;
- PL_utf8_swash_ptrs[classnum] =
- _core_swash_init("utf8",
- "",
- &PL_sv_undef, 1, 0,
- PL_XPosix_ptrs[classnum], &flags);
- }
-
- return cBOOL(swash_fetch(PL_utf8_swash_ptrs[classnum], (U8 *)
- character,
- TRUE /* is UTF */ ));
- }
+ _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(character, e);
switch ((_char_class_number) classnum) {
case _CC_ENUM_SPACE: return is_XPERLSPACE_high(character);
case _CC_ENUM_BLANK: return is_HORIZWS_high(character);
case _CC_ENUM_XDIGIT: return is_XDIGIT_high(character);
case _CC_ENUM_VERTSPACE: return is_VERTWS_high(character);
- default: break;
+ default:
+ return _invlist_contains_cp(PL_XPosix_ptrs[classnum],
+ utf8_to_uvchr_buf(character, e, NULL));
}
return FALSE; /* Things like CNTRL are always below 256 */
PERL_UINTMAX_T complemented = ~ * (PERL_UINTMAX_T *) s;
if (complemented & PERL_VARIANTS_WORD_MASK) {
-#if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 \
- || BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
+# if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 \
+ || BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
s += _variant_byte_number(complemented);
return s;
-#else /* If weird byte order, drop into next loop to do byte-at-a-time
+# else /* If weird byte order, drop into next loop to do byte-at-a-time
checks. */
break;
-#endif
+# endif
}
s += PERL_WORDSIZE;
}
+STATIC U8 *
+S_find_span_end(U8 * s, const U8 * send, const U8 span_byte)
+{
+ /* Returns the position of the first byte in the sequence between 's' and
+ * 'send-1' inclusive that isn't 'span_byte'; returns 'send' if none found.
+ * */
+
+ PERL_ARGS_ASSERT_FIND_SPAN_END;
+
+ assert(send >= s);
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+ PERL_UINTMAX_T span_word;
+
+ /* Process per-byte until reach word boundary. XXX This loop could be
+ * eliminated if we knew that this platform had fast unaligned reads */
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (*s != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ /* Create a word filled with the bytes we are spanning */
+ span_word = PERL_COUNT_MULTIPLIER * span_byte;
+
+ /* Process per-word as long as we have at least a full word left */
+ do {
+
+ /* Keep going if the whole word is composed of 'span_byte's */
+ if ((* (PERL_UINTMAX_T *) s) == span_word) {
+ s += PERL_WORDSIZE;
+ continue;
+ }
+
+ /* Here, at least one byte in the word isn't 'span_byte'. */
+
+#ifdef EBCDIC
+
+ break;
+
+#else
+
+ /* This xor leaves 1 bits only in those non-matching bytes */
+ span_word ^= * (PERL_UINTMAX_T *) s;
+
+ /* Make sure the upper bit of each non-matching byte is set. This
+ * makes each such byte look like an ASCII platform variant byte */
+ span_word |= span_word << 1;
+ span_word |= span_word << 2;
+ span_word |= span_word << 4;
+
+ /* That reduces the problem to what this function solves */
+ return s + _variant_byte_number(span_word);
+
+#endif
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+ /* Process the straggler bytes beyond the final word boundary */
+ while (s < send) {
+ if (*s != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ return s;
+}
+
+STATIC U8 *
+S_find_next_masked(U8 * s, const U8 * send, const U8 byte, const U8 mask)
+{
+ /* Returns the position of the first byte in the sequence between 's'
+ * and 'send-1' inclusive that when ANDed with 'mask' yields 'byte';
+ * returns 'send' if none found. It uses word-level operations instead of
+ * byte to speed up the process */
+
+ PERL_ARGS_ASSERT_FIND_NEXT_MASKED;
+
+ assert(send >= s);
+ assert((byte & mask) == byte);
+
+#ifndef EBCDIC
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+ PERL_UINTMAX_T word_complemented, mask_word;
+
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (((*s) & mask) == byte) {
+ return s;
+ }
+ s++;
+ }
+
+ word_complemented = ~ (PERL_COUNT_MULTIPLIER * byte);
+ mask_word = PERL_COUNT_MULTIPLIER * mask;
+
+ do {
+ PERL_UINTMAX_T masked = (* (PERL_UINTMAX_T *) s) & mask_word;
+
+ /* If 'masked' contains 'byte' within it, anding with the
+ * complement will leave those 8 bits 0 */
+ masked &= word_complemented;
+
+ /* This causes the most significant bit to be set to 1 for any
+ * bytes in the word that aren't completely 0 */
+ masked |= masked << 1;
+ masked |= masked << 2;
+ masked |= masked << 4;
+
+ /* The msbits are the same as what marks a byte as variant, so we
+ * can use this mask. If all msbits are 1, the word doesn't
+ * contain 'byte' */
+ if ((masked & PERL_VARIANTS_WORD_MASK) == PERL_VARIANTS_WORD_MASK) {
+ s += PERL_WORDSIZE;
+ continue;
+ }
+
+ /* Here, the msbit of bytes in the word that aren't 'byte' are 1,
+ * and any that are, are 0. Complement and re-AND to swap that */
+ masked = ~ masked;
+ masked &= PERL_VARIANTS_WORD_MASK;
+
+ /* This reduces the problem to that solved by this function */
+ s += _variant_byte_number(masked);
+ return s;
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+#endif
+
+ while (s < send) {
+ if (((*s) & mask) == byte) {
+ return s;
+ }
+ s++;
+ }
+
+ return s;
+}
+
+STATIC U8 *
+S_find_span_end_mask(U8 * s, const U8 * send, const U8 span_byte, const U8 mask)
+{
+ /* Returns the position of the first byte in the sequence between 's' and
+ * 'send-1' inclusive that when ANDed with 'mask' isn't 'span_byte'.
+ * 'span_byte' should have been ANDed with 'mask' in the call of this
+ * function. Returns 'send' if none found. Works like find_span_end(),
+ * except for the AND */
+
+ PERL_ARGS_ASSERT_FIND_SPAN_END_MASK;
+
+ assert(send >= s);
+ assert((span_byte & mask) == span_byte);
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+ PERL_UINTMAX_T span_word, mask_word;
+
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (((*s) & mask) != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ span_word = PERL_COUNT_MULTIPLIER * span_byte;
+ mask_word = PERL_COUNT_MULTIPLIER * mask;
+
+ do {
+ PERL_UINTMAX_T masked = (* (PERL_UINTMAX_T *) s) & mask_word;
+
+ if (masked == span_word) {
+ s += PERL_WORDSIZE;
+ continue;
+ }
+
+#ifdef EBCDIC
+
+ break;
+
+#else
+
+ masked ^= span_word;
+ masked |= masked << 1;
+ masked |= masked << 2;
+ masked |= masked << 4;
+ return s + _variant_byte_number(masked);
+
+#endif
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+ while (s < send) {
+ if (((*s) & mask) != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ return s;
+}
+
/*
* pregexec and friends
*/
if (check_len > targ_len) {
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
- "Anchored string too short...\n"));
+ "Target string too short to match required substring...\n"));
goto fail_finish;
}
end_point - check_len
)
+ check_len;
+ if (end_point < start_point)
+ goto fail_finish;
}
}
#define DECL_TRIE_TYPE(scan) \
const enum { trie_plain, trie_utf8, trie_utf8_fold, trie_latin_utf8_fold, \
trie_utf8_exactfa_fold, trie_latin_utf8_exactfa_fold, \
- trie_utf8l, trie_flu8 } \
+ trie_utf8l, trie_flu8, trie_flu8_latin } \
trie_type = ((scan->flags == EXACT) \
? (utf8_target ? trie_utf8 : trie_plain) \
: (scan->flags == EXACTL) \
? (utf8_target ? trie_utf8l : trie_plain) \
- : (scan->flags == EXACTFA) \
+ : (scan->flags == EXACTFAA) \
? (utf8_target \
? trie_utf8_exactfa_fold \
: trie_latin_utf8_exactfa_fold) \
: (scan->flags == EXACTFLU8 \
- ? trie_flu8 \
+ ? (utf8_target \
+ ? trie_flu8 \
+ : trie_flu8_latin) \
: (utf8_target \
? trie_utf8_fold \
- : trie_latin_utf8_fold)))
+ : trie_latin_utf8_fold)))
-#define REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc, uscan, len, uvc, charid, foldlen, foldbuf, uniflags) \
+/* 'uscan' is set to foldbuf, and incremented, so below the end of uscan is
+ * 'foldbuf+sizeof(foldbuf)' */
+#define REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc, uc_end, uscan, len, uvc, charid, foldlen, foldbuf, uniflags) \
STMT_START { \
STRLEN skiplen; \
U8 flags = FOLD_FLAGS_FULL; \
switch (trie_type) { \
case trie_flu8: \
_CHECK_AND_WARN_PROBLEMATIC_LOCALE; \
- if (utf8_target && UTF8_IS_ABOVE_LATIN1(*uc)) { \
- _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(uc, uc + UTF8SKIP(uc)); \
+ if (UTF8_IS_ABOVE_LATIN1(*uc)) { \
+ _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(uc, uc_end); \
} \
goto do_trie_utf8_fold; \
case trie_utf8_exactfa_fold: \
case trie_utf8_fold: \
do_trie_utf8_fold: \
if ( foldlen>0 ) { \
- uvc = utf8n_to_uvchr( (const U8*) uscan, UTF8_MAXLEN, &len, uniflags ); \
+ uvc = utf8n_to_uvchr( (const U8*) uscan, foldlen, &len, uniflags ); \
foldlen -= len; \
uscan += len; \
len=0; \
} else { \
- len = UTF8SKIP(uc); \
- uvc = _toFOLD_utf8_flags( (const U8*) uc, uc + len, foldbuf, &foldlen, \
+ uvc = _toFOLD_utf8_flags( (const U8*) uc, uc_end, foldbuf, &foldlen, \
flags); \
+ len = UTF8SKIP(uc); \
skiplen = UVCHR_SKIP( uvc ); \
foldlen -= skiplen; \
uscan = foldbuf + skiplen; \
} \
break; \
+ case trie_flu8_latin: \
+ _CHECK_AND_WARN_PROBLEMATIC_LOCALE; \
+ goto do_trie_latin_utf8_fold; \
case trie_latin_utf8_exactfa_fold: \
flags |= FOLD_FLAGS_NOMIX_ASCII; \
/* FALLTHROUGH */ \
case trie_latin_utf8_fold: \
+ do_trie_latin_utf8_fold: \
if ( foldlen>0 ) { \
- uvc = utf8n_to_uvchr( (const U8*) uscan, UTF8_MAXLEN, &len, uniflags ); \
+ uvc = utf8n_to_uvchr( (const U8*) uscan, foldlen, &len, uniflags ); \
foldlen -= len; \
uscan += len; \
len=0; \
} \
/* FALLTHROUGH */ \
case trie_utf8: \
- uvc = utf8n_to_uvchr( (const U8*) uc, UTF8_MAXLEN, &len, uniflags ); \
+ uvc = utf8n_to_uvchr( (const U8*) uc, uc_end - uc, &len, uniflags ); \
break; \
case trie_plain: \
uvc = (UV)*uc; \
dump_exec_pos(li,s,(reginfo->strend),(reginfo->strbeg), \
startpos, doutf8, depth)
-#define REXEC_FBC_EXACTISH_SCAN(COND) \
-STMT_START { \
- while (s <= e) { \
- if ( (COND) \
- && (ln == 1 || folder(s, pat_string, ln)) \
- && (reginfo->intuit || regtry(reginfo, &s)) )\
- goto got_it; \
- s++; \
- } \
-} STMT_END
-
-#define REXEC_FBC_UTF8_SCAN(CODE) \
-STMT_START { \
- while (s < strend) { \
- CODE \
- s += UTF8SKIP(s); \
- } \
-} STMT_END
-
-#define REXEC_FBC_SCAN(CODE) \
-STMT_START { \
- while (s < strend) { \
- CODE \
- s++; \
- } \
-} STMT_END
+#define REXEC_FBC_SCAN(UTF8, CODE) \
+ STMT_START { \
+ while (s < strend) { \
+ CODE \
+ s += ((UTF8) ? UTF8SKIP(s) : 1); \
+ } \
+ } STMT_END
-/* In the next few macros, 'try_it' is a bool indicating whether to actually
- * try the match or not. It is used for when the flags indicate that only the
- * first occurrence of 'x' in a string of them should be considered for
- * matching. try_it is initialized to 1, and set to 1 on every failure of the
- * condition, thus it will be 1 whenever a 'x' happens to be first. But when
- * the condition is met, and we don't exit the loop because we have ultimate
- * success, try_it is set to 'doevery', the latter being FALSE if we only want
- * the first in a string; otherwise TRUE, so try_it will be 0 when the previous
- * thing was 'x' and we only want the first 'x' */
-
-#define REXEC_FBC_UTF8_CLASS_SCAN(COND) \
-REXEC_FBC_UTF8_SCAN( /* Loops while (s < strend) */ \
- if (COND) { \
- if (try_it && (reginfo->intuit || regtry(reginfo, &s)))\
- goto got_it; \
- else \
- try_it = doevery; \
- } \
- else \
- try_it = 1; \
-)
+#define REXEC_FBC_CLASS_SCAN(UTF8, COND) \
+ STMT_START { \
+ while (s < strend) { \
+ REXEC_FBC_CLASS_SCAN_GUTS(UTF8, COND) \
+ } \
+ } STMT_END
-#define REXEC_FBC_CLASS_SCAN(COND) \
-REXEC_FBC_SCAN( /* Loops while (s < strend) */ \
+#define REXEC_FBC_CLASS_SCAN_GUTS(UTF8, COND) \
if (COND) { \
- if (try_it && (reginfo->intuit || regtry(reginfo, &s)))\
- goto got_it; \
- else \
- try_it = doevery; \
+ FBC_CHECK_AND_TRY \
+ s += ((UTF8) ? UTF8SKIP(s) : 1); \
+ previous_occurrence_end = s; \
} \
- else \
- try_it = 1; \
-)
+ else { \
+ s += ((UTF8) ? UTF8SKIP(s) : 1); \
+ }
#define REXEC_FBC_CSCAN(CONDUTF8,COND) \
if (utf8_target) { \
- REXEC_FBC_UTF8_CLASS_SCAN(CONDUTF8); \
+ REXEC_FBC_CLASS_SCAN(1, CONDUTF8); \
} \
else { \
- REXEC_FBC_CLASS_SCAN(COND); \
+ REXEC_FBC_CLASS_SCAN(0, COND); \
+ }
+
+/* We keep track of where the next character should start after an occurrence
+ * of the one we're looking for. Knowing that, we can see right away if the
+ * next occurrence is adjacent to the previous. When 'doevery' is FALSE, we
+ * don't accept the 2nd and succeeding adjacent occurrences */
+#define FBC_CHECK_AND_TRY \
+ if ( ( doevery \
+ || s != previous_occurrence_end) \
+ && (reginfo->intuit || regtry(reginfo, &s))) \
+ { \
+ goto got_it; \
+ }
+
+
+/* This differs from the above macros in that it calls a function which returns
+ * the next occurrence of the thing being looked for in 's'; and 'strend' if
+ * there is no such occurrence. */
+#define REXEC_FBC_FIND_NEXT_SCAN(UTF8, f) \
+ while (s < strend) { \
+ s = (f); \
+ if (s >= strend) { \
+ break; \
+ } \
+ \
+ FBC_CHECK_AND_TRY \
+ s += (UTF8) ? UTF8SKIP(s) : 1; \
+ previous_occurrence_end = s; \
}
/* The three macros below are slightly different versions of the same logic.
* here. And vice-versa if we are looking for a non-boundary.
*
* 'tmp' below in the next three macros in the REXEC_FBC_SCAN and
- * REXEC_FBC_UTF8_SCAN loops is a loop invariant, a bool giving the return of
+ * REXEC_FBC_SCAN loops is a loop invariant, a bool giving the return of
* TEST_NON_UTF8(s-1). To see this, note that that's what it is defined to be
* at entry to the loop, and to get to the IF_FAIL branch, tmp must equal
* TEST_NON_UTF8(s), and in the opposite branch, IF_SUCCESS, tmp is that
#define FBC_UTF8_A(TEST_NON_UTF8, IF_SUCCESS, IF_FAIL) \
tmp = (s != reginfo->strbeg) ? UCHARAT(s - 1) : '\n'; \
tmp = TEST_NON_UTF8(tmp); \
- REXEC_FBC_UTF8_SCAN( /* advances s while s < strend */ \
+ REXEC_FBC_SCAN(1, /* 1=>is-utf8; advances s while s < strend */ \
if (tmp == ! TEST_NON_UTF8((U8) *s)) { \
tmp = !tmp; \
IF_SUCCESS; /* Is a boundary if values for s-1 and s differ */ \
0, UTF8_ALLOW_DEFAULT); \
} \
tmp = TEST_UV(tmp); \
- LOAD_UTF8_CHARCLASS_ALNUM(); \
- REXEC_FBC_UTF8_SCAN( /* advances s while s < strend */ \
+ REXEC_FBC_SCAN(1, /* 1=>is-utf8; advances s while s < strend */ \
if (tmp == ! (TEST_UTF8((U8 *) s, (U8 *) reginfo->strend))) { \
tmp = !tmp; \
IF_SUCCESS; \
else { /* Not utf8 */ \
tmp = (s != reginfo->strbeg) ? UCHARAT(s - 1) : '\n'; \
tmp = TEST_NON_UTF8(tmp); \
- REXEC_FBC_SCAN( /* advances s while s < strend */ \
+ REXEC_FBC_SCAN(0, /* 0=>not-utf8; advances s while s < strend */ \
if (tmp == ! TEST_NON_UTF8((U8) *s)) { \
IF_SUCCESS; \
tmp = !tmp; \
#ifdef DEBUGGING
static IV
S_get_break_val_cp_checked(SV* const invlist, const UV cp_in) {
- IV cp_out = Perl__invlist_search(invlist, cp_in);
+ IV cp_out = _invlist_search(invlist, cp_in);
assert(cp_out >= 0);
return cp_out;
}
const char *strend, regmatch_info *reginfo)
{
dVAR;
+
+ /* TRUE if x+ need not match at just the 1st pos of run of x's */
const I32 doevery = (prog->intflags & PREGf_SKIP) == 0;
+
char *pat_string; /* The pattern's exactish string */
char *pat_end; /* ptr to end char of pat_string */
re_fold_t folder; /* Function for computing non-utf8 folds */
U8 c1;
U8 c2;
char *e;
- bool try_it = 1; /* Use in some macros to control whether to accept this
- occurrence of what's being matched, or not */
+
+ /* In some cases we accept only the first occurence of 'x' in a sequence of
+ * them. This variable points to just beyond the end of the previous
+ * occurrence of 'x', hence we can tell if we are in a sequence. (Having
+ * it point to beyond the 'x' allows us to work for UTF-8 without having to
+ * hop back.) */
+ char * previous_occurrence_end = 0;
+
I32 tmp; /* Scratch variable */
const bool utf8_target = reginfo->is_utf8_target;
UV utf8_fold_flags = 0;
/* We know what class it must start with. */
switch (OP(c)) {
+ case ANYOFPOSIXL:
case ANYOFL:
_CHECK_AND_WARN_PROBLEMATIC_LOCALE;
case ANYOFD:
case ANYOF:
if (utf8_target) {
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1, /* 1=>is-utf8 */
reginclass(prog, c, (U8*)s, (U8*) strend, utf8_target));
}
else if (ANYOF_FLAGS(c)) {
- REXEC_FBC_CLASS_SCAN(reginclass(prog,c, (U8*)s, (U8*)s+1, 0));
+ REXEC_FBC_CLASS_SCAN(0, reginclass(prog,c, (U8*)s, (U8*)s+1, 0));
}
else {
- REXEC_FBC_CLASS_SCAN(ANYOF_BITMAP_TEST(c, *((U8*)s)));
+ REXEC_FBC_CLASS_SCAN(0, ANYOF_BITMAP_TEST(c, *((U8*)s)));
}
break;
- case EXACTFA_NO_TRIE: /* This node only generated for non-utf8 patterns */
+ case ANYOFM: /* ARG() is the base byte; FLAGS() the mask byte */
+ /* UTF-8ness doesn't matter, so use 0 */
+ REXEC_FBC_FIND_NEXT_SCAN(0,
+ (char *) find_next_masked((U8 *) s, (U8 *) strend,
+ (U8) ARG(c), FLAGS(c)));
+ break;
+
+ case EXACTFAA_NO_TRIE: /* This node only generated for non-utf8 patterns */
assert(! is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA:
+ case EXACTFAA:
if (is_utf8_pat || utf8_target) {
utf8_fold_flags = FOLDEQ_UTF8_NOMIX_ASCII;
goto do_exactf_utf8;
c1 = *pat_string;
c2 = fold_array[c1];
if (c1 == c2) { /* If char and fold are the same */
- REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1);
+ while (s <= e) {
+ s = (char *) memchr(s, c1, e + 1 - s);
+ if (s == NULL) {
+ break;
+ }
+
+ /* Check that the rest of the node matches */
+ if ( (ln == 1 || folder(s + 1, pat_string + 1, ln - 1))
+ && (reginfo->intuit || regtry(reginfo, &s)) )
+ {
+ goto got_it;
+ }
+ s++;
+ }
}
else {
- REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1 || *(U8*)s == c2);
+ U8 bits_differing = c1 ^ c2;
+
+ /* If the folds differ in one bit position only, we can mask to
+ * match either of them, and can use this faster find method. Both
+ * ASCII and EBCDIC tend to have their case folds differ in only
+ * one position, so this is very likely */
+ if (LIKELY(PL_bitcount[bits_differing] == 1)) {
+ bits_differing = ~ bits_differing;
+ while (s <= e) {
+ s = (char *) find_next_masked((U8 *) s, (U8 *) e + 1,
+ (c1 & bits_differing), bits_differing);
+ if (s > e) {
+ break;
+ }
+
+ if ( (ln == 1 || folder(s + 1, pat_string + 1, ln - 1))
+ && (reginfo->intuit || regtry(reginfo, &s)) )
+ {
+ goto got_it;
+ }
+ s++;
+ }
+ }
+ else { /* Otherwise, stuck with looking byte-at-a-time. This
+ should actually happen only in EXACTFL nodes */
+ while (s <= e) {
+ if ( (*(U8*)s == c1 || *(U8*)s == c2)
+ && (ln == 1 || folder(s + 1, pat_string + 1, ln - 1))
+ && (reginfo->intuit || regtry(reginfo, &s)) )
+ {
+ goto got_it;
+ }
+ s++;
+ }
+ }
}
break;
break;
case ASCII:
- s = find_next_ascii(s, strend, utf8_target);
- if (s < strend && (reginfo->intuit || regtry(reginfo, &s))) {
- goto got_it;
- }
-
+ REXEC_FBC_FIND_NEXT_SCAN(0, find_next_ascii(s, strend, utf8_target));
break;
case NASCII:
- s = find_next_non_ascii(s, strend, utf8_target);
- if (s < strend && (reginfo->intuit || regtry(reginfo, &s))) {
- goto got_it;
+ if (utf8_target) {
+ REXEC_FBC_FIND_NEXT_SCAN(1, find_next_non_ascii(s, strend,
+ utf8_target));
+ }
+ else {
+ REXEC_FBC_FIND_NEXT_SCAN(0, find_next_non_ascii(s, strend,
+ utf8_target));
}
break;
case POSIXL:
_CHECK_AND_WARN_PROBLEMATIC_LOCALE;
- REXEC_FBC_CSCAN(to_complement ^ cBOOL(isFOO_utf8_lc(FLAGS(c), (U8 *) s)),
+ REXEC_FBC_CSCAN(to_complement ^ cBOOL(isFOO_utf8_lc(FLAGS(c), (U8 *) s, (U8 *) strend)),
to_complement ^ cBOOL(isFOO_lc(FLAGS(c), *s)));
break;
if (utf8_target) {
/* The complement of something that matches only ASCII matches all
* non-ASCII, plus everything in ASCII that isn't in the class. */
- REXEC_FBC_UTF8_CLASS_SCAN( ! isASCII_utf8_safe(s, strend)
- || ! _generic_isCC_A(*s, FLAGS(c)));
+ REXEC_FBC_CLASS_SCAN(1, ! isASCII_utf8_safe(s, strend)
+ || ! _generic_isCC_A(*s, FLAGS(c)));
break;
}
* as otherwise we would have to examine all the continuation
* characters */
if (utf8_target) {
- REXEC_FBC_UTF8_CLASS_SCAN(_generic_isCC_A(*s, FLAGS(c)));
+ REXEC_FBC_CLASS_SCAN(1, _generic_isCC_A(*s, FLAGS(c)));
break;
}
posixa:
- REXEC_FBC_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(0, /* 0=>not-utf8 */
to_complement ^ cBOOL(_generic_isCC_A(*s, FLAGS(c))));
break;
case POSIXU:
if (! utf8_target) {
- REXEC_FBC_CLASS_SCAN(to_complement ^ cBOOL(_generic_isCC(*s,
+ REXEC_FBC_CLASS_SCAN(0, /* 0=>not-utf8 */
+ to_complement ^ cBOOL(_generic_isCC(*s,
FLAGS(c))));
}
else {
posix_utf8:
classnum = (_char_class_number) FLAGS(c);
- if (classnum < _FIRST_NON_SWASH_CC) {
- while (s < strend) {
-
- /* We avoid loading in the swash as long as possible, but
- * should we have to, we jump to a separate loop. This
- * extra 'if' statement is what keeps this code from being
- * just a call to REXEC_FBC_UTF8_CLASS_SCAN() */
- if (UTF8_IS_ABOVE_LATIN1(*s)) {
- goto found_above_latin1;
- }
- if ((UTF8_IS_INVARIANT(*s)
- && to_complement ^ cBOOL(_generic_isCC((U8) *s,
- classnum)))
- || ( UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, strend)
- && to_complement ^ cBOOL(
- _generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*s,
- *(s + 1)),
- classnum))))
- {
- if (try_it && (reginfo->intuit || regtry(reginfo, &s)))
- goto got_it;
- else {
- try_it = doevery;
- }
- }
- else {
- try_it = 1;
- }
- s += UTF8SKIP(s);
- }
- }
- else switch (classnum) { /* These classes are implemented as
- macros */
+ switch (classnum) {
+ default:
+ REXEC_FBC_CLASS_SCAN(1, /* 1=>is-utf8 */
+ to_complement ^ cBOOL(_invlist_contains_cp(
+ PL_XPosix_ptrs[classnum],
+ utf8_to_uvchr_buf((U8 *) s,
+ (U8 *) strend,
+ NULL))));
+ break;
case _CC_ENUM_SPACE:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1, /* 1=>is-utf8 */
to_complement ^ cBOOL(isSPACE_utf8_safe(s, strend)));
break;
case _CC_ENUM_BLANK:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isBLANK_utf8_safe(s, strend)));
break;
case _CC_ENUM_XDIGIT:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isXDIGIT_utf8_safe(s, strend)));
break;
case _CC_ENUM_VERTSPACE:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isVERTWS_utf8_safe(s, strend)));
break;
case _CC_ENUM_CNTRL:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isCNTRL_utf8_safe(s, strend)));
break;
-
- default:
- Perl_croak(aTHX_ "panic: find_byclass() node %d='%s' has an unexpected character class '%d'", OP(c), PL_reg_name[OP(c)], classnum);
- NOT_REACHED; /* NOTREACHED */
}
}
break;
- found_above_latin1: /* Here we have to load a swash to get the result
- for the current code point */
- if (! PL_utf8_swash_ptrs[classnum]) {
- U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST;
- PL_utf8_swash_ptrs[classnum] =
- _core_swash_init("utf8",
- "",
- &PL_sv_undef, 1, 0,
- PL_XPosix_ptrs[classnum], &flags);
- }
-
- /* This is a copy of the loop above for swash classes, though using the
- * FBC macro instead of being expanded out. Since we've loaded the
- * swash, we don't have to check for that each time through the loop */
- REXEC_FBC_UTF8_CLASS_SCAN(
- to_complement ^ cBOOL(_generic_utf8_safe(
- classnum,
- s,
- strend,
- swash_fetch(PL_utf8_swash_ptrs[classnum],
- (U8 *) s, TRUE))));
- break;
-
case AHOCORASICKC:
case AHOCORASICK:
{
}
points[pointpos++ % maxlen]= uc;
if (foldlen || uc < (U8*)strend) {
- REXEC_TRIE_READ_CHAR(trie_type, trie,
- widecharmap, uc,
- uscan, len, uvc, charid, foldlen,
- foldbuf, uniflags);
+ REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc,
+ (U8 *) strend, uscan, len, uvc,
+ charid, foldlen, foldbuf,
+ uniflags);
DEBUG_TRIE_EXECUTE_r({
dump_exec_pos( (char *)uc, c, strend,
real_start, s, utf8_target, 0);
to_utf8_substr(prog);
}
ch = SvPVX_const(prog->anchored_utf8)[0];
- REXEC_FBC_SCAN(
+ REXEC_FBC_SCAN(0, /* 0=>not-utf8 */
if (*s == ch) {
DEBUG_EXECUTE_r( did_match = 1 );
if (regtry(reginfo, &s)) goto got_it;
}
}
ch = SvPVX_const(prog->anchored_substr)[0];
- REXEC_FBC_SCAN(
+ REXEC_FBC_SCAN(0, /* 0=>not-utf8 */
if (*s == ch) {
DEBUG_EXECUTE_r( did_match = 1 );
if (regtry(reginfo, &s)) goto got_it;
else { /* an EXACTFish node which doesn't begin with a multi-char fold */
c1 = is_utf8_pat ? valid_utf8_to_uvchr(pat, NULL) : *pat;
if (c1 > 255) {
- /* Load the folds hash, if not already done */
- SV** listp;
- if (! PL_utf8_foldclosures) {
- _load_PL_utf8_foldclosures();
+ const unsigned int * remaining_folds_to_list;
+ unsigned int first_folds_to;
+
+ /* Look up what code points (besides c1) fold to c1; e.g.,
+ * [ 'K', KELVIN_SIGN ] both fold to 'k'. */
+ Size_t folds_to_count = _inverse_folds(c1,
+ &first_folds_to,
+ &remaining_folds_to_list);
+ if (folds_to_count == 0) {
+ c2 = c1; /* there is only a single character that could
+ match */
}
-
- /* The fold closures data structure is a hash with the keys
- * being the UTF-8 of every character that is folded to, like
- * 'k', and the values each an array of all code points that
- * fold to its key. e.g. [ 'k', 'K', KELVIN_SIGN ].
- * Multi-character folds are not included */
- if ((! (listp = hv_fetch(PL_utf8_foldclosures,
- (char *) pat,
- UTF8SKIP(pat),
- FALSE))))
- {
- /* Not found in the hash, therefore there are no folds
- * containing it, so there is only a single character that
- * could match */
- c2 = c1;
+ else if (folds_to_count != 1) {
+ /* If there aren't exactly two folds to this (itself and
+ * another), it is outside the scope of this function */
+ use_chrtest_void = TRUE;
}
- else { /* Does participate in folds */
- AV* list = (AV*) *listp;
- if (av_tindex_skip_len_mg(list) != 1) {
-
- /* If there aren't exactly two folds to this, it is
- * outside the scope of this function */
- use_chrtest_void = TRUE;
- }
- else { /* There are two. Get them */
- SV** c_p = av_fetch(list, 0, FALSE);
- if (c_p == NULL) {
- Perl_croak(aTHX_ "panic: invalid PL_utf8_foldclosures structure");
- }
- c1 = SvUV(*c_p);
-
- c_p = av_fetch(list, 1, FALSE);
- if (c_p == NULL) {
- Perl_croak(aTHX_ "panic: invalid PL_utf8_foldclosures structure");
- }
- c2 = SvUV(*c_p);
-
- /* Folds that cross the 255/256 boundary are forbidden
- * if EXACTFL (and isnt a UTF8 locale), or EXACTFA and
- * one is ASCIII. Since the pattern character is above
- * 255, and its only other match is below 256, the only
- * legal match will be to itself. We have thrown away
- * the original, so have to compute which is the one
- * above 255. */
- if ((c1 < 256) != (c2 < 256)) {
- if ((OP(text_node) == EXACTFL
- && ! IN_UTF8_CTYPE_LOCALE)
- || ((OP(text_node) == EXACTFA
- || OP(text_node) == EXACTFA_NO_TRIE)
- && (isASCII(c1) || isASCII(c2))))
- {
- if (c1 < 256) {
- c1 = c2;
- }
- else {
- c2 = c1;
- }
- }
- }
+ else { /* There are two. We already have one, get the other */
+ c2 = first_folds_to;
+
+ /* Folds that cross the 255/256 boundary are forbidden if
+ * EXACTFL (and isnt a UTF8 locale), or EXACTFAA and one is
+ * ASCIII. The only other match to c1 is c2, and since c1
+ * is above 255, c2 better be as well under these
+ * circumstances. If it isn't, it means the only legal
+ * match of c1 is itself. */
+ if ( c2 < 256
+ && ( ( OP(text_node) == EXACTFL
+ && ! IN_UTF8_CTYPE_LOCALE)
+ || (( OP(text_node) == EXACTFAA
+ || OP(text_node) == EXACTFAA_NO_TRIE)
+ && (isASCII(c1) || isASCII(c2)))))
+ {
+ c2 = c1;
}
}
}
if (utf8_target
&& HAS_NONLATIN1_FOLD_CLOSURE(c1)
&& ( ! (OP(text_node) == EXACTFL && ! IN_UTF8_CTYPE_LOCALE))
- && ((OP(text_node) != EXACTFA
- && OP(text_node) != EXACTFA_NO_TRIE)
+ && ((OP(text_node) != EXACTFAA
+ && OP(text_node) != EXACTFAA_NO_TRIE)
|| ! isASCII(c1)))
{
/* Here, there could be something above Latin1 in the target
}
/* FALLTHROUGH */
/* /u rules for all these. This happens to work for
- * EXACTFA as nothing in Latin1 folds to ASCII */
- case EXACTFA_NO_TRIE: /* This node only generated for
- non-utf8 patterns */
+ * EXACTFAA as nothing in Latin1 folds to ASCII */
+ case EXACTFAA_NO_TRIE: /* This node only generated for
+ non-utf8 patterns */
assert(! is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA:
+ case EXACTFAA:
case EXACTFU_SS:
case EXACTFU:
c2 = PL_fold_latin1[c1];
return prev != GCB_E_Base && prev != GCB_E_Base_GAZ;
}
+ case GCB_Maybe_Emoji_NonBreak:
+
+ {
+
+ /* Do not break within emoji modifier sequences or emoji zwj sequences.
+ GB11 \p{Extended_Pictographic} Extend* ZWJ × \p{Extended_Pictographic}
+ */
+ U8 * temp_pos = (U8 *) curpos;
+ GCB_enum prev;
+
+ do {
+ prev = backup_one_GCB(strbeg, &temp_pos, utf8_target);
+ }
+ while (prev == GCB_Extend);
+
+ return prev != GCB_XPG_XX;
+ }
+
default:
break;
}
*/
if(!NEXTCHR_IS_EOS && !ANYOF_BITMAP_TEST(scan, nextchr)) {
DEBUG_EXECUTE_r(
- Perl_re_exec_indentf( aTHX_ "%sfailed to match trie start class...%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sTRIE: failed to match trie start class...%s\n",
depth, PL_colors[4], PL_colors[5])
);
sayNO_SILENT;
if (scan->flags == EXACTL || scan->flags == EXACTFLU8) {
_CHECK_AND_WARN_PROBLEMATIC_LOCALE;
if (utf8_target
- && nextchr >= 0 /* guard against negative EOS value in nextchr */
+ && ! NEXTCHR_IS_EOS
&& UTF8_IS_ABOVE_LATIN1(nextchr)
&& scan->flags == EXACTL)
{
{
if (trie->states[ state ].wordnum) {
DEBUG_EXECUTE_r(
- Perl_re_exec_indentf( aTHX_ "%smatched empty string...%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sTRIE: matched empty string...%s\n",
depth, PL_colors[4], PL_colors[5])
);
if (!trie->jump)
break;
} else {
DEBUG_EXECUTE_r(
- Perl_re_exec_indentf( aTHX_ "%sfailed to match trie start class...%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sTRIE: failed to match trie start class...%s\n",
depth, PL_colors[4], PL_colors[5])
);
sayNO_SILENT;
DUMP_EXEC_POS( (char *)uc, scan, utf8_target, depth );
/* HERE */
PerlIO_printf( Perl_debug_log,
- "%*s%sState: %4" UVxf " Accepted: %c ",
+ "%*s%sTRIE: State: %4" UVxf " Accepted: %c ",
INDENT_CHARS(depth), "", PL_colors[4],
(UV)state, (accepted ? 'Y' : 'N'));
});
if ( base && (foldlen || uc < (U8*)(reginfo->strend))) {
I32 offset;
REXEC_TRIE_READ_CHAR(trie_type, trie, widecharmap, uc,
- uscan, len, uvc, charid, foldlen,
- foldbuf, uniflags);
+ (U8 *) reginfo->strend, uscan,
+ len, uvc, charid, foldlen,
+ foldbuf, uniflags);
charcount++;
if (foldlen>0)
ST.longfold = TRUE;
}
DEBUG_TRIE_EXECUTE_r(
Perl_re_printf( aTHX_
- "Charid:%3x CP:%4" UVxf " After State: %4" UVxf "%s\n",
+ "TRIE: Charid:%3x CP:%4" UVxf " After State: %4" UVxf "%s\n",
charid, uvc, (UV)state, PL_colors[5] );
);
}
}
DEBUG_EXECUTE_r(
- Perl_re_exec_indentf( aTHX_ "%sgot %" IVdf " possible matches%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sTRIE: got %" IVdf " possible matches%s\n",
depth,
PL_colors[4], (IV)ST.accepted, PL_colors[5] );
);
while (foldlen) {
if (!--chars)
break;
- uvc = utf8n_to_uvchr(uscan, UTF8_MAXLEN, &len,
- uniflags);
+ uvc = utf8n_to_uvchr(uscan, foldlen, &len,
+ uniflags);
uscan += len;
foldlen -= len;
}
? av_fetch(trie_words, ST.nextword - 1, 0) : NULL;
SV *sv= tmp ? sv_newmortal() : NULL;
- Perl_re_exec_indentf( aTHX_ "%sonly one match left, short-circuiting: #%d <%s>%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sTRIE: only one match left, short-circuiting: #%d <%s>%s\n",
depth, PL_colors[4],
ST.nextword,
tmp ? pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), 0,
fold_utf8_flags = is_utf8_pat ? FOLDEQ_S1_ALREADY_FOLDED : 0;
goto do_exactf;
- case EXACTFA_NO_TRIE: /* This node only generated for non-utf8
+ case EXACTFAA_NO_TRIE: /* This node only generated for non-utf8
patterns */
assert(! is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA: /* /abc/iaa */
+ case EXACTFAA: /* /abc/iaa */
folder = foldEQ_latin1;
fold_array = PL_fold_latin1;
fold_utf8_flags = FOLDEQ_UTF8_NOMIX_ASCII;
}
break;
+ case ANYOFPOSIXL:
case ANYOFL: /* /[abc]/l */
_CHECK_AND_WARN_PROBLEMATIC_LOCALE;
}
break;
+ case ANYOFM:
+ if (NEXTCHR_IS_EOS || (UCHARAT(locinput) & FLAGS(scan)) != ARG(scan)) {
+ sayNO;
+ }
+ locinput++;
+ break;
+
case ASCII:
if (NEXTCHR_IS_EOS || ! isASCII(UCHARAT(locinput))) {
sayNO;
else { /* Handle above Latin-1 code points */
utf8_posix_above_latin1:
classnum = (_char_class_number) FLAGS(scan);
- if (classnum < _FIRST_NON_SWASH_CC) {
-
- /* Here, uses a swash to find such code points. Load if if
- * not done already */
- if (! PL_utf8_swash_ptrs[classnum]) {
- U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST;
- PL_utf8_swash_ptrs[classnum]
- = _core_swash_init("utf8",
- "",
- &PL_sv_undef, 1, 0,
- PL_XPosix_ptrs[classnum], &flags);
- }
- if (! (to_complement
- ^ cBOOL(swash_fetch(PL_utf8_swash_ptrs[classnum],
- (U8 *) locinput, TRUE))))
- {
- sayNO;
- }
- }
- else { /* Here, uses macros to find above Latin-1 code points */
- switch (classnum) {
- case _CC_ENUM_SPACE:
- if (! (to_complement
- ^ cBOOL(is_XPERLSPACE_high(locinput))))
- {
- sayNO;
- }
- break;
- case _CC_ENUM_BLANK:
- if (! (to_complement
- ^ cBOOL(is_HORIZWS_high(locinput))))
- {
- sayNO;
- }
- break;
- case _CC_ENUM_XDIGIT:
- if (! (to_complement
- ^ cBOOL(is_XDIGIT_high(locinput))))
- {
- sayNO;
- }
- break;
- case _CC_ENUM_VERTSPACE:
- if (! (to_complement
- ^ cBOOL(is_VERTWS_high(locinput))))
- {
- sayNO;
- }
- break;
- default: /* The rest, e.g. [:cntrl:], can't match
- above Latin1 */
- if (! to_complement) {
- sayNO;
- }
- break;
- }
+ switch (classnum) {
+ default:
+ if (! (to_complement
+ ^ cBOOL(_invlist_contains_cp(
+ PL_XPosix_ptrs[classnum],
+ utf8_to_uvchr_buf((U8 *) locinput,
+ (U8 *) reginfo->strend,
+ NULL)))))
+ {
+ sayNO;
+ }
+ break;
+ case _CC_ENUM_SPACE:
+ if (! (to_complement
+ ^ cBOOL(is_XPERLSPACE_high(locinput))))
+ {
+ sayNO;
+ }
+ break;
+ case _CC_ENUM_BLANK:
+ if (! (to_complement
+ ^ cBOOL(is_HORIZWS_high(locinput))))
+ {
+ sayNO;
+ }
+ break;
+ case _CC_ENUM_XDIGIT:
+ if (! (to_complement
+ ^ cBOOL(is_XDIGIT_high(locinput))))
+ {
+ sayNO;
+ }
+ break;
+ case _CC_ENUM_VERTSPACE:
+ if (! (to_complement
+ ^ cBOOL(is_VERTWS_high(locinput))))
+ {
+ sayNO;
+ }
+ break;
+ case _CC_ENUM_CNTRL: /* These can't match above Latin1 */
+ case _CC_ENUM_ASCII:
+ if (! to_complement) {
+ sayNO;
+ }
+ break;
}
locinput += UTF8SKIP(locinput);
}
rei = RXi_GET(re);
DEBUG_EXECUTE_r(
debug_start_match(re_sv, utf8_target, locinput,
- reginfo->strend, "Matching embedded");
+ reginfo->strend, "EVAL/GOSUB: Matching embedded");
);
startpoint = rei->program + 1;
EVAL_CLOSE_PAREN_CLEAR(st); /* ST.close_paren = 0;
if (n > maxopenparen)
maxopenparen = n;
DEBUG_BUFFERS_r(Perl_re_exec_indentf( aTHX_
- "rex=0x%" UVxf " offs=0x%" UVxf ": \\%" UVuf ": set %" IVdf " tmp; maxopenparen=%" UVuf "\n",
+ "OPEN: rex=0x%" UVxf " offs=0x%" UVxf ": \\%" UVuf ": set %" IVdf " tmp; maxopenparen=%" UVuf "\n",
depth,
PTR2UV(rex),
PTR2UV(rex->offs),
script_run_begin = (U8 *) locinput;
break;
-/* XXX really need to log other places start/end are set too */
-#define CLOSE_CAPTURE \
- rex->offs[n].start = rex->offs[n].start_tmp; \
- rex->offs[n].end = locinput - reginfo->strbeg; \
- DEBUG_BUFFERS_r(Perl_re_exec_indentf( aTHX_ \
- "rex=0x%" UVxf " offs=0x%" UVxf ": \\%" UVuf ": set %" IVdf "..%" IVdf "\n", \
- depth, \
- PTR2UV(rex), \
- PTR2UV(rex->offs), \
- (UV)n, \
- (IV)rex->offs[n].start, \
- (IV)rex->offs[n].end \
- ))
case CLOSE: /* ) */
n = ARG(scan); /* which paren pair */
- CLOSE_CAPTURE;
- if (n > rex->lastparen)
- rex->lastparen = n;
- rex->lastcloseparen = n;
+ CLOSE_CAPTURE(n, rex->offs[n].start_tmp,
+ locinput - reginfo->strbeg);
if ( EVAL_CLOSE_PAREN_IS( cur_eval, n ) )
goto fake_end;
if ( OP(cursor)==CLOSE ){
n = ARG(cursor);
if ( n <= lastopen ) {
- CLOSE_CAPTURE;
- if (n > rex->lastparen)
- rex->lastparen = n;
- rex->lastcloseparen = n;
+ CLOSE_CAPTURE(n, rex->offs[n].start_tmp,
+ locinput - reginfo->strbeg);
if ( n == ARG(scan) || EVAL_CLOSE_PAREN_IS(cur_eval, n) )
break;
}
ST.cache_mask = 0;
- DEBUG_EXECUTE_r( Perl_re_exec_indentf( aTHX_ "whilem: matched %ld out of %d..%d\n",
+ DEBUG_EXECUTE_r( Perl_re_exec_indentf( aTHX_ "WHILEM: matched %ld out of %d..%d\n",
depth, (long)n, min, max)
);
/* If degenerate A matches "", assume A done. */
if (locinput == cur_curlyx->u.curlyx.lastloc) {
- DEBUG_EXECUTE_r( Perl_re_exec_indentf( aTHX_ "whilem: empty match detected, trying continuation...\n",
+ DEBUG_EXECUTE_r( Perl_re_exec_indentf( aTHX_ "WHILEM: empty match detected, trying continuation...\n",
depth)
);
goto do_whilem_B_max;
Newxz(aux->poscache, size, char);
}
DEBUG_EXECUTE_r( Perl_re_printf( aTHX_
- "%swhilem: Detected a super-linear match, switching on caching%s...\n",
+ "%sWHILEM: Detected a super-linear match, switching on caching%s...\n",
PL_colors[4], PL_colors[5])
);
}
mask = 1 << (offset % 8);
offset /= 8;
if (reginfo->info_aux->poscache[offset] & mask) {
- DEBUG_EXECUTE_r( Perl_re_exec_indentf( aTHX_ "whilem: (cache) already tried at this position...\n",
+ DEBUG_EXECUTE_r( Perl_re_exec_indentf( aTHX_ "WHILEM: (cache) already tried at this position...\n",
depth)
);
cur_curlyx->u.curlyx.count--;
CACHEsayNO;
NOT_REACHED; /* NOTREACHED */
+ case WHILEM_A_min_fail: /* just failed to match A in a minimal match */
+ /* FALLTHROUGH */
case WHILEM_A_pre_fail: /* just failed to match even minimal A */
REGCP_UNWIND(ST.lastcp);
regcppop(rex, &maxopenparen);
- /* FALLTHROUGH */
- case WHILEM_A_min_fail: /* just failed to match A in a minimal match */
cur_curlyx->u.curlyx.lastloc = ST.save_lastloc;
cur_curlyx->u.curlyx.count--;
CACHEsayNO;
case WHILEM_A_max_fail: /* just failed to match A in a maximal match */
REGCP_UNWIND(ST.lastcp);
regcppop(rex, &maxopenparen); /* Restore some previous $<digit>s? */
- DEBUG_EXECUTE_r(Perl_re_exec_indentf( aTHX_ "whilem: failed, trying continuation...\n",
+ DEBUG_EXECUTE_r(Perl_re_exec_indentf( aTHX_ "WHILEM: failed, trying continuation...\n",
depth)
);
do_whilem_B_max:
CACHEsayNO;
}
- DEBUG_EXECUTE_r(Perl_re_exec_indentf( aTHX_ "trying longer...\n", depth)
+ DEBUG_EXECUTE_r(Perl_re_exec_indentf( aTHX_ "WHILEM: B min fail: trying longer...\n", depth)
);
/* Try grabbing another A and see if it helps. */
cur_curlyx->u.curlyx.lastloc = locinput;
+ ST.cp = regcppush(rex, cur_curlyx->u.curlyx.parenfloor,
+ maxopenparen);
+ REGCP_SET(ST.lastcp);
PUSH_STATE_GOTO(WHILEM_A_min,
/*A*/ NEXTOPER(ST.save_curlyx->u.curlyx.me) + EXTRA_STEP_2ARGS,
locinput);
if (ST.me->flags) {
/* emulate CLOSE: mark current A as captured */
- I32 paren = ST.me->flags;
+ U32 paren = (U32)ST.me->flags;
if (ST.count) {
- rex->offs[paren].start
- = HOPc(locinput, -ST.alen) - reginfo->strbeg;
- rex->offs[paren].end = locinput - reginfo->strbeg;
- if ((U32)paren > rex->lastparen)
- rex->lastparen = paren;
- rex->lastcloseparen = paren;
+ CLOSE_CAPTURE(paren,
+ HOPc(locinput, -ST.alen) - reginfo->strbeg,
+ locinput - reginfo->strbeg);
}
else
rex->offs[paren].end = -1;
#define CURLY_SETPAREN(paren, success) \
if (paren) { \
if (success) { \
- rex->offs[paren].start = HOPc(locinput, -1) - reginfo->strbeg; \
- rex->offs[paren].end = locinput - reginfo->strbeg; \
- if (paren > rex->lastparen) \
- rex->lastparen = paren; \
- rex->lastcloseparen = paren; \
+ CLOSE_CAPTURE(paren, HOPc(locinput, -1) - reginfo->strbeg, \
+ locinput - reginfo->strbeg); \
} \
else { \
rex->offs[paren].end = -1; \
maxopenparen = ST.paren;
ST.min = ARG1(scan); /* min to match */
ST.max = ARG2(scan); /* max to match */
+ scan = regnext(NEXTOPER(scan) + NODE_STEP_REGNODE);
+
+ /* handle the single-char capture called as a GOSUB etc */
if (EVAL_CLOSE_PAREN_IS_TRUE(cur_eval,(U32)ST.paren))
{
- ST.min=1;
- ST.max=1;
+ char *li = locinput;
+ if (!regrepeat(rex, &li, scan, reginfo, 1))
+ sayNO;
+ SET_locinput(li);
+ goto fake_end;
}
- scan = regnext(NEXTOPER(scan) + NODE_STEP_REGNODE);
+
goto repeat;
case CURLY: /* /A{m,n}B/ where A is width 1 char */
}
NOT_REACHED; /* NOTREACHED */
- case CURLY_B_min_known_fail:
- /* failed to find B in a non-greedy match where c1,c2 valid */
+ case CURLY_B_min_fail:
+ /* failed to find B in a non-greedy match.
+ * Handles both cases where c1,c2 valid or not */
REGCP_UNWIND(ST.cp);
if (ST.paren) {
UNWIND_PAREN(ST.lastparen, ST.lastcloseparen);
}
- /* Couldn't or didn't -- move forward. */
- ST.oldloc = locinput;
- if (utf8_target)
- locinput += UTF8SKIP(locinput);
- else
- locinput++;
- ST.count++;
- curly_try_B_min_known:
- /* find the next place where 'B' could work, then call B */
- {
+
+ if (ST.c1 == CHRTEST_VOID) {
+ /* failed -- move forward one */
+ char *li = locinput;
+ if (!regrepeat(rex, &li, ST.A, reginfo, 1)) {
+ sayNO;
+ }
+ locinput = li;
+ ST.count++;
+ if (!( ST.count <= ST.max
+ /* count overflow ? */
+ || (ST.max == REG_INFTY && ST.count > 0))
+ )
+ sayNO;
+ }
+ else {
int n;
+ /* Couldn't or didn't -- move forward. */
+ ST.oldloc = locinput;
+ if (utf8_target)
+ locinput += UTF8SKIP(locinput);
+ else
+ locinput++;
+ ST.count++;
+
+ curly_try_B_min_known:
+ /* find the next place where 'B' could work, then call B */
if (utf8_target) {
n = (ST.oldloc == locinput) ? 0 : 1;
if (ST.c1 == ST.c2) {
sayNO;
assert(n == REG_INFTY || locinput == li);
}
- CURLY_SETPAREN(ST.paren, ST.count);
- if (EVAL_CLOSE_PAREN_IS_TRUE(cur_eval,(U32)ST.paren))
- goto fake_end;
- PUSH_STATE_GOTO(CURLY_B_min_known, ST.B, locinput);
}
- NOT_REACHED; /* NOTREACHED */
-
- case CURLY_B_min_fail:
- /* failed to find B in a non-greedy match where c1,c2 invalid */
- REGCP_UNWIND(ST.cp);
- if (ST.paren) {
- UNWIND_PAREN(ST.lastparen, ST.lastcloseparen);
- }
- /* failed -- move forward one */
- {
- char *li = locinput;
- if (!regrepeat(rex, &li, ST.A, reginfo, 1)) {
- sayNO;
- }
- locinput = li;
- }
- {
- ST.count++;
- if (ST.count <= ST.max || (ST.max == REG_INFTY &&
- ST.count > 0)) /* count overflow ? */
- {
- curly_try_B_min:
- CURLY_SETPAREN(ST.paren, ST.count);
- if (EVAL_CLOSE_PAREN_IS_TRUE(cur_eval,(U32)ST.paren))
- goto fake_end;
- PUSH_STATE_GOTO(CURLY_B_min, ST.B, locinput);
- }
- }
- sayNO;
+ curly_try_B_min:
+ CURLY_SETPAREN(ST.paren, ST.count);
+ PUSH_STATE_GOTO(CURLY_B_min, ST.B, locinput);
NOT_REACHED; /* NOTREACHED */
+
curly_try_B_max:
/* a successful greedy match: now try to match B */
- if (EVAL_CLOSE_PAREN_IS_TRUE(cur_eval,(U32)ST.paren))
- goto fake_end;
{
bool could_match = locinput < reginfo->strend;
st->u.eval.prev_eval = cur_eval;
cur_eval = CUR_EVAL.prev_eval;
DEBUG_EXECUTE_r(
- Perl_re_exec_indentf( aTHX_ "EVAL trying tail ... (cur_eval=%p)\n",
+ Perl_re_exec_indentf( aTHX_ "END: EVAL trying tail ... (cur_eval=%p)\n",
depth, cur_eval););
if ( nochange_depth )
nochange_depth--;
if (locinput < reginfo->till) {
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
- "%sMatch possible, but length=%ld is smaller than requested=%ld, failing!%s\n",
+ "%sEND: Match possible, but length=%ld is smaller than requested=%ld, failing!%s\n",
PL_colors[4],
(long)(locinput - startpos),
(long)(reginfo->till - startpos),
case SUCCEED: /* successful SUSPEND/UNLESSM/IFMATCH/CURLYM */
DEBUG_EXECUTE_r(
- Perl_re_exec_indentf( aTHX_ "%ssubpattern success...%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sSUCCEED: subpattern success...%s\n",
depth, PL_colors[4], PL_colors[5]));
sayYES; /* Success! */
sv_commit = ST.mark_name;
DEBUG_EXECUTE_r({
- Perl_re_exec_indentf( aTHX_ "%ssetting cutpoint to mark:%" SVf "...%s\n",
+ Perl_re_exec_indentf( aTHX_ "%sMARKPOINT: next fail: setting cutpoint to mark:%" SVf "...%s\n",
depth,
PL_colors[4], SVfARG(sv_commit), PL_colors[5]);
});
hardcount++;
}
} else {
- while (scan < loceol && *scan != '\n')
- scan++;
+ scan = (char *) memchr(scan, '\n', loceol - scan);
+ if (! scan) {
+ scan = loceol;
+ }
}
break;
case SANY:
c = (U8)*STRING(p);
- /* Can use a simple loop if the pattern char to match on is invariant
+ /* Can use a simple find if the pattern char to match on is invariant
* under UTF-8, or both target and pattern aren't UTF-8. Note that we
* can use UTF8_IS_INVARIANT() even if the pattern isn't UTF-8, as it's
* true iff it doesn't matter if the argument is in UTF-8 or not */
* since here, to match at all, 1 char == 1 byte */
loceol = scan + max;
}
- while (scan < loceol && UCHARAT(scan) == c) {
- scan++;
- }
+ scan = (char *) find_span_end((U8 *) scan, (U8 *) loceol, (U8) c);
}
else if (reginfo->is_utf8_pat) {
if (utf8_target) {
else if (! UTF8_IS_ABOVE_LATIN1(c)) {
/* Target isn't utf8; convert the character in the UTF-8
- * pattern to non-UTF8, and do a simple loop */
+ * pattern to non-UTF8, and do a simple find */
c = EIGHT_BIT_UTF8_TO_NATIVE(c, *(STRING(p) + 1));
- while (scan < loceol && UCHARAT(scan) == c) {
- scan++;
- }
+ scan = (char *) find_span_end((U8 *) scan, (U8 *) loceol, (U8) c);
} /* else pattern char is above Latin1, can't possibly match the
non-UTF-8 target */
}
}
break;
- case EXACTFA_NO_TRIE: /* This node only generated for non-utf8 patterns */
+ case EXACTFAA_NO_TRIE: /* This node only generated for non-utf8 patterns */
assert(! reginfo->is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA:
+ case EXACTFAA:
utf8_flags = FOLDEQ_UTF8_NOMIX_ASCII;
goto do_exactf;
}
}
else if (c1 == c2) {
- while (scan < loceol && UCHARAT(scan) == c1) {
- scan++;
- }
+ scan = (char *) find_span_end((U8 *) scan, (U8 *) loceol, (U8) c1);
}
else {
/* See comments in regmatch() CURLY_B_min_known_fail. We avoid
U8 c1_c2_bits_differing = c1 ^ c2;
if (isPOWER_OF_2(c1_c2_bits_differing)) {
- U8 c1_masked = c1 & ~ c1_c2_bits_differing;
U8 c1_c2_mask = ~ c1_c2_bits_differing;
- while ( scan < loceol
- && (UCHARAT(scan) & c1_c2_mask) == c1_masked)
- {
- scan++;
- }
+ scan = (char *) find_span_end_mask((U8 *) scan,
+ (U8 *) loceol,
+ c1 & c1_c2_mask,
+ c1_c2_mask);
}
else {
while ( scan < loceol
}
break;
}
+ case ANYOFPOSIXL:
case ANYOFL:
_CHECK_AND_WARN_PROBLEMATIC_LOCALE;
}
break;
- case ASCII:
+ case ANYOFM:
if (utf8_target && loceol - scan > max) {
/* We didn't adjust <loceol> at the beginning of this routine
loceol = scan + max;
}
+ scan = (char *) find_span_end_mask((U8 *) scan, (U8 *) loceol, (U8) ARG(p), FLAGS(p));
+ break;
+
+ case ASCII:
+ if (utf8_target && loceol - scan > max) {
+ loceol = scan + max;
+ }
+
scan = find_next_non_ascii(scan, loceol, utf8_target);
break;
} else {
while (hardcount < max && scan < loceol
&& to_complement ^ cBOOL(isFOO_utf8_lc(FLAGS(p),
- (U8 *) scan)))
+ (U8 *) scan,
+ (U8 *) loceol)))
{
scan += UTF8SKIP(scan);
hardcount++;
else {
utf8_posix:
classnum = (_char_class_number) FLAGS(p);
- if (classnum < _FIRST_NON_SWASH_CC) {
-
- /* Here, a swash is needed for above-Latin1 code points.
- * Process as many Latin1 code points using the built-in rules.
- * Go to another loop to finish processing upon encountering
- * the first Latin1 code point. We could do that in this loop
- * as well, but the other way saves having to test if the swash
- * has been loaded every time through the loop: extra space to
- * save a test. */
- while (hardcount < max && scan < loceol) {
- if (UTF8_IS_INVARIANT(*scan)) {
- if (! (to_complement ^ cBOOL(_generic_isCC((U8) *scan,
- classnum))))
- {
- break;
- }
- scan++;
+ switch (classnum) {
+ default:
+ while ( hardcount < max && scan < loceol
+ && to_complement ^ cBOOL(_invlist_contains_cp(
+ PL_XPosix_ptrs[classnum],
+ utf8_to_uvchr_buf((U8 *) scan,
+ (U8 *) loceol,
+ NULL))))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
}
- else if (UTF8_IS_DOWNGRADEABLE_START(*scan)) {
- if (! (to_complement
- ^ cBOOL(_generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*scan,
- *(scan + 1)),
- classnum))))
- {
- break;
- }
- scan += 2;
+ break;
+
+ /* For the classes below, the knowledge of how to handle
+ * every code point is compiled in to Perl via a macro.
+ * This code is written for making the loops as tight as
+ * possible. It could be refactored to save space instead.
+ * */
+
+ case _CC_ENUM_SPACE:
+ while (hardcount < max
+ && scan < loceol
+ && (to_complement
+ ^ cBOOL(isSPACE_utf8_safe(scan, loceol))))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
}
- else {
- goto found_above_latin1;
+ break;
+ case _CC_ENUM_BLANK:
+ while (hardcount < max
+ && scan < loceol
+ && (to_complement
+ ^ cBOOL(isBLANK_utf8_safe(scan, loceol))))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
}
-
- hardcount++;
- }
- }
- else {
- /* For these character classes, the knowledge of how to handle
- * every code point is compiled in to Perl via a macro. This
- * code is written for making the loops as tight as possible.
- * It could be refactored to save space instead */
- switch (classnum) {
- case _CC_ENUM_SPACE:
- while (hardcount < max
- && scan < loceol
- && (to_complement
- ^ cBOOL(isSPACE_utf8_safe(scan, loceol))))
- {
- scan += UTF8SKIP(scan);
- hardcount++;
- }
- break;
- case _CC_ENUM_BLANK:
- while (hardcount < max
- && scan < loceol
- && (to_complement
- ^ cBOOL(isBLANK_utf8_safe(scan, loceol))))
- {
- scan += UTF8SKIP(scan);
- hardcount++;
- }
- break;
- case _CC_ENUM_XDIGIT:
- while (hardcount < max
- && scan < loceol
- && (to_complement
- ^ cBOOL(isXDIGIT_utf8_safe(scan, loceol))))
- {
- scan += UTF8SKIP(scan);
- hardcount++;
- }
- break;
- case _CC_ENUM_VERTSPACE:
- while (hardcount < max
- && scan < loceol
- && (to_complement
- ^ cBOOL(isVERTWS_utf8_safe(scan, loceol))))
- {
- scan += UTF8SKIP(scan);
- hardcount++;
- }
- break;
- case _CC_ENUM_CNTRL:
- while (hardcount < max
- && scan < loceol
- && (to_complement
- ^ cBOOL(isCNTRL_utf8_safe(scan, loceol))))
- {
- scan += UTF8SKIP(scan);
- hardcount++;
- }
- break;
- default:
- Perl_croak(aTHX_ "panic: regrepeat() node %d='%s' has an unexpected character class '%d'", OP(p), PL_reg_name[OP(p)], classnum);
- }
+ break;
+ case _CC_ENUM_XDIGIT:
+ while (hardcount < max
+ && scan < loceol
+ && (to_complement
+ ^ cBOOL(isXDIGIT_utf8_safe(scan, loceol))))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
+ }
+ break;
+ case _CC_ENUM_VERTSPACE:
+ while (hardcount < max
+ && scan < loceol
+ && (to_complement
+ ^ cBOOL(isVERTWS_utf8_safe(scan, loceol))))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
+ }
+ break;
+ case _CC_ENUM_CNTRL:
+ while (hardcount < max
+ && scan < loceol
+ && (to_complement
+ ^ cBOOL(isCNTRL_utf8_safe(scan, loceol))))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
+ }
+ break;
}
}
break;
- found_above_latin1: /* Continuation of POSIXU and NPOSIXU */
-
- /* Load the swash if not already present */
- if (! PL_utf8_swash_ptrs[classnum]) {
- U8 flags = _CORE_SWASH_INIT_ACCEPT_INVLIST;
- PL_utf8_swash_ptrs[classnum] = _core_swash_init(
- "utf8",
- "",
- &PL_sv_undef, 1, 0,
- PL_XPosix_ptrs[classnum], &flags);
- }
-
- while (hardcount < max && scan < loceol
- && to_complement ^ cBOOL(_generic_utf8_safe(
- classnum,
- scan,
- loceol,
- swash_fetch(PL_utf8_swash_ptrs[classnum],
- (U8 *) scan,
- TRUE))))
- {
- scan += UTF8SKIP(scan);
- hardcount++;
- }
- break;
-
case LNBREAK:
if (utf8_target) {
while (hardcount < max && scan < loceol &&
1 /* 1 means die */ );
NOT_REACHED; /* NOTREACHED */
}
- if (c > 255 && OP(n) == ANYOFL && ! ANYOFL_UTF8_LOCALE_REQD(flags)) {
+ if ( c > 255
+ && (OP(n) == ANYOFL || OP(n) == ANYOFPOSIXL)
+ && ! ANYOFL_UTF8_LOCALE_REQD(flags))
+ {
_CHECK_AND_OUTPUT_WIDE_LOCALE_CP_MSG(c);
}
}
PERL_ARGS_ASSERT__IS_GRAPHEME;
- /* Unassigned code points are forbidden */
+ if ( UNLIKELY(UNICODE_IS_SUPER(cp))
+ || UNLIKELY(UNICODE_IS_NONCHAR(cp)))
+ {
+ /* These are considered graphemes */
+ return TRUE;
+ }
+
+ /* Otherwise, unassigned code points are forbidden */
if (UNLIKELY(! ELEMENT_RANGE_MATCHES_INVLIST(
_invlist_search(PL_Assigned_invlist, cp))))
{
return isGCB(cp_gcb_val, next_cp_gcb_val, strbeg, s, TRUE);
}
+/*
+=head1 Unicode Support
+
+=for apidoc isSCRIPT_RUN
+
+Returns a bool as to whether or not the sequence of bytes from C<s> up to but
+not including C<send> form a "script run". C<utf8_target> is TRUE iff the
+sequence starting at C<s> is to be treated as UTF-8. To be precise, except for
+two degenerate cases given below, this function returns TRUE iff all code
+points in it come from any combination of three "scripts" given by the Unicode
+"Script Extensions" property: Common, Inherited, and possibly one other.
+Additionally all decimal digits must come from the same consecutive sequence of
+10.
+
+For example, if all the characters in the sequence are Greek, or Common, or
+Inherited, this function will return TRUE, provided any decimal digits in it
+are the ASCII digits "0".."9". For scripts (unlike Greek) that have their own
+digits defined this will accept either digits from that set or from 0..9, but
+not a combination of the two. Some scripts, such as Arabic, have more than one
+set of digits. All digits must come from the same set for this function to
+return TRUE.
+
+C<*ret_script>, if C<ret_script> is not NULL, will on return of TRUE
+contain the script found, using the C<SCX_enum> typedef. Its value will be
+C<SCX_INVALID> if the function returns FALSE.
+
+If the sequence is empty, TRUE is returned, but C<*ret_script> (if asked for)
+will be C<SCX_INVALID>.
+
+If the sequence contains a single code point which is unassigned to a character
+in the version of Unicode being used, the function will return TRUE, and the
+script will be C<SCX_Unknown>. Any other combination of unassigned code points
+in the input sequence will result in the function treating the input as not
+being a script run.
+
+The returned script will be C<SCX_Inherited> iff all the code points in it are
+from the Inherited script.
+
+Otherwise, the returned script will be C<SCX_Common> iff all the code points in
+it are from the Inherited or Common scripts.
+
+=cut
+
+*/
+
bool
Perl_isSCRIPT_RUN(pTHX_ const U8 * s, const U8 * send, const bool utf8_target)
{
- /* Checks that every character in the sequence from 's' to 'send' is one of
- * three scripts: Common, Inherited, and possibly one other. Additionally
- * all decimal digits must come from the same consecutive sequence of 10.
- * 'utf8_target' is TRUE iff the sequence is encoded in UTF-8.
- *
- * Basically, it looks at each character in the sequence to see if the
+ /* Basically, it looks at each character in the sequence to see if the
* above conditions are met; if not it fails. It uses an inversion map to
* find the enum corresponding to the script of each character. But this
* is complicated by the fact that a few code points can be in any of
* These are all defined in charclass_invlists.h */
/* XXX Here are the additional things UTS 39 says could be done:
- * Mark Chinese strings as “mixed script” if they contain both simplified
- * (S) and traditional (T) Chinese characters, using the Unihan data in the
- * Unicode Character Database [UCD]. The criterion can only be applied if
- * the language of the string is known to be Chinese. So, for example, the
- * string “写真だけの結婚式 ” is Japanese, and should not be marked as
- * mixed script because of a mixture of S and T characters. Testing for
- * whether a character is S or T needs to be based not on whether the
- * character has a S or T variant , but whether the character is an S or T
- * variant. khw notes that the sample contains a Hiragana character, and it
- * is unclear if absence of any foreign script marks the script as
- * "Chinese"
*
* Forbid sequences of the same nonspacing mark
*
SV * decimals_invlist = PL_XPosix_ptrs[_CC_DIGIT];
UV * decimals_array = invlist_array(decimals_invlist);
- /* What code point is the digit '0' of the script run? */
+ /* What code point is the digit '0' of the script run? (0 meaning FALSE if
+ * not currently known) */
UV zero_of_run = 0;
+
SCX_enum script_of_run = SCX_INVALID; /* Illegal value */
SCX_enum script_of_char = SCX_INVALID;
PERL_UINT_FAST8_T intersection_len = 0;
bool retval = TRUE;
+ SCX_enum * ret_script = NULL;
- assert(send > s);
+ assert(send >= s);
PERL_ARGS_ASSERT_ISSCRIPT_RUN;
+ /* All code points in 0..255 are either Common or Latin, so must be a
+ * script run. We can return immediately unless we need to know which
+ * script it is. */
+ if (! utf8_target && LIKELY(send > s)) {
+ if (ret_script == NULL) {
+ return TRUE;
+ }
+
+ /* If any character is Latin, the run is Latin */
+ while (s < send) {
+ if (isALPHA_L1(*s) && LIKELY(*s != MICRO_SIGN_NATIVE)) {
+ *ret_script = SCX_Latin;
+ return TRUE;
+ }
+ }
+
+ /* Here, all are Common */
+ *ret_script = SCX_Common;
+ return TRUE;
+ }
+
/* Look at each character in the sequence */
while (s < send) {
+ /* If the current character being examined is a digit, this is the code
+ * point of the zero for its sequence of 10 */
+ UV zero_of_char;
+
UV cp;
/* The code allows all scripts to use the ASCII digits. This is
* because they are used in commerce even in scripts that have their
- * own set. Hence any ASCII ones found are ok, unless a digit from
- * another set has already been encountered. (The other digit ranges
- * in Common are not similarly blessed */
+ * own set. Hence any ASCII ones found are ok, unless and until a
+ * digit from another set has already been encountered. (The other
+ * digit ranges in Common are not similarly blessed) */
if (UNLIKELY(isDIGIT(*s))) {
- if (zero_of_run > 0) {
+ if (UNLIKELY(script_of_run == SCX_Unknown)) {
+ retval = FALSE;
+ break;
+ }
+ if (zero_of_run) {
if (zero_of_run != '0') {
retval = FALSE;
break;
}
/* Here, isn't an ASCII digit. Find the code point of the character */
- if (utf8_target && ! UTF8_IS_INVARIANT(*s)) {
+ if (! UTF8_IS_INVARIANT(*s)) {
Size_t len;
cp = valid_utf8_to_uvchr((U8 *) s, &len);
s += len;
/* If is within the range [+0 .. +9] of the script's zero, it also is a
* digit in that script. We can skip the rest of this code for this
* character. */
- if (UNLIKELY( zero_of_run > 0
+ if (UNLIKELY( zero_of_run
&& cp >= zero_of_run
&& cp - zero_of_run <= 9))
{
break;
}
+ /* For the first character, or the run is inherited, the run's script
+ * is set to the char's */
+ if ( UNLIKELY(script_of_run == SCX_INVALID)
+ || UNLIKELY(script_of_run == SCX_Inherited))
+ {
+ script_of_run = script_of_char;
+ }
+
+ /* For the character's script to be Unknown, it must be the first
+ * character in the sequence (for otherwise a test above would have
+ * prevented us from reaching here), and we have set the run's script
+ * to it. Nothing further to be done for this character */
if (UNLIKELY(script_of_char == SCX_Unknown)) {
- script_of_run = SCX_Unknown;
- continue;
+ continue;
}
/* We accept 'inherited' script characters currently even at the
continue;
}
- /* If unknown, the run's script is set to the char's */
- if (UNLIKELY(script_of_run == SCX_INVALID)) {
- script_of_run = script_of_char;
- }
-
- /* All decimal digits must be from the same sequence of 10. Above, we
- * handled any ASCII digits without descending to here. We also
- * handled the case where we already knew what digit sequence is the
- * one to use, and the character is in that sequence. Now that we know
- * the script, we can use script_zeros[] to directly find which
- * sequence the script uses, except in a few cases it returns 0 */
- if (UNLIKELY(zero_of_run == 0) && script_of_char >= 0) {
- zero_of_run = script_zeros[script_of_char];
- }
-
- /* Now we can see if the script of the character is the same as that of
- * the run */
- if (LIKELY(script_of_char == script_of_run)) {
- /* By far the most common case */
- goto scripts_match;
- }
-
- /* Here, the scripts of the run and the current character don't match
- * exactly. The run could so far have been entirely characters from
- * Common. It's now time to change its script to that of this
- * non-Common character */
- if (script_of_run == SCX_Common) {
+ /* If the run so far is Common, and the new character isn't, change the
+ * run's script to that of this character */
+ if (script_of_run == SCX_Common && script_of_char != SCX_Common) {
/* But Common contains several sets of digits. Only the '0' set
* can be part of another script. */
- if (zero_of_run > 0 && zero_of_run != '0') {
+ if (zero_of_run && zero_of_run != '0') {
retval = FALSE;
break;
}
script_of_run = script_of_char;
+ }
+
+ /* Now we can see if the script of the character is the same as that of
+ * the run */
+ if (LIKELY(script_of_char == script_of_run)) {
+ /* By far the most common case */
goto scripts_match;
}
/* Too early a Unicode version to have a code point belonging to more
* than one script, so, if the scripts don't exactly match, fail */
+ PERL_UNUSED_VAR(intersection_len);
retval = FALSE;
break;
/* If there is only a single script in common, set to that.
* Otherwise, use the intersection going forward */
Safefree(intersection);
+ intersection = NULL;
if (intersection_len == 1) {
script_of_run = script_of_char = new_overlap[0];
Safefree(new_overlap);
+ new_overlap = NULL;
}
else {
intersection = new_overlap;
scripts_match:
/* Here, the script of the character is compatible with that of the
- * run. Either they match exactly, or one or both can be any of
- * several scripts, and the intersection is not empty. If the
- * character is not a decimal digit, we are done with it. Otherwise,
- * it could still fail if it is from a different set of 10 than seen
- * already (or we may not have seen any, and we need to set the
- * sequence). If we have determined a single script and that script
- * only has one set of digits (almost all scripts are like that), then
- * this isn't a problem, as any digit must come from the same sequence.
- * The only scripts that have multiple sequences have been constructed
- * to be 0 in 'script_zeros[]'.
- *
- * Here we check if it is a digit. */
- if ( cp >= FIRST_NON_ASCII_DECIMAL_DIGIT
- && ( ( zero_of_run == 0
- || ( ( script_of_char >= 0
- && script_zeros[script_of_char] == 0)
- || intersection))))
+ * run. That means that in most cases, it continues the script run.
+ * Either it and the run match exactly, or one or both can be in any of
+ * several scripts, and the intersection is not empty. However, if the
+ * character is a decimal digit, it could still mean failure if it is
+ * from the wrong sequence of 10. So, we need to look at if it's a
+ * digit. We've already handled the 10 decimal digits, and the next
+ * lowest one is this one: */
+ if (cp < FIRST_NON_ASCII_DECIMAL_DIGIT) {
+ continue; /* Not a digit; this character is part of the run */
+ }
+
+ /* If we have a definitive '0' for the script of this character, we
+ * know that for this to be a digit, it must be in the range of +0..+9
+ * of that zero. */
+ if ( script_of_char >= 0
+ && (zero_of_char = script_zeros[script_of_char]))
{
- SSize_t range_zero_index;
- range_zero_index = _invlist_search(decimals_invlist, cp);
- if ( LIKELY(range_zero_index >= 0)
- && ELEMENT_RANGE_MATCHES_INVLIST(range_zero_index))
+ if ( cp < zero_of_char
+ || cp > zero_of_char + 9)
{
- UV range_zero = decimals_array[range_zero_index];
- if (zero_of_run) {
- if (zero_of_run != range_zero) {
- retval = FALSE;
- break;
- }
- }
- else {
- zero_of_run = range_zero;
- }
+ continue; /* Not a digit; this character is part of the run
+ */
}
+
+ }
+ else { /* Need to look up if this character is a digit or not */
+ SSize_t index_of_zero_of_char;
+ index_of_zero_of_char = _invlist_search(decimals_invlist, cp);
+ if ( UNLIKELY(index_of_zero_of_char < 0)
+ || ! ELEMENT_RANGE_MATCHES_INVLIST(index_of_zero_of_char))
+ {
+ continue; /* Not a digit; this character is part of the run.
+ */
+ }
+
+ zero_of_char = decimals_array[index_of_zero_of_char];
+ }
+
+ /* Here, the character is a decimal digit, and the zero of its sequence
+ * of 10 is in 'zero_of_char'. If we already have a zero for this run,
+ * they better be the same. */
+ if (zero_of_run) {
+ if (zero_of_run != zero_of_char) {
+ retval = FALSE;
+ break;
+ }
+ }
+ else if (script_of_char == SCX_Common && script_of_run != SCX_Common) {
+
+ /* Here, the script run isn't Common, but the current digit is in
+ * Common, and isn't '0'-'9' (those were handled earlier). Only
+ * '0'-'9' are acceptable in non-Common scripts. */
+ retval = FALSE;
+ break;
+ }
+ else { /* Otherwise we now have a zero for this run */
+ zero_of_run = zero_of_char;
}
} /* end of looping through CLOSESR text */
Safefree(intersection);
+
+ if (ret_script != NULL) {
+ if (retval) {
+ *ret_script = script_of_run;
+ }
+ else {
+ *ret_script = SCX_INVALID;
+ }
+ }
+
return retval;
}
https://perl5.git.perl.org / perl5.git / blobdiff