= "Can't match, because target string needs to be in UTF-8\n";
#endif
+/* Returns a boolean as to whether the input unsigned number is a power of 2
+ * (2**0, 2**1, etc). In other words if it has just a single bit set.
+ * If not, subtracting 1 would leave the uppermost bit set, so the & would
+ * yield non-zero */
+#define isPOWER_OF_2(n) ((n & (n-1)) == 0)
+
#define NON_UTF8_TARGET_BUT_UTF8_REQUIRED(target) STMT_START { \
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_ "%s", non_utf8_target_but_utf8_required));\
goto target; \
/* Currently these are only used when PL_regkind[OP(rn)] == EXACT so
we don't need this definition. XXX These are now out-of-sync*/
#define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==REF || OP(rn)==NREF )
-#define IS_TEXTF(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn)==EXACTFA || OP(rn)==EXACTFA_NO_TRIE || OP(rn)==EXACTF || OP(rn)==REFF || OP(rn)==NREFF )
+#define IS_TEXTF(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFU_SS || OP(rn)==EXACTFAA || OP(rn)==EXACTFAA_NO_TRIE || OP(rn)==EXACTF || OP(rn)==REFF || OP(rn)==NREFF )
#define IS_TEXTFL(rn) ( OP(rn)==EXACTFL || OP(rn)==REFFL || OP(rn)==NREFFL )
#else
/* ... so we use this as its faster. */
#define IS_TEXT(rn) ( OP(rn)==EXACT || OP(rn)==EXACTL )
-#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFLU8 || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFA || OP(rn) == EXACTFA_NO_TRIE)
+#define IS_TEXTFU(rn) ( OP(rn)==EXACTFU || OP(rn)==EXACTFLU8 || OP(rn)==EXACTFU_SS || OP(rn) == EXACTFAA || OP(rn) == EXACTFAA_NO_TRIE)
#define IS_TEXTF(rn) ( OP(rn)==EXACTF )
#define IS_TEXTFL(rn) ( OP(rn)==EXACTFL )
return FALSE; /* Things like CNTRL are always below 256 */
}
+STATIC char *
+S_find_next_ascii(char * s, const char * send, const bool utf8_target)
+{
+ /* Returns the position of the first ASCII byte in the sequence between 's'
+ * and 'send-1' inclusive; returns 'send' if none found */
+
+ PERL_ARGS_ASSERT_FIND_NEXT_ASCII;
+
+#ifndef EBCDIC
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+
+ /* This term is wordsize if subword; 0 if not */
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+
+ /* 'offset' */
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+
+ /* Process per-byte until reach word boundary. XXX This loop could be
+ * eliminated if we knew that this platform had fast unaligned reads */
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (isASCII(*s)) {
+ return s;
+ }
+ s++; /* khw didn't bother creating a separate loop for
+ utf8_target */
+ }
+
+ /* Here, we know we have at least one full word to process. Process
+ * per-word as long as we have at least a full word left */
+ do {
+ PERL_UINTMAX_T complemented = ~ * (PERL_UINTMAX_T *) s;
+ if (complemented & PERL_VARIANTS_WORD_MASK) {
+
+#if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 \
+ || BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
+
+ s += _variant_byte_number(complemented);
+ return s;
+
+#else /* If weird byte order, drop into next loop to do byte-at-a-time
+ checks. */
+
+ break;
+#endif
+ }
+
+ s += PERL_WORDSIZE;
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+#endif
+
+ /* Process per-character */
+ if (utf8_target) {
+ while (s < send) {
+ if (isASCII(*s)) {
+ return s;
+ }
+ s += UTF8SKIP(s);
+ }
+ }
+ else {
+ while (s < send) {
+ if (isASCII(*s)) {
+ return s;
+ }
+ s++;
+ }
+ }
+
+ return s;
+}
+
+STATIC char *
+S_find_next_non_ascii(char * s, const char * send, const bool utf8_target)
+{
+ /* Returns the position of the first non-ASCII byte in the sequence between
+ * 's' and 'send-1' inclusive; returns 'send' if none found */
+
+#ifdef EBCDIC
+
+ PERL_ARGS_ASSERT_FIND_NEXT_NON_ASCII;
+
+ if (utf8_target) {
+ while (s < send) {
+ if ( ! isASCII(*s)) {
+ return s;
+ }
+ s += UTF8SKIP(s);
+ }
+ }
+ else {
+ while (s < send) {
+ if ( ! isASCII(*s)) {
+ return s;
+ }
+ s++;
+ }
+ }
+
+ return s;
+
+#else
+
+ const U8 * next_non_ascii = NULL;
+
+ PERL_ARGS_ASSERT_FIND_NEXT_NON_ASCII;
+ PERL_UNUSED_ARG(utf8_target);
+
+ /* On ASCII platforms invariants and ASCII are identical, so if the string
+ * is entirely invariants, there is no non-ASCII character */
+ return (is_utf8_invariant_string_loc((U8 *) s,
+ (STRLEN) (send - s),
+ &next_non_ascii))
+ ? (char *) send
+ : (char *) next_non_ascii;
+
+#endif
+
+}
+
+STATIC char *
+S_find_span_end(char * s, const char * send, const char span_byte)
+{
+ /* Returns the position of the first byte in the sequence between 's' and
+ * 'send-1' inclusive that isn't 'span_byte'; returns 'send' if none found.
+ * */
+
+ PERL_ARGS_ASSERT_FIND_SPAN_END;
+
+ assert(send >= s);
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+ PERL_UINTMAX_T span_word;
+
+ /* Process per-byte until reach word boundary. XXX This loop could be
+ * eliminated if we knew that this platform had fast unaligned reads */
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (*s != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ /* Create a word filled with the bytes we are spanning */
+ span_word = PERL_COUNT_MULTIPLIER * span_byte;
+
+ /* Process per-word as long as we have at least a full word left */
+ do {
+
+ /* Keep going if the whole word is composed of 'span_byte's */
+ if ((* (PERL_UINTMAX_T *) s) == span_word) {
+ s += PERL_WORDSIZE;
+ continue;
+ }
+
+ /* Here, at least one byte in the word isn't 'span_byte'. This xor
+ * leaves 1 bits only in those non-matching bytes */
+ span_word ^= * (PERL_UINTMAX_T *) s;
+
+ /* Make sure the upper bit of each non-matching byte is set. This
+ * makes each such byte look like an ASCII platform variant byte */
+ span_word |= span_word << 1;
+ span_word |= span_word << 2;
+ span_word |= span_word << 4;
+
+ /* That reduces the problem to what this function solves */
+ return s + _variant_byte_number(span_word);
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+ /* Process the straggler bytes beyond the final word boundary */
+ while (s < send) {
+ if (*s != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ return s;
+}
+
+STATIC char *
+S_find_next_masked(char * s, const char * send, const U8 byte, const U8 mask)
+{
+ /* Returns the position of the first byte in the sequence between 's'
+ * and 'send-1' inclusive that when ANDed with 'mask' yields 'byte';
+ * returns 'send' if none found. It uses word-level operations instead of
+ * byte to speed up the process */
+
+ PERL_ARGS_ASSERT_FIND_NEXT_MASKED;
+
+ assert(send >= s);
+ assert((byte & mask) == byte);
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+ PERL_UINTMAX_T word_complemented, mask_word;
+
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (((* (U8 *) s) & mask) == byte) {
+ return s;
+ }
+ s++;
+ }
+
+ word_complemented = ~ (PERL_COUNT_MULTIPLIER * byte);
+ mask_word = PERL_COUNT_MULTIPLIER * mask;
+
+ do {
+ PERL_UINTMAX_T masked = (* (PERL_UINTMAX_T *) s) & mask_word;
+
+ /* If 'masked' contains 'byte' within it, anding with the
+ * complement will leave those 8 bits 0 */
+ masked &= word_complemented;
+
+ /* This causes the most significant bit to be set to 1 for any
+ * bytes in the word that aren't completely 0 */
+ masked |= masked << 1;
+ masked |= masked << 2;
+ masked |= masked << 4;
+
+ /* The msbits are the same as what marks a byte as variant, so we
+ * can use this mask. If all msbits are 1, the word doesn't
+ * contain 'byte' */
+ if ((masked & PERL_VARIANTS_WORD_MASK) == PERL_VARIANTS_WORD_MASK) {
+ s += PERL_WORDSIZE;
+ continue;
+ }
+
+ /* Here, the msbit of bytes in the word that aren't 'byte' are 1,
+ * and any that are, are 0. Complement and re-AND to swap that */
+ masked = ~ masked;
+ masked &= PERL_VARIANTS_WORD_MASK;
+
+ /* This reduces the problem to that solved by this function */
+ s += _variant_byte_number(masked);
+ return s;
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+ while (s < send) {
+ if (((* (U8 *) s) & mask) == byte) {
+ return s;
+ }
+ s++;
+ }
+
+ return s;
+}
+
+STATIC U8 *
+S_find_span_end_mask(U8 * s, const U8 * send, const U8 span_byte, const U8 mask)
+{
+ /* Returns the position of the first byte in the sequence between 's' and
+ * 'send-1' inclusive that when ANDed with 'mask' isn't 'span_byte'.
+ * 'span_byte' should have been ANDed with 'mask' in the call of this
+ * function. Returns 'send' if none found. Works like find_span_end(),
+ * except for the AND */
+
+ PERL_ARGS_ASSERT_FIND_SPAN_END_MASK;
+
+ assert(send >= s);
+ assert((span_byte & mask) == span_byte);
+
+ if ((STRLEN) (send - s) >= PERL_WORDSIZE
+ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(s)
+ - (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK))
+ {
+ PERL_UINTMAX_T span_word, mask_word;
+
+ while (PTR2nat(s) & PERL_WORD_BOUNDARY_MASK) {
+ if (((* (U8 *) s) & mask) != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ span_word = PERL_COUNT_MULTIPLIER * span_byte;
+ mask_word = PERL_COUNT_MULTIPLIER * mask;
+
+ do {
+ PERL_UINTMAX_T masked = (* (PERL_UINTMAX_T *) s) & mask_word;
+
+ if (masked == span_word) {
+ s += PERL_WORDSIZE;
+ continue;
+ }
+
+ masked ^= span_word;
+ masked |= masked << 1;
+ masked |= masked << 2;
+ masked |= masked << 4;
+ return s + _variant_byte_number(masked);
+
+ } while (s + PERL_WORDSIZE <= send);
+ }
+
+ while (s < send) {
+ if (((* (U8 *) s) & mask) != span_byte) {
+ return s;
+ }
+ s++;
+ }
+
+ return s;
+}
+
/*
* pregexec and friends
*/
* /\d+abc/: "abc" is anchored to neither the pattern nor the string;
* /^\d+abc/: "abc" is anchored to neither the pattern nor the string,
* but the pattern is anchored to the string.
- *
- * Note the variables names suffixed with _c represent character counts
- * while _b represent byte counts
*/
char *
re_scream_pos_data *data)
{
struct regexp *const prog = ReANY(rx);
-
- /* Minimum number of chars which *must* precede the check substring to
- * be capable of matching, e.g. 2 in /[ab]cd?substring/. */
- SSize_t check_min_offset_c = prog->check_offset_min;
-
- /* Minimum number of chars which *must* follow the check substring to
- * be capable of matching, e.g. 2 in /substring[ab]cd?/.
- * Should be nonnegative! */
-
- SSize_t check_end_shift_c = 0;
+ SSize_t start_shift = prog->check_offset_min;
+ /* Should be nonnegative! */
+ SSize_t end_shift = 0;
/* current lowest pos in string where the regex can start matching */
char *rx_origin = strpos;
SV *check;
if (prog->check_offset_min == prog->check_offset_max) {
/* Substring at constant offset from beg-of-str... */
- SSize_t slen_b = SvCUR(check);
+ SSize_t slen = SvCUR(check);
char *s = HOP3c(strpos, prog->check_offset_min, strend);
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
if (SvTAIL(check)) {
/* In this case, the regex is anchored at the end too.
* Unless it's a multiline match, the lengths must match
- * exactly, give or take a \n. NB: slen_b >= 1 since
+ * exactly, give or take a \n. NB: slen >= 1 since
* the last char of check is \n */
if (!multiline
- && ( strend - s > slen_b
- || strend - s < slen_b - 1
- || (strend - s == slen_b && strend[-1] != '\n')))
+ && ( strend - s > slen
+ || strend - s < slen - 1
+ || (strend - s == slen && strend[-1] != '\n')))
{
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
" String too long...\n"));
goto fail_finish;
}
- /* Now should match s[0..slen_b-2] */
- slen_b--;
+ /* Now should match s[0..slen-2] */
+ slen--;
}
- if (slen_b && (strend - s < slen_b
+ if (slen && (strend - s < slen
|| *SvPVX_const(check) != *s
- || (slen_b > 1 && (memNE(SvPVX_const(check), s, slen_b)))))
+ || (slen > 1 && (memNE(SvPVX_const(check), s, slen)))))
{
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
" String not equal...\n"));
}
}
- check_end_shift_c = prog->check_end_shift;
+ end_shift = prog->check_end_shift;
#ifdef DEBUGGING /* 7/99: reports of failure (with the older version) */
- if (check_end_shift_c < 0)
+ if (end_shift < 0)
Perl_croak(aTHX_ "panic: end_shift: %" IVdf " pattern:\n%s\n ",
- (IV)check_end_shift_c, RX_PRECOMP(rx));
+ (IV)end_shift, RX_PRECOMP(rx));
#endif
restart:
/* This is the (re)entry point of the main loop in this function.
* The goal of this loop is to:
* 1) find the "check" substring in the region rx_origin..strend
- * (adjusted by check_min_offset_c / check_end_shift_c). If not found, reject
+ * (adjusted by start_shift / end_shift). If not found, reject
* immediately.
* 2) If it exists, look for the "other" substr too if defined; for
* example, if the check substr maps to the anchored substr, then
" Real end Shift: %" IVdf "\n",
(IV)(rx_origin - strbeg),
(IV)prog->check_offset_min,
- (IV)check_min_offset_c,
- (IV)check_end_shift_c,
+ (IV)start_shift,
+ (IV)end_shift,
(IV)prog->check_end_shift);
});
- end_point = HOPBACK3(strend, check_end_shift_c, rx_origin);
+ end_point = HOPBACK3(strend, end_shift, rx_origin);
if (!end_point)
goto fail_finish;
- start_point = HOPMAYBE3(rx_origin, check_min_offset_c, end_point);
+ start_point = HOPMAYBE3(rx_origin, start_shift, end_point);
if (!start_point)
goto fail_finish;
&& prog->intflags & PREGf_ANCH
&& prog->check_offset_max != SSize_t_MAX)
{
- SSize_t check_len_b = SvCUR(check) - !!SvTAIL(check);
+ SSize_t check_len = SvCUR(check) - !!SvTAIL(check);
const char * const anchor =
(prog->intflags & PREGf_ANCH_GPOS ? strpos : strbeg);
- SSize_t targ_len_b = (char*)end_point - anchor;
+ SSize_t targ_len = (char*)end_point - anchor;
- if (check_len_b > targ_len_b) {
+ if (check_len > targ_len) {
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
- "Anchored string too short...\n"));
+ "Target string too short to match required substring...\n"));
goto fail_finish;
}
* so it skips doing the HOP if the result can't possibly end
* up earlier than the old value of end_point.
*/
- assert(anchor + check_len_b <= (char *)end_point);
- if (prog->check_offset_max + check_len_b < targ_len_b) {
+ assert(anchor + check_len <= (char *)end_point);
+ if (prog->check_offset_max + check_len < targ_len) {
end_point = HOP3lim((U8*)anchor,
prog->check_offset_max,
- end_point - check_len_b
+ end_point - check_len
)
- + check_len_b;
+ + check_len;
+ if (end_point < start_point)
+ goto fail_finish;
}
}
: (char*)HOP3lim(rx_origin, other->max_offset, last1);
}
else {
- assert(strpos + check_min_offset_c <= check_at);
- last = HOP4c(check_at, other->min_offset - check_min_offset_c,
+ assert(strpos + start_shift <= check_at);
+ last = HOP4c(check_at, other->min_offset - start_shift,
strbeg, strend);
}
if (progi->regstclass && PL_regkind[OP(progi->regstclass)]!=TRIE) {
const U8* const str = (U8*)STRING(progi->regstclass);
- /* The length (in chars) if the character class. This is almost
- * always 1.
- * XXX this value could be pre-computed */
- const int class_len_c = (PL_regkind[OP(progi->regstclass)] == EXACT
+ /* XXX this value could be pre-computed */
+ const int cl_l = (PL_regkind[OP(progi->regstclass)] == EXACT
? (reginfo->is_utf8_pat
? utf8_distance(str + STR_LEN(progi->regstclass), str)
: STR_LEN(progi->regstclass))
*/
if (prog->anchored_substr || prog->anchored_utf8 || ml_anch)
- endpos = HOP3clim(rx_origin,
- (prog->minlen ? class_len_c : 0),
- strend);
+ endpos = HOP3clim(rx_origin, (prog->minlen ? cl_l : 0), strend);
else if (prog->float_substr || prog->float_utf8) {
- rx_max_float = HOP3c(check_at, -check_min_offset_c, strbeg);
- endpos = HOP3clim(rx_max_float, class_len_c, strend);
+ rx_max_float = HOP3c(check_at, -start_shift, strbeg);
+ endpos = HOP3clim(rx_max_float, cl_l, strend);
}
else
endpos= strend;
DEBUG_EXECUTE_r(Perl_re_printf( aTHX_
- " looking for class: check_min_offset: %" IVdf " check_at: %" IVdf
+ " looking for class: start_shift: %" IVdf " check_at: %" IVdf
" rx_origin: %" IVdf " endpos: %" IVdf "\n",
- (IV)check_min_offset_c, (IV)(check_at - strbeg),
+ (IV)start_shift, (IV)(check_at - strbeg),
(IV)(rx_origin - strbeg), (IV)(endpos - strbeg)));
s = find_byclass(prog, progi->regstclass, rx_origin, endpos,
if (prog->anchored_substr || prog->anchored_utf8) {
if (prog->substrs->check_ix == 1) { /* check is float */
/* Have both, check_string is floating */
- assert(rx_origin + check_min_offset_c <= check_at);
- if (rx_origin + check_min_offset_c != check_at) {
+ assert(rx_origin + start_shift <= check_at);
+ if (rx_origin + start_shift != check_at) {
/* not at latest position float substr could match:
* Recheck anchored substring, but not floating.
* The condition above is in bytes rather than
/* uses bytes rather than char calculations for efficiency.
* It's conservative: it errs on the side of doing 'goto restart',
* where there is code that does a proper char-based test */
- if (rx_origin + check_min_offset_c + check_end_shift_c > strend) {
+ if (rx_origin + start_shift + end_shift > strend) {
DEBUG_EXECUTE_r( Perl_re_printf( aTHX_
" Could not match STCLASS...\n") );
goto fail;
DEBUG_EXECUTE_r( Perl_re_printf( aTHX_
" about to look for %s substr starting at offset %ld (rx_origin now %" IVdf ")...\n",
(prog->substrs->check_ix ? "floating" : "anchored"),
- (long)(rx_origin + check_min_offset_c - strbeg),
+ (long)(rx_origin + start_shift - strbeg),
(IV)(rx_origin - strbeg)
));
goto restart;
? (utf8_target ? trie_utf8 : trie_plain) \
: (scan->flags == EXACTL) \
? (utf8_target ? trie_utf8l : trie_plain) \
- : (scan->flags == EXACTFA) \
+ : (scan->flags == EXACTFAA) \
? (utf8_target \
? trie_utf8_exactfa_fold \
: trie_latin_utf8_exactfa_fold) \
dump_exec_pos(li,s,(reginfo->strend),(reginfo->strbeg), \
startpos, doutf8, depth)
-#define REXEC_FBC_EXACTISH_SCAN(COND) \
-STMT_START { \
- while (s <= e) { \
- if ( (COND) \
- && (ln == 1 || folder(s, pat_string, ln)) \
- && (reginfo->intuit || regtry(reginfo, &s)) )\
- goto got_it; \
- s++; \
- } \
-} STMT_END
-
-#define REXEC_FBC_UTF8_SCAN(CODE) \
-STMT_START { \
- while (s < strend) { \
- CODE \
- s += UTF8SKIP(s); \
- } \
-} STMT_END
-
-#define REXEC_FBC_SCAN(CODE) \
-STMT_START { \
- while (s < strend) { \
- CODE \
- s++; \
- } \
-} STMT_END
+#define REXEC_FBC_SCAN(UTF8, CODE) \
+ STMT_START { \
+ while (s < strend) { \
+ CODE \
+ s += ((UTF8) ? UTF8SKIP(s) : 1); \
+ } \
+ } STMT_END
-#define REXEC_FBC_UTF8_CLASS_SCAN(COND) \
-REXEC_FBC_UTF8_SCAN( /* Loops while (s < strend) */ \
- if (COND) { \
- if (tmp && (reginfo->intuit || regtry(reginfo, &s))) \
- goto got_it; \
- else \
- tmp = doevery; \
- } \
- else \
- tmp = 1; \
-)
+#define REXEC_FBC_CLASS_SCAN(UTF8, COND) \
+ STMT_START { \
+ while (s < strend) { \
+ REXEC_FBC_CLASS_SCAN_GUTS(UTF8, COND) \
+ } \
+ } STMT_END
-#define REXEC_FBC_CLASS_SCAN(COND) \
-REXEC_FBC_SCAN( /* Loops while (s < strend) */ \
+#define REXEC_FBC_CLASS_SCAN_GUTS(UTF8, COND) \
if (COND) { \
- if (tmp && (reginfo->intuit || regtry(reginfo, &s))) \
- goto got_it; \
- else \
- tmp = doevery; \
+ FBC_CHECK_AND_TRY \
+ s += ((UTF8) ? UTF8SKIP(s) : 1); \
+ previous_occurrence_end = s; \
} \
- else \
- tmp = 1; \
-)
+ else { \
+ s += ((UTF8) ? UTF8SKIP(s) : 1); \
+ }
#define REXEC_FBC_CSCAN(CONDUTF8,COND) \
if (utf8_target) { \
- REXEC_FBC_UTF8_CLASS_SCAN(CONDUTF8); \
+ REXEC_FBC_CLASS_SCAN(1, CONDUTF8); \
} \
else { \
- REXEC_FBC_CLASS_SCAN(COND); \
+ REXEC_FBC_CLASS_SCAN(0, COND); \
+ }
+
+/* We keep track of where the next character should start after an occurrence
+ * of the one we're looking for. Knowing that, we can see right away if the
+ * next occurrence is adjacent to the previous. When 'doevery' is FALSE, we
+ * don't accept the 2nd and succeeding adjacent occurrences */
+#define FBC_CHECK_AND_TRY \
+ if ( ( doevery \
+ || s != previous_occurrence_end) \
+ && (reginfo->intuit || regtry(reginfo, &s))) \
+ { \
+ goto got_it; \
+ }
+
+
+/* This differs from the above macros in that it calls a function which returns
+ * the next occurrence of the thing being looked for in 's'; and 'strend' if
+ * there is no such occurrence. */
+#define REXEC_FBC_FIND_NEXT_SCAN(UTF8, f) \
+ while (s < strend) { \
+ s = f; \
+ if (s >= strend) { \
+ break; \
+ } \
+ \
+ FBC_CHECK_AND_TRY \
+ s += (UTF8) ? UTF8SKIP(s) : 1; \
+ previous_occurrence_end = s; \
}
/* The three macros below are slightly different versions of the same logic.
* here. And vice-versa if we are looking for a non-boundary.
*
* 'tmp' below in the next three macros in the REXEC_FBC_SCAN and
- * REXEC_FBC_UTF8_SCAN loops is a loop invariant, a bool giving the return of
+ * REXEC_FBC_SCAN loops is a loop invariant, a bool giving the return of
* TEST_NON_UTF8(s-1). To see this, note that that's what it is defined to be
* at entry to the loop, and to get to the IF_FAIL branch, tmp must equal
* TEST_NON_UTF8(s), and in the opposite branch, IF_SUCCESS, tmp is that
#define FBC_UTF8_A(TEST_NON_UTF8, IF_SUCCESS, IF_FAIL) \
tmp = (s != reginfo->strbeg) ? UCHARAT(s - 1) : '\n'; \
tmp = TEST_NON_UTF8(tmp); \
- REXEC_FBC_UTF8_SCAN( /* advances s while s < strend */ \
+ REXEC_FBC_SCAN(1, /* 1=>is-utf8; advances s while s < strend */ \
if (tmp == ! TEST_NON_UTF8((U8) *s)) { \
tmp = !tmp; \
IF_SUCCESS; /* Is a boundary if values for s-1 and s differ */ \
} \
tmp = TEST_UV(tmp); \
LOAD_UTF8_CHARCLASS_ALNUM(); \
- REXEC_FBC_UTF8_SCAN( /* advances s while s < strend */ \
+ REXEC_FBC_SCAN(1, /* 1=>is-utf8; advances s while s < strend */ \
if (tmp == ! (TEST_UTF8((U8 *) s, (U8 *) reginfo->strend))) { \
tmp = !tmp; \
IF_SUCCESS; \
else { /* Not utf8 */ \
tmp = (s != reginfo->strbeg) ? UCHARAT(s - 1) : '\n'; \
tmp = TEST_NON_UTF8(tmp); \
- REXEC_FBC_SCAN( /* advances s while s < strend */ \
+ REXEC_FBC_SCAN(0, /* 0=>not-utf8; advances s while s < strend */ \
if (tmp == ! TEST_NON_UTF8((U8) *s)) { \
IF_SUCCESS; \
tmp = !tmp; \
const char *strend, regmatch_info *reginfo)
{
dVAR;
+
+ /* TRUE if x+ need not match at just the 1st pos of run of x's */
const I32 doevery = (prog->intflags & PREGf_SKIP) == 0;
+
char *pat_string; /* The pattern's exactish string */
char *pat_end; /* ptr to end char of pat_string */
re_fold_t folder; /* Function for computing non-utf8 folds */
U8 c1;
U8 c2;
char *e;
- I32 tmp = 1; /* Scratch variable? */
+
+ /* In some cases we accept only the first occurence of 'x' in a sequence of
+ * them. This variable points to just beyond the end of the previous
+ * occurrence of 'x', hence we can tell if we are in a sequence. (Having
+ * it point to beyond the 'x' allows us to work for UTF-8 without having to
+ * hop back.) */
+ char * previous_occurrence_end = 0;
+
+ I32 tmp; /* Scratch variable */
const bool utf8_target = reginfo->is_utf8_target;
UV utf8_fold_flags = 0;
const bool is_utf8_pat = reginfo->is_utf8_pat;
case ANYOFD:
case ANYOF:
if (utf8_target) {
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1, /* 1=>is-utf8 */
reginclass(prog, c, (U8*)s, (U8*) strend, utf8_target));
}
else if (ANYOF_FLAGS(c)) {
- REXEC_FBC_CLASS_SCAN(reginclass(prog,c, (U8*)s, (U8*)s+1, 0));
+ REXEC_FBC_CLASS_SCAN(0, reginclass(prog,c, (U8*)s, (U8*)s+1, 0));
}
else {
- REXEC_FBC_CLASS_SCAN(ANYOF_BITMAP_TEST(c, *((U8*)s)));
+ REXEC_FBC_CLASS_SCAN(0, ANYOF_BITMAP_TEST(c, *((U8*)s)));
}
break;
- case EXACTFA_NO_TRIE: /* This node only generated for non-utf8 patterns */
+ case ANYOFM: /* ARG() is the base byte; FLAGS() the mask byte */
+ /* UTF-8ness doesn't matter, so use 0 */
+ REXEC_FBC_FIND_NEXT_SCAN(0,
+ find_next_masked(s, strend, ARG(c), FLAGS(c)));
+ break;
+
+ case EXACTFAA_NO_TRIE: /* This node only generated for non-utf8 patterns */
assert(! is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA:
+ case EXACTFAA:
if (is_utf8_pat || utf8_target) {
utf8_fold_flags = FOLDEQ_UTF8_NOMIX_ASCII;
goto do_exactf_utf8;
c1 = *pat_string;
c2 = fold_array[c1];
if (c1 == c2) { /* If char and fold are the same */
- REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1);
+ while (s <= e) {
+ s = (char *) memchr(s, c1, e + 1 - s);
+ if (s == NULL) {
+ break;
+ }
+
+ /* Check that the rest of the node matches */
+ if ( (ln == 1 || folder(s + 1, pat_string + 1, ln - 1))
+ && (reginfo->intuit || regtry(reginfo, &s)) )
+ {
+ goto got_it;
+ }
+ s++;
+ }
}
else {
- REXEC_FBC_EXACTISH_SCAN(*(U8*)s == c1 || *(U8*)s == c2);
+ U8 bits_differing = c1 ^ c2;
+
+ /* If the folds differ in one bit position only, we can mask to
+ * match either of them, and can use this faster find method. Both
+ * ASCII and EBCDIC tend to have their case folds differ in only
+ * one position, so this is very likely */
+ if (LIKELY(PL_bitcount[bits_differing] == 1)) {
+ bits_differing = ~ bits_differing;
+ while (s <= e) {
+ s = find_next_masked(s, e + 1,
+ (c1 & bits_differing), bits_differing);
+ if (s > e) {
+ break;
+ }
+
+ if ( (ln == 1 || folder(s + 1, pat_string + 1, ln - 1))
+ && (reginfo->intuit || regtry(reginfo, &s)) )
+ {
+ goto got_it;
+ }
+ s++;
+ }
+ }
+ else { /* Otherwise, stuck with looking byte-at-a-time. This
+ should actually happen only in EXACTFL nodes */
+ while (s <= e) {
+ if ( (*(U8*)s == c1 || *(U8*)s == c2)
+ && (ln == 1 || folder(s + 1, pat_string + 1, ln - 1))
+ && (reginfo->intuit || regtry(reginfo, &s)) )
+ {
+ goto got_it;
+ }
+ s++;
+ }
+ }
}
break;
);
break;
+ case ASCII:
+ REXEC_FBC_FIND_NEXT_SCAN(0, find_next_ascii(s, strend, utf8_target));
+ break;
+
+ case NASCII:
+ if (utf8_target) {
+ REXEC_FBC_FIND_NEXT_SCAN(1, find_next_non_ascii(s, strend,
+ utf8_target));
+ }
+ else {
+ REXEC_FBC_FIND_NEXT_SCAN(0, find_next_non_ascii(s, strend,
+ utf8_target));
+ }
+
+ break;
+
/* The argument to all the POSIX node types is the class number to pass to
* _generic_isCC() to build a mask for searching in PL_charclass[] */
if (utf8_target) {
/* The complement of something that matches only ASCII matches all
* non-ASCII, plus everything in ASCII that isn't in the class. */
- REXEC_FBC_UTF8_CLASS_SCAN( ! isASCII_utf8_safe(s, strend)
- || ! _generic_isCC_A(*s, FLAGS(c)));
+ REXEC_FBC_CLASS_SCAN(1, ! isASCII_utf8_safe(s, strend)
+ || ! _generic_isCC_A(*s, FLAGS(c)));
break;
}
to_complement = 1;
- /* FALLTHROUGH */
+ goto posixa;
case POSIXA:
- posixa:
/* Don't need to worry about utf8, as it can match only a single
- * byte invariant character. */
- REXEC_FBC_CLASS_SCAN(
+ * byte invariant character. But we do anyway for performance reasons,
+ * as otherwise we would have to examine all the continuation
+ * characters */
+ if (utf8_target) {
+ REXEC_FBC_CLASS_SCAN(1, _generic_isCC_A(*s, FLAGS(c)));
+ break;
+ }
+
+ posixa:
+ REXEC_FBC_CLASS_SCAN(0, /* 0=>not-utf8 */
to_complement ^ cBOOL(_generic_isCC_A(*s, FLAGS(c))));
break;
case POSIXU:
if (! utf8_target) {
- REXEC_FBC_CLASS_SCAN(to_complement ^ cBOOL(_generic_isCC(*s,
+ REXEC_FBC_CLASS_SCAN(0, /* 0=>not-utf8 */
+ to_complement ^ cBOOL(_generic_isCC(*s,
FLAGS(c))));
}
else {
/* We avoid loading in the swash as long as possible, but
* should we have to, we jump to a separate loop. This
* extra 'if' statement is what keeps this code from being
- * just a call to REXEC_FBC_UTF8_CLASS_SCAN() */
+ * just a call to REXEC_FBC_CLASS_SCAN() */
if (UTF8_IS_ABOVE_LATIN1(*s)) {
goto found_above_latin1;
}
- if ((UTF8_IS_INVARIANT(*s)
+
+ REXEC_FBC_CLASS_SCAN_GUTS(1, (UTF8_IS_INVARIANT(*s)
&& to_complement ^ cBOOL(_generic_isCC((U8) *s,
classnum)))
|| ( UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, strend)
&& to_complement ^ cBOOL(
_generic_isCC(EIGHT_BIT_UTF8_TO_NATIVE(*s,
*(s + 1)),
- classnum))))
- {
- if (tmp && (reginfo->intuit || regtry(reginfo, &s)))
- goto got_it;
- else {
- tmp = doevery;
- }
- }
- else {
- tmp = 1;
- }
- s += UTF8SKIP(s);
+ classnum))));
}
}
else switch (classnum) { /* These classes are implemented as
macros */
case _CC_ENUM_SPACE:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1, /* 1=>is-utf8 */
to_complement ^ cBOOL(isSPACE_utf8_safe(s, strend)));
break;
case _CC_ENUM_BLANK:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isBLANK_utf8_safe(s, strend)));
break;
case _CC_ENUM_XDIGIT:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isXDIGIT_utf8_safe(s, strend)));
break;
case _CC_ENUM_VERTSPACE:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isVERTWS_utf8_safe(s, strend)));
break;
case _CC_ENUM_CNTRL:
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1,
to_complement ^ cBOOL(isCNTRL_utf8_safe(s, strend)));
break;
/* This is a copy of the loop above for swash classes, though using the
* FBC macro instead of being expanded out. Since we've loaded the
* swash, we don't have to check for that each time through the loop */
- REXEC_FBC_UTF8_CLASS_SCAN(
+ REXEC_FBC_CLASS_SCAN(1, /* 1=>is-utf8 */
to_complement ^ cBOOL(_generic_utf8_safe(
classnum,
s,
to_utf8_substr(prog);
}
ch = SvPVX_const(prog->anchored_utf8)[0];
- REXEC_FBC_SCAN(
+ REXEC_FBC_SCAN(0, /* 0=>not-utf8 */
if (*s == ch) {
DEBUG_EXECUTE_r( did_match = 1 );
if (regtry(reginfo, &s)) goto got_it;
}
}
ch = SvPVX_const(prog->anchored_substr)[0];
- REXEC_FBC_SCAN(
+ REXEC_FBC_SCAN(0, /* 0=>not-utf8 */
if (*s == ch) {
DEBUG_EXECUTE_r( did_match = 1 );
if (regtry(reginfo, &s)) goto got_it;
c2 = SvUV(*c_p);
/* Folds that cross the 255/256 boundary are forbidden
- * if EXACTFL (and isnt a UTF8 locale), or EXACTFA and
+ * if EXACTFL (and isnt a UTF8 locale), or EXACTFAA and
* one is ASCIII. Since the pattern character is above
* 255, and its only other match is below 256, the only
* legal match will be to itself. We have thrown away
if ((c1 < 256) != (c2 < 256)) {
if ((OP(text_node) == EXACTFL
&& ! IN_UTF8_CTYPE_LOCALE)
- || ((OP(text_node) == EXACTFA
- || OP(text_node) == EXACTFA_NO_TRIE)
+ || ((OP(text_node) == EXACTFAA
+ || OP(text_node) == EXACTFAA_NO_TRIE)
&& (isASCII(c1) || isASCII(c2))))
{
if (c1 < 256) {
if (utf8_target
&& HAS_NONLATIN1_FOLD_CLOSURE(c1)
&& ( ! (OP(text_node) == EXACTFL && ! IN_UTF8_CTYPE_LOCALE))
- && ((OP(text_node) != EXACTFA
- && OP(text_node) != EXACTFA_NO_TRIE)
+ && ((OP(text_node) != EXACTFAA
+ && OP(text_node) != EXACTFAA_NO_TRIE)
|| ! isASCII(c1)))
{
/* Here, there could be something above Latin1 in the target
}
/* FALLTHROUGH */
/* /u rules for all these. This happens to work for
- * EXACTFA as nothing in Latin1 folds to ASCII */
- case EXACTFA_NO_TRIE: /* This node only generated for
- non-utf8 patterns */
+ * EXACTFAA as nothing in Latin1 folds to ASCII */
+ case EXACTFAA_NO_TRIE: /* This node only generated for
+ non-utf8 patterns */
assert(! is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA:
+ case EXACTFAA:
case EXACTFU_SS:
case EXACTFU:
c2 = PL_fold_latin1[c1];
bool is_utf8_pat = reginfo->is_utf8_pat;
bool match = FALSE;
I32 orig_savestack_ix = PL_savestack_ix;
+ U8 * script_run_begin = NULL;
/* Solaris Studio 12.3 messes up fetching PL_charclass['\n'] */
#if (defined(__SUNPRO_C) && (__SUNPRO_C == 0x5120) && defined(__x86_64) && defined(USE_64_BIT_ALL))
}));
while (scan != NULL) {
-
-
next = scan + NEXT_OFF(scan);
if (next == scan)
next = NULL;
fold_utf8_flags = is_utf8_pat ? FOLDEQ_S1_ALREADY_FOLDED : 0;
goto do_exactf;
- case EXACTFA_NO_TRIE: /* This node only generated for non-utf8
+ case EXACTFAA_NO_TRIE: /* This node only generated for non-utf8
patterns */
assert(! is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA: /* /abc/iaa */
+ case EXACTFAA: /* /abc/iaa */
folder = foldEQ_latin1;
fold_array = PL_fold_latin1;
fold_utf8_flags = FOLDEQ_UTF8_NOMIX_ASCII;
}
break;
+ case ANYOFM:
+ if (NEXTCHR_IS_EOS || (UCHARAT(locinput) & FLAGS(scan)) != ARG(scan)) {
+ sayNO;
+ }
+ locinput++;
+ break;
+
+ case ASCII:
+ if (NEXTCHR_IS_EOS || ! isASCII(UCHARAT(locinput))) {
+ sayNO;
+ }
+
+ locinput++; /* ASCII is always single byte */
+ break;
+
+ case NASCII:
+ if (NEXTCHR_IS_EOS || isASCII(UCHARAT(locinput))) {
+ sayNO;
+ }
+
+ goto increment_locinput;
+ break;
+
/* The argument (FLAGS) to all the POSIX node types is the class number
* */
lastopen = n;
break;
+ case SROPEN: /* (*SCRIPT_RUN: */
+ script_run_begin = (U8 *) locinput;
+ break;
+
/* XXX really need to log other places start/end are set too */
#define CLOSE_CAPTURE \
rex->offs[n].start = rex->offs[n].start_tmp; \
break;
+ case SRCLOSE: /* (*SCRIPT_RUN: ... ) */
+
+ if (! isSCRIPT_RUN(script_run_begin, (U8 *) locinput, utf8_target, NULL))
+ {
+ sayNO;
+ }
+
+ break;
+
+
case ACCEPT: /* (*ACCEPT) */
if (scan->flags)
sv_yes_mark = MUTABLE_SV(rexi->data->data[ ARG( scan ) ]);
}
else { /* Not utf8_target */
if (ST.c1 == ST.c2) {
- while (locinput <= ST.maxpos &&
- UCHARAT(locinput) != ST.c1)
- locinput++;
- }
- else {
- while (locinput <= ST.maxpos
- && UCHARAT(locinput) != ST.c1
- && UCHARAT(locinput) != ST.c2)
- locinput++;
+ locinput = (char *) memchr(locinput,
+ ST.c1,
+ ST.maxpos + 1 - locinput);
+ if (! locinput) {
+ locinput = ST.maxpos + 1;
+ }
}
+ else {
+ U8 c1_c2_bits_differing = ST.c1 ^ ST.c2;
+
+ if (! isPOWER_OF_2(c1_c2_bits_differing)) {
+ while ( locinput <= ST.maxpos
+ && UCHARAT(locinput) != ST.c1
+ && UCHARAT(locinput) != ST.c2)
+ {
+ locinput++;
+ }
+ }
+ else {
+ /* If c1 and c2 only differ by a single bit, we can
+ * avoid a conditional each time through the loop,
+ * at the expense of a little preliminary setup and
+ * an extra mask each iteration. By masking out
+ * that bit, we match exactly two characters, c1
+ * and c2, and so we don't have to test for both.
+ * On both ASCII and EBCDIC platforms, most of the
+ * ASCII-range and Latin1-range folded equivalents
+ * differ only in a single bit, so this is actually
+ * the most common case. (e.g. 'A' 0x41 vs 'a'
+ * 0x61). */
+ U8 c1_masked = ST.c1 &~ c1_c2_bits_differing;
+ U8 c1_c2_mask = ~ c1_c2_bits_differing;
+ while ( locinput <= ST.maxpos
+ && (UCHARAT(locinput) & c1_c2_mask)
+ != c1_masked)
+ {
+ locinput++;
+ }
+ }
+ }
n = locinput - ST.oldloc;
}
if (locinput > ST.maxpos)
hardcount++;
}
} else {
- while (scan < loceol && *scan != '\n')
- scan++;
+ scan = (char *) memchr(scan, '\n', loceol - scan);
+ if (! scan) {
+ scan = loceol;
+ }
}
break;
case SANY:
c = (U8)*STRING(p);
- /* Can use a simple loop if the pattern char to match on is invariant
+ /* Can use a simple find if the pattern char to match on is invariant
* under UTF-8, or both target and pattern aren't UTF-8. Note that we
* can use UTF8_IS_INVARIANT() even if the pattern isn't UTF-8, as it's
* true iff it doesn't matter if the argument is in UTF-8 or not */
* since here, to match at all, 1 char == 1 byte */
loceol = scan + max;
}
- while (scan < loceol && UCHARAT(scan) == c) {
- scan++;
- }
+ scan = find_span_end(scan, loceol, (U8) c);
}
else if (reginfo->is_utf8_pat) {
if (utf8_target) {
else if (! UTF8_IS_ABOVE_LATIN1(c)) {
/* Target isn't utf8; convert the character in the UTF-8
- * pattern to non-UTF8, and do a simple loop */
+ * pattern to non-UTF8, and do a simple find */
c = EIGHT_BIT_UTF8_TO_NATIVE(c, *(STRING(p) + 1));
- while (scan < loceol && UCHARAT(scan) == c) {
- scan++;
- }
+ scan = find_span_end(scan, loceol, (U8) c);
} /* else pattern char is above Latin1, can't possibly match the
non-UTF-8 target */
}
}
break;
- case EXACTFA_NO_TRIE: /* This node only generated for non-utf8 patterns */
+ case EXACTFAA_NO_TRIE: /* This node only generated for non-utf8 patterns */
assert(! reginfo->is_utf8_pat);
/* FALLTHROUGH */
- case EXACTFA:
+ case EXACTFAA:
utf8_flags = FOLDEQ_UTF8_NOMIX_ASCII;
goto do_exactf;
}
}
else if (c1 == c2) {
- while (scan < loceol && UCHARAT(scan) == c1) {
- scan++;
- }
+ scan = find_span_end(scan, loceol, c1);
}
else {
- while (scan < loceol &&
- (UCHARAT(scan) == c1 || UCHARAT(scan) == c2))
- {
- scan++;
+ /* See comments in regmatch() CURLY_B_min_known_fail. We avoid
+ * a conditional each time through the loop if the characters
+ * differ only in a single bit, as is the usual situation */
+ U8 c1_c2_bits_differing = c1 ^ c2;
+
+ if (isPOWER_OF_2(c1_c2_bits_differing)) {
+ U8 c1_c2_mask = ~ c1_c2_bits_differing;
+
+ scan = (char *) find_span_end_mask((U8 *) scan,
+ (U8 *) loceol,
+ c1 & c1_c2_mask,
+ c1_c2_mask);
+ }
+ else {
+ while ( scan < loceol
+ && (UCHARAT(scan) == c1 || UCHARAT(scan) == c2))
+ {
+ scan++;
+ }
}
}
}
}
break;
+ case ANYOFM:
+ if (utf8_target && loceol - scan > max) {
+
+ /* We didn't adjust <loceol> at the beginning of this routine
+ * because is UTF-8, but it is actually ok to do so, since here, to
+ * match, 1 char == 1 byte. */
+ loceol = scan + max;
+ }
+
+ scan = (char *) find_span_end_mask((U8 *) scan, (U8 *) loceol, (U8) ARG(p), FLAGS(p));
+ break;
+
+ case ASCII:
+ if (utf8_target && loceol - scan > max) {
+ loceol = scan + max;
+ }
+
+ scan = find_next_non_ascii(scan, loceol, utf8_target);
+ break;
+
+ case NASCII:
+ if (utf8_target) {
+ while ( hardcount < max
+ && scan < loceol
+ && ! isASCII_utf8_safe(scan, loceol))
+ {
+ scan += UTF8SKIP(scan);
+ hardcount++;
+ }
+ }
+ else {
+ scan = find_next_ascii(scan, loceol, utf8_target);
+ }
+ break;
+
/* The argument (FLAGS) to all the POSIX node types is the class number */
case NPOSIXL:
if (off >= 0) {
while (off-- && s < lim) {
/* XXX could check well-formedness here */
- s += UTF8SKIP(s);
+ U8 *new_s = s + UTF8SKIP(s);
+ if (new_s > lim) /* lim may be in the middle of a long character */
+ return s;
+ s = new_s;
}
}
else {
return isGCB(cp_gcb_val, next_cp_gcb_val, strbeg, s, TRUE);
}
+/*
+=head1 Unicode Support
+
+=for apidoc isSCRIPT_RUN
+
+Returns a bool as to whether or not the sequence of bytes from C<s> up to but
+not including C<send> form a "script run". C<utf8_target> is TRUE iff the
+sequence starting at C<s> is to be treated as UTF-8. To be precise, except for
+two degenerate cases given below, this function returns TRUE iff all code
+points in it come from any combination of three "scripts" given by the Unicode
+"Script Extensions" property: Common, Inherited, and possibly one other.
+Additionally all decimal digits must come from the same consecutive sequence of
+10.
+
+For example, if all the characters in the sequence are Greek, or Common, or
+Inherited, this function will return TRUE, provided any decimal digits in it
+are the ASCII digits "0".."9". For scripts (unlike Greek) that have their own
+digits defined this will accept either digits from that set or from 0..9, but
+not a combination of the two. Some scripts, such as Arabic, have more than one
+set of digits. All digits must come from the same set for this function to
+return TRUE.
+
+C<*ret_script>, if C<ret_script> is not NULL, will on return of TRUE
+contain the script found, using the C<SCX_enum> typedef. Its value will be
+C<SCX_INVALID> if the function returns FALSE.
+
+If the sequence is empty, TRUE is returned, but C<*ret_script> (if asked for)
+will be C<SCX_INVALID>.
+
+If the sequence contains a single code point which is unassigned to a character
+in the version of Unicode being used, the function will return TRUE, and the
+script will be C<SCX_Unknown>. Any other combination of unassigned code points
+in the input sequence will result in the function treating the input as not
+being a script run.
+
+The returned script will be C<SCX_Inherited> iff all the code points in it are
+from the Inherited script.
+
+Otherwise, the returned script will be C<SCX_Common> iff all the code points in
+it are from the Inherited or Common scripts.
+
+=cut
+
+*/
+
+bool
+Perl_isSCRIPT_RUN(pTHX_ const U8 * s, const U8 * send, const bool utf8_target, SCX_enum * ret_script)
+{
+ /* Basically, it looks at each character in the sequence to see if the
+ * above conditions are met; if not it fails. It uses an inversion map to
+ * find the enum corresponding to the script of each character. But this
+ * is complicated by the fact that a few code points can be in any of
+ * several scripts. The data has been constructed so that there are
+ * additional enum values (all negative) for these situations. The
+ * absolute value of those is an index into another table which contains
+ * pointers to auxiliary tables for each such situation. Each aux array
+ * lists all the scripts for the given situation. There is another,
+ * parallel, table that gives the number of entries in each aux table.
+ * These are all defined in charclass_invlists.h */
+
+ /* XXX Here are the additional things UTS 39 says could be done:
+ * Mark Chinese strings as “mixed script” if they contain both simplified
+ * (S) and traditional (T) Chinese characters, using the Unihan data in the
+ * Unicode Character Database [UCD]. The criterion can only be applied if
+ * the language of the string is known to be Chinese. So, for example, the
+ * string “写真だけの結婚式 ” is Japanese, and should not be marked as
+ * mixed script because of a mixture of S and T characters. Testing for
+ * whether a character is S or T needs to be based not on whether the
+ * character has a S or T variant , but whether the character is an S or T
+ * variant. khw notes that the sample contains a Hiragana character, and it
+ * is unclear if absence of any foreign script marks the script as
+ * "Chinese"
+ *
+ * Forbid sequences of the same nonspacing mark
+ *
+ * Check to see that all the characters are in the sets of exemplar
+ * characters for at least one language in the Unicode Common Locale Data
+ * Repository [CLDR]. */
+
+
+ /* Things that match /\d/u */
+ SV * decimals_invlist = PL_XPosix_ptrs[_CC_DIGIT];
+ UV * decimals_array = invlist_array(decimals_invlist);
+
+ /* What code point is the digit '0' of the script run? */
+ UV zero_of_run = 0;
+ SCX_enum script_of_run = SCX_INVALID; /* Illegal value */
+ SCX_enum script_of_char = SCX_INVALID;
+
+ /* If the script remains not fully determined from iteration to iteration,
+ * this is the current intersection of the possiblities. */
+ SCX_enum * intersection = NULL;
+ PERL_UINT_FAST8_T intersection_len = 0;
+
+ bool retval = TRUE;
+
+ assert(send >= s);
+
+ PERL_ARGS_ASSERT_ISSCRIPT_RUN;
+
+ /* All code points in 0..255 are either Common or Latin, so must be a
+ * script run. We can special case it */
+ if (! utf8_target && LIKELY(send > s)) {
+ if (ret_script == NULL) {
+ return TRUE;
+ }
+
+ /* If any character is Latin, the run is Latin */
+ while (s < send) {
+ if (isALPHA_L1(*s) && LIKELY(*s != MICRO_SIGN_NATIVE)) {
+ *ret_script = SCX_Latin;
+ return TRUE;
+ }
+ }
+
+ /* If all are Common ... */
+ *ret_script = SCX_Common;
+ return TRUE;
+ }
+
+ /* Look at each character in the sequence */
+ while (s < send) {
+ UV cp;
+
+ /* The code allows all scripts to use the ASCII digits. This is
+ * because they are used in commerce even in scripts that have their
+ * own set. Hence any ASCII ones found are ok, unless a digit from
+ * another set has already been encountered. (The other digit ranges
+ * in Common are not similarly blessed) */
+ if (UNLIKELY(isDIGIT(*s))) {
+ if (UNLIKELY(script_of_run == SCX_Unknown)) {
+ retval = FALSE;
+ break;
+ }
+ if (zero_of_run > 0) {
+ if (zero_of_run != '0') {
+ retval = FALSE;
+ break;
+ }
+ }
+ else {
+ zero_of_run = '0';
+ }
+ s++;
+ continue;
+ }
+
+ /* Here, isn't an ASCII digit. Find the code point of the character */
+ if (! UTF8_IS_INVARIANT(*s)) {
+ Size_t len;
+ cp = valid_utf8_to_uvchr((U8 *) s, &len);
+ s += len;
+ }
+ else {
+ cp = *(s++);
+ }
+
+ /* If is within the range [+0 .. +9] of the script's zero, it also is a
+ * digit in that script. We can skip the rest of this code for this
+ * character. */
+ if (UNLIKELY( zero_of_run > 0
+ && cp >= zero_of_run
+ && cp - zero_of_run <= 9))
+ {
+ continue;
+ }
+
+ /* Find the character's script. The correct values are hard-coded here
+ * for small-enough code points. */
+ if (cp < 0x2B9) { /* From inspection of Unicode db; extremely
+ unlikely to change */
+ if ( cp > 255
+ || ( isALPHA_L1(cp)
+ && LIKELY(cp != MICRO_SIGN_NATIVE)))
+ {
+ script_of_char = SCX_Latin;
+ }
+ else {
+ script_of_char = SCX_Common;
+ }
+ }
+ else {
+ script_of_char = _Perl_SCX_invmap[
+ _invlist_search(PL_SCX_invlist, cp)];
+ }
+
+ /* We arbitrarily accept a single unassigned character, but not in
+ * combination with anything else, and not a run of them. */
+ if ( UNLIKELY(script_of_run == SCX_Unknown)
+ || UNLIKELY( script_of_run != SCX_INVALID
+ && script_of_char == SCX_Unknown))
+ {
+ retval = FALSE;
+ break;
+ }
+
+ /* For the first character, or the run is inherited, the run's script
+ * is set to the char's */
+ if ( UNLIKELY(script_of_run == SCX_INVALID)
+ || UNLIKELY(script_of_run == SCX_Inherited))
+ {
+ script_of_run = script_of_char;
+ }
+
+ /* For the character's script to be Unknown, it must be the first
+ * character in the sequence (for otherwise a test above would have
+ * prevented us from reaching here), and we have set the run's script
+ * to it. Nothing further to be done for this character */
+ if (UNLIKELY(script_of_char == SCX_Unknown)) {
+ continue;
+ }
+
+ /* We accept 'inherited' script characters currently even at the
+ * beginning. (We know that no characters in Inherited are digits, or
+ * we'd have to check for that) */
+ if (UNLIKELY(script_of_char == SCX_Inherited)) {
+ continue;
+ }
+
+ /* If the run so far is Common, and the new character isn't, change the
+ * run's script to that of this character */
+ if (script_of_run == SCX_Common && script_of_char != SCX_Common) {
+
+ /* But Common contains several sets of digits. Only the '0' set
+ * can be part of another script. */
+ if (zero_of_run > 0 && zero_of_run != '0') {
+ retval = FALSE;
+ break;
+ }
+
+ script_of_run = script_of_char;
+ }
+
+ /* All decimal digits must be from the same sequence of 10. Above, we
+ * handled any ASCII digits without descending to here. We also
+ * handled the case where we already knew what digit sequence is the
+ * one to use, and the character is in that sequence. Now that we know
+ * the script, we can use script_zeros[] to directly find which
+ * sequence the script uses, except in a few cases it returns 0 */
+ if (UNLIKELY(zero_of_run == 0) && script_of_char >= 0) {
+ zero_of_run = script_zeros[script_of_char];
+ }
+
+ /* Now we can see if the script of the character is the same as that of
+ * the run */
+ if (LIKELY(script_of_char == script_of_run)) {
+ /* By far the most common case */
+ goto scripts_match;
+ }
+
+
+ /* Here, the script of the run isn't Common. But characters in Common
+ * match any script */
+ if (script_of_char == SCX_Common) {
+ goto scripts_match;
+ }
+
+#ifndef HAS_SCX_AUX_TABLES
+
+ /* Too early a Unicode version to have a code point belonging to more
+ * than one script, so, if the scripts don't exactly match, fail */
+ retval = FALSE;
+ break;
+
+#else
+
+ /* Here there is no exact match between the character's script and the
+ * run's. And we've handled the special cases of scripts Unknown,
+ * Inherited, and Common.
+ *
+ * Negative script numbers signify that the value may be any of several
+ * scripts, and we need to look at auxiliary information to make our
+ * deterimination. But if both are non-negative, we can fail now */
+ if (LIKELY(script_of_char >= 0)) {
+ const SCX_enum * search_in;
+ PERL_UINT_FAST8_T search_in_len;
+ PERL_UINT_FAST8_T i;
+
+ if (LIKELY(script_of_run >= 0)) {
+ retval = FALSE;
+ break;
+ }
+
+ /* Use the previously constructed set of possible scripts, if any.
+ * */
+ if (intersection) {
+ search_in = intersection;
+ search_in_len = intersection_len;
+ }
+ else {
+ search_in = SCX_AUX_TABLE_ptrs[-script_of_run];
+ search_in_len = SCX_AUX_TABLE_lengths[-script_of_run];
+ }
+
+ for (i = 0; i < search_in_len; i++) {
+ if (search_in[i] == script_of_char) {
+ script_of_run = script_of_char;
+ goto scripts_match;
+ }
+ }
+
+ retval = FALSE;
+ break;
+ }
+ else if (LIKELY(script_of_run >= 0)) {
+ /* script of character could be one of several, but run is a single
+ * script */
+ const SCX_enum * search_in = SCX_AUX_TABLE_ptrs[-script_of_char];
+ const PERL_UINT_FAST8_T search_in_len
+ = SCX_AUX_TABLE_lengths[-script_of_char];
+ PERL_UINT_FAST8_T i;
+
+ for (i = 0; i < search_in_len; i++) {
+ if (search_in[i] == script_of_run) {
+ script_of_char = script_of_run;
+ goto scripts_match;
+ }
+ }
+
+ retval = FALSE;
+ break;
+ }
+ else {
+ /* Both run and char could be in one of several scripts. If the
+ * intersection is empty, then this character isn't in this script
+ * run. Otherwise, we need to calculate the intersection to use
+ * for future iterations of the loop, unless we are already at the
+ * final character */
+ const SCX_enum * search_char = SCX_AUX_TABLE_ptrs[-script_of_char];
+ const PERL_UINT_FAST8_T char_len
+ = SCX_AUX_TABLE_lengths[-script_of_char];
+ const SCX_enum * search_run;
+ PERL_UINT_FAST8_T run_len;
+
+ SCX_enum * new_overlap = NULL;
+ PERL_UINT_FAST8_T i, j;
+
+ if (intersection) {
+ search_run = intersection;
+ run_len = intersection_len;
+ }
+ else {
+ search_run = SCX_AUX_TABLE_ptrs[-script_of_run];
+ run_len = SCX_AUX_TABLE_lengths[-script_of_run];
+ }
+
+ intersection_len = 0;
+
+ for (i = 0; i < run_len; i++) {
+ for (j = 0; j < char_len; j++) {
+ if (search_run[i] == search_char[j]) {
+
+ /* Here, the script at i,j matches. That means this
+ * character is in the run. But continue on to find
+ * the complete intersection, for the next loop
+ * iteration, and for the digit check after it.
+ *
+ * On the first found common script, we malloc space
+ * for the intersection list for the worst case of the
+ * intersection, which is the minimum of the number of
+ * scripts remaining in each set. */
+ if (intersection_len == 0) {
+ Newx(new_overlap,
+ MIN(run_len - i, char_len - j),
+ SCX_enum);
+ }
+ new_overlap[intersection_len++] = search_run[i];
+ }
+ }
+ }
+
+ /* Here we've looked through everything. If they have no scripts
+ * in common, not a run */
+ if (intersection_len == 0) {
+ retval = FALSE;
+ break;
+ }
+
+ /* If there is only a single script in common, set to that.
+ * Otherwise, use the intersection going forward */
+ Safefree(intersection);
+ if (intersection_len == 1) {
+ script_of_run = script_of_char = new_overlap[0];
+ Safefree(new_overlap);
+ }
+ else {
+ intersection = new_overlap;
+ }
+ }
+
#endif
+ scripts_match:
+
+ /* Here, the script of the character is compatible with that of the
+ * run. Either they match exactly, or one or both can be any of
+ * several scripts, and the intersection is not empty. If the
+ * character is not a decimal digit, we are done with it. Otherwise,
+ * it could still fail if it is from a different set of 10 than seen
+ * already (or we may not have seen any, and we need to set the
+ * sequence). If we have determined a single script and that script
+ * only has one set of digits (almost all scripts are like that), then
+ * this isn't a problem, as any digit must come from the same sequence.
+ * The only scripts that have multiple sequences have been constructed
+ * to be 0 in 'script_zeros[]'.
+ *
+ * Here we check if it is a digit. */
+ if ( cp >= FIRST_NON_ASCII_DECIMAL_DIGIT
+ && ( ( zero_of_run == 0
+ || ( ( script_of_char >= 0
+ && script_zeros[script_of_char] == 0)
+ || intersection))))
+ {
+ SSize_t range_zero_index;
+ range_zero_index = _invlist_search(decimals_invlist, cp);
+ if ( LIKELY(range_zero_index >= 0)
+ && ELEMENT_RANGE_MATCHES_INVLIST(range_zero_index))
+ {
+ UV range_zero = decimals_array[range_zero_index];
+ if (zero_of_run) {
+ if (zero_of_run != range_zero) {
+ retval = FALSE;
+ break;
+ }
+ }
+ else {
+ zero_of_run = range_zero;
+ }
+ }
+ }
+ } /* end of looping through CLOSESR text */
+
+ Safefree(intersection);
+
+ if (ret_script != NULL) {
+ if (retval) {
+ *ret_script = script_of_run;
+ }
+ else {
+ *ret_script = SCX_INVALID;
+ }
+ }
+
+ return retval;
+}
+#endif /* ifndef PERL_IN_XSUB_RE */
/*
* ex: set ts=8 sts=4 sw=4 et:
https://perl5.git.perl.org / perl5.git / blobdiff