UTF8fARG(UTF, \
(xI(xC) > eC) /* Don't run off end */ \
? eC - sC /* Length before the <--HERE */ \
- : ( __ASSERT_(xI_offset(xC) >= 0) xI_offset(xC) ), \
+ : ((xI_offset(xC) >= 0) \
+ ? xI_offset(xC) \
+ : (Perl_croak(aTHX_ "panic: %s: %d: negative offset: %" \
+ IVdf " trying to output message for " \
+ " pattern %.*s", \
+ __FILE__, __LINE__, xI_offset(xC), \
+ ((int) (eC - sC)), sC), 0)), \
sC), /* The input pattern printed up to the <--HERE */ \
UTF8fARG(UTF, \
(xI(xC) > eC) ? 0 : eC - xI(xC), /* Length after <--HERE */ \
/* Cannot expect anything... */
scan_commit(pRExC_state, data, minlenp, is_inf);
data->pos_min += 1;
- data->pos_delta += 1;
+ if (data->pos_delta != SSize_t_MAX) {
+ data->pos_delta += 1;
+ }
data->cur_is_floating = 1; /* float */
}
}
invert = 1;
/* FALLTHROUGH */
case ASCII:
- my_invlist = invlist_clone(PL_XPosix_ptrs[_CC_ASCII]);
+ my_invlist = invlist_clone(PL_Posix_ptrs[_CC_ASCII]);
/* This can be handled as a Posix class */
goto join_posix_and_ascii;
/* FALLTHROUGH */
case POSIXA:
assert(FLAGS(scan) != _CC_ASCII);
- _invlist_intersection(PL_XPosix_ptrs[FLAGS(scan)],
- PL_XPosix_ptrs[_CC_ASCII],
- &my_invlist);
+ my_invlist = invlist_clone(PL_Posix_ptrs[FLAGS(scan)]);
goto join_posix_and_ascii;
case NPOSIXD:
* it is properly null terminated or we will fail asserts
* later. In theory we probably shouldn't get such SV's,
* but if we do we should handle it gracefully. */
- if ( SvTYPE(msv) != SVt_PV || (SvLEN(msv) > SvCUR(msv) && *(SvEND(msv)) == 0) ) {
+ if ( SvTYPE(msv) != SVt_PV || (SvLEN(msv) > SvCUR(msv) && *(SvEND(msv)) == 0) || SvIsCOW_shared_hash(msv) ) {
/* not a string, or a string with a trailing null */
pat = msv;
} else {
/* a string with no trailing null, we need to copy it
- * so it we have a trailing null */
- pat = newSVsv(msv);
+ * so it has a trailing null */
+ pat = sv_2mortal(newSVsv(msv));
}
}
/* Initialize these here instead of as-needed, as is quick and avoids
* having to test them each time otherwise */
- if (! PL_AboveLatin1) {
+ if (! PL_InBitmap) {
#ifdef DEBUGGING
char * dump_len_string;
#endif
- PL_AboveLatin1 = _new_invlist_C_array(AboveLatin1_invlist);
- PL_Latin1 = _new_invlist_C_array(Latin1_invlist);
- PL_UpperLatin1 = _new_invlist_C_array(UpperLatin1_invlist);
- PL_utf8_foldable = _new_invlist_C_array(_Perl_Any_Folds_invlist);
- PL_HasMultiCharFold =
- _new_invlist_C_array(_Perl_Folds_To_Multi_Char_invlist);
-
/* This is calculated here, because the Perl program that generates the
* static global ones doesn't currently have access to
* NUM_ANYOF_CODE_POINTS */
}
}
-void
-Perl__load_PL_utf8_foldclosures (pTHX)
-{
- assert(! PL_utf8_foldclosures);
-
- /* If the folds haven't been read in, call a fold function
- * to force that */
- if (! PL_utf8_tofold) {
- U8 dummy[UTF8_MAXBYTES_CASE+1];
- const U8 hyphen[] = HYPHEN_UTF8;
-
- /* This string is just a short named one above \xff */
- toFOLD_utf8_safe(hyphen, hyphen + sizeof(hyphen) - 1, dummy, NULL);
- assert(PL_utf8_tofold); /* Verify that worked */
- }
- PL_utf8_foldclosures = _swash_inversion_hash(PL_utf8_tofold);
-}
#endif
#if defined(PERL_ARGS_ASSERT__INVLISTEQ) && !defined(PERL_IN_XSUB_RE)
}
else { /* Pattern is UTF-8 */
U8 folded[UTF8_MAX_FOLD_CHAR_EXPAND * UTF8_MAXBYTES_CASE + 1] = { '\0' };
- STRLEN foldlen = UTF8SKIP(s);
const U8* e = s + bytelen;
- SV** listp;
+ IV fc;
- uc = utf8_to_uvchr_buf(s, s + bytelen, NULL);
+ fc = uc = utf8_to_uvchr_buf(s, s + bytelen, NULL);
/* The only code points that aren't folded in a UTF EXACTFish
* node are are the problematic ones in EXACTFL nodes */
U8 *d = folded;
int i;
+ fc = -1;
for (i = 0; i < UTF8_MAX_FOLD_CHAR_EXPAND && s < e; i++) {
if (isASCII(*s)) {
*(d++) = (U8) toFOLD(*s);
+ if (fc < 0) { /* Save the first fold */
+ fc = *(d-1);
+ }
s++;
}
else {
STRLEN len;
- toFOLD_utf8_safe(s, e, d, &len);
+ UV fold = toFOLD_utf8_safe(s, e, d, &len);
+ if (fc < 0) { /* Save the first fold */
+ fc = fold;
+ }
d += len;
s += UTF8SKIP(s);
}
/* And set up so the code below that looks in this folded
* buffer instead of the node's string */
e = d;
- foldlen = UTF8SKIP(folded);
s = folded;
}
/* When we reach here 's' points to the fold of the first
* character(s) of the node; and 'e' points to far enough along
* the folded string to be just past any possible multi-char
- * fold. 'foldlen' is the length in bytes of the first
- * character in 's'
+ * fold.
*
* Unlike the non-UTF-8 case, the macro for determining if a
* string is a multi-char fold requires all the characters to
invlist = _add_range_to_invlist(invlist, 0, UV_MAX);
}
else { /* Single char fold */
-
- /* It matches all the things that fold to it, which are
- * found in PL_utf8_foldclosures (including itself) */
- invlist = add_cp_to_invlist(invlist, uc);
- if (! PL_utf8_foldclosures)
- _load_PL_utf8_foldclosures();
- if ((listp = hv_fetch(PL_utf8_foldclosures,
- (char *) s, foldlen, FALSE)))
- {
- AV* list = (AV*) *listp;
- IV k;
- for (k = 0; k <= av_tindex_skip_len_mg(list); k++) {
- SV** c_p = av_fetch(list, k, FALSE);
- UV c;
- assert(c_p);
-
- c = SvUV(*c_p);
-
- /* /aa doesn't allow folds between ASCII and non- */
- if ((OP(node) == EXACTFAA || OP(node) == EXACTFAA_NO_TRIE)
- && isASCII(c) != isASCII(uc))
- {
- continue;
- }
-
- invlist = add_cp_to_invlist(invlist, c);
+ unsigned int k;
+ unsigned int first_folds_to;
+ const unsigned int * remaining_folds_to_list;
+ Size_t folds_to_count;
+
+ /* It matches itself */
+ invlist = add_cp_to_invlist(invlist, fc);
+
+ /* ... plus all the things that fold to it, which are found in
+ * PL_utf8_foldclosures */
+ folds_to_count = _inverse_folds(fc, &first_folds_to,
+ &remaining_folds_to_list);
+ for (k = 0; k < folds_to_count; k++) {
+ UV c = (k == 0) ? first_folds_to : remaining_folds_to_list[k-1];
+
+ /* /aa doesn't allow folds between ASCII and non- */
+ if ( (OP(node) == EXACTFAA || OP(node) == EXACTFAA_NO_TRIE)
+ && isASCII(c) != isASCII(fc))
+ {
+ continue;
}
+
+ invlist = add_cp_to_invlist(invlist, c);
}
}
}
* RExC_parse beyond the '('. Things like '(?' are indivisible tokens, and
* this flag alerts us to the need to check for that */
{
- regnode *ret; /* Will be the head of the group. */
+ regnode *ret = NULL; /* Will be the head of the group. */
regnode *br;
regnode *lastbr;
regnode *ender = NULL;
vFAIL("Unmatched (");
}
- if ( *RExC_parse == '*') { /* (*VERB:ARG), (*construct:...) */
+ if (paren == 'r') { /* Atomic script run */
+ paren = '>';
+ goto parse_rest;
+ }
+ else if ( *RExC_parse == '*') { /* (*VERB:ARG), (*construct:...) */
char *start_verb = RExC_parse + 1;
STRLEN verb_len;
char *start_arg = NULL;
}
break;
case 'a':
- if (memEQs(start_verb, verb_len, "atomic")) {
+ if ( memEQs(start_verb, verb_len, "asr")
+ || memEQs(start_verb, verb_len, "atomic_script_run"))
+ {
+ paren = 'r'; /* Mnemonic: recursed run */
+ goto script_run;
+ }
+ else if (memEQs(start_verb, verb_len, "atomic")) {
paren = 't'; /* AtOMIC */
goto alpha_assertions;
}
if ( memEQs(start_verb, verb_len, "sr")
|| memEQs(start_verb, verb_len, "script_run"))
{
+ regnode * atomic;
+
paren = 's';
+ script_run:
+
/* This indicates Unicode rules. */
REQUIRE_UNI_RULES(flagp, NULL);
RExC_parse = start_arg;
+ if (RExC_in_script_run) {
+
+ /* Nested script runs are treated as no-ops, because
+ * if the nested one fails, the outer one must as
+ * well. It could fail sooner, and avoid (??{} with
+ * side effects, but that is explicitly documented as
+ * undefined behavior. */
+
+ ret = NULL;
+
+ if (paren == 's') {
+ paren = ':';
+ goto parse_rest;
+ }
+
+ /* But, the atomic part of a nested atomic script run
+ * isn't a no-op, but can be treated just like a '(?>'
+ * */
+ paren = '>';
+ goto parse_rest;
+ }
+
+ /* By doing this here, we avoid extra warnings for nested
+ * script runs */
if (PASS2) {
Perl_ck_warner_d(aTHX_
packWARN(WARN_EXPERIMENTAL__SCRIPT_RUN),
}
- if (RExC_in_script_run) {
- paren = ':';
- nextchar(pRExC_state);
- ret = NULL;
+ if (paren == 's') {
+ /* Here, we're starting a new regular script run */
+ ret = reg_node(pRExC_state, SROPEN);
+ RExC_in_script_run = 1;
+ is_open = 1;
goto parse_rest;
}
- RExC_in_script_run = 1;
+
+ /* Here, we are starting an atomic script run. This is
+ * handled by recursing to deal with the atomic portion
+ * separately, enclosed in SROPEN ... SRCLOSE nodes */
ret = reg_node(pRExC_state, SROPEN);
- is_open = 1;
- goto parse_rest;
+ RExC_in_script_run = 1;
+
+ atomic = reg(pRExC_state, 'r', &flags, depth);
+ if (flags & (RESTART_PASS1|NEED_UTF8)) {
+ *flagp = flags & (RESTART_PASS1|NEED_UTF8);
+ return NULL;
+ }
+
+ REGTAIL(pRExC_state, ret, atomic);
+
+ REGTAIL(pRExC_state, atomic,
+ reg_node(pRExC_state, SRCLOSE));
+
+ RExC_in_script_run = 0;
+ return ret;
}
break;
RExC_parse++;
is_neg = TRUE;
}
+ endptr = RExC_end;
if (grok_atoUV(RExC_parse, &unum, &endptr)
&& unum <= I32_MAX
) {
}
else if (RExC_parse[0] >= '1' && RExC_parse[0] <= '9' ) {
UV uv;
+ endptr = RExC_end;
if (grok_atoUV(RExC_parse, &uv, &endptr)
&& uv <= I32_MAX
) {
/* (?(1)...) */
char c;
UV uv;
+ endptr = RExC_end;
if (grok_atoUV(RExC_parse, &uv, &endptr)
&& uv <= I32_MAX
) {
if (paren == '>' || paren == 't') {
node = SUSPEND, flag = 0;
}
+
reginsert(pRExC_state, node,ret, depth+1);
Set_Node_Cur_Length(ret, parse_start);
Set_Node_Offset(ret, parse_start + 1);
maxpos = next;
RExC_parse++;
if (isDIGIT(*RExC_parse)) {
+ endptr = RExC_end;
if (!grok_atoUV(RExC_parse, &uv, &endptr))
vFAIL("Invalid quantifier in {,}");
if (uv >= REG_INFTY)
else
maxpos = RExC_parse;
if (isDIGIT(*maxpos)) {
+ endptr = RExC_end;
if (!grok_atoUV(maxpos, &uv, &endptr))
vFAIL("Invalid quantifier in {,}");
if (uv >= REG_INFTY)
* *node_p, nor *code_point_p, nor *flagp.
*
* If <cp_count> is not NULL, the caller wants to know the length (in code
- * points) that this \N sequence matches. This is set even if the function
- * returns FALSE, as detailed below.
+ * points) that this \N sequence matches. This is set, and the input is
+ * parsed for errors, even if the function returns FALSE, as detailed below.
*
* There are 5 possibilities here, as detailed in the next 5 paragraphs.
*
*/
char * endbrace; /* points to '}' following the name */
- char *endchar; /* Points to '.' or '}' ending cur char in the input
- stream */
char* p = RExC_parse; /* Temporary */
+ SV * substitute_parse = NULL;
+ char *orig_end;
+ char *save_start;
+ I32 flags;
+ Size_t count = 0; /* code point count kept internally by this function */
+
GET_RE_DEBUG_FLAGS_DECL;
PERL_ARGS_ASSERT_GROK_BSLASH_N;
* [^\n]. The latter is assumed when the {...} following the \N is a legal
* quantifier, or there is no '{' at all */
if (*p != '{' || regcurly(p)) {
- RExC_parse = p;
+ RExC_parse = p;
if (cp_count) {
*cp_count = -1;
}
- if (! node_p) {
+ if (! node_p) {
return FALSE;
}
- *node_p = reg_node(pRExC_state, REG_ANY);
- *flagp |= HASWIDTH|SIMPLE;
- MARK_NAUGHTY(1);
+ *node_p = reg_node(pRExC_state, REG_ANY);
+ *flagp |= HASWIDTH|SIMPLE;
+ MARK_NAUGHTY(1);
Set_Node_Length(*node_p, 1); /* MJD */
- return TRUE;
+ return TRUE;
}
- /* Here, we have decided it should be a named character or sequence */
-
/* The test above made sure that the next real character is a '{', but
* under the /x modifier, it could be separated by space (or a comment and
* \n) and this is not allowed (for consistency with \x{...} and the
* tokenizer handling of \N{NAME}). */
if (*RExC_parse != '{') {
- vFAIL("Missing braces on \\N{}");
+ vFAIL("Missing braces on \\N{}");
}
- RExC_parse++; /* Skip past the '{' */
+ RExC_parse++; /* Skip past the '{' */
endbrace = (char *) memchr(RExC_parse, '}', RExC_end - RExC_parse);
if (! endbrace) { /* no trailing brace */
vFAIL2("Missing right brace on \\%c{}", 'N');
}
- else if (!( endbrace == RExC_parse /* nothing between the {} */
- || memBEGINs(RExC_parse, /* U+ (bad hex is checked below
- for a better error msg) */
- (STRLEN) (RExC_end - RExC_parse),
- "U+")))
- {
- RExC_parse = endbrace; /* position msg's '<--HERE' */
- vFAIL("\\N{NAME} must be resolved by the lexer");
- }
+ /* Here, we have decided it should be a named character or sequence */
REQUIRE_UNI_RULES(flagp, FALSE); /* Unicode named chars imply Unicode
semantics */
*cp_count = 0;
}
nextchar(pRExC_state);
- if (! node_p) {
+ if (! node_p) {
return FALSE;
}
return TRUE;
}
- RExC_parse += 2; /* Skip past the 'U+' */
+ /* If we haven't got something that begins with 'U+', then it didn't get lexed. */
+ if ( endbrace - RExC_parse < 2
+ || strnNE(RExC_parse, "U+", 2))
+ {
+ RExC_parse = endbrace; /* position msg's '<--HERE' */
+ vFAIL("\\N{NAME} must be resolved by the lexer");
+ }
- /* Because toke.c has generated a special construct for us guaranteed not
- * to have NULs, we can use a str function */
- endchar = RExC_parse + strcspn(RExC_parse, ".}");
+ /* This code purposely indented below because of future changes coming */
- /* Code points are separated by dots. If none, there is only one code
- * point, and is terminated by the brace */
+ /* We can get to here when the input is \N{U+...} or when toke.c has
+ * converted a name to the \N{U+...} form. This include changing a
+ * name that evaluates to multiple code points to \N{U+c1.c2.c3 ...} */
- if (endchar >= endbrace) {
- STRLEN length_of_hex;
- I32 grok_hex_flags;
+ RExC_parse += 2; /* Skip past the 'U+' */
- /* Here, exactly one code point. If that isn't what is wanted, fail */
- if (! code_point_p) {
- RExC_parse = p;
- return FALSE;
- }
+ /* Code points are separated by dots. The '}' terminates the whole
+ * thing. */
- /* Convert code point from hex */
- length_of_hex = (STRLEN)(endchar - RExC_parse);
- grok_hex_flags = PERL_SCAN_ALLOW_UNDERSCORES
- | PERL_SCAN_DISALLOW_PREFIX
-
- /* No errors in the first pass (See [perl
- * #122671].) We let the code below find the
- * errors when there are multiple chars. */
- | ((SIZE_ONLY)
- ? PERL_SCAN_SILENT_ILLDIGIT
- : 0);
-
- /* This routine is the one place where both single- and double-quotish
- * \N{U+xxxx} are evaluated. The value is a Unicode code point which
- * must be converted to native. */
- *code_point_p = UNI_TO_NATIVE(grok_hex(RExC_parse,
- &length_of_hex,
- &grok_hex_flags,
- NULL));
-
- /* The tokenizer should have guaranteed validity, but it's possible to
- * bypass it by using single quoting, so check. Don't do the check
- * here when there are multiple chars; we do it below anyway. */
- if (length_of_hex == 0
- || length_of_hex != (STRLEN)(endchar - RExC_parse) )
- {
- RExC_parse += length_of_hex; /* Includes all the valid */
- RExC_parse += (RExC_orig_utf8) /* point to after 1st invalid */
- ? UTF8SKIP(RExC_parse)
- : 1;
- /* Guard against malformed utf8 */
- if (RExC_parse >= endchar) {
- RExC_parse = endchar;
+ do { /* Loop until the ending brace */
+ UV cp = 0;
+ char * start_digit; /* The first of the current code point */
+ if (! isXDIGIT(*RExC_parse)) {
+ RExC_parse++;
+ vFAIL("Invalid hexadecimal number in \\N{U+...}");
}
- vFAIL("Invalid hexadecimal number in \\N{U+...}");
- }
- RExC_parse = endbrace + 1;
- return TRUE;
- }
- else { /* Is a multiple character sequence */
- SV * substitute_parse;
- STRLEN len;
- char *orig_end = RExC_end;
- char *save_start = RExC_start;
- I32 flags;
+ start_digit = RExC_parse;
+ count++;
- /* Count the code points, if desired, in the sequence */
- if (cp_count) {
- *cp_count = 0;
- while (RExC_parse < endbrace) {
- /* Point to the beginning of the next character in the sequence. */
- RExC_parse = endchar + 1;
- endchar = RExC_parse + strcspn(RExC_parse, ".}");
- (*cp_count)++;
+ /* Loop through the hex digits of the current code point */
+ do {
+ /* Adding this digit will shift the result 4 bits. If that
+ * result would be above IV_MAX, it's overflow */
+ if (cp > IV_MAX >> 4) {
+
+ /* Find the end of the code point */
+ do {
+ RExC_parse ++;
+ } while (isXDIGIT(*RExC_parse) || *RExC_parse == '_');
+
+ /* Be sure to synchronize this message with the similar one
+ * in utf8.c */
+ vFAIL4("Use of code point 0x%.*s is not allowed; the"
+ " permissible max is 0x%" UVxf,
+ (int) (RExC_parse - start_digit), start_digit, IV_MAX);
+ }
+
+ /* Accumulate this (valid) digit into the running total */
+ cp = (cp << 4) + READ_XDIGIT(RExC_parse);
+
+ /* READ_XDIGIT advanced the input pointer. Ignore a single
+ * underscore separator */
+ if (*RExC_parse == '_' && isXDIGIT(RExC_parse[1])) {
+ RExC_parse++;
+ }
+ } while (isXDIGIT(*RExC_parse));
+
+ /* Here, have accumulated the next code point */
+ if (RExC_parse >= endbrace) { /* If done ... */
+ if (count != 1) {
+ goto do_concat;
+ }
+
+ /* Here, is a single code point; fail if doesn't want that */
+ if (! code_point_p) {
+ RExC_parse = p;
+ return FALSE;
+ }
+
+ /* A single code point is easy to handle; just return it */
+ *code_point_p = UNI_TO_NATIVE(cp);
+ RExC_parse = endbrace;
+ nextchar(pRExC_state);
+ return TRUE;
}
- }
- /* Fail if caller doesn't want to handle a multi-code-point sequence.
- * But don't backup up the pointer if the caller wants to know how many
- * code points there are (they can then handle things) */
- if (! node_p) {
- if (! cp_count) {
- RExC_parse = p;
+ /* Here, the only legal thing would be a multiple character
+ * sequence (of the form "\N{U+c1.c2. ... }". So the next
+ * character must be a dot (and the one after that can't be the
+ * endbrace, or we'd have something like \N{U+100.} ) */
+ if (*RExC_parse != '.' || RExC_parse + 1 >= endbrace) {
+ RExC_parse += (RExC_orig_utf8) /* point to after 1st invalid */
+ ? UTF8SKIP(RExC_parse)
+ : 1;
+ if (RExC_parse >= endbrace) { /* Guard against malformed utf8 */
+ RExC_parse = endbrace;
+ }
+ vFAIL("Invalid hexadecimal number in \\N{U+...}");
}
- return FALSE;
- }
- /* What is done here is to convert this to a sub-pattern of the form
- * \x{char1}\x{char2}... and then call reg recursively to parse it
- * (enclosing in "(?: ... )" ). That way, it retains its atomicness,
- * while not having to worry about special handling that some code
- * points may have. */
+ /* Here, looks like its really a multiple character sequence. Fail
+ * if that's not what the caller wants. But continue with counting
+ * and error checking if they still want a count */
+ if (! node_p && ! cp_count) {
+ return FALSE;
+ }
- substitute_parse = newSVpvs("?:");
+ /* What is done here is to convert this to a sub-pattern of the
+ * form \x{char1}\x{char2}... and then call reg recursively to
+ * parse it (enclosing in "(?: ... )" ). That way, it retains its
+ * atomicness, while not having to worry about special handling
+ * that some code points may have. We don't create a subpattern,
+ * but go through the motions of code point counting and error
+ * checking, if the caller doesn't want a node returned. */
- while (RExC_parse < endbrace) {
+ if (node_p && count == 1) {
+ substitute_parse = newSVpvs("?:");
+ }
- /* Convert to notation the rest of the code understands */
- sv_catpv(substitute_parse, "\\x{");
- sv_catpvn(substitute_parse, RExC_parse, endchar - RExC_parse);
- sv_catpv(substitute_parse, "}");
+ do_concat:
- /* Point to the beginning of the next character in the sequence. */
- RExC_parse = endchar + 1;
- endchar = RExC_parse + strcspn(RExC_parse, ".}");
+ if (node_p) {
+ /* Convert to notation the rest of the code understands */
+ sv_catpv(substitute_parse, "\\x{");
+ sv_catpvn(substitute_parse, start_digit,
+ RExC_parse - start_digit);
+ sv_catpv(substitute_parse, "}");
+ }
- }
- sv_catpv(substitute_parse, ")");
+ /* Move to after the dot (or ending brace the final time through.)
+ * */
+ RExC_parse++;
+ count++;
- len = SvCUR(substitute_parse);
+ } while (RExC_parse < endbrace);
- /* Don't allow empty number */
- if (len < (STRLEN) 8) {
- RExC_parse = endbrace;
- vFAIL("Invalid hexadecimal number in \\N{U+...}");
- }
+ if (! node_p) { /* Doesn't want the node */
+ assert (cp_count);
- RExC_parse = RExC_start = RExC_adjusted_start
- = SvPV_nolen(substitute_parse);
- RExC_end = RExC_parse + len;
+ *cp_count = count;
+ return FALSE;
+ }
+
+ sv_catpv(substitute_parse, ")");
- /* The values are Unicode, and therefore not subject to recoding, but
- * have to be converted to native on a non-Unicode (meaning non-ASCII)
- * platform. */
#ifdef EBCDIC
+ /* The values are Unicode, and therefore have to be converted to native
+ * on a non-Unicode (meaning non-ASCII) platform. */
RExC_recode_x_to_native = 1;
#endif
- *node_p = reg(pRExC_state, 1, &flags, depth+1);
+ /* Here, we have the string the name evaluates to, ready to be parsed,
+ * stored in 'substitute_parse' as a series of valid "\x{...}\x{...}"
+ * constructs. This can be called from within a substitute parse already.
+ * The error reporting mechanism doesn't work for 2 levels of this, but the
+ * code above has validated this new construct, so there should be no
+ * errors generated by the below.*/
+ save_start = RExC_start;
+ orig_end = RExC_end;
- /* Restore the saved values */
- RExC_start = RExC_adjusted_start = save_start;
- RExC_parse = endbrace;
- RExC_end = orig_end;
+ RExC_parse = RExC_start = SvPVX(substitute_parse);
+ RExC_end = RExC_parse + SvCUR(substitute_parse);
+
+ *node_p = reg(pRExC_state, 1, &flags, depth+1);
+
+ /* Restore the saved values */
+ RExC_start = save_start;
+ RExC_parse = endbrace;
+ RExC_end = orig_end;
#ifdef EBCDIC
- RExC_recode_x_to_native = 0;
+ RExC_recode_x_to_native = 0;
#endif
- SvREFCNT_dec_NN(substitute_parse);
-
- if (! *node_p) {
- RETURN_X_ON_RESTART(FALSE, flags,flagp);
- FAIL2("panic: reg returned NULL to grok_bslash_N, flags=%#" UVxf,
- (UV) flags);
- }
- *flagp |= flags&(HASWIDTH|SPSTART|SIMPLE|POSTPONED);
- nextchar(pRExC_state);
+ SvREFCNT_dec_NN(substitute_parse);
- return TRUE;
+ if (! *node_p) {
+ RETURN_X_ON_RESTART(FALSE, flags,flagp);
+ FAIL2("panic: reg returned NULL to grok_bslash_N, flags=%#" UVxf,
+ (UV) flags);
}
+ *flagp |= flags&(HASWIDTH|SPSTART|SIMPLE|POSTPONED);
+
+ nextchar(pRExC_state);
+
+ return TRUE;
}
* in which case return I32_MAX (rather than possibly 32-bit wrapping) */
static I32
-S_backref_value(char *p)
+S_backref_value(char *p, char *e)
{
- const char* endptr;
+ const char* endptr = e;
UV val;
if (grok_atoUV(p, &val, &endptr) && val <= I32_MAX)
return (I32)val;
if (RExC_parse >= RExC_end) {
goto unterminated_g;
}
- num = S_backref_value(RExC_parse);
+ num = S_backref_value(RExC_parse, RExC_end);
if (num == 0)
vFAIL("Reference to invalid group 0");
else if (num == I32_MAX) {
}
}
else {
- num = S_backref_value(RExC_parse);
+ num = S_backref_value(RExC_parse, RExC_end);
/* bare \NNN might be backref or octal - if it is larger
* than or equal RExC_npar then it is assumed to be an
* octal escape. Note RExC_npar is +1 from the actual
* need to figure this out until pass 2) */
bool maybe_exactfu = PASS2;
+ /* To see if RExC_uni_semantics changes during parsing of the node.
+ * */
+ bool uni_semantics_at_node_start;
+
/* The node_type may change below, but since the size of the node
* doesn't change, it works */
ret = reg_node(pRExC_state, node_type);
|| UTF8_IS_INVARIANT(UCHARAT(RExC_parse))
|| UTF8_IS_START(UCHARAT(RExC_parse)));
+ uni_semantics_at_node_start = RExC_uni_semantics;
+
/* Here, we have a literal character. Find the maximal string of
* them in the input that we can fit into a single EXACTish node.
* We quit at the first non-literal or when the node gets full, or
/* NOTE, RExC_npar is 1 more than the actual number of
* parens we have seen so far, hence the < RExC_npar below. */
- if ( !isDIGIT(p[1]) || S_backref_value(p) < RExC_npar)
+ if ( !isDIGIT(p[1]) || S_backref_value(p, RExC_end) < RExC_npar)
{ /* Not to be treated as an octal constant, go
find backref */
--p;
} /* End of switch on '\' */
break;
case '{':
- /* Currently we allow an lbrace at the start of a construct
- * without raising a warning. This is because we think we
- * will never want such a brace to be meant to be other
- * than taken literally. */
+ /* Trying to gain new uses for '{' without breaking too
+ * much existing code is hard. The solution currently
+ * adopted is:
+ * 1) If there is no ambiguity that a '{' should always
+ * be taken literally, at the start of a construct, we
+ * just do so.
+ * 2) If the literal '{' conflicts with our desired use
+ * of it as a metacharacter, we die. The deprecation
+ * cycles for this have come and gone.
+ * 3) If there is ambiguity, we raise a simple warning.
+ * This could happen, for example, if the user
+ * intended it to introduce a quantifier, but slightly
+ * misspelled the quantifier. Without this warning,
+ * the quantifier would silently be taken as a literal
+ * string of characters instead of a meta construct */
if (len || (p > RExC_start && isALPHA_A(*(p - 1)))) {
-
- /* But, we raise a fatal warning otherwise, as the
- * deprecation cycle has come and gone. Except that it
- * turns out that some heavily-relied on upstream
- * software, notably GNU Autoconf, have failed to fix
- * their uses. For these, don't make it fatal unless
- * we anticipate using the '{' for something else.
- * This happens after any alpha, and for a looser {m,n}
- * quantifier specification */
if ( RExC_strict
|| ( p > parse_start + 1
&& isALPHA_A(*(p - 1))
"illegal here");
}
if (PASS2) {
- ckWARNregdep(p + 1,
- "Unescaped left brace in regex is "
- "deprecated here (and will be fatal "
- "in Perl 5.30), passed through");
+ ckWARNreg(p + 1, "Unescaped left brace in regex is"
+ " passed through");
}
}
goto normal_default;
ender = 's';
added_len = 2;
}
+ else if ( uni_semantics_at_node_start
+ != RExC_uni_semantics)
+ {
+ /* Here, we are supossed to be using Unicode
+ * rules, but this folding node is not. This
+ * happens during pass 1 when the node started
+ * out not under Unicode rules, but a \N{} was
+ * encountered during the processing of it,
+ * causing Unicode rules to be switched into.
+ * Pass 1 continues uninterrupted, as by the
+ * time we get to pass 2, we will know enough
+ * to generate the correct folds. Except in
+ * this one case, we need to restart the node,
+ * because the fold of the sharp s requires 2
+ * characters, and the sizing needs to account
+ * for that. */
+ p = oldp;
+ goto loopdone;
+ }
else {
RExC_seen_unfolded_sharp_s = 1;
maybe_exactfu = FALSE;
len = s - s0 + 1;
}
else {
- if (! PL_NonL1NonFinalFold) {
- PL_NonL1NonFinalFold = _new_invlist_C_array(
- NonL1_Perl_Non_Final_Folds_invlist);
- }
/* Point to the first byte of the final character */
s = (char *) utf8_hop((U8 *) s, -1);
/* Position parse to next real character */
skip_to_be_ignored_text(pRExC_state, &RExC_parse,
FALSE /* Don't force to /x */ );
- if (PASS2 && *RExC_parse == '{' && OP(ret) != SBOL && ! regcurly(RExC_parse)) {
- ckWARNregdep(RExC_parse + 1, "Unescaped left brace in regex is deprecated here (and will be fatal in Perl 5.30), passed through");
+ if ( PASS2 && *RExC_parse == '{'
+ && OP(ret) != SBOL && ! regcurly(RExC_parse))
+ {
+ if (RExC_strict || new_regcurly(RExC_parse, RExC_end)) {
+ RExC_parse++;
+ vFAIL("Unescaped left brace in regex is illegal here");
+ }
+ ckWARNreg(RExC_parse + 1, "Unescaped left brace in regex is"
+ " passed through");
}
return(ret);
* fence. Get rid of it */
fence_ptr = av_pop(fence_stack);
assert(fence_ptr);
- fence = SvIV(fence_ptr) - 1;
+ fence = SvIV(fence_ptr);
SvREFCNT_dec_NN(fence_ptr);
fence_ptr = NULL;
STATIC void
S_add_above_Latin1_folds(pTHX_ RExC_state_t *pRExC_state, const U8 cp, SV** invlist)
{
- /* This hard-codes the Latin1/above-Latin1 folding rules, so that an
- * innocent-looking character class, like /[ks]/i won't have to go out to
- * disk to find the possible matches.
+ /* This adds the Latin1/above-Latin1 folding rules.
*
* This should be called only for a Latin1-range code points, cp, which is
* known to be involved in a simple fold with other code points above
* Latin1. It would give false results if /aa has been specified.
* Multi-char folds are outside the scope of this, and must be handled
- * specially.
- *
- * XXX It would be better to generate these via regen, in case a new
- * version of the Unicode standard adds new mappings, though that is not
- * really likely, and may be caught by the default: case of the switch
- * below. */
+ * specially. */
PERL_ARGS_ASSERT_ADD_ABOVE_LATIN1_FOLDS;
assert(HAS_NONLATIN1_SIMPLE_FOLD_CLOSURE(cp));
+ /* The rules that are valid for all Unicode versions are hard-coded in */
switch (cp) {
case 'k':
case 'K':
LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS);
break;
-#ifdef LATIN_CAPITAL_LETTER_SHARP_S /* not defined in early Unicode releases */
+ default: /* Other code points are checked against the data for the
+ current Unicode version */
+ {
+ Size_t folds_to_count;
+ unsigned int first_folds_to;
+ const unsigned int * remaining_folds_to_list;
+ UV folded_cp;
- case LATIN_SMALL_LETTER_SHARP_S:
- *invlist = add_cp_to_invlist(*invlist, LATIN_CAPITAL_LETTER_SHARP_S);
- break;
+ if (isASCII(cp)) {
+ folded_cp = toFOLD(cp);
+ }
+ else {
+ U8 dummy_fold[UTF8_MAXBYTES_CASE+1];
+ Size_t dummy_len;
+ folded_cp = _to_fold_latin1(cp, dummy_fold, &dummy_len, 0);
+ }
-#endif
+ if (folded_cp > 255) {
+ *invlist = add_cp_to_invlist(*invlist, folded_cp);
+ }
-#if UNICODE_MAJOR_VERSION < 3 \
- || (UNICODE_MAJOR_VERSION == 3 && UNICODE_DOT_VERSION == 0)
+ folds_to_count = _inverse_folds(folded_cp, &first_folds_to,
+ &remaining_folds_to_list);
+ if (folds_to_count == 0) {
- /* In 3.0 and earlier, U+0130 folded simply to 'i'; and in 3.0.1 so did
- * U+0131. */
- case 'i':
- case 'I':
- *invlist =
- add_cp_to_invlist(*invlist, LATIN_CAPITAL_LETTER_I_WITH_DOT_ABOVE);
-# if UNICODE_DOT_DOT_VERSION == 1
- *invlist = add_cp_to_invlist(*invlist, LATIN_SMALL_LETTER_DOTLESS_I);
-# endif
- break;
-#endif
+ /* Use deprecated warning to increase the chances of this being
+ * output */
+ if (PASS2) {
+ ckWARN2reg_d(RExC_parse,
+ "Perl folding rules are not up-to-date for 0x%02X;"
+ " please use the perlbug utility to report;", cp);
+ }
+ }
+ else {
+ unsigned int i;
- default:
- /* Use deprecated warning to increase the chances of this being
- * output */
- if (PASS2) {
- ckWARN2reg_d(RExC_parse, "Perl folding rules are not up-to-date for 0x%02X; please use the perlbug utility to report;", cp);
+ if (first_folds_to > 255) {
+ *invlist = add_cp_to_invlist(*invlist, first_folds_to);
+ }
+ for (i = 0; i < folds_to_count - 1; i++) {
+ if (remaining_folds_to_list[i] > 255) {
+ *invlist = add_cp_to_invlist(*invlist,
+ remaining_folds_to_list[i]);
+ }
+ }
}
break;
+ }
}
}
case 'P':
{
char *e;
+ char *i;
+
/* We will handle any undefined properties ourselves */
U8 swash_init_flags = _CORE_SWASH_INIT_RETURN_IF_UNDEF
* anyway, to save a little time */
|_CORE_SWASH_INIT_ACCEPT_INVLIST;
+ SvREFCNT_dec(swash); /* Free any left-overs */
if (RExC_parse >= RExC_end)
vFAIL2("Empty \\%c", (U8)value);
if (*RExC_parse == '{') {
n = e - RExC_parse;
while (isSPACE(*(RExC_parse + n - 1)))
n--;
+
} /* The \p isn't immediately followed by a '{' */
else if (! isALPHA(*RExC_parse)) {
RExC_parse += (UTF) ? UTF8SKIP(RExC_parse) : 1;
n = 1;
}
if (!SIZE_ONLY) {
- SV* invlist;
- char* name;
+ char* name = RExC_parse;
char* base_name; /* name after any packages are stripped */
char* lookup_name = NULL;
const char * const colon_colon = "::";
+ bool invert;
+
+ SV* invlist;
+
+ /* Temporary workaround for [perl #133136]. For this
+ * precise input that is in the .t that is failing, use the
+ * old method so that that .t passes */
+ if (memEQs(RExC_start, e + 1 - RExC_start, "foo\\p{Alnum}"))
+ {
+ invlist = NULL;
+ }
+ else {
+ invlist = parse_uniprop_string(name, n, FOLD, &invert);
+ }
+ if (invlist) {
+ if (invert) {
+ value ^= 'P' ^ 'p';
+ }
+ }
+ else {
/* Try to get the definition of the property into
* <invlist>. If /i is in effect, the effective property
* 2f833f5208e26b208886e51e09e2c072b5eabb46 */
name = savepv(Perl_form(aTHX_ "%.*s", (int)n, RExC_parse));
SAVEFREEPV(name);
+
+ for (i = RExC_parse; i < RExC_parse + n; i++) {
+ if (isCNTRL(*i) && *i != '\t') {
+ RExC_parse = e + 1;
+ vFAIL2("Can't find Unicode property definition \"%s\"", name);
+ }
+ }
+
if (FOLD) {
lookup_name = savepv(Perl_form(aTHX_ "__%s_i", name));
/* Look up the property name, and get its swash and
* inversion list, if the property is found */
- SvREFCNT_dec(swash); /* Free any left-overs */
swash = _core_swash_init("utf8",
(lookup_name)
? lookup_name
{
has_user_defined_property = TRUE;
}
- else if
+ }
+ }
+ if (invlist) {
+ if (! has_user_defined_property &&
/* We warn on matching an above-Unicode code point
* if the match would return true, except don't
* warn for \p{All}, which has exactly one element
* = 0 */
(_invlist_contains_cp(invlist, 0x110000)
&& (! (_invlist_len(invlist) == 1
- && *invlist_array(invlist) == 0)))
+ && *invlist_array(invlist) == 0))))
{
warn_super = TRUE;
}
-
/* Invert if asking for the complement */
if (value == 'P') {
_invlist_union_complement_2nd(properties,
/* The swash can't be used as-is, because we've
* inverted things; delay removing it to here after
* have copied its invlist above */
- SvREFCNT_dec_NN(swash);
+ if (! swash) {
+ SvREFCNT_dec_NN(invlist);
+ }
+ SvREFCNT_dec(swash);
swash = NULL;
}
else {
_invlist_union(properties, invlist, &properties);
+ if (! swash) {
+ SvREFCNT_dec_NN(invlist);
+ }
}
- }
- }
+ }
+ }
RExC_parse = e + 1;
namedclass = ANYOF_UNIPROP; /* no official name, but it's
named */
}
}
else if ( UNI_SEMANTICS
+ || AT_LEAST_ASCII_RESTRICTED
|| classnum == _CC_ASCII
|| (DEPENDS_SEMANTICS && ( classnum == _CC_DIGIT
|| classnum == _CC_XDIGIT)))
{
- /* We usually have to worry about /d and /a affecting what
- * POSIX classes match, with special code needed for /d
- * because we won't know until runtime what all matches.
- * But there is no extra work needed under /u, and
- * [:ascii:] is unaffected by /a and /d; and :digit: and
- * :xdigit: don't have runtime differences under /d. So we
- * can special case these, and avoid some extra work below,
- * and at runtime. */
+ /* We usually have to worry about /d a affecting what POSIX
+ * classes match, with special code needed because we won't
+ * know until runtime what all matches. But there is no
+ * extra work needed under /u and /a; and [:ascii:] is
+ * unaffected by /d; and :digit: and :xdigit: don't have
+ * runtime differences under /d. So we can special case
+ * these, and avoid some extra work below, and at runtime.
+ * */
_invlist_union_maybe_complement_2nd(
simple_posixes,
- PL_XPosix_ptrs[classnum],
+ ((AT_LEAST_ASCII_RESTRICTED)
+ ? PL_Posix_ptrs[classnum]
+ : PL_XPosix_ptrs[classnum]),
namedclass % 2 != 0,
&simple_posixes);
}
_invlist_intersection(PL_utf8_foldable, cp_foldable_list,
&fold_intersection);
- /* The folds for all the Latin1 characters are hard-coded into this
- * program, but we have to go out to disk to get the others. */
- if (invlist_highest(cp_foldable_list) >= 256) {
-
- /* This is a hash that for a particular fold gives all
- * characters that are involved in it */
- if (! PL_utf8_foldclosures) {
- _load_PL_utf8_foldclosures();
- }
- }
-
/* Now look at the foldable characters in this class individually */
invlist_iterinit(fold_intersection);
while (invlist_iternext(fold_intersection, &start, &end)) {
UV j;
+ UV folded;
/* Look at every character in the range */
for (j = start; j <= end; j++) {
U8 foldbuf[UTF8_MAXBYTES_CASE+1];
STRLEN foldlen;
- SV** listp;
+ unsigned int k;
+ Size_t folds_to_count;
+ unsigned int first_folds_to;
+ const unsigned int * remaining_folds_to_list;
if (j < 256) {
* rules hard-coded for it. First, get its fold. This is
* the simple fold, as the multi-character folds have been
* handled earlier and separated out */
- _to_uni_fold_flags(j, foldbuf, &foldlen,
+ folded = _to_uni_fold_flags(j, foldbuf, &foldlen,
(ASCII_FOLD_RESTRICTED)
? FOLD_FLAGS_NOMIX_ASCII
: 0);
- /* Single character fold of above Latin1. Add everything in
- * its fold closure to the list that this node should match.
- * The fold closures data structure is a hash with the keys
- * being the UTF-8 of every character that is folded to, like
- * 'k', and the values each an array of all code points that
- * fold to its key. e.g. [ 'k', 'K', KELVIN_SIGN ].
- * Multi-character folds are not included */
- if ((listp = hv_fetch(PL_utf8_foldclosures,
- (char *) foldbuf, foldlen, FALSE)))
- {
- AV* list = (AV*) *listp;
- IV k;
- for (k = 0; k <= av_tindex_skip_len_mg(list); k++) {
- SV** c_p = av_fetch(list, k, FALSE);
- UV c;
- assert(c_p);
-
- c = SvUV(*c_p);
-
- /* /aa doesn't allow folds between ASCII and non- */
- if ((ASCII_FOLD_RESTRICTED
- && (isASCII(c) != isASCII(j))))
- {
- continue;
- }
+ /* Single character fold of above Latin1. Add everything
+ * in its fold closure to the list that this node should
+ * match. */
+ folds_to_count = _inverse_folds(folded, &first_folds_to,
+ &remaining_folds_to_list);
+ for (k = 0; k <= folds_to_count; k++) {
+ UV c = (k == 0) /* First time through use itself */
+ ? folded
+ : (k == 1) /* 2nd time use, the first fold */
+ ? first_folds_to
+
+ /* Then the remaining ones */
+ : remaining_folds_to_list[k-2];
+
+ /* /aa doesn't allow folds between ASCII and non- */
+ if (( ASCII_FOLD_RESTRICTED
+ && (isASCII(c) != isASCII(j))))
+ {
+ continue;
+ }
- /* Folds under /l which cross the 255/256 boundary
- * are added to a separate list. (These are valid
- * only when the locale is UTF-8.) */
- if (c < 256 && LOC) {
- *use_list = add_cp_to_invlist(*use_list, c);
- continue;
- }
+ /* Folds under /l which cross the 255/256 boundary are
+ * added to a separate list. (These are valid only
+ * when the locale is UTF-8.) */
+ if (c < 256 && LOC) {
+ *use_list = add_cp_to_invlist(*use_list, c);
+ continue;
+ }
- if (isASCII(c) || c > 255 || AT_LEAST_UNI_SEMANTICS)
- {
- cp_list = add_cp_to_invlist(cp_list, c);
- }
- else {
- /* Similarly folds involving non-ascii Latin1
- * characters under /d are added to their list */
- has_upper_latin1_only_utf8_matches
- = add_cp_to_invlist(
- has_upper_latin1_only_utf8_matches,
- c);
- }
+ if (isASCII(c) || c > 255 || AT_LEAST_UNI_SEMANTICS)
+ {
+ cp_list = add_cp_to_invlist(cp_list, c);
+ }
+ else {
+ /* Similarly folds involving non-ascii Latin1
+ * characters under /d are added to their list */
+ has_upper_latin1_only_utf8_matches
+ = add_cp_to_invlist(
+ has_upper_latin1_only_utf8_matches,
+ c);
}
}
}
}
}
if (posixes || nposixes) {
-
- /* We have to adjust /a and /aa */
- if (AT_LEAST_ASCII_RESTRICTED) {
-
- /* Under /a and /aa, nothing above ASCII matches these */
- if (posixes) {
- _invlist_intersection(posixes,
- PL_XPosix_ptrs[_CC_ASCII],
- &posixes);
- }
-
- /* Under /a and /aa, everything above ASCII matches these
- * complements */
- if (nposixes) {
- _invlist_union_complement_2nd(nposixes,
- PL_XPosix_ptrs[_CC_ASCII],
- &nposixes);
- }
- }
-
if (! DEPENDS_SEMANTICS) {
/* For everything but /d, we can just add the current 'posixes' and
#endif /* DEBUGGING */
+#ifndef PERL_IN_XSUB_RE
+
+#include "uni_keywords.h"
+
+void
+Perl_init_uniprops(pTHX)
+{
+ /* Set up the inversion list global variables */
+
+ PL_XPosix_ptrs[_CC_ASCII] = _new_invlist_C_array(uni_prop_ptrs[UNI_ASCII]);
+ PL_XPosix_ptrs[_CC_ALPHANUMERIC] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXALNUM]);
+ PL_XPosix_ptrs[_CC_ALPHA] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXALPHA]);
+ PL_XPosix_ptrs[_CC_BLANK] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXBLANK]);
+ PL_XPosix_ptrs[_CC_CASED] = _new_invlist_C_array(uni_prop_ptrs[UNI_CASED]);
+ PL_XPosix_ptrs[_CC_CNTRL] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXCNTRL]);
+ PL_XPosix_ptrs[_CC_DIGIT] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXDIGIT]);
+ PL_XPosix_ptrs[_CC_GRAPH] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXGRAPH]);
+ PL_XPosix_ptrs[_CC_LOWER] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXLOWER]);
+ PL_XPosix_ptrs[_CC_PRINT] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXPRINT]);
+ PL_XPosix_ptrs[_CC_PUNCT] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXPUNCT]);
+ PL_XPosix_ptrs[_CC_SPACE] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXSPACE]);
+ PL_XPosix_ptrs[_CC_UPPER] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXUPPER]);
+ PL_XPosix_ptrs[_CC_VERTSPACE] = _new_invlist_C_array(uni_prop_ptrs[UNI_VERTSPACE]);
+ PL_XPosix_ptrs[_CC_WORDCHAR] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXWORD]);
+ PL_XPosix_ptrs[_CC_XDIGIT] = _new_invlist_C_array(uni_prop_ptrs[UNI_XPOSIXXDIGIT]);
+
+ PL_Posix_ptrs[_CC_ASCII] = _new_invlist_C_array(uni_prop_ptrs[UNI_ASCII]);
+ PL_Posix_ptrs[_CC_ALPHANUMERIC] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXALNUM]);
+ PL_Posix_ptrs[_CC_ALPHA] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXALPHA]);
+ PL_Posix_ptrs[_CC_BLANK] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXBLANK]);
+ PL_Posix_ptrs[_CC_CASED] = _new_invlist_C_array(uni_prop_ptrs[UNI_CASED]);
+ PL_Posix_ptrs[_CC_CNTRL] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXCNTRL]);
+ PL_Posix_ptrs[_CC_DIGIT] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXDIGIT]);
+ PL_Posix_ptrs[_CC_GRAPH] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXGRAPH]);
+ PL_Posix_ptrs[_CC_LOWER] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXLOWER]);
+ PL_Posix_ptrs[_CC_PRINT] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXPRINT]);
+ PL_Posix_ptrs[_CC_PUNCT] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXPUNCT]);
+ PL_Posix_ptrs[_CC_SPACE] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXSPACE]);
+ PL_Posix_ptrs[_CC_UPPER] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXUPPER]);
+ PL_Posix_ptrs[_CC_VERTSPACE] = _new_invlist_C_array(uni_prop_ptrs[UNI_VERTSPACE]);
+ PL_Posix_ptrs[_CC_WORDCHAR] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXWORD]);
+ PL_Posix_ptrs[_CC_XDIGIT] = _new_invlist_C_array(uni_prop_ptrs[UNI_POSIXXDIGIT]);
+
+ PL_GCB_invlist = _new_invlist_C_array(_Perl_GCB_invlist);
+ PL_SB_invlist = _new_invlist_C_array(_Perl_SB_invlist);
+ PL_WB_invlist = _new_invlist_C_array(_Perl_WB_invlist);
+ PL_LB_invlist = _new_invlist_C_array(_Perl_LB_invlist);
+ PL_SCX_invlist = _new_invlist_C_array(_Perl_SCX_invlist);
+
+ PL_AboveLatin1 = _new_invlist_C_array(AboveLatin1_invlist);
+ PL_Latin1 = _new_invlist_C_array(Latin1_invlist);
+ PL_UpperLatin1 = _new_invlist_C_array(UpperLatin1_invlist);
+
+ PL_Assigned_invlist = _new_invlist_C_array(uni_prop_ptrs[UNI_ASSIGNED]);
+
+ PL_utf8_perl_idstart = _new_invlist_C_array(uni_prop_ptrs[UNI__PERL_IDSTART]);
+ PL_utf8_perl_idcont = _new_invlist_C_array(uni_prop_ptrs[UNI__PERL_IDCONT]);
+
+ PL_utf8_charname_begin = _new_invlist_C_array(uni_prop_ptrs[UNI__PERL_CHARNAME_BEGIN]);
+ PL_utf8_charname_continue = _new_invlist_C_array(uni_prop_ptrs[UNI__PERL_CHARNAME_CONTINUE]);
+
+ PL_utf8_foldable = _new_invlist_C_array(uni_prop_ptrs[UNI__PERL_ANY_FOLDS]);
+ PL_HasMultiCharFold = _new_invlist_C_array(uni_prop_ptrs[
+ UNI__PERL_FOLDS_TO_MULTI_CHAR]);
+ PL_NonL1NonFinalFold = _new_invlist_C_array(
+ NonL1_Perl_Non_Final_Folds_invlist);
+
+ PL_utf8_toupper = _new_invlist_C_array(Uppercase_Mapping_invlist);
+ PL_utf8_tolower = _new_invlist_C_array(Lowercase_Mapping_invlist);
+ PL_utf8_totitle = _new_invlist_C_array(Titlecase_Mapping_invlist);
+ PL_utf8_tofold = _new_invlist_C_array(Case_Folding_invlist);
+ PL_utf8_tosimplefold = _new_invlist_C_array(Simple_Case_Folding_invlist);
+ PL_utf8_foldclosures = _new_invlist_C_array(_Perl_IVCF_invlist);
+ PL_utf8_mark = _new_invlist_C_array(uni_prop_ptrs[UNI_M]);
+
+ /* The below are used only by deprecated functions. They could be removed */
+ PL_utf8_xidcont = _new_invlist_C_array(uni_prop_ptrs[UNI_XIDC]);
+ PL_utf8_idcont = _new_invlist_C_array(uni_prop_ptrs[UNI_IDC]);
+ PL_utf8_xidstart = _new_invlist_C_array(uni_prop_ptrs[UNI_XIDS]);
+}
+
+SV *
+Perl_parse_uniprop_string(pTHX_ const char * const name, const Size_t len, const bool to_fold, bool * invert)
+{
+ /* Parse the interior meat of \p{} passed to this in 'name' with length 'len',
+ * and return an inversion list if a property with 'name' is found, or NULL
+ * if not. 'name' point to the input with leading and trailing space trimmed.
+ * 'to_fold' indicates if /i is in effect.
+ *
+ * When the return is an inversion list, '*invert' will be set to a boolean
+ * indicating if it should be inverted or not
+ *
+ * This currently doesn't handle all cases. A NULL return indicates the
+ * caller should try a different approach
+ */
+
+ char* lookup_name;
+ bool stricter = FALSE;
+ bool is_nv_type = FALSE; /* nv= or numeric_value=, or possibly one
+ of the cjk numeric properties (though
+ it requires extra effort to compile
+ them) */
+ unsigned int i;
+ unsigned int j = 0, lookup_len;
+ int equals_pos = -1; /* Where the '=' is found, or negative if none */
+ int slash_pos = -1; /* Where the '/' is found, or negative if none */
+ int table_index = 0;
+ bool starts_with_In_or_Is = FALSE;
+ Size_t lookup_offset = 0;
+
+ PERL_ARGS_ASSERT_PARSE_UNIPROP_STRING;
+
+ /* The input will be modified into 'lookup_name' */
+ Newx(lookup_name, len, char);
+ SAVEFREEPV(lookup_name);
+
+ /* Parse the input. */
+ for (i = 0; i < len; i++) {
+ char cur = name[i];
+
+ /* These characters can be freely ignored in most situations. Later it
+ * may turn out we shouldn't have ignored them, and we have to reparse,
+ * but we don't have enough information yet to make that decision */
+ if (cur == '-' || cur == '_' || isSPACE(cur)) {
+ continue;
+ }
+
+ /* Case differences are also ignored. Our lookup routine assumes
+ * everything is lowercase */
+ if (isUPPER(cur)) {
+ lookup_name[j++] = toLOWER(cur);
+ continue;
+ }
+
+ /* A double colon is either an error, or a package qualifier to a
+ * subroutine user-defined property; neither of which do we currently
+ * handle
+ *
+ * But a single colon is a synonym for '=' */
+ if (cur == ':') {
+ if (i < len - 1 && name[i+1] == ':') {
+ return NULL;
+ }
+ cur = '=';
+ }
+
+ /* Otherwise, this character is part of the name. */
+ lookup_name[j++] = cur;
+
+ /* Only the equals sign needs further processing */
+ if (cur == '=') {
+ equals_pos = j; /* Note where it occurred in the input */
+ break;
+ }
+ }
+
+ /* Here, we are either done with the whole property name, if it was simple;
+ * or are positioned just after the '=' if it is compound. */
+
+ if (equals_pos >= 0) {
+ assert(! stricter); /* We shouldn't have set this yet */
+
+ /* Space immediately after the '=' is ignored */
+ i++;
+ for (; i < len; i++) {
+ if (! isSPACE(name[i])) {
+ break;
+ }
+ }
+
+ /* Certain properties need special handling. They may optionally be
+ * prefixed by 'is'. Ignore that prefix for the purposes of checking
+ * if this is one of those properties */
+ if (memBEGINPs(lookup_name, len, "is")) {
+ lookup_offset = 2;
+ }
+
+ /* Then check if it is one of these properties. This is hard-coded
+ * because easier this way, and the list is unlikely to change. There
+ * are several properties like this in the Unihan DB, which is unlikely
+ * to be compiled, and they all end with 'numeric'. The interiors
+ * aren't checked for the precise property. This would stop working if
+ * a cjk property were to be created that ended with 'numeric' and
+ * wasn't a numeric type */
+ is_nv_type = memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "numericvalue")
+ || memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "nv")
+ || ( memENDPs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "numeric")
+ && ( memBEGINPs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "cjk")
+ || memBEGINPs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "k")));
+ if ( is_nv_type
+ || memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "canonicalcombiningclass")
+ || memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "ccc")
+ || memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "age")
+ || memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "in")
+ || memEQs(lookup_name + lookup_offset,
+ j - 1 - lookup_offset, "presentin"))
+ {
+ unsigned int k;
+
+ /* What makes these properties special is that the stuff after the
+ * '=' is a number. Therefore, we can't throw away '-'
+ * willy-nilly, as those could be a minus sign. Other stricter
+ * rules also apply. However, these properties all can have the
+ * rhs not be a number, in which case they contain at least one
+ * alphabetic. In those cases, the stricter rules don't apply.
+ * But the numeric type properties can have the alphas [Ee] to
+ * signify an exponent, and it is still a number with stricter
+ * rules. So look for an alpha that signifys not-strict */
+ stricter = TRUE;
+ for (k = i; k < len; k++) {
+ if ( isALPHA(name[k])
+ && (! is_nv_type || ! isALPHA_FOLD_EQ(name[k], 'E')))
+ {
+ stricter = FALSE;
+ break;
+ }
+ }
+ }
+
+ if (stricter) {
+
+ /* A number may have a leading '+' or '-'. The latter is retained
+ * */
+ if (name[i] == '+') {
+ i++;
+ }
+ else if (name[i] == '-') {
+ lookup_name[j++] = '-';
+ i++;
+ }
+
+ /* Skip leading zeros including single underscores separating the
+ * zeros, or between the final leading zero and the first other
+ * digit */
+ for (; i < len - 1; i++) {
+ if ( name[i] != '0'
+ && (name[i] != '_' || ! isDIGIT(name[i+1])))
+ {
+ break;
+ }
+ }
+ }
+ }
+ else { /* No '=' */
+
+ /* We are now in a position to determine if this property should have
+ * been parsed using stricter rules. Only a few are like that, and
+ * unlikely to change. */
+ if ( memBEGINPs(lookup_name, j, "perl")
+ && memNEs(lookup_name + 4, j - 4, "space")
+ && memNEs(lookup_name + 4, j - 4, "word"))
+ {
+ stricter = TRUE;
+
+ /* We set the inputs back to 0 and the code below will reparse,
+ * using strict */
+ i = j = 0;
+ }
+ }
+
+ /* Here, we have either finished the property, or are positioned to parse
+ * the remainder, and we know if stricter rules apply. Finish out, if not
+ * already done */
+ for (; i < len; i++) {
+ char cur = name[i];
+
+ /* In all instances, case differences are ignored, and we normalize to
+ * lowercase */
+ if (isUPPER(cur)) {
+ lookup_name[j++] = toLOWER(cur);
+ continue;
+ }
+
+ /* An underscore is skipped, but not under strict rules unless it
+ * separates two digits */
+ if (cur == '_') {
+ if ( stricter
+ && ( i == 0 || (int) i == equals_pos || i == len- 1
+ || ! isDIGIT(name[i-1]) || ! isDIGIT(name[i+1])))
+ {
+ lookup_name[j++] = '_';
+ }
+ continue;
+ }
+
+ /* Hyphens are skipped except under strict */
+ if (cur == '-' && ! stricter) {
+ continue;
+ }
+
+ /* XXX Bug in documentation. It says white space skipped adjacent to
+ * non-word char. Maybe we should, but shouldn't skip it next to a dot
+ * in a number */
+ if (isSPACE(cur) && ! stricter) {
+ continue;
+ }
+
+ lookup_name[j++] = cur;
+
+ /* Unless this is a non-trailing slash, we are done with it */
+ if (i >= len - 1 || cur != '/') {
+ continue;
+ }
+
+ slash_pos = j;
+
+ /* A slash in the 'numeric value' property indicates that what follows
+ * is a denominator. It can have a leading '+' and '0's that should be
+ * skipped. But we have never allowed a negative denominator, so treat
+ * a minus like every other character. (No need to rule out a second
+ * '/', as that won't match anything anyway */
+ if (is_nv_type) {
+ i++;
+ if (i < len && name[i] == '+') {
+ i++;
+ }
+
+ /* Skip leading zeros including underscores separating digits */
+ for (; i < len - 1; i++) {
+ if ( name[i] != '0'
+ && (name[i] != '_' || ! isDIGIT(name[i+1])))
+ {
+ break;
+ }
+ }
+
+ /* Store the first real character in the denominator */
+ lookup_name[j++] = name[i];
+ }
+ }
+
+ /* Here are completely done parsing the input 'name', and 'lookup_name'
+ * contains a copy, normalized.
+ *
+ * This special case is grandfathered in: 'L_' and 'GC=L_' are accepted and
+ * different from without the underscores. */
+ if ( ( UNLIKELY(memEQs(lookup_name, j, "l"))
+ || UNLIKELY(memEQs(lookup_name, j, "gc=l")))
+ && UNLIKELY(name[len-1] == '_'))
+ {
+ lookup_name[j++] = '&';
+ }
+ else if (len > 2 && name[0] == 'I' && ( name[1] == 'n' || name[1] == 's'))
+ {
+
+ /* Also, if the original input began with 'In' or 'Is', it could be a
+ * subroutine call instead of a property names, which currently isn't
+ * handled by this function. Subroutine calls can't happen if there is
+ * an '=' in the name */
+ if (equals_pos < 0 && get_cvn_flags(name, len, GV_NOTQUAL) != NULL) {
+ return NULL;
+ }
+
+ starts_with_In_or_Is = TRUE;
+ }
+
+ lookup_len = j; /* Use a more mnemonic name starting here */
+
+ /* Get the index into our pointer table of the inversion list corresponding
+ * to the property */
+ table_index = match_uniprop((U8 *) lookup_name, lookup_len);
+
+ /* If it didn't find the property */
+ if (table_index == 0) {
+
+ /* If didn't find the property, we try again stripping off any initial
+ * 'In' or 'Is' */
+ if (starts_with_In_or_Is) {
+ lookup_name += 2;
+ lookup_len -= 2;
+ equals_pos -= 2;
+ slash_pos -= 2;
+
+ table_index = match_uniprop((U8 *) lookup_name, lookup_len);
+ }
+
+ if (table_index == 0) {
+ char * canonical;
+
+ /* If not found, and not a numeric type property, isn't a legal
+ * property */
+ if (! is_nv_type) {
+ return NULL;
+ }
+
+ /* But the numeric type properties need more work to decide. What
+ * we do is make sure we have the number in canonical form and look
+ * that up. */
+
+ if (slash_pos < 0) { /* No slash */
+
+ /* When it isn't a rational, take the input, convert it to a
+ * NV, then create a canonical string representation of that
+ * NV. */
+
+ NV value;
+
+ /* Get the value */
+ if (my_atof3(lookup_name + equals_pos, &value,
+ lookup_len - equals_pos)
+ != lookup_name + lookup_len)
+ {
+ return NULL;
+ }
+
+ /* If the value is an integer, the canonical value is integral */
+ if (Perl_ceil(value) == value) {
+ canonical = Perl_form(aTHX_ "%.*s%.0" NVff,
+ equals_pos, lookup_name, value);
+ }
+ else { /* Otherwise, it is %e with a known precision */
+ canonical = Perl_form(aTHX_ "%.*s%.*" NVef,
+ equals_pos, lookup_name,
+ PL_E_FORMAT_PRECISION, value);
+ }
+ }
+ else { /* Has a slash. Create a rational in canonical form */
+ UV numerator, denominator, gcd, trial;
+ const char * end_ptr;
+ const char * sign = "";
+
+ /* We can't just find the numerator, denominator, and do the
+ * division, then use the method above, because that is
+ * inexact. And the input could be a rational that is within
+ * epsilon (given our precision) of a valid rational, and would
+ * then incorrectly compare valid.
+ *
+ * We're only interested in the part after the '=' */
+ const char * this_lookup_name = lookup_name + equals_pos;
+ lookup_len -= equals_pos;
+ slash_pos -= equals_pos;
+
+ /* Handle any leading minus */
+ if (this_lookup_name[0] == '-') {
+ sign = "-";
+ this_lookup_name++;
+ lookup_len--;
+ slash_pos--;
+ }
+
+ /* Convert the numerator to numeric */
+ end_ptr = this_lookup_name + slash_pos;
+ if (! grok_atoUV(this_lookup_name, &numerator, &end_ptr)) {
+ return NULL;
+ }
+
+ /* It better have included all characters before the slash */
+ if (*end_ptr != '/') {
+ return NULL;
+ }
+
+ /* Set to look at just the denominator */
+ this_lookup_name += slash_pos;
+ lookup_len -= slash_pos;
+ end_ptr = this_lookup_name + lookup_len;
+
+ /* Convert the denominator to numeric */
+ if (! grok_atoUV(this_lookup_name, &denominator, &end_ptr)) {
+ return NULL;
+ }
+
+ /* It better be the rest of the characters, and don't divide by
+ * 0 */
+ if ( end_ptr != this_lookup_name + lookup_len
+ || denominator == 0)
+ {
+ return NULL;
+ }
+
+ /* Get the greatest common denominator using
+ http://en.wikipedia.org/wiki/Euclidean_algorithm */
+ gcd = numerator;
+ trial = denominator;
+ while (trial != 0) {
+ UV temp = trial;
+ trial = gcd % trial;
+ gcd = temp;
+ }
+
+ /* If already in lowest possible terms, we have already tried
+ * looking this up */
+ if (gcd == 1) {
+ return NULL;
+ }
+
+ /* Reduce the rational, which should put it in canonical form.
+ * Then look it up */
+ numerator /= gcd;
+ denominator /= gcd;
+
+ canonical = Perl_form(aTHX_ "%.*s%s%" UVuf "/%" UVuf,
+ equals_pos, lookup_name, sign, numerator, denominator);
+ }
+
+ /* Here, we have the number in canonical form. Try that */
+ table_index = match_uniprop((U8 *) canonical, strlen(canonical));
+ if (table_index == 0) {
+ return NULL;
+ }
+ }
+ }
+
+ /* The return is an index into a table of ptrs. A negative return
+ * signifies that the real index is the absolute value, but the result
+ * needs to be inverted */
+ if (table_index < 0) {
+ *invert = TRUE;
+ table_index = -table_index;
+ }
+ else {
+ *invert = FALSE;
+ }
+
+ /* Out-of band indices indicate a deprecated property. The proper index is
+ * modulo it with the table size. And dividing by the table size yields
+ * an offset into a table constructed to contain the corresponding warning
+ * message */
+ if (table_index > MAX_UNI_KEYWORD_INDEX) {
+ Size_t warning_offset = table_index / MAX_UNI_KEYWORD_INDEX;
+ table_index %= MAX_UNI_KEYWORD_INDEX;
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_DEPRECATED),
+ "Use of '%.*s' in \\p{} or \\P{} is deprecated because: %s",
+ (int) len, name, deprecated_property_msgs[warning_offset]);
+ }
+
+ /* In a few properties, a different property is used under /i. These are
+ * unlikely to change, so are hard-coded here. */
+ if (to_fold) {
+ if ( table_index == UNI_XPOSIXUPPER
+ || table_index == UNI_XPOSIXLOWER
+ || table_index == UNI_TITLE)
+ {
+ table_index = UNI_CASED;
+ }
+ else if ( table_index == UNI_UPPERCASELETTER
+ || table_index == UNI_LOWERCASELETTER
+# ifdef UNI_TITLECASELETTER /* Missing from early Unicodes */
+ || table_index == UNI_TITLECASELETTER
+# endif
+ ) {
+ table_index = UNI_CASEDLETTER;
+ }
+ else if ( table_index == UNI_POSIXUPPER
+ || table_index == UNI_POSIXLOWER)
+ {
+ table_index = UNI_POSIXALPHA;
+ }
+ }
+
+ /* Create and return the inversion list */
+ return _new_invlist_C_array(uni_prop_ptrs[table_index]);
+}
+
+#endif
+
/*
* ex: set ts=8 sts=4 sw=4 et:
*/
https://perl5.git.perl.org / perl5.git / blobdiff