* License or the Artistic License, as specified in the README file.
*
*/
+
+#ifndef PERL_REGCOMP_H_
+#define PERL_REGCOMP_H_
+
#include "regcharclass.h"
/* Convert branch sequences to more efficient trie ops? */
char string[1];
};
+struct regnode_lstring { /* Constructed this way to keep the string aligned. */
+ U8 flags;
+ U8 type;
+ U16 next_off;
+ U32 str_len; /* Only 16 bits allowed before would overflow 'next_off' */
+ char string[1];
+};
+
/* Argument bearing node - workhorse,
arg1 is often for the data field */
struct regnode_1 {
* Cyrillic, Greek, Hebrew, Indian subcontinent, Latin, and Thai; but not Han,
* Japanese, nor Korean. (The regarglen structure in regnodes.h is a U8, and
* the trie types TRIEC and AHOCORASICKC are larger than U8 for shift values
- * below above 12.) Be sure to benchmark before changing, as larger sizes do
+ * above 12.) Be sure to benchmark before changing, as larger sizes do
* significantly slow down the test suite */
#define NUM_ANYOF_CODE_POINTS (1 << 8)
#undef OP
#undef OPERAND
-#undef MASK
#undef STRING
#define OP(p) ((p)->type)
#define FLAGS(p) ((p)->flags) /* Caution: Doesn't apply to all \
regnode types. For some, it's the \
character set of the regnode */
-#define OPERAND(p) (((struct regnode_string *)p)->string)
-#define MASK(p) ((char*)OPERAND(p))
-#define STR_LEN(p) (((struct regnode_string *)p)->str_len)
-#define STRING(p) (((struct regnode_string *)p)->string)
-#define STR_SZ(l) ((l + sizeof(regnode) - 1) / sizeof(regnode))
-#define NODE_SZ_STR(p) (STR_SZ(STR_LEN(p))+1)
+#define STR_LENs(p) (__ASSERT_(OP(p) != LEXACT && OP(p) != LEXACT_ONLY8) \
+ ((struct regnode_string *)p)->str_len)
+#define STRINGs(p) (__ASSERT_(OP(p) != LEXACT && OP(p) != LEXACT_ONLY8) \
+ ((struct regnode_string *)p)->string)
+#define OPERANDs(p) STRINGs(p)
+
+/* Long strings. Currently limited to length 18 bits, which handles a 262000
+ * byte string. The limiting factor is the 16 bit 'next_off' field, which
+ * points to the next regnode, so the furthest away it can be is 2**16. On
+ * most architectures, regnodes are 2**2 bytes long, so that yields 2**18
+ * bytes. Should a longer string be desired, we could increase it to 26 bits
+ * fairly easily, by changing this node to have longj type which causes the ARG
+ * field to be used for the link to the next regnode (although code would have
+ * to be changed to account for this), and then use a combination of the flags
+ * and next_off fields for the length. To get 34 bit length, also change the
+ * node to be an ARG2L, using the second 32 bit field for the length, and not
+ * using the flags nor next_off fields at all. One could have an llstring node
+ * and even an lllstring type. */
+#define STR_LENl(p) (__ASSERT_(OP(p) == LEXACT || OP(p) == LEXACT_ONLY8) \
+ (((struct regnode_lstring *)p)->str_len))
+#define STRINGl(p) (__ASSERT_(OP(p) == LEXACT || OP(p) == LEXACT_ONLY8) \
+ (((struct regnode_lstring *)p)->string))
+#define OPERANDl(p) STRINGl(p)
+
+#define STR_LEN(p) ((OP(p) == LEXACT || OP(p) == LEXACT_ONLY8) \
+ ? STR_LENl(p) : STR_LENs(p))
+#define STRING(p) ((OP(p) == LEXACT || OP(p) == LEXACT_ONLY8) \
+ ? STRINGl(p) : STRINGs(p))
+#define OPERAND(p) STRING(p)
+
+/* The number of (smallest) regnode equivalents that a string of length l bytes
+ * occupies */
+#define STR_SZ(l) (((l) + sizeof(regnode) - 1) / sizeof(regnode))
+
+/* The number of (smallest) regnode equivalents that the EXACTISH node 'p'
+ * occupies */
+#define NODE_SZ_STR(p) (STR_SZ(STR_LEN(p)) + 1 + regarglen[(p)->type])
+
+#define setSTR_LEN(p,v) \
+ STMT_START{ \
+ if (OP(p) == LEXACT || OP(p) == LEXACT_ONLY8) \
+ ((struct regnode_lstring *)(p))->str_len = (v); \
+ else \
+ ((struct regnode_string *)(p))->str_len = (v); \
+ } STMT_END
#undef NODE_ALIGN
#undef ARG_LOC
*
* 1) The bitmap has a compiled-in very finite size. So something else needs
* to be used to specify if a code point that is too large for the bitmap
- * actually matches. The mechanism currently is a swash or inversion
+ * actually matches. The mechanism currently is an inversion
* list. ANYOF_ONLY_HAS_BITMAP, described above, being TRUE indicates
* there are no matches of too-large code points. But if it is FALSE,
* then almost certainly there are matches too large for the bitmap. (The
* 2) A subset of item 1) is if all possible code points outside the bitmap
* match. This is a common occurrence when the class is complemented,
* like /[^ij]/. Therefore a bit is reserved to indicate this,
- * rather than having an expensive swash created,
+ * rather than having a more expensive inversion list created,
* ANYOF_MATCHES_ALL_ABOVE_BITMAP.
* 3) Under /d rules, it can happen that code points that are in the upper
* latin1 range (\x80-\xFF or their equivalents on EBCDIC platforms) match
* handled. But it can be a shared flag: see 5) below.
* 4) Also under /d rules, something like /[\Wfoo]/ will match everything in
* the \x80-\xFF range, unless the string being matched against is UTF-8.
- * A swash could be created for this case, but this is relatively common,
- * and it turns out that it's all or nothing: if any one of these code
- * points matches, they all do. Hence a single bit suffices. We use a
- * shared flag that doesn't take up space by itself:
- * ANYOF_SHARED_d_MATCHES_ALL_NON_UTF8_NON_ASCII_non_d_WARN_SUPER.
- * This also implies 1), with one exception: [:^cntrl:].
+ * An inversion list could be created for this case, but this is
+ * relatively common, and it turns out that it's all or nothing: if any
+ * one of these code points matches, they all do. Hence a single bit
+ * suffices. We use a shared flag that doesn't take up space by itself:
+ * ANYOF_SHARED_d_MATCHES_ALL_NON_UTF8_NON_ASCII_non_d_WARN_SUPER. This
+ * also implies 1), with one exception: [:^cntrl:].
* 5) A user-defined \p{} property may not have been defined by the time the
* regex is compiled. In this case, we don't know until runtime what it
* will match, so we have to assume it could match anything, including
#define ANYOFL_FOLD 0x04
/* Shared bit set only with ANYOFL and SSC nodes:
- * If ANYOFL_FOLD is set, this means there are potential matches valid
- * only if the locale is a UTF-8 one.
- * If ANYOFL_FOLD is NOT set, this means to warn if the runtime locale
+ * If ANYOFL_FOLD is set, this flag indicates there are potential matches
+ * valid only if the locale is a UTF-8 one.
+ * If ANYOFL_FOLD is NOT set, this flag means to warn if the runtime locale
* isn't a UTF-8 one (and the generated node assumes a UTF-8 locale).
* None of INVERT, POSIXL,
* ANYOF_SHARED_d_UPPER_LATIN1_UTF8_STRING_MATCHES_non_d_RUNTIME_USER_PROP
/* Shared bit:
* Under /d it means the ANYOFD node matches more things if the target
* string is encoded in UTF-8; any such things will be non-ASCII,
- * characters that are < 256, and can be accessed via the swash.
+ * characters that are < 256, and can be accessed via the inversion
+ * list.
* When not under /d, it means the ANYOF node contains a user-defined
* property that wasn't yet defined at the time the regex was compiled,
- * and so must be looked up at runtime, by creating a swash
+ * and so must be looked up at runtime, by creating an inversion list.
* (These uses are mutually exclusive because a user-defined property is
* specified by \p{}, and \p{} implies /u which deselects /d). The long macro
* name is to make sure that you are cautioned about its shared nature. Only
# define UCHARAT(p) ((int)*(p)&CHARMASK)
#endif
+/* Number of regnode equivalents that 'guy' occupies beyond the size of the
+ * smallest regnode. */
#define EXTRA_SIZE(guy) ((sizeof(guy)-1)/sizeof(struct regnode))
#define REG_ZERO_LEN_SEEN 0x00000001
* l - start op for literal (?{EVAL}) item
* L - start op for literal (?{EVAL}) item, with separate CV (qr//)
* r - pointer to an embedded code-containing qr, e.g. /ab$qr/
- * s - swash for Unicode-style character class, and the multicharacter
- * strings resulting from casefolding the single-character entries
- * in the character class
+ * s - inversion list for Unicode-style character class, and the
+ * multicharacter strings resulting from casefolding the single-character
+ * entries in the character class
* t - trie struct
* u - trie struct's widecharmap (a HV, so can't share, must dup)
* also used for revcharmap and words under DEBUGGING
#define RE_TRIE_MAXBUF_NAME "\022E_TRIE_MAXBUF"
#define RE_DEBUG_FLAGS "\022E_DEBUG_FLAGS"
+#define RE_COMPILE_RECURSION_INIT 1000
+#define RE_COMPILE_RECURSION_LIMIT "\022E_COMPILE_RECURSION_LIMIT"
+
/*
RE_DEBUG_FLAGS is used to control what debug output is emitted
#define RE_DEBUG_EXECUTE_TRIE 0x000400
/* Extra */
-#define RE_DEBUG_EXTRA_MASK 0xFF0000
-#define RE_DEBUG_EXTRA_TRIE 0x010000
-#define RE_DEBUG_EXTRA_OFFSETS 0x020000
-#define RE_DEBUG_EXTRA_OFFDEBUG 0x040000
-#define RE_DEBUG_EXTRA_STATE 0x080000
-#define RE_DEBUG_EXTRA_OPTIMISE 0x100000
-#define RE_DEBUG_EXTRA_BUFFERS 0x400000
-#define RE_DEBUG_EXTRA_GPOS 0x800000
+#define RE_DEBUG_EXTRA_MASK 0x1FF0000
+#define RE_DEBUG_EXTRA_TRIE 0x0010000
+#define RE_DEBUG_EXTRA_OFFSETS 0x0020000
+#define RE_DEBUG_EXTRA_OFFDEBUG 0x0040000
+#define RE_DEBUG_EXTRA_STATE 0x0080000
+#define RE_DEBUG_EXTRA_OPTIMISE 0x0100000
+#define RE_DEBUG_EXTRA_BUFFERS 0x0400000
+#define RE_DEBUG_EXTRA_GPOS 0x0800000
+#define RE_DEBUG_EXTRA_DUMP_PRE_OPTIMIZE 0x1000000
/* combined */
-#define RE_DEBUG_EXTRA_STACK 0x280000
+#define RE_DEBUG_EXTRA_STACK 0x0280000
#define RE_DEBUG_FLAG(x) (re_debug_flags & x)
/* Compile */
#define DEBUG_GPOS_r(x) DEBUG_r( \
if (DEBUG_v_TEST || (re_debug_flags & RE_DEBUG_EXTRA_GPOS)) x )
+#define DEBUG_DUMP_PRE_OPTIMIZE_r(x) DEBUG_r( \
+ if (DEBUG_v_TEST || (re_debug_flags & RE_DEBUG_EXTRA_DUMP_PRE_OPTIMIZE)) x )
+
/* initialization */
/* get_sv() can return NULL during global destruction. */
#define GET_RE_DEBUG_FLAGS DEBUG_r({ \
WB_BOUND
} bound_type;
+/* This unpacks the FLAGS field of ANYOFHx nodes. The value it contains
+ * gives the strict lower bound for the UTF-8 start byte of any code point
+ * matchable by the node, and a loose upper bound as well.
+ *
+ * The low bound is stored in the upper 6 bits, plus 0xC0.
+ * The loose upper bound is determined from the lowest 2 bits and the low bound
+ * (called x) as follows:
+ *
+ * 11 The upper limit of the range can be as much as (EF - x) / 8
+ * 10 The upper limit of the range can be as much as (EF - x) / 4
+ * 01 The upper limit of the range can be as much as (EF - x) / 2
+ * 00 The upper limit of the range can be as much as EF
+ *
+ * For motivation of this design, see commit message in
+ * 3146c00a633e9cbed741e10146662fbcedfdb8d3 */
+#ifdef EBCDIC
+# define MAX_ANYOF_HRx_BYTE 0xF4
+#else
+# define MAX_ANYOF_HRx_BYTE 0xEF
+#endif
+#define LOWEST_ANYOF_HRx_BYTE(b) (((b) >> 2) + 0xC0)
+#define HIGHEST_ANYOF_HRx_BYTE(b) \
+ (LOWEST_ANYOF_HRx_BYTE(b) \
+ + ((MAX_ANYOF_HRx_BYTE - LOWEST_ANYOF_HRx_BYTE(b)) >> ((b) & 3)))
+
+#endif /* PERL_REGCOMP_H_ */
+
/*
* ex: set ts=8 sts=4 sw=4 et:
*/
https://perl5.git.perl.org / perl5.git / blobdiff