This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
charset.t: Skip locale tests depending on platform
[perl5.git] / regcomp.h
CommitLineData
a0d0e21e 1/* regcomp.h
d6376244 2 *
4bb101f2 3 * Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
663f364b 4 * 2000, 2001, 2002, 2003, 2005, 2006, 2007, by Larry Wall and others
d6376244
JH
5 *
6 * You may distribute under the terms of either the GNU General Public
7 * License or the Artistic License, as specified in the README file.
8 *
a687059c 9 */
e1d1eefb 10#include "regcharclass.h"
a687059c 11
c277df42
IZ
12typedef OP OP_4tree; /* Will be redefined later. */
13
07be1b83 14
58e23c8d 15/* Convert branch sequences to more efficient trie ops? */
07be1b83 16#define PERL_ENABLE_TRIE_OPTIMISATION 1
58e23c8d 17
486ec47a 18/* Be really aggressive about optimising patterns with trie sequences? */
07be1b83 19#define PERL_ENABLE_EXTENDED_TRIE_OPTIMISATION 1
58e23c8d
YO
20
21/* Should the optimiser take positive assertions into account? */
68ba3a3f 22#define PERL_ENABLE_POSITIVE_ASSERTION_STUDY 0
58e23c8d
YO
23
24/* Not for production use: */
07be1b83 25#define PERL_ENABLE_EXPERIMENTAL_REGEX_OPTIMISATIONS 0
58e23c8d 26
7122b237
YO
27/* Activate offsets code - set to if 1 to enable */
28#ifdef DEBUGGING
29#define RE_TRACK_PATTERN_OFFSETS
30#endif
31
b9b4dddf
YO
32/* Unless the next line is uncommented it is illegal to combine lazy
33 matching with possessive matching. Frankly it doesn't make much sense
34 to allow it as X*?+ matches nothing, X+?+ matches a single char only,
35 and X{min,max}?+ matches min times only.
36 */
37/* #define REG_ALLOW_MINMOD_SUSPEND */
07be1b83 38
a687059c
LW
39/*
40 * The "internal use only" fields in regexp.h are present to pass info from
41 * compile to execute that permits the execute phase to run lots faster on
42 * simple cases. They are:
43 *
bd61b366 44 * regstart sv that must begin a match; NULL if none obvious
a687059c
LW
45 * reganch is the match anchored (at beginning-of-line only)?
46 * regmust string (pointer into program) that match must include, or NULL
79072805 47 * [regmust changed to SV* for bminstr()--law]
a687059c
LW
48 * regmlen length of regmust string
49 * [regmlen not used currently]
50 *
51 * Regstart and reganch permit very fast decisions on suitable starting points
52 * for a match, cutting down the work a lot. Regmust permits fast rejection
53 * of lines that cannot possibly match. The regmust tests are costly enough
e50aee73 54 * that pregcomp() supplies a regmust only if the r.e. contains something
a687059c
LW
55 * potentially expensive (at present, the only such thing detected is * or +
56 * at the start of the r.e., which can involve a lot of backup). Regmlen is
e50aee73 57 * supplied because the test in pregexec() needs it and pregcomp() is computing
a687059c
LW
58 * it anyway.
59 * [regmust is now supplied always. The tests that use regmust have a
60 * heuristic that disables the test if it usually matches.]
61 *
62 * [In fact, we now use regmust in many cases to locate where the search
63 * starts in the string, so if regback is >= 0, the regmust search is never
64 * wasted effort. The regback variable says how many characters back from
65 * where regmust matched is the earliest possible start of the match.
66 * For instance, /[a-z].foo/ has a regmust of 'foo' and a regback of 2.]
67 */
68
69/*
70 * Structure for regexp "program". This is essentially a linear encoding
71 * of a nondeterministic finite-state machine (aka syntax charts or
72 * "railroad normal form" in parsing technology). Each node is an opcode
73 * plus a "next" pointer, possibly plus an operand. "Next" pointers of
74 * all nodes except BRANCH implement concatenation; a "next" pointer with
75 * a BRANCH on both ends of it is connecting two alternatives. (Here we
76 * have one of the subtle syntax dependencies: an individual BRANCH (as
77 * opposed to a collection of them) is never concatenated with anything
78 * because of operator precedence.) The operand of some types of node is
79 * a literal string; for others, it is a node leading into a sub-FSM. In
80 * particular, the operand of a BRANCH node is the first node of the branch.
81 * (NB this is *not* a tree structure: the tail of the branch connects
a3621e74
YO
82 * to the thing following the set of BRANCHes.) The opcodes are defined
83 * in regnodes.h which is generated from regcomp.sym by regcomp.pl.
a687059c
LW
84 */
85
a687059c
LW
86/*
87 * A node is one char of opcode followed by two chars of "next" pointer.
88 * "Next" pointers are stored as two 8-bit pieces, high order first. The
89 * value is a positive offset from the opcode of the node containing it.
90 * An operand, if any, simply follows the node. (Note that much of the
91 * code generation knows about this implicit relationship.)
92 *
93 * Using two bytes for the "next" pointer is vast overkill for most things,
94 * but allows patterns to get big without disasters.
95 *
e91177ed 96 * [The "next" pointer is always aligned on an even
a687059c
LW
97 * boundary, and reads the offset directly as a short. Also, there is no
98 * special test to reverse the sign of BACK pointers since the offset is
99 * stored negative.]
100 */
785a26d5
YO
101
102/* This is the stuff that used to live in regexp.h that was truly
103 private to the engine itself. It now lives here. */
104
28d8d7f4 105
785a26d5
YO
106
107 typedef struct regexp_internal {
1f1031fe 108 int name_list_idx; /* Optional data index of an array of paren names */
7122b237
YO
109 union {
110 U32 *offsets; /* offset annotations 20001228 MJD
111 data about mapping the program to the
112 string -
113 offsets[0] is proglen when this is used
114 */
115 U32 proglen;
116 } u;
117
6d5c990f
RGS
118 regnode *regstclass; /* Optional startclass as identified or constructed
119 by the optimiser */
120 struct reg_data *data; /* Additional miscellaneous data used by the program.
121 Used to make it easier to clone and free arbitrary
122 data that the regops need. Often the ARG field of
123 a regop is an index into this structure */
124 regnode program[1]; /* Unwarranted chumminess with compiler. */
125} regexp_internal;
126
127#define RXi_SET(x,y) (x)->pprivate = (void*)(y)
128#define RXi_GET(x) ((regexp_internal *)((x)->pprivate))
129#define RXi_GET_DECL(r,ri) regexp_internal *ri = RXi_GET(r)
785a26d5
YO
130/*
131 * Flags stored in regexp->intflags
132 * These are used only internally to the regexp engine
133 *
134 * See regexp.h for flags used externally to the regexp engine
135 */
136#define PREGf_SKIP 0x00000001
137#define PREGf_IMPLICIT 0x00000002 /* Converted .* to ^.* */
138#define PREGf_NAUGHTY 0x00000004 /* how exponential is this pattern? */
139#define PREGf_VERBARG_SEEN 0x00000008
140#define PREGf_CUTGROUP_SEEN 0x00000010
141
142
143/* this is where the old regcomp.h started */
a687059c 144
c277df42 145struct regnode_string {
cd439c50 146 U8 str_len;
c277df42
IZ
147 U8 type;
148 U16 next_off;
cd439c50 149 char string[1];
c277df42 150};
a687059c 151
6bda09f9
YO
152/* Argument bearing node - workhorse,
153 arg1 is often for the data field */
c277df42
IZ
154struct regnode_1 {
155 U8 flags;
156 U8 type;
157 U16 next_off;
158 U32 arg1;
159};
160
6bda09f9
YO
161/* Similar to a regnode_1 but with an extra signed argument */
162struct regnode_2L {
163 U8 flags;
164 U8 type;
165 U16 next_off;
166 U32 arg1;
167 I32 arg2;
168};
169
170/* 'Two field' -- Two 16 bit unsigned args */
c277df42
IZ
171struct regnode_2 {
172 U8 flags;
173 U8 type;
174 U16 next_off;
175 U16 arg1;
176 U16 arg2;
177};
178
6bda09f9 179
936ed897 180#define ANYOF_BITMAP_SIZE 32 /* 256 b/(8 b/B) */
d1eb3177 181#define ANYOF_CLASSBITMAP_SIZE 4 /* up to 32 (8*4) named classes */
936ed897 182
786e8c11 183/* also used by trie */
936ed897
IZ
184struct regnode_charclass {
185 U8 flags;
186 U8 type;
187 U16 next_off;
bf3f2d85 188 U32 arg1; /* used as ptr in S_regclass */
19103b91 189 char bitmap[ANYOF_BITMAP_SIZE]; /* only compile-time */
936ed897
IZ
190};
191
bf3f2d85
KW
192/* has runtime (locale) \d, \w, ..., [:posix:] classes */
193struct regnode_charclass_class {
194 U8 flags; /* ANYOF_CLASS bit must go here */
936ed897
IZ
195 U8 type;
196 U16 next_off;
bf3f2d85 197 U32 arg1; /* used as ptr in S_regclass */
19103b91
JH
198 char bitmap[ANYOF_BITMAP_SIZE]; /* both compile-time */
199 char classflags[ANYOF_CLASSBITMAP_SIZE]; /* and run-time */
936ed897
IZ
200};
201
9c7e81e8
AD
202/* XXX fix this description.
203 Impose a limit of REG_INFTY on various pattern matching operations
204 to limit stack growth and to avoid "infinite" recursions.
205*/
206/* The default size for REG_INFTY is I16_MAX, which is the same as
207 SHORT_MAX (see perl.h). Unfortunately I16 isn't necessarily 16 bits
208 (see handy.h). On the Cray C90, sizeof(short)==4 and hence I16_MAX is
209 ((1<<31)-1), while on the Cray T90, sizeof(short)==8 and I16_MAX is
210 ((1<<63)-1). To limit stack growth to reasonable sizes, supply a
211 smaller default.
212 --Andy Dougherty 11 June 1998
213*/
214#if SHORTSIZE > 2
215# ifndef REG_INFTY
216# define REG_INFTY ((1<<15)-1)
217# endif
218#endif
219
220#ifndef REG_INFTY
83cfe48c
IZ
221# define REG_INFTY I16_MAX
222#endif
a687059c 223
e91177ed
IZ
224#define ARG_VALUE(arg) (arg)
225#define ARG__SET(arg,val) ((arg) = (val))
c277df42 226
e8e3b0fb
JH
227#undef ARG
228#undef ARG1
229#undef ARG2
230
c277df42
IZ
231#define ARG(p) ARG_VALUE(ARG_LOC(p))
232#define ARG1(p) ARG_VALUE(ARG1_LOC(p))
233#define ARG2(p) ARG_VALUE(ARG2_LOC(p))
6bda09f9 234#define ARG2L(p) ARG_VALUE(ARG2L_LOC(p))
786e8c11 235
c277df42
IZ
236#define ARG_SET(p, val) ARG__SET(ARG_LOC(p), (val))
237#define ARG1_SET(p, val) ARG__SET(ARG1_LOC(p), (val))
238#define ARG2_SET(p, val) ARG__SET(ARG2_LOC(p), (val))
6bda09f9 239#define ARG2L_SET(p, val) ARG__SET(ARG2L_LOC(p), (val))
c277df42 240
e8e3b0fb
JH
241#undef NEXT_OFF
242#undef NODE_ALIGN
243
075abff3
AL
244#define NEXT_OFF(p) ((p)->next_off)
245#define NODE_ALIGN(node)
246#define NODE_ALIGN_FILL(node) ((node)->flags = 0xde) /* deadbeef */
a687059c 247
c277df42
IZ
248#define SIZE_ALIGN NODE_ALIGN
249
e8e3b0fb
JH
250#undef OP
251#undef OPERAND
252#undef MASK
253#undef STRING
254
e91177ed 255#define OP(p) ((p)->type)
b988e673
KW
256#define FLAGS(p) ((p)->flags) /* Caution: Doesn't apply to all \
257 regnode types. For some, it's the \
258 character set of the regnode */
e91177ed 259#define OPERAND(p) (((struct regnode_string *)p)->string)
cd439c50
IZ
260#define MASK(p) ((char*)OPERAND(p))
261#define STR_LEN(p) (((struct regnode_string *)p)->str_len)
262#define STRING(p) (((struct regnode_string *)p)->string)
263#define STR_SZ(l) ((l + sizeof(regnode) - 1) / sizeof(regnode))
264#define NODE_SZ_STR(p) (STR_SZ(STR_LEN(p))+1)
265
e8e3b0fb
JH
266#undef NODE_ALIGN
267#undef ARG_LOC
268#undef NEXTOPER
269#undef PREVOPER
270
e91177ed
IZ
271#define NODE_ALIGN(node)
272#define ARG_LOC(p) (((struct regnode_1 *)p)->arg1)
273#define ARG1_LOC(p) (((struct regnode_2 *)p)->arg1)
274#define ARG2_LOC(p) (((struct regnode_2 *)p)->arg2)
6bda09f9 275#define ARG2L_LOC(p) (((struct regnode_2L *)p)->arg2)
786e8c11 276
e91177ed
IZ
277#define NODE_STEP_REGNODE 1 /* sizeof(regnode)/sizeof(regnode) */
278#define EXTRA_STEP_2ARGS EXTRA_SIZE(struct regnode_2)
279
280#define NODE_STEP_B 4
c277df42
IZ
281
282#define NEXTOPER(p) ((p) + NODE_STEP_REGNODE)
283#define PREVOPER(p) ((p) - NODE_STEP_REGNODE)
284
e91177ed 285#define FILL_ADVANCE_NODE(ptr, op) STMT_START { \
c277df42 286 (ptr)->type = op; (ptr)->next_off = 0; (ptr)++; } STMT_END
e91177ed 287#define FILL_ADVANCE_NODE_ARG(ptr, op, arg) STMT_START { \
c277df42 288 ARG_SET(ptr, arg); FILL_ADVANCE_NODE(ptr, op); (ptr) += 1; } STMT_END
a687059c 289
22c35a8c 290#define REG_MAGIC 0234
a687059c 291
830247a4 292#define SIZE_ONLY (RExC_emit == &PL_regdummy)
c277df42 293
137165a6
KW
294/* If the bitmap doesn't fully represent what this ANYOF node can match, the
295 * ARG is set to this special value (since 0, 1, ... are legal, but will never
296 * reach this high). */
297#define ANYOF_NONBITMAP_EMPTY ((U32) -1)
298
299/* The information used to be stored as as combination of the ANYOF_UTF8 and
300 * ANYOF_NONBITMAP_NON_UTF8 bits in the flags field, but was moved out of there
301 * to free up a bit for other uses. This tries to hide the change from
302 * existing code as much as possible. Now, the data structure that goes in ARG
303 * is not allocated unless it is needed, and that is what is used to determine
304 * if there is something outside the bitmap. The code now assumes that if
305 * that structure exists, that any UTF-8 encoded string should be tried against
306 * it, but a non-UTF8-encoded string will be tried only if the
307 * ANYOF_NONBITMAP_NON_UTF8 bit is also set. */
308#define ANYOF_NONBITMAP(node) (ARG(node) != ANYOF_NONBITMAP_EMPTY)
309
d5788240 310/* Flags for node->flags of ANYOF. These are in short supply, so some games
f4244008
KW
311 * are done to share them, as described below. If necessary, the ANYOF_LOCALE
312 * and ANYOF_CLASS bits could be shared with a space penalty for locale nodes
313 * (and the code at the time this comment was written, is written so that all
314 * that is necessary to make the change would be to redefine the ANYOF_CLASS
315 * define). Once the planned change to compile all the above-latin1 code points
316 * is done, then the UNICODE_ALL bit can be freed up. If flags need to be
317 * added that are applicable to the synthetic start class only, with some work,
318 * they could be put in the next-node field, or in an unused bit of the
319 * classflags field. */
b8c5462f 320
39065660
KW
321#define ANYOF_LOCALE 0x01
322
323/* The fold is calculated and stored in the bitmap where possible at compile
324 * time. However there are two cases where it isn't possible. These share
325 * this bit: 1) under locale, where the actual folding varies depending on
326 * what the locale is at the time of execution; and 2) where the folding is
327 * specified in a swash, not the bitmap, such as characters which aren't
328 * specified in the bitmap, or properties that aren't looked at at compile time
329 */
330#define ANYOF_LOC_NONBITMAP_FOLD 0x02
331
332#define ANYOF_INVERT 0x04
1984f6a0 333
c8d3cd88
KW
334/* CLASS is never set unless LOCALE is too: has runtime \d, \w, [:posix:], ...
335 * The non-locale ones are resolved at compile-time */
336#define ANYOF_CLASS 0x08
337#define ANYOF_LARGE ANYOF_CLASS /* Same; name retained for back compat */
338
f4244008 339/* EOS, meaning that it can match an empty string too, is used for the
c613755a 340 * synthetic start class only. */
84193928 341#define ANYOF_EOS 0x10
f4244008 342
c613755a
KW
343/* ? Is this node the synthetic start class (ssc). This bit is shared with
344 * ANYOF_EOS, as the latter is used only for the ssc, and then not used by
345 * regexec.c. And, the code is structured so that if it is set, the ssc is
346 * not used, so it is guaranteed to be 0 for the ssc by the time regexec.c
347 * gets executed, and 0 for a non-ssc ANYOF node, as it only ever gets set for
348 * a potential ssc candidate. Thus setting it to 1 after it has been
349 * determined that the ssc will be used is not ambiguous */
350#define ANYOF_IS_SYNTHETIC ANYOF_EOS
351
ef87b810
KW
352/* Can match something outside the bitmap that isn't in utf8 */
353#define ANYOF_NONBITMAP_NON_UTF8 0x20
354
61ef9872
KW
355/* Matches every code point 0x100 and above*/
356#define ANYOF_UNICODE_ALL 0x40
ffc61ed2 357
11454c59
KW
358/* Match all Latin1 characters that aren't ASCII when the target string is not
359 * in utf8. */
360#define ANYOF_NON_UTF8_LATIN1_ALL 0x80
361
ffc61ed2 362#define ANYOF_FLAGS_ALL 0xff
653099ff 363
ace6b0e4
KW
364/* These are the flags that ANYOF_INVERT being set or not doesn't affect
365 * whether they are operative or not. e.g., the node still has LOCALE
366 * regardless of being inverted; whereas ANYOF_UNICODE_ALL means something
367 * different if inverted */
368#define INVERSION_UNAFFECTED_FLAGS (ANYOF_LOCALE \
369 |ANYOF_LOC_NONBITMAP_FOLD \
370 |ANYOF_CLASS \
371 |ANYOF_EOS \
372 |ANYOF_NONBITMAP_NON_UTF8)
373
936ed897 374/* Character classes for node->classflags of ANYOF */
653099ff
GS
375/* Should be synchronized with a table in regprop() */
376/* 2n should pair with 2n+1 */
b8c5462f 377
289d4f09 378#define ANYOF_ALNUM 0 /* \w, PL_utf8_alnum, utf8::IsWord, ALNUM */
b8c5462f 379#define ANYOF_NALNUM 1
aaa51d5e 380#define ANYOF_SPACE 2 /* \s */
b8c5462f 381#define ANYOF_NSPACE 3
24d786f4 382#define ANYOF_DIGIT 4 /* \d */
b8c5462f 383#define ANYOF_NDIGIT 5
24d786f4 384#define ANYOF_ALNUMC 6 /* [[:alnum:]] isalnum(3), utf8::IsAlnum, ALNUMC */
b8c5462f
JH
385#define ANYOF_NALNUMC 7
386#define ANYOF_ALPHA 8
387#define ANYOF_NALPHA 9
388#define ANYOF_ASCII 10
389#define ANYOF_NASCII 11
390#define ANYOF_CNTRL 12
391#define ANYOF_NCNTRL 13
392#define ANYOF_GRAPH 14
393#define ANYOF_NGRAPH 15
394#define ANYOF_LOWER 16
395#define ANYOF_NLOWER 17
396#define ANYOF_PRINT 18
397#define ANYOF_NPRINT 19
398#define ANYOF_PUNCT 20
399#define ANYOF_NPUNCT 21
400#define ANYOF_UPPER 22
401#define ANYOF_NUPPER 23
402#define ANYOF_XDIGIT 24
403#define ANYOF_NXDIGIT 25
aaa51d5e
JF
404#define ANYOF_PSXSPC 26 /* POSIX space: \s plus the vertical tab */
405#define ANYOF_NPSXSPC 27
ffc61ed2 406#define ANYOF_BLANK 28 /* GNU extension: space and tab: non-vertical space */
289d4f09 407#define ANYOF_NBLANK 29
b8c5462f 408
aaa51d5e 409#define ANYOF_MAX 32
b8c5462f 410
e1d1eefb
YO
411/* pseudo classes, not stored in the class bitmap, but used as flags
412 during compilation of char classes */
413
414#define ANYOF_VERTWS (ANYOF_MAX+1)
415#define ANYOF_NVERTWS (ANYOF_MAX+2)
416#define ANYOF_HORIZWS (ANYOF_MAX+3)
417#define ANYOF_NHORIZWS (ANYOF_MAX+4)
418
b8c5462f
JH
419/* Backward source code compatibility. */
420
421#define ANYOF_ALNUML ANYOF_ALNUM
422#define ANYOF_NALNUML ANYOF_NALNUM
423#define ANYOF_SPACEL ANYOF_SPACE
424#define ANYOF_NSPACEL ANYOF_NSPACE
425
426/* Utility macros for the bitmap and classes of ANYOF */
427
936ed897
IZ
428#define ANYOF_SIZE (sizeof(struct regnode_charclass))
429#define ANYOF_CLASS_SIZE (sizeof(struct regnode_charclass_class))
b8c5462f 430
936ed897 431#define ANYOF_FLAGS(p) ((p)->flags)
b8c5462f
JH
432
433#define ANYOF_BIT(c) (1 << ((c) & 7))
434
936ed897 435#define ANYOF_CLASS_BYTE(p, c) (((struct regnode_charclass_class*)(p))->classflags[((c) >> 3) & 3])
b8c5462f
JH
436#define ANYOF_CLASS_SET(p, c) (ANYOF_CLASS_BYTE(p, c) |= ANYOF_BIT(c))
437#define ANYOF_CLASS_CLEAR(p, c) (ANYOF_CLASS_BYTE(p, c) &= ~ANYOF_BIT(c))
438#define ANYOF_CLASS_TEST(p, c) (ANYOF_CLASS_BYTE(p, c) & ANYOF_BIT(c))
439
936ed897
IZ
440#define ANYOF_CLASS_ZERO(ret) Zero(((struct regnode_charclass_class*)(ret))->classflags, ANYOF_CLASSBITMAP_SIZE, char)
441#define ANYOF_BITMAP_ZERO(ret) Zero(((struct regnode_charclass*)(ret))->bitmap, ANYOF_BITMAP_SIZE, char)
442
443#define ANYOF_BITMAP(p) (((struct regnode_charclass*)(p))->bitmap)
786e8c11 444#define ANYOF_BITMAP_BYTE(p, c) (ANYOF_BITMAP(p)[(((U8)(c)) >> 3) & 31])
b8c5462f
JH
445#define ANYOF_BITMAP_SET(p, c) (ANYOF_BITMAP_BYTE(p, c) |= ANYOF_BIT(c))
446#define ANYOF_BITMAP_CLEAR(p,c) (ANYOF_BITMAP_BYTE(p, c) &= ~ANYOF_BIT(c))
447#define ANYOF_BITMAP_TEST(p, c) (ANYOF_BITMAP_BYTE(p, c) & ANYOF_BIT(c))
448
f8bef550
NC
449#define ANYOF_BITMAP_SETALL(p) \
450 memset (ANYOF_BITMAP(p), 255, ANYOF_BITMAP_SIZE)
451#define ANYOF_BITMAP_CLEARALL(p) \
452 Zero (ANYOF_BITMAP(p), ANYOF_BITMAP_SIZE)
453/* Check that all 256 bits are all set. Used in S_cl_is_anything() */
c355c09a 454#define ANYOF_BITMAP_TESTALLSET(p) /* Assumes sizeof(p) == 32 */ \
f8bef550
NC
455 memEQ (ANYOF_BITMAP(p), "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377", ANYOF_BITMAP_SIZE)
456
936ed897
IZ
457#define ANYOF_SKIP ((ANYOF_SIZE - 1)/sizeof(regnode))
458#define ANYOF_CLASS_SKIP ((ANYOF_CLASS_SIZE - 1)/sizeof(regnode))
3a15e693
KW
459
460/* The class bit can be set to the locale one if necessary to save bits at the
461 * expense of having locale ANYOF nodes always have a class bit map, and hence
462 * take up extra space. This allows convenient changing it as development
463 * proceeds on this */
464#if ANYOF_CLASS == ANYOF_LOCALE
465# undef ANYOF_CLASS_ADD_SKIP
466# define ANYOF_ADD_LOC_SKIP (ANYOF_CLASS_SKIP - ANYOF_SKIP)
467
468 /* Quicker way to see if there are actually any tests. This is because
469 * currently the set of tests can be empty even when the class bitmap is
470 * allocated */
471# if ANYOF_CLASSBITMAP_SIZE != 4
472# error ANYOF_CLASSBITMAP_SIZE is expected to be 4
473# endif
474# define ANYOF_CLASS_TEST_ANY_SET(p) /* assumes sizeof(p) = 4 */ \
475 memNE (((struct regnode_charclass_class*)(p))->classflags, \
476 "\0\0\0\0", ANYOF_CLASSBITMAP_SIZE)
477#else
478# define ANYOF_CLASS_ADD_SKIP (ANYOF_CLASS_SKIP - ANYOF_SKIP)
479# undef ANYOF_ADD_LOC_SKIP
480# define ANYOF_CLASS_TEST_ANY_SET(p) (ANYOF_FLAGS(p) & ANYOF_CLASS)
481#endif
c277df42 482
786e8c11 483
a687059c
LW
484/*
485 * Utility definitions.
486 */
2304df62 487#ifndef CHARMASK
075abff3 488# define UCHARAT(p) ((int)*(const U8*)(p))
a687059c 489#else
075abff3 490# define UCHARAT(p) ((int)*(p)&CHARMASK)
a687059c 491#endif
a687059c 492
c277df42
IZ
493#define EXTRA_SIZE(guy) ((sizeof(guy)-1)/sizeof(struct regnode))
494
3dab1dad
YO
495#define REG_SEEN_ZERO_LEN 0x00000001
496#define REG_SEEN_LOOKBEHIND 0x00000002
497#define REG_SEEN_GPOS 0x00000004
498#define REG_SEEN_EVAL 0x00000008
499#define REG_SEEN_CANY 0x00000010
cce850e4 500#define REG_SEEN_SANY REG_SEEN_CANY /* src bckwrd cmpt */
6bda09f9 501#define REG_SEEN_RECURSE 0x00000020
eaf3ca90 502#define REG_TOP_LEVEL_BRANCHES 0x00000040
e2e6a0f1 503#define REG_SEEN_VERBARG 0x00000080
5d458dd8 504#define REG_SEEN_CUTGROUP 0x00000100
bcdf7404 505#define REG_SEEN_RUN_ON_COMMENT 0x00000200
d09b2d29 506
73c4f7a1
GS
507START_EXTERN_C
508
be8e71aa
YO
509#ifdef PLUGGABLE_RE_EXTENSION
510#include "re_nodes.h"
511#else
d09b2d29 512#include "regnodes.h"
be8e71aa 513#endif
d09b2d29 514
f9f4320a
YO
515#ifndef PLUGGABLE_RE_EXTENSION
516#ifndef DOINIT
517EXTCONST regexp_engine PL_core_reg_engine;
518#else /* DOINIT */
519EXTCONST regexp_engine PL_core_reg_engine = {
fe578d7f 520 Perl_re_compile,
2fdbfb4d 521 Perl_regexec_flags,
f9f4320a
YO
522 Perl_re_intuit_start,
523 Perl_re_intuit_string,
2fdbfb4d
AB
524 Perl_regfree_internal,
525 Perl_reg_numbered_buff_fetch,
526 Perl_reg_numbered_buff_store,
527 Perl_reg_numbered_buff_length,
192b9cd1
AB
528 Perl_reg_named_buff,
529 Perl_reg_named_buff_iter,
49d7dfbc 530 Perl_reg_qr_package,
f9f4320a 531#if defined(USE_ITHREADS)
f8149455 532 Perl_regdupe_internal
f9f4320a
YO
533#endif
534};
535#endif /* DOINIT */
536#endif /* PLUGGABLE_RE_EXTENSION */
537
538
73c4f7a1 539END_EXTERN_C
cad2e5aa 540
cad2e5aa 541
261faec3
MJD
542/* .what is a character array with one character for each member of .data
543 * The character describes the function of the corresponding .data item:
af534a04 544 * a - AV for paren_name_list under DEBUGGING
0111c4fd 545 * f - start-class data for regstclass optimization
261faec3
MJD
546 * n - Root of op tree for (?{EVAL}) item
547 * o - Start op for (?{EVAL}) item
3dab1dad 548 * p - Pad for (?{EVAL}) item
38a44b82 549 * s - swash for Unicode-style character class, and the multicharacter
9e55ce06
JH
550 * strings resulting from casefolding the single-character entries
551 * in the character class
a3621e74 552 * t - trie struct
55eed653 553 * u - trie struct's widecharmap (a HV, so can't share, must dup)
2b8b4781 554 * also used for revcharmap and words under DEBUGGING
07be1b83 555 * T - aho-trie struct
81714fb9 556 * S - sv for named capture lookup
261faec3 557 * 20010712 mjd@plover.com
c3613ed7 558 * (Remember to update re_dup() and pregfree() if you add any items.)
261faec3 559 */
cad2e5aa
JH
560struct reg_data {
561 U32 count;
562 U8 *what;
563 void* data[1];
564};
565
a1cac82e
NC
566/* Code in S_to_utf8_substr() and S_to_byte_substr() in regexec.c accesses
567 anchored* and float* via array indexes 0 and 1. */
cad2e5aa 568#define anchored_substr substrs->data[0].substr
33b8afdf 569#define anchored_utf8 substrs->data[0].utf8_substr
cad2e5aa 570#define anchored_offset substrs->data[0].min_offset
1de06328
YO
571#define anchored_end_shift substrs->data[0].end_shift
572
cad2e5aa 573#define float_substr substrs->data[1].substr
33b8afdf 574#define float_utf8 substrs->data[1].utf8_substr
cad2e5aa
JH
575#define float_min_offset substrs->data[1].min_offset
576#define float_max_offset substrs->data[1].max_offset
1de06328
YO
577#define float_end_shift substrs->data[1].end_shift
578
cad2e5aa 579#define check_substr substrs->data[2].substr
33b8afdf 580#define check_utf8 substrs->data[2].utf8_substr
cad2e5aa
JH
581#define check_offset_min substrs->data[2].min_offset
582#define check_offset_max substrs->data[2].max_offset
1de06328 583#define check_end_shift substrs->data[2].end_shift
a3621e74 584
288b8c02
NC
585#define RX_ANCHORED_SUBSTR(rx) (((struct regexp *)SvANY(rx))->anchored_substr)
586#define RX_ANCHORED_UTF8(rx) (((struct regexp *)SvANY(rx))->anchored_utf8)
587#define RX_FLOAT_SUBSTR(rx) (((struct regexp *)SvANY(rx))->float_substr)
588#define RX_FLOAT_UTF8(rx) (((struct regexp *)SvANY(rx))->float_utf8)
a3621e74
YO
589
590/* trie related stuff */
5d9a96ca 591
a3621e74
YO
592/* a transition record for the state machine. the
593 check field determines which state "owns" the
594 transition. the char the transition is for is
595 determined by offset from the owning states base
596 field. the next field determines which state
597 is to be transitioned to if any.
598*/
599struct _reg_trie_trans {
600 U32 next;
601 U32 check;
602};
603
604/* a transition list element for the list based representation */
605struct _reg_trie_trans_list_elem {
606 U16 forid;
607 U32 newstate;
608};
609typedef struct _reg_trie_trans_list_elem reg_trie_trans_le;
610
611/* a state for compressed nodes. base is an offset
612 into an array of reg_trie_trans array. If wordnum is
613 nonzero the state is accepting. if base is zero then
614 the state has no children (and will be accepting)
615*/
616struct _reg_trie_state {
617 U16 wordnum;
618 union {
619 U32 base;
620 reg_trie_trans_le* list;
621 } trans;
622};
623
2e64971a
DM
624/* info per word; indexed by wordnum */
625typedef struct {
626 U16 prev; /* previous word in acceptance chain; eg in
627 * zzz|abc|ab/ after matching the chars abc, the
628 * accepted word is #2, and the previous accepted
629 * word is #3 */
630 U32 len; /* how many chars long is this word? */
631 U32 accept; /* accept state for this word */
632} reg_trie_wordinfo;
a3621e74
YO
633
634
a3621e74
YO
635typedef struct _reg_trie_state reg_trie_state;
636typedef struct _reg_trie_trans reg_trie_trans;
637
638
639/* anything in here that needs to be freed later
23eab42c
NC
640 should be dealt with in pregfree.
641 refcount is first in both this and _reg_ac_data to allow a space
642 optimisation in Perl_regdupe. */
a3621e74 643struct _reg_trie_data {
23eab42c 644 U32 refcount; /* number of times this trie is referenced */
786e8c11
YO
645 U32 lasttrans; /* last valid transition element */
646 U16 *charmap; /* byte to charid lookup array */
786e8c11
YO
647 reg_trie_state *states; /* state data */
648 reg_trie_trans *trans; /* array of transition elements */
649 char *bitmap; /* stclass bitmap */
786e8c11
YO
650 U16 *jump; /* optional 1 indexed array of offsets before tail
651 for the node following a given word. */
2e64971a 652 reg_trie_wordinfo *wordinfo; /* array of info per word */
1b5c067c
NC
653 U16 uniquecharcount; /* unique chars in trie (width of trans table) */
654 U32 startstate; /* initial state - used for common prefix optimisation */
655 STRLEN minlen; /* minimum length of words in trie - build/opt only? */
656 STRLEN maxlen; /* maximum length of words in trie - build/opt only? */
2e64971a 657 U32 prefixlen; /* #chars in common prefix */
1e2e3d02
YO
658 U32 statecount; /* Build only - number of states in the states array
659 (including the unused zero state) */
786e8c11 660 U32 wordcount; /* Build only */
a3621e74 661#ifdef DEBUGGING
786e8c11 662 STRLEN charcount; /* Build only */
a3621e74
YO
663#endif
664};
2b8b4781
NC
665/* There is one (3 under DEBUGGING) pointers that logically belong in this
666 structure, but are held outside as they need duplication on thread cloning,
667 whereas the rest of the structure can be read only:
668 HV *widecharmap; code points > 255 to charid
669#ifdef DEBUGGING
670 AV *words; Array of words contained in trie, for dumping
671 AV *revcharmap; Map of each charid back to its character representation
672#endif
673*/
674
675#define TRIE_WORDS_OFFSET 2
676
a3621e74
YO
677typedef struct _reg_trie_data reg_trie_data;
678
23eab42c
NC
679/* refcount is first in both this and _reg_trie_data to allow a space
680 optimisation in Perl_regdupe. */
07be1b83 681struct _reg_ac_data {
23eab42c 682 U32 refcount;
1b5c067c 683 U32 trie;
07be1b83
YO
684 U32 *fail;
685 reg_trie_state *states;
07be1b83
YO
686};
687typedef struct _reg_ac_data reg_ac_data;
688
486ec47a 689/* ANY_BIT doesn't use the structure, so we can borrow it here.
3dab1dad
YO
690 This is simpler than refactoring all of it as wed end up with
691 three different sets... */
692
693#define TRIE_BITMAP(p) (((reg_trie_data *)(p))->bitmap)
786e8c11 694#define TRIE_BITMAP_BYTE(p, c) (TRIE_BITMAP(p)[(((U8)(c)) >> 3) & 31])
5dfe062f
YO
695#define TRIE_BITMAP_SET(p, c) (TRIE_BITMAP_BYTE(p, c) |= ANYOF_BIT((U8)c))
696#define TRIE_BITMAP_CLEAR(p,c) (TRIE_BITMAP_BYTE(p, c) &= ~ANYOF_BIT((U8)c))
697#define TRIE_BITMAP_TEST(p, c) (TRIE_BITMAP_BYTE(p, c) & ANYOF_BIT((U8)c))
3dab1dad 698
1de06328
YO
699#define IS_ANYOF_TRIE(op) ((op)==TRIEC || (op)==AHOCORASICKC)
700#define IS_TRIE_AC(op) ((op)>=AHOCORASICK)
701
702
786e8c11
YO
703#define BITMAP_BYTE(p, c) (((U8*)p)[(((U8)(c)) >> 3) & 31])
704#define BITMAP_TEST(p, c) (BITMAP_BYTE(p, c) & ANYOF_BIT((U8)c))
3dab1dad 705
a3621e74
YO
706/* these defines assume uniquecharcount is the correct variable, and state may be evaluated twice */
707#define TRIE_NODENUM(state) (((state)-1)/(trie->uniquecharcount)+1)
708#define SAFE_TRIE_NODENUM(state) ((state) ? (((state)-1)/(trie->uniquecharcount)+1) : (state))
709#define TRIE_NODEIDX(state) ((state) ? (((state)-1)*(trie->uniquecharcount)+1) : (state))
710
3dab1dad 711#ifdef DEBUGGING
3dab1dad 712#define TRIE_CHARCOUNT(trie) ((trie)->charcount)
3dab1dad 713#else
3dab1dad 714#define TRIE_CHARCOUNT(trie) (trie_charcount)
3dab1dad 715#endif
a3621e74 716
0111c4fd
RGS
717#define RE_TRIE_MAXBUF_INIT 65536
718#define RE_TRIE_MAXBUF_NAME "\022E_TRIE_MAXBUF"
a3621e74
YO
719#define RE_DEBUG_FLAGS "\022E_DEBUG_FLAGS"
720
be8e71aa
YO
721/*
722
723RE_DEBUG_FLAGS is used to control what debug output is emitted
724its divided into three groups of options, some of which interact.
725The three groups are: Compile, Execute, Extra. There is room for a
726further group, as currently only the low three bytes are used.
727
728 Compile Options:
07be1b83 729
be8e71aa
YO
730 PARSE
731 PEEP
732 TRIE
733 PROGRAM
734 OFFSETS
a3621e74 735
be8e71aa 736 Execute Options:
a3621e74 737
be8e71aa
YO
738 INTUIT
739 MATCH
740 TRIE
a3621e74 741
be8e71aa 742 Extra Options
a3621e74 743
be8e71aa
YO
744 TRIE
745 OFFSETS
746
747If you modify any of these make sure you make corresponding changes to
748re.pm, especially to the documentation.
a3621e74 749
be8e71aa
YO
750*/
751
752
753/* Compile */
754#define RE_DEBUG_COMPILE_MASK 0x0000FF
755#define RE_DEBUG_COMPILE_PARSE 0x000001
756#define RE_DEBUG_COMPILE_OPTIMISE 0x000002
757#define RE_DEBUG_COMPILE_TRIE 0x000004
758#define RE_DEBUG_COMPILE_DUMP 0x000008
f7819f85 759#define RE_DEBUG_COMPILE_FLAGS 0x000010
be8e71aa
YO
760
761/* Execute */
762#define RE_DEBUG_EXECUTE_MASK 0x00FF00
763#define RE_DEBUG_EXECUTE_INTUIT 0x000100
764#define RE_DEBUG_EXECUTE_MATCH 0x000200
765#define RE_DEBUG_EXECUTE_TRIE 0x000400
766
767/* Extra */
768#define RE_DEBUG_EXTRA_MASK 0xFF0000
769#define RE_DEBUG_EXTRA_TRIE 0x010000
770#define RE_DEBUG_EXTRA_OFFSETS 0x020000
a5ca303d
YO
771#define RE_DEBUG_EXTRA_OFFDEBUG 0x040000
772#define RE_DEBUG_EXTRA_STATE 0x080000
773#define RE_DEBUG_EXTRA_OPTIMISE 0x100000
e7707071 774#define RE_DEBUG_EXTRA_BUFFERS 0x400000
2c296965 775#define RE_DEBUG_EXTRA_GPOS 0x800000
24b23f37
YO
776/* combined */
777#define RE_DEBUG_EXTRA_STACK 0x280000
be8e71aa 778
e68ec53f 779#define RE_DEBUG_FLAG(x) (re_debug_flags & x)
be8e71aa
YO
780/* Compile */
781#define DEBUG_COMPILE_r(x) DEBUG_r( \
e68ec53f 782 if (re_debug_flags & RE_DEBUG_COMPILE_MASK) x )
be8e71aa 783#define DEBUG_PARSE_r(x) DEBUG_r( \
e68ec53f 784 if (re_debug_flags & RE_DEBUG_COMPILE_PARSE) x )
be8e71aa 785#define DEBUG_OPTIMISE_r(x) DEBUG_r( \
e68ec53f 786 if (re_debug_flags & RE_DEBUG_COMPILE_OPTIMISE) x )
be8e71aa 787#define DEBUG_PARSE_r(x) DEBUG_r( \
e68ec53f 788 if (re_debug_flags & RE_DEBUG_COMPILE_PARSE) x )
be8e71aa 789#define DEBUG_DUMP_r(x) DEBUG_r( \
e68ec53f 790 if (re_debug_flags & RE_DEBUG_COMPILE_DUMP) x )
be8e71aa 791#define DEBUG_TRIE_COMPILE_r(x) DEBUG_r( \
e68ec53f 792 if (re_debug_flags & RE_DEBUG_COMPILE_TRIE) x )
f7819f85
A
793#define DEBUG_FLAGS_r(x) DEBUG_r( \
794 if (re_debug_flags & RE_DEBUG_COMPILE_FLAGS) x )
be8e71aa
YO
795/* Execute */
796#define DEBUG_EXECUTE_r(x) DEBUG_r( \
e68ec53f 797 if (re_debug_flags & RE_DEBUG_EXECUTE_MASK) x )
be8e71aa 798#define DEBUG_INTUIT_r(x) DEBUG_r( \
e68ec53f 799 if (re_debug_flags & RE_DEBUG_EXECUTE_INTUIT) x )
be8e71aa 800#define DEBUG_MATCH_r(x) DEBUG_r( \
e68ec53f 801 if (re_debug_flags & RE_DEBUG_EXECUTE_MATCH) x )
be8e71aa 802#define DEBUG_TRIE_EXECUTE_r(x) DEBUG_r( \
e68ec53f 803 if (re_debug_flags & RE_DEBUG_EXECUTE_TRIE) x )
be8e71aa
YO
804
805/* Extra */
806#define DEBUG_EXTRA_r(x) DEBUG_r( \
e68ec53f 807 if (re_debug_flags & RE_DEBUG_EXTRA_MASK) x )
a5ca303d
YO
808#define DEBUG_OFFSETS_r(x) DEBUG_r( \
809 if (re_debug_flags & RE_DEBUG_EXTRA_OFFSETS) x )
ab3bbdeb
YO
810#define DEBUG_STATE_r(x) DEBUG_r( \
811 if (re_debug_flags & RE_DEBUG_EXTRA_STATE) x )
24b23f37
YO
812#define DEBUG_STACK_r(x) DEBUG_r( \
813 if (re_debug_flags & RE_DEBUG_EXTRA_STACK) x )
e7707071
YO
814#define DEBUG_BUFFERS_r(x) DEBUG_r( \
815 if (re_debug_flags & RE_DEBUG_EXTRA_BUFFERS) x )
816
a5ca303d
YO
817#define DEBUG_OPTIMISE_MORE_r(x) DEBUG_r( \
818 if ((RE_DEBUG_EXTRA_OPTIMISE|RE_DEBUG_COMPILE_OPTIMISE) == \
819 (re_debug_flags & (RE_DEBUG_EXTRA_OPTIMISE|RE_DEBUG_COMPILE_OPTIMISE)) ) x )
07be1b83 820#define MJD_OFFSET_DEBUG(x) DEBUG_r( \
a5ca303d 821 if (re_debug_flags & RE_DEBUG_EXTRA_OFFDEBUG) \
be8e71aa
YO
822 Perl_warn_nocontext x )
823#define DEBUG_TRIE_COMPILE_MORE_r(x) DEBUG_TRIE_COMPILE_r( \
e68ec53f 824 if (re_debug_flags & RE_DEBUG_EXTRA_TRIE) x )
be8e71aa 825#define DEBUG_TRIE_EXECUTE_MORE_r(x) DEBUG_TRIE_EXECUTE_r( \
e68ec53f 826 if (re_debug_flags & RE_DEBUG_EXTRA_TRIE) x )
be8e71aa
YO
827
828#define DEBUG_TRIE_r(x) DEBUG_r( \
e68ec53f 829 if (re_debug_flags & (RE_DEBUG_COMPILE_TRIE \
be8e71aa 830 | RE_DEBUG_EXECUTE_TRIE )) x )
2c296965
YO
831#define DEBUG_GPOS_r(x) DEBUG_r( \
832 if (re_debug_flags & RE_DEBUG_EXTRA_GPOS) x )
be8e71aa
YO
833
834/* initialization */
595598ee
KW
835/* get_sv() can return NULL during global destruction. re_debug_flags can get
836 * clobbered by a longjmp, so must be initialized */
e68ec53f
YO
837#define GET_RE_DEBUG_FLAGS DEBUG_r({ \
838 SV * re_debug_flags_sv = NULL; \
595598ee 839 re_debug_flags = 0; \
e68ec53f
YO
840 re_debug_flags_sv = get_sv(RE_DEBUG_FLAGS, 1); \
841 if (re_debug_flags_sv) { \
8e9a8a48 842 if (!SvIOK(re_debug_flags_sv)) \
e68ec53f 843 sv_setuv(re_debug_flags_sv, RE_DEBUG_COMPILE_DUMP | RE_DEBUG_EXECUTE_MASK ); \
8e9a8a48 844 re_debug_flags=SvIV(re_debug_flags_sv); \
e68ec53f
YO
845 }\
846})
a3621e74
YO
847
848#ifdef DEBUGGING
ab3bbdeb 849
d53d27f9 850#define GET_RE_DEBUG_FLAGS_DECL VOL IV re_debug_flags = 0; GET_RE_DEBUG_FLAGS;
ab3bbdeb
YO
851
852#define RE_PV_COLOR_DECL(rpv,rlen,isuni,dsv,pv,l,m,c1,c2) \
853 const char * const rpv = \
ddc5bc0f 854 pv_pretty((dsv), (pv), (l), (m), \
ab3bbdeb 855 PL_colors[(c1)],PL_colors[(c2)], \
c89df6cf 856 PERL_PV_ESCAPE_RE|PERL_PV_ESCAPE_NONASCII |((isuni) ? PERL_PV_ESCAPE_UNI : 0) ); \
0df25f3d 857 const int rlen = SvCUR(dsv)
ab3bbdeb
YO
858
859#define RE_SV_ESCAPE(rpv,isuni,dsv,sv,m) \
860 const char * const rpv = \
ddc5bc0f 861 pv_pretty((dsv), (SvPV_nolen_const(sv)), (SvCUR(sv)), (m), \
ab3bbdeb 862 PL_colors[(c1)],PL_colors[(c2)], \
c89df6cf 863 PERL_PV_ESCAPE_RE|PERL_PV_ESCAPE_NONASCII |((isuni) ? PERL_PV_ESCAPE_UNI : 0) )
ab3bbdeb
YO
864
865#define RE_PV_QUOTED_DECL(rpv,isuni,dsv,pv,l,m) \
866 const char * const rpv = \
ddc5bc0f 867 pv_pretty((dsv), (pv), (l), (m), \
ab3bbdeb 868 PL_colors[0], PL_colors[1], \
c89df6cf 869 ( PERL_PV_PRETTY_QUOTE | PERL_PV_ESCAPE_RE | PERL_PV_ESCAPE_NONASCII | PERL_PV_PRETTY_ELLIPSES | \
ab3bbdeb 870 ((isuni) ? PERL_PV_ESCAPE_UNI : 0)) \
0a0e9afc 871 )
ab3bbdeb
YO
872
873#define RE_SV_DUMPLEN(ItEm) (SvCUR(ItEm) - (SvTAIL(ItEm)!=0))
874#define RE_SV_TAIL(ItEm) (SvTAIL(ItEm) ? "$" : "")
875
876#else /* if not DEBUGGING */
877
a3621e74 878#define GET_RE_DEBUG_FLAGS_DECL
ab3bbdeb
YO
879#define RE_PV_COLOR_DECL(rpv,rlen,isuni,dsv,pv,l,m,c1,c2)
880#define RE_SV_ESCAPE(rpv,isuni,dsv,sv,m)
881#define RE_PV_QUOTED_DECL(rpv,isuni,dsv,pv,l,m)
882#define RE_SV_DUMPLEN(ItEm)
883#define RE_SV_TAIL(ItEm)
884
885#endif /* DEBUG RELATED DEFINES */
a3621e74 886
e9a8c099
MHM
887/*
888 * Local variables:
889 * c-indentation-style: bsd
890 * c-basic-offset: 4
891 * indent-tabs-mode: t
892 * End:
893 *
894 * ex: set ts=8 sts=4 sw=4 noet:
895 */