This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Revert change 28962 (at least until a better solution is found)
[perl5.git] / regcomp.c
CommitLineData
a0d0e21e
LW
1/* regcomp.c
2 */
3
4/*
5 * "A fair jaw-cracker dwarf-language must be." --Samwise Gamgee
6 */
7
61296642
DM
8/* This file contains functions for compiling a regular expression. See
9 * also regexec.c which funnily enough, contains functions for executing
166f8a29 10 * a regular expression.
e4a054ea
DM
11 *
12 * This file is also copied at build time to ext/re/re_comp.c, where
13 * it's built with -DPERL_EXT_RE_BUILD -DPERL_EXT_RE_DEBUG -DPERL_EXT.
14 * This causes the main functions to be compiled under new names and with
15 * debugging support added, which makes "use re 'debug'" work.
166f8a29
DM
16 */
17
a687059c
LW
18/* NOTE: this is derived from Henry Spencer's regexp code, and should not
19 * confused with the original package (see point 3 below). Thanks, Henry!
20 */
21
22/* Additional note: this code is very heavily munged from Henry's version
23 * in places. In some spots I've traded clarity for efficiency, so don't
24 * blame Henry for some of the lack of readability.
25 */
26
e50aee73
AD
27/* The names of the functions have been changed from regcomp and
28 * regexec to pregcomp and pregexec in order to avoid conflicts
29 * with the POSIX routines of the same names.
30*/
31
b9d5759e 32#ifdef PERL_EXT_RE_BUILD
54df2634 33#include "re_top.h"
b81d288d 34#endif
56953603 35
a687059c 36/*
e50aee73 37 * pregcomp and pregexec -- regsub and regerror are not used in perl
a687059c
LW
38 *
39 * Copyright (c) 1986 by University of Toronto.
40 * Written by Henry Spencer. Not derived from licensed software.
41 *
42 * Permission is granted to anyone to use this software for any
43 * purpose on any computer system, and to redistribute it freely,
44 * subject to the following restrictions:
45 *
46 * 1. The author is not responsible for the consequences of use of
47 * this software, no matter how awful, even if they arise
48 * from defects in it.
49 *
50 * 2. The origin of this software must not be misrepresented, either
51 * by explicit claim or by omission.
52 *
53 * 3. Altered versions must be plainly marked as such, and must not
54 * be misrepresented as being the original software.
55 *
56 *
57 **** Alterations to Henry's code are...
58 ****
4bb101f2 59 **** Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
b94e2f88 60 **** 2000, 2001, 2002, 2003, 2004, 2005, 2006, by Larry Wall and others
a687059c 61 ****
9ef589d8
LW
62 **** You may distribute under the terms of either the GNU General Public
63 **** License or the Artistic License, as specified in the README file.
64
a687059c
LW
65 *
66 * Beware that some of this code is subtly aware of the way operator
67 * precedence is structured in regular expressions. Serious changes in
68 * regular-expression syntax might require a total rethink.
69 */
70#include "EXTERN.h"
864dbfa3 71#define PERL_IN_REGCOMP_C
a687059c 72#include "perl.h"
d06ea78c 73
acfe0abc 74#ifndef PERL_IN_XSUB_RE
d06ea78c
GS
75# include "INTERN.h"
76#endif
c277df42
IZ
77
78#define REG_COMP_C
54df2634
NC
79#ifdef PERL_IN_XSUB_RE
80# include "re_comp.h"
81#else
82# include "regcomp.h"
83#endif
a687059c 84
d4cce5f1 85#ifdef op
11343788 86#undef op
d4cce5f1 87#endif /* op */
11343788 88
fe14fcc3 89#ifdef MSDOS
7e4e8c89 90# if defined(BUGGY_MSC6)
fe14fcc3 91 /* MSC 6.00A breaks on op/regexp.t test 85 unless we turn this off */
7e4e8c89 92# pragma optimize("a",off)
fe14fcc3 93 /* But MSC 6.00A is happy with 'w', for aliases only across function calls*/
7e4e8c89
NC
94# pragma optimize("w",on )
95# endif /* BUGGY_MSC6 */
fe14fcc3
LW
96#endif /* MSDOS */
97
a687059c
LW
98#ifndef STATIC
99#define STATIC static
100#endif
101
830247a4 102typedef struct RExC_state_t {
e2509266 103 U32 flags; /* are we folding, multilining? */
830247a4
IZ
104 char *precomp; /* uncompiled string. */
105 regexp *rx;
fac92740 106 char *start; /* Start of input for compile */
830247a4
IZ
107 char *end; /* End of input for compile */
108 char *parse; /* Input-scan pointer. */
109 I32 whilem_seen; /* number of WHILEM in this expr */
fac92740 110 regnode *emit_start; /* Start of emitted-code area */
ffc61ed2 111 regnode *emit; /* Code-emit pointer; &regdummy = don't = compiling */
830247a4
IZ
112 I32 naughty; /* How bad is this pattern? */
113 I32 sawback; /* Did we see \1, ...? */
114 U32 seen;
115 I32 size; /* Code size. */
116 I32 npar; /* () count. */
117 I32 extralen;
118 I32 seen_zerolen;
119 I32 seen_evals;
6bda09f9 120 regnode **parens; /* offsets of each paren */
1aa99e6b 121 I32 utf8;
6bda09f9 122 HV *charnames; /* cache of named sequences */
81714fb9 123 HV *paren_names; /* Paren names */
830247a4
IZ
124#if ADD_TO_REGEXEC
125 char *starttry; /* -Dr: where regtry was called. */
126#define RExC_starttry (pRExC_state->starttry)
127#endif
3dab1dad 128#ifdef DEBUGGING
be8e71aa 129 const char *lastparse;
3dab1dad
YO
130 I32 lastnum;
131#define RExC_lastparse (pRExC_state->lastparse)
132#define RExC_lastnum (pRExC_state->lastnum)
133#endif
830247a4
IZ
134} RExC_state_t;
135
e2509266 136#define RExC_flags (pRExC_state->flags)
830247a4
IZ
137#define RExC_precomp (pRExC_state->precomp)
138#define RExC_rx (pRExC_state->rx)
fac92740 139#define RExC_start (pRExC_state->start)
830247a4
IZ
140#define RExC_end (pRExC_state->end)
141#define RExC_parse (pRExC_state->parse)
142#define RExC_whilem_seen (pRExC_state->whilem_seen)
fac92740 143#define RExC_offsets (pRExC_state->rx->offsets) /* I am not like the others */
830247a4 144#define RExC_emit (pRExC_state->emit)
fac92740 145#define RExC_emit_start (pRExC_state->emit_start)
830247a4
IZ
146#define RExC_naughty (pRExC_state->naughty)
147#define RExC_sawback (pRExC_state->sawback)
148#define RExC_seen (pRExC_state->seen)
149#define RExC_size (pRExC_state->size)
150#define RExC_npar (pRExC_state->npar)
151#define RExC_extralen (pRExC_state->extralen)
152#define RExC_seen_zerolen (pRExC_state->seen_zerolen)
153#define RExC_seen_evals (pRExC_state->seen_evals)
1aa99e6b 154#define RExC_utf8 (pRExC_state->utf8)
fc8cd66c 155#define RExC_charnames (pRExC_state->charnames)
6bda09f9 156#define RExC_parens (pRExC_state->parens)
81714fb9 157#define RExC_paren_names (pRExC_state->paren_names)
830247a4 158
a687059c
LW
159#define ISMULT1(c) ((c) == '*' || (c) == '+' || (c) == '?')
160#define ISMULT2(s) ((*s) == '*' || (*s) == '+' || (*s) == '?' || \
161 ((*s) == '{' && regcurly(s)))
a687059c 162
35c8bce7
LW
163#ifdef SPSTART
164#undef SPSTART /* dratted cpp namespace... */
165#endif
a687059c
LW
166/*
167 * Flags to be passed up and down.
168 */
a687059c 169#define WORST 0 /* Worst case. */
821b33a5 170#define HASWIDTH 0x1 /* Known to match non-null strings. */
a0d0e21e
LW
171#define SIMPLE 0x2 /* Simple enough to be STAR/PLUS operand. */
172#define SPSTART 0x4 /* Starts with * or +. */
173#define TRYAGAIN 0x8 /* Weeded out a declaration. */
a687059c 174
3dab1dad
YO
175#define REG_NODE_NUM(x) ((x) ? (int)((x)-RExC_emit_start) : -1)
176
07be1b83
YO
177/* whether trie related optimizations are enabled */
178#if PERL_ENABLE_EXTENDED_TRIE_OPTIMISATION
179#define TRIE_STUDY_OPT
786e8c11 180#define FULL_TRIE_STUDY
07be1b83
YO
181#define TRIE_STCLASS
182#endif
1de06328
YO
183
184
185/* About scan_data_t.
186
187 During optimisation we recurse through the regexp program performing
188 various inplace (keyhole style) optimisations. In addition study_chunk
189 and scan_commit populate this data structure with information about
190 what strings MUST appear in the pattern. We look for the longest
191 string that must appear for at a fixed location, and we look for the
192 longest string that may appear at a floating location. So for instance
193 in the pattern:
194
195 /FOO[xX]A.*B[xX]BAR/
196
197 Both 'FOO' and 'A' are fixed strings. Both 'B' and 'BAR' are floating
198 strings (because they follow a .* construct). study_chunk will identify
199 both FOO and BAR as being the longest fixed and floating strings respectively.
200
201 The strings can be composites, for instance
202
203 /(f)(o)(o)/
204
205 will result in a composite fixed substring 'foo'.
206
207 For each string some basic information is maintained:
208
209 - offset or min_offset
210 This is the position the string must appear at, or not before.
211 It also implicitly (when combined with minlenp) tells us how many
212 character must match before the string we are searching.
213 Likewise when combined with minlenp and the length of the string
214 tells us how many characters must appear after the string we have
215 found.
216
217 - max_offset
218 Only used for floating strings. This is the rightmost point that
219 the string can appear at. Ifset to I32 max it indicates that the
220 string can occur infinitely far to the right.
221
222 - minlenp
223 A pointer to the minimum length of the pattern that the string
224 was found inside. This is important as in the case of positive
225 lookahead or positive lookbehind we can have multiple patterns
226 involved. Consider
227
228 /(?=FOO).*F/
229
230 The minimum length of the pattern overall is 3, the minimum length
231 of the lookahead part is 3, but the minimum length of the part that
232 will actually match is 1. So 'FOO's minimum length is 3, but the
233 minimum length for the F is 1. This is important as the minimum length
234 is used to determine offsets in front of and behind the string being
235 looked for. Since strings can be composites this is the length of the
236 pattern at the time it was commited with a scan_commit. Note that
237 the length is calculated by study_chunk, so that the minimum lengths
238 are not known until the full pattern has been compiled, thus the
239 pointer to the value.
240
241 - lookbehind
242
243 In the case of lookbehind the string being searched for can be
244 offset past the start point of the final matching string.
245 If this value was just blithely removed from the min_offset it would
246 invalidate some of the calculations for how many chars must match
247 before or after (as they are derived from min_offset and minlen and
248 the length of the string being searched for).
249 When the final pattern is compiled and the data is moved from the
250 scan_data_t structure into the regexp structure the information
251 about lookbehind is factored in, with the information that would
252 have been lost precalculated in the end_shift field for the
253 associated string.
254
255 The fields pos_min and pos_delta are used to store the minimum offset
256 and the delta to the maximum offset at the current point in the pattern.
257
258*/
2c2d71f5
JH
259
260typedef struct scan_data_t {
1de06328
YO
261 /*I32 len_min; unused */
262 /*I32 len_delta; unused */
2c2d71f5
JH
263 I32 pos_min;
264 I32 pos_delta;
265 SV *last_found;
1de06328 266 I32 last_end; /* min value, <0 unless valid. */
2c2d71f5
JH
267 I32 last_start_min;
268 I32 last_start_max;
1de06328
YO
269 SV **longest; /* Either &l_fixed, or &l_float. */
270 SV *longest_fixed; /* longest fixed string found in pattern */
271 I32 offset_fixed; /* offset where it starts */
272 I32 *minlen_fixed; /* pointer to the minlen relevent to the string */
273 I32 lookbehind_fixed; /* is the position of the string modfied by LB */
274 SV *longest_float; /* longest floating string found in pattern */
275 I32 offset_float_min; /* earliest point in string it can appear */
276 I32 offset_float_max; /* latest point in string it can appear */
277 I32 *minlen_float; /* pointer to the minlen relevent to the string */
278 I32 lookbehind_float; /* is the position of the string modified by LB */
2c2d71f5
JH
279 I32 flags;
280 I32 whilem_c;
cb434fcc 281 I32 *last_closep;
653099ff 282 struct regnode_charclass_class *start_class;
2c2d71f5
JH
283} scan_data_t;
284
a687059c 285/*
e50aee73 286 * Forward declarations for pregcomp()'s friends.
a687059c 287 */
a0d0e21e 288
27da23d5 289static const scan_data_t zero_scan_data =
1de06328 290 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ,0};
c277df42
IZ
291
292#define SF_BEFORE_EOL (SF_BEFORE_SEOL|SF_BEFORE_MEOL)
07be1b83
YO
293#define SF_BEFORE_SEOL 0x0001
294#define SF_BEFORE_MEOL 0x0002
c277df42
IZ
295#define SF_FIX_BEFORE_EOL (SF_FIX_BEFORE_SEOL|SF_FIX_BEFORE_MEOL)
296#define SF_FL_BEFORE_EOL (SF_FL_BEFORE_SEOL|SF_FL_BEFORE_MEOL)
297
09b7f37c
CB
298#ifdef NO_UNARY_PLUS
299# define SF_FIX_SHIFT_EOL (0+2)
300# define SF_FL_SHIFT_EOL (0+4)
301#else
302# define SF_FIX_SHIFT_EOL (+2)
303# define SF_FL_SHIFT_EOL (+4)
304#endif
c277df42
IZ
305
306#define SF_FIX_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FIX_SHIFT_EOL)
307#define SF_FIX_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FIX_SHIFT_EOL)
308
309#define SF_FL_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FL_SHIFT_EOL)
310#define SF_FL_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FL_SHIFT_EOL) /* 0x20 */
07be1b83
YO
311#define SF_IS_INF 0x0040
312#define SF_HAS_PAR 0x0080
313#define SF_IN_PAR 0x0100
314#define SF_HAS_EVAL 0x0200
315#define SCF_DO_SUBSTR 0x0400
653099ff
GS
316#define SCF_DO_STCLASS_AND 0x0800
317#define SCF_DO_STCLASS_OR 0x1000
318#define SCF_DO_STCLASS (SCF_DO_STCLASS_AND|SCF_DO_STCLASS_OR)
e1901655 319#define SCF_WHILEM_VISITED_POS 0x2000
c277df42 320
786e8c11
YO
321#define SCF_TRIE_RESTUDY 0x4000 /* Do restudy? */
322
07be1b83 323
eb160463 324#define UTF (RExC_utf8 != 0)
e2509266
JH
325#define LOC ((RExC_flags & PMf_LOCALE) != 0)
326#define FOLD ((RExC_flags & PMf_FOLD) != 0)
a0ed51b3 327
ffc61ed2 328#define OOB_UNICODE 12345678
93733859 329#define OOB_NAMEDCLASS -1
b8c5462f 330
a0ed51b3
LW
331#define CHR_SVLEN(sv) (UTF ? sv_len_utf8(sv) : SvCUR(sv))
332#define CHR_DIST(a,b) (UTF ? utf8_distance(a,b) : a - b)
333
8615cb43 334
b45f050a
JF
335/* length of regex to show in messages that don't mark a position within */
336#define RegexLengthToShowInErrorMessages 127
337
338/*
339 * If MARKER[12] are adjusted, be sure to adjust the constants at the top
340 * of t/op/regmesg.t, the tests in t/op/re_tests, and those in
341 * op/pragma/warn/regcomp.
342 */
7253e4e3
RK
343#define MARKER1 "<-- HERE" /* marker as it appears in the description */
344#define MARKER2 " <-- HERE " /* marker as it appears within the regex */
b81d288d 345
7253e4e3 346#define REPORT_LOCATION " in regex; marked by " MARKER1 " in m/%.*s" MARKER2 "%s/"
b45f050a
JF
347
348/*
349 * Calls SAVEDESTRUCTOR_X if needed, then calls Perl_croak with the given
350 * arg. Show regex, up to a maximum length. If it's too long, chop and add
351 * "...".
352 */
ccb2c380 353#define FAIL(msg) STMT_START { \
bfed75c6 354 const char *ellipses = ""; \
ccb2c380
MP
355 IV len = RExC_end - RExC_precomp; \
356 \
357 if (!SIZE_ONLY) \
358 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
359 if (len > RegexLengthToShowInErrorMessages) { \
360 /* chop 10 shorter than the max, to ensure meaning of "..." */ \
361 len = RegexLengthToShowInErrorMessages - 10; \
362 ellipses = "..."; \
363 } \
364 Perl_croak(aTHX_ "%s in regex m/%.*s%s/", \
365 msg, (int)len, RExC_precomp, ellipses); \
366} STMT_END
8615cb43 367
b45f050a 368/*
b45f050a
JF
369 * Simple_vFAIL -- like FAIL, but marks the current location in the scan
370 */
ccb2c380 371#define Simple_vFAIL(m) STMT_START { \
a28509cc 372 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
373 Perl_croak(aTHX_ "%s" REPORT_LOCATION, \
374 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
375} STMT_END
b45f050a
JF
376
377/*
378 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL()
379 */
ccb2c380
MP
380#define vFAIL(m) STMT_START { \
381 if (!SIZE_ONLY) \
382 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
383 Simple_vFAIL(m); \
384} STMT_END
b45f050a
JF
385
386/*
387 * Like Simple_vFAIL(), but accepts two arguments.
388 */
ccb2c380 389#define Simple_vFAIL2(m,a1) STMT_START { \
a28509cc 390 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
391 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, \
392 (int)offset, RExC_precomp, RExC_precomp + offset); \
393} STMT_END
b45f050a
JF
394
395/*
396 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL2().
397 */
ccb2c380
MP
398#define vFAIL2(m,a1) STMT_START { \
399 if (!SIZE_ONLY) \
400 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
401 Simple_vFAIL2(m, a1); \
402} STMT_END
b45f050a
JF
403
404
405/*
406 * Like Simple_vFAIL(), but accepts three arguments.
407 */
ccb2c380 408#define Simple_vFAIL3(m, a1, a2) STMT_START { \
a28509cc 409 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
410 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, \
411 (int)offset, RExC_precomp, RExC_precomp + offset); \
412} STMT_END
b45f050a
JF
413
414/*
415 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL3().
416 */
ccb2c380
MP
417#define vFAIL3(m,a1,a2) STMT_START { \
418 if (!SIZE_ONLY) \
419 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
420 Simple_vFAIL3(m, a1, a2); \
421} STMT_END
b45f050a
JF
422
423/*
424 * Like Simple_vFAIL(), but accepts four arguments.
425 */
ccb2c380 426#define Simple_vFAIL4(m, a1, a2, a3) STMT_START { \
a28509cc 427 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
428 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, a3, \
429 (int)offset, RExC_precomp, RExC_precomp + offset); \
430} STMT_END
b45f050a 431
ccb2c380 432#define vWARN(loc,m) STMT_START { \
a28509cc 433 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
434 Perl_warner(aTHX_ packWARN(WARN_REGEXP), "%s" REPORT_LOCATION, \
435 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
436} STMT_END
437
438#define vWARNdep(loc,m) STMT_START { \
a28509cc 439 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
440 Perl_warner(aTHX_ packWARN2(WARN_DEPRECATED, WARN_REGEXP), \
441 "%s" REPORT_LOCATION, \
442 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
443} STMT_END
444
445
446#define vWARN2(loc, m, a1) STMT_START { \
a28509cc 447 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
448 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
449 a1, (int)offset, RExC_precomp, RExC_precomp + offset); \
450} STMT_END
451
452#define vWARN3(loc, m, a1, a2) STMT_START { \
a28509cc 453 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
454 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
455 a1, a2, (int)offset, RExC_precomp, RExC_precomp + offset); \
456} STMT_END
457
458#define vWARN4(loc, m, a1, a2, a3) STMT_START { \
a28509cc 459 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
460 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
461 a1, a2, a3, (int)offset, RExC_precomp, RExC_precomp + offset); \
462} STMT_END
463
464#define vWARN5(loc, m, a1, a2, a3, a4) STMT_START { \
a28509cc 465 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
466 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
467 a1, a2, a3, a4, (int)offset, RExC_precomp, RExC_precomp + offset); \
468} STMT_END
9d1d55b5 469
8615cb43 470
cd439c50 471/* Allow for side effects in s */
ccb2c380
MP
472#define REGC(c,s) STMT_START { \
473 if (!SIZE_ONLY) *(s) = (c); else (void)(s); \
474} STMT_END
cd439c50 475
fac92740
MJD
476/* Macros for recording node offsets. 20001227 mjd@plover.com
477 * Nodes are numbered 1, 2, 3, 4. Node #n's position is recorded in
478 * element 2*n-1 of the array. Element #2n holds the byte length node #n.
479 * Element 0 holds the number n.
07be1b83 480 * Position is 1 indexed.
fac92740
MJD
481 */
482
ccb2c380
MP
483#define Set_Node_Offset_To_R(node,byte) STMT_START { \
484 if (! SIZE_ONLY) { \
485 MJD_OFFSET_DEBUG(("** (%d) offset of node %d is %d.\n", \
07be1b83 486 __LINE__, (node), (int)(byte))); \
ccb2c380 487 if((node) < 0) { \
551405c4 488 Perl_croak(aTHX_ "value of node is %d in Offset macro", (int)(node)); \
ccb2c380
MP
489 } else { \
490 RExC_offsets[2*(node)-1] = (byte); \
491 } \
492 } \
493} STMT_END
494
495#define Set_Node_Offset(node,byte) \
496 Set_Node_Offset_To_R((node)-RExC_emit_start, (byte)-RExC_start)
497#define Set_Cur_Node_Offset Set_Node_Offset(RExC_emit, RExC_parse)
498
499#define Set_Node_Length_To_R(node,len) STMT_START { \
500 if (! SIZE_ONLY) { \
501 MJD_OFFSET_DEBUG(("** (%d) size of node %d is %d.\n", \
551405c4 502 __LINE__, (int)(node), (int)(len))); \
ccb2c380 503 if((node) < 0) { \
551405c4 504 Perl_croak(aTHX_ "value of node is %d in Length macro", (int)(node)); \
ccb2c380
MP
505 } else { \
506 RExC_offsets[2*(node)] = (len); \
507 } \
508 } \
509} STMT_END
510
511#define Set_Node_Length(node,len) \
512 Set_Node_Length_To_R((node)-RExC_emit_start, len)
513#define Set_Cur_Node_Length(len) Set_Node_Length(RExC_emit, len)
514#define Set_Node_Cur_Length(node) \
515 Set_Node_Length(node, RExC_parse - parse_start)
fac92740
MJD
516
517/* Get offsets and lengths */
518#define Node_Offset(n) (RExC_offsets[2*((n)-RExC_emit_start)-1])
519#define Node_Length(n) (RExC_offsets[2*((n)-RExC_emit_start)])
520
07be1b83
YO
521#define Set_Node_Offset_Length(node,offset,len) STMT_START { \
522 Set_Node_Offset_To_R((node)-RExC_emit_start, (offset)); \
523 Set_Node_Length_To_R((node)-RExC_emit_start, (len)); \
524} STMT_END
525
526
527#if PERL_ENABLE_EXPERIMENTAL_REGEX_OPTIMISATIONS
528#define EXPERIMENTAL_INPLACESCAN
529#endif
530
1de06328 531#define DEBUG_STUDYDATA(data,depth) \
a5ca303d 532DEBUG_OPTIMISE_MORE_r(if(data){ \
1de06328
YO
533 PerlIO_printf(Perl_debug_log, \
534 "%*s"/* Len:%"IVdf"/%"IVdf" */" Pos:%"IVdf"/%"IVdf \
535 " Flags: %"IVdf" Whilem_c: %"IVdf" Lcp: %"IVdf" ", \
536 (int)(depth)*2, "", \
537 (IV)((data)->pos_min), \
538 (IV)((data)->pos_delta), \
539 (IV)((data)->flags), \
540 (IV)((data)->whilem_c), \
541 (IV)((data)->last_closep ? *((data)->last_closep) : -1) \
542 ); \
543 if ((data)->last_found) \
544 PerlIO_printf(Perl_debug_log, \
545 "Last:'%s' %"IVdf":%"IVdf"/%"IVdf" %sFixed:'%s' @ %"IVdf \
546 " %sFloat: '%s' @ %"IVdf"/%"IVdf"", \
547 SvPVX_const((data)->last_found), \
548 (IV)((data)->last_end), \
549 (IV)((data)->last_start_min), \
550 (IV)((data)->last_start_max), \
551 ((data)->longest && \
552 (data)->longest==&((data)->longest_fixed)) ? "*" : "", \
553 SvPVX_const((data)->longest_fixed), \
554 (IV)((data)->offset_fixed), \
555 ((data)->longest && \
556 (data)->longest==&((data)->longest_float)) ? "*" : "", \
557 SvPVX_const((data)->longest_float), \
558 (IV)((data)->offset_float_min), \
559 (IV)((data)->offset_float_max) \
560 ); \
561 PerlIO_printf(Perl_debug_log,"\n"); \
562});
563
acfe0abc 564static void clear_re(pTHX_ void *r);
4327152a 565
653099ff 566/* Mark that we cannot extend a found fixed substring at this point.
786e8c11 567 Update the longest found anchored substring and the longest found
653099ff
GS
568 floating substrings if needed. */
569
4327152a 570STATIC void
1de06328 571S_scan_commit(pTHX_ const RExC_state_t *pRExC_state, scan_data_t *data, I32 *minlenp)
c277df42 572{
e1ec3a88
AL
573 const STRLEN l = CHR_SVLEN(data->last_found);
574 const STRLEN old_l = CHR_SVLEN(*data->longest);
1de06328 575 GET_RE_DEBUG_FLAGS_DECL;
b81d288d 576
c277df42 577 if ((l >= old_l) && ((l > old_l) || (data->flags & SF_BEFORE_EOL))) {
6b43b216 578 SvSetMagicSV(*data->longest, data->last_found);
c277df42
IZ
579 if (*data->longest == data->longest_fixed) {
580 data->offset_fixed = l ? data->last_start_min : data->pos_min;
581 if (data->flags & SF_BEFORE_EOL)
b81d288d 582 data->flags
c277df42
IZ
583 |= ((data->flags & SF_BEFORE_EOL) << SF_FIX_SHIFT_EOL);
584 else
585 data->flags &= ~SF_FIX_BEFORE_EOL;
1de06328
YO
586 data->minlen_fixed=minlenp;
587 data->lookbehind_fixed=0;
a0ed51b3
LW
588 }
589 else {
c277df42 590 data->offset_float_min = l ? data->last_start_min : data->pos_min;
b81d288d
AB
591 data->offset_float_max = (l
592 ? data->last_start_max
c277df42 593 : data->pos_min + data->pos_delta);
9051bda5
HS
594 if ((U32)data->offset_float_max > (U32)I32_MAX)
595 data->offset_float_max = I32_MAX;
c277df42 596 if (data->flags & SF_BEFORE_EOL)
b81d288d 597 data->flags
c277df42
IZ
598 |= ((data->flags & SF_BEFORE_EOL) << SF_FL_SHIFT_EOL);
599 else
600 data->flags &= ~SF_FL_BEFORE_EOL;
1de06328
YO
601 data->minlen_float=minlenp;
602 data->lookbehind_float=0;
c277df42
IZ
603 }
604 }
605 SvCUR_set(data->last_found, 0);
0eda9292 606 {
a28509cc 607 SV * const sv = data->last_found;
097eb12c
AL
608 if (SvUTF8(sv) && SvMAGICAL(sv)) {
609 MAGIC * const mg = mg_find(sv, PERL_MAGIC_utf8);
610 if (mg)
611 mg->mg_len = 0;
612 }
0eda9292 613 }
c277df42
IZ
614 data->last_end = -1;
615 data->flags &= ~SF_BEFORE_EOL;
1de06328 616 DEBUG_STUDYDATA(data,0);
c277df42
IZ
617}
618
653099ff
GS
619/* Can match anything (initialization) */
620STATIC void
097eb12c 621S_cl_anything(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
653099ff 622{
653099ff 623 ANYOF_CLASS_ZERO(cl);
f8bef550 624 ANYOF_BITMAP_SETALL(cl);
1aa99e6b 625 cl->flags = ANYOF_EOS|ANYOF_UNICODE_ALL;
653099ff
GS
626 if (LOC)
627 cl->flags |= ANYOF_LOCALE;
628}
629
630/* Can match anything (initialization) */
631STATIC int
5f66b61c 632S_cl_is_anything(const struct regnode_charclass_class *cl)
653099ff
GS
633{
634 int value;
635
aaa51d5e 636 for (value = 0; value <= ANYOF_MAX; value += 2)
653099ff
GS
637 if (ANYOF_CLASS_TEST(cl, value) && ANYOF_CLASS_TEST(cl, value + 1))
638 return 1;
1aa99e6b
IH
639 if (!(cl->flags & ANYOF_UNICODE_ALL))
640 return 0;
10edeb5d 641 if (!ANYOF_BITMAP_TESTALLSET((const void*)cl))
f8bef550 642 return 0;
653099ff
GS
643 return 1;
644}
645
646/* Can match anything (initialization) */
647STATIC void
097eb12c 648S_cl_init(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
653099ff 649{
8ecf7187 650 Zero(cl, 1, struct regnode_charclass_class);
653099ff 651 cl->type = ANYOF;
830247a4 652 cl_anything(pRExC_state, cl);
653099ff
GS
653}
654
655STATIC void
097eb12c 656S_cl_init_zero(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
653099ff 657{
8ecf7187 658 Zero(cl, 1, struct regnode_charclass_class);
653099ff 659 cl->type = ANYOF;
830247a4 660 cl_anything(pRExC_state, cl);
653099ff
GS
661 if (LOC)
662 cl->flags |= ANYOF_LOCALE;
663}
664
665/* 'And' a given class with another one. Can create false positives */
666/* We assume that cl is not inverted */
667STATIC void
5f66b61c 668S_cl_and(struct regnode_charclass_class *cl,
a28509cc 669 const struct regnode_charclass_class *and_with)
653099ff 670{
653099ff
GS
671 if (!(and_with->flags & ANYOF_CLASS)
672 && !(cl->flags & ANYOF_CLASS)
673 && (and_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
674 && !(and_with->flags & ANYOF_FOLD)
675 && !(cl->flags & ANYOF_FOLD)) {
676 int i;
677
678 if (and_with->flags & ANYOF_INVERT)
679 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
680 cl->bitmap[i] &= ~and_with->bitmap[i];
681 else
682 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
683 cl->bitmap[i] &= and_with->bitmap[i];
684 } /* XXXX: logic is complicated otherwise, leave it along for a moment. */
685 if (!(and_with->flags & ANYOF_EOS))
686 cl->flags &= ~ANYOF_EOS;
1aa99e6b 687
14ebb1a2
JH
688 if (cl->flags & ANYOF_UNICODE_ALL && and_with->flags & ANYOF_UNICODE &&
689 !(and_with->flags & ANYOF_INVERT)) {
1aa99e6b
IH
690 cl->flags &= ~ANYOF_UNICODE_ALL;
691 cl->flags |= ANYOF_UNICODE;
692 ARG_SET(cl, ARG(and_with));
693 }
14ebb1a2
JH
694 if (!(and_with->flags & ANYOF_UNICODE_ALL) &&
695 !(and_with->flags & ANYOF_INVERT))
1aa99e6b 696 cl->flags &= ~ANYOF_UNICODE_ALL;
14ebb1a2
JH
697 if (!(and_with->flags & (ANYOF_UNICODE|ANYOF_UNICODE_ALL)) &&
698 !(and_with->flags & ANYOF_INVERT))
1aa99e6b 699 cl->flags &= ~ANYOF_UNICODE;
653099ff
GS
700}
701
702/* 'OR' a given class with another one. Can create false positives */
703/* We assume that cl is not inverted */
704STATIC void
097eb12c 705S_cl_or(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl, const struct regnode_charclass_class *or_with)
653099ff 706{
653099ff
GS
707 if (or_with->flags & ANYOF_INVERT) {
708 /* We do not use
709 * (B1 | CL1) | (!B2 & !CL2) = (B1 | !B2 & !CL2) | (CL1 | (!B2 & !CL2))
710 * <= (B1 | !B2) | (CL1 | !CL2)
711 * which is wasteful if CL2 is small, but we ignore CL2:
712 * (B1 | CL1) | (!B2 & !CL2) <= (B1 | CL1) | !B2 = (B1 | !B2) | CL1
713 * XXXX Can we handle case-fold? Unclear:
714 * (OK1(i) | OK1(i')) | !(OK1(i) | OK1(i')) =
715 * (OK1(i) | OK1(i')) | (!OK1(i) & !OK1(i'))
716 */
717 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
718 && !(or_with->flags & ANYOF_FOLD)
719 && !(cl->flags & ANYOF_FOLD) ) {
720 int i;
721
722 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
723 cl->bitmap[i] |= ~or_with->bitmap[i];
724 } /* XXXX: logic is complicated otherwise */
725 else {
830247a4 726 cl_anything(pRExC_state, cl);
653099ff
GS
727 }
728 } else {
729 /* (B1 | CL1) | (B2 | CL2) = (B1 | B2) | (CL1 | CL2)) */
730 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
b81d288d 731 && (!(or_with->flags & ANYOF_FOLD)
653099ff
GS
732 || (cl->flags & ANYOF_FOLD)) ) {
733 int i;
734
735 /* OR char bitmap and class bitmap separately */
736 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
737 cl->bitmap[i] |= or_with->bitmap[i];
738 if (or_with->flags & ANYOF_CLASS) {
739 for (i = 0; i < ANYOF_CLASSBITMAP_SIZE; i++)
740 cl->classflags[i] |= or_with->classflags[i];
741 cl->flags |= ANYOF_CLASS;
742 }
743 }
744 else { /* XXXX: logic is complicated, leave it along for a moment. */
830247a4 745 cl_anything(pRExC_state, cl);
653099ff
GS
746 }
747 }
748 if (or_with->flags & ANYOF_EOS)
749 cl->flags |= ANYOF_EOS;
1aa99e6b
IH
750
751 if (cl->flags & ANYOF_UNICODE && or_with->flags & ANYOF_UNICODE &&
752 ARG(cl) != ARG(or_with)) {
753 cl->flags |= ANYOF_UNICODE_ALL;
754 cl->flags &= ~ANYOF_UNICODE;
755 }
756 if (or_with->flags & ANYOF_UNICODE_ALL) {
757 cl->flags |= ANYOF_UNICODE_ALL;
758 cl->flags &= ~ANYOF_UNICODE;
759 }
653099ff
GS
760}
761
a3621e74
YO
762#define TRIE_LIST_ITEM(state,idx) (trie->states[state].trans.list)[ idx ]
763#define TRIE_LIST_CUR(state) ( TRIE_LIST_ITEM( state, 0 ).forid )
764#define TRIE_LIST_LEN(state) ( TRIE_LIST_ITEM( state, 0 ).newstate )
765#define TRIE_LIST_USED(idx) ( trie->states[state].trans.list ? (TRIE_LIST_CUR( idx ) - 1) : 0 )
766
3dab1dad
YO
767
768#ifdef DEBUGGING
07be1b83 769/*
3dab1dad
YO
770 dump_trie(trie)
771 dump_trie_interim_list(trie,next_alloc)
772 dump_trie_interim_table(trie,next_alloc)
773
774 These routines dump out a trie in a somewhat readable format.
07be1b83
YO
775 The _interim_ variants are used for debugging the interim
776 tables that are used to generate the final compressed
777 representation which is what dump_trie expects.
778
3dab1dad
YO
779 Part of the reason for their existance is to provide a form
780 of documentation as to how the different representations function.
07be1b83
YO
781
782*/
3dab1dad
YO
783
784/*
785 dump_trie(trie)
786 Dumps the final compressed table form of the trie to Perl_debug_log.
787 Used for debugging make_trie().
788*/
789
790STATIC void
791S_dump_trie(pTHX_ const struct _reg_trie_data *trie,U32 depth)
792{
793 U32 state;
ab3bbdeb
YO
794 SV *sv=sv_newmortal();
795 int colwidth= trie->widecharmap ? 6 : 4;
3dab1dad
YO
796 GET_RE_DEBUG_FLAGS_DECL;
797
ab3bbdeb 798
3dab1dad
YO
799 PerlIO_printf( Perl_debug_log, "%*sChar : %-6s%-6s%-4s ",
800 (int)depth * 2 + 2,"",
801 "Match","Base","Ofs" );
802
803 for( state = 0 ; state < trie->uniquecharcount ; state++ ) {
be8e71aa 804 SV ** const tmp = av_fetch( trie->revcharmap, state, 0);
3dab1dad 805 if ( tmp ) {
ab3bbdeb
YO
806 PerlIO_printf( Perl_debug_log, "%*s",
807 colwidth,
ddc5bc0f 808 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
ab3bbdeb
YO
809 PL_colors[0], PL_colors[1],
810 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
811 PERL_PV_ESCAPE_FIRSTCHAR
812 )
813 );
3dab1dad
YO
814 }
815 }
816 PerlIO_printf( Perl_debug_log, "\n%*sState|-----------------------",
817 (int)depth * 2 + 2,"");
818
819 for( state = 0 ; state < trie->uniquecharcount ; state++ )
ab3bbdeb 820 PerlIO_printf( Perl_debug_log, "%.*s", colwidth, "--------");
3dab1dad
YO
821 PerlIO_printf( Perl_debug_log, "\n");
822
786e8c11 823 for( state = 1 ; state < trie->laststate ; state++ ) {
be8e71aa 824 const U32 base = trie->states[ state ].trans.base;
3dab1dad
YO
825
826 PerlIO_printf( Perl_debug_log, "%*s#%4"UVXf"|", (int)depth * 2 + 2,"", (UV)state);
827
828 if ( trie->states[ state ].wordnum ) {
829 PerlIO_printf( Perl_debug_log, " W%4X", trie->states[ state ].wordnum );
830 } else {
831 PerlIO_printf( Perl_debug_log, "%6s", "" );
832 }
833
834 PerlIO_printf( Perl_debug_log, " @%4"UVXf" ", (UV)base );
835
836 if ( base ) {
837 U32 ofs = 0;
838
839 while( ( base + ofs < trie->uniquecharcount ) ||
840 ( base + ofs - trie->uniquecharcount < trie->lasttrans
841 && trie->trans[ base + ofs - trie->uniquecharcount ].check != state))
842 ofs++;
843
844 PerlIO_printf( Perl_debug_log, "+%2"UVXf"[ ", (UV)ofs);
845
846 for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) {
847 if ( ( base + ofs >= trie->uniquecharcount ) &&
848 ( base + ofs - trie->uniquecharcount < trie->lasttrans ) &&
849 trie->trans[ base + ofs - trie->uniquecharcount ].check == state )
850 {
ab3bbdeb
YO
851 PerlIO_printf( Perl_debug_log, "%*"UVXf,
852 colwidth,
3dab1dad
YO
853 (UV)trie->trans[ base + ofs - trie->uniquecharcount ].next );
854 } else {
ab3bbdeb 855 PerlIO_printf( Perl_debug_log, "%*s",colwidth," ." );
3dab1dad
YO
856 }
857 }
858
859 PerlIO_printf( Perl_debug_log, "]");
860
861 }
862 PerlIO_printf( Perl_debug_log, "\n" );
863 }
864}
865/*
866 dump_trie_interim_list(trie,next_alloc)
867 Dumps a fully constructed but uncompressed trie in list form.
868 List tries normally only are used for construction when the number of
869 possible chars (trie->uniquecharcount) is very high.
870 Used for debugging make_trie().
871*/
872STATIC void
873S_dump_trie_interim_list(pTHX_ const struct _reg_trie_data *trie, U32 next_alloc,U32 depth)
874{
875 U32 state;
ab3bbdeb
YO
876 SV *sv=sv_newmortal();
877 int colwidth= trie->widecharmap ? 6 : 4;
3dab1dad
YO
878 GET_RE_DEBUG_FLAGS_DECL;
879 /* print out the table precompression. */
ab3bbdeb
YO
880 PerlIO_printf( Perl_debug_log, "%*sState :Word | Transition Data\n%*s%s",
881 (int)depth * 2 + 2,"", (int)depth * 2 + 2,"",
882 "------:-----+-----------------\n" );
3dab1dad
YO
883
884 for( state=1 ; state < next_alloc ; state ++ ) {
885 U16 charid;
886
ab3bbdeb 887 PerlIO_printf( Perl_debug_log, "%*s %4"UVXf" :",
3dab1dad
YO
888 (int)depth * 2 + 2,"", (UV)state );
889 if ( ! trie->states[ state ].wordnum ) {
890 PerlIO_printf( Perl_debug_log, "%5s| ","");
891 } else {
892 PerlIO_printf( Perl_debug_log, "W%4x| ",
893 trie->states[ state ].wordnum
894 );
895 }
896 for( charid = 1 ; charid <= TRIE_LIST_USED( state ) ; charid++ ) {
be8e71aa 897 SV ** const tmp = av_fetch( trie->revcharmap, TRIE_LIST_ITEM(state,charid).forid, 0);
ab3bbdeb
YO
898 if ( tmp ) {
899 PerlIO_printf( Perl_debug_log, "%*s:%3X=%4"UVXf" | ",
900 colwidth,
ddc5bc0f 901 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
ab3bbdeb
YO
902 PL_colors[0], PL_colors[1],
903 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
904 PERL_PV_ESCAPE_FIRSTCHAR
905 ) ,
3dab1dad
YO
906 TRIE_LIST_ITEM(state,charid).forid,
907 (UV)TRIE_LIST_ITEM(state,charid).newstate
908 );
909 }
ab3bbdeb
YO
910 }
911 PerlIO_printf( Perl_debug_log, "\n");
3dab1dad
YO
912 }
913}
914
915/*
916 dump_trie_interim_table(trie,next_alloc)
917 Dumps a fully constructed but uncompressed trie in table form.
918 This is the normal DFA style state transition table, with a few
919 twists to facilitate compression later.
920 Used for debugging make_trie().
921*/
922STATIC void
923S_dump_trie_interim_table(pTHX_ const struct _reg_trie_data *trie, U32 next_alloc, U32 depth)
924{
925 U32 state;
926 U16 charid;
ab3bbdeb
YO
927 SV *sv=sv_newmortal();
928 int colwidth= trie->widecharmap ? 6 : 4;
3dab1dad
YO
929 GET_RE_DEBUG_FLAGS_DECL;
930
931 /*
932 print out the table precompression so that we can do a visual check
933 that they are identical.
934 */
935
936 PerlIO_printf( Perl_debug_log, "%*sChar : ",(int)depth * 2 + 2,"" );
937
938 for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) {
be8e71aa 939 SV ** const tmp = av_fetch( trie->revcharmap, charid, 0);
3dab1dad 940 if ( tmp ) {
ab3bbdeb
YO
941 PerlIO_printf( Perl_debug_log, "%*s",
942 colwidth,
ddc5bc0f 943 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
ab3bbdeb
YO
944 PL_colors[0], PL_colors[1],
945 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
946 PERL_PV_ESCAPE_FIRSTCHAR
947 )
948 );
3dab1dad
YO
949 }
950 }
951
952 PerlIO_printf( Perl_debug_log, "\n%*sState+-",(int)depth * 2 + 2,"" );
953
954 for( charid=0 ; charid < trie->uniquecharcount ; charid++ ) {
ab3bbdeb 955 PerlIO_printf( Perl_debug_log, "%.*s", colwidth,"--------");
3dab1dad
YO
956 }
957
958 PerlIO_printf( Perl_debug_log, "\n" );
959
960 for( state=1 ; state < next_alloc ; state += trie->uniquecharcount ) {
961
962 PerlIO_printf( Perl_debug_log, "%*s%4"UVXf" : ",
963 (int)depth * 2 + 2,"",
964 (UV)TRIE_NODENUM( state ) );
965
966 for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) {
ab3bbdeb
YO
967 UV v=(UV)SAFE_TRIE_NODENUM( trie->trans[ state + charid ].next );
968 if (v)
969 PerlIO_printf( Perl_debug_log, "%*"UVXf, colwidth, v );
970 else
971 PerlIO_printf( Perl_debug_log, "%*s", colwidth, "." );
3dab1dad
YO
972 }
973 if ( ! trie->states[ TRIE_NODENUM( state ) ].wordnum ) {
974 PerlIO_printf( Perl_debug_log, " (%4"UVXf")\n", (UV)trie->trans[ state ].check );
975 } else {
976 PerlIO_printf( Perl_debug_log, " (%4"UVXf") W%4X\n", (UV)trie->trans[ state ].check,
977 trie->states[ TRIE_NODENUM( state ) ].wordnum );
978 }
979 }
07be1b83 980}
3dab1dad
YO
981
982#endif
983
786e8c11
YO
984/* make_trie(startbranch,first,last,tail,word_count,flags,depth)
985 startbranch: the first branch in the whole branch sequence
986 first : start branch of sequence of branch-exact nodes.
987 May be the same as startbranch
988 last : Thing following the last branch.
989 May be the same as tail.
990 tail : item following the branch sequence
991 count : words in the sequence
992 flags : currently the OP() type we will be building one of /EXACT(|F|Fl)/
993 depth : indent depth
3dab1dad 994
786e8c11 995Inplace optimizes a sequence of 2 or more Branch-Exact nodes into a TRIE node.
07be1b83 996
786e8c11
YO
997A trie is an N'ary tree where the branches are determined by digital
998decomposition of the key. IE, at the root node you look up the 1st character and
999follow that branch repeat until you find the end of the branches. Nodes can be
1000marked as "accepting" meaning they represent a complete word. Eg:
07be1b83 1001
786e8c11 1002 /he|she|his|hers/
72f13be8 1003
786e8c11
YO
1004would convert into the following structure. Numbers represent states, letters
1005following numbers represent valid transitions on the letter from that state, if
1006the number is in square brackets it represents an accepting state, otherwise it
1007will be in parenthesis.
07be1b83 1008
786e8c11
YO
1009 +-h->+-e->[3]-+-r->(8)-+-s->[9]
1010 | |
1011 | (2)
1012 | |
1013 (1) +-i->(6)-+-s->[7]
1014 |
1015 +-s->(3)-+-h->(4)-+-e->[5]
07be1b83 1016
786e8c11
YO
1017 Accept Word Mapping: 3=>1 (he),5=>2 (she), 7=>3 (his), 9=>4 (hers)
1018
1019This shows that when matching against the string 'hers' we will begin at state 1
1020read 'h' and move to state 2, read 'e' and move to state 3 which is accepting,
1021then read 'r' and go to state 8 followed by 's' which takes us to state 9 which
1022is also accepting. Thus we know that we can match both 'he' and 'hers' with a
1023single traverse. We store a mapping from accepting to state to which word was
1024matched, and then when we have multiple possibilities we try to complete the
1025rest of the regex in the order in which they occured in the alternation.
1026
1027The only prior NFA like behaviour that would be changed by the TRIE support is
1028the silent ignoring of duplicate alternations which are of the form:
1029
1030 / (DUPE|DUPE) X? (?{ ... }) Y /x
1031
1032Thus EVAL blocks follwing a trie may be called a different number of times with
1033and without the optimisation. With the optimisations dupes will be silently
1034ignored. This inconsistant behaviour of EVAL type nodes is well established as
1035the following demonstrates:
1036
1037 'words'=~/(word|word|word)(?{ print $1 })[xyz]/
1038
1039which prints out 'word' three times, but
1040
1041 'words'=~/(word|word|word)(?{ print $1 })S/
1042
1043which doesnt print it out at all. This is due to other optimisations kicking in.
1044
1045Example of what happens on a structural level:
1046
1047The regexp /(ac|ad|ab)+/ will produce the folowing debug output:
1048
1049 1: CURLYM[1] {1,32767}(18)
1050 5: BRANCH(8)
1051 6: EXACT <ac>(16)
1052 8: BRANCH(11)
1053 9: EXACT <ad>(16)
1054 11: BRANCH(14)
1055 12: EXACT <ab>(16)
1056 16: SUCCEED(0)
1057 17: NOTHING(18)
1058 18: END(0)
1059
1060This would be optimizable with startbranch=5, first=5, last=16, tail=16
1061and should turn into:
1062
1063 1: CURLYM[1] {1,32767}(18)
1064 5: TRIE(16)
1065 [Words:3 Chars Stored:6 Unique Chars:4 States:5 NCP:1]
1066 <ac>
1067 <ad>
1068 <ab>
1069 16: SUCCEED(0)
1070 17: NOTHING(18)
1071 18: END(0)
1072
1073Cases where tail != last would be like /(?foo|bar)baz/:
1074
1075 1: BRANCH(4)
1076 2: EXACT <foo>(8)
1077 4: BRANCH(7)
1078 5: EXACT <bar>(8)
1079 7: TAIL(8)
1080 8: EXACT <baz>(10)
1081 10: END(0)
1082
1083which would be optimizable with startbranch=1, first=1, last=7, tail=8
1084and would end up looking like:
1085
1086 1: TRIE(8)
1087 [Words:2 Chars Stored:6 Unique Chars:5 States:7 NCP:1]
1088 <foo>
1089 <bar>
1090 7: TAIL(8)
1091 8: EXACT <baz>(10)
1092 10: END(0)
1093
1094 d = uvuni_to_utf8_flags(d, uv, 0);
1095
1096is the recommended Unicode-aware way of saying
1097
1098 *(d++) = uv;
1099*/
1100
1101#define TRIE_STORE_REVCHAR \
1102 STMT_START { \
1103 SV *tmp = Perl_newSVpvf_nocontext( "%c", (int)uvc ); \
1104 if (UTF) SvUTF8_on(tmp); \
1105 av_push( TRIE_REVCHARMAP(trie), tmp ); \
1106 } STMT_END
1107
1108#define TRIE_READ_CHAR STMT_START { \
1109 wordlen++; \
1110 if ( UTF ) { \
1111 if ( folder ) { \
1112 if ( foldlen > 0 ) { \
1113 uvc = utf8n_to_uvuni( scan, UTF8_MAXLEN, &len, uniflags ); \
1114 foldlen -= len; \
1115 scan += len; \
1116 len = 0; \
1117 } else { \
1118 uvc = utf8n_to_uvuni( (const U8*)uc, UTF8_MAXLEN, &len, uniflags);\
1119 uvc = to_uni_fold( uvc, foldbuf, &foldlen ); \
1120 foldlen -= UNISKIP( uvc ); \
1121 scan = foldbuf + UNISKIP( uvc ); \
1122 } \
1123 } else { \
1124 uvc = utf8n_to_uvuni( (const U8*)uc, UTF8_MAXLEN, &len, uniflags);\
1125 } \
1126 } else { \
1127 uvc = (U32)*uc; \
1128 len = 1; \
1129 } \
1130} STMT_END
1131
1132
1133
1134#define TRIE_LIST_PUSH(state,fid,ns) STMT_START { \
1135 if ( TRIE_LIST_CUR( state ) >=TRIE_LIST_LEN( state ) ) { \
1136 TRIE_LIST_LEN( state ) *= 2; \
1137 Renew( trie->states[ state ].trans.list, \
1138 TRIE_LIST_LEN( state ), reg_trie_trans_le ); \
1139 } \
1140 TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).forid = fid; \
1141 TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).newstate = ns; \
1142 TRIE_LIST_CUR( state )++; \
1143} STMT_END
07be1b83 1144
786e8c11
YO
1145#define TRIE_LIST_NEW(state) STMT_START { \
1146 Newxz( trie->states[ state ].trans.list, \
1147 4, reg_trie_trans_le ); \
1148 TRIE_LIST_CUR( state ) = 1; \
1149 TRIE_LIST_LEN( state ) = 4; \
1150} STMT_END
07be1b83 1151
786e8c11
YO
1152#define TRIE_HANDLE_WORD(state) STMT_START { \
1153 U16 dupe= trie->states[ state ].wordnum; \
1154 regnode * const noper_next = regnext( noper ); \
1155 \
1156 if (trie->wordlen) \
1157 trie->wordlen[ curword ] = wordlen; \
1158 DEBUG_r({ \
1159 /* store the word for dumping */ \
1160 SV* tmp; \
1161 if (OP(noper) != NOTHING) \
1162 tmp = newSVpvn(STRING(noper), STR_LEN(noper)); \
1163 else \
1164 tmp = newSVpvn( "", 0 ); \
1165 if ( UTF ) SvUTF8_on( tmp ); \
1166 av_push( trie->words, tmp ); \
1167 }); \
1168 \
1169 curword++; \
1170 \
1171 if ( noper_next < tail ) { \
1172 if (!trie->jump) \
1173 Newxz( trie->jump, word_count + 1, U16); \
1174 trie->jump[curword] = (U16)(tail - noper_next); \
1175 if (!jumper) \
1176 jumper = noper_next; \
1177 if (!nextbranch) \
1178 nextbranch= regnext(cur); \
1179 } \
1180 \
1181 if ( dupe ) { \
1182 /* So it's a dupe. This means we need to maintain a */\
1183 /* linked-list from the first to the next. */\
1184 /* we only allocate the nextword buffer when there */\
1185 /* a dupe, so first time we have to do the allocation */\
1186 if (!trie->nextword) \
1187 Newxz( trie->nextword, word_count + 1, U16); \
1188 while ( trie->nextword[dupe] ) \
1189 dupe= trie->nextword[dupe]; \
1190 trie->nextword[dupe]= curword; \
1191 } else { \
1192 /* we haven't inserted this word yet. */ \
1193 trie->states[ state ].wordnum = curword; \
1194 } \
1195} STMT_END
07be1b83 1196
3dab1dad 1197
786e8c11
YO
1198#define TRIE_TRANS_STATE(state,base,ucharcount,charid,special) \
1199 ( ( base + charid >= ucharcount \
1200 && base + charid < ubound \
1201 && state == trie->trans[ base - ucharcount + charid ].check \
1202 && trie->trans[ base - ucharcount + charid ].next ) \
1203 ? trie->trans[ base - ucharcount + charid ].next \
1204 : ( state==1 ? special : 0 ) \
1205 )
3dab1dad 1206
786e8c11
YO
1207#define MADE_TRIE 1
1208#define MADE_JUMP_TRIE 2
1209#define MADE_EXACT_TRIE 4
3dab1dad 1210
a3621e74 1211STATIC I32
786e8c11 1212S_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch, regnode *first, regnode *last, regnode *tail, U32 word_count, U32 flags, U32 depth)
a3621e74 1213{
27da23d5 1214 dVAR;
a3621e74
YO
1215 /* first pass, loop through and scan words */
1216 reg_trie_data *trie;
1217 regnode *cur;
9f7f3913 1218 const U32 uniflags = UTF8_ALLOW_DEFAULT;
a3621e74
YO
1219 STRLEN len = 0;
1220 UV uvc = 0;
1221 U16 curword = 0;
1222 U32 next_alloc = 0;
786e8c11
YO
1223 regnode *jumper = NULL;
1224 regnode *nextbranch = NULL;
a3621e74 1225 /* we just use folder as a flag in utf8 */
e1ec3a88 1226 const U8 * const folder = ( flags == EXACTF
a3621e74
YO
1227 ? PL_fold
1228 : ( flags == EXACTFL
1229 ? PL_fold_locale
1230 : NULL
1231 )
1232 );
1233
e1ec3a88 1234 const U32 data_slot = add_data( pRExC_state, 1, "t" );
a3621e74 1235 SV *re_trie_maxbuff;
3dab1dad
YO
1236#ifndef DEBUGGING
1237 /* these are only used during construction but are useful during
8e11feef 1238 * debugging so we store them in the struct when debugging.
8e11feef 1239 */
3dab1dad 1240 STRLEN trie_charcount=0;
3dab1dad
YO
1241 AV *trie_revcharmap;
1242#endif
a3621e74 1243 GET_RE_DEBUG_FLAGS_DECL;
72f13be8
YO
1244#ifndef DEBUGGING
1245 PERL_UNUSED_ARG(depth);
1246#endif
a3621e74 1247
a02a5408 1248 Newxz( trie, 1, reg_trie_data );
a3621e74 1249 trie->refcount = 1;
3dab1dad 1250 trie->startstate = 1;
786e8c11 1251 trie->wordcount = word_count;
a3621e74 1252 RExC_rx->data->data[ data_slot ] = (void*)trie;
a02a5408 1253 Newxz( trie->charmap, 256, U16 );
3dab1dad
YO
1254 if (!(UTF && folder))
1255 Newxz( trie->bitmap, ANYOF_BITMAP_SIZE, char );
a3621e74
YO
1256 DEBUG_r({
1257 trie->words = newAV();
a3621e74 1258 });
3dab1dad 1259 TRIE_REVCHARMAP(trie) = newAV();
a3621e74 1260
0111c4fd 1261 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1);
a3621e74 1262 if (!SvIOK(re_trie_maxbuff)) {
0111c4fd 1263 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT);
a3621e74 1264 }
3dab1dad
YO
1265 DEBUG_OPTIMISE_r({
1266 PerlIO_printf( Perl_debug_log,
786e8c11 1267 "%*smake_trie start==%d, first==%d, last==%d, tail==%d depth=%d\n",
3dab1dad
YO
1268 (int)depth * 2 + 2, "",
1269 REG_NODE_NUM(startbranch),REG_NODE_NUM(first),
786e8c11 1270 REG_NODE_NUM(last), REG_NODE_NUM(tail),
85c3142d 1271 (int)depth);
3dab1dad 1272 });
a3621e74
YO
1273 /* -- First loop and Setup --
1274
1275 We first traverse the branches and scan each word to determine if it
1276 contains widechars, and how many unique chars there are, this is
1277 important as we have to build a table with at least as many columns as we
1278 have unique chars.
1279
1280 We use an array of integers to represent the character codes 0..255
1281 (trie->charmap) and we use a an HV* to store unicode characters. We use the
1282 native representation of the character value as the key and IV's for the
1283 coded index.
1284
1285 *TODO* If we keep track of how many times each character is used we can
1286 remap the columns so that the table compression later on is more
1287 efficient in terms of memory by ensuring most common value is in the
1288 middle and the least common are on the outside. IMO this would be better
1289 than a most to least common mapping as theres a decent chance the most
1290 common letter will share a node with the least common, meaning the node
1291 will not be compressable. With a middle is most common approach the worst
1292 case is when we have the least common nodes twice.
1293
1294 */
1295
a3621e74 1296 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
c445ea15 1297 regnode * const noper = NEXTOPER( cur );
e1ec3a88 1298 const U8 *uc = (U8*)STRING( noper );
a28509cc 1299 const U8 * const e = uc + STR_LEN( noper );
a3621e74
YO
1300 STRLEN foldlen = 0;
1301 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
2af232bd 1302 const U8 *scan = (U8*)NULL;
07be1b83 1303 U32 wordlen = 0; /* required init */
3dab1dad 1304 STRLEN chars=0;
a3621e74 1305
3dab1dad
YO
1306 if (OP(noper) == NOTHING) {
1307 trie->minlen= 0;
1308 continue;
1309 }
1310 if (trie->bitmap) {
1311 TRIE_BITMAP_SET(trie,*uc);
1312 if ( folder ) TRIE_BITMAP_SET(trie,folder[ *uc ]);
1313 }
a3621e74 1314 for ( ; uc < e ; uc += len ) {
3dab1dad 1315 TRIE_CHARCOUNT(trie)++;
a3621e74 1316 TRIE_READ_CHAR;
3dab1dad 1317 chars++;
a3621e74
YO
1318 if ( uvc < 256 ) {
1319 if ( !trie->charmap[ uvc ] ) {
1320 trie->charmap[ uvc ]=( ++trie->uniquecharcount );
1321 if ( folder )
1322 trie->charmap[ folder[ uvc ] ] = trie->charmap[ uvc ];
3dab1dad 1323 TRIE_STORE_REVCHAR;
a3621e74
YO
1324 }
1325 } else {
1326 SV** svpp;
1327 if ( !trie->widecharmap )
1328 trie->widecharmap = newHV();
1329
1330 svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 1 );
1331
1332 if ( !svpp )
e4584336 1333 Perl_croak( aTHX_ "error creating/fetching widecharmap entry for 0x%"UVXf, uvc );
a3621e74
YO
1334
1335 if ( !SvTRUE( *svpp ) ) {
1336 sv_setiv( *svpp, ++trie->uniquecharcount );
3dab1dad 1337 TRIE_STORE_REVCHAR;
a3621e74
YO
1338 }
1339 }
1340 }
3dab1dad
YO
1341 if( cur == first ) {
1342 trie->minlen=chars;
1343 trie->maxlen=chars;
1344 } else if (chars < trie->minlen) {
1345 trie->minlen=chars;
1346 } else if (chars > trie->maxlen) {
1347 trie->maxlen=chars;
1348 }
1349
a3621e74
YO
1350 } /* end first pass */
1351 DEBUG_TRIE_COMPILE_r(
3dab1dad
YO
1352 PerlIO_printf( Perl_debug_log, "%*sTRIE(%s): W:%d C:%d Uq:%d Min:%d Max:%d\n",
1353 (int)depth * 2 + 2,"",
85c3142d 1354 ( trie->widecharmap ? "UTF8" : "NATIVE" ), (int)word_count,
be8e71aa
YO
1355 (int)TRIE_CHARCOUNT(trie), trie->uniquecharcount,
1356 (int)trie->minlen, (int)trie->maxlen )
a3621e74 1357 );
786e8c11 1358 Newxz( trie->wordlen, word_count, U32 );
a3621e74
YO
1359
1360 /*
1361 We now know what we are dealing with in terms of unique chars and
1362 string sizes so we can calculate how much memory a naive
0111c4fd
RGS
1363 representation using a flat table will take. If it's over a reasonable
1364 limit (as specified by ${^RE_TRIE_MAXBUF}) we use a more memory
a3621e74
YO
1365 conservative but potentially much slower representation using an array
1366 of lists.
1367
1368 At the end we convert both representations into the same compressed
1369 form that will be used in regexec.c for matching with. The latter
1370 is a form that cannot be used to construct with but has memory
1371 properties similar to the list form and access properties similar
1372 to the table form making it both suitable for fast searches and
1373 small enough that its feasable to store for the duration of a program.
1374
1375 See the comment in the code where the compressed table is produced
1376 inplace from the flat tabe representation for an explanation of how
1377 the compression works.
1378
1379 */
1380
1381
3dab1dad 1382 if ( (IV)( ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1) > SvIV(re_trie_maxbuff) ) {
a3621e74
YO
1383 /*
1384 Second Pass -- Array Of Lists Representation
1385
1386 Each state will be represented by a list of charid:state records
1387 (reg_trie_trans_le) the first such element holds the CUR and LEN
1388 points of the allocated array. (See defines above).
1389
1390 We build the initial structure using the lists, and then convert
1391 it into the compressed table form which allows faster lookups
1392 (but cant be modified once converted).
a3621e74
YO
1393 */
1394
a3621e74
YO
1395 STRLEN transcount = 1;
1396
3dab1dad 1397 Newxz( trie->states, TRIE_CHARCOUNT(trie) + 2, reg_trie_state );
a3621e74
YO
1398 TRIE_LIST_NEW(1);
1399 next_alloc = 2;
1400
1401 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1402
c445ea15
AL
1403 regnode * const noper = NEXTOPER( cur );
1404 U8 *uc = (U8*)STRING( noper );
1405 const U8 * const e = uc + STR_LEN( noper );
1406 U32 state = 1; /* required init */
1407 U16 charid = 0; /* sanity init */
1408 U8 *scan = (U8*)NULL; /* sanity init */
1409 STRLEN foldlen = 0; /* required init */
07be1b83 1410 U32 wordlen = 0; /* required init */
c445ea15
AL
1411 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1412
3dab1dad 1413 if (OP(noper) != NOTHING) {
786e8c11 1414 for ( ; uc < e ; uc += len ) {
c445ea15 1415
786e8c11 1416 TRIE_READ_CHAR;
c445ea15 1417
786e8c11
YO
1418 if ( uvc < 256 ) {
1419 charid = trie->charmap[ uvc ];
c445ea15 1420 } else {
786e8c11
YO
1421 SV** const svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 0);
1422 if ( !svpp ) {
1423 charid = 0;
1424 } else {
1425 charid=(U16)SvIV( *svpp );
1426 }
c445ea15 1427 }
786e8c11
YO
1428 /* charid is now 0 if we dont know the char read, or nonzero if we do */
1429 if ( charid ) {
a3621e74 1430
786e8c11
YO
1431 U16 check;
1432 U32 newstate = 0;
a3621e74 1433
786e8c11
YO
1434 charid--;
1435 if ( !trie->states[ state ].trans.list ) {
1436 TRIE_LIST_NEW( state );
c445ea15 1437 }
786e8c11
YO
1438 for ( check = 1; check <= TRIE_LIST_USED( state ); check++ ) {
1439 if ( TRIE_LIST_ITEM( state, check ).forid == charid ) {
1440 newstate = TRIE_LIST_ITEM( state, check ).newstate;
1441 break;
1442 }
1443 }
1444 if ( ! newstate ) {
1445 newstate = next_alloc++;
1446 TRIE_LIST_PUSH( state, charid, newstate );
1447 transcount++;
1448 }
1449 state = newstate;
1450 } else {
1451 Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc );
c445ea15 1452 }
a28509cc 1453 }
c445ea15 1454 }
3dab1dad 1455 TRIE_HANDLE_WORD(state);
a3621e74
YO
1456
1457 } /* end second pass */
1458
786e8c11 1459 trie->laststate = next_alloc;
a3621e74
YO
1460 Renew( trie->states, next_alloc, reg_trie_state );
1461
3dab1dad
YO
1462 /* and now dump it out before we compress it */
1463 DEBUG_TRIE_COMPILE_MORE_r(
1464 dump_trie_interim_list(trie,next_alloc,depth+1)
a3621e74 1465 );
a3621e74 1466
a02a5408 1467 Newxz( trie->trans, transcount ,reg_trie_trans );
a3621e74
YO
1468 {
1469 U32 state;
a3621e74
YO
1470 U32 tp = 0;
1471 U32 zp = 0;
1472
1473
1474 for( state=1 ; state < next_alloc ; state ++ ) {
1475 U32 base=0;
1476
1477 /*
1478 DEBUG_TRIE_COMPILE_MORE_r(
1479 PerlIO_printf( Perl_debug_log, "tp: %d zp: %d ",tp,zp)
1480 );
1481 */
1482
1483 if (trie->states[state].trans.list) {
1484 U16 minid=TRIE_LIST_ITEM( state, 1).forid;
1485 U16 maxid=minid;
a28509cc 1486 U16 idx;
a3621e74
YO
1487
1488 for( idx = 2 ; idx <= TRIE_LIST_USED( state ) ; idx++ ) {
c445ea15
AL
1489 const U16 forid = TRIE_LIST_ITEM( state, idx).forid;
1490 if ( forid < minid ) {
1491 minid=forid;
1492 } else if ( forid > maxid ) {
1493 maxid=forid;
1494 }
a3621e74
YO
1495 }
1496 if ( transcount < tp + maxid - minid + 1) {
1497 transcount *= 2;
1498 Renew( trie->trans, transcount, reg_trie_trans );
1499 Zero( trie->trans + (transcount / 2), transcount / 2 , reg_trie_trans );
1500 }
1501 base = trie->uniquecharcount + tp - minid;
1502 if ( maxid == minid ) {
1503 U32 set = 0;
1504 for ( ; zp < tp ; zp++ ) {
1505 if ( ! trie->trans[ zp ].next ) {
1506 base = trie->uniquecharcount + zp - minid;
1507 trie->trans[ zp ].next = TRIE_LIST_ITEM( state, 1).newstate;
1508 trie->trans[ zp ].check = state;
1509 set = 1;
1510 break;
1511 }
1512 }
1513 if ( !set ) {
1514 trie->trans[ tp ].next = TRIE_LIST_ITEM( state, 1).newstate;
1515 trie->trans[ tp ].check = state;
1516 tp++;
1517 zp = tp;
1518 }
1519 } else {
1520 for ( idx=1; idx <= TRIE_LIST_USED( state ) ; idx++ ) {
c445ea15 1521 const U32 tid = base - trie->uniquecharcount + TRIE_LIST_ITEM( state, idx ).forid;
a3621e74
YO
1522 trie->trans[ tid ].next = TRIE_LIST_ITEM( state, idx ).newstate;
1523 trie->trans[ tid ].check = state;
1524 }
1525 tp += ( maxid - minid + 1 );
1526 }
1527 Safefree(trie->states[ state ].trans.list);
1528 }
1529 /*
1530 DEBUG_TRIE_COMPILE_MORE_r(
1531 PerlIO_printf( Perl_debug_log, " base: %d\n",base);
1532 );
1533 */
1534 trie->states[ state ].trans.base=base;
1535 }
cc601c31 1536 trie->lasttrans = tp + 1;
a3621e74
YO
1537 }
1538 } else {
1539 /*
1540 Second Pass -- Flat Table Representation.
1541
1542 we dont use the 0 slot of either trans[] or states[] so we add 1 to each.
1543 We know that we will need Charcount+1 trans at most to store the data
1544 (one row per char at worst case) So we preallocate both structures
1545 assuming worst case.
1546
1547 We then construct the trie using only the .next slots of the entry
1548 structs.
1549
1550 We use the .check field of the first entry of the node temporarily to
1551 make compression both faster and easier by keeping track of how many non
1552 zero fields are in the node.
1553
1554 Since trans are numbered from 1 any 0 pointer in the table is a FAIL
1555 transition.
1556
1557 There are two terms at use here: state as a TRIE_NODEIDX() which is a
1558 number representing the first entry of the node, and state as a
1559 TRIE_NODENUM() which is the trans number. state 1 is TRIE_NODEIDX(1) and
1560 TRIE_NODENUM(1), state 2 is TRIE_NODEIDX(2) and TRIE_NODENUM(3) if there
1561 are 2 entrys per node. eg:
1562
1563 A B A B
1564 1. 2 4 1. 3 7
1565 2. 0 3 3. 0 5
1566 3. 0 0 5. 0 0
1567 4. 0 0 7. 0 0
1568
1569 The table is internally in the right hand, idx form. However as we also
1570 have to deal with the states array which is indexed by nodenum we have to
1571 use TRIE_NODENUM() to convert.
1572
1573 */
1574
3dab1dad
YO
1575
1576 Newxz( trie->trans, ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1,
a3621e74 1577 reg_trie_trans );
3dab1dad 1578 Newxz( trie->states, TRIE_CHARCOUNT(trie) + 2, reg_trie_state );
a3621e74
YO
1579 next_alloc = trie->uniquecharcount + 1;
1580
3dab1dad 1581
a3621e74
YO
1582 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1583
c445ea15 1584 regnode * const noper = NEXTOPER( cur );
a28509cc
AL
1585 const U8 *uc = (U8*)STRING( noper );
1586 const U8 * const e = uc + STR_LEN( noper );
a3621e74
YO
1587
1588 U32 state = 1; /* required init */
1589
1590 U16 charid = 0; /* sanity init */
1591 U32 accept_state = 0; /* sanity init */
1592 U8 *scan = (U8*)NULL; /* sanity init */
1593
1594 STRLEN foldlen = 0; /* required init */
07be1b83 1595 U32 wordlen = 0; /* required init */
a3621e74
YO
1596 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1597
3dab1dad 1598 if ( OP(noper) != NOTHING ) {
786e8c11 1599 for ( ; uc < e ; uc += len ) {
a3621e74 1600
786e8c11 1601 TRIE_READ_CHAR;
a3621e74 1602
786e8c11
YO
1603 if ( uvc < 256 ) {
1604 charid = trie->charmap[ uvc ];
1605 } else {
1606 SV* const * const svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 0);
1607 charid = svpp ? (U16)SvIV(*svpp) : 0;
a3621e74 1608 }
786e8c11
YO
1609 if ( charid ) {
1610 charid--;
1611 if ( !trie->trans[ state + charid ].next ) {
1612 trie->trans[ state + charid ].next = next_alloc;
1613 trie->trans[ state ].check++;
1614 next_alloc += trie->uniquecharcount;
1615 }
1616 state = trie->trans[ state + charid ].next;
1617 } else {
1618 Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc );
1619 }
1620 /* charid is now 0 if we dont know the char read, or nonzero if we do */
a3621e74 1621 }
a3621e74 1622 }
3dab1dad
YO
1623 accept_state = TRIE_NODENUM( state );
1624 TRIE_HANDLE_WORD(accept_state);
a3621e74
YO
1625
1626 } /* end second pass */
1627
3dab1dad
YO
1628 /* and now dump it out before we compress it */
1629 DEBUG_TRIE_COMPILE_MORE_r(
1630 dump_trie_interim_table(trie,next_alloc,depth+1)
1631 );
a3621e74 1632
a3621e74
YO
1633 {
1634 /*
1635 * Inplace compress the table.*
1636
1637 For sparse data sets the table constructed by the trie algorithm will
1638 be mostly 0/FAIL transitions or to put it another way mostly empty.
1639 (Note that leaf nodes will not contain any transitions.)
1640
1641 This algorithm compresses the tables by eliminating most such
1642 transitions, at the cost of a modest bit of extra work during lookup:
1643
1644 - Each states[] entry contains a .base field which indicates the
1645 index in the state[] array wheres its transition data is stored.
1646
1647 - If .base is 0 there are no valid transitions from that node.
1648
1649 - If .base is nonzero then charid is added to it to find an entry in
1650 the trans array.
1651
1652 -If trans[states[state].base+charid].check!=state then the
1653 transition is taken to be a 0/Fail transition. Thus if there are fail
1654 transitions at the front of the node then the .base offset will point
1655 somewhere inside the previous nodes data (or maybe even into a node
1656 even earlier), but the .check field determines if the transition is
1657 valid.
1658
786e8c11 1659 XXX - wrong maybe?
a3621e74
YO
1660 The following process inplace converts the table to the compressed
1661 table: We first do not compress the root node 1,and mark its all its
1662 .check pointers as 1 and set its .base pointer as 1 as well. This
1663 allows to do a DFA construction from the compressed table later, and
1664 ensures that any .base pointers we calculate later are greater than
1665 0.
1666
1667 - We set 'pos' to indicate the first entry of the second node.
1668
1669 - We then iterate over the columns of the node, finding the first and
1670 last used entry at l and m. We then copy l..m into pos..(pos+m-l),
1671 and set the .check pointers accordingly, and advance pos
1672 appropriately and repreat for the next node. Note that when we copy
1673 the next pointers we have to convert them from the original
1674 NODEIDX form to NODENUM form as the former is not valid post
1675 compression.
1676
1677 - If a node has no transitions used we mark its base as 0 and do not
1678 advance the pos pointer.
1679
1680 - If a node only has one transition we use a second pointer into the
1681 structure to fill in allocated fail transitions from other states.
1682 This pointer is independent of the main pointer and scans forward
1683 looking for null transitions that are allocated to a state. When it
1684 finds one it writes the single transition into the "hole". If the
786e8c11 1685 pointer doesnt find one the single transition is appended as normal.
a3621e74
YO
1686
1687 - Once compressed we can Renew/realloc the structures to release the
1688 excess space.
1689
1690 See "Table-Compression Methods" in sec 3.9 of the Red Dragon,
1691 specifically Fig 3.47 and the associated pseudocode.
1692
1693 demq
1694 */
a3b680e6 1695 const U32 laststate = TRIE_NODENUM( next_alloc );
a28509cc 1696 U32 state, charid;
a3621e74 1697 U32 pos = 0, zp=0;
786e8c11 1698 trie->laststate = laststate;
a3621e74
YO
1699
1700 for ( state = 1 ; state < laststate ; state++ ) {
1701 U8 flag = 0;
a28509cc
AL
1702 const U32 stateidx = TRIE_NODEIDX( state );
1703 const U32 o_used = trie->trans[ stateidx ].check;
1704 U32 used = trie->trans[ stateidx ].check;
a3621e74
YO
1705 trie->trans[ stateidx ].check = 0;
1706
1707 for ( charid = 0 ; used && charid < trie->uniquecharcount ; charid++ ) {
1708 if ( flag || trie->trans[ stateidx + charid ].next ) {
1709 if ( trie->trans[ stateidx + charid ].next ) {
1710 if (o_used == 1) {
1711 for ( ; zp < pos ; zp++ ) {
1712 if ( ! trie->trans[ zp ].next ) {
1713 break;
1714 }
1715 }
1716 trie->states[ state ].trans.base = zp + trie->uniquecharcount - charid ;
1717 trie->trans[ zp ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next );
1718 trie->trans[ zp ].check = state;
1719 if ( ++zp > pos ) pos = zp;
1720 break;
1721 }
1722 used--;
1723 }
1724 if ( !flag ) {
1725 flag = 1;
1726 trie->states[ state ].trans.base = pos + trie->uniquecharcount - charid ;
1727 }
1728 trie->trans[ pos ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next );
1729 trie->trans[ pos ].check = state;
1730 pos++;
1731 }
1732 }
1733 }
cc601c31 1734 trie->lasttrans = pos + 1;
a3621e74
YO
1735 Renew( trie->states, laststate + 1, reg_trie_state);
1736 DEBUG_TRIE_COMPILE_MORE_r(
e4584336 1737 PerlIO_printf( Perl_debug_log,
3dab1dad
YO
1738 "%*sAlloc: %d Orig: %"IVdf" elements, Final:%"IVdf". Savings of %%%5.2f\n",
1739 (int)depth * 2 + 2,"",
1740 (int)( ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1 ),
5d7488b2
AL
1741 (IV)next_alloc,
1742 (IV)pos,
a3621e74
YO
1743 ( ( next_alloc - pos ) * 100 ) / (double)next_alloc );
1744 );
1745
1746 } /* end table compress */
1747 }
cc601c31
YO
1748 /* resize the trans array to remove unused space */
1749 Renew( trie->trans, trie->lasttrans, reg_trie_trans);
a3621e74 1750
3dab1dad
YO
1751 /* and now dump out the compressed format */
1752 DEBUG_TRIE_COMPILE_r(
1753 dump_trie(trie,depth+1)
1754 );
07be1b83 1755
3dab1dad
YO
1756 { /* Modify the program and insert the new TRIE node*/
1757 regnode *convert;
1758 U8 nodetype =(U8)(flags & 0xFF);
1759 char *str=NULL;
786e8c11 1760
07be1b83 1761#ifdef DEBUGGING
a5ca303d 1762 regnode *optimize;
b57a0404
JH
1763 U32 mjd_offset = 0;
1764 U32 mjd_nodelen = 0;
07be1b83 1765#endif
a3621e74 1766 /*
3dab1dad
YO
1767 This means we convert either the first branch or the first Exact,
1768 depending on whether the thing following (in 'last') is a branch
1769 or not and whther first is the startbranch (ie is it a sub part of
1770 the alternation or is it the whole thing.)
1771 Assuming its a sub part we conver the EXACT otherwise we convert
1772 the whole branch sequence, including the first.
a3621e74 1773 */
3dab1dad
YO
1774 /* Find the node we are going to overwrite */
1775 if ( first == startbranch && OP( last ) != BRANCH ) {
07be1b83 1776 /* whole branch chain */
3dab1dad 1777 convert = first;
07be1b83
YO
1778 DEBUG_r({
1779 const regnode *nop = NEXTOPER( convert );
1780 mjd_offset= Node_Offset((nop));
1781 mjd_nodelen= Node_Length((nop));
1782 });
1783 } else {
1784 /* branch sub-chain */
3dab1dad
YO
1785 convert = NEXTOPER( first );
1786 NEXT_OFF( first ) = (U16)(last - first);
07be1b83
YO
1787 DEBUG_r({
1788 mjd_offset= Node_Offset((convert));
1789 mjd_nodelen= Node_Length((convert));
1790 });
1791 }
1792 DEBUG_OPTIMISE_r(
1793 PerlIO_printf(Perl_debug_log, "%*sMJD offset:%"UVuf" MJD length:%"UVuf"\n",
1794 (int)depth * 2 + 2, "",
786e8c11 1795 (UV)mjd_offset, (UV)mjd_nodelen)
07be1b83 1796 );
a3621e74 1797
3dab1dad
YO
1798 /* But first we check to see if there is a common prefix we can
1799 split out as an EXACT and put in front of the TRIE node. */
1800 trie->startstate= 1;
786e8c11 1801 if ( trie->bitmap && !trie->widecharmap && !trie->jump ) {
3dab1dad
YO
1802 U32 state;
1803 DEBUG_OPTIMISE_r(
8e11feef
RGS
1804 PerlIO_printf(Perl_debug_log, "%*sLaststate:%"UVuf"\n",
1805 (int)depth * 2 + 2, "",
786e8c11 1806 (UV)trie->laststate)
8e11feef 1807 );
786e8c11 1808 for ( state = 1 ; state < trie->laststate-1 ; state++ ) {
a3621e74 1809 U32 ofs = 0;
8e11feef
RGS
1810 I32 idx = -1;
1811 U32 count = 0;
1812 const U32 base = trie->states[ state ].trans.base;
a3621e74 1813
3dab1dad 1814 if ( trie->states[state].wordnum )
8e11feef 1815 count = 1;
a3621e74 1816
8e11feef 1817 for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) {
cc601c31
YO
1818 if ( ( base + ofs >= trie->uniquecharcount ) &&
1819 ( base + ofs - trie->uniquecharcount < trie->lasttrans ) &&
a3621e74
YO
1820 trie->trans[ base + ofs - trie->uniquecharcount ].check == state )
1821 {
3dab1dad 1822 if ( ++count > 1 ) {
8e11feef 1823 SV **tmp = av_fetch( TRIE_REVCHARMAP(trie), ofs, 0);
07be1b83 1824 const U8 *ch = (U8*)SvPV_nolen_const( *tmp );
8e11feef 1825 if ( state == 1 ) break;
3dab1dad
YO
1826 if ( count == 2 ) {
1827 Zero(trie->bitmap, ANYOF_BITMAP_SIZE, char);
1828 DEBUG_OPTIMISE_r(
8e11feef
RGS
1829 PerlIO_printf(Perl_debug_log,
1830 "%*sNew Start State=%"UVuf" Class: [",
1831 (int)depth * 2 + 2, "",
786e8c11 1832 (UV)state));
be8e71aa
YO
1833 if (idx >= 0) {
1834 SV ** const tmp = av_fetch( TRIE_REVCHARMAP(trie), idx, 0);
1835 const U8 * const ch = (U8*)SvPV_nolen_const( *tmp );
8e11feef 1836
3dab1dad 1837 TRIE_BITMAP_SET(trie,*ch);
8e11feef
RGS
1838 if ( folder )
1839 TRIE_BITMAP_SET(trie, folder[ *ch ]);
3dab1dad 1840 DEBUG_OPTIMISE_r(
07be1b83 1841 PerlIO_printf(Perl_debug_log, (char*)ch)
3dab1dad 1842 );
8e11feef
RGS
1843 }
1844 }
1845 TRIE_BITMAP_SET(trie,*ch);
1846 if ( folder )
1847 TRIE_BITMAP_SET(trie,folder[ *ch ]);
1848 DEBUG_OPTIMISE_r(PerlIO_printf( Perl_debug_log,"%s", ch));
1849 }
1850 idx = ofs;
1851 }
3dab1dad
YO
1852 }
1853 if ( count == 1 ) {
1854 SV **tmp = av_fetch( TRIE_REVCHARMAP(trie), idx, 0);
8e11feef 1855 const char *ch = SvPV_nolen_const( *tmp );
3dab1dad 1856 DEBUG_OPTIMISE_r(
8e11feef
RGS
1857 PerlIO_printf( Perl_debug_log,
1858 "%*sPrefix State: %"UVuf" Idx:%"UVuf" Char='%s'\n",
1859 (int)depth * 2 + 2, "",
786e8c11 1860 (UV)state, (UV)idx, ch)
3dab1dad
YO
1861 );
1862 if ( state==1 ) {
1863 OP( convert ) = nodetype;
1864 str=STRING(convert);
1865 STR_LEN(convert)=0;
1866 }
1867 *str++=*ch;
1868 STR_LEN(convert)++;
a3621e74 1869
8e11feef 1870 } else {
f9049ba1 1871#ifdef DEBUGGING
8e11feef
RGS
1872 if (state>1)
1873 DEBUG_OPTIMISE_r(PerlIO_printf( Perl_debug_log,"]\n"));
f9049ba1 1874#endif
8e11feef
RGS
1875 break;
1876 }
1877 }
3dab1dad 1878 if (str) {
8e11feef 1879 regnode *n = convert+NODE_SZ_STR(convert);
07be1b83 1880 NEXT_OFF(convert) = NODE_SZ_STR(convert);
8e11feef 1881 trie->startstate = state;
07be1b83
YO
1882 trie->minlen -= (state - 1);
1883 trie->maxlen -= (state - 1);
1884 DEBUG_r({
1885 regnode *fix = convert;
1886 mjd_nodelen++;
1887 Set_Node_Offset_Length(convert, mjd_offset, state - 1);
1888 while( ++fix < n ) {
1889 Set_Node_Offset_Length(fix, 0, 0);
1890 }
1891 });
8e11feef
RGS
1892 if (trie->maxlen) {
1893 convert = n;
1894 } else {
3dab1dad 1895 NEXT_OFF(convert) = (U16)(tail - convert);
a5ca303d 1896 DEBUG_r(optimize= n);
3dab1dad
YO
1897 }
1898 }
1899 }
a5ca303d
YO
1900 if (!jumper)
1901 jumper = last;
3dab1dad 1902 if ( trie->maxlen ) {
8e11feef
RGS
1903 NEXT_OFF( convert ) = (U16)(tail - convert);
1904 ARG_SET( convert, data_slot );
786e8c11
YO
1905 /* Store the offset to the first unabsorbed branch in
1906 jump[0], which is otherwise unused by the jump logic.
1907 We use this when dumping a trie and during optimisation. */
1908 if (trie->jump)
1909 trie->jump[0] = (U16)(tail - nextbranch);
a5ca303d 1910
786e8c11
YO
1911 /* XXXX */
1912 if ( !trie->states[trie->startstate].wordnum && trie->bitmap &&
1de06328 1913 ( (char *)jumper - (char *)convert) >= (int)sizeof(struct regnode_charclass) )
786e8c11
YO
1914 {
1915 OP( convert ) = TRIEC;
1916 Copy(trie->bitmap, ((struct regnode_charclass *)convert)->bitmap, ANYOF_BITMAP_SIZE, char);
1917 Safefree(trie->bitmap);
1918 trie->bitmap= NULL;
1919 } else
1920 OP( convert ) = TRIE;
a3621e74 1921
3dab1dad
YO
1922 /* store the type in the flags */
1923 convert->flags = nodetype;
a5ca303d
YO
1924 DEBUG_r({
1925 optimize = convert
1926 + NODE_STEP_REGNODE
1927 + regarglen[ OP( convert ) ];
1928 });
1929 /* XXX We really should free up the resource in trie now,
1930 as we won't use them - (which resources?) dmq */
3dab1dad 1931 }
a3621e74
YO
1932 /* needed for dumping*/
1933 DEBUG_r({
07be1b83
YO
1934 regnode *opt = convert;
1935 while (++opt<optimize) {
1936 Set_Node_Offset_Length(opt,0,0);
1937 }
786e8c11
YO
1938 /*
1939 Try to clean up some of the debris left after the
1940 optimisation.
a3621e74 1941 */
786e8c11 1942 while( optimize < jumper ) {
07be1b83 1943 mjd_nodelen += Node_Length((optimize));
a3621e74 1944 OP( optimize ) = OPTIMIZED;
07be1b83 1945 Set_Node_Offset_Length(optimize,0,0);
a3621e74
YO
1946 optimize++;
1947 }
07be1b83 1948 Set_Node_Offset_Length(convert,mjd_offset,mjd_nodelen);
a3621e74
YO
1949 });
1950 } /* end node insert */
07be1b83 1951#ifndef DEBUGGING
6e8b4190 1952 SvREFCNT_dec(TRIE_REVCHARMAP(trie));
07be1b83 1953#endif
786e8c11
YO
1954 return trie->jump
1955 ? MADE_JUMP_TRIE
1956 : trie->startstate>1
1957 ? MADE_EXACT_TRIE
1958 : MADE_TRIE;
1959}
1960
1961STATIC void
1962S_make_trie_failtable(pTHX_ RExC_state_t *pRExC_state, regnode *source, regnode *stclass, U32 depth)
1963{
1964/* The Trie is constructed and compressed now so we can build a fail array now if its needed
1965
1966 This is basically the Aho-Corasick algorithm. Its from exercise 3.31 and 3.32 in the
1967 "Red Dragon" -- Compilers, principles, techniques, and tools. Aho, Sethi, Ullman 1985/88
1968 ISBN 0-201-10088-6
1969
1970 We find the fail state for each state in the trie, this state is the longest proper
1971 suffix of the current states 'word' that is also a proper prefix of another word in our
1972 trie. State 1 represents the word '' and is the thus the default fail state. This allows
1973 the DFA not to have to restart after its tried and failed a word at a given point, it
1974 simply continues as though it had been matching the other word in the first place.
1975 Consider
1976 'abcdgu'=~/abcdefg|cdgu/
1977 When we get to 'd' we are still matching the first word, we would encounter 'g' which would
1978 fail, which would bring use to the state representing 'd' in the second word where we would
1979 try 'g' and succeed, prodceding to match 'cdgu'.
1980 */
1981 /* add a fail transition */
1982 reg_trie_data *trie=(reg_trie_data *)RExC_rx->data->data[ARG(source)];
1983 U32 *q;
1984 const U32 ucharcount = trie->uniquecharcount;
1985 const U32 numstates = trie->laststate;
1986 const U32 ubound = trie->lasttrans + ucharcount;
1987 U32 q_read = 0;
1988 U32 q_write = 0;
1989 U32 charid;
1990 U32 base = trie->states[ 1 ].trans.base;
1991 U32 *fail;
1992 reg_ac_data *aho;
1993 const U32 data_slot = add_data( pRExC_state, 1, "T" );
1994 GET_RE_DEBUG_FLAGS_DECL;
1995#ifndef DEBUGGING
1996 PERL_UNUSED_ARG(depth);
1997#endif
1998
1999
2000 ARG_SET( stclass, data_slot );
2001 Newxz( aho, 1, reg_ac_data );
2002 RExC_rx->data->data[ data_slot ] = (void*)aho;
2003 aho->trie=trie;
2004 aho->states=(reg_trie_state *)savepvn((const char*)trie->states,
2005 (trie->laststate+1)*sizeof(reg_trie_state));
2006 Newxz( q, numstates, U32);
2007 Newxz( aho->fail, numstates, U32 );
2008 aho->refcount = 1;
2009 fail = aho->fail;
2010 /* initialize fail[0..1] to be 1 so that we always have
2011 a valid final fail state */
2012 fail[ 0 ] = fail[ 1 ] = 1;
2013
2014 for ( charid = 0; charid < ucharcount ; charid++ ) {
2015 const U32 newstate = TRIE_TRANS_STATE( 1, base, ucharcount, charid, 0 );
2016 if ( newstate ) {
2017 q[ q_write ] = newstate;
2018 /* set to point at the root */
2019 fail[ q[ q_write++ ] ]=1;
2020 }
2021 }
2022 while ( q_read < q_write) {
2023 const U32 cur = q[ q_read++ % numstates ];
2024 base = trie->states[ cur ].trans.base;
2025
2026 for ( charid = 0 ; charid < ucharcount ; charid++ ) {
2027 const U32 ch_state = TRIE_TRANS_STATE( cur, base, ucharcount, charid, 1 );
2028 if (ch_state) {
2029 U32 fail_state = cur;
2030 U32 fail_base;
2031 do {
2032 fail_state = fail[ fail_state ];
2033 fail_base = aho->states[ fail_state ].trans.base;
2034 } while ( !TRIE_TRANS_STATE( fail_state, fail_base, ucharcount, charid, 1 ) );
2035
2036 fail_state = TRIE_TRANS_STATE( fail_state, fail_base, ucharcount, charid, 1 );
2037 fail[ ch_state ] = fail_state;
2038 if ( !aho->states[ ch_state ].wordnum && aho->states[ fail_state ].wordnum )
2039 {
2040 aho->states[ ch_state ].wordnum = aho->states[ fail_state ].wordnum;
2041 }
2042 q[ q_write++ % numstates] = ch_state;
2043 }
2044 }
2045 }
2046 /* restore fail[0..1] to 0 so that we "fall out" of the AC loop
2047 when we fail in state 1, this allows us to use the
2048 charclass scan to find a valid start char. This is based on the principle
2049 that theres a good chance the string being searched contains lots of stuff
2050 that cant be a start char.
2051 */
2052 fail[ 0 ] = fail[ 1 ] = 0;
2053 DEBUG_TRIE_COMPILE_r({
2054 PerlIO_printf(Perl_debug_log, "%*sStclass Failtable: 0", (int)(depth * 2), "");
2055 for( q_read=1; q_read<numstates; q_read++ ) {
2056 PerlIO_printf(Perl_debug_log, ", %"UVuf, (UV)fail[q_read]);
2057 }
2058 PerlIO_printf(Perl_debug_log, "\n");
2059 });
2060 Safefree(q);
2061 /*RExC_seen |= REG_SEEN_TRIEDFA;*/
a3621e74
YO
2062}
2063
786e8c11 2064
a3621e74 2065/*
5d1c421c
JH
2066 * There are strange code-generation bugs caused on sparc64 by gcc-2.95.2.
2067 * These need to be revisited when a newer toolchain becomes available.
2068 */
2069#if defined(__sparc64__) && defined(__GNUC__)
2070# if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
2071# undef SPARC64_GCC_WORKAROUND
2072# define SPARC64_GCC_WORKAROUND 1
2073# endif
2074#endif
2075
07be1b83
YO
2076#define DEBUG_PEEP(str,scan,depth) \
2077 DEBUG_OPTIMISE_r({ \
2078 SV * const mysv=sv_newmortal(); \
2079 regnode *Next = regnext(scan); \
2080 regprop(RExC_rx, mysv, scan); \
2081 PerlIO_printf(Perl_debug_log, "%*s" str ">%3d: %s [%d]\n", \
2082 (int)depth*2, "", REG_NODE_NUM(scan), SvPV_nolen_const(mysv),\
2083 Next ? (REG_NODE_NUM(Next)) : 0 ); \
2084 });
2085
1de06328
YO
2086
2087
2088
2089
07be1b83
YO
2090#define JOIN_EXACT(scan,min,flags) \
2091 if (PL_regkind[OP(scan)] == EXACT) \
2092 join_exact(pRExC_state,(scan),(min),(flags),NULL,depth+1)
2093
be8e71aa 2094STATIC U32
07be1b83
YO
2095S_join_exact(pTHX_ RExC_state_t *pRExC_state, regnode *scan, I32 *min, U32 flags,regnode *val, U32 depth) {
2096 /* Merge several consecutive EXACTish nodes into one. */
2097 regnode *n = regnext(scan);
2098 U32 stringok = 1;
2099 regnode *next = scan + NODE_SZ_STR(scan);
2100 U32 merged = 0;
2101 U32 stopnow = 0;
2102#ifdef DEBUGGING
2103 regnode *stop = scan;
72f13be8 2104 GET_RE_DEBUG_FLAGS_DECL;
f9049ba1 2105#else
d47053eb
RGS
2106 PERL_UNUSED_ARG(depth);
2107#endif
2108#ifndef EXPERIMENTAL_INPLACESCAN
f9049ba1
SP
2109 PERL_UNUSED_ARG(flags);
2110 PERL_UNUSED_ARG(val);
07be1b83 2111#endif
07be1b83
YO
2112 DEBUG_PEEP("join",scan,depth);
2113
2114 /* Skip NOTHING, merge EXACT*. */
2115 while (n &&
2116 ( PL_regkind[OP(n)] == NOTHING ||
2117 (stringok && (OP(n) == OP(scan))))
2118 && NEXT_OFF(n)
2119 && NEXT_OFF(scan) + NEXT_OFF(n) < I16_MAX) {
2120
2121 if (OP(n) == TAIL || n > next)
2122 stringok = 0;
2123 if (PL_regkind[OP(n)] == NOTHING) {
07be1b83
YO
2124 DEBUG_PEEP("skip:",n,depth);
2125 NEXT_OFF(scan) += NEXT_OFF(n);
2126 next = n + NODE_STEP_REGNODE;
2127#ifdef DEBUGGING
2128 if (stringok)
2129 stop = n;
2130#endif
2131 n = regnext(n);
2132 }
2133 else if (stringok) {
786e8c11 2134 const unsigned int oldl = STR_LEN(scan);
07be1b83
YO
2135 regnode * const nnext = regnext(n);
2136
2137 DEBUG_PEEP("merg",n,depth);
2138
2139 merged++;
2140 if (oldl + STR_LEN(n) > U8_MAX)
2141 break;
2142 NEXT_OFF(scan) += NEXT_OFF(n);
2143 STR_LEN(scan) += STR_LEN(n);
2144 next = n + NODE_SZ_STR(n);
2145 /* Now we can overwrite *n : */
2146 Move(STRING(n), STRING(scan) + oldl, STR_LEN(n), char);
2147#ifdef DEBUGGING
2148 stop = next - 1;
2149#endif
2150 n = nnext;
2151 if (stopnow) break;
2152 }
2153
d47053eb
RGS
2154#ifdef EXPERIMENTAL_INPLACESCAN
2155 if (flags && !NEXT_OFF(n)) {
2156 DEBUG_PEEP("atch", val, depth);
2157 if (reg_off_by_arg[OP(n)]) {
2158 ARG_SET(n, val - n);
2159 }
2160 else {
2161 NEXT_OFF(n) = val - n;
2162 }
2163 stopnow = 1;
2164 }
07be1b83
YO
2165#endif
2166 }
2167
2168 if (UTF && ( OP(scan) == EXACTF ) && ( STR_LEN(scan) >= 6 ) ) {
2169 /*
2170 Two problematic code points in Unicode casefolding of EXACT nodes:
2171
2172 U+0390 - GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
2173 U+03B0 - GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
2174
2175 which casefold to
2176
2177 Unicode UTF-8
2178
2179 U+03B9 U+0308 U+0301 0xCE 0xB9 0xCC 0x88 0xCC 0x81
2180 U+03C5 U+0308 U+0301 0xCF 0x85 0xCC 0x88 0xCC 0x81
2181
2182 This means that in case-insensitive matching (or "loose matching",
2183 as Unicode calls it), an EXACTF of length six (the UTF-8 encoded byte
2184 length of the above casefolded versions) can match a target string
2185 of length two (the byte length of UTF-8 encoded U+0390 or U+03B0).
2186 This would rather mess up the minimum length computation.
2187
2188 What we'll do is to look for the tail four bytes, and then peek
2189 at the preceding two bytes to see whether we need to decrease
2190 the minimum length by four (six minus two).
2191
2192 Thanks to the design of UTF-8, there cannot be false matches:
2193 A sequence of valid UTF-8 bytes cannot be a subsequence of
2194 another valid sequence of UTF-8 bytes.
2195
2196 */
2197 char * const s0 = STRING(scan), *s, *t;
2198 char * const s1 = s0 + STR_LEN(scan) - 1;
2199 char * const s2 = s1 - 4;
e294cc5d
JH
2200#ifdef EBCDIC /* RD tunifold greek 0390 and 03B0 */
2201 const char t0[] = "\xaf\x49\xaf\x42";
2202#else
07be1b83 2203 const char t0[] = "\xcc\x88\xcc\x81";
e294cc5d 2204#endif
07be1b83
YO
2205 const char * const t1 = t0 + 3;
2206
2207 for (s = s0 + 2;
2208 s < s2 && (t = ninstr(s, s1, t0, t1));
2209 s = t + 4) {
e294cc5d
JH
2210#ifdef EBCDIC
2211 if (((U8)t[-1] == 0x68 && (U8)t[-2] == 0xB4) ||
2212 ((U8)t[-1] == 0x46 && (U8)t[-2] == 0xB5))
2213#else
07be1b83
YO
2214 if (((U8)t[-1] == 0xB9 && (U8)t[-2] == 0xCE) ||
2215 ((U8)t[-1] == 0x85 && (U8)t[-2] == 0xCF))
e294cc5d 2216#endif
07be1b83
YO
2217 *min -= 4;
2218 }
2219 }
2220
2221#ifdef DEBUGGING
2222 /* Allow dumping */
2223 n = scan + NODE_SZ_STR(scan);
2224 while (n <= stop) {
2225 if (PL_regkind[OP(n)] != NOTHING || OP(n) == NOTHING) {
2226 OP(n) = OPTIMIZED;
2227 NEXT_OFF(n) = 0;
2228 }
2229 n++;
2230 }
2231#endif
2232 DEBUG_OPTIMISE_r(if (merged){DEBUG_PEEP("finl",scan,depth)});
2233 return stopnow;
2234}
2235
653099ff
GS
2236/* REx optimizer. Converts nodes into quickier variants "in place".
2237 Finds fixed substrings. */
2238
a0288114 2239/* Stops at toplevel WHILEM as well as at "last". At end *scanp is set
c277df42
IZ
2240 to the position after last scanned or to NULL. */
2241
07be1b83
YO
2242
2243
76e3520e 2244STATIC I32
1de06328
YO
2245S_study_chunk(pTHX_ RExC_state_t *pRExC_state, regnode **scanp,
2246 I32 *minlenp, I32 *deltap,
9a957fbc 2247 regnode *last, scan_data_t *data, U32 flags, U32 depth)
c277df42
IZ
2248 /* scanp: Start here (read-write). */
2249 /* deltap: Write maxlen-minlen here. */
2250 /* last: Stop before this one. */
2251{
97aff369 2252 dVAR;
c277df42
IZ
2253 I32 min = 0, pars = 0, code;
2254 regnode *scan = *scanp, *next;
2255 I32 delta = 0;
2256 int is_inf = (flags & SCF_DO_SUBSTR) && (data->flags & SF_IS_INF);
aca2d497 2257 int is_inf_internal = 0; /* The studied chunk is infinite */
c277df42
IZ
2258 I32 is_par = OP(scan) == OPEN ? ARG(scan) : 0;
2259 scan_data_t data_fake;
653099ff 2260 struct regnode_charclass_class and_with; /* Valid if flags & SCF_DO_STCLASS_OR */
a3621e74 2261 SV *re_trie_maxbuff = NULL;
786e8c11
YO
2262 regnode *first_non_open = scan;
2263
a3621e74
YO
2264
2265 GET_RE_DEBUG_FLAGS_DECL;
13a24bad
YO
2266#ifdef DEBUGGING
2267 StructCopy(&zero_scan_data, &data_fake, scan_data_t);
2268#endif
786e8c11
YO
2269 if ( depth == 0 ) {
2270 while (first_non_open && OP(first_non_open) == OPEN)
2271 first_non_open=regnext(first_non_open);
2272 }
2273
b81d288d 2274
c277df42
IZ
2275 while (scan && OP(scan) != END && scan < last) {
2276 /* Peephole optimizer: */
1de06328 2277 DEBUG_STUDYDATA(data,depth);
07be1b83 2278 DEBUG_PEEP("Peep",scan,depth);
07be1b83 2279 JOIN_EXACT(scan,&min,0);
a3621e74 2280
653099ff
GS
2281 /* Follow the next-chain of the current node and optimize
2282 away all the NOTHINGs from it. */
c277df42 2283 if (OP(scan) != CURLYX) {
a3b680e6 2284 const int max = (reg_off_by_arg[OP(scan)]
048cfca1
GS
2285 ? I32_MAX
2286 /* I32 may be smaller than U16 on CRAYs! */
2287 : (I32_MAX < U16_MAX ? I32_MAX : U16_MAX));
c277df42
IZ
2288 int off = (reg_off_by_arg[OP(scan)] ? ARG(scan) : NEXT_OFF(scan));
2289 int noff;
2290 regnode *n = scan;
b81d288d 2291
c277df42
IZ
2292 /* Skip NOTHING and LONGJMP. */
2293 while ((n = regnext(n))
3dab1dad 2294 && ((PL_regkind[OP(n)] == NOTHING && (noff = NEXT_OFF(n)))
c277df42
IZ
2295 || ((OP(n) == LONGJMP) && (noff = ARG(n))))
2296 && off + noff < max)
2297 off += noff;
2298 if (reg_off_by_arg[OP(scan)])
2299 ARG(scan) = off;
b81d288d 2300 else
c277df42
IZ
2301 NEXT_OFF(scan) = off;
2302 }
a3621e74 2303
07be1b83 2304
3dab1dad 2305
653099ff
GS
2306 /* The principal pseudo-switch. Cannot be a switch, since we
2307 look into several different things. */
b81d288d 2308 if (OP(scan) == BRANCH || OP(scan) == BRANCHJ
c277df42
IZ
2309 || OP(scan) == IFTHEN || OP(scan) == SUSPEND) {
2310 next = regnext(scan);
2311 code = OP(scan);
a3621e74 2312 /* demq: the op(next)==code check is to see if we have "branch-branch" AFAICT */
b81d288d
AB
2313
2314 if (OP(next) == code || code == IFTHEN || code == SUSPEND) {
786e8c11
YO
2315 /* NOTE - There is similar code to this block below for handling
2316 TRIE nodes on a re-study. If you change stuff here check there
2317 too. */
c277df42 2318 I32 max1 = 0, min1 = I32_MAX, num = 0;
653099ff 2319 struct regnode_charclass_class accum;
d4c19fe8 2320 regnode * const startbranch=scan;
c277df42 2321
653099ff 2322 if (flags & SCF_DO_SUBSTR) /* XXXX Add !SUSPEND? */
1de06328 2323 scan_commit(pRExC_state, data, minlenp); /* Cannot merge strings after this. */
653099ff 2324 if (flags & SCF_DO_STCLASS)
830247a4 2325 cl_init_zero(pRExC_state, &accum);
a3621e74 2326
c277df42 2327 while (OP(scan) == code) {
830247a4 2328 I32 deltanext, minnext, f = 0, fake;
653099ff 2329 struct regnode_charclass_class this_class;
c277df42
IZ
2330
2331 num++;
2332 data_fake.flags = 0;
b81d288d 2333 if (data) {
2c2d71f5 2334 data_fake.whilem_c = data->whilem_c;
cb434fcc
IZ
2335 data_fake.last_closep = data->last_closep;
2336 }
2337 else
2338 data_fake.last_closep = &fake;
c277df42
IZ
2339 next = regnext(scan);
2340 scan = NEXTOPER(scan);
2341 if (code != BRANCH)
2342 scan = NEXTOPER(scan);
653099ff 2343 if (flags & SCF_DO_STCLASS) {
830247a4 2344 cl_init(pRExC_state, &this_class);
653099ff
GS
2345 data_fake.start_class = &this_class;
2346 f = SCF_DO_STCLASS_AND;
b81d288d 2347 }
e1901655
IZ
2348 if (flags & SCF_WHILEM_VISITED_POS)
2349 f |= SCF_WHILEM_VISITED_POS;
a3621e74 2350
653099ff 2351 /* we suppose the run is continuous, last=next...*/
1de06328 2352 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext,
a3621e74 2353 next, &data_fake, f,depth+1);
b81d288d 2354 if (min1 > minnext)
c277df42
IZ
2355 min1 = minnext;
2356 if (max1 < minnext + deltanext)
2357 max1 = minnext + deltanext;
2358 if (deltanext == I32_MAX)
aca2d497 2359 is_inf = is_inf_internal = 1;
c277df42
IZ
2360 scan = next;
2361 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
2362 pars++;
3dab1dad
YO
2363 if (data) {
2364 if (data_fake.flags & SF_HAS_EVAL)
07be1b83 2365 data->flags |= SF_HAS_EVAL;
2c2d71f5 2366 data->whilem_c = data_fake.whilem_c;
3dab1dad 2367 }
653099ff 2368 if (flags & SCF_DO_STCLASS)
830247a4 2369 cl_or(pRExC_state, &accum, &this_class);
b81d288d 2370 if (code == SUSPEND)
c277df42
IZ
2371 break;
2372 }
2373 if (code == IFTHEN && num < 2) /* Empty ELSE branch */
2374 min1 = 0;
2375 if (flags & SCF_DO_SUBSTR) {
2376 data->pos_min += min1;
2377 data->pos_delta += max1 - min1;
2378 if (max1 != min1 || is_inf)
2379 data->longest = &(data->longest_float);
2380 }
2381 min += min1;
2382 delta += max1 - min1;
653099ff 2383 if (flags & SCF_DO_STCLASS_OR) {
830247a4 2384 cl_or(pRExC_state, data->start_class, &accum);
653099ff
GS
2385 if (min1) {
2386 cl_and(data->start_class, &and_with);
2387 flags &= ~SCF_DO_STCLASS;
2388 }
2389 }
2390 else if (flags & SCF_DO_STCLASS_AND) {
de0c8cb8
GS
2391 if (min1) {
2392 cl_and(data->start_class, &accum);
653099ff 2393 flags &= ~SCF_DO_STCLASS;
de0c8cb8
GS
2394 }
2395 else {
b81d288d 2396 /* Switch to OR mode: cache the old value of
de0c8cb8
GS
2397 * data->start_class */
2398 StructCopy(data->start_class, &and_with,
2399 struct regnode_charclass_class);
2400 flags &= ~SCF_DO_STCLASS_AND;
2401 StructCopy(&accum, data->start_class,
2402 struct regnode_charclass_class);
2403 flags |= SCF_DO_STCLASS_OR;
2404 data->start_class->flags |= ANYOF_EOS;
2405 }
653099ff 2406 }
a3621e74 2407
786e8c11 2408 if (PERL_ENABLE_TRIE_OPTIMISATION && OP( startbranch ) == BRANCH ) {
a3621e74
YO
2409 /* demq.
2410
2411 Assuming this was/is a branch we are dealing with: 'scan' now
2412 points at the item that follows the branch sequence, whatever
2413 it is. We now start at the beginning of the sequence and look
2414 for subsequences of
2415
786e8c11
YO
2416 BRANCH->EXACT=>x1
2417 BRANCH->EXACT=>x2
2418 tail
a3621e74
YO
2419
2420 which would be constructed from a pattern like /A|LIST|OF|WORDS/
2421
2422 If we can find such a subseqence we need to turn the first
2423 element into a trie and then add the subsequent branch exact
2424 strings to the trie.
2425
2426 We have two cases
2427
786e8c11 2428 1. patterns where the whole set of branch can be converted.
a3621e74 2429
786e8c11 2430 2. patterns where only a subset can be converted.
a3621e74
YO
2431
2432 In case 1 we can replace the whole set with a single regop
2433 for the trie. In case 2 we need to keep the start and end
2434 branchs so
2435
2436 'BRANCH EXACT; BRANCH EXACT; BRANCH X'
2437 becomes BRANCH TRIE; BRANCH X;
2438
786e8c11
YO
2439 There is an additional case, that being where there is a
2440 common prefix, which gets split out into an EXACT like node
2441 preceding the TRIE node.
2442
2443 If x(1..n)==tail then we can do a simple trie, if not we make
2444 a "jump" trie, such that when we match the appropriate word
2445 we "jump" to the appopriate tail node. Essentailly we turn
2446 a nested if into a case structure of sorts.
a3621e74
YO
2447
2448 */
786e8c11 2449
3dab1dad 2450 int made=0;
0111c4fd
RGS
2451 if (!re_trie_maxbuff) {
2452 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1);
2453 if (!SvIOK(re_trie_maxbuff))
2454 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT);
2455 }
786e8c11 2456 if ( SvIV(re_trie_maxbuff)>=0 ) {
a3621e74
YO
2457 regnode *cur;
2458 regnode *first = (regnode *)NULL;
2459 regnode *last = (regnode *)NULL;
2460 regnode *tail = scan;
2461 U8 optype = 0;
2462 U32 count=0;
2463
2464#ifdef DEBUGGING
c445ea15 2465 SV * const mysv = sv_newmortal(); /* for dumping */
a3621e74
YO
2466#endif
2467 /* var tail is used because there may be a TAIL
2468 regop in the way. Ie, the exacts will point to the
2469 thing following the TAIL, but the last branch will
2470 point at the TAIL. So we advance tail. If we
2471 have nested (?:) we may have to move through several
2472 tails.
2473 */
2474
2475 while ( OP( tail ) == TAIL ) {
2476 /* this is the TAIL generated by (?:) */
2477 tail = regnext( tail );
2478 }
2479
3dab1dad 2480
a3621e74 2481 DEBUG_OPTIMISE_r({
32fc9b6a 2482 regprop(RExC_rx, mysv, tail );
3dab1dad
YO
2483 PerlIO_printf( Perl_debug_log, "%*s%s%s\n",
2484 (int)depth * 2 + 2, "",
2485 "Looking for TRIE'able sequences. Tail node is: ",
2486 SvPV_nolen_const( mysv )
a3621e74
YO
2487 );
2488 });
3dab1dad 2489
a3621e74
YO
2490 /*
2491
2492 step through the branches, cur represents each
2493 branch, noper is the first thing to be matched
2494 as part of that branch and noper_next is the
2495 regnext() of that node. if noper is an EXACT
2496 and noper_next is the same as scan (our current
2497 position in the regex) then the EXACT branch is
2498 a possible optimization target. Once we have
2499 two or more consequetive such branches we can
2500 create a trie of the EXACT's contents and stich
2501 it in place. If the sequence represents all of
2502 the branches we eliminate the whole thing and
2503 replace it with a single TRIE. If it is a
2504 subsequence then we need to stitch it in. This
2505 means the first branch has to remain, and needs
2506 to be repointed at the item on the branch chain
2507 following the last branch optimized. This could
2508 be either a BRANCH, in which case the
2509 subsequence is internal, or it could be the
2510 item following the branch sequence in which
2511 case the subsequence is at the end.
2512
2513 */
2514
2515 /* dont use tail as the end marker for this traverse */
2516 for ( cur = startbranch ; cur != scan ; cur = regnext( cur ) ) {
aec46f14 2517 regnode * const noper = NEXTOPER( cur );
be981c67 2518#if defined(DEBUGGING) || defined(NOJUMPTRIE)
aec46f14 2519 regnode * const noper_next = regnext( noper );
be981c67 2520#endif
a3621e74 2521
a3621e74 2522 DEBUG_OPTIMISE_r({
32fc9b6a 2523 regprop(RExC_rx, mysv, cur);
3dab1dad
YO
2524 PerlIO_printf( Perl_debug_log, "%*s- %s (%d)",
2525 (int)depth * 2 + 2,"", SvPV_nolen_const( mysv ), REG_NODE_NUM(cur) );
a3621e74 2526
32fc9b6a 2527 regprop(RExC_rx, mysv, noper);
a3621e74 2528 PerlIO_printf( Perl_debug_log, " -> %s",
cfd0369c 2529 SvPV_nolen_const(mysv));
a3621e74
YO
2530
2531 if ( noper_next ) {
32fc9b6a 2532 regprop(RExC_rx, mysv, noper_next );
a3621e74 2533 PerlIO_printf( Perl_debug_log,"\t=> %s\t",
cfd0369c 2534 SvPV_nolen_const(mysv));
a3621e74 2535 }
3dab1dad
YO
2536 PerlIO_printf( Perl_debug_log, "(First==%d,Last==%d,Cur==%d)\n",
2537 REG_NODE_NUM(first), REG_NODE_NUM(last), REG_NODE_NUM(cur) );
a3621e74 2538 });
3dab1dad
YO
2539 if ( (((first && optype!=NOTHING) ? OP( noper ) == optype
2540 : PL_regkind[ OP( noper ) ] == EXACT )
2541 || OP(noper) == NOTHING )
786e8c11
YO
2542#ifdef NOJUMPTRIE
2543 && noper_next == tail
2544#endif
2545 && count < U16_MAX)
a3621e74
YO
2546 {
2547 count++;
3dab1dad
YO
2548 if ( !first || optype == NOTHING ) {
2549 if (!first) first = cur;
a3621e74
YO
2550 optype = OP( noper );
2551 } else {
a3621e74 2552 last = cur;
a3621e74
YO
2553 }
2554 } else {
2555 if ( last ) {
786e8c11
YO
2556 make_trie( pRExC_state,
2557 startbranch, first, cur, tail, count,
2558 optype, depth+1 );
a3621e74 2559 }
3dab1dad 2560 if ( PL_regkind[ OP( noper ) ] == EXACT
786e8c11
YO
2561#ifdef NOJUMPTRIE
2562 && noper_next == tail
2563#endif
2564 ){
a3621e74
YO
2565 count = 1;
2566 first = cur;
2567 optype = OP( noper );
2568 } else {
2569 count = 0;
2570 first = NULL;
2571 optype = 0;
2572 }
2573 last = NULL;
2574 }
2575 }
2576 DEBUG_OPTIMISE_r({
32fc9b6a 2577 regprop(RExC_rx, mysv, cur);
a3621e74 2578 PerlIO_printf( Perl_debug_log,
3dab1dad
YO
2579 "%*s- %s (%d) <SCAN FINISHED>\n", (int)depth * 2 + 2,
2580 "", SvPV_nolen_const( mysv ),REG_NODE_NUM(cur));
a3621e74
YO
2581
2582 });
2583 if ( last ) {
786e8c11 2584 made= make_trie( pRExC_state, startbranch, first, scan, tail, count, optype, depth+1 );
3dab1dad 2585#ifdef TRIE_STUDY_OPT
786e8c11
YO
2586 if ( ((made == MADE_EXACT_TRIE &&
2587 startbranch == first)
2588 || ( first_non_open == first )) &&
2589 depth==0 )
2590 flags |= SCF_TRIE_RESTUDY;
3dab1dad 2591#endif
07be1b83 2592 }
a3621e74 2593 }
3dab1dad
YO
2594
2595 } /* do trie */
786e8c11 2596
a0ed51b3 2597 }
a3621e74 2598 else if ( code == BRANCHJ ) { /* single branch is optimized. */
c277df42 2599 scan = NEXTOPER(NEXTOPER(scan));
a3621e74 2600 } else /* single branch is optimized. */
c277df42
IZ
2601 scan = NEXTOPER(scan);
2602 continue;
a0ed51b3
LW
2603 }
2604 else if (OP(scan) == EXACT) {
cd439c50 2605 I32 l = STR_LEN(scan);
c445ea15 2606 UV uc;
a0ed51b3 2607 if (UTF) {
a3b680e6 2608 const U8 * const s = (U8*)STRING(scan);
1aa99e6b 2609 l = utf8_length(s, s + l);
9041c2e3 2610 uc = utf8_to_uvchr(s, NULL);
c445ea15
AL
2611 } else {
2612 uc = *((U8*)STRING(scan));
a0ed51b3
LW
2613 }
2614 min += l;
c277df42 2615 if (flags & SCF_DO_SUBSTR) { /* Update longest substr. */
c277df42
IZ
2616 /* The code below prefers earlier match for fixed
2617 offset, later match for variable offset. */
2618 if (data->last_end == -1) { /* Update the start info. */
2619 data->last_start_min = data->pos_min;
2620 data->last_start_max = is_inf
b81d288d 2621 ? I32_MAX : data->pos_min + data->pos_delta;
c277df42 2622 }
cd439c50 2623 sv_catpvn(data->last_found, STRING(scan), STR_LEN(scan));
45f47268
NC
2624 if (UTF)
2625 SvUTF8_on(data->last_found);
0eda9292 2626 {
9a957fbc 2627 SV * const sv = data->last_found;
a28509cc 2628 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ?
0eda9292
JH
2629 mg_find(sv, PERL_MAGIC_utf8) : NULL;
2630 if (mg && mg->mg_len >= 0)
5e43f467
JH
2631 mg->mg_len += utf8_length((U8*)STRING(scan),
2632 (U8*)STRING(scan)+STR_LEN(scan));
0eda9292 2633 }
c277df42
IZ
2634 data->last_end = data->pos_min + l;
2635 data->pos_min += l; /* As in the first entry. */
2636 data->flags &= ~SF_BEFORE_EOL;
2637 }
653099ff
GS
2638 if (flags & SCF_DO_STCLASS_AND) {
2639 /* Check whether it is compatible with what we know already! */
2640 int compat = 1;
2641
1aa99e6b 2642 if (uc >= 0x100 ||
516a5887 2643 (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE))
1aa99e6b 2644 && !ANYOF_BITMAP_TEST(data->start_class, uc)
653099ff 2645 && (!(data->start_class->flags & ANYOF_FOLD)
1aa99e6b 2646 || !ANYOF_BITMAP_TEST(data->start_class, PL_fold[uc])))
516a5887 2647 )
653099ff
GS
2648 compat = 0;
2649 ANYOF_CLASS_ZERO(data->start_class);
2650 ANYOF_BITMAP_ZERO(data->start_class);
2651 if (compat)
1aa99e6b 2652 ANYOF_BITMAP_SET(data->start_class, uc);
653099ff 2653 data->start_class->flags &= ~ANYOF_EOS;
9b877dbb
IH
2654 if (uc < 0x100)
2655 data->start_class->flags &= ~ANYOF_UNICODE_ALL;
653099ff
GS
2656 }
2657 else if (flags & SCF_DO_STCLASS_OR) {
2658 /* false positive possible if the class is case-folded */
1aa99e6b 2659 if (uc < 0x100)
9b877dbb
IH
2660 ANYOF_BITMAP_SET(data->start_class, uc);
2661 else
2662 data->start_class->flags |= ANYOF_UNICODE_ALL;
653099ff
GS
2663 data->start_class->flags &= ~ANYOF_EOS;
2664 cl_and(data->start_class, &and_with);
2665 }
2666 flags &= ~SCF_DO_STCLASS;
a0ed51b3 2667 }
3dab1dad 2668 else if (PL_regkind[OP(scan)] == EXACT) { /* But OP != EXACT! */
cd439c50 2669 I32 l = STR_LEN(scan);
1aa99e6b 2670 UV uc = *((U8*)STRING(scan));
653099ff
GS
2671
2672 /* Search for fixed substrings supports EXACT only. */
ecaa9b9c
NC
2673 if (flags & SCF_DO_SUBSTR) {
2674 assert(data);
1de06328 2675 scan_commit(pRExC_state, data, minlenp);
ecaa9b9c 2676 }
a0ed51b3 2677 if (UTF) {
6136c704 2678 const U8 * const s = (U8 *)STRING(scan);
1aa99e6b 2679 l = utf8_length(s, s + l);
9041c2e3 2680 uc = utf8_to_uvchr(s, NULL);
a0ed51b3
LW
2681 }
2682 min += l;
ecaa9b9c 2683 if (flags & SCF_DO_SUBSTR)
a0ed51b3 2684 data->pos_min += l;
653099ff
GS
2685 if (flags & SCF_DO_STCLASS_AND) {
2686 /* Check whether it is compatible with what we know already! */
2687 int compat = 1;
2688
1aa99e6b 2689 if (uc >= 0x100 ||
516a5887 2690 (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE))
1aa99e6b 2691 && !ANYOF_BITMAP_TEST(data->start_class, uc)
516a5887 2692 && !ANYOF_BITMAP_TEST(data->start_class, PL_fold[uc])))
653099ff
GS
2693 compat = 0;
2694 ANYOF_CLASS_ZERO(data->start_class);
2695 ANYOF_BITMAP_ZERO(data->start_class);
2696 if (compat) {
1aa99e6b 2697 ANYOF_BITMAP_SET(data->start_class, uc);
653099ff
GS
2698 data->start_class->flags &= ~ANYOF_EOS;
2699 data->start_class->flags |= ANYOF_FOLD;
2700 if (OP(scan) == EXACTFL)
2701 data->start_class->flags |= ANYOF_LOCALE;
2702 }
2703 }
2704 else if (flags & SCF_DO_STCLASS_OR) {
2705 if (data->start_class->flags & ANYOF_FOLD) {
2706 /* false positive possible if the class is case-folded.
2707 Assume that the locale settings are the same... */
1aa99e6b
IH
2708 if (uc < 0x100)
2709 ANYOF_BITMAP_SET(data->start_class, uc);
653099ff
GS
2710 data->start_class->flags &= ~ANYOF_EOS;
2711 }
2712 cl_and(data->start_class, &and_with);
2713 }
2714 flags &= ~SCF_DO_STCLASS;
a0ed51b3 2715 }
6bda09f9
YO
2716 else if (OP(scan)==RECURSE) {
2717 ARG2L_SET( scan, RExC_parens[ARG(scan)-1] - scan );
2718 }
bfed75c6 2719 else if (strchr((const char*)PL_varies,OP(scan))) {
9c5ffd7c 2720 I32 mincount, maxcount, minnext, deltanext, fl = 0;
aa7a4b56 2721 I32 f = flags, pos_before = 0;
d4c19fe8 2722 regnode * const oscan = scan;
653099ff
GS
2723 struct regnode_charclass_class this_class;
2724 struct regnode_charclass_class *oclass = NULL;
727f22e3 2725 I32 next_is_eval = 0;
653099ff 2726
3dab1dad 2727 switch (PL_regkind[OP(scan)]) {
653099ff 2728 case WHILEM: /* End of (?:...)* . */
c277df42
IZ
2729 scan = NEXTOPER(scan);
2730 goto finish;
2731 case PLUS:
653099ff 2732 if (flags & (SCF_DO_SUBSTR | SCF_DO_STCLASS)) {
c277df42 2733 next = NEXTOPER(scan);
653099ff 2734 if (OP(next) == EXACT || (flags & SCF_DO_STCLASS)) {
b81d288d
AB
2735 mincount = 1;
2736 maxcount = REG_INFTY;
c277df42
IZ
2737 next = regnext(scan);
2738 scan = NEXTOPER(scan);
2739 goto do_curly;
2740 }
2741 }
2742 if (flags & SCF_DO_SUBSTR)
2743 data->pos_min++;
2744 min++;
2745 /* Fall through. */
2746 case STAR:
653099ff
GS
2747 if (flags & SCF_DO_STCLASS) {
2748 mincount = 0;
b81d288d 2749 maxcount = REG_INFTY;
653099ff
GS
2750 next = regnext(scan);
2751 scan = NEXTOPER(scan);
2752 goto do_curly;
2753 }
b81d288d 2754 is_inf = is_inf_internal = 1;
c277df42
IZ
2755 scan = regnext(scan);
2756 if (flags & SCF_DO_SUBSTR) {
1de06328 2757 scan_commit(pRExC_state, data, minlenp); /* Cannot extend fixed substrings */
c277df42
IZ
2758 data->longest = &(data->longest_float);
2759 }
2760 goto optimize_curly_tail;
2761 case CURLY:
b81d288d 2762 mincount = ARG1(scan);
c277df42
IZ
2763 maxcount = ARG2(scan);
2764 next = regnext(scan);
cb434fcc
IZ
2765 if (OP(scan) == CURLYX) {
2766 I32 lp = (data ? *(data->last_closep) : 0);
786e8c11 2767 scan->flags = ((lp <= (I32)U8_MAX) ? (U8)lp : U8_MAX);
cb434fcc 2768 }
c277df42 2769 scan = NEXTOPER(scan) + EXTRA_STEP_2ARGS;
727f22e3 2770 next_is_eval = (OP(scan) == EVAL);
c277df42
IZ
2771 do_curly:
2772 if (flags & SCF_DO_SUBSTR) {
1de06328 2773 if (mincount == 0) scan_commit(pRExC_state,data,minlenp); /* Cannot extend fixed substrings */
c277df42
IZ
2774 pos_before = data->pos_min;
2775 }
2776 if (data) {
2777 fl = data->flags;
2778 data->flags &= ~(SF_HAS_PAR|SF_IN_PAR|SF_HAS_EVAL);
2779 if (is_inf)
2780 data->flags |= SF_IS_INF;
2781 }
653099ff 2782 if (flags & SCF_DO_STCLASS) {
830247a4 2783 cl_init(pRExC_state, &this_class);
653099ff
GS
2784 oclass = data->start_class;
2785 data->start_class = &this_class;
2786 f |= SCF_DO_STCLASS_AND;
2787 f &= ~SCF_DO_STCLASS_OR;
2788 }
e1901655
IZ
2789 /* These are the cases when once a subexpression
2790 fails at a particular position, it cannot succeed
2791 even after backtracking at the enclosing scope.
b81d288d 2792
e1901655
IZ
2793 XXXX what if minimal match and we are at the
2794 initial run of {n,m}? */
2795 if ((mincount != maxcount - 1) && (maxcount != REG_INFTY))
2796 f &= ~SCF_WHILEM_VISITED_POS;
653099ff 2797
c277df42 2798 /* This will finish on WHILEM, setting scan, or on NULL: */
1de06328 2799 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext, last, data,
a3621e74
YO
2800 (mincount == 0
2801 ? (f & ~SCF_DO_SUBSTR) : f),depth+1);
653099ff
GS
2802
2803 if (flags & SCF_DO_STCLASS)
2804 data->start_class = oclass;
2805 if (mincount == 0 || minnext == 0) {
2806 if (flags & SCF_DO_STCLASS_OR) {
830247a4 2807 cl_or(pRExC_state, data->start_class, &this_class);
653099ff
GS
2808 }
2809 else if (flags & SCF_DO_STCLASS_AND) {
b81d288d 2810 /* Switch to OR mode: cache the old value of
653099ff
GS
2811 * data->start_class */
2812 StructCopy(data->start_class, &and_with,
2813 struct regnode_charclass_class);
2814 flags &= ~SCF_DO_STCLASS_AND;
2815 StructCopy(&this_class, data->start_class,
2816 struct regnode_charclass_class);
2817 flags |= SCF_DO_STCLASS_OR;
2818 data->start_class->flags |= ANYOF_EOS;
2819 }
2820 } else { /* Non-zero len */
2821 if (flags & SCF_DO_STCLASS_OR) {
830247a4 2822 cl_or(pRExC_state, data->start_class, &this_class);
653099ff
GS
2823 cl_and(data->start_class, &and_with);
2824 }
2825 else if (flags & SCF_DO_STCLASS_AND)
2826 cl_and(data->start_class, &this_class);
2827 flags &= ~SCF_DO_STCLASS;
2828 }
c277df42
IZ
2829 if (!scan) /* It was not CURLYX, but CURLY. */
2830 scan = next;
041457d9
DM
2831 if ( /* ? quantifier ok, except for (?{ ... }) */
2832 (next_is_eval || !(mincount == 0 && maxcount == 1))
84037bb0 2833 && (minnext == 0) && (deltanext == 0)
99799961 2834 && data && !(data->flags & (SF_HAS_PAR|SF_IN_PAR))
041457d9
DM
2835 && maxcount <= REG_INFTY/3 /* Complement check for big count */
2836 && ckWARN(WARN_REGEXP))
b45f050a 2837 {
830247a4 2838 vWARN(RExC_parse,
b45f050a
JF
2839 "Quantifier unexpected on zero-length expression");
2840 }
2841
c277df42 2842 min += minnext * mincount;
b81d288d 2843 is_inf_internal |= ((maxcount == REG_INFTY
155aba94
GS
2844 && (minnext + deltanext) > 0)
2845 || deltanext == I32_MAX);
aca2d497 2846 is_inf |= is_inf_internal;
c277df42
IZ
2847 delta += (minnext + deltanext) * maxcount - minnext * mincount;
2848
2849 /* Try powerful optimization CURLYX => CURLYN. */
b81d288d 2850 if ( OP(oscan) == CURLYX && data
c277df42
IZ
2851 && data->flags & SF_IN_PAR
2852 && !(data->flags & SF_HAS_EVAL)
2853 && !deltanext && minnext == 1 ) {
2854 /* Try to optimize to CURLYN. */
2855 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS;
d4c19fe8 2856 regnode * const nxt1 = nxt;
497b47a8
JH
2857#ifdef DEBUGGING
2858 regnode *nxt2;
2859#endif
c277df42
IZ
2860
2861 /* Skip open. */
2862 nxt = regnext(nxt);
bfed75c6 2863 if (!strchr((const char*)PL_simple,OP(nxt))
3dab1dad 2864 && !(PL_regkind[OP(nxt)] == EXACT
b81d288d 2865 && STR_LEN(nxt) == 1))
c277df42 2866 goto nogo;
497b47a8 2867#ifdef DEBUGGING
c277df42 2868 nxt2 = nxt;
497b47a8 2869#endif
c277df42 2870 nxt = regnext(nxt);
b81d288d 2871 if (OP(nxt) != CLOSE)
c277df42
IZ
2872 goto nogo;
2873 /* Now we know that nxt2 is the only contents: */
eb160463 2874 oscan->flags = (U8)ARG(nxt);
c277df42
IZ
2875 OP(oscan) = CURLYN;
2876 OP(nxt1) = NOTHING; /* was OPEN. */
2877#ifdef DEBUGGING
2878 OP(nxt1 + 1) = OPTIMIZED; /* was count. */
2879 NEXT_OFF(nxt1+ 1) = 0; /* just for consistancy. */
2880 NEXT_OFF(nxt2) = 0; /* just for consistancy with CURLY. */
2881 OP(nxt) = OPTIMIZED; /* was CLOSE. */
2882 OP(nxt + 1) = OPTIMIZED; /* was count. */
2883 NEXT_OFF(nxt+ 1) = 0; /* just for consistancy. */
b81d288d 2884#endif
c277df42 2885 }
c277df42
IZ
2886 nogo:
2887
2888 /* Try optimization CURLYX => CURLYM. */
b81d288d 2889 if ( OP(oscan) == CURLYX && data
c277df42 2890 && !(data->flags & SF_HAS_PAR)
c277df42 2891 && !(data->flags & SF_HAS_EVAL)
0e788c72
HS
2892 && !deltanext /* atom is fixed width */
2893 && minnext != 0 /* CURLYM can't handle zero width */
2894 ) {
c277df42
IZ
2895 /* XXXX How to optimize if data == 0? */
2896 /* Optimize to a simpler form. */
2897 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN */
2898 regnode *nxt2;
2899
2900 OP(oscan) = CURLYM;
2901 while ( (nxt2 = regnext(nxt)) /* skip over embedded stuff*/
b81d288d 2902 && (OP(nxt2) != WHILEM))
c277df42
IZ
2903 nxt = nxt2;
2904 OP(nxt2) = SUCCEED; /* Whas WHILEM */
c277df42
IZ
2905 /* Need to optimize away parenths. */
2906 if (data->flags & SF_IN_PAR) {
2907 /* Set the parenth number. */
2908 regnode *nxt1 = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN*/
2909
b81d288d 2910 if (OP(nxt) != CLOSE)
b45f050a 2911 FAIL("Panic opt close");
eb160463 2912 oscan->flags = (U8)ARG(nxt);
c277df42
IZ
2913 OP(nxt1) = OPTIMIZED; /* was OPEN. */
2914 OP(nxt) = OPTIMIZED; /* was CLOSE. */
2915#ifdef DEBUGGING
2916 OP(nxt1 + 1) = OPTIMIZED; /* was count. */
2917 OP(nxt + 1) = OPTIMIZED; /* was count. */
2918 NEXT_OFF(nxt1 + 1) = 0; /* just for consistancy. */
2919 NEXT_OFF(nxt + 1) = 0; /* just for consistancy. */
b81d288d 2920#endif
c277df42
IZ
2921#if 0
2922 while ( nxt1 && (OP(nxt1) != WHILEM)) {
2923 regnode *nnxt = regnext(nxt1);
b81d288d 2924
c277df42
IZ
2925 if (nnxt == nxt) {
2926 if (reg_off_by_arg[OP(nxt1)])
2927 ARG_SET(nxt1, nxt2 - nxt1);
2928 else if (nxt2 - nxt1 < U16_MAX)
2929 NEXT_OFF(nxt1) = nxt2 - nxt1;
2930 else
2931 OP(nxt) = NOTHING; /* Cannot beautify */
2932 }
2933 nxt1 = nnxt;
2934 }
2935#endif
2936 /* Optimize again: */
1de06328 2937 study_chunk(pRExC_state, &nxt1, minlenp, &deltanext, nxt,
a3621e74 2938 NULL, 0,depth+1);
a0ed51b3
LW
2939 }
2940 else
c277df42 2941 oscan->flags = 0;
c277df42 2942 }
e1901655
IZ
2943 else if ((OP(oscan) == CURLYX)
2944 && (flags & SCF_WHILEM_VISITED_POS)
2945 /* See the comment on a similar expression above.
2946 However, this time it not a subexpression
2947 we care about, but the expression itself. */
2948 && (maxcount == REG_INFTY)
2949 && data && ++data->whilem_c < 16) {
2950 /* This stays as CURLYX, we can put the count/of pair. */
2c2d71f5
JH
2951 /* Find WHILEM (as in regexec.c) */
2952 regnode *nxt = oscan + NEXT_OFF(oscan);
2953
2954 if (OP(PREVOPER(nxt)) == NOTHING) /* LONGJMP */
2955 nxt += ARG(nxt);
eb160463
GS
2956 PREVOPER(nxt)->flags = (U8)(data->whilem_c
2957 | (RExC_whilem_seen << 4)); /* On WHILEM */
2c2d71f5 2958 }
b81d288d 2959 if (data && fl & (SF_HAS_PAR|SF_IN_PAR))
c277df42
IZ
2960 pars++;
2961 if (flags & SCF_DO_SUBSTR) {
c445ea15 2962 SV *last_str = NULL;
c277df42
IZ
2963 int counted = mincount != 0;
2964
2965 if (data->last_end > 0 && mincount != 0) { /* Ends with a string. */
5d1c421c
JH
2966#if defined(SPARC64_GCC_WORKAROUND)
2967 I32 b = 0;
2968 STRLEN l = 0;
cfd0369c 2969 const char *s = NULL;
5d1c421c
JH
2970 I32 old = 0;
2971
2972 if (pos_before >= data->last_start_min)
2973 b = pos_before;
2974 else
2975 b = data->last_start_min;
2976
2977 l = 0;
cfd0369c 2978 s = SvPV_const(data->last_found, l);
5d1c421c
JH
2979 old = b - data->last_start_min;
2980
2981#else
b81d288d 2982 I32 b = pos_before >= data->last_start_min
c277df42
IZ
2983 ? pos_before : data->last_start_min;
2984 STRLEN l;
d4c19fe8 2985 const char * const s = SvPV_const(data->last_found, l);
a0ed51b3 2986 I32 old = b - data->last_start_min;
5d1c421c 2987#endif
a0ed51b3
LW
2988
2989 if (UTF)
2990 old = utf8_hop((U8*)s, old) - (U8*)s;
c277df42 2991
a0ed51b3 2992 l -= old;
c277df42 2993 /* Get the added string: */
79cb57f6 2994 last_str = newSVpvn(s + old, l);
0e933229
IH
2995 if (UTF)
2996 SvUTF8_on(last_str);
c277df42
IZ
2997 if (deltanext == 0 && pos_before == b) {
2998 /* What was added is a constant string */
2999 if (mincount > 1) {
3000 SvGROW(last_str, (mincount * l) + 1);
b81d288d 3001 repeatcpy(SvPVX(last_str) + l,
3f7c398e 3002 SvPVX_const(last_str), l, mincount - 1);
b162af07 3003 SvCUR_set(last_str, SvCUR(last_str) * mincount);
c277df42 3004 /* Add additional parts. */
b81d288d 3005 SvCUR_set(data->last_found,
c277df42
IZ
3006 SvCUR(data->last_found) - l);
3007 sv_catsv(data->last_found, last_str);
0eda9292
JH
3008 {
3009 SV * sv = data->last_found;
3010 MAGIC *mg =
3011 SvUTF8(sv) && SvMAGICAL(sv) ?
3012 mg_find(sv, PERL_MAGIC_utf8) : NULL;
3013 if (mg && mg->mg_len >= 0)
3014 mg->mg_len += CHR_SVLEN(last_str);
3015 }
c277df42
IZ
3016 data->last_end += l * (mincount - 1);
3017 }
2a8d9689
HS
3018 } else {
3019 /* start offset must point into the last copy */
3020 data->last_start_min += minnext * (mincount - 1);
c152dc43
HS
3021 data->last_start_max += is_inf ? I32_MAX
3022 : (maxcount - 1) * (minnext + data->pos_delta);
c277df42
IZ
3023 }
3024 }
3025 /* It is counted once already... */
3026 data->pos_min += minnext * (mincount - counted);
3027 data->pos_delta += - counted * deltanext +
3028 (minnext + deltanext) * maxcount - minnext * mincount;
3029 if (mincount != maxcount) {
653099ff
GS
3030 /* Cannot extend fixed substrings found inside
3031 the group. */
1de06328 3032 scan_commit(pRExC_state,data,minlenp);
c277df42 3033 if (mincount && last_str) {
d4c19fe8
AL
3034 SV * const sv = data->last_found;
3035 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ?
45f47268
NC
3036 mg_find(sv, PERL_MAGIC_utf8) : NULL;
3037
3038 if (mg)
3039 mg->mg_len = -1;
3040 sv_setsv(sv, last_str);
c277df42 3041 data->last_end = data->pos_min;
b81d288d 3042 data->last_start_min =
a0ed51b3 3043 data->pos_min - CHR_SVLEN(last_str);
b81d288d
AB
3044 data->last_start_max = is_inf
3045 ? I32_MAX
c277df42 3046 : data->pos_min + data->pos_delta
a0ed51b3 3047 - CHR_SVLEN(last_str);
c277df42
IZ
3048 }
3049 data->longest = &(data->longest_float);
3050 }
aca2d497 3051 SvREFCNT_dec(last_str);
c277df42 3052 }
405ff068 3053 if (data && (fl & SF_HAS_EVAL))
c277df42
IZ
3054 data->flags |= SF_HAS_EVAL;
3055 optimize_curly_tail:
c277df42 3056 if (OP(oscan) != CURLYX) {
3dab1dad 3057 while (PL_regkind[OP(next = regnext(oscan))] == NOTHING
c277df42
IZ
3058 && NEXT_OFF(next))
3059 NEXT_OFF(oscan) += NEXT_OFF(next);
3060 }
c277df42 3061 continue;
653099ff 3062 default: /* REF and CLUMP only? */
c277df42 3063 if (flags & SCF_DO_SUBSTR) {
1de06328 3064 scan_commit(pRExC_state,data,minlenp); /* Cannot expect anything... */
c277df42
IZ
3065 data->longest = &(data->longest_float);
3066 }
aca2d497 3067 is_inf = is_inf_internal = 1;
653099ff 3068 if (flags & SCF_DO_STCLASS_OR)
830247a4 3069 cl_anything(pRExC_state, data->start_class);
653099ff 3070 flags &= ~SCF_DO_STCLASS;
c277df42
IZ
3071 break;
3072 }
a0ed51b3 3073 }
bfed75c6 3074 else if (strchr((const char*)PL_simple,OP(scan))) {
9c5ffd7c 3075 int value = 0;
653099ff 3076
c277df42 3077 if (flags & SCF_DO_SUBSTR) {
1de06328 3078 scan_commit(pRExC_state,data,minlenp);
c277df42
IZ
3079 data->pos_min++;
3080 }
3081 min++;
653099ff
GS
3082 if (flags & SCF_DO_STCLASS) {
3083 data->start_class->flags &= ~ANYOF_EOS; /* No match on empty */
3084
3085 /* Some of the logic below assumes that switching
3086 locale on will only add false positives. */
3dab1dad 3087 switch (PL_regkind[OP(scan)]) {
653099ff 3088 case SANY:
653099ff
GS
3089 default:
3090 do_default:
3091 /* Perl_croak(aTHX_ "panic: unexpected simple REx opcode %d", OP(scan)); */
3092 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
830247a4 3093 cl_anything(pRExC_state, data->start_class);
653099ff
GS
3094 break;
3095 case REG_ANY:
3096 if (OP(scan) == SANY)
3097 goto do_default;
3098 if (flags & SCF_DO_STCLASS_OR) { /* Everything but \n */
3099 value = (ANYOF_BITMAP_TEST(data->start_class,'\n')
3100 || (data->start_class->flags & ANYOF_CLASS));
830247a4 3101 cl_anything(pRExC_state, data->start_class);
653099ff
GS
3102 }
3103 if (flags & SCF_DO_STCLASS_AND || !value)
3104 ANYOF_BITMAP_CLEAR(data->start_class,'\n');
3105 break;
3106 case ANYOF:
3107 if (flags & SCF_DO_STCLASS_AND)
3108 cl_and(data->start_class,
3109 (struct regnode_charclass_class*)scan);
3110 else
830247a4 3111 cl_or(pRExC_state, data->start_class,
653099ff
GS
3112 (struct regnode_charclass_class*)scan);
3113 break;
3114 case ALNUM:
3115 if (flags & SCF_DO_STCLASS_AND) {
3116 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3117 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM);
3118 for (value = 0; value < 256; value++)
3119 if (!isALNUM(value))
3120 ANYOF_BITMAP_CLEAR(data->start_class, value);
3121 }
3122 }
3123 else {
3124 if (data->start_class->flags & ANYOF_LOCALE)
3125 ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM);
3126 else {
3127 for (value = 0; value < 256; value++)
3128 if (isALNUM(value))
b81d288d 3129 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3130 }
3131 }
3132 break;
3133 case ALNUML:
3134 if (flags & SCF_DO_STCLASS_AND) {
3135 if (data->start_class->flags & ANYOF_LOCALE)
3136 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM);
3137 }
3138 else {
3139 ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM);
3140 data->start_class->flags |= ANYOF_LOCALE;
3141 }
3142 break;
3143 case NALNUM:
3144 if (flags & SCF_DO_STCLASS_AND) {
3145 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3146 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM);
3147 for (value = 0; value < 256; value++)
3148 if (isALNUM(value))
3149 ANYOF_BITMAP_CLEAR(data->start_class, value);
3150 }
3151 }
3152 else {
3153 if (data->start_class->flags & ANYOF_LOCALE)
3154 ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM);
3155 else {
3156 for (value = 0; value < 256; value++)
3157 if (!isALNUM(value))
b81d288d 3158 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3159 }
3160 }
3161 break;
3162 case NALNUML:
3163 if (flags & SCF_DO_STCLASS_AND) {
3164 if (data->start_class->flags & ANYOF_LOCALE)
3165 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM);
3166 }
3167 else {
3168 data->start_class->flags |= ANYOF_LOCALE;
3169 ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM);
3170 }
3171 break;
3172 case SPACE:
3173 if (flags & SCF_DO_STCLASS_AND) {
3174 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3175 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE);
3176 for (value = 0; value < 256; value++)
3177 if (!isSPACE(value))
3178 ANYOF_BITMAP_CLEAR(data->start_class, value);
3179 }
3180 }
3181 else {
3182 if (data->start_class->flags & ANYOF_LOCALE)
3183 ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE);
3184 else {
3185 for (value = 0; value < 256; value++)
3186 if (isSPACE(value))
b81d288d 3187 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3188 }
3189 }
3190 break;
3191 case SPACEL:
3192 if (flags & SCF_DO_STCLASS_AND) {
3193 if (data->start_class->flags & ANYOF_LOCALE)
3194 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE);
3195 }
3196 else {
3197 data->start_class->flags |= ANYOF_LOCALE;
3198 ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE);
3199 }
3200 break;
3201 case NSPACE:
3202 if (flags & SCF_DO_STCLASS_AND) {
3203 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3204 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE);
3205 for (value = 0; value < 256; value++)
3206 if (isSPACE(value))
3207 ANYOF_BITMAP_CLEAR(data->start_class, value);
3208 }
3209 }
3210 else {
3211 if (data->start_class->flags & ANYOF_LOCALE)
3212 ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE);
3213 else {
3214 for (value = 0; value < 256; value++)
3215 if (!isSPACE(value))
b81d288d 3216 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3217 }
3218 }
3219 break;
3220 case NSPACEL:
3221 if (flags & SCF_DO_STCLASS_AND) {
3222 if (data->start_class->flags & ANYOF_LOCALE) {
3223 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE);
3224 for (value = 0; value < 256; value++)
3225 if (!isSPACE(value))
3226 ANYOF_BITMAP_CLEAR(data->start_class, value);
3227 }
3228 }
3229 else {
3230 data->start_class->flags |= ANYOF_LOCALE;
3231 ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE);
3232 }
3233 break;
3234 case DIGIT:
3235 if (flags & SCF_DO_STCLASS_AND) {
3236 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NDIGIT);
3237 for (value = 0; value < 256; value++)
3238 if (!isDIGIT(value))
3239 ANYOF_BITMAP_CLEAR(data->start_class, value);
3240 }
3241 else {
3242 if (data->start_class->flags & ANYOF_LOCALE)
3243 ANYOF_CLASS_SET(data->start_class,ANYOF_DIGIT);
3244 else {
3245 for (value = 0; value < 256; value++)
3246 if (isDIGIT(value))
b81d288d 3247 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3248 }
3249 }
3250 break;
3251 case NDIGIT:
3252 if (flags & SCF_DO_STCLASS_AND) {
3253 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_DIGIT);
3254 for (value = 0; value < 256; value++)
3255 if (isDIGIT(value))
3256 ANYOF_BITMAP_CLEAR(data->start_class, value);
3257 }
3258 else {
3259 if (data->start_class->flags & ANYOF_LOCALE)
3260 ANYOF_CLASS_SET(data->start_class,ANYOF_NDIGIT);
3261 else {
3262 for (value = 0; value < 256; value++)
3263 if (!isDIGIT(value))
b81d288d 3264 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3265 }
3266 }
3267 break;
3268 }
3269 if (flags & SCF_DO_STCLASS_OR)
3270 cl_and(data->start_class, &and_with);
3271 flags &= ~SCF_DO_STCLASS;
3272 }
a0ed51b3 3273 }
3dab1dad 3274 else if (PL_regkind[OP(scan)] == EOL && flags & SCF_DO_SUBSTR) {
c277df42
IZ
3275 data->flags |= (OP(scan) == MEOL
3276 ? SF_BEFORE_MEOL
3277 : SF_BEFORE_SEOL);
a0ed51b3 3278 }
3dab1dad 3279 else if ( PL_regkind[OP(scan)] == BRANCHJ
653099ff
GS
3280 /* Lookbehind, or need to calculate parens/evals/stclass: */
3281 && (scan->flags || data || (flags & SCF_DO_STCLASS))
c277df42 3282 && (OP(scan) == IFMATCH || OP(scan) == UNLESSM)) {
1de06328
YO
3283 if ( !PERL_ENABLE_POSITIVE_ASSERTION_STUDY
3284 || OP(scan) == UNLESSM )
3285 {
3286 /* Negative Lookahead/lookbehind
3287 In this case we can't do fixed string optimisation.
3288 */
3289
3290 I32 deltanext, minnext, fake = 0;
3291 regnode *nscan;
3292 struct regnode_charclass_class intrnl;
3293 int f = 0;
3294
3295 data_fake.flags = 0;
3296 if (data) {
3297 data_fake.whilem_c = data->whilem_c;
3298 data_fake.last_closep = data->last_closep;
a0ed51b3 3299 }
1de06328
YO
3300 else
3301 data_fake.last_closep = &fake;
3302 if ( flags & SCF_DO_STCLASS && !scan->flags
3303 && OP(scan) == IFMATCH ) { /* Lookahead */
3304 cl_init(pRExC_state, &intrnl);
3305 data_fake.start_class = &intrnl;
3306 f |= SCF_DO_STCLASS_AND;
c277df42 3307 }
1de06328
YO
3308 if (flags & SCF_WHILEM_VISITED_POS)
3309 f |= SCF_WHILEM_VISITED_POS;
3310 next = regnext(scan);
3311 nscan = NEXTOPER(NEXTOPER(scan));
3312 minnext = study_chunk(pRExC_state, &nscan, minlenp, &deltanext, last, &data_fake, f,depth+1);
3313 if (scan->flags) {
3314 if (deltanext) {
3315 vFAIL("Variable length lookbehind not implemented");
3316 }
3317 else if (minnext > (I32)U8_MAX) {
3318 vFAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX);
3319 }
3320 scan->flags = (U8)minnext;
3321 }
3322 if (data) {
3323 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3324 pars++;
3325 if (data_fake.flags & SF_HAS_EVAL)
3326 data->flags |= SF_HAS_EVAL;
3327 data->whilem_c = data_fake.whilem_c;
3328 }
3329 if (f & SCF_DO_STCLASS_AND) {
3330 const int was = (data->start_class->flags & ANYOF_EOS);
3331
3332 cl_and(data->start_class, &intrnl);
3333 if (was)
3334 data->start_class->flags |= ANYOF_EOS;
3335 }
be8e71aa 3336 }
1de06328
YO
3337#if PERL_ENABLE_POSITIVE_ASSERTION_STUDY
3338 else {
3339 /* Positive Lookahead/lookbehind
3340 In this case we can do fixed string optimisation,
3341 but we must be careful about it. Note in the case of
3342 lookbehind the positions will be offset by the minimum
3343 length of the pattern, something we won't know about
3344 until after the recurse.
3345 */
3346 I32 deltanext, fake = 0;
3347 regnode *nscan;
3348 struct regnode_charclass_class intrnl;
3349 int f = 0;
3350 /* We use SAVEFREEPV so that when the full compile
3351 is finished perl will clean up the allocated
3352 minlens when its all done. This was we don't
3353 have to worry about freeing them when we know
3354 they wont be used, which would be a pain.
3355 */
3356 I32 *minnextp;
3357 Newx( minnextp, 1, I32 );
3358 SAVEFREEPV(minnextp);
3359
3360 if (data) {
3361 StructCopy(data, &data_fake, scan_data_t);
3362 if ((flags & SCF_DO_SUBSTR) && data->last_found) {
3363 f |= SCF_DO_SUBSTR;
3364 if (scan->flags)
3365 scan_commit(pRExC_state, &data_fake,minlenp);
3366 data_fake.last_found=newSVsv(data->last_found);
3367 }
3368 }
3369 else
3370 data_fake.last_closep = &fake;
3371 data_fake.flags = 0;
3372 if (is_inf)
3373 data_fake.flags |= SF_IS_INF;
3374 if ( flags & SCF_DO_STCLASS && !scan->flags
3375 && OP(scan) == IFMATCH ) { /* Lookahead */
3376 cl_init(pRExC_state, &intrnl);
3377 data_fake.start_class = &intrnl;
3378 f |= SCF_DO_STCLASS_AND;
3379 }
3380 if (flags & SCF_WHILEM_VISITED_POS)
3381 f |= SCF_WHILEM_VISITED_POS;
3382 next = regnext(scan);
3383 nscan = NEXTOPER(NEXTOPER(scan));
3384
3385 *minnextp = study_chunk(pRExC_state, &nscan, minnextp, &deltanext, last, &data_fake, f,depth+1);
3386 if (scan->flags) {
3387 if (deltanext) {
3388 vFAIL("Variable length lookbehind not implemented");
3389 }
3390 else if (*minnextp > (I32)U8_MAX) {
3391 vFAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX);
3392 }
3393 scan->flags = (U8)*minnextp;
3394 }
3395
3396 *minnextp += min;
3397
3398
3399 if (f & SCF_DO_STCLASS_AND) {
3400 const int was = (data->start_class->flags & ANYOF_EOS);
3401
3402 cl_and(data->start_class, &intrnl);
3403 if (was)
3404 data->start_class->flags |= ANYOF_EOS;
3405 }
3406 if (data) {
3407 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3408 pars++;
3409 if (data_fake.flags & SF_HAS_EVAL)
3410 data->flags |= SF_HAS_EVAL;
3411 data->whilem_c = data_fake.whilem_c;
3412 if ((flags & SCF_DO_SUBSTR) && data_fake.last_found) {
3413 if (RExC_rx->minlen<*minnextp)
3414 RExC_rx->minlen=*minnextp;
3415 scan_commit(pRExC_state, &data_fake, minnextp);
3416 SvREFCNT_dec(data_fake.last_found);
3417
3418 if ( data_fake.minlen_fixed != minlenp )
3419 {
3420 data->offset_fixed= data_fake.offset_fixed;
3421 data->minlen_fixed= data_fake.minlen_fixed;
3422 data->lookbehind_fixed+= scan->flags;
3423 }
3424 if ( data_fake.minlen_float != minlenp )
3425 {
3426 data->minlen_float= data_fake.minlen_float;
3427 data->offset_float_min=data_fake.offset_float_min;
3428 data->offset_float_max=data_fake.offset_float_max;
3429 data->lookbehind_float+= scan->flags;
3430 }
3431 }
3432 }
3433
653099ff 3434
653099ff 3435 }
1de06328 3436#endif
a0ed51b3
LW
3437 }
3438 else if (OP(scan) == OPEN) {
c277df42 3439 pars++;
a0ed51b3 3440 }
cb434fcc 3441 else if (OP(scan) == CLOSE) {
eb160463 3442 if ((I32)ARG(scan) == is_par) {
cb434fcc 3443 next = regnext(scan);
c277df42 3444
cb434fcc
IZ
3445 if ( next && (OP(next) != WHILEM) && next < last)
3446 is_par = 0; /* Disable optimization */
3447 }
3448 if (data)
3449 *(data->last_closep) = ARG(scan);
a0ed51b3
LW
3450 }
3451 else if (OP(scan) == EVAL) {
c277df42
IZ
3452 if (data)
3453 data->flags |= SF_HAS_EVAL;
3454 }
96776eda 3455 else if (OP(scan) == LOGICAL && scan->flags == 2) { /* Embedded follows */
0f5d15d6 3456 if (flags & SCF_DO_SUBSTR) {
1de06328 3457 scan_commit(pRExC_state,data,minlenp);
0f5d15d6
IZ
3458 data->longest = &(data->longest_float);
3459 }
3460 is_inf = is_inf_internal = 1;
653099ff 3461 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
830247a4 3462 cl_anything(pRExC_state, data->start_class);
96776eda 3463 flags &= ~SCF_DO_STCLASS;
0f5d15d6 3464 }
786e8c11
YO
3465#ifdef TRIE_STUDY_OPT
3466#ifdef FULL_TRIE_STUDY
3467 else if (PL_regkind[OP(scan)] == TRIE) {
3468 /* NOTE - There is similar code to this block above for handling
3469 BRANCH nodes on the initial study. If you change stuff here
3470 check there too. */
3471 regnode *tail= regnext(scan);
3472 reg_trie_data *trie = (reg_trie_data*)RExC_rx->data->data[ ARG(scan) ];
3473 I32 max1 = 0, min1 = I32_MAX;
3474 struct regnode_charclass_class accum;
3475
3476 if (flags & SCF_DO_SUBSTR) /* XXXX Add !SUSPEND? */
1de06328 3477 scan_commit(pRExC_state, data,minlenp); /* Cannot merge strings after this. */
786e8c11
YO
3478 if (flags & SCF_DO_STCLASS)
3479 cl_init_zero(pRExC_state, &accum);
3480
3481 if (!trie->jump) {
3482 min1= trie->minlen;
3483 max1= trie->maxlen;
3484 } else {
3485 const regnode *nextbranch= NULL;
3486 U32 word;
3487
3488 for ( word=1 ; word <= trie->wordcount ; word++)
3489 {
3490 I32 deltanext=0, minnext=0, f = 0, fake;
3491 struct regnode_charclass_class this_class;
3492
3493 data_fake.flags = 0;
3494 if (data) {
3495 data_fake.whilem_c = data->whilem_c;
3496 data_fake.last_closep = data->last_closep;
3497 }
3498 else
3499 data_fake.last_closep = &fake;
3500
3501 if (flags & SCF_DO_STCLASS) {
3502 cl_init(pRExC_state, &this_class);
3503 data_fake.start_class = &this_class;
3504 f = SCF_DO_STCLASS_AND;
3505 }
3506 if (flags & SCF_WHILEM_VISITED_POS)
3507 f |= SCF_WHILEM_VISITED_POS;
3508
3509 if (trie->jump[word]) {
3510 if (!nextbranch)
3511 nextbranch = tail - trie->jump[0];
3512 scan= tail - trie->jump[word];
3513 /* We go from the jump point to the branch that follows
3514 it. Note this means we need the vestigal unused branches
3515 even though they arent otherwise used.
3516 */
1de06328 3517 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext,
786e8c11
YO
3518 (regnode *)nextbranch, &data_fake, f,depth+1);
3519 }
3520 if (nextbranch && PL_regkind[OP(nextbranch)]==BRANCH)
3521 nextbranch= regnext((regnode*)nextbranch);
3522
3523 if (min1 > (I32)(minnext + trie->minlen))
3524 min1 = minnext + trie->minlen;
3525 if (max1 < (I32)(minnext + deltanext + trie->maxlen))
3526 max1 = minnext + deltanext + trie->maxlen;
3527 if (deltanext == I32_MAX)
3528 is_inf = is_inf_internal = 1;
3529
3530 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3531 pars++;
3532
3533 if (data) {
3534 if (data_fake.flags & SF_HAS_EVAL)
3535 data->flags |= SF_HAS_EVAL;
3536 data->whilem_c = data_fake.whilem_c;
3537 }
3538 if (flags & SCF_DO_STCLASS)
3539 cl_or(pRExC_state, &accum, &this_class);
3540 }
3541 }
3542 if (flags & SCF_DO_SUBSTR) {
3543 data->pos_min += min1;
3544 data->pos_delta += max1 - min1;
3545 if (max1 != min1 || is_inf)
3546 data->longest = &(data->longest_float);
3547 }
3548 min += min1;
3549 delta += max1 - min1;
3550 if (flags & SCF_DO_STCLASS_OR) {
3551 cl_or(pRExC_state, data->start_class, &accum);
3552 if (min1) {
3553 cl_and(data->start_class, &and_with);
3554 flags &= ~SCF_DO_STCLASS;
3555 }
3556 }
3557 else if (flags & SCF_DO_STCLASS_AND) {
3558 if (min1) {
3559 cl_and(data->start_class, &accum);
3560 flags &= ~SCF_DO_STCLASS;
3561 }
3562 else {
3563 /* Switch to OR mode: cache the old value of
3564 * data->start_class */
3565 StructCopy(data->start_class, &and_with,
3566 struct regnode_charclass_class);
3567 flags &= ~SCF_DO_STCLASS_AND;
3568 StructCopy(&accum, data->start_class,
3569 struct regnode_charclass_class);
3570 flags |= SCF_DO_STCLASS_OR;
3571 data->start_class->flags |= ANYOF_EOS;
3572 }
3573 }
3574 scan= tail;
3575 continue;
3576 }
3577#else
3578 else if (PL_regkind[OP(scan)] == TRIE) {
3579 reg_trie_data *trie = (reg_trie_data*)RExC_rx->data->data[ ARG(scan) ];
3580 U8*bang=NULL;
3581
3582 min += trie->minlen;
3583 delta += (trie->maxlen - trie->minlen);
3584 flags &= ~SCF_DO_STCLASS; /* xxx */
3585 if (flags & SCF_DO_SUBSTR) {
1de06328 3586 scan_commit(pRExC_state,data,minlenp); /* Cannot expect anything... */
786e8c11
YO
3587 data->pos_min += trie->minlen;
3588 data->pos_delta += (trie->maxlen - trie->minlen);
3589 if (trie->maxlen != trie->minlen)
3590 data->longest = &(data->longest_float);
3591 }
3592 if (trie->jump) /* no more substrings -- for now /grr*/
3593 flags &= ~SCF_DO_SUBSTR;
3594 }
3595#endif /* old or new */
3596#endif /* TRIE_STUDY_OPT */
c277df42
IZ
3597 /* Else: zero-length, ignore. */
3598 scan = regnext(scan);
3599 }
3600
3601 finish:
3602 *scanp = scan;
aca2d497 3603 *deltap = is_inf_internal ? I32_MAX : delta;
b81d288d 3604 if (flags & SCF_DO_SUBSTR && is_inf)
c277df42 3605 data->pos_delta = I32_MAX - data->pos_min;
786e8c11 3606 if (is_par > (I32)U8_MAX)
c277df42
IZ
3607 is_par = 0;
3608 if (is_par && pars==1 && data) {
3609 data->flags |= SF_IN_PAR;
3610 data->flags &= ~SF_HAS_PAR;
a0ed51b3
LW
3611 }
3612 else if (pars && data) {
c277df42
IZ
3613 data->flags |= SF_HAS_PAR;
3614 data->flags &= ~SF_IN_PAR;
3615 }
653099ff
GS
3616 if (flags & SCF_DO_STCLASS_OR)
3617 cl_and(data->start_class, &and_with);
786e8c11
YO
3618 if (flags & SCF_TRIE_RESTUDY)
3619 data->flags |= SCF_TRIE_RESTUDY;
1de06328
YO
3620
3621 DEBUG_STUDYDATA(data,depth);
3622
c277df42
IZ
3623 return min;
3624}
3625
76e3520e 3626STATIC I32
5f66b61c 3627S_add_data(RExC_state_t *pRExC_state, I32 n, const char *s)
c277df42 3628{
830247a4 3629 if (RExC_rx->data) {
b81d288d
AB
3630 Renewc(RExC_rx->data,
3631 sizeof(*RExC_rx->data) + sizeof(void*) * (RExC_rx->data->count + n - 1),
c277df42 3632 char, struct reg_data);
830247a4
IZ
3633 Renew(RExC_rx->data->what, RExC_rx->data->count + n, U8);
3634 RExC_rx->data->count += n;
a0ed51b3
LW
3635 }
3636 else {
a02a5408 3637 Newxc(RExC_rx->data, sizeof(*RExC_rx->data) + sizeof(void*) * (n - 1),
c277df42 3638 char, struct reg_data);
a02a5408 3639 Newx(RExC_rx->data->what, n, U8);
830247a4 3640 RExC_rx->data->count = n;
c277df42 3641 }
830247a4
IZ
3642 Copy(s, RExC_rx->data->what + RExC_rx->data->count - n, n, U8);
3643 return RExC_rx->data->count - n;
c277df42
IZ
3644}
3645
76234dfb 3646#ifndef PERL_IN_XSUB_RE
d88dccdf 3647void
864dbfa3 3648Perl_reginitcolors(pTHX)
d88dccdf 3649{
97aff369 3650 dVAR;
1df70142 3651 const char * const s = PerlEnv_getenv("PERL_RE_COLORS");
d88dccdf 3652 if (s) {
1df70142
AL
3653 char *t = savepv(s);
3654 int i = 0;
3655 PL_colors[0] = t;
d88dccdf 3656 while (++i < 6) {
1df70142
AL
3657 t = strchr(t, '\t');
3658 if (t) {
3659 *t = '\0';
3660 PL_colors[i] = ++t;
d88dccdf
IZ
3661 }
3662 else
1df70142 3663 PL_colors[i] = t = (char *)"";
d88dccdf
IZ
3664 }
3665 } else {
1df70142 3666 int i = 0;
b81d288d 3667 while (i < 6)
06b5626a 3668 PL_colors[i++] = (char *)"";
d88dccdf
IZ
3669 }
3670 PL_colorset = 1;
3671}
76234dfb 3672#endif
8615cb43 3673
07be1b83 3674
786e8c11
YO
3675#ifdef TRIE_STUDY_OPT
3676#define CHECK_RESTUDY_GOTO \
3677 if ( \
3678 (data.flags & SCF_TRIE_RESTUDY) \
3679 && ! restudied++ \
3680 ) goto reStudy
3681#else
3682#define CHECK_RESTUDY_GOTO
3683#endif
f9f4320a 3684
a687059c 3685/*
e50aee73 3686 - pregcomp - compile a regular expression into internal code
a687059c
LW
3687 *
3688 * We can't allocate space until we know how big the compiled form will be,
3689 * but we can't compile it (and thus know how big it is) until we've got a
3690 * place to put the code. So we cheat: we compile it twice, once with code
3691 * generation turned off and size counting turned on, and once "for real".
3692 * This also means that we don't allocate space until we are sure that the
3693 * thing really will compile successfully, and we never have to move the
3694 * code and thus invalidate pointers into it. (Note that it has to be in
3695 * one piece because free() must be able to free it all.) [NB: not true in perl]
3696 *
3697 * Beware that the optimization-preparation code in here knows about some
3698 * of the structure of the compiled regexp. [I'll say.]
3699 */
f9f4320a
YO
3700#ifndef PERL_IN_XSUB_RE
3701#define CORE_ONLY_BLOCK(c) {c}{
3702#define RE_ENGINE_PTR &PL_core_reg_engine
3703#else
3704#define CORE_ONLY_BLOCK(c) {
3705extern const struct regexp_engine my_reg_engine;
3706#define RE_ENGINE_PTR &my_reg_engine
3707#endif
3708#define END_BLOCK }
3709
a687059c 3710regexp *
864dbfa3 3711Perl_pregcomp(pTHX_ char *exp, char *xend, PMOP *pm)
a687059c 3712{
97aff369 3713 dVAR;
f9f4320a
YO
3714 GET_RE_DEBUG_FLAGS_DECL;
3715 DEBUG_r(if (!PL_colorset) reginitcolors());
3716 CORE_ONLY_BLOCK(
3717 /* Dispatch a request to compile a regexp to correct
3718 regexp engine. */
3719 HV * const table = GvHV(PL_hintgv);
3720 if (table) {
3721 SV **ptr= hv_fetchs(table, "regcomp", FALSE);
3722 if (ptr && SvIOK(*ptr)) {
3723 const regexp_engine *eng=INT2PTR(regexp_engine*,SvIV(*ptr));
3724 DEBUG_COMPILE_r({
8d8756e7 3725 PerlIO_printf(Perl_debug_log, "Using engine %"UVxf"\n",