This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Avoid undefined behaviour for -DPERL_MEM_LOG by not using a direct
[perl5.git] / regcomp.c
CommitLineData
a0d0e21e
LW
1/* regcomp.c
2 */
3
4/*
5 * "A fair jaw-cracker dwarf-language must be." --Samwise Gamgee
6 */
7
61296642
DM
8/* This file contains functions for compiling a regular expression. See
9 * also regexec.c which funnily enough, contains functions for executing
166f8a29 10 * a regular expression.
e4a054ea
DM
11 *
12 * This file is also copied at build time to ext/re/re_comp.c, where
13 * it's built with -DPERL_EXT_RE_BUILD -DPERL_EXT_RE_DEBUG -DPERL_EXT.
14 * This causes the main functions to be compiled under new names and with
15 * debugging support added, which makes "use re 'debug'" work.
166f8a29
DM
16 */
17
a687059c
LW
18/* NOTE: this is derived from Henry Spencer's regexp code, and should not
19 * confused with the original package (see point 3 below). Thanks, Henry!
20 */
21
22/* Additional note: this code is very heavily munged from Henry's version
23 * in places. In some spots I've traded clarity for efficiency, so don't
24 * blame Henry for some of the lack of readability.
25 */
26
e50aee73
AD
27/* The names of the functions have been changed from regcomp and
28 * regexec to pregcomp and pregexec in order to avoid conflicts
29 * with the POSIX routines of the same names.
30*/
31
b9d5759e 32#ifdef PERL_EXT_RE_BUILD
54df2634 33#include "re_top.h"
b81d288d 34#endif
56953603 35
a687059c 36/*
e50aee73 37 * pregcomp and pregexec -- regsub and regerror are not used in perl
a687059c
LW
38 *
39 * Copyright (c) 1986 by University of Toronto.
40 * Written by Henry Spencer. Not derived from licensed software.
41 *
42 * Permission is granted to anyone to use this software for any
43 * purpose on any computer system, and to redistribute it freely,
44 * subject to the following restrictions:
45 *
46 * 1. The author is not responsible for the consequences of use of
47 * this software, no matter how awful, even if they arise
48 * from defects in it.
49 *
50 * 2. The origin of this software must not be misrepresented, either
51 * by explicit claim or by omission.
52 *
53 * 3. Altered versions must be plainly marked as such, and must not
54 * be misrepresented as being the original software.
55 *
56 *
57 **** Alterations to Henry's code are...
58 ****
4bb101f2 59 **** Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
b94e2f88 60 **** 2000, 2001, 2002, 2003, 2004, 2005, 2006, by Larry Wall and others
a687059c 61 ****
9ef589d8
LW
62 **** You may distribute under the terms of either the GNU General Public
63 **** License or the Artistic License, as specified in the README file.
64
a687059c
LW
65 *
66 * Beware that some of this code is subtly aware of the way operator
67 * precedence is structured in regular expressions. Serious changes in
68 * regular-expression syntax might require a total rethink.
69 */
70#include "EXTERN.h"
864dbfa3 71#define PERL_IN_REGCOMP_C
a687059c 72#include "perl.h"
d06ea78c 73
acfe0abc 74#ifndef PERL_IN_XSUB_RE
d06ea78c
GS
75# include "INTERN.h"
76#endif
c277df42
IZ
77
78#define REG_COMP_C
54df2634
NC
79#ifdef PERL_IN_XSUB_RE
80# include "re_comp.h"
81#else
82# include "regcomp.h"
83#endif
a687059c 84
d4cce5f1 85#ifdef op
11343788 86#undef op
d4cce5f1 87#endif /* op */
11343788 88
fe14fcc3 89#ifdef MSDOS
7e4e8c89 90# if defined(BUGGY_MSC6)
fe14fcc3 91 /* MSC 6.00A breaks on op/regexp.t test 85 unless we turn this off */
7e4e8c89 92# pragma optimize("a",off)
fe14fcc3 93 /* But MSC 6.00A is happy with 'w', for aliases only across function calls*/
7e4e8c89
NC
94# pragma optimize("w",on )
95# endif /* BUGGY_MSC6 */
fe14fcc3
LW
96#endif /* MSDOS */
97
a687059c
LW
98#ifndef STATIC
99#define STATIC static
100#endif
101
830247a4 102typedef struct RExC_state_t {
e2509266 103 U32 flags; /* are we folding, multilining? */
830247a4
IZ
104 char *precomp; /* uncompiled string. */
105 regexp *rx;
fac92740 106 char *start; /* Start of input for compile */
830247a4
IZ
107 char *end; /* End of input for compile */
108 char *parse; /* Input-scan pointer. */
109 I32 whilem_seen; /* number of WHILEM in this expr */
fac92740 110 regnode *emit_start; /* Start of emitted-code area */
ffc61ed2 111 regnode *emit; /* Code-emit pointer; &regdummy = don't = compiling */
830247a4
IZ
112 I32 naughty; /* How bad is this pattern? */
113 I32 sawback; /* Did we see \1, ...? */
114 U32 seen;
115 I32 size; /* Code size. */
116 I32 npar; /* () count. */
117 I32 extralen;
118 I32 seen_zerolen;
119 I32 seen_evals;
6bda09f9 120 regnode **parens; /* offsets of each paren */
1aa99e6b 121 I32 utf8;
6bda09f9 122 HV *charnames; /* cache of named sequences */
81714fb9 123 HV *paren_names; /* Paren names */
830247a4
IZ
124#if ADD_TO_REGEXEC
125 char *starttry; /* -Dr: where regtry was called. */
126#define RExC_starttry (pRExC_state->starttry)
127#endif
3dab1dad 128#ifdef DEBUGGING
be8e71aa 129 const char *lastparse;
3dab1dad
YO
130 I32 lastnum;
131#define RExC_lastparse (pRExC_state->lastparse)
132#define RExC_lastnum (pRExC_state->lastnum)
133#endif
830247a4
IZ
134} RExC_state_t;
135
e2509266 136#define RExC_flags (pRExC_state->flags)
830247a4
IZ
137#define RExC_precomp (pRExC_state->precomp)
138#define RExC_rx (pRExC_state->rx)
fac92740 139#define RExC_start (pRExC_state->start)
830247a4
IZ
140#define RExC_end (pRExC_state->end)
141#define RExC_parse (pRExC_state->parse)
142#define RExC_whilem_seen (pRExC_state->whilem_seen)
fac92740 143#define RExC_offsets (pRExC_state->rx->offsets) /* I am not like the others */
830247a4 144#define RExC_emit (pRExC_state->emit)
fac92740 145#define RExC_emit_start (pRExC_state->emit_start)
830247a4
IZ
146#define RExC_naughty (pRExC_state->naughty)
147#define RExC_sawback (pRExC_state->sawback)
148#define RExC_seen (pRExC_state->seen)
149#define RExC_size (pRExC_state->size)
150#define RExC_npar (pRExC_state->npar)
151#define RExC_extralen (pRExC_state->extralen)
152#define RExC_seen_zerolen (pRExC_state->seen_zerolen)
153#define RExC_seen_evals (pRExC_state->seen_evals)
1aa99e6b 154#define RExC_utf8 (pRExC_state->utf8)
fc8cd66c 155#define RExC_charnames (pRExC_state->charnames)
6bda09f9 156#define RExC_parens (pRExC_state->parens)
81714fb9 157#define RExC_paren_names (pRExC_state->paren_names)
830247a4 158
a687059c
LW
159#define ISMULT1(c) ((c) == '*' || (c) == '+' || (c) == '?')
160#define ISMULT2(s) ((*s) == '*' || (*s) == '+' || (*s) == '?' || \
161 ((*s) == '{' && regcurly(s)))
a687059c 162
35c8bce7
LW
163#ifdef SPSTART
164#undef SPSTART /* dratted cpp namespace... */
165#endif
a687059c
LW
166/*
167 * Flags to be passed up and down.
168 */
a687059c 169#define WORST 0 /* Worst case. */
821b33a5 170#define HASWIDTH 0x1 /* Known to match non-null strings. */
a0d0e21e
LW
171#define SIMPLE 0x2 /* Simple enough to be STAR/PLUS operand. */
172#define SPSTART 0x4 /* Starts with * or +. */
173#define TRYAGAIN 0x8 /* Weeded out a declaration. */
a687059c 174
3dab1dad
YO
175#define REG_NODE_NUM(x) ((x) ? (int)((x)-RExC_emit_start) : -1)
176
07be1b83
YO
177/* whether trie related optimizations are enabled */
178#if PERL_ENABLE_EXTENDED_TRIE_OPTIMISATION
179#define TRIE_STUDY_OPT
786e8c11 180#define FULL_TRIE_STUDY
07be1b83
YO
181#define TRIE_STCLASS
182#endif
1de06328
YO
183
184
185/* About scan_data_t.
186
187 During optimisation we recurse through the regexp program performing
188 various inplace (keyhole style) optimisations. In addition study_chunk
189 and scan_commit populate this data structure with information about
190 what strings MUST appear in the pattern. We look for the longest
191 string that must appear for at a fixed location, and we look for the
192 longest string that may appear at a floating location. So for instance
193 in the pattern:
194
195 /FOO[xX]A.*B[xX]BAR/
196
197 Both 'FOO' and 'A' are fixed strings. Both 'B' and 'BAR' are floating
198 strings (because they follow a .* construct). study_chunk will identify
199 both FOO and BAR as being the longest fixed and floating strings respectively.
200
201 The strings can be composites, for instance
202
203 /(f)(o)(o)/
204
205 will result in a composite fixed substring 'foo'.
206
207 For each string some basic information is maintained:
208
209 - offset or min_offset
210 This is the position the string must appear at, or not before.
211 It also implicitly (when combined with minlenp) tells us how many
212 character must match before the string we are searching.
213 Likewise when combined with minlenp and the length of the string
214 tells us how many characters must appear after the string we have
215 found.
216
217 - max_offset
218 Only used for floating strings. This is the rightmost point that
219 the string can appear at. Ifset to I32 max it indicates that the
220 string can occur infinitely far to the right.
221
222 - minlenp
223 A pointer to the minimum length of the pattern that the string
224 was found inside. This is important as in the case of positive
225 lookahead or positive lookbehind we can have multiple patterns
226 involved. Consider
227
228 /(?=FOO).*F/
229
230 The minimum length of the pattern overall is 3, the minimum length
231 of the lookahead part is 3, but the minimum length of the part that
232 will actually match is 1. So 'FOO's minimum length is 3, but the
233 minimum length for the F is 1. This is important as the minimum length
234 is used to determine offsets in front of and behind the string being
235 looked for. Since strings can be composites this is the length of the
236 pattern at the time it was commited with a scan_commit. Note that
237 the length is calculated by study_chunk, so that the minimum lengths
238 are not known until the full pattern has been compiled, thus the
239 pointer to the value.
240
241 - lookbehind
242
243 In the case of lookbehind the string being searched for can be
244 offset past the start point of the final matching string.
245 If this value was just blithely removed from the min_offset it would
246 invalidate some of the calculations for how many chars must match
247 before or after (as they are derived from min_offset and minlen and
248 the length of the string being searched for).
249 When the final pattern is compiled and the data is moved from the
250 scan_data_t structure into the regexp structure the information
251 about lookbehind is factored in, with the information that would
252 have been lost precalculated in the end_shift field for the
253 associated string.
254
255 The fields pos_min and pos_delta are used to store the minimum offset
256 and the delta to the maximum offset at the current point in the pattern.
257
258*/
2c2d71f5
JH
259
260typedef struct scan_data_t {
1de06328
YO
261 /*I32 len_min; unused */
262 /*I32 len_delta; unused */
2c2d71f5
JH
263 I32 pos_min;
264 I32 pos_delta;
265 SV *last_found;
1de06328 266 I32 last_end; /* min value, <0 unless valid. */
2c2d71f5
JH
267 I32 last_start_min;
268 I32 last_start_max;
1de06328
YO
269 SV **longest; /* Either &l_fixed, or &l_float. */
270 SV *longest_fixed; /* longest fixed string found in pattern */
271 I32 offset_fixed; /* offset where it starts */
272 I32 *minlen_fixed; /* pointer to the minlen relevent to the string */
273 I32 lookbehind_fixed; /* is the position of the string modfied by LB */
274 SV *longest_float; /* longest floating string found in pattern */
275 I32 offset_float_min; /* earliest point in string it can appear */
276 I32 offset_float_max; /* latest point in string it can appear */
277 I32 *minlen_float; /* pointer to the minlen relevent to the string */
278 I32 lookbehind_float; /* is the position of the string modified by LB */
2c2d71f5
JH
279 I32 flags;
280 I32 whilem_c;
cb434fcc 281 I32 *last_closep;
653099ff 282 struct regnode_charclass_class *start_class;
2c2d71f5
JH
283} scan_data_t;
284
a687059c 285/*
e50aee73 286 * Forward declarations for pregcomp()'s friends.
a687059c 287 */
a0d0e21e 288
27da23d5 289static const scan_data_t zero_scan_data =
1de06328 290 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ,0};
c277df42
IZ
291
292#define SF_BEFORE_EOL (SF_BEFORE_SEOL|SF_BEFORE_MEOL)
07be1b83
YO
293#define SF_BEFORE_SEOL 0x0001
294#define SF_BEFORE_MEOL 0x0002
c277df42
IZ
295#define SF_FIX_BEFORE_EOL (SF_FIX_BEFORE_SEOL|SF_FIX_BEFORE_MEOL)
296#define SF_FL_BEFORE_EOL (SF_FL_BEFORE_SEOL|SF_FL_BEFORE_MEOL)
297
09b7f37c
CB
298#ifdef NO_UNARY_PLUS
299# define SF_FIX_SHIFT_EOL (0+2)
300# define SF_FL_SHIFT_EOL (0+4)
301#else
302# define SF_FIX_SHIFT_EOL (+2)
303# define SF_FL_SHIFT_EOL (+4)
304#endif
c277df42
IZ
305
306#define SF_FIX_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FIX_SHIFT_EOL)
307#define SF_FIX_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FIX_SHIFT_EOL)
308
309#define SF_FL_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FL_SHIFT_EOL)
310#define SF_FL_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FL_SHIFT_EOL) /* 0x20 */
07be1b83
YO
311#define SF_IS_INF 0x0040
312#define SF_HAS_PAR 0x0080
313#define SF_IN_PAR 0x0100
314#define SF_HAS_EVAL 0x0200
315#define SCF_DO_SUBSTR 0x0400
653099ff
GS
316#define SCF_DO_STCLASS_AND 0x0800
317#define SCF_DO_STCLASS_OR 0x1000
318#define SCF_DO_STCLASS (SCF_DO_STCLASS_AND|SCF_DO_STCLASS_OR)
e1901655 319#define SCF_WHILEM_VISITED_POS 0x2000
c277df42 320
786e8c11
YO
321#define SCF_TRIE_RESTUDY 0x4000 /* Do restudy? */
322
07be1b83 323
eb160463 324#define UTF (RExC_utf8 != 0)
e2509266
JH
325#define LOC ((RExC_flags & PMf_LOCALE) != 0)
326#define FOLD ((RExC_flags & PMf_FOLD) != 0)
a0ed51b3 327
ffc61ed2 328#define OOB_UNICODE 12345678
93733859 329#define OOB_NAMEDCLASS -1
b8c5462f 330
a0ed51b3
LW
331#define CHR_SVLEN(sv) (UTF ? sv_len_utf8(sv) : SvCUR(sv))
332#define CHR_DIST(a,b) (UTF ? utf8_distance(a,b) : a - b)
333
8615cb43 334
b45f050a
JF
335/* length of regex to show in messages that don't mark a position within */
336#define RegexLengthToShowInErrorMessages 127
337
338/*
339 * If MARKER[12] are adjusted, be sure to adjust the constants at the top
340 * of t/op/regmesg.t, the tests in t/op/re_tests, and those in
341 * op/pragma/warn/regcomp.
342 */
7253e4e3
RK
343#define MARKER1 "<-- HERE" /* marker as it appears in the description */
344#define MARKER2 " <-- HERE " /* marker as it appears within the regex */
b81d288d 345
7253e4e3 346#define REPORT_LOCATION " in regex; marked by " MARKER1 " in m/%.*s" MARKER2 "%s/"
b45f050a
JF
347
348/*
349 * Calls SAVEDESTRUCTOR_X if needed, then calls Perl_croak with the given
350 * arg. Show regex, up to a maximum length. If it's too long, chop and add
351 * "...".
352 */
ccb2c380 353#define FAIL(msg) STMT_START { \
bfed75c6 354 const char *ellipses = ""; \
ccb2c380
MP
355 IV len = RExC_end - RExC_precomp; \
356 \
357 if (!SIZE_ONLY) \
358 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
359 if (len > RegexLengthToShowInErrorMessages) { \
360 /* chop 10 shorter than the max, to ensure meaning of "..." */ \
361 len = RegexLengthToShowInErrorMessages - 10; \
362 ellipses = "..."; \
363 } \
364 Perl_croak(aTHX_ "%s in regex m/%.*s%s/", \
365 msg, (int)len, RExC_precomp, ellipses); \
366} STMT_END
8615cb43 367
b45f050a 368/*
b45f050a
JF
369 * Simple_vFAIL -- like FAIL, but marks the current location in the scan
370 */
ccb2c380 371#define Simple_vFAIL(m) STMT_START { \
a28509cc 372 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
373 Perl_croak(aTHX_ "%s" REPORT_LOCATION, \
374 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
375} STMT_END
b45f050a
JF
376
377/*
378 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL()
379 */
ccb2c380
MP
380#define vFAIL(m) STMT_START { \
381 if (!SIZE_ONLY) \
382 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
383 Simple_vFAIL(m); \
384} STMT_END
b45f050a
JF
385
386/*
387 * Like Simple_vFAIL(), but accepts two arguments.
388 */
ccb2c380 389#define Simple_vFAIL2(m,a1) STMT_START { \
a28509cc 390 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
391 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, \
392 (int)offset, RExC_precomp, RExC_precomp + offset); \
393} STMT_END
b45f050a
JF
394
395/*
396 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL2().
397 */
ccb2c380
MP
398#define vFAIL2(m,a1) STMT_START { \
399 if (!SIZE_ONLY) \
400 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
401 Simple_vFAIL2(m, a1); \
402} STMT_END
b45f050a
JF
403
404
405/*
406 * Like Simple_vFAIL(), but accepts three arguments.
407 */
ccb2c380 408#define Simple_vFAIL3(m, a1, a2) STMT_START { \
a28509cc 409 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
410 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, \
411 (int)offset, RExC_precomp, RExC_precomp + offset); \
412} STMT_END
b45f050a
JF
413
414/*
415 * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL3().
416 */
ccb2c380
MP
417#define vFAIL3(m,a1,a2) STMT_START { \
418 if (!SIZE_ONLY) \
419 SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \
420 Simple_vFAIL3(m, a1, a2); \
421} STMT_END
b45f050a
JF
422
423/*
424 * Like Simple_vFAIL(), but accepts four arguments.
425 */
ccb2c380 426#define Simple_vFAIL4(m, a1, a2, a3) STMT_START { \
a28509cc 427 const IV offset = RExC_parse - RExC_precomp; \
ccb2c380
MP
428 S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, a3, \
429 (int)offset, RExC_precomp, RExC_precomp + offset); \
430} STMT_END
b45f050a 431
ccb2c380 432#define vWARN(loc,m) STMT_START { \
a28509cc 433 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
434 Perl_warner(aTHX_ packWARN(WARN_REGEXP), "%s" REPORT_LOCATION, \
435 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
436} STMT_END
437
438#define vWARNdep(loc,m) STMT_START { \
a28509cc 439 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
440 Perl_warner(aTHX_ packWARN2(WARN_DEPRECATED, WARN_REGEXP), \
441 "%s" REPORT_LOCATION, \
442 m, (int)offset, RExC_precomp, RExC_precomp + offset); \
443} STMT_END
444
445
446#define vWARN2(loc, m, a1) STMT_START { \
a28509cc 447 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
448 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
449 a1, (int)offset, RExC_precomp, RExC_precomp + offset); \
450} STMT_END
451
452#define vWARN3(loc, m, a1, a2) STMT_START { \
a28509cc 453 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
454 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
455 a1, a2, (int)offset, RExC_precomp, RExC_precomp + offset); \
456} STMT_END
457
458#define vWARN4(loc, m, a1, a2, a3) STMT_START { \
a28509cc 459 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
460 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
461 a1, a2, a3, (int)offset, RExC_precomp, RExC_precomp + offset); \
462} STMT_END
463
464#define vWARN5(loc, m, a1, a2, a3, a4) STMT_START { \
a28509cc 465 const IV offset = loc - RExC_precomp; \
ccb2c380
MP
466 Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \
467 a1, a2, a3, a4, (int)offset, RExC_precomp, RExC_precomp + offset); \
468} STMT_END
9d1d55b5 469
8615cb43 470
cd439c50 471/* Allow for side effects in s */
ccb2c380
MP
472#define REGC(c,s) STMT_START { \
473 if (!SIZE_ONLY) *(s) = (c); else (void)(s); \
474} STMT_END
cd439c50 475
fac92740
MJD
476/* Macros for recording node offsets. 20001227 mjd@plover.com
477 * Nodes are numbered 1, 2, 3, 4. Node #n's position is recorded in
478 * element 2*n-1 of the array. Element #2n holds the byte length node #n.
479 * Element 0 holds the number n.
07be1b83 480 * Position is 1 indexed.
fac92740
MJD
481 */
482
ccb2c380
MP
483#define Set_Node_Offset_To_R(node,byte) STMT_START { \
484 if (! SIZE_ONLY) { \
485 MJD_OFFSET_DEBUG(("** (%d) offset of node %d is %d.\n", \
07be1b83 486 __LINE__, (node), (int)(byte))); \
ccb2c380 487 if((node) < 0) { \
551405c4 488 Perl_croak(aTHX_ "value of node is %d in Offset macro", (int)(node)); \
ccb2c380
MP
489 } else { \
490 RExC_offsets[2*(node)-1] = (byte); \
491 } \
492 } \
493} STMT_END
494
495#define Set_Node_Offset(node,byte) \
496 Set_Node_Offset_To_R((node)-RExC_emit_start, (byte)-RExC_start)
497#define Set_Cur_Node_Offset Set_Node_Offset(RExC_emit, RExC_parse)
498
499#define Set_Node_Length_To_R(node,len) STMT_START { \
500 if (! SIZE_ONLY) { \
501 MJD_OFFSET_DEBUG(("** (%d) size of node %d is %d.\n", \
551405c4 502 __LINE__, (int)(node), (int)(len))); \
ccb2c380 503 if((node) < 0) { \
551405c4 504 Perl_croak(aTHX_ "value of node is %d in Length macro", (int)(node)); \
ccb2c380
MP
505 } else { \
506 RExC_offsets[2*(node)] = (len); \
507 } \
508 } \
509} STMT_END
510
511#define Set_Node_Length(node,len) \
512 Set_Node_Length_To_R((node)-RExC_emit_start, len)
513#define Set_Cur_Node_Length(len) Set_Node_Length(RExC_emit, len)
514#define Set_Node_Cur_Length(node) \
515 Set_Node_Length(node, RExC_parse - parse_start)
fac92740
MJD
516
517/* Get offsets and lengths */
518#define Node_Offset(n) (RExC_offsets[2*((n)-RExC_emit_start)-1])
519#define Node_Length(n) (RExC_offsets[2*((n)-RExC_emit_start)])
520
07be1b83
YO
521#define Set_Node_Offset_Length(node,offset,len) STMT_START { \
522 Set_Node_Offset_To_R((node)-RExC_emit_start, (offset)); \
523 Set_Node_Length_To_R((node)-RExC_emit_start, (len)); \
524} STMT_END
525
526
527#if PERL_ENABLE_EXPERIMENTAL_REGEX_OPTIMISATIONS
528#define EXPERIMENTAL_INPLACESCAN
529#endif
530
1de06328 531#define DEBUG_STUDYDATA(data,depth) \
a5ca303d 532DEBUG_OPTIMISE_MORE_r(if(data){ \
1de06328
YO
533 PerlIO_printf(Perl_debug_log, \
534 "%*s"/* Len:%"IVdf"/%"IVdf" */" Pos:%"IVdf"/%"IVdf \
535 " Flags: %"IVdf" Whilem_c: %"IVdf" Lcp: %"IVdf" ", \
536 (int)(depth)*2, "", \
537 (IV)((data)->pos_min), \
538 (IV)((data)->pos_delta), \
539 (IV)((data)->flags), \
540 (IV)((data)->whilem_c), \
541 (IV)((data)->last_closep ? *((data)->last_closep) : -1) \
542 ); \
543 if ((data)->last_found) \
544 PerlIO_printf(Perl_debug_log, \
545 "Last:'%s' %"IVdf":%"IVdf"/%"IVdf" %sFixed:'%s' @ %"IVdf \
546 " %sFloat: '%s' @ %"IVdf"/%"IVdf"", \
547 SvPVX_const((data)->last_found), \
548 (IV)((data)->last_end), \
549 (IV)((data)->last_start_min), \
550 (IV)((data)->last_start_max), \
551 ((data)->longest && \
552 (data)->longest==&((data)->longest_fixed)) ? "*" : "", \
553 SvPVX_const((data)->longest_fixed), \
554 (IV)((data)->offset_fixed), \
555 ((data)->longest && \
556 (data)->longest==&((data)->longest_float)) ? "*" : "", \
557 SvPVX_const((data)->longest_float), \
558 (IV)((data)->offset_float_min), \
559 (IV)((data)->offset_float_max) \
560 ); \
561 PerlIO_printf(Perl_debug_log,"\n"); \
562});
563
acfe0abc 564static void clear_re(pTHX_ void *r);
4327152a 565
653099ff 566/* Mark that we cannot extend a found fixed substring at this point.
786e8c11 567 Update the longest found anchored substring and the longest found
653099ff
GS
568 floating substrings if needed. */
569
4327152a 570STATIC void
1de06328 571S_scan_commit(pTHX_ const RExC_state_t *pRExC_state, scan_data_t *data, I32 *minlenp)
c277df42 572{
e1ec3a88
AL
573 const STRLEN l = CHR_SVLEN(data->last_found);
574 const STRLEN old_l = CHR_SVLEN(*data->longest);
1de06328 575 GET_RE_DEBUG_FLAGS_DECL;
b81d288d 576
c277df42 577 if ((l >= old_l) && ((l > old_l) || (data->flags & SF_BEFORE_EOL))) {
6b43b216 578 SvSetMagicSV(*data->longest, data->last_found);
c277df42
IZ
579 if (*data->longest == data->longest_fixed) {
580 data->offset_fixed = l ? data->last_start_min : data->pos_min;
581 if (data->flags & SF_BEFORE_EOL)
b81d288d 582 data->flags
c277df42
IZ
583 |= ((data->flags & SF_BEFORE_EOL) << SF_FIX_SHIFT_EOL);
584 else
585 data->flags &= ~SF_FIX_BEFORE_EOL;
1de06328
YO
586 data->minlen_fixed=minlenp;
587 data->lookbehind_fixed=0;
a0ed51b3
LW
588 }
589 else {
c277df42 590 data->offset_float_min = l ? data->last_start_min : data->pos_min;
b81d288d
AB
591 data->offset_float_max = (l
592 ? data->last_start_max
c277df42 593 : data->pos_min + data->pos_delta);
9051bda5
HS
594 if ((U32)data->offset_float_max > (U32)I32_MAX)
595 data->offset_float_max = I32_MAX;
c277df42 596 if (data->flags & SF_BEFORE_EOL)
b81d288d 597 data->flags
c277df42
IZ
598 |= ((data->flags & SF_BEFORE_EOL) << SF_FL_SHIFT_EOL);
599 else
600 data->flags &= ~SF_FL_BEFORE_EOL;
1de06328
YO
601 data->minlen_float=minlenp;
602 data->lookbehind_float=0;
c277df42
IZ
603 }
604 }
605 SvCUR_set(data->last_found, 0);
0eda9292 606 {
a28509cc 607 SV * const sv = data->last_found;
097eb12c
AL
608 if (SvUTF8(sv) && SvMAGICAL(sv)) {
609 MAGIC * const mg = mg_find(sv, PERL_MAGIC_utf8);
610 if (mg)
611 mg->mg_len = 0;
612 }
0eda9292 613 }
c277df42
IZ
614 data->last_end = -1;
615 data->flags &= ~SF_BEFORE_EOL;
1de06328 616 DEBUG_STUDYDATA(data,0);
c277df42
IZ
617}
618
653099ff
GS
619/* Can match anything (initialization) */
620STATIC void
097eb12c 621S_cl_anything(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
653099ff 622{
653099ff 623 ANYOF_CLASS_ZERO(cl);
f8bef550 624 ANYOF_BITMAP_SETALL(cl);
1aa99e6b 625 cl->flags = ANYOF_EOS|ANYOF_UNICODE_ALL;
653099ff
GS
626 if (LOC)
627 cl->flags |= ANYOF_LOCALE;
628}
629
630/* Can match anything (initialization) */
631STATIC int
5f66b61c 632S_cl_is_anything(const struct regnode_charclass_class *cl)
653099ff
GS
633{
634 int value;
635
aaa51d5e 636 for (value = 0; value <= ANYOF_MAX; value += 2)
653099ff
GS
637 if (ANYOF_CLASS_TEST(cl, value) && ANYOF_CLASS_TEST(cl, value + 1))
638 return 1;
1aa99e6b
IH
639 if (!(cl->flags & ANYOF_UNICODE_ALL))
640 return 0;
10edeb5d 641 if (!ANYOF_BITMAP_TESTALLSET((const void*)cl))
f8bef550 642 return 0;
653099ff
GS
643 return 1;
644}
645
646/* Can match anything (initialization) */
647STATIC void
097eb12c 648S_cl_init(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
653099ff 649{
8ecf7187 650 Zero(cl, 1, struct regnode_charclass_class);
653099ff 651 cl->type = ANYOF;
830247a4 652 cl_anything(pRExC_state, cl);
653099ff
GS
653}
654
655STATIC void
097eb12c 656S_cl_init_zero(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl)
653099ff 657{
8ecf7187 658 Zero(cl, 1, struct regnode_charclass_class);
653099ff 659 cl->type = ANYOF;
830247a4 660 cl_anything(pRExC_state, cl);
653099ff
GS
661 if (LOC)
662 cl->flags |= ANYOF_LOCALE;
663}
664
665/* 'And' a given class with another one. Can create false positives */
666/* We assume that cl is not inverted */
667STATIC void
5f66b61c 668S_cl_and(struct regnode_charclass_class *cl,
a28509cc 669 const struct regnode_charclass_class *and_with)
653099ff 670{
653099ff
GS
671 if (!(and_with->flags & ANYOF_CLASS)
672 && !(cl->flags & ANYOF_CLASS)
673 && (and_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
674 && !(and_with->flags & ANYOF_FOLD)
675 && !(cl->flags & ANYOF_FOLD)) {
676 int i;
677
678 if (and_with->flags & ANYOF_INVERT)
679 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
680 cl->bitmap[i] &= ~and_with->bitmap[i];
681 else
682 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
683 cl->bitmap[i] &= and_with->bitmap[i];
684 } /* XXXX: logic is complicated otherwise, leave it along for a moment. */
685 if (!(and_with->flags & ANYOF_EOS))
686 cl->flags &= ~ANYOF_EOS;
1aa99e6b 687
14ebb1a2
JH
688 if (cl->flags & ANYOF_UNICODE_ALL && and_with->flags & ANYOF_UNICODE &&
689 !(and_with->flags & ANYOF_INVERT)) {
1aa99e6b
IH
690 cl->flags &= ~ANYOF_UNICODE_ALL;
691 cl->flags |= ANYOF_UNICODE;
692 ARG_SET(cl, ARG(and_with));
693 }
14ebb1a2
JH
694 if (!(and_with->flags & ANYOF_UNICODE_ALL) &&
695 !(and_with->flags & ANYOF_INVERT))
1aa99e6b 696 cl->flags &= ~ANYOF_UNICODE_ALL;
14ebb1a2
JH
697 if (!(and_with->flags & (ANYOF_UNICODE|ANYOF_UNICODE_ALL)) &&
698 !(and_with->flags & ANYOF_INVERT))
1aa99e6b 699 cl->flags &= ~ANYOF_UNICODE;
653099ff
GS
700}
701
702/* 'OR' a given class with another one. Can create false positives */
703/* We assume that cl is not inverted */
704STATIC void
097eb12c 705S_cl_or(const RExC_state_t *pRExC_state, struct regnode_charclass_class *cl, const struct regnode_charclass_class *or_with)
653099ff 706{
653099ff
GS
707 if (or_with->flags & ANYOF_INVERT) {
708 /* We do not use
709 * (B1 | CL1) | (!B2 & !CL2) = (B1 | !B2 & !CL2) | (CL1 | (!B2 & !CL2))
710 * <= (B1 | !B2) | (CL1 | !CL2)
711 * which is wasteful if CL2 is small, but we ignore CL2:
712 * (B1 | CL1) | (!B2 & !CL2) <= (B1 | CL1) | !B2 = (B1 | !B2) | CL1
713 * XXXX Can we handle case-fold? Unclear:
714 * (OK1(i) | OK1(i')) | !(OK1(i) | OK1(i')) =
715 * (OK1(i) | OK1(i')) | (!OK1(i) & !OK1(i'))
716 */
717 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
718 && !(or_with->flags & ANYOF_FOLD)
719 && !(cl->flags & ANYOF_FOLD) ) {
720 int i;
721
722 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
723 cl->bitmap[i] |= ~or_with->bitmap[i];
724 } /* XXXX: logic is complicated otherwise */
725 else {
830247a4 726 cl_anything(pRExC_state, cl);
653099ff
GS
727 }
728 } else {
729 /* (B1 | CL1) | (B2 | CL2) = (B1 | B2) | (CL1 | CL2)) */
730 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE)
b81d288d 731 && (!(or_with->flags & ANYOF_FOLD)
653099ff
GS
732 || (cl->flags & ANYOF_FOLD)) ) {
733 int i;
734
735 /* OR char bitmap and class bitmap separately */
736 for (i = 0; i < ANYOF_BITMAP_SIZE; i++)
737 cl->bitmap[i] |= or_with->bitmap[i];
738 if (or_with->flags & ANYOF_CLASS) {
739 for (i = 0; i < ANYOF_CLASSBITMAP_SIZE; i++)
740 cl->classflags[i] |= or_with->classflags[i];
741 cl->flags |= ANYOF_CLASS;
742 }
743 }
744 else { /* XXXX: logic is complicated, leave it along for a moment. */
830247a4 745 cl_anything(pRExC_state, cl);
653099ff
GS
746 }
747 }
748 if (or_with->flags & ANYOF_EOS)
749 cl->flags |= ANYOF_EOS;
1aa99e6b
IH
750
751 if (cl->flags & ANYOF_UNICODE && or_with->flags & ANYOF_UNICODE &&
752 ARG(cl) != ARG(or_with)) {
753 cl->flags |= ANYOF_UNICODE_ALL;
754 cl->flags &= ~ANYOF_UNICODE;
755 }
756 if (or_with->flags & ANYOF_UNICODE_ALL) {
757 cl->flags |= ANYOF_UNICODE_ALL;
758 cl->flags &= ~ANYOF_UNICODE;
759 }
653099ff
GS
760}
761
a3621e74
YO
762#define TRIE_LIST_ITEM(state,idx) (trie->states[state].trans.list)[ idx ]
763#define TRIE_LIST_CUR(state) ( TRIE_LIST_ITEM( state, 0 ).forid )
764#define TRIE_LIST_LEN(state) ( TRIE_LIST_ITEM( state, 0 ).newstate )
765#define TRIE_LIST_USED(idx) ( trie->states[state].trans.list ? (TRIE_LIST_CUR( idx ) - 1) : 0 )
766
3dab1dad
YO
767
768#ifdef DEBUGGING
07be1b83 769/*
3dab1dad
YO
770 dump_trie(trie)
771 dump_trie_interim_list(trie,next_alloc)
772 dump_trie_interim_table(trie,next_alloc)
773
774 These routines dump out a trie in a somewhat readable format.
07be1b83
YO
775 The _interim_ variants are used for debugging the interim
776 tables that are used to generate the final compressed
777 representation which is what dump_trie expects.
778
3dab1dad
YO
779 Part of the reason for their existance is to provide a form
780 of documentation as to how the different representations function.
07be1b83
YO
781
782*/
3dab1dad
YO
783
784/*
785 dump_trie(trie)
786 Dumps the final compressed table form of the trie to Perl_debug_log.
787 Used for debugging make_trie().
788*/
789
790STATIC void
791S_dump_trie(pTHX_ const struct _reg_trie_data *trie,U32 depth)
792{
793 U32 state;
ab3bbdeb
YO
794 SV *sv=sv_newmortal();
795 int colwidth= trie->widecharmap ? 6 : 4;
3dab1dad
YO
796 GET_RE_DEBUG_FLAGS_DECL;
797
ab3bbdeb 798
3dab1dad
YO
799 PerlIO_printf( Perl_debug_log, "%*sChar : %-6s%-6s%-4s ",
800 (int)depth * 2 + 2,"",
801 "Match","Base","Ofs" );
802
803 for( state = 0 ; state < trie->uniquecharcount ; state++ ) {
be8e71aa 804 SV ** const tmp = av_fetch( trie->revcharmap, state, 0);
3dab1dad 805 if ( tmp ) {
ab3bbdeb
YO
806 PerlIO_printf( Perl_debug_log, "%*s",
807 colwidth,
ddc5bc0f 808 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
ab3bbdeb
YO
809 PL_colors[0], PL_colors[1],
810 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
811 PERL_PV_ESCAPE_FIRSTCHAR
812 )
813 );
3dab1dad
YO
814 }
815 }
816 PerlIO_printf( Perl_debug_log, "\n%*sState|-----------------------",
817 (int)depth * 2 + 2,"");
818
819 for( state = 0 ; state < trie->uniquecharcount ; state++ )
ab3bbdeb 820 PerlIO_printf( Perl_debug_log, "%.*s", colwidth, "--------");
3dab1dad
YO
821 PerlIO_printf( Perl_debug_log, "\n");
822
786e8c11 823 for( state = 1 ; state < trie->laststate ; state++ ) {
be8e71aa 824 const U32 base = trie->states[ state ].trans.base;
3dab1dad
YO
825
826 PerlIO_printf( Perl_debug_log, "%*s#%4"UVXf"|", (int)depth * 2 + 2,"", (UV)state);
827
828 if ( trie->states[ state ].wordnum ) {
829 PerlIO_printf( Perl_debug_log, " W%4X", trie->states[ state ].wordnum );
830 } else {
831 PerlIO_printf( Perl_debug_log, "%6s", "" );
832 }
833
834 PerlIO_printf( Perl_debug_log, " @%4"UVXf" ", (UV)base );
835
836 if ( base ) {
837 U32 ofs = 0;
838
839 while( ( base + ofs < trie->uniquecharcount ) ||
840 ( base + ofs - trie->uniquecharcount < trie->lasttrans
841 && trie->trans[ base + ofs - trie->uniquecharcount ].check != state))
842 ofs++;
843
844 PerlIO_printf( Perl_debug_log, "+%2"UVXf"[ ", (UV)ofs);
845
846 for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) {
847 if ( ( base + ofs >= trie->uniquecharcount ) &&
848 ( base + ofs - trie->uniquecharcount < trie->lasttrans ) &&
849 trie->trans[ base + ofs - trie->uniquecharcount ].check == state )
850 {
ab3bbdeb
YO
851 PerlIO_printf( Perl_debug_log, "%*"UVXf,
852 colwidth,
3dab1dad
YO
853 (UV)trie->trans[ base + ofs - trie->uniquecharcount ].next );
854 } else {
ab3bbdeb 855 PerlIO_printf( Perl_debug_log, "%*s",colwidth," ." );
3dab1dad
YO
856 }
857 }
858
859 PerlIO_printf( Perl_debug_log, "]");
860
861 }
862 PerlIO_printf( Perl_debug_log, "\n" );
863 }
864}
865/*
866 dump_trie_interim_list(trie,next_alloc)
867 Dumps a fully constructed but uncompressed trie in list form.
868 List tries normally only are used for construction when the number of
869 possible chars (trie->uniquecharcount) is very high.
870 Used for debugging make_trie().
871*/
872STATIC void
873S_dump_trie_interim_list(pTHX_ const struct _reg_trie_data *trie, U32 next_alloc,U32 depth)
874{
875 U32 state;
ab3bbdeb
YO
876 SV *sv=sv_newmortal();
877 int colwidth= trie->widecharmap ? 6 : 4;
3dab1dad
YO
878 GET_RE_DEBUG_FLAGS_DECL;
879 /* print out the table precompression. */
ab3bbdeb
YO
880 PerlIO_printf( Perl_debug_log, "%*sState :Word | Transition Data\n%*s%s",
881 (int)depth * 2 + 2,"", (int)depth * 2 + 2,"",
882 "------:-----+-----------------\n" );
3dab1dad
YO
883
884 for( state=1 ; state < next_alloc ; state ++ ) {
885 U16 charid;
886
ab3bbdeb 887 PerlIO_printf( Perl_debug_log, "%*s %4"UVXf" :",
3dab1dad
YO
888 (int)depth * 2 + 2,"", (UV)state );
889 if ( ! trie->states[ state ].wordnum ) {
890 PerlIO_printf( Perl_debug_log, "%5s| ","");
891 } else {
892 PerlIO_printf( Perl_debug_log, "W%4x| ",
893 trie->states[ state ].wordnum
894 );
895 }
896 for( charid = 1 ; charid <= TRIE_LIST_USED( state ) ; charid++ ) {
be8e71aa 897 SV ** const tmp = av_fetch( trie->revcharmap, TRIE_LIST_ITEM(state,charid).forid, 0);
ab3bbdeb
YO
898 if ( tmp ) {
899 PerlIO_printf( Perl_debug_log, "%*s:%3X=%4"UVXf" | ",
900 colwidth,
ddc5bc0f 901 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
ab3bbdeb
YO
902 PL_colors[0], PL_colors[1],
903 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
904 PERL_PV_ESCAPE_FIRSTCHAR
905 ) ,
3dab1dad
YO
906 TRIE_LIST_ITEM(state,charid).forid,
907 (UV)TRIE_LIST_ITEM(state,charid).newstate
908 );
909 }
ab3bbdeb
YO
910 }
911 PerlIO_printf( Perl_debug_log, "\n");
3dab1dad
YO
912 }
913}
914
915/*
916 dump_trie_interim_table(trie,next_alloc)
917 Dumps a fully constructed but uncompressed trie in table form.
918 This is the normal DFA style state transition table, with a few
919 twists to facilitate compression later.
920 Used for debugging make_trie().
921*/
922STATIC void
923S_dump_trie_interim_table(pTHX_ const struct _reg_trie_data *trie, U32 next_alloc, U32 depth)
924{
925 U32 state;
926 U16 charid;
ab3bbdeb
YO
927 SV *sv=sv_newmortal();
928 int colwidth= trie->widecharmap ? 6 : 4;
3dab1dad
YO
929 GET_RE_DEBUG_FLAGS_DECL;
930
931 /*
932 print out the table precompression so that we can do a visual check
933 that they are identical.
934 */
935
936 PerlIO_printf( Perl_debug_log, "%*sChar : ",(int)depth * 2 + 2,"" );
937
938 for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) {
be8e71aa 939 SV ** const tmp = av_fetch( trie->revcharmap, charid, 0);
3dab1dad 940 if ( tmp ) {
ab3bbdeb
YO
941 PerlIO_printf( Perl_debug_log, "%*s",
942 colwidth,
ddc5bc0f 943 pv_pretty(sv, SvPV_nolen_const(*tmp), SvCUR(*tmp), colwidth,
ab3bbdeb
YO
944 PL_colors[0], PL_colors[1],
945 (SvUTF8(*tmp) ? PERL_PV_ESCAPE_UNI : 0) |
946 PERL_PV_ESCAPE_FIRSTCHAR
947 )
948 );
3dab1dad
YO
949 }
950 }
951
952 PerlIO_printf( Perl_debug_log, "\n%*sState+-",(int)depth * 2 + 2,"" );
953
954 for( charid=0 ; charid < trie->uniquecharcount ; charid++ ) {
ab3bbdeb 955 PerlIO_printf( Perl_debug_log, "%.*s", colwidth,"--------");
3dab1dad
YO
956 }
957
958 PerlIO_printf( Perl_debug_log, "\n" );
959
960 for( state=1 ; state < next_alloc ; state += trie->uniquecharcount ) {
961
962 PerlIO_printf( Perl_debug_log, "%*s%4"UVXf" : ",
963 (int)depth * 2 + 2,"",
964 (UV)TRIE_NODENUM( state ) );
965
966 for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) {
ab3bbdeb
YO
967 UV v=(UV)SAFE_TRIE_NODENUM( trie->trans[ state + charid ].next );
968 if (v)
969 PerlIO_printf( Perl_debug_log, "%*"UVXf, colwidth, v );
970 else
971 PerlIO_printf( Perl_debug_log, "%*s", colwidth, "." );
3dab1dad
YO
972 }
973 if ( ! trie->states[ TRIE_NODENUM( state ) ].wordnum ) {
974 PerlIO_printf( Perl_debug_log, " (%4"UVXf")\n", (UV)trie->trans[ state ].check );
975 } else {
976 PerlIO_printf( Perl_debug_log, " (%4"UVXf") W%4X\n", (UV)trie->trans[ state ].check,
977 trie->states[ TRIE_NODENUM( state ) ].wordnum );
978 }
979 }
07be1b83 980}
3dab1dad
YO
981
982#endif
983
786e8c11
YO
984/* make_trie(startbranch,first,last,tail,word_count,flags,depth)
985 startbranch: the first branch in the whole branch sequence
986 first : start branch of sequence of branch-exact nodes.
987 May be the same as startbranch
988 last : Thing following the last branch.
989 May be the same as tail.
990 tail : item following the branch sequence
991 count : words in the sequence
992 flags : currently the OP() type we will be building one of /EXACT(|F|Fl)/
993 depth : indent depth
3dab1dad 994
786e8c11 995Inplace optimizes a sequence of 2 or more Branch-Exact nodes into a TRIE node.
07be1b83 996
786e8c11
YO
997A trie is an N'ary tree where the branches are determined by digital
998decomposition of the key. IE, at the root node you look up the 1st character and
999follow that branch repeat until you find the end of the branches. Nodes can be
1000marked as "accepting" meaning they represent a complete word. Eg:
07be1b83 1001
786e8c11 1002 /he|she|his|hers/
72f13be8 1003
786e8c11
YO
1004would convert into the following structure. Numbers represent states, letters
1005following numbers represent valid transitions on the letter from that state, if
1006the number is in square brackets it represents an accepting state, otherwise it
1007will be in parenthesis.
07be1b83 1008
786e8c11
YO
1009 +-h->+-e->[3]-+-r->(8)-+-s->[9]
1010 | |
1011 | (2)
1012 | |
1013 (1) +-i->(6)-+-s->[7]
1014 |
1015 +-s->(3)-+-h->(4)-+-e->[5]
07be1b83 1016
786e8c11
YO
1017 Accept Word Mapping: 3=>1 (he),5=>2 (she), 7=>3 (his), 9=>4 (hers)
1018
1019This shows that when matching against the string 'hers' we will begin at state 1
1020read 'h' and move to state 2, read 'e' and move to state 3 which is accepting,
1021then read 'r' and go to state 8 followed by 's' which takes us to state 9 which
1022is also accepting. Thus we know that we can match both 'he' and 'hers' with a
1023single traverse. We store a mapping from accepting to state to which word was
1024matched, and then when we have multiple possibilities we try to complete the
1025rest of the regex in the order in which they occured in the alternation.
1026
1027The only prior NFA like behaviour that would be changed by the TRIE support is
1028the silent ignoring of duplicate alternations which are of the form:
1029
1030 / (DUPE|DUPE) X? (?{ ... }) Y /x
1031
1032Thus EVAL blocks follwing a trie may be called a different number of times with
1033and without the optimisation. With the optimisations dupes will be silently
1034ignored. This inconsistant behaviour of EVAL type nodes is well established as
1035the following demonstrates:
1036
1037 'words'=~/(word|word|word)(?{ print $1 })[xyz]/
1038
1039which prints out 'word' three times, but
1040
1041 'words'=~/(word|word|word)(?{ print $1 })S/
1042
1043which doesnt print it out at all. This is due to other optimisations kicking in.
1044
1045Example of what happens on a structural level:
1046
1047The regexp /(ac|ad|ab)+/ will produce the folowing debug output:
1048
1049 1: CURLYM[1] {1,32767}(18)
1050 5: BRANCH(8)
1051 6: EXACT <ac>(16)
1052 8: BRANCH(11)
1053 9: EXACT <ad>(16)
1054 11: BRANCH(14)
1055 12: EXACT <ab>(16)
1056 16: SUCCEED(0)
1057 17: NOTHING(18)
1058 18: END(0)
1059
1060This would be optimizable with startbranch=5, first=5, last=16, tail=16
1061and should turn into:
1062
1063 1: CURLYM[1] {1,32767}(18)
1064 5: TRIE(16)
1065 [Words:3 Chars Stored:6 Unique Chars:4 States:5 NCP:1]
1066 <ac>
1067 <ad>
1068 <ab>
1069 16: SUCCEED(0)
1070 17: NOTHING(18)
1071 18: END(0)
1072
1073Cases where tail != last would be like /(?foo|bar)baz/:
1074
1075 1: BRANCH(4)
1076 2: EXACT <foo>(8)
1077 4: BRANCH(7)
1078 5: EXACT <bar>(8)
1079 7: TAIL(8)
1080 8: EXACT <baz>(10)
1081 10: END(0)
1082
1083which would be optimizable with startbranch=1, first=1, last=7, tail=8
1084and would end up looking like:
1085
1086 1: TRIE(8)
1087 [Words:2 Chars Stored:6 Unique Chars:5 States:7 NCP:1]
1088 <foo>
1089 <bar>
1090 7: TAIL(8)
1091 8: EXACT <baz>(10)
1092 10: END(0)
1093
1094 d = uvuni_to_utf8_flags(d, uv, 0);
1095
1096is the recommended Unicode-aware way of saying
1097
1098 *(d++) = uv;
1099*/
1100
1101#define TRIE_STORE_REVCHAR \
1102 STMT_START { \
1103 SV *tmp = Perl_newSVpvf_nocontext( "%c", (int)uvc ); \
1104 if (UTF) SvUTF8_on(tmp); \
1105 av_push( TRIE_REVCHARMAP(trie), tmp ); \
1106 } STMT_END
1107
1108#define TRIE_READ_CHAR STMT_START { \
1109 wordlen++; \
1110 if ( UTF ) { \
1111 if ( folder ) { \
1112 if ( foldlen > 0 ) { \
1113 uvc = utf8n_to_uvuni( scan, UTF8_MAXLEN, &len, uniflags ); \
1114 foldlen -= len; \
1115 scan += len; \
1116 len = 0; \
1117 } else { \
1118 uvc = utf8n_to_uvuni( (const U8*)uc, UTF8_MAXLEN, &len, uniflags);\
1119 uvc = to_uni_fold( uvc, foldbuf, &foldlen ); \
1120 foldlen -= UNISKIP( uvc ); \
1121 scan = foldbuf + UNISKIP( uvc ); \
1122 } \
1123 } else { \
1124 uvc = utf8n_to_uvuni( (const U8*)uc, UTF8_MAXLEN, &len, uniflags);\
1125 } \
1126 } else { \
1127 uvc = (U32)*uc; \
1128 len = 1; \
1129 } \
1130} STMT_END
1131
1132
1133
1134#define TRIE_LIST_PUSH(state,fid,ns) STMT_START { \
1135 if ( TRIE_LIST_CUR( state ) >=TRIE_LIST_LEN( state ) ) { \
1136 TRIE_LIST_LEN( state ) *= 2; \
1137 Renew( trie->states[ state ].trans.list, \
1138 TRIE_LIST_LEN( state ), reg_trie_trans_le ); \
1139 } \
1140 TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).forid = fid; \
1141 TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).newstate = ns; \
1142 TRIE_LIST_CUR( state )++; \
1143} STMT_END
07be1b83 1144
786e8c11
YO
1145#define TRIE_LIST_NEW(state) STMT_START { \
1146 Newxz( trie->states[ state ].trans.list, \
1147 4, reg_trie_trans_le ); \
1148 TRIE_LIST_CUR( state ) = 1; \
1149 TRIE_LIST_LEN( state ) = 4; \
1150} STMT_END
07be1b83 1151
786e8c11
YO
1152#define TRIE_HANDLE_WORD(state) STMT_START { \
1153 U16 dupe= trie->states[ state ].wordnum; \
1154 regnode * const noper_next = regnext( noper ); \
1155 \
1156 if (trie->wordlen) \
1157 trie->wordlen[ curword ] = wordlen; \
1158 DEBUG_r({ \
1159 /* store the word for dumping */ \
1160 SV* tmp; \
1161 if (OP(noper) != NOTHING) \
1162 tmp = newSVpvn(STRING(noper), STR_LEN(noper)); \
1163 else \
1164 tmp = newSVpvn( "", 0 ); \
1165 if ( UTF ) SvUTF8_on( tmp ); \
1166 av_push( trie->words, tmp ); \
1167 }); \
1168 \
1169 curword++; \
1170 \
1171 if ( noper_next < tail ) { \
1172 if (!trie->jump) \
1173 Newxz( trie->jump, word_count + 1, U16); \
1174 trie->jump[curword] = (U16)(tail - noper_next); \
1175 if (!jumper) \
1176 jumper = noper_next; \
1177 if (!nextbranch) \
1178 nextbranch= regnext(cur); \
1179 } \
1180 \
1181 if ( dupe ) { \
1182 /* So it's a dupe. This means we need to maintain a */\
1183 /* linked-list from the first to the next. */\
1184 /* we only allocate the nextword buffer when there */\
1185 /* a dupe, so first time we have to do the allocation */\
1186 if (!trie->nextword) \
1187 Newxz( trie->nextword, word_count + 1, U16); \
1188 while ( trie->nextword[dupe] ) \
1189 dupe= trie->nextword[dupe]; \
1190 trie->nextword[dupe]= curword; \
1191 } else { \
1192 /* we haven't inserted this word yet. */ \
1193 trie->states[ state ].wordnum = curword; \
1194 } \
1195} STMT_END
07be1b83 1196
3dab1dad 1197
786e8c11
YO
1198#define TRIE_TRANS_STATE(state,base,ucharcount,charid,special) \
1199 ( ( base + charid >= ucharcount \
1200 && base + charid < ubound \
1201 && state == trie->trans[ base - ucharcount + charid ].check \
1202 && trie->trans[ base - ucharcount + charid ].next ) \
1203 ? trie->trans[ base - ucharcount + charid ].next \
1204 : ( state==1 ? special : 0 ) \
1205 )
3dab1dad 1206
786e8c11
YO
1207#define MADE_TRIE 1
1208#define MADE_JUMP_TRIE 2
1209#define MADE_EXACT_TRIE 4
3dab1dad 1210
a3621e74 1211STATIC I32
786e8c11 1212S_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch, regnode *first, regnode *last, regnode *tail, U32 word_count, U32 flags, U32 depth)
a3621e74 1213{
27da23d5 1214 dVAR;
a3621e74
YO
1215 /* first pass, loop through and scan words */
1216 reg_trie_data *trie;
1217 regnode *cur;
9f7f3913 1218 const U32 uniflags = UTF8_ALLOW_DEFAULT;
a3621e74
YO
1219 STRLEN len = 0;
1220 UV uvc = 0;
1221 U16 curword = 0;
1222 U32 next_alloc = 0;
786e8c11
YO
1223 regnode *jumper = NULL;
1224 regnode *nextbranch = NULL;
a3621e74 1225 /* we just use folder as a flag in utf8 */
e1ec3a88 1226 const U8 * const folder = ( flags == EXACTF
a3621e74
YO
1227 ? PL_fold
1228 : ( flags == EXACTFL
1229 ? PL_fold_locale
1230 : NULL
1231 )
1232 );
1233
e1ec3a88 1234 const U32 data_slot = add_data( pRExC_state, 1, "t" );
a3621e74 1235 SV *re_trie_maxbuff;
3dab1dad
YO
1236#ifndef DEBUGGING
1237 /* these are only used during construction but are useful during
8e11feef 1238 * debugging so we store them in the struct when debugging.
8e11feef 1239 */
3dab1dad 1240 STRLEN trie_charcount=0;
3dab1dad
YO
1241 AV *trie_revcharmap;
1242#endif
a3621e74 1243 GET_RE_DEBUG_FLAGS_DECL;
72f13be8
YO
1244#ifndef DEBUGGING
1245 PERL_UNUSED_ARG(depth);
1246#endif
a3621e74 1247
a02a5408 1248 Newxz( trie, 1, reg_trie_data );
a3621e74 1249 trie->refcount = 1;
3dab1dad 1250 trie->startstate = 1;
786e8c11 1251 trie->wordcount = word_count;
a3621e74 1252 RExC_rx->data->data[ data_slot ] = (void*)trie;
a02a5408 1253 Newxz( trie->charmap, 256, U16 );
3dab1dad
YO
1254 if (!(UTF && folder))
1255 Newxz( trie->bitmap, ANYOF_BITMAP_SIZE, char );
a3621e74
YO
1256 DEBUG_r({
1257 trie->words = newAV();
a3621e74 1258 });
3dab1dad 1259 TRIE_REVCHARMAP(trie) = newAV();
a3621e74 1260
0111c4fd 1261 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1);
a3621e74 1262 if (!SvIOK(re_trie_maxbuff)) {
0111c4fd 1263 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT);
a3621e74 1264 }
3dab1dad
YO
1265 DEBUG_OPTIMISE_r({
1266 PerlIO_printf( Perl_debug_log,
786e8c11 1267 "%*smake_trie start==%d, first==%d, last==%d, tail==%d depth=%d\n",
3dab1dad
YO
1268 (int)depth * 2 + 2, "",
1269 REG_NODE_NUM(startbranch),REG_NODE_NUM(first),
786e8c11 1270 REG_NODE_NUM(last), REG_NODE_NUM(tail),
85c3142d 1271 (int)depth);
3dab1dad 1272 });
a3621e74
YO
1273 /* -- First loop and Setup --
1274
1275 We first traverse the branches and scan each word to determine if it
1276 contains widechars, and how many unique chars there are, this is
1277 important as we have to build a table with at least as many columns as we
1278 have unique chars.
1279
1280 We use an array of integers to represent the character codes 0..255
1281 (trie->charmap) and we use a an HV* to store unicode characters. We use the
1282 native representation of the character value as the key and IV's for the
1283 coded index.
1284
1285 *TODO* If we keep track of how many times each character is used we can
1286 remap the columns so that the table compression later on is more
1287 efficient in terms of memory by ensuring most common value is in the
1288 middle and the least common are on the outside. IMO this would be better
1289 than a most to least common mapping as theres a decent chance the most
1290 common letter will share a node with the least common, meaning the node
1291 will not be compressable. With a middle is most common approach the worst
1292 case is when we have the least common nodes twice.
1293
1294 */
1295
a3621e74 1296 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
c445ea15 1297 regnode * const noper = NEXTOPER( cur );
e1ec3a88 1298 const U8 *uc = (U8*)STRING( noper );
a28509cc 1299 const U8 * const e = uc + STR_LEN( noper );
a3621e74
YO
1300 STRLEN foldlen = 0;
1301 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
2af232bd 1302 const U8 *scan = (U8*)NULL;
07be1b83 1303 U32 wordlen = 0; /* required init */
3dab1dad 1304 STRLEN chars=0;
a3621e74 1305
3dab1dad
YO
1306 if (OP(noper) == NOTHING) {
1307 trie->minlen= 0;
1308 continue;
1309 }
1310 if (trie->bitmap) {
1311 TRIE_BITMAP_SET(trie,*uc);
1312 if ( folder ) TRIE_BITMAP_SET(trie,folder[ *uc ]);
1313 }
a3621e74 1314 for ( ; uc < e ; uc += len ) {
3dab1dad 1315 TRIE_CHARCOUNT(trie)++;
a3621e74 1316 TRIE_READ_CHAR;
3dab1dad 1317 chars++;
a3621e74
YO
1318 if ( uvc < 256 ) {
1319 if ( !trie->charmap[ uvc ] ) {
1320 trie->charmap[ uvc ]=( ++trie->uniquecharcount );
1321 if ( folder )
1322 trie->charmap[ folder[ uvc ] ] = trie->charmap[ uvc ];
3dab1dad 1323 TRIE_STORE_REVCHAR;
a3621e74
YO
1324 }
1325 } else {
1326 SV** svpp;
1327 if ( !trie->widecharmap )
1328 trie->widecharmap = newHV();
1329
1330 svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 1 );
1331
1332 if ( !svpp )
e4584336 1333 Perl_croak( aTHX_ "error creating/fetching widecharmap entry for 0x%"UVXf, uvc );
a3621e74
YO
1334
1335 if ( !SvTRUE( *svpp ) ) {
1336 sv_setiv( *svpp, ++trie->uniquecharcount );
3dab1dad 1337 TRIE_STORE_REVCHAR;
a3621e74
YO
1338 }
1339 }
1340 }
3dab1dad
YO
1341 if( cur == first ) {
1342 trie->minlen=chars;
1343 trie->maxlen=chars;
1344 } else if (chars < trie->minlen) {
1345 trie->minlen=chars;
1346 } else if (chars > trie->maxlen) {
1347 trie->maxlen=chars;
1348 }
1349
a3621e74
YO
1350 } /* end first pass */
1351 DEBUG_TRIE_COMPILE_r(
3dab1dad
YO
1352 PerlIO_printf( Perl_debug_log, "%*sTRIE(%s): W:%d C:%d Uq:%d Min:%d Max:%d\n",
1353 (int)depth * 2 + 2,"",
85c3142d 1354 ( trie->widecharmap ? "UTF8" : "NATIVE" ), (int)word_count,
be8e71aa
YO
1355 (int)TRIE_CHARCOUNT(trie), trie->uniquecharcount,
1356 (int)trie->minlen, (int)trie->maxlen )
a3621e74 1357 );
786e8c11 1358 Newxz( trie->wordlen, word_count, U32 );
a3621e74
YO
1359
1360 /*
1361 We now know what we are dealing with in terms of unique chars and
1362 string sizes so we can calculate how much memory a naive
0111c4fd
RGS
1363 representation using a flat table will take. If it's over a reasonable
1364 limit (as specified by ${^RE_TRIE_MAXBUF}) we use a more memory
a3621e74
YO
1365 conservative but potentially much slower representation using an array
1366 of lists.
1367
1368 At the end we convert both representations into the same compressed
1369 form that will be used in regexec.c for matching with. The latter
1370 is a form that cannot be used to construct with but has memory
1371 properties similar to the list form and access properties similar
1372 to the table form making it both suitable for fast searches and
1373 small enough that its feasable to store for the duration of a program.
1374
1375 See the comment in the code where the compressed table is produced
1376 inplace from the flat tabe representation for an explanation of how
1377 the compression works.
1378
1379 */
1380
1381
3dab1dad 1382 if ( (IV)( ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1) > SvIV(re_trie_maxbuff) ) {
a3621e74
YO
1383 /*
1384 Second Pass -- Array Of Lists Representation
1385
1386 Each state will be represented by a list of charid:state records
1387 (reg_trie_trans_le) the first such element holds the CUR and LEN
1388 points of the allocated array. (See defines above).
1389
1390 We build the initial structure using the lists, and then convert
1391 it into the compressed table form which allows faster lookups
1392 (but cant be modified once converted).
a3621e74
YO
1393 */
1394
a3621e74
YO
1395 STRLEN transcount = 1;
1396
3dab1dad 1397 Newxz( trie->states, TRIE_CHARCOUNT(trie) + 2, reg_trie_state );
a3621e74
YO
1398 TRIE_LIST_NEW(1);
1399 next_alloc = 2;
1400
1401 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1402
c445ea15
AL
1403 regnode * const noper = NEXTOPER( cur );
1404 U8 *uc = (U8*)STRING( noper );
1405 const U8 * const e = uc + STR_LEN( noper );
1406 U32 state = 1; /* required init */
1407 U16 charid = 0; /* sanity init */
1408 U8 *scan = (U8*)NULL; /* sanity init */
1409 STRLEN foldlen = 0; /* required init */
07be1b83 1410 U32 wordlen = 0; /* required init */
c445ea15
AL
1411 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1412
3dab1dad 1413 if (OP(noper) != NOTHING) {
786e8c11 1414 for ( ; uc < e ; uc += len ) {
c445ea15 1415
786e8c11 1416 TRIE_READ_CHAR;
c445ea15 1417
786e8c11
YO
1418 if ( uvc < 256 ) {
1419 charid = trie->charmap[ uvc ];
c445ea15 1420 } else {
786e8c11
YO
1421 SV** const svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 0);
1422 if ( !svpp ) {
1423 charid = 0;
1424 } else {
1425 charid=(U16)SvIV( *svpp );
1426 }
c445ea15 1427 }
786e8c11
YO
1428 /* charid is now 0 if we dont know the char read, or nonzero if we do */
1429 if ( charid ) {
a3621e74 1430
786e8c11
YO
1431 U16 check;
1432 U32 newstate = 0;
a3621e74 1433
786e8c11
YO
1434 charid--;
1435 if ( !trie->states[ state ].trans.list ) {
1436 TRIE_LIST_NEW( state );
c445ea15 1437 }
786e8c11
YO
1438 for ( check = 1; check <= TRIE_LIST_USED( state ); check++ ) {
1439 if ( TRIE_LIST_ITEM( state, check ).forid == charid ) {
1440 newstate = TRIE_LIST_ITEM( state, check ).newstate;
1441 break;
1442 }
1443 }
1444 if ( ! newstate ) {
1445 newstate = next_alloc++;
1446 TRIE_LIST_PUSH( state, charid, newstate );
1447 transcount++;
1448 }
1449 state = newstate;
1450 } else {
1451 Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc );
c445ea15 1452 }
a28509cc 1453 }
c445ea15 1454 }
3dab1dad 1455 TRIE_HANDLE_WORD(state);
a3621e74
YO
1456
1457 } /* end second pass */
1458
786e8c11 1459 trie->laststate = next_alloc;
a3621e74
YO
1460 Renew( trie->states, next_alloc, reg_trie_state );
1461
3dab1dad
YO
1462 /* and now dump it out before we compress it */
1463 DEBUG_TRIE_COMPILE_MORE_r(
1464 dump_trie_interim_list(trie,next_alloc,depth+1)
a3621e74 1465 );
a3621e74 1466
a02a5408 1467 Newxz( trie->trans, transcount ,reg_trie_trans );
a3621e74
YO
1468 {
1469 U32 state;
a3621e74
YO
1470 U32 tp = 0;
1471 U32 zp = 0;
1472
1473
1474 for( state=1 ; state < next_alloc ; state ++ ) {
1475 U32 base=0;
1476
1477 /*
1478 DEBUG_TRIE_COMPILE_MORE_r(
1479 PerlIO_printf( Perl_debug_log, "tp: %d zp: %d ",tp,zp)
1480 );
1481 */
1482
1483 if (trie->states[state].trans.list) {
1484 U16 minid=TRIE_LIST_ITEM( state, 1).forid;
1485 U16 maxid=minid;
a28509cc 1486 U16 idx;
a3621e74
YO
1487
1488 for( idx = 2 ; idx <= TRIE_LIST_USED( state ) ; idx++ ) {
c445ea15
AL
1489 const U16 forid = TRIE_LIST_ITEM( state, idx).forid;
1490 if ( forid < minid ) {
1491 minid=forid;
1492 } else if ( forid > maxid ) {
1493 maxid=forid;
1494 }
a3621e74
YO
1495 }
1496 if ( transcount < tp + maxid - minid + 1) {
1497 transcount *= 2;
1498 Renew( trie->trans, transcount, reg_trie_trans );
1499 Zero( trie->trans + (transcount / 2), transcount / 2 , reg_trie_trans );
1500 }
1501 base = trie->uniquecharcount + tp - minid;
1502 if ( maxid == minid ) {
1503 U32 set = 0;
1504 for ( ; zp < tp ; zp++ ) {
1505 if ( ! trie->trans[ zp ].next ) {
1506 base = trie->uniquecharcount + zp - minid;
1507 trie->trans[ zp ].next = TRIE_LIST_ITEM( state, 1).newstate;
1508 trie->trans[ zp ].check = state;
1509 set = 1;
1510 break;
1511 }
1512 }
1513 if ( !set ) {
1514 trie->trans[ tp ].next = TRIE_LIST_ITEM( state, 1).newstate;
1515 trie->trans[ tp ].check = state;
1516 tp++;
1517 zp = tp;
1518 }
1519 } else {
1520 for ( idx=1; idx <= TRIE_LIST_USED( state ) ; idx++ ) {
c445ea15 1521 const U32 tid = base - trie->uniquecharcount + TRIE_LIST_ITEM( state, idx ).forid;
a3621e74
YO
1522 trie->trans[ tid ].next = TRIE_LIST_ITEM( state, idx ).newstate;
1523 trie->trans[ tid ].check = state;
1524 }
1525 tp += ( maxid - minid + 1 );
1526 }
1527 Safefree(trie->states[ state ].trans.list);
1528 }
1529 /*
1530 DEBUG_TRIE_COMPILE_MORE_r(
1531 PerlIO_printf( Perl_debug_log, " base: %d\n",base);
1532 );
1533 */
1534 trie->states[ state ].trans.base=base;
1535 }
cc601c31 1536 trie->lasttrans = tp + 1;
a3621e74
YO
1537 }
1538 } else {
1539 /*
1540 Second Pass -- Flat Table Representation.
1541
1542 we dont use the 0 slot of either trans[] or states[] so we add 1 to each.
1543 We know that we will need Charcount+1 trans at most to store the data
1544 (one row per char at worst case) So we preallocate both structures
1545 assuming worst case.
1546
1547 We then construct the trie using only the .next slots of the entry
1548 structs.
1549
1550 We use the .check field of the first entry of the node temporarily to
1551 make compression both faster and easier by keeping track of how many non
1552 zero fields are in the node.
1553
1554 Since trans are numbered from 1 any 0 pointer in the table is a FAIL
1555 transition.
1556
1557 There are two terms at use here: state as a TRIE_NODEIDX() which is a
1558 number representing the first entry of the node, and state as a
1559 TRIE_NODENUM() which is the trans number. state 1 is TRIE_NODEIDX(1) and
1560 TRIE_NODENUM(1), state 2 is TRIE_NODEIDX(2) and TRIE_NODENUM(3) if there
1561 are 2 entrys per node. eg:
1562
1563 A B A B
1564 1. 2 4 1. 3 7
1565 2. 0 3 3. 0 5
1566 3. 0 0 5. 0 0
1567 4. 0 0 7. 0 0
1568
1569 The table is internally in the right hand, idx form. However as we also
1570 have to deal with the states array which is indexed by nodenum we have to
1571 use TRIE_NODENUM() to convert.
1572
1573 */
1574
3dab1dad
YO
1575
1576 Newxz( trie->trans, ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1,
a3621e74 1577 reg_trie_trans );
3dab1dad 1578 Newxz( trie->states, TRIE_CHARCOUNT(trie) + 2, reg_trie_state );
a3621e74
YO
1579 next_alloc = trie->uniquecharcount + 1;
1580
3dab1dad 1581
a3621e74
YO
1582 for ( cur = first ; cur < last ; cur = regnext( cur ) ) {
1583
c445ea15 1584 regnode * const noper = NEXTOPER( cur );
a28509cc
AL
1585 const U8 *uc = (U8*)STRING( noper );
1586 const U8 * const e = uc + STR_LEN( noper );
a3621e74
YO
1587
1588 U32 state = 1; /* required init */
1589
1590 U16 charid = 0; /* sanity init */
1591 U32 accept_state = 0; /* sanity init */
1592 U8 *scan = (U8*)NULL; /* sanity init */
1593
1594 STRLEN foldlen = 0; /* required init */
07be1b83 1595 U32 wordlen = 0; /* required init */
a3621e74
YO
1596 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ];
1597
3dab1dad 1598 if ( OP(noper) != NOTHING ) {
786e8c11 1599 for ( ; uc < e ; uc += len ) {
a3621e74 1600
786e8c11 1601 TRIE_READ_CHAR;
a3621e74 1602
786e8c11
YO
1603 if ( uvc < 256 ) {
1604 charid = trie->charmap[ uvc ];
1605 } else {
1606 SV* const * const svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 0);
1607 charid = svpp ? (U16)SvIV(*svpp) : 0;
a3621e74 1608 }
786e8c11
YO
1609 if ( charid ) {
1610 charid--;
1611 if ( !trie->trans[ state + charid ].next ) {
1612 trie->trans[ state + charid ].next = next_alloc;
1613 trie->trans[ state ].check++;
1614 next_alloc += trie->uniquecharcount;
1615 }
1616 state = trie->trans[ state + charid ].next;
1617 } else {
1618 Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc );
1619 }
1620 /* charid is now 0 if we dont know the char read, or nonzero if we do */
a3621e74 1621 }
a3621e74 1622 }
3dab1dad
YO
1623 accept_state = TRIE_NODENUM( state );
1624 TRIE_HANDLE_WORD(accept_state);
a3621e74
YO
1625
1626 } /* end second pass */
1627
3dab1dad
YO
1628 /* and now dump it out before we compress it */
1629 DEBUG_TRIE_COMPILE_MORE_r(
1630 dump_trie_interim_table(trie,next_alloc,depth+1)
1631 );
a3621e74 1632
a3621e74
YO
1633 {
1634 /*
1635 * Inplace compress the table.*
1636
1637 For sparse data sets the table constructed by the trie algorithm will
1638 be mostly 0/FAIL transitions or to put it another way mostly empty.
1639 (Note that leaf nodes will not contain any transitions.)
1640
1641 This algorithm compresses the tables by eliminating most such
1642 transitions, at the cost of a modest bit of extra work during lookup:
1643
1644 - Each states[] entry contains a .base field which indicates the
1645 index in the state[] array wheres its transition data is stored.
1646
1647 - If .base is 0 there are no valid transitions from that node.
1648
1649 - If .base is nonzero then charid is added to it to find an entry in
1650 the trans array.
1651
1652 -If trans[states[state].base+charid].check!=state then the
1653 transition is taken to be a 0/Fail transition. Thus if there are fail
1654 transitions at the front of the node then the .base offset will point
1655 somewhere inside the previous nodes data (or maybe even into a node
1656 even earlier), but the .check field determines if the transition is
1657 valid.
1658
786e8c11 1659 XXX - wrong maybe?
a3621e74
YO
1660 The following process inplace converts the table to the compressed
1661 table: We first do not compress the root node 1,and mark its all its
1662 .check pointers as 1 and set its .base pointer as 1 as well. This
1663 allows to do a DFA construction from the compressed table later, and
1664 ensures that any .base pointers we calculate later are greater than
1665 0.
1666
1667 - We set 'pos' to indicate the first entry of the second node.
1668
1669 - We then iterate over the columns of the node, finding the first and
1670 last used entry at l and m. We then copy l..m into pos..(pos+m-l),
1671 and set the .check pointers accordingly, and advance pos
1672 appropriately and repreat for the next node. Note that when we copy
1673 the next pointers we have to convert them from the original
1674 NODEIDX form to NODENUM form as the former is not valid post
1675 compression.
1676
1677 - If a node has no transitions used we mark its base as 0 and do not
1678 advance the pos pointer.
1679
1680 - If a node only has one transition we use a second pointer into the
1681 structure to fill in allocated fail transitions from other states.
1682 This pointer is independent of the main pointer and scans forward
1683 looking for null transitions that are allocated to a state. When it
1684 finds one it writes the single transition into the "hole". If the
786e8c11 1685 pointer doesnt find one the single transition is appended as normal.
a3621e74
YO
1686
1687 - Once compressed we can Renew/realloc the structures to release the
1688 excess space.
1689
1690 See "Table-Compression Methods" in sec 3.9 of the Red Dragon,
1691 specifically Fig 3.47 and the associated pseudocode.
1692
1693 demq
1694 */
a3b680e6 1695 const U32 laststate = TRIE_NODENUM( next_alloc );
a28509cc 1696 U32 state, charid;
a3621e74 1697 U32 pos = 0, zp=0;
786e8c11 1698 trie->laststate = laststate;
a3621e74
YO
1699
1700 for ( state = 1 ; state < laststate ; state++ ) {
1701 U8 flag = 0;
a28509cc
AL
1702 const U32 stateidx = TRIE_NODEIDX( state );
1703 const U32 o_used = trie->trans[ stateidx ].check;
1704 U32 used = trie->trans[ stateidx ].check;
a3621e74
YO
1705 trie->trans[ stateidx ].check = 0;
1706
1707 for ( charid = 0 ; used && charid < trie->uniquecharcount ; charid++ ) {
1708 if ( flag || trie->trans[ stateidx + charid ].next ) {
1709 if ( trie->trans[ stateidx + charid ].next ) {
1710 if (o_used == 1) {
1711 for ( ; zp < pos ; zp++ ) {
1712 if ( ! trie->trans[ zp ].next ) {
1713 break;
1714 }
1715 }
1716 trie->states[ state ].trans.base = zp + trie->uniquecharcount - charid ;
1717 trie->trans[ zp ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next );
1718 trie->trans[ zp ].check = state;
1719 if ( ++zp > pos ) pos = zp;
1720 break;
1721 }
1722 used--;
1723 }
1724 if ( !flag ) {
1725 flag = 1;
1726 trie->states[ state ].trans.base = pos + trie->uniquecharcount - charid ;
1727 }
1728 trie->trans[ pos ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next );
1729 trie->trans[ pos ].check = state;
1730 pos++;
1731 }
1732 }
1733 }
cc601c31 1734 trie->lasttrans = pos + 1;
a3621e74
YO
1735 Renew( trie->states, laststate + 1, reg_trie_state);
1736 DEBUG_TRIE_COMPILE_MORE_r(
e4584336 1737 PerlIO_printf( Perl_debug_log,
3dab1dad
YO
1738 "%*sAlloc: %d Orig: %"IVdf" elements, Final:%"IVdf". Savings of %%%5.2f\n",
1739 (int)depth * 2 + 2,"",
1740 (int)( ( TRIE_CHARCOUNT(trie) + 1 ) * trie->uniquecharcount + 1 ),
5d7488b2
AL
1741 (IV)next_alloc,
1742 (IV)pos,
a3621e74
YO
1743 ( ( next_alloc - pos ) * 100 ) / (double)next_alloc );
1744 );
1745
1746 } /* end table compress */
1747 }
cc601c31
YO
1748 /* resize the trans array to remove unused space */
1749 Renew( trie->trans, trie->lasttrans, reg_trie_trans);
a3621e74 1750
3dab1dad
YO
1751 /* and now dump out the compressed format */
1752 DEBUG_TRIE_COMPILE_r(
1753 dump_trie(trie,depth+1)
1754 );
07be1b83 1755
3dab1dad
YO
1756 { /* Modify the program and insert the new TRIE node*/
1757 regnode *convert;
1758 U8 nodetype =(U8)(flags & 0xFF);
1759 char *str=NULL;
786e8c11 1760
07be1b83 1761#ifdef DEBUGGING
e62cc96a 1762 regnode *optimize = NULL;
b57a0404
JH
1763 U32 mjd_offset = 0;
1764 U32 mjd_nodelen = 0;
07be1b83 1765#endif
a3621e74 1766 /*
3dab1dad
YO
1767 This means we convert either the first branch or the first Exact,
1768 depending on whether the thing following (in 'last') is a branch
1769 or not and whther first is the startbranch (ie is it a sub part of
1770 the alternation or is it the whole thing.)
1771 Assuming its a sub part we conver the EXACT otherwise we convert
1772 the whole branch sequence, including the first.
a3621e74 1773 */
3dab1dad
YO
1774 /* Find the node we are going to overwrite */
1775 if ( first == startbranch && OP( last ) != BRANCH ) {
07be1b83 1776 /* whole branch chain */
3dab1dad 1777 convert = first;
07be1b83
YO
1778 DEBUG_r({
1779 const regnode *nop = NEXTOPER( convert );
1780 mjd_offset= Node_Offset((nop));
1781 mjd_nodelen= Node_Length((nop));
1782 });
1783 } else {
1784 /* branch sub-chain */
3dab1dad
YO
1785 convert = NEXTOPER( first );
1786 NEXT_OFF( first ) = (U16)(last - first);
07be1b83
YO
1787 DEBUG_r({
1788 mjd_offset= Node_Offset((convert));
1789 mjd_nodelen= Node_Length((convert));
1790 });
1791 }
1792 DEBUG_OPTIMISE_r(
1793 PerlIO_printf(Perl_debug_log, "%*sMJD offset:%"UVuf" MJD length:%"UVuf"\n",
1794 (int)depth * 2 + 2, "",
786e8c11 1795 (UV)mjd_offset, (UV)mjd_nodelen)
07be1b83 1796 );
a3621e74 1797
3dab1dad
YO
1798 /* But first we check to see if there is a common prefix we can
1799 split out as an EXACT and put in front of the TRIE node. */
1800 trie->startstate= 1;
786e8c11 1801 if ( trie->bitmap && !trie->widecharmap && !trie->jump ) {
3dab1dad
YO
1802 U32 state;
1803 DEBUG_OPTIMISE_r(
8e11feef
RGS
1804 PerlIO_printf(Perl_debug_log, "%*sLaststate:%"UVuf"\n",
1805 (int)depth * 2 + 2, "",
786e8c11 1806 (UV)trie->laststate)
8e11feef 1807 );
786e8c11 1808 for ( state = 1 ; state < trie->laststate-1 ; state++ ) {
a3621e74 1809 U32 ofs = 0;
8e11feef
RGS
1810 I32 idx = -1;
1811 U32 count = 0;
1812 const U32 base = trie->states[ state ].trans.base;
a3621e74 1813
3dab1dad 1814 if ( trie->states[state].wordnum )
8e11feef 1815 count = 1;
a3621e74 1816
8e11feef 1817 for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) {
cc601c31
YO
1818 if ( ( base + ofs >= trie->uniquecharcount ) &&
1819 ( base + ofs - trie->uniquecharcount < trie->lasttrans ) &&
a3621e74
YO
1820 trie->trans[ base + ofs - trie->uniquecharcount ].check == state )
1821 {
3dab1dad 1822 if ( ++count > 1 ) {
8e11feef 1823 SV **tmp = av_fetch( TRIE_REVCHARMAP(trie), ofs, 0);
07be1b83 1824 const U8 *ch = (U8*)SvPV_nolen_const( *tmp );
8e11feef 1825 if ( state == 1 ) break;
3dab1dad
YO
1826 if ( count == 2 ) {
1827 Zero(trie->bitmap, ANYOF_BITMAP_SIZE, char);
1828 DEBUG_OPTIMISE_r(
8e11feef
RGS
1829 PerlIO_printf(Perl_debug_log,
1830 "%*sNew Start State=%"UVuf" Class: [",
1831 (int)depth * 2 + 2, "",
786e8c11 1832 (UV)state));
be8e71aa
YO
1833 if (idx >= 0) {
1834 SV ** const tmp = av_fetch( TRIE_REVCHARMAP(trie), idx, 0);
1835 const U8 * const ch = (U8*)SvPV_nolen_const( *tmp );
8e11feef 1836
3dab1dad 1837 TRIE_BITMAP_SET(trie,*ch);
8e11feef
RGS
1838 if ( folder )
1839 TRIE_BITMAP_SET(trie, folder[ *ch ]);
3dab1dad 1840 DEBUG_OPTIMISE_r(
07be1b83 1841 PerlIO_printf(Perl_debug_log, (char*)ch)
3dab1dad 1842 );
8e11feef
RGS
1843 }
1844 }
1845 TRIE_BITMAP_SET(trie,*ch);
1846 if ( folder )
1847 TRIE_BITMAP_SET(trie,folder[ *ch ]);
1848 DEBUG_OPTIMISE_r(PerlIO_printf( Perl_debug_log,"%s", ch));
1849 }
1850 idx = ofs;
1851 }
3dab1dad
YO
1852 }
1853 if ( count == 1 ) {
1854 SV **tmp = av_fetch( TRIE_REVCHARMAP(trie), idx, 0);
8e11feef 1855 const char *ch = SvPV_nolen_const( *tmp );
3dab1dad 1856 DEBUG_OPTIMISE_r(
8e11feef
RGS
1857 PerlIO_printf( Perl_debug_log,
1858 "%*sPrefix State: %"UVuf" Idx:%"UVuf" Char='%s'\n",
1859 (int)depth * 2 + 2, "",
786e8c11 1860 (UV)state, (UV)idx, ch)
3dab1dad
YO
1861 );
1862 if ( state==1 ) {
1863 OP( convert ) = nodetype;
1864 str=STRING(convert);
1865 STR_LEN(convert)=0;
1866 }
1867 *str++=*ch;
1868 STR_LEN(convert)++;
a3621e74 1869
8e11feef 1870 } else {
f9049ba1 1871#ifdef DEBUGGING
8e11feef
RGS
1872 if (state>1)
1873 DEBUG_OPTIMISE_r(PerlIO_printf( Perl_debug_log,"]\n"));
f9049ba1 1874#endif
8e11feef
RGS
1875 break;
1876 }
1877 }
3dab1dad 1878 if (str) {
8e11feef 1879 regnode *n = convert+NODE_SZ_STR(convert);
07be1b83 1880 NEXT_OFF(convert) = NODE_SZ_STR(convert);
8e11feef 1881 trie->startstate = state;
07be1b83
YO
1882 trie->minlen -= (state - 1);
1883 trie->maxlen -= (state - 1);
1884 DEBUG_r({
1885 regnode *fix = convert;
1886 mjd_nodelen++;
1887 Set_Node_Offset_Length(convert, mjd_offset, state - 1);
1888 while( ++fix < n ) {
1889 Set_Node_Offset_Length(fix, 0, 0);
1890 }
1891 });
8e11feef
RGS
1892 if (trie->maxlen) {
1893 convert = n;
1894 } else {
3dab1dad 1895 NEXT_OFF(convert) = (U16)(tail - convert);
a5ca303d 1896 DEBUG_r(optimize= n);
3dab1dad
YO
1897 }
1898 }
1899 }
a5ca303d
YO
1900 if (!jumper)
1901 jumper = last;
3dab1dad 1902 if ( trie->maxlen ) {
8e11feef
RGS
1903 NEXT_OFF( convert ) = (U16)(tail - convert);
1904 ARG_SET( convert, data_slot );
786e8c11
YO
1905 /* Store the offset to the first unabsorbed branch in
1906 jump[0], which is otherwise unused by the jump logic.
1907 We use this when dumping a trie and during optimisation. */
1908 if (trie->jump)
1909 trie->jump[0] = (U16)(tail - nextbranch);
a5ca303d 1910
786e8c11
YO
1911 /* XXXX */
1912 if ( !trie->states[trie->startstate].wordnum && trie->bitmap &&
1de06328 1913 ( (char *)jumper - (char *)convert) >= (int)sizeof(struct regnode_charclass) )
786e8c11
YO
1914 {
1915 OP( convert ) = TRIEC;
1916 Copy(trie->bitmap, ((struct regnode_charclass *)convert)->bitmap, ANYOF_BITMAP_SIZE, char);
1917 Safefree(trie->bitmap);
1918 trie->bitmap= NULL;
1919 } else
1920 OP( convert ) = TRIE;
a3621e74 1921
3dab1dad
YO
1922 /* store the type in the flags */
1923 convert->flags = nodetype;
a5ca303d
YO
1924 DEBUG_r({
1925 optimize = convert
1926 + NODE_STEP_REGNODE
1927 + regarglen[ OP( convert ) ];
1928 });
1929 /* XXX We really should free up the resource in trie now,
1930 as we won't use them - (which resources?) dmq */
3dab1dad 1931 }
a3621e74 1932 /* needed for dumping*/
e62cc96a 1933 DEBUG_r(if (optimize) {
07be1b83 1934 regnode *opt = convert;
e62cc96a 1935 while ( ++opt < optimize) {
07be1b83
YO
1936 Set_Node_Offset_Length(opt,0,0);
1937 }
786e8c11
YO
1938 /*
1939 Try to clean up some of the debris left after the
1940 optimisation.
a3621e74 1941 */
786e8c11 1942 while( optimize < jumper ) {
07be1b83 1943 mjd_nodelen += Node_Length((optimize));
a3621e74 1944 OP( optimize ) = OPTIMIZED;
07be1b83 1945 Set_Node_Offset_Length(optimize,0,0);
a3621e74
YO
1946 optimize++;
1947 }
07be1b83 1948 Set_Node_Offset_Length(convert,mjd_offset,mjd_nodelen);
a3621e74
YO
1949 });
1950 } /* end node insert */
07be1b83 1951#ifndef DEBUGGING
6e8b4190 1952 SvREFCNT_dec(TRIE_REVCHARMAP(trie));
07be1b83 1953#endif
786e8c11
YO
1954 return trie->jump
1955 ? MADE_JUMP_TRIE
1956 : trie->startstate>1
1957 ? MADE_EXACT_TRIE
1958 : MADE_TRIE;
1959}
1960
1961STATIC void
1962S_make_trie_failtable(pTHX_ RExC_state_t *pRExC_state, regnode *source, regnode *stclass, U32 depth)
1963{
1964/* The Trie is constructed and compressed now so we can build a fail array now if its needed
1965
1966 This is basically the Aho-Corasick algorithm. Its from exercise 3.31 and 3.32 in the
1967 "Red Dragon" -- Compilers, principles, techniques, and tools. Aho, Sethi, Ullman 1985/88
1968 ISBN 0-201-10088-6
1969
1970 We find the fail state for each state in the trie, this state is the longest proper
1971 suffix of the current states 'word' that is also a proper prefix of another word in our
1972 trie. State 1 represents the word '' and is the thus the default fail state. This allows
1973 the DFA not to have to restart after its tried and failed a word at a given point, it
1974 simply continues as though it had been matching the other word in the first place.
1975 Consider
1976 'abcdgu'=~/abcdefg|cdgu/
1977 When we get to 'd' we are still matching the first word, we would encounter 'g' which would
1978 fail, which would bring use to the state representing 'd' in the second word where we would
1979 try 'g' and succeed, prodceding to match 'cdgu'.
1980 */
1981 /* add a fail transition */
1982 reg_trie_data *trie=(reg_trie_data *)RExC_rx->data->data[ARG(source)];
1983 U32 *q;
1984 const U32 ucharcount = trie->uniquecharcount;
1985 const U32 numstates = trie->laststate;
1986 const U32 ubound = trie->lasttrans + ucharcount;
1987 U32 q_read = 0;
1988 U32 q_write = 0;
1989 U32 charid;
1990 U32 base = trie->states[ 1 ].trans.base;
1991 U32 *fail;
1992 reg_ac_data *aho;
1993 const U32 data_slot = add_data( pRExC_state, 1, "T" );
1994 GET_RE_DEBUG_FLAGS_DECL;
1995#ifndef DEBUGGING
1996 PERL_UNUSED_ARG(depth);
1997#endif
1998
1999
2000 ARG_SET( stclass, data_slot );
2001 Newxz( aho, 1, reg_ac_data );
2002 RExC_rx->data->data[ data_slot ] = (void*)aho;
2003 aho->trie=trie;
2004 aho->states=(reg_trie_state *)savepvn((const char*)trie->states,
2005 (trie->laststate+1)*sizeof(reg_trie_state));
2006 Newxz( q, numstates, U32);
2007 Newxz( aho->fail, numstates, U32 );
2008 aho->refcount = 1;
2009 fail = aho->fail;
2010 /* initialize fail[0..1] to be 1 so that we always have
2011 a valid final fail state */
2012 fail[ 0 ] = fail[ 1 ] = 1;
2013
2014 for ( charid = 0; charid < ucharcount ; charid++ ) {
2015 const U32 newstate = TRIE_TRANS_STATE( 1, base, ucharcount, charid, 0 );
2016 if ( newstate ) {
2017 q[ q_write ] = newstate;
2018 /* set to point at the root */
2019 fail[ q[ q_write++ ] ]=1;
2020 }
2021 }
2022 while ( q_read < q_write) {
2023 const U32 cur = q[ q_read++ % numstates ];
2024 base = trie->states[ cur ].trans.base;
2025
2026 for ( charid = 0 ; charid < ucharcount ; charid++ ) {
2027 const U32 ch_state = TRIE_TRANS_STATE( cur, base, ucharcount, charid, 1 );
2028 if (ch_state) {
2029 U32 fail_state = cur;
2030 U32 fail_base;
2031 do {
2032 fail_state = fail[ fail_state ];
2033 fail_base = aho->states[ fail_state ].trans.base;
2034 } while ( !TRIE_TRANS_STATE( fail_state, fail_base, ucharcount, charid, 1 ) );
2035
2036 fail_state = TRIE_TRANS_STATE( fail_state, fail_base, ucharcount, charid, 1 );
2037 fail[ ch_state ] = fail_state;
2038 if ( !aho->states[ ch_state ].wordnum && aho->states[ fail_state ].wordnum )
2039 {
2040 aho->states[ ch_state ].wordnum = aho->states[ fail_state ].wordnum;
2041 }
2042 q[ q_write++ % numstates] = ch_state;
2043 }
2044 }
2045 }
2046 /* restore fail[0..1] to 0 so that we "fall out" of the AC loop
2047 when we fail in state 1, this allows us to use the
2048 charclass scan to find a valid start char. This is based on the principle
2049 that theres a good chance the string being searched contains lots of stuff
2050 that cant be a start char.
2051 */
2052 fail[ 0 ] = fail[ 1 ] = 0;
2053 DEBUG_TRIE_COMPILE_r({
2054 PerlIO_printf(Perl_debug_log, "%*sStclass Failtable: 0", (int)(depth * 2), "");
2055 for( q_read=1; q_read<numstates; q_read++ ) {
2056 PerlIO_printf(Perl_debug_log, ", %"UVuf, (UV)fail[q_read]);
2057 }
2058 PerlIO_printf(Perl_debug_log, "\n");
2059 });
2060 Safefree(q);
2061 /*RExC_seen |= REG_SEEN_TRIEDFA;*/
a3621e74
YO
2062}
2063
786e8c11 2064
a3621e74 2065/*
5d1c421c
JH
2066 * There are strange code-generation bugs caused on sparc64 by gcc-2.95.2.
2067 * These need to be revisited when a newer toolchain becomes available.
2068 */
2069#if defined(__sparc64__) && defined(__GNUC__)
2070# if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
2071# undef SPARC64_GCC_WORKAROUND
2072# define SPARC64_GCC_WORKAROUND 1
2073# endif
2074#endif
2075
07be1b83
YO
2076#define DEBUG_PEEP(str,scan,depth) \
2077 DEBUG_OPTIMISE_r({ \
2078 SV * const mysv=sv_newmortal(); \
2079 regnode *Next = regnext(scan); \
2080 regprop(RExC_rx, mysv, scan); \
2081 PerlIO_printf(Perl_debug_log, "%*s" str ">%3d: %s [%d]\n", \
2082 (int)depth*2, "", REG_NODE_NUM(scan), SvPV_nolen_const(mysv),\
2083 Next ? (REG_NODE_NUM(Next)) : 0 ); \
2084 });
2085
1de06328
YO
2086
2087
2088
2089
07be1b83
YO
2090#define JOIN_EXACT(scan,min,flags) \
2091 if (PL_regkind[OP(scan)] == EXACT) \
2092 join_exact(pRExC_state,(scan),(min),(flags),NULL,depth+1)
2093
be8e71aa 2094STATIC U32
07be1b83
YO
2095S_join_exact(pTHX_ RExC_state_t *pRExC_state, regnode *scan, I32 *min, U32 flags,regnode *val, U32 depth) {
2096 /* Merge several consecutive EXACTish nodes into one. */
2097 regnode *n = regnext(scan);
2098 U32 stringok = 1;
2099 regnode *next = scan + NODE_SZ_STR(scan);
2100 U32 merged = 0;
2101 U32 stopnow = 0;
2102#ifdef DEBUGGING
2103 regnode *stop = scan;
72f13be8 2104 GET_RE_DEBUG_FLAGS_DECL;
f9049ba1 2105#else
d47053eb
RGS
2106 PERL_UNUSED_ARG(depth);
2107#endif
2108#ifndef EXPERIMENTAL_INPLACESCAN
f9049ba1
SP
2109 PERL_UNUSED_ARG(flags);
2110 PERL_UNUSED_ARG(val);
07be1b83 2111#endif
07be1b83
YO
2112 DEBUG_PEEP("join",scan,depth);
2113
2114 /* Skip NOTHING, merge EXACT*. */
2115 while (n &&
2116 ( PL_regkind[OP(n)] == NOTHING ||
2117 (stringok && (OP(n) == OP(scan))))
2118 && NEXT_OFF(n)
2119 && NEXT_OFF(scan) + NEXT_OFF(n) < I16_MAX) {
2120
2121 if (OP(n) == TAIL || n > next)
2122 stringok = 0;
2123 if (PL_regkind[OP(n)] == NOTHING) {
07be1b83
YO
2124 DEBUG_PEEP("skip:",n,depth);
2125 NEXT_OFF(scan) += NEXT_OFF(n);
2126 next = n + NODE_STEP_REGNODE;
2127#ifdef DEBUGGING
2128 if (stringok)
2129 stop = n;
2130#endif
2131 n = regnext(n);
2132 }
2133 else if (stringok) {
786e8c11 2134 const unsigned int oldl = STR_LEN(scan);
07be1b83
YO
2135 regnode * const nnext = regnext(n);
2136
2137 DEBUG_PEEP("merg",n,depth);
2138
2139 merged++;
2140 if (oldl + STR_LEN(n) > U8_MAX)
2141 break;
2142 NEXT_OFF(scan) += NEXT_OFF(n);
2143 STR_LEN(scan) += STR_LEN(n);
2144 next = n + NODE_SZ_STR(n);
2145 /* Now we can overwrite *n : */
2146 Move(STRING(n), STRING(scan) + oldl, STR_LEN(n), char);
2147#ifdef DEBUGGING
2148 stop = next - 1;
2149#endif
2150 n = nnext;
2151 if (stopnow) break;
2152 }
2153
d47053eb
RGS
2154#ifdef EXPERIMENTAL_INPLACESCAN
2155 if (flags && !NEXT_OFF(n)) {
2156 DEBUG_PEEP("atch", val, depth);
2157 if (reg_off_by_arg[OP(n)]) {
2158 ARG_SET(n, val - n);
2159 }
2160 else {
2161 NEXT_OFF(n) = val - n;
2162 }
2163 stopnow = 1;
2164 }
07be1b83
YO
2165#endif
2166 }
2167
2168 if (UTF && ( OP(scan) == EXACTF ) && ( STR_LEN(scan) >= 6 ) ) {
2169 /*
2170 Two problematic code points in Unicode casefolding of EXACT nodes:
2171
2172 U+0390 - GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
2173 U+03B0 - GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
2174
2175 which casefold to
2176
2177 Unicode UTF-8
2178
2179 U+03B9 U+0308 U+0301 0xCE 0xB9 0xCC 0x88 0xCC 0x81
2180 U+03C5 U+0308 U+0301 0xCF 0x85 0xCC 0x88 0xCC 0x81
2181
2182 This means that in case-insensitive matching (or "loose matching",
2183 as Unicode calls it), an EXACTF of length six (the UTF-8 encoded byte
2184 length of the above casefolded versions) can match a target string
2185 of length two (the byte length of UTF-8 encoded U+0390 or U+03B0).
2186 This would rather mess up the minimum length computation.
2187
2188 What we'll do is to look for the tail four bytes, and then peek
2189 at the preceding two bytes to see whether we need to decrease
2190 the minimum length by four (six minus two).
2191
2192 Thanks to the design of UTF-8, there cannot be false matches:
2193 A sequence of valid UTF-8 bytes cannot be a subsequence of
2194 another valid sequence of UTF-8 bytes.
2195
2196 */
2197 char * const s0 = STRING(scan), *s, *t;
2198 char * const s1 = s0 + STR_LEN(scan) - 1;
2199 char * const s2 = s1 - 4;
e294cc5d
JH
2200#ifdef EBCDIC /* RD tunifold greek 0390 and 03B0 */
2201 const char t0[] = "\xaf\x49\xaf\x42";
2202#else
07be1b83 2203 const char t0[] = "\xcc\x88\xcc\x81";
e294cc5d 2204#endif
07be1b83
YO
2205 const char * const t1 = t0 + 3;
2206
2207 for (s = s0 + 2;
2208 s < s2 && (t = ninstr(s, s1, t0, t1));
2209 s = t + 4) {
e294cc5d
JH
2210#ifdef EBCDIC
2211 if (((U8)t[-1] == 0x68 && (U8)t[-2] == 0xB4) ||
2212 ((U8)t[-1] == 0x46 && (U8)t[-2] == 0xB5))
2213#else
07be1b83
YO
2214 if (((U8)t[-1] == 0xB9 && (U8)t[-2] == 0xCE) ||
2215 ((U8)t[-1] == 0x85 && (U8)t[-2] == 0xCF))
e294cc5d 2216#endif
07be1b83
YO
2217 *min -= 4;
2218 }
2219 }
2220
2221#ifdef DEBUGGING
2222 /* Allow dumping */
2223 n = scan + NODE_SZ_STR(scan);
2224 while (n <= stop) {
2225 if (PL_regkind[OP(n)] != NOTHING || OP(n) == NOTHING) {
2226 OP(n) = OPTIMIZED;
2227 NEXT_OFF(n) = 0;
2228 }
2229 n++;
2230 }
2231#endif
2232 DEBUG_OPTIMISE_r(if (merged){DEBUG_PEEP("finl",scan,depth)});
2233 return stopnow;
2234}
2235
653099ff
GS
2236/* REx optimizer. Converts nodes into quickier variants "in place".
2237 Finds fixed substrings. */
2238
a0288114 2239/* Stops at toplevel WHILEM as well as at "last". At end *scanp is set
c277df42
IZ
2240 to the position after last scanned or to NULL. */
2241
07be1b83
YO
2242
2243
76e3520e 2244STATIC I32
1de06328
YO
2245S_study_chunk(pTHX_ RExC_state_t *pRExC_state, regnode **scanp,
2246 I32 *minlenp, I32 *deltap,
9a957fbc 2247 regnode *last, scan_data_t *data, U32 flags, U32 depth)
c277df42
IZ
2248 /* scanp: Start here (read-write). */
2249 /* deltap: Write maxlen-minlen here. */
2250 /* last: Stop before this one. */
2251{
97aff369 2252 dVAR;
c277df42
IZ
2253 I32 min = 0, pars = 0, code;
2254 regnode *scan = *scanp, *next;
2255 I32 delta = 0;
2256 int is_inf = (flags & SCF_DO_SUBSTR) && (data->flags & SF_IS_INF);
aca2d497 2257 int is_inf_internal = 0; /* The studied chunk is infinite */
c277df42
IZ
2258 I32 is_par = OP(scan) == OPEN ? ARG(scan) : 0;
2259 scan_data_t data_fake;
653099ff 2260 struct regnode_charclass_class and_with; /* Valid if flags & SCF_DO_STCLASS_OR */
a3621e74 2261 SV *re_trie_maxbuff = NULL;
786e8c11
YO
2262 regnode *first_non_open = scan;
2263
a3621e74
YO
2264
2265 GET_RE_DEBUG_FLAGS_DECL;
13a24bad
YO
2266#ifdef DEBUGGING
2267 StructCopy(&zero_scan_data, &data_fake, scan_data_t);
2268#endif
786e8c11
YO
2269 if ( depth == 0 ) {
2270 while (first_non_open && OP(first_non_open) == OPEN)
2271 first_non_open=regnext(first_non_open);
2272 }
2273
b81d288d 2274
c277df42
IZ
2275 while (scan && OP(scan) != END && scan < last) {
2276 /* Peephole optimizer: */
1de06328 2277 DEBUG_STUDYDATA(data,depth);
07be1b83 2278 DEBUG_PEEP("Peep",scan,depth);
07be1b83 2279 JOIN_EXACT(scan,&min,0);
a3621e74 2280
653099ff
GS
2281 /* Follow the next-chain of the current node and optimize
2282 away all the NOTHINGs from it. */
c277df42 2283 if (OP(scan) != CURLYX) {
a3b680e6 2284 const int max = (reg_off_by_arg[OP(scan)]
048cfca1
GS
2285 ? I32_MAX
2286 /* I32 may be smaller than U16 on CRAYs! */
2287 : (I32_MAX < U16_MAX ? I32_MAX : U16_MAX));
c277df42
IZ
2288 int off = (reg_off_by_arg[OP(scan)] ? ARG(scan) : NEXT_OFF(scan));
2289 int noff;
2290 regnode *n = scan;
b81d288d 2291
c277df42
IZ
2292 /* Skip NOTHING and LONGJMP. */
2293 while ((n = regnext(n))
3dab1dad 2294 && ((PL_regkind[OP(n)] == NOTHING && (noff = NEXT_OFF(n)))
c277df42
IZ
2295 || ((OP(n) == LONGJMP) && (noff = ARG(n))))
2296 && off + noff < max)
2297 off += noff;
2298 if (reg_off_by_arg[OP(scan)])
2299 ARG(scan) = off;
b81d288d 2300 else
c277df42
IZ
2301 NEXT_OFF(scan) = off;
2302 }
a3621e74 2303
07be1b83 2304
3dab1dad 2305
653099ff
GS
2306 /* The principal pseudo-switch. Cannot be a switch, since we
2307 look into several different things. */
b81d288d 2308 if (OP(scan) == BRANCH || OP(scan) == BRANCHJ
c277df42
IZ
2309 || OP(scan) == IFTHEN || OP(scan) == SUSPEND) {
2310 next = regnext(scan);
2311 code = OP(scan);
a3621e74 2312 /* demq: the op(next)==code check is to see if we have "branch-branch" AFAICT */
b81d288d
AB
2313
2314 if (OP(next) == code || code == IFTHEN || code == SUSPEND) {
786e8c11
YO
2315 /* NOTE - There is similar code to this block below for handling
2316 TRIE nodes on a re-study. If you change stuff here check there
2317 too. */
c277df42 2318 I32 max1 = 0, min1 = I32_MAX, num = 0;
653099ff 2319 struct regnode_charclass_class accum;
d4c19fe8 2320 regnode * const startbranch=scan;
c277df42 2321
653099ff 2322 if (flags & SCF_DO_SUBSTR) /* XXXX Add !SUSPEND? */
1de06328 2323 scan_commit(pRExC_state, data, minlenp); /* Cannot merge strings after this. */
653099ff 2324 if (flags & SCF_DO_STCLASS)
830247a4 2325 cl_init_zero(pRExC_state, &accum);
a3621e74 2326
c277df42 2327 while (OP(scan) == code) {
830247a4 2328 I32 deltanext, minnext, f = 0, fake;
653099ff 2329 struct regnode_charclass_class this_class;
c277df42
IZ
2330
2331 num++;
2332 data_fake.flags = 0;
b81d288d 2333 if (data) {
2c2d71f5 2334 data_fake.whilem_c = data->whilem_c;
cb434fcc
IZ
2335 data_fake.last_closep = data->last_closep;
2336 }
2337 else
2338 data_fake.last_closep = &fake;
c277df42
IZ
2339 next = regnext(scan);
2340 scan = NEXTOPER(scan);
2341 if (code != BRANCH)
2342 scan = NEXTOPER(scan);
653099ff 2343 if (flags & SCF_DO_STCLASS) {
830247a4 2344 cl_init(pRExC_state, &this_class);
653099ff
GS
2345 data_fake.start_class = &this_class;
2346 f = SCF_DO_STCLASS_AND;
b81d288d 2347 }
e1901655
IZ
2348 if (flags & SCF_WHILEM_VISITED_POS)
2349 f |= SCF_WHILEM_VISITED_POS;
a3621e74 2350
653099ff 2351 /* we suppose the run is continuous, last=next...*/
1de06328 2352 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext,
a3621e74 2353 next, &data_fake, f,depth+1);
b81d288d 2354 if (min1 > minnext)
c277df42
IZ
2355 min1 = minnext;
2356 if (max1 < minnext + deltanext)
2357 max1 = minnext + deltanext;
2358 if (deltanext == I32_MAX)
aca2d497 2359 is_inf = is_inf_internal = 1;
c277df42
IZ
2360 scan = next;
2361 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
2362 pars++;
3dab1dad
YO
2363 if (data) {
2364 if (data_fake.flags & SF_HAS_EVAL)
07be1b83 2365 data->flags |= SF_HAS_EVAL;
2c2d71f5 2366 data->whilem_c = data_fake.whilem_c;
3dab1dad 2367 }
653099ff 2368 if (flags & SCF_DO_STCLASS)
830247a4 2369 cl_or(pRExC_state, &accum, &this_class);
b81d288d 2370 if (code == SUSPEND)
c277df42
IZ
2371 break;
2372 }
2373 if (code == IFTHEN && num < 2) /* Empty ELSE branch */
2374 min1 = 0;
2375 if (flags & SCF_DO_SUBSTR) {
2376 data->pos_min += min1;
2377 data->pos_delta += max1 - min1;
2378 if (max1 != min1 || is_inf)
2379 data->longest = &(data->longest_float);
2380 }
2381 min += min1;
2382 delta += max1 - min1;
653099ff 2383 if (flags & SCF_DO_STCLASS_OR) {
830247a4 2384 cl_or(pRExC_state, data->start_class, &accum);
653099ff
GS
2385 if (min1) {
2386 cl_and(data->start_class, &and_with);
2387 flags &= ~SCF_DO_STCLASS;
2388 }
2389 }
2390 else if (flags & SCF_DO_STCLASS_AND) {
de0c8cb8
GS
2391 if (min1) {
2392 cl_and(data->start_class, &accum);
653099ff 2393 flags &= ~SCF_DO_STCLASS;
de0c8cb8
GS
2394 }
2395 else {
b81d288d 2396 /* Switch to OR mode: cache the old value of
de0c8cb8
GS
2397 * data->start_class */
2398 StructCopy(data->start_class, &and_with,
2399 struct regnode_charclass_class);
2400 flags &= ~SCF_DO_STCLASS_AND;
2401 StructCopy(&accum, data->start_class,
2402 struct regnode_charclass_class);
2403 flags |= SCF_DO_STCLASS_OR;
2404 data->start_class->flags |= ANYOF_EOS;
2405 }
653099ff 2406 }
a3621e74 2407
786e8c11 2408 if (PERL_ENABLE_TRIE_OPTIMISATION && OP( startbranch ) == BRANCH ) {
a3621e74
YO
2409 /* demq.
2410
2411 Assuming this was/is a branch we are dealing with: 'scan' now
2412 points at the item that follows the branch sequence, whatever
2413 it is. We now start at the beginning of the sequence and look
2414 for subsequences of
2415
786e8c11
YO
2416 BRANCH->EXACT=>x1
2417 BRANCH->EXACT=>x2
2418 tail
a3621e74
YO
2419
2420 which would be constructed from a pattern like /A|LIST|OF|WORDS/
2421
2422 If we can find such a subseqence we need to turn the first
2423 element into a trie and then add the subsequent branch exact
2424 strings to the trie.
2425
2426 We have two cases
2427
786e8c11 2428 1. patterns where the whole set of branch can be converted.
a3621e74 2429
786e8c11 2430 2. patterns where only a subset can be converted.
a3621e74
YO
2431
2432 In case 1 we can replace the whole set with a single regop
2433 for the trie. In case 2 we need to keep the start and end
2434 branchs so
2435
2436 'BRANCH EXACT; BRANCH EXACT; BRANCH X'
2437 becomes BRANCH TRIE; BRANCH X;
2438
786e8c11
YO
2439 There is an additional case, that being where there is a
2440 common prefix, which gets split out into an EXACT like node
2441 preceding the TRIE node.
2442
2443 If x(1..n)==tail then we can do a simple trie, if not we make
2444 a "jump" trie, such that when we match the appropriate word
2445 we "jump" to the appopriate tail node. Essentailly we turn
2446 a nested if into a case structure of sorts.
a3621e74
YO
2447
2448 */
786e8c11 2449
3dab1dad 2450 int made=0;
0111c4fd
RGS
2451 if (!re_trie_maxbuff) {
2452 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1);
2453 if (!SvIOK(re_trie_maxbuff))
2454 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT);
2455 }
786e8c11 2456 if ( SvIV(re_trie_maxbuff)>=0 ) {
a3621e74
YO
2457 regnode *cur;
2458 regnode *first = (regnode *)NULL;
2459 regnode *last = (regnode *)NULL;
2460 regnode *tail = scan;
2461 U8 optype = 0;
2462 U32 count=0;
2463
2464#ifdef DEBUGGING
c445ea15 2465 SV * const mysv = sv_newmortal(); /* for dumping */
a3621e74
YO
2466#endif
2467 /* var tail is used because there may be a TAIL
2468 regop in the way. Ie, the exacts will point to the
2469 thing following the TAIL, but the last branch will
2470 point at the TAIL. So we advance tail. If we
2471 have nested (?:) we may have to move through several
2472 tails.
2473 */
2474
2475 while ( OP( tail ) == TAIL ) {
2476 /* this is the TAIL generated by (?:) */
2477 tail = regnext( tail );
2478 }
2479
3dab1dad 2480
a3621e74 2481 DEBUG_OPTIMISE_r({
32fc9b6a 2482 regprop(RExC_rx, mysv, tail );
3dab1dad
YO
2483 PerlIO_printf( Perl_debug_log, "%*s%s%s\n",
2484 (int)depth * 2 + 2, "",
2485 "Looking for TRIE'able sequences. Tail node is: ",
2486 SvPV_nolen_const( mysv )
a3621e74
YO
2487 );
2488 });
3dab1dad 2489
a3621e74
YO
2490 /*
2491
2492 step through the branches, cur represents each
2493 branch, noper is the first thing to be matched
2494 as part of that branch and noper_next is the
2495 regnext() of that node. if noper is an EXACT
2496 and noper_next is the same as scan (our current
2497 position in the regex) then the EXACT branch is
2498 a possible optimization target. Once we have
2499 two or more consequetive such branches we can
2500 create a trie of the EXACT's contents and stich
2501 it in place. If the sequence represents all of
2502 the branches we eliminate the whole thing and
2503 replace it with a single TRIE. If it is a
2504 subsequence then we need to stitch it in. This
2505 means the first branch has to remain, and needs
2506 to be repointed at the item on the branch chain
2507 following the last branch optimized. This could
2508 be either a BRANCH, in which case the
2509 subsequence is internal, or it could be the
2510 item following the branch sequence in which
2511 case the subsequence is at the end.
2512
2513 */
2514
2515 /* dont use tail as the end marker for this traverse */
2516 for ( cur = startbranch ; cur != scan ; cur = regnext( cur ) ) {
aec46f14 2517 regnode * const noper = NEXTOPER( cur );
be981c67 2518#if defined(DEBUGGING) || defined(NOJUMPTRIE)
aec46f14 2519 regnode * const noper_next = regnext( noper );
be981c67 2520#endif
a3621e74 2521
a3621e74 2522 DEBUG_OPTIMISE_r({
32fc9b6a 2523 regprop(RExC_rx, mysv, cur);
3dab1dad
YO
2524 PerlIO_printf( Perl_debug_log, "%*s- %s (%d)",
2525 (int)depth * 2 + 2,"", SvPV_nolen_const( mysv ), REG_NODE_NUM(cur) );
a3621e74 2526
32fc9b6a 2527 regprop(RExC_rx, mysv, noper);
a3621e74 2528 PerlIO_printf( Perl_debug_log, " -> %s",
cfd0369c 2529 SvPV_nolen_const(mysv));
a3621e74
YO
2530
2531 if ( noper_next ) {
32fc9b6a 2532 regprop(RExC_rx, mysv, noper_next );
a3621e74 2533 PerlIO_printf( Perl_debug_log,"\t=> %s\t",
cfd0369c 2534 SvPV_nolen_const(mysv));
a3621e74 2535 }
3dab1dad
YO
2536 PerlIO_printf( Perl_debug_log, "(First==%d,Last==%d,Cur==%d)\n",
2537 REG_NODE_NUM(first), REG_NODE_NUM(last), REG_NODE_NUM(cur) );
a3621e74 2538 });
3dab1dad
YO
2539 if ( (((first && optype!=NOTHING) ? OP( noper ) == optype
2540 : PL_regkind[ OP( noper ) ] == EXACT )
2541 || OP(noper) == NOTHING )
786e8c11
YO
2542#ifdef NOJUMPTRIE
2543 && noper_next == tail
2544#endif
2545 && count < U16_MAX)
a3621e74
YO
2546 {
2547 count++;
3dab1dad
YO
2548 if ( !first || optype == NOTHING ) {
2549 if (!first) first = cur;
a3621e74
YO
2550 optype = OP( noper );
2551 } else {
a3621e74 2552 last = cur;
a3621e74
YO
2553 }
2554 } else {
2555 if ( last ) {
786e8c11
YO
2556 make_trie( pRExC_state,
2557 startbranch, first, cur, tail, count,
2558 optype, depth+1 );
a3621e74 2559 }
3dab1dad 2560 if ( PL_regkind[ OP( noper ) ] == EXACT
786e8c11
YO
2561#ifdef NOJUMPTRIE
2562 && noper_next == tail
2563#endif
2564 ){
a3621e74
YO
2565 count = 1;
2566 first = cur;
2567 optype = OP( noper );
2568 } else {
2569 count = 0;
2570 first = NULL;
2571 optype = 0;
2572 }
2573 last = NULL;
2574 }
2575 }
2576 DEBUG_OPTIMISE_r({
32fc9b6a 2577 regprop(RExC_rx, mysv, cur);
a3621e74 2578 PerlIO_printf( Perl_debug_log,
3dab1dad
YO
2579 "%*s- %s (%d) <SCAN FINISHED>\n", (int)depth * 2 + 2,
2580 "", SvPV_nolen_const( mysv ),REG_NODE_NUM(cur));
a3621e74
YO
2581
2582 });
2583 if ( last ) {
786e8c11 2584 made= make_trie( pRExC_state, startbranch, first, scan, tail, count, optype, depth+1 );
3dab1dad 2585#ifdef TRIE_STUDY_OPT
786e8c11
YO
2586 if ( ((made == MADE_EXACT_TRIE &&
2587 startbranch == first)
2588 || ( first_non_open == first )) &&
2589 depth==0 )
2590 flags |= SCF_TRIE_RESTUDY;
3dab1dad 2591#endif
07be1b83 2592 }
a3621e74 2593 }
3dab1dad
YO
2594
2595 } /* do trie */
786e8c11 2596
a0ed51b3 2597 }
a3621e74 2598 else if ( code == BRANCHJ ) { /* single branch is optimized. */
c277df42 2599 scan = NEXTOPER(NEXTOPER(scan));
a3621e74 2600 } else /* single branch is optimized. */
c277df42
IZ
2601 scan = NEXTOPER(scan);
2602 continue;
a0ed51b3
LW
2603 }
2604 else if (OP(scan) == EXACT) {
cd439c50 2605 I32 l = STR_LEN(scan);
c445ea15 2606 UV uc;
a0ed51b3 2607 if (UTF) {
a3b680e6 2608 const U8 * const s = (U8*)STRING(scan);
1aa99e6b 2609 l = utf8_length(s, s + l);
9041c2e3 2610 uc = utf8_to_uvchr(s, NULL);
c445ea15
AL
2611 } else {
2612 uc = *((U8*)STRING(scan));
a0ed51b3
LW
2613 }
2614 min += l;
c277df42 2615 if (flags & SCF_DO_SUBSTR) { /* Update longest substr. */
c277df42
IZ
2616 /* The code below prefers earlier match for fixed
2617 offset, later match for variable offset. */
2618 if (data->last_end == -1) { /* Update the start info. */
2619 data->last_start_min = data->pos_min;
2620 data->last_start_max = is_inf
b81d288d 2621 ? I32_MAX : data->pos_min + data->pos_delta;
c277df42 2622 }
cd439c50 2623 sv_catpvn(data->last_found, STRING(scan), STR_LEN(scan));
45f47268
NC
2624 if (UTF)
2625 SvUTF8_on(data->last_found);
0eda9292 2626 {
9a957fbc 2627 SV * const sv = data->last_found;
a28509cc 2628 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ?
0eda9292
JH
2629 mg_find(sv, PERL_MAGIC_utf8) : NULL;
2630 if (mg && mg->mg_len >= 0)
5e43f467
JH
2631 mg->mg_len += utf8_length((U8*)STRING(scan),
2632 (U8*)STRING(scan)+STR_LEN(scan));
0eda9292 2633 }
c277df42
IZ
2634 data->last_end = data->pos_min + l;
2635 data->pos_min += l; /* As in the first entry. */
2636 data->flags &= ~SF_BEFORE_EOL;
2637 }
653099ff
GS
2638 if (flags & SCF_DO_STCLASS_AND) {
2639 /* Check whether it is compatible with what we know already! */
2640 int compat = 1;
2641
1aa99e6b 2642 if (uc >= 0x100 ||
516a5887 2643 (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE))
1aa99e6b 2644 && !ANYOF_BITMAP_TEST(data->start_class, uc)
653099ff 2645 && (!(data->start_class->flags & ANYOF_FOLD)
1aa99e6b 2646 || !ANYOF_BITMAP_TEST(data->start_class, PL_fold[uc])))
516a5887 2647 )
653099ff
GS
2648 compat = 0;
2649 ANYOF_CLASS_ZERO(data->start_class);
2650 ANYOF_BITMAP_ZERO(data->start_class);
2651 if (compat)
1aa99e6b 2652 ANYOF_BITMAP_SET(data->start_class, uc);
653099ff 2653 data->start_class->flags &= ~ANYOF_EOS;
9b877dbb
IH
2654 if (uc < 0x100)
2655 data->start_class->flags &= ~ANYOF_UNICODE_ALL;
653099ff
GS
2656 }
2657 else if (flags & SCF_DO_STCLASS_OR) {
2658 /* false positive possible if the class is case-folded */
1aa99e6b 2659 if (uc < 0x100)
9b877dbb
IH
2660 ANYOF_BITMAP_SET(data->start_class, uc);
2661 else
2662 data->start_class->flags |= ANYOF_UNICODE_ALL;
653099ff
GS
2663 data->start_class->flags &= ~ANYOF_EOS;
2664 cl_and(data->start_class, &and_with);
2665 }
2666 flags &= ~SCF_DO_STCLASS;
a0ed51b3 2667 }
3dab1dad 2668 else if (PL_regkind[OP(scan)] == EXACT) { /* But OP != EXACT! */
cd439c50 2669 I32 l = STR_LEN(scan);
1aa99e6b 2670 UV uc = *((U8*)STRING(scan));
653099ff
GS
2671
2672 /* Search for fixed substrings supports EXACT only. */
ecaa9b9c
NC
2673 if (flags & SCF_DO_SUBSTR) {
2674 assert(data);
1de06328 2675 scan_commit(pRExC_state, data, minlenp);
ecaa9b9c 2676 }
a0ed51b3 2677 if (UTF) {
6136c704 2678 const U8 * const s = (U8 *)STRING(scan);
1aa99e6b 2679 l = utf8_length(s, s + l);
9041c2e3 2680 uc = utf8_to_uvchr(s, NULL);
a0ed51b3
LW
2681 }
2682 min += l;
ecaa9b9c 2683 if (flags & SCF_DO_SUBSTR)
a0ed51b3 2684 data->pos_min += l;
653099ff
GS
2685 if (flags & SCF_DO_STCLASS_AND) {
2686 /* Check whether it is compatible with what we know already! */
2687 int compat = 1;
2688
1aa99e6b 2689 if (uc >= 0x100 ||
516a5887 2690 (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE))
1aa99e6b 2691 && !ANYOF_BITMAP_TEST(data->start_class, uc)
516a5887 2692 && !ANYOF_BITMAP_TEST(data->start_class, PL_fold[uc])))
653099ff
GS
2693 compat = 0;
2694 ANYOF_CLASS_ZERO(data->start_class);
2695 ANYOF_BITMAP_ZERO(data->start_class);
2696 if (compat) {
1aa99e6b 2697 ANYOF_BITMAP_SET(data->start_class, uc);
653099ff
GS
2698 data->start_class->flags &= ~ANYOF_EOS;
2699 data->start_class->flags |= ANYOF_FOLD;
2700 if (OP(scan) == EXACTFL)
2701 data->start_class->flags |= ANYOF_LOCALE;
2702 }
2703 }
2704 else if (flags & SCF_DO_STCLASS_OR) {
2705 if (data->start_class->flags & ANYOF_FOLD) {
2706 /* false positive possible if the class is case-folded.
2707 Assume that the locale settings are the same... */
1aa99e6b
IH
2708 if (uc < 0x100)
2709 ANYOF_BITMAP_SET(data->start_class, uc);
653099ff
GS
2710 data->start_class->flags &= ~ANYOF_EOS;
2711 }
2712 cl_and(data->start_class, &and_with);
2713 }
2714 flags &= ~SCF_DO_STCLASS;
a0ed51b3 2715 }
bfed75c6 2716 else if (strchr((const char*)PL_varies,OP(scan))) {
9c5ffd7c 2717 I32 mincount, maxcount, minnext, deltanext, fl = 0;
aa7a4b56 2718 I32 f = flags, pos_before = 0;
d4c19fe8 2719 regnode * const oscan = scan;
653099ff
GS
2720 struct regnode_charclass_class this_class;
2721 struct regnode_charclass_class *oclass = NULL;
727f22e3 2722 I32 next_is_eval = 0;
653099ff 2723
3dab1dad 2724 switch (PL_regkind[OP(scan)]) {
653099ff 2725 case WHILEM: /* End of (?:...)* . */
c277df42
IZ
2726 scan = NEXTOPER(scan);
2727 goto finish;
2728 case PLUS:
653099ff 2729 if (flags & (SCF_DO_SUBSTR | SCF_DO_STCLASS)) {
c277df42 2730 next = NEXTOPER(scan);
653099ff 2731 if (OP(next) == EXACT || (flags & SCF_DO_STCLASS)) {
b81d288d
AB
2732 mincount = 1;
2733 maxcount = REG_INFTY;
c277df42
IZ
2734 next = regnext(scan);
2735 scan = NEXTOPER(scan);
2736 goto do_curly;
2737 }
2738 }
2739 if (flags & SCF_DO_SUBSTR)
2740 data->pos_min++;
2741 min++;
2742 /* Fall through. */
2743 case STAR:
653099ff
GS
2744 if (flags & SCF_DO_STCLASS) {
2745 mincount = 0;
b81d288d 2746 maxcount = REG_INFTY;
653099ff
GS
2747 next = regnext(scan);
2748 scan = NEXTOPER(scan);
2749 goto do_curly;
2750 }
b81d288d 2751 is_inf = is_inf_internal = 1;
c277df42
IZ
2752 scan = regnext(scan);
2753 if (flags & SCF_DO_SUBSTR) {
1de06328 2754 scan_commit(pRExC_state, data, minlenp); /* Cannot extend fixed substrings */
c277df42
IZ
2755 data->longest = &(data->longest_float);
2756 }
2757 goto optimize_curly_tail;
2758 case CURLY:
b81d288d 2759 mincount = ARG1(scan);
c277df42
IZ
2760 maxcount = ARG2(scan);
2761 next = regnext(scan);
cb434fcc
IZ
2762 if (OP(scan) == CURLYX) {
2763 I32 lp = (data ? *(data->last_closep) : 0);
786e8c11 2764 scan->flags = ((lp <= (I32)U8_MAX) ? (U8)lp : U8_MAX);
cb434fcc 2765 }
c277df42 2766 scan = NEXTOPER(scan) + EXTRA_STEP_2ARGS;
727f22e3 2767 next_is_eval = (OP(scan) == EVAL);
c277df42
IZ
2768 do_curly:
2769 if (flags & SCF_DO_SUBSTR) {
1de06328 2770 if (mincount == 0) scan_commit(pRExC_state,data,minlenp); /* Cannot extend fixed substrings */
c277df42
IZ
2771 pos_before = data->pos_min;
2772 }
2773 if (data) {
2774 fl = data->flags;
2775 data->flags &= ~(SF_HAS_PAR|SF_IN_PAR|SF_HAS_EVAL);
2776 if (is_inf)
2777 data->flags |= SF_IS_INF;
2778 }
653099ff 2779 if (flags & SCF_DO_STCLASS) {
830247a4 2780 cl_init(pRExC_state, &this_class);
653099ff
GS
2781 oclass = data->start_class;
2782 data->start_class = &this_class;
2783 f |= SCF_DO_STCLASS_AND;
2784 f &= ~SCF_DO_STCLASS_OR;
2785 }
e1901655
IZ
2786 /* These are the cases when once a subexpression
2787 fails at a particular position, it cannot succeed
2788 even after backtracking at the enclosing scope.
b81d288d 2789
e1901655
IZ
2790 XXXX what if minimal match and we are at the
2791 initial run of {n,m}? */
2792 if ((mincount != maxcount - 1) && (maxcount != REG_INFTY))
2793 f &= ~SCF_WHILEM_VISITED_POS;
653099ff 2794
c277df42 2795 /* This will finish on WHILEM, setting scan, or on NULL: */
1de06328 2796 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext, last, data,
a3621e74
YO
2797 (mincount == 0
2798 ? (f & ~SCF_DO_SUBSTR) : f),depth+1);
653099ff
GS
2799
2800 if (flags & SCF_DO_STCLASS)
2801 data->start_class = oclass;
2802 if (mincount == 0 || minnext == 0) {
2803 if (flags & SCF_DO_STCLASS_OR) {
830247a4 2804 cl_or(pRExC_state, data->start_class, &this_class);
653099ff
GS
2805 }
2806 else if (flags & SCF_DO_STCLASS_AND) {
b81d288d 2807 /* Switch to OR mode: cache the old value of
653099ff
GS
2808 * data->start_class */
2809 StructCopy(data->start_class, &and_with,
2810 struct regnode_charclass_class);
2811 flags &= ~SCF_DO_STCLASS_AND;
2812 StructCopy(&this_class, data->start_class,
2813 struct regnode_charclass_class);
2814 flags |= SCF_DO_STCLASS_OR;
2815 data->start_class->flags |= ANYOF_EOS;
2816 }
2817 } else { /* Non-zero len */
2818 if (flags & SCF_DO_STCLASS_OR) {
830247a4 2819 cl_or(pRExC_state, data->start_class, &this_class);
653099ff
GS
2820 cl_and(data->start_class, &and_with);
2821 }
2822 else if (flags & SCF_DO_STCLASS_AND)
2823 cl_and(data->start_class, &this_class);
2824 flags &= ~SCF_DO_STCLASS;
2825 }
c277df42
IZ
2826 if (!scan) /* It was not CURLYX, but CURLY. */
2827 scan = next;
041457d9
DM
2828 if ( /* ? quantifier ok, except for (?{ ... }) */
2829 (next_is_eval || !(mincount == 0 && maxcount == 1))
84037bb0 2830 && (minnext == 0) && (deltanext == 0)
99799961 2831 && data && !(data->flags & (SF_HAS_PAR|SF_IN_PAR))
041457d9
DM
2832 && maxcount <= REG_INFTY/3 /* Complement check for big count */
2833 && ckWARN(WARN_REGEXP))
b45f050a 2834 {
830247a4 2835 vWARN(RExC_parse,
b45f050a
JF
2836 "Quantifier unexpected on zero-length expression");
2837 }
2838
c277df42 2839 min += minnext * mincount;
b81d288d 2840 is_inf_internal |= ((maxcount == REG_INFTY
155aba94
GS
2841 && (minnext + deltanext) > 0)
2842 || deltanext == I32_MAX);
aca2d497 2843 is_inf |= is_inf_internal;
c277df42
IZ
2844 delta += (minnext + deltanext) * maxcount - minnext * mincount;
2845
2846 /* Try powerful optimization CURLYX => CURLYN. */
b81d288d 2847 if ( OP(oscan) == CURLYX && data
c277df42
IZ
2848 && data->flags & SF_IN_PAR
2849 && !(data->flags & SF_HAS_EVAL)
2850 && !deltanext && minnext == 1 ) {
2851 /* Try to optimize to CURLYN. */
2852 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS;
d4c19fe8 2853 regnode * const nxt1 = nxt;
497b47a8
JH
2854#ifdef DEBUGGING
2855 regnode *nxt2;
2856#endif
c277df42
IZ
2857
2858 /* Skip open. */
2859 nxt = regnext(nxt);
bfed75c6 2860 if (!strchr((const char*)PL_simple,OP(nxt))
3dab1dad 2861 && !(PL_regkind[OP(nxt)] == EXACT
b81d288d 2862 && STR_LEN(nxt) == 1))
c277df42 2863 goto nogo;
497b47a8 2864#ifdef DEBUGGING
c277df42 2865 nxt2 = nxt;
497b47a8 2866#endif
c277df42 2867 nxt = regnext(nxt);
b81d288d 2868 if (OP(nxt) != CLOSE)
c277df42
IZ
2869 goto nogo;
2870 /* Now we know that nxt2 is the only contents: */
eb160463 2871 oscan->flags = (U8)ARG(nxt);
c277df42
IZ
2872 OP(oscan) = CURLYN;
2873 OP(nxt1) = NOTHING; /* was OPEN. */
2874#ifdef DEBUGGING
2875 OP(nxt1 + 1) = OPTIMIZED; /* was count. */
2876 NEXT_OFF(nxt1+ 1) = 0; /* just for consistancy. */
2877 NEXT_OFF(nxt2) = 0; /* just for consistancy with CURLY. */
2878 OP(nxt) = OPTIMIZED; /* was CLOSE. */
2879 OP(nxt + 1) = OPTIMIZED; /* was count. */
2880 NEXT_OFF(nxt+ 1) = 0; /* just for consistancy. */
b81d288d 2881#endif
c277df42 2882 }
c277df42
IZ
2883 nogo:
2884
2885 /* Try optimization CURLYX => CURLYM. */
b81d288d 2886 if ( OP(oscan) == CURLYX && data
c277df42 2887 && !(data->flags & SF_HAS_PAR)
c277df42 2888 && !(data->flags & SF_HAS_EVAL)
0e788c72
HS
2889 && !deltanext /* atom is fixed width */
2890 && minnext != 0 /* CURLYM can't handle zero width */
2891 ) {
c277df42
IZ
2892 /* XXXX How to optimize if data == 0? */
2893 /* Optimize to a simpler form. */
2894 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN */
2895 regnode *nxt2;
2896
2897 OP(oscan) = CURLYM;
2898 while ( (nxt2 = regnext(nxt)) /* skip over embedded stuff*/
b81d288d 2899 && (OP(nxt2) != WHILEM))
c277df42
IZ
2900 nxt = nxt2;
2901 OP(nxt2) = SUCCEED; /* Whas WHILEM */
c277df42
IZ
2902 /* Need to optimize away parenths. */
2903 if (data->flags & SF_IN_PAR) {
2904 /* Set the parenth number. */
2905 regnode *nxt1 = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN*/
2906
b81d288d 2907 if (OP(nxt) != CLOSE)
b45f050a 2908 FAIL("Panic opt close");
eb160463 2909 oscan->flags = (U8)ARG(nxt);
c277df42
IZ
2910 OP(nxt1) = OPTIMIZED; /* was OPEN. */
2911 OP(nxt) = OPTIMIZED; /* was CLOSE. */
2912#ifdef DEBUGGING
2913 OP(nxt1 + 1) = OPTIMIZED; /* was count. */
2914 OP(nxt + 1) = OPTIMIZED; /* was count. */
2915 NEXT_OFF(nxt1 + 1) = 0; /* just for consistancy. */
2916 NEXT_OFF(nxt + 1) = 0; /* just for consistancy. */
b81d288d 2917#endif
c277df42
IZ
2918#if 0
2919 while ( nxt1 && (OP(nxt1) != WHILEM)) {
2920 regnode *nnxt = regnext(nxt1);
b81d288d 2921
c277df42
IZ
2922 if (nnxt == nxt) {
2923 if (reg_off_by_arg[OP(nxt1)])
2924 ARG_SET(nxt1, nxt2 - nxt1);
2925 else if (nxt2 - nxt1 < U16_MAX)
2926 NEXT_OFF(nxt1) = nxt2 - nxt1;
2927 else
2928 OP(nxt) = NOTHING; /* Cannot beautify */
2929 }
2930 nxt1 = nnxt;
2931 }
2932#endif
2933 /* Optimize again: */
1de06328 2934 study_chunk(pRExC_state, &nxt1, minlenp, &deltanext, nxt,
a3621e74 2935 NULL, 0,depth+1);
a0ed51b3
LW
2936 }
2937 else
c277df42 2938 oscan->flags = 0;
c277df42 2939 }
e1901655
IZ
2940 else if ((OP(oscan) == CURLYX)
2941 && (flags & SCF_WHILEM_VISITED_POS)
2942 /* See the comment on a similar expression above.
2943 However, this time it not a subexpression
2944 we care about, but the expression itself. */
2945 && (maxcount == REG_INFTY)
2946 && data && ++data->whilem_c < 16) {
2947 /* This stays as CURLYX, we can put the count/of pair. */
2c2d71f5
JH
2948 /* Find WHILEM (as in regexec.c) */
2949 regnode *nxt = oscan + NEXT_OFF(oscan);
2950
2951 if (OP(PREVOPER(nxt)) == NOTHING) /* LONGJMP */
2952 nxt += ARG(nxt);
eb160463
GS
2953 PREVOPER(nxt)->flags = (U8)(data->whilem_c
2954 | (RExC_whilem_seen << 4)); /* On WHILEM */
2c2d71f5 2955 }
b81d288d 2956 if (data && fl & (SF_HAS_PAR|SF_IN_PAR))
c277df42
IZ
2957 pars++;
2958 if (flags & SCF_DO_SUBSTR) {
c445ea15 2959 SV *last_str = NULL;
c277df42
IZ
2960 int counted = mincount != 0;
2961
2962 if (data->last_end > 0 && mincount != 0) { /* Ends with a string. */
5d1c421c
JH
2963#if defined(SPARC64_GCC_WORKAROUND)
2964 I32 b = 0;
2965 STRLEN l = 0;
cfd0369c 2966 const char *s = NULL;
5d1c421c
JH
2967 I32 old = 0;
2968
2969 if (pos_before >= data->last_start_min)
2970 b = pos_before;
2971 else
2972 b = data->last_start_min;
2973
2974 l = 0;
cfd0369c 2975 s = SvPV_const(data->last_found, l);
5d1c421c
JH
2976 old = b - data->last_start_min;
2977
2978#else
b81d288d 2979 I32 b = pos_before >= data->last_start_min
c277df42
IZ
2980 ? pos_before : data->last_start_min;
2981 STRLEN l;
d4c19fe8 2982 const char * const s = SvPV_const(data->last_found, l);
a0ed51b3 2983 I32 old = b - data->last_start_min;
5d1c421c 2984#endif
a0ed51b3
LW
2985
2986 if (UTF)
2987 old = utf8_hop((U8*)s, old) - (U8*)s;
c277df42 2988
a0ed51b3 2989 l -= old;
c277df42 2990 /* Get the added string: */
79cb57f6 2991 last_str = newSVpvn(s + old, l);
0e933229
IH
2992 if (UTF)
2993 SvUTF8_on(last_str);
c277df42
IZ
2994 if (deltanext == 0 && pos_before == b) {
2995 /* What was added is a constant string */
2996 if (mincount > 1) {
2997 SvGROW(last_str, (mincount * l) + 1);
b81d288d 2998 repeatcpy(SvPVX(last_str) + l,
3f7c398e 2999 SvPVX_const(last_str), l, mincount - 1);
b162af07 3000 SvCUR_set(last_str, SvCUR(last_str) * mincount);
c277df42 3001 /* Add additional parts. */
b81d288d 3002 SvCUR_set(data->last_found,
c277df42
IZ
3003 SvCUR(data->last_found) - l);
3004 sv_catsv(data->last_found, last_str);
0eda9292
JH
3005 {
3006 SV * sv = data->last_found;
3007 MAGIC *mg =
3008 SvUTF8(sv) && SvMAGICAL(sv) ?
3009 mg_find(sv, PERL_MAGIC_utf8) : NULL;
3010 if (mg && mg->mg_len >= 0)
3011 mg->mg_len += CHR_SVLEN(last_str);
3012 }
c277df42
IZ
3013 data->last_end += l * (mincount - 1);
3014 }
2a8d9689
HS
3015 } else {
3016 /* start offset must point into the last copy */
3017 data->last_start_min += minnext * (mincount - 1);
c152dc43
HS
3018 data->last_start_max += is_inf ? I32_MAX
3019 : (maxcount - 1) * (minnext + data->pos_delta);
c277df42
IZ
3020 }
3021 }
3022 /* It is counted once already... */
3023 data->pos_min += minnext * (mincount - counted);
3024 data->pos_delta += - counted * deltanext +
3025 (minnext + deltanext) * maxcount - minnext * mincount;
3026 if (mincount != maxcount) {
653099ff
GS
3027 /* Cannot extend fixed substrings found inside
3028 the group. */
1de06328 3029 scan_commit(pRExC_state,data,minlenp);
c277df42 3030 if (mincount && last_str) {
d4c19fe8
AL
3031 SV * const sv = data->last_found;
3032 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ?
45f47268
NC
3033 mg_find(sv, PERL_MAGIC_utf8) : NULL;
3034
3035 if (mg)
3036 mg->mg_len = -1;
3037 sv_setsv(sv, last_str);
c277df42 3038 data->last_end = data->pos_min;
b81d288d 3039 data->last_start_min =
a0ed51b3 3040 data->pos_min - CHR_SVLEN(last_str);
b81d288d
AB
3041 data->last_start_max = is_inf
3042 ? I32_MAX
c277df42 3043 : data->pos_min + data->pos_delta
a0ed51b3 3044 - CHR_SVLEN(last_str);
c277df42
IZ
3045 }
3046 data->longest = &(data->longest_float);
3047 }
aca2d497 3048 SvREFCNT_dec(last_str);
c277df42 3049 }
405ff068 3050 if (data && (fl & SF_HAS_EVAL))
c277df42
IZ
3051 data->flags |= SF_HAS_EVAL;
3052 optimize_curly_tail:
c277df42 3053 if (OP(oscan) != CURLYX) {
3dab1dad 3054 while (PL_regkind[OP(next = regnext(oscan))] == NOTHING
c277df42
IZ
3055 && NEXT_OFF(next))
3056 NEXT_OFF(oscan) += NEXT_OFF(next);
3057 }
c277df42 3058 continue;
653099ff 3059 default: /* REF and CLUMP only? */
c277df42 3060 if (flags & SCF_DO_SUBSTR) {
1de06328 3061 scan_commit(pRExC_state,data,minlenp); /* Cannot expect anything... */
c277df42
IZ
3062 data->longest = &(data->longest_float);
3063 }
aca2d497 3064 is_inf = is_inf_internal = 1;
653099ff 3065 if (flags & SCF_DO_STCLASS_OR)
830247a4 3066 cl_anything(pRExC_state, data->start_class);
653099ff 3067 flags &= ~SCF_DO_STCLASS;
c277df42
IZ
3068 break;
3069 }
a0ed51b3 3070 }
bfed75c6 3071 else if (strchr((const char*)PL_simple,OP(scan))) {
9c5ffd7c 3072 int value = 0;
653099ff 3073
c277df42 3074 if (flags & SCF_DO_SUBSTR) {
1de06328 3075 scan_commit(pRExC_state,data,minlenp);
c277df42
IZ
3076 data->pos_min++;
3077 }
3078 min++;
653099ff
GS
3079 if (flags & SCF_DO_STCLASS) {
3080 data->start_class->flags &= ~ANYOF_EOS; /* No match on empty */
3081
3082 /* Some of the logic below assumes that switching
3083 locale on will only add false positives. */
3dab1dad 3084 switch (PL_regkind[OP(scan)]) {
653099ff 3085 case SANY:
653099ff
GS
3086 default:
3087 do_default:
3088 /* Perl_croak(aTHX_ "panic: unexpected simple REx opcode %d", OP(scan)); */
3089 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
830247a4 3090 cl_anything(pRExC_state, data->start_class);
653099ff
GS
3091 break;
3092 case REG_ANY:
3093 if (OP(scan) == SANY)
3094 goto do_default;
3095 if (flags & SCF_DO_STCLASS_OR) { /* Everything but \n */
3096 value = (ANYOF_BITMAP_TEST(data->start_class,'\n')
3097 || (data->start_class->flags & ANYOF_CLASS));
830247a4 3098 cl_anything(pRExC_state, data->start_class);
653099ff
GS
3099 }
3100 if (flags & SCF_DO_STCLASS_AND || !value)
3101 ANYOF_BITMAP_CLEAR(data->start_class,'\n');
3102 break;
3103 case ANYOF:
3104 if (flags & SCF_DO_STCLASS_AND)
3105 cl_and(data->start_class,
3106 (struct regnode_charclass_class*)scan);
3107 else
830247a4 3108 cl_or(pRExC_state, data->start_class,
653099ff
GS
3109 (struct regnode_charclass_class*)scan);
3110 break;
3111 case ALNUM:
3112 if (flags & SCF_DO_STCLASS_AND) {
3113 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3114 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM);
3115 for (value = 0; value < 256; value++)
3116 if (!isALNUM(value))
3117 ANYOF_BITMAP_CLEAR(data->start_class, value);
3118 }
3119 }
3120 else {
3121 if (data->start_class->flags & ANYOF_LOCALE)
3122 ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM);
3123 else {
3124 for (value = 0; value < 256; value++)
3125 if (isALNUM(value))
b81d288d 3126 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3127 }
3128 }
3129 break;
3130 case ALNUML:
3131 if (flags & SCF_DO_STCLASS_AND) {
3132 if (data->start_class->flags & ANYOF_LOCALE)
3133 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM);
3134 }
3135 else {
3136 ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM);
3137 data->start_class->flags |= ANYOF_LOCALE;
3138 }
3139 break;
3140 case NALNUM:
3141 if (flags & SCF_DO_STCLASS_AND) {
3142 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3143 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM);
3144 for (value = 0; value < 256; value++)
3145 if (isALNUM(value))
3146 ANYOF_BITMAP_CLEAR(data->start_class, value);
3147 }
3148 }
3149 else {
3150 if (data->start_class->flags & ANYOF_LOCALE)
3151 ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM);
3152 else {
3153 for (value = 0; value < 256; value++)
3154 if (!isALNUM(value))
b81d288d 3155 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3156 }
3157 }
3158 break;
3159 case NALNUML:
3160 if (flags & SCF_DO_STCLASS_AND) {
3161 if (data->start_class->flags & ANYOF_LOCALE)
3162 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM);
3163 }
3164 else {
3165 data->start_class->flags |= ANYOF_LOCALE;
3166 ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM);
3167 }
3168 break;
3169 case SPACE:
3170 if (flags & SCF_DO_STCLASS_AND) {
3171 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3172 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE);
3173 for (value = 0; value < 256; value++)
3174 if (!isSPACE(value))
3175 ANYOF_BITMAP_CLEAR(data->start_class, value);
3176 }
3177 }
3178 else {
3179 if (data->start_class->flags & ANYOF_LOCALE)
3180 ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE);
3181 else {
3182 for (value = 0; value < 256; value++)
3183 if (isSPACE(value))
b81d288d 3184 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3185 }
3186 }
3187 break;
3188 case SPACEL:
3189 if (flags & SCF_DO_STCLASS_AND) {
3190 if (data->start_class->flags & ANYOF_LOCALE)
3191 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE);
3192 }
3193 else {
3194 data->start_class->flags |= ANYOF_LOCALE;
3195 ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE);
3196 }
3197 break;
3198 case NSPACE:
3199 if (flags & SCF_DO_STCLASS_AND) {
3200 if (!(data->start_class->flags & ANYOF_LOCALE)) {
3201 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE);
3202 for (value = 0; value < 256; value++)
3203 if (isSPACE(value))
3204 ANYOF_BITMAP_CLEAR(data->start_class, value);
3205 }
3206 }
3207 else {
3208 if (data->start_class->flags & ANYOF_LOCALE)
3209 ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE);
3210 else {
3211 for (value = 0; value < 256; value++)
3212 if (!isSPACE(value))
b81d288d 3213 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3214 }
3215 }
3216 break;
3217 case NSPACEL:
3218 if (flags & SCF_DO_STCLASS_AND) {
3219 if (data->start_class->flags & ANYOF_LOCALE) {
3220 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE);
3221 for (value = 0; value < 256; value++)
3222 if (!isSPACE(value))
3223 ANYOF_BITMAP_CLEAR(data->start_class, value);
3224 }
3225 }
3226 else {
3227 data->start_class->flags |= ANYOF_LOCALE;
3228 ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE);
3229 }
3230 break;
3231 case DIGIT:
3232 if (flags & SCF_DO_STCLASS_AND) {
3233 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NDIGIT);
3234 for (value = 0; value < 256; value++)
3235 if (!isDIGIT(value))
3236 ANYOF_BITMAP_CLEAR(data->start_class, value);
3237 }
3238 else {
3239 if (data->start_class->flags & ANYOF_LOCALE)
3240 ANYOF_CLASS_SET(data->start_class,ANYOF_DIGIT);
3241 else {
3242 for (value = 0; value < 256; value++)
3243 if (isDIGIT(value))
b81d288d 3244 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3245 }
3246 }
3247 break;
3248 case NDIGIT:
3249 if (flags & SCF_DO_STCLASS_AND) {
3250 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_DIGIT);
3251 for (value = 0; value < 256; value++)
3252 if (isDIGIT(value))
3253 ANYOF_BITMAP_CLEAR(data->start_class, value);
3254 }
3255 else {
3256 if (data->start_class->flags & ANYOF_LOCALE)
3257 ANYOF_CLASS_SET(data->start_class,ANYOF_NDIGIT);
3258 else {
3259 for (value = 0; value < 256; value++)
3260 if (!isDIGIT(value))
b81d288d 3261 ANYOF_BITMAP_SET(data->start_class, value);
653099ff
GS
3262 }
3263 }
3264 break;
3265 }
3266 if (flags & SCF_DO_STCLASS_OR)
3267 cl_and(data->start_class, &and_with);
3268 flags &= ~SCF_DO_STCLASS;
3269 }
a0ed51b3 3270 }
3dab1dad 3271 else if (PL_regkind[OP(scan)] == EOL && flags & SCF_DO_SUBSTR) {
c277df42
IZ
3272 data->flags |= (OP(scan) == MEOL
3273 ? SF_BEFORE_MEOL
3274 : SF_BEFORE_SEOL);
a0ed51b3 3275 }
3dab1dad 3276 else if ( PL_regkind[OP(scan)] == BRANCHJ
653099ff
GS
3277 /* Lookbehind, or need to calculate parens/evals/stclass: */
3278 && (scan->flags || data || (flags & SCF_DO_STCLASS))
c277df42 3279 && (OP(scan) == IFMATCH || OP(scan) == UNLESSM)) {
1de06328
YO
3280 if ( !PERL_ENABLE_POSITIVE_ASSERTION_STUDY
3281 || OP(scan) == UNLESSM )
3282 {
3283 /* Negative Lookahead/lookbehind
3284 In this case we can't do fixed string optimisation.
3285 */
3286
3287 I32 deltanext, minnext, fake = 0;
3288 regnode *nscan;
3289 struct regnode_charclass_class intrnl;
3290 int f = 0;
3291
3292 data_fake.flags = 0;
3293 if (data) {
3294 data_fake.whilem_c = data->whilem_c;
3295 data_fake.last_closep = data->last_closep;
a0ed51b3 3296 }
1de06328
YO
3297 else
3298 data_fake.last_closep = &fake;
3299 if ( flags & SCF_DO_STCLASS && !scan->flags
3300 && OP(scan) == IFMATCH ) { /* Lookahead */
3301 cl_init(pRExC_state, &intrnl);
3302 data_fake.start_class = &intrnl;
3303 f |= SCF_DO_STCLASS_AND;
c277df42 3304 }
1de06328
YO
3305 if (flags & SCF_WHILEM_VISITED_POS)
3306 f |= SCF_WHILEM_VISITED_POS;
3307 next = regnext(scan);
3308 nscan = NEXTOPER(NEXTOPER(scan));
3309 minnext = study_chunk(pRExC_state, &nscan, minlenp, &deltanext, last, &data_fake, f,depth+1);
3310 if (scan->flags) {
3311 if (deltanext) {
3312 vFAIL("Variable length lookbehind not implemented");
3313 }
3314 else if (minnext > (I32)U8_MAX) {
3315 vFAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX);
3316 }
3317 scan->flags = (U8)minnext;
3318 }
3319 if (data) {
3320 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3321 pars++;
3322 if (data_fake.flags & SF_HAS_EVAL)
3323 data->flags |= SF_HAS_EVAL;
3324 data->whilem_c = data_fake.whilem_c;
3325 }
3326 if (f & SCF_DO_STCLASS_AND) {
3327 const int was = (data->start_class->flags & ANYOF_EOS);
3328
3329 cl_and(data->start_class, &intrnl);
3330 if (was)
3331 data->start_class->flags |= ANYOF_EOS;
3332 }
be8e71aa 3333 }
1de06328
YO
3334#if PERL_ENABLE_POSITIVE_ASSERTION_STUDY
3335 else {
3336 /* Positive Lookahead/lookbehind
3337 In this case we can do fixed string optimisation,
3338 but we must be careful about it. Note in the case of
3339 lookbehind the positions will be offset by the minimum
3340 length of the pattern, something we won't know about
3341 until after the recurse.
3342 */
3343 I32 deltanext, fake = 0;
3344 regnode *nscan;
3345 struct regnode_charclass_class intrnl;
3346 int f = 0;
3347 /* We use SAVEFREEPV so that when the full compile
3348 is finished perl will clean up the allocated
3349 minlens when its all done. This was we don't
3350 have to worry about freeing them when we know
3351 they wont be used, which would be a pain.
3352 */
3353 I32 *minnextp;
3354 Newx( minnextp, 1, I32 );
3355 SAVEFREEPV(minnextp);
3356
3357 if (data) {
3358 StructCopy(data, &data_fake, scan_data_t);
3359 if ((flags & SCF_DO_SUBSTR) && data->last_found) {
3360 f |= SCF_DO_SUBSTR;
3361 if (scan->flags)
3362 scan_commit(pRExC_state, &data_fake,minlenp);
3363 data_fake.last_found=newSVsv(data->last_found);
3364 }
3365 }
3366 else
3367 data_fake.last_closep = &fake;
3368 data_fake.flags = 0;
3369 if (is_inf)
3370 data_fake.flags |= SF_IS_INF;
3371 if ( flags & SCF_DO_STCLASS && !scan->flags
3372 && OP(scan) == IFMATCH ) { /* Lookahead */
3373 cl_init(pRExC_state, &intrnl);
3374 data_fake.start_class = &intrnl;
3375 f |= SCF_DO_STCLASS_AND;
3376 }
3377 if (flags & SCF_WHILEM_VISITED_POS)
3378 f |= SCF_WHILEM_VISITED_POS;
3379 next = regnext(scan);
3380 nscan = NEXTOPER(NEXTOPER(scan));
3381
3382 *minnextp = study_chunk(pRExC_state, &nscan, minnextp, &deltanext, last, &data_fake, f,depth+1);
3383 if (scan->flags) {
3384 if (deltanext) {
3385 vFAIL("Variable length lookbehind not implemented");
3386 }
3387 else if (*minnextp > (I32)U8_MAX) {
3388 vFAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX);
3389 }
3390 scan->flags = (U8)*minnextp;
3391 }
3392
3393 *minnextp += min;
3394
3395
3396 if (f & SCF_DO_STCLASS_AND) {
3397 const int was = (data->start_class->flags & ANYOF_EOS);
3398
3399 cl_and(data->start_class, &intrnl);
3400 if (was)
3401 data->start_class->flags |= ANYOF_EOS;
3402 }
3403 if (data) {
3404 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3405 pars++;
3406 if (data_fake.flags & SF_HAS_EVAL)
3407 data->flags |= SF_HAS_EVAL;
3408 data->whilem_c = data_fake.whilem_c;
3409 if ((flags & SCF_DO_SUBSTR) && data_fake.last_found) {
3410 if (RExC_rx->minlen<*minnextp)
3411 RExC_rx->minlen=*minnextp;
3412 scan_commit(pRExC_state, &data_fake, minnextp);
3413 SvREFCNT_dec(data_fake.last_found);
3414
3415 if ( data_fake.minlen_fixed != minlenp )
3416 {
3417 data->offset_fixed= data_fake.offset_fixed;
3418 data->minlen_fixed= data_fake.minlen_fixed;
3419 data->lookbehind_fixed+= scan->flags;
3420 }
3421 if ( data_fake.minlen_float != minlenp )
3422 {
3423 data->minlen_float= data_fake.minlen_float;
3424 data->offset_float_min=data_fake.offset_float_min;
3425 data->offset_float_max=data_fake.offset_float_max;
3426 data->lookbehind_float+= scan->flags;
3427 }
3428 }
3429 }
3430
653099ff 3431
653099ff 3432 }
1de06328 3433#endif
a0ed51b3
LW
3434 }
3435 else if (OP(scan) == OPEN) {
c277df42 3436 pars++;
a0ed51b3 3437 }
cb434fcc 3438 else if (OP(scan) == CLOSE) {
eb160463 3439 if ((I32)ARG(scan) == is_par) {
cb434fcc 3440 next = regnext(scan);
c277df42 3441
cb434fcc
IZ
3442 if ( next && (OP(next) != WHILEM) && next < last)
3443 is_par = 0; /* Disable optimization */
3444 }
3445 if (data)
3446 *(data->last_closep) = ARG(scan);
a0ed51b3
LW
3447 }
3448 else if (OP(scan) == EVAL) {
c277df42
IZ
3449 if (data)
3450 data->flags |= SF_HAS_EVAL;
3451 }
0a4db386
YO
3452 else if ( (OP(scan) == LOGICAL && scan->flags == 2) /* Embedded follows */
3453 || OP(scan)==RECURSE) /* recursion */
3454 {
3455 if (OP(scan)==RECURSE) {
3456 ARG2L_SET( scan, RExC_parens[ARG(scan)-1] - scan );
3457 }
0f5d15d6 3458 if (flags & SCF_DO_SUBSTR) {
1de06328 3459 scan_commit(pRExC_state,data,minlenp);
0f5d15d6
IZ
3460 data->longest = &(data->longest_float);
3461 }
3462 is_inf = is_inf_internal = 1;
653099ff 3463 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */
830247a4 3464 cl_anything(pRExC_state, data->start_class);
96776eda 3465 flags &= ~SCF_DO_STCLASS;
0f5d15d6 3466 }
786e8c11
YO
3467#ifdef TRIE_STUDY_OPT
3468#ifdef FULL_TRIE_STUDY
3469 else if (PL_regkind[OP(scan)] == TRIE) {
3470 /* NOTE - There is similar code to this block above for handling
3471 BRANCH nodes on the initial study. If you change stuff here
3472 check there too. */
3473 regnode *tail= regnext(scan);
3474 reg_trie_data *trie = (reg_trie_data*)RExC_rx->data->data[ ARG(scan) ];
3475 I32 max1 = 0, min1 = I32_MAX;
3476 struct regnode_charclass_class accum;
3477
3478 if (flags & SCF_DO_SUBSTR) /* XXXX Add !SUSPEND? */
1de06328 3479 scan_commit(pRExC_state, data,minlenp); /* Cannot merge strings after this. */
786e8c11
YO
3480 if (flags & SCF_DO_STCLASS)
3481 cl_init_zero(pRExC_state, &accum);
3482
3483 if (!trie->jump) {
3484 min1= trie->minlen;
3485 max1= trie->maxlen;
3486 } else {
3487 const regnode *nextbranch= NULL;
3488 U32 word;
3489
3490 for ( word=1 ; word <= trie->wordcount ; word++)
3491 {
3492 I32 deltanext=0, minnext=0, f = 0, fake;
3493 struct regnode_charclass_class this_class;
3494
3495 data_fake.flags = 0;
3496 if (data) {
3497 data_fake.whilem_c = data->whilem_c;
3498 data_fake.last_closep = data->last_closep;
3499 }
3500 else
3501 data_fake.last_closep = &fake;
3502
3503 if (flags & SCF_DO_STCLASS) {
3504 cl_init(pRExC_state, &this_class);
3505 data_fake.start_class = &this_class;
3506 f = SCF_DO_STCLASS_AND;
3507 }
3508 if (flags & SCF_WHILEM_VISITED_POS)
3509 f |= SCF_WHILEM_VISITED_POS;
3510
3511 if (trie->jump[word]) {
3512 if (!nextbranch)
3513 nextbranch = tail - trie->jump[0];
3514 scan= tail - trie->jump[word];
3515 /* We go from the jump point to the branch that follows
3516 it. Note this means we need the vestigal unused branches
3517 even though they arent otherwise used.
3518 */
1de06328 3519 minnext = study_chunk(pRExC_state, &scan, minlenp, &deltanext,
786e8c11
YO
3520 (regnode *)nextbranch, &data_fake, f,depth+1);
3521 }
3522 if (nextbranch && PL_regkind[OP(nextbranch)]==BRANCH)
3523 nextbranch= regnext((regnode*)nextbranch);
3524
3525 if (min1 > (I32)(minnext + trie->minlen))
3526 min1 = minnext + trie->minlen;
3527 if (max1 < (I32)(minnext + deltanext + trie->maxlen))
3528 max1 = minnext + deltanext + trie->maxlen;
3529 if (deltanext == I32_MAX)
3530 is_inf = is_inf_internal = 1;
3531
3532 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR))
3533 pars++;
3534
3535 if (data) {
3536 if (data_fake.flags & SF_HAS_EVAL)
3537 data->flags |= SF_HAS_EVAL;
3538 data->whilem_c = data_fake.whilem_c;
3539 }
3540 if (flags & SCF_DO_STCLASS)
3541 cl_or(pRExC_state, &accum, &this_class);
3542 }
3543 }
3544 if (flags & SCF_DO_SUBSTR) {
3545 data->pos_min += min1;
3546 data->pos_delta += max1 - min1;
3547 if (max1 != min1 || is_inf)
3548 data->longest = &(data->longest_float);
3549 }
3550 min += min1;
3551 delta += max1 - min1;
3552 if (flags & SCF_DO_STCLASS_OR) {
3553 cl_or(pRExC_state, data->start_class, &accum);
3554 if (min1) {
3555 cl_and(data->start_class, &and_with);
3556 flags &= ~SCF_DO_STCLASS;
3557 }
3558 }
3559 else if (flags & SCF_DO_STCLASS_AND) {
3560 if (min1) {
3561 cl_and(data->start_class, &accum);
3562 flags &= ~SCF_DO_STCLASS;
3563 }
3564 else {
3565 /* Switch to OR mode: cache the old value of
3566 * data->start_class */
3567 StructCopy(data->start_class, &and_with,
3568 struct regnode_charclass_class);
3569 flags &= ~SCF_DO_STCLASS_AND;
3570 StructCopy(&accum, data->start_class,
3571 struct regnode_charclass_class);
3572 flags |= SCF_DO_STCLASS_OR;
3573 data->start_class->flags |= ANYOF_EOS;
3574 }
3575 }
3576 scan= tail;
3577 continue;
3578 }
3579#else
3580 else if (PL_regkind[OP(scan)] == TRIE) {
3581 reg_trie_data *trie = (reg_trie_data*)RExC_rx->data->data[ ARG(scan) ];
3582 U8*bang=NULL;
3583
3584 min += trie->minlen;
3585 delta += (trie->maxlen - trie->minlen);
3586 flags &= ~SCF_DO_STCLASS; /* xxx */
3587 if (flags & SCF_DO_SUBSTR) {
1de06328 3588 scan_commit(pRExC_state,data,minlenp); /* Cannot expect anything... */
786e8c11
YO
3589 data->pos_min += trie->minlen;
3590 data->pos_delta += (trie->maxlen - trie->minlen);
3591 if (trie->maxlen != trie->minlen)
3592 data->longest = &(data->longest_float);
3593 }
3594 if (trie->jump) /* no more substrings -- for now /grr*/
3595 flags &= ~SCF_DO_SUBSTR;
3596 }
3597#endif /* old or new */
3598#endif /* TRIE_STUDY_OPT */
c277df42
IZ
3599 /* Else: zero-length, ignore. */
3600 scan = regnext(scan);
3601 }
3602
3603 finish:
3604 *scanp = scan;
aca2d497 3605 *deltap = is_inf_internal ? I32_MAX : delta;
b81d288d 3606 if (flags & SCF_DO_SUBSTR && is_inf)
c277df42 3607 data->pos_delta = I32_MAX - data->pos_min;
786e8c11 3608 if (is_par > (I32)U8_MAX)
c277df42
IZ
3609 is_par = 0;
3610 if (is_par && pars==1 && data) {
3611 data->flags |= SF_IN_PAR;
3612 data->flags &= ~SF_HAS_PAR;
a0ed51b3
LW
3613 }
3614 else if (pars && data) {
c277df42
IZ
3615 data->flags |= SF_HAS_PAR;
3616 data->flags &= ~SF_IN_PAR;
3617 }
653099ff
GS
3618 if (flags & SCF_DO_STCLASS_OR)
3619 cl_and(data->start_class, &and_with);
786e8c11
YO
3620 if (flags & SCF_TRIE_RESTUDY)
3621 data->flags |= SCF_TRIE_RESTUDY;
1de06328
YO
3622
3623 DEBUG_STUDYDATA(data,depth);
3624
c277df42
IZ
3625 return min;
3626}
3627
76e3520e 3628STATIC I32
5f66b61c 3629S_add_data(RExC_state_t *pRExC_state, I32 n, const char *s)
c277df42 3630{
830247a4 3631 if (RExC_rx->data) {
2eb97020 3632 const U32 count = RExC_rx->data->count;
b81d288d 3633 Renewc(RExC_rx->data,
2eb97020 3634 sizeof(*RExC_rx->data) + sizeof(void*) * (count + n - 1),
c277df42 3635 char, struct reg_data);
2eb97020 3636 Renew(RExC_rx->data->what, count + n, U8);
830247a4 3637 RExC_rx->data->count += n;
a0ed51b3
LW
3638 }
3639 else {
a02a5408 3640 Newxc(RExC_rx->data, sizeof(*RExC_rx->data) + sizeof(void*) * (n - 1),
c277df42 3641 char, struct reg_data);
a02a5408 3642 Newx(RExC_rx->data->what, n, U8);
830247a4 3643 RExC_rx->data->count = n;
c277df42 3644 }
830247a4
IZ
3645 Copy(s, RExC_rx->data->what + RExC_rx->data->count - n, n, U8);
3646 return RExC_rx->data->count - n;
c277df42
IZ
3647}
3648
76234dfb 3649#ifndef PERL_IN_XSUB_RE
d88dccdf 3650void
864dbfa3 3651Perl_reginitcolors(pTHX)
d88dccdf 3652{
97aff369 3653 dVAR;
1df70142 3654 const char * const s = PerlEnv_getenv("PERL_RE_COLORS");
d88dccdf 3655 if (s) {
1df70142
AL
3656 char *t = savepv(s);
3657 int i = 0;
3658 PL_colors[0] = t;
d88dccdf 3659 while (++i < 6) {
1df70142
AL
3660 t = strchr(t, '\t');
3661 if (t) {
3662 *t = '\0';
3663 PL_colors[i] = ++t;
d88dccdf
IZ
3664 }
3665 else
1df70142 3666 PL_colors[i] = t = (char *)"";
d88dccdf
IZ
3667 }
3668 } else {
1df70142 3669 int i = 0;
b81d288d 3670 while (i < 6)
06b5626a 3671 PL_colors[i++] = (char *)"";
d88dccdf
IZ
3672 }
3673 PL_colorset = 1;
3674}
76234dfb 3675#endif
8615cb43 3676
07be1b83 3677
786e8c11
YO
3678#ifdef TRIE_STUDY_OPT
3679#define CHECK_RESTUDY_GOTO \
3680 if ( \
3681 (data.flags & SCF_TRIE_RESTUDY) \
3682 && ! restudied++ \
3683 ) goto reStudy
3684#else
3685#define CHECK_RESTUDY_GOTO
3686#endif
f9f4320a 3687
a687059c 3688/*
e50aee73 3689 - pregcomp - compile a regular expression into internal code
a687059c
LW
3690 *
3691 * We can't allocate space until we know how big the compiled form will be,
3692 * but we can't compile it (and thus know how big it is) until we've got a
3693 * place to put the code. So we cheat: we compile it twice, once with code
3694 * generation turned off and size counting turned on, and once "for real".
3695 * This also means that we don't allocate space until we are sure that the
3696 * thing really will compile successfully, and we never have to move the
3697 * code and thus invalidate pointers into it. (Note that it has to be in
3698 * one piece because free() must be able to free it all.) [NB: not true in perl]
3699 *
3700 * Beware that the optimization-preparation code in here knows about some
3701 * of the structure of the compiled regexp. [I'll say.]
3702 */
b9b4dddf
YO
3703
3704
3705
f9f4320a 3706#ifndef PERL_IN_XSUB_RE
f9f4320a
YO
3707#define RE_ENGINE_PTR &PL_core_reg_engine
3708#else
f9f4320a
YO
3709extern const struct regexp_engine my_reg_engine;
3710#define RE_ENGINE_PTR &my_reg_engine
3711#endif
b9b4dddf
YO
3712/* these make a few things look better, to avoid indentation */
3713#define BEGIN_BLOCK {
f9f4320a
YO
3714#define END_BLOCK }
3715
a687059c 3716regexp *
864dbfa3 3717Perl_pregcomp(pTHX_ char *exp, char *xend, PMOP *pm)
a687059c 3718{
97aff369 3719 dVAR;
f9f4320a
YO
3720 GET_RE_DEBUG_FLAGS_DECL;
3721