3 * Copyright (C) 2012 by Larry Wall and others
5 * You may distribute under the terms of either the GNU General Public
6 * License or the Artistic License, as specified in the README file.
8 * This file contains tables and code adapted from
9 * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which requires this
12 Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
14 Permission is hereby granted, free of charge, to any person obtaining a copy of
15 this software and associated documentation files (the "Software"), to deal in
16 the Software without restriction, including without limitation the rights to
17 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
18 of the Software, and to permit persons to whom the Software is furnished to do
19 so, subject to the following conditions:
21 The above copyright notice and this permission notice shall be included in all
22 copies or substantial portions of the Software.
24 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
29 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * This file is a home for static inline functions that cannot go in other
34 * header files, because they depend on proto.h (included after most other
35 * headers) or struct definitions.
37 * Note also perlstatic.h for functions that can't or shouldn't be inlined, but
38 * whose details should be exposed to the compiler, for such things as tail
41 * Each section names the header file that the functions "belong" to.
44 /* ------------------------------- av.h ------------------------------- */
47 =for apidoc_section $AV
49 Returns the number of elements in the array C<av>. This is the true length of
50 the array, including any undefined elements. It is always the same as
51 S<C<av_top_index(av) + 1>>.
55 PERL_STATIC_INLINE Size_t
56 Perl_av_count(pTHX_ AV *av)
58 PERL_ARGS_ASSERT_AV_COUNT;
59 assert(SvTYPE(av) == SVt_PVAV);
61 return AvFILL(av) + 1;
64 /* ------------------------------- av.c ------------------------------- */
67 =for apidoc av_store_simple
69 This is a cut-down version of av_store that assumes that the array is
70 very straightforward - no magic, not readonly, and AvREAL - and that
71 C<key> is not negative. This function MUST NOT be used in situations
72 where any of those assumptions may not hold.
74 Stores an SV in an array. The array index is specified as C<key>. It
75 can be dereferenced to get the C<SV*> that was stored there (= C<val>)).
77 Note that the caller is responsible for suitably incrementing the reference
78 count of C<val> before the call.
80 Approximate Perl equivalent: C<splice(@myarray, $key, 1, $val)>.
85 PERL_STATIC_INLINE SV**
86 Perl_av_store_simple(pTHX_ AV *av, SSize_t key, SV *val)
90 PERL_ARGS_ASSERT_AV_STORE_SIMPLE;
91 assert(SvTYPE(av) == SVt_PVAV);
92 assert(!SvMAGICAL(av));
93 assert(!SvREADONLY(av));
99 if (AvFILLp(av) < key) {
100 if (key > AvMAX(av)) {
106 SvREFCNT_dec(ary[key]);
113 =for apidoc av_fetch_simple
115 This is a cut-down version of av_fetch that assumes that the array is
116 very straightforward - no magic, not readonly, and AvREAL - and that
117 C<key> is not negative. This function MUST NOT be used in situations
118 where any of those assumptions may not hold.
120 Returns the SV at the specified index in the array. The C<key> is the
121 index. If lval is true, you are guaranteed to get a real SV back (in case
122 it wasn't real before), which you can then modify. Check that the return
123 value is non-null before dereferencing it to a C<SV*>.
125 The rough perl equivalent is C<$myarray[$key]>.
130 PERL_STATIC_INLINE SV**
131 Perl_av_fetch_simple(pTHX_ AV *av, SSize_t key, I32 lval)
133 PERL_ARGS_ASSERT_AV_FETCH_SIMPLE;
134 assert(SvTYPE(av) == SVt_PVAV);
135 assert(!SvMAGICAL(av));
136 assert(!SvREADONLY(av));
140 if ( (key > AvFILLp(av)) || !AvARRAY(av)[key]) {
141 return lval ? av_store_simple(av,key,newSV_type(SVt_NULL)) : NULL;
143 return &AvARRAY(av)[key];
148 =for apidoc av_push_simple
150 This is a cut-down version of av_push that assumes that the array is very
151 straightforward - no magic, not readonly, and AvREAL - and that C<key> is
152 not less than -1. This function MUST NOT be used in situations where any
153 of those assumptions may not hold.
155 Pushes an SV (transferring control of one reference count) onto the end of the
156 array. The array will grow automatically to accommodate the addition.
158 Perl equivalent: C<push @myarray, $val;>.
163 PERL_STATIC_INLINE void
164 Perl_av_push_simple(pTHX_ AV *av, SV *val)
166 PERL_ARGS_ASSERT_AV_PUSH_SIMPLE;
167 assert(SvTYPE(av) == SVt_PVAV);
168 assert(!SvMAGICAL(av));
169 assert(!SvREADONLY(av));
171 assert(AvFILLp(av) > -2);
173 (void)av_store_simple(av,AvFILLp(av)+1,val);
177 =for apidoc av_new_alloc
179 This implements L<perlapi/C<newAV_alloc_x>>
180 and L<perlapi/C<newAV_alloc_xz>>, which are the public API for this
183 Creates a new AV and allocates its SV* array.
185 This is similar to, but more efficient than doing:
190 The size parameter is used to pre-allocate a SV* array large enough to
191 hold at least elements C<0..(size-1)>. C<size> must be at least 1.
193 The C<zeroflag> parameter controls whether or not the array is NULL
199 PERL_STATIC_INLINE AV *
200 Perl_av_new_alloc(pTHX_ SSize_t size, bool zeroflag)
202 AV * const av = newAV();
204 PERL_ARGS_ASSERT_AV_NEW_ALLOC;
207 Newx(ary, size, SV*); /* Newx performs the memwrap check */
210 AvMAX(av) = size - 1;
213 Zero(ary, size, SV*);
219 /* ------------------------------- cv.h ------------------------------- */
222 =for apidoc_section $CV
224 Returns the GV associated with the CV C<sv>, reifying it if necessary.
228 PERL_STATIC_INLINE GV *
229 Perl_CvGV(pTHX_ CV *sv)
231 PERL_ARGS_ASSERT_CVGV;
234 ? Perl_cvgv_from_hek(aTHX_ sv)
235 : ((XPVCV*)MUTABLE_PTR(SvANY(sv)))->xcv_gv_u.xcv_gv;
240 Returns the recursion level of the CV C<sv>. Hence >= 2 indicates we are in a
245 PERL_STATIC_INLINE I32 *
246 Perl_CvDEPTH(const CV * const sv)
248 PERL_ARGS_ASSERT_CVDEPTH;
249 assert(SvTYPE(sv) == SVt_PVCV || SvTYPE(sv) == SVt_PVFM);
251 return &((XPVCV*)SvANY(sv))->xcv_depth;
255 CvPROTO returns the prototype as stored, which is not necessarily what
256 the interpreter should be using. Specifically, the interpreter assumes
257 that spaces have been stripped, which has been the case if the prototype
258 was added by toke.c, but is generally not the case if it was added elsewhere.
259 Since we can't enforce the spacelessness at assignment time, this routine
260 provides a temporary copy at parse time with spaces removed.
261 I<orig> is the start of the original buffer, I<len> is the length of the
262 prototype and will be updated when this returns.
266 PERL_STATIC_INLINE char *
267 S_strip_spaces(pTHX_ const char * orig, STRLEN * const len)
271 tmpsv = newSVpvn_flags(orig, *len, SVs_TEMP);
279 *len = tmps - SvPVX(tmpsv);
284 /* ------------------------------- mg.h ------------------------------- */
286 #if defined(PERL_CORE) || defined(PERL_EXT)
287 /* assumes get-magic and stringification have already occurred */
288 PERL_STATIC_INLINE STRLEN
289 S_MgBYTEPOS(pTHX_ MAGIC *mg, SV *sv, const char *s, STRLEN len)
291 assert(mg->mg_type == PERL_MAGIC_regex_global);
292 assert(mg->mg_len != -1);
293 if (mg->mg_flags & MGf_BYTES || !DO_UTF8(sv))
294 return (STRLEN)mg->mg_len;
296 const STRLEN pos = (STRLEN)mg->mg_len;
297 /* Without this check, we may read past the end of the buffer: */
298 if (pos > sv_or_pv_len_utf8(sv, s, len)) return len+1;
299 return sv_or_pv_pos_u2b(sv, s, pos, NULL);
304 /* ------------------------------- pad.h ------------------------------ */
306 #if defined(PERL_IN_PAD_C) || defined(PERL_IN_OP_C)
307 PERL_STATIC_INLINE bool
308 S_PadnameIN_SCOPE(const PADNAME * const pn, const U32 seq)
310 PERL_ARGS_ASSERT_PADNAMEIN_SCOPE;
312 /* is seq within the range _LOW to _HIGH ?
313 * This is complicated by the fact that PL_cop_seqmax
314 * may have wrapped around at some point */
315 if (COP_SEQ_RANGE_LOW(pn) == PERL_PADSEQ_INTRO)
316 return FALSE; /* not yet introduced */
318 if (COP_SEQ_RANGE_HIGH(pn) == PERL_PADSEQ_INTRO) {
319 /* in compiling scope */
321 (seq > COP_SEQ_RANGE_LOW(pn))
322 ? (seq - COP_SEQ_RANGE_LOW(pn) < (U32_MAX >> 1))
323 : (COP_SEQ_RANGE_LOW(pn) - seq > (U32_MAX >> 1))
328 (COP_SEQ_RANGE_LOW(pn) > COP_SEQ_RANGE_HIGH(pn))
330 ( seq > COP_SEQ_RANGE_LOW(pn)
331 || seq <= COP_SEQ_RANGE_HIGH(pn))
333 : ( seq > COP_SEQ_RANGE_LOW(pn)
334 && seq <= COP_SEQ_RANGE_HIGH(pn))
341 /* ------------------------------- pp.h ------------------------------- */
343 PERL_STATIC_INLINE I32
346 DEBUG_s(DEBUG_v(PerlIO_printf(Perl_debug_log,
347 "MARK top %p %" IVdf "\n",
349 (IV)*PL_markstack_ptr)));
350 return *PL_markstack_ptr;
353 PERL_STATIC_INLINE I32
356 DEBUG_s(DEBUG_v(PerlIO_printf(Perl_debug_log,
357 "MARK pop %p %" IVdf "\n",
358 (PL_markstack_ptr-1),
359 (IV)*(PL_markstack_ptr-1))));
360 assert((PL_markstack_ptr > PL_markstack) || !"MARK underflow");
361 return *PL_markstack_ptr--;
364 /* ----------------------------- regexp.h ----------------------------- */
366 /* PVLVs need to act as a superset of all scalar types - they are basically
367 * PVMGs with a few extra fields.
368 * REGEXPs are first class scalars, but have many fields that can't be copied
371 * Hence we take a different approach - instead of a copy, PVLVs store a pointer
372 * back to the original body. To avoid increasing the size of PVLVs just for the
373 * rare case of REGEXP assignment, this pointer is stored in the memory usually
374 * used for SvLEN(). Hence the check for SVt_PVLV below, and the ? : ternary to
375 * read the pointer from the two possible locations. The macro SvLEN() wraps the
376 * access to the union's member xpvlenu_len, but there is no equivalent macro
377 * for wrapping the union's member xpvlenu_rx, hence the direct reference here.
379 * See commit df6b4bd56551f2d3 for more details. */
381 PERL_STATIC_INLINE struct regexp *
382 Perl_ReANY(const REGEXP * const re)
384 XPV* const p = (XPV*)SvANY(re);
386 PERL_ARGS_ASSERT_REANY;
387 assert(isREGEXP(re));
389 return SvTYPE(re) == SVt_PVLV ? p->xpv_len_u.xpvlenu_rx
390 : (struct regexp *)p;
393 /* ------------------------------- utf8.h ------------------------------- */
396 =for apidoc_section $unicode
399 PERL_STATIC_INLINE void
400 Perl_append_utf8_from_native_byte(const U8 byte, U8** dest)
402 /* Takes an input 'byte' (Latin1 or EBCDIC) and appends it to the UTF-8
403 * encoded string at '*dest', updating '*dest' to include it */
405 PERL_ARGS_ASSERT_APPEND_UTF8_FROM_NATIVE_BYTE;
407 if (NATIVE_BYTE_IS_INVARIANT(byte))
410 *((*dest)++) = UTF8_EIGHT_BIT_HI(byte);
411 *((*dest)++) = UTF8_EIGHT_BIT_LO(byte);
416 =for apidoc valid_utf8_to_uvchr
417 Like C<L<perlapi/utf8_to_uvchr_buf>>, but should only be called when it is
418 known that the next character in the input UTF-8 string C<s> is well-formed
419 (I<e.g.>, it passes C<L<perlapi/isUTF8_CHAR>>. Surrogates, non-character code
420 points, and non-Unicode code points are allowed.
426 PERL_STATIC_INLINE UV
427 Perl_valid_utf8_to_uvchr(const U8 *s, STRLEN *retlen)
429 const UV expectlen = UTF8SKIP(s);
430 const U8* send = s + expectlen;
433 PERL_ARGS_ASSERT_VALID_UTF8_TO_UVCHR;
439 /* An invariant is trivially returned */
440 if (expectlen == 1) {
444 /* Remove the leading bits that indicate the number of bytes, leaving just
445 * the bits that are part of the value */
446 uv = NATIVE_UTF8_TO_I8(uv) & UTF_START_MASK(expectlen);
448 /* Now, loop through the remaining bytes, accumulating each into the
449 * working total as we go. (I khw tried unrolling the loop for up to 4
450 * bytes, but there was no performance improvement) */
451 for (++s; s < send; s++) {
452 uv = UTF8_ACCUMULATE(uv, *s);
455 return UNI_TO_NATIVE(uv);
460 =for apidoc is_utf8_invariant_string
462 Returns TRUE if the first C<len> bytes of the string C<s> are the same
463 regardless of the UTF-8 encoding of the string (or UTF-EBCDIC encoding on
464 EBCDIC machines); otherwise it returns FALSE. That is, it returns TRUE if they
465 are UTF-8 invariant. On ASCII-ish machines, all the ASCII characters and only
466 the ASCII characters fit this definition. On EBCDIC machines, the ASCII-range
467 characters are invariant, but so also are the C1 controls.
469 If C<len> is 0, it will be calculated using C<strlen(s)>, (which means if you
470 use this option, that C<s> can't have embedded C<NUL> characters and has to
471 have a terminating C<NUL> byte).
474 C<L</is_utf8_string>>,
475 C<L</is_utf8_string_flags>>,
476 C<L</is_utf8_string_loc>>,
477 C<L</is_utf8_string_loc_flags>>,
478 C<L</is_utf8_string_loclen>>,
479 C<L</is_utf8_string_loclen_flags>>,
480 C<L</is_utf8_fixed_width_buf_flags>>,
481 C<L</is_utf8_fixed_width_buf_loc_flags>>,
482 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
483 C<L</is_strict_utf8_string>>,
484 C<L</is_strict_utf8_string_loc>>,
485 C<L</is_strict_utf8_string_loclen>>,
486 C<L</is_c9strict_utf8_string>>,
487 C<L</is_c9strict_utf8_string_loc>>,
489 C<L</is_c9strict_utf8_string_loclen>>.
495 #define is_utf8_invariant_string(s, len) \
496 is_utf8_invariant_string_loc(s, len, NULL)
499 =for apidoc is_utf8_invariant_string_loc
501 Like C<L</is_utf8_invariant_string>> but upon failure, stores the location of
502 the first UTF-8 variant character in the C<ep> pointer; if all characters are
503 UTF-8 invariant, this function does not change the contents of C<*ep>.
509 PERL_STATIC_INLINE bool
510 Perl_is_utf8_invariant_string_loc(const U8* const s, STRLEN len, const U8 ** ep)
515 PERL_ARGS_ASSERT_IS_UTF8_INVARIANT_STRING_LOC;
518 len = strlen((const char *)s);
523 /* This looks like 0x010101... */
524 # define PERL_COUNT_MULTIPLIER (~ (UINTMAX_C(0)) / 0xFF)
526 /* This looks like 0x808080... */
527 # define PERL_VARIANTS_WORD_MASK (PERL_COUNT_MULTIPLIER * 0x80)
528 # define PERL_WORDSIZE sizeof(PERL_UINTMAX_T)
529 # define PERL_WORD_BOUNDARY_MASK (PERL_WORDSIZE - 1)
531 /* Evaluates to 0 if 'x' is at a word boundary; otherwise evaluates to 1, by
532 * or'ing together the lowest bits of 'x'. Hopefully the final term gets
533 * optimized out completely on a 32-bit system, and its mask gets optimized out
534 * on a 64-bit system */
535 # define PERL_IS_SUBWORD_ADDR(x) (1 & ( PTR2nat(x) \
536 | ( PTR2nat(x) >> 1) \
538 & PERL_WORD_BOUNDARY_MASK) >> 2))))
542 /* Do the word-at-a-time iff there is at least one usable full word. That
543 * means that after advancing to a word boundary, there still is at least a
544 * full word left. The number of bytes needed to advance is 'wordsize -
545 * offset' unless offset is 0. */
546 if ((STRLEN) (send - x) >= PERL_WORDSIZE
548 /* This term is wordsize if subword; 0 if not */
549 + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(x)
552 - (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK))
555 /* Process per-byte until reach word boundary. XXX This loop could be
556 * eliminated if we knew that this platform had fast unaligned reads */
557 while (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK) {
558 if (! UTF8_IS_INVARIANT(*x)) {
568 /* Here, we know we have at least one full word to process. Process
569 * per-word as long as we have at least a full word left */
571 if ((* (PERL_UINTMAX_T *) x) & PERL_VARIANTS_WORD_MASK) {
573 /* Found a variant. Just return if caller doesn't want its
579 # if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 \
580 || BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
582 *ep = x + variant_byte_number(* (PERL_UINTMAX_T *) x);
583 assert(*ep >= s && *ep < send);
587 # else /* If weird byte order, drop into next loop to do byte-at-a-time
596 } while (x + PERL_WORDSIZE <= send);
599 #endif /* End of ! EBCDIC */
601 /* Process per-byte */
603 if (! UTF8_IS_INVARIANT(*x)) {
617 /* See if the platform has builtins for finding the most/least significant bit,
618 * and which one is right for using on 32 and 64 bit operands */
619 #if (__has_builtin(__builtin_clz) || PERL_GCC_VERSION_GE(3,4,0))
620 # if U32SIZE == INTSIZE
621 # define PERL_CLZ_32 __builtin_clz
623 # if defined(U64TYPE) && U64SIZE == INTSIZE
624 # define PERL_CLZ_64 __builtin_clz
627 #if (__has_builtin(__builtin_ctz) || PERL_GCC_VERSION_GE(3,4,0))
628 # if U32SIZE == INTSIZE
629 # define PERL_CTZ_32 __builtin_ctz
631 # if defined(U64TYPE) && U64SIZE == INTSIZE
632 # define PERL_CTZ_64 __builtin_ctz
636 #if (__has_builtin(__builtin_clzl) || PERL_GCC_VERSION_GE(3,4,0))
637 # if U32SIZE == LONGSIZE && ! defined(PERL_CLZ_32)
638 # define PERL_CLZ_32 __builtin_clzl
640 # if defined(U64TYPE) && U64SIZE == LONGSIZE && ! defined(PERL_CLZ_64)
641 # define PERL_CLZ_64 __builtin_clzl
644 #if (__has_builtin(__builtin_ctzl) || PERL_GCC_VERSION_GE(3,4,0))
645 # if U32SIZE == LONGSIZE && ! defined(PERL_CTZ_32)
646 # define PERL_CTZ_32 __builtin_ctzl
648 # if defined(U64TYPE) && U64SIZE == LONGSIZE && ! defined(PERL_CTZ_64)
649 # define PERL_CTZ_64 __builtin_ctzl
653 #if (__has_builtin(__builtin_clzll) || PERL_GCC_VERSION_GE(3,4,0))
654 # if U32SIZE == LONGLONGSIZE && ! defined(PERL_CLZ_32)
655 # define PERL_CLZ_32 __builtin_clzll
657 # if defined(U64TYPE) && U64SIZE == LONGLONGSIZE && ! defined(PERL_CLZ_64)
658 # define PERL_CLZ_64 __builtin_clzll
661 #if (__has_builtin(__builtin_ctzll) || PERL_GCC_VERSION_GE(3,4,0))
662 # if U32SIZE == LONGLONGSIZE && ! defined(PERL_CTZ_32)
663 # define PERL_CTZ_32 __builtin_ctzll
665 # if defined(U64TYPE) && U64SIZE == LONGLONGSIZE && ! defined(PERL_CTZ_64)
666 # define PERL_CTZ_64 __builtin_ctzll
670 #if defined(_MSC_VER)
672 # pragma intrinsic(_BitScanForward)
673 # pragma intrinsic(_BitScanReverse)
675 # pragma intrinsic(_BitScanForward64)
676 # pragma intrinsic(_BitScanReverse64)
680 /* The reason there are not checks to see if ffs() and ffsl() are available for
681 * determining the lsb, is because these don't improve on the deBruijn method
682 * fallback, which is just a branchless integer multiply, array element
683 * retrieval, and shift. The others, even if the function call overhead is
684 * optimized out, have to cope with the possibility of the input being all
685 * zeroes, and almost certainly will have conditionals for this eventuality.
686 * khw, at the time of this commit, looked at the source for both gcc and clang
687 * to verify this. (gcc used a method inferior to deBruijn.) */
689 /* Below are functions to find the first, last, or only set bit in a word. On
690 * platforms with 64-bit capability, there is a pair for each operation; the
691 * first taking a 64 bit operand, and the second a 32 bit one. The logic is
692 * the same in each pair, so the second is stripped of most comments. */
694 #ifdef U64TYPE /* HAS_QUAD not usable outside the core */
696 PERL_STATIC_INLINE unsigned
697 Perl_lsbit_pos64(U64 word)
699 /* Find the position (0..63) of the least significant set bit in the input
704 /* If we can determine that the platform has a usable fast method to get
705 * this info, use that */
707 # if defined(PERL_CTZ_64)
708 # define PERL_HAS_FAST_GET_LSB_POS64
710 return (unsigned) PERL_CTZ_64(word);
712 # elif U64SIZE == 8 && defined(_WIN64) && defined(_MSC_VER)
713 # define PERL_HAS_FAST_GET_LSB_POS64
717 _BitScanForward64(&index, word);
718 return (unsigned)index;
723 /* Here, we didn't find a fast method for finding the lsb. Fall back to
724 * making the lsb the only set bit in the word, and use our function that
725 * works on words with a single bit set.
728 * https://stackoverflow.com/questions/757059/position-of-least-significant-bit-that-is-set
730 * The word will look like this, with a rightmost set bit in position 's':
731 * ('x's are don't cares, and 'y's are their complements)
734 * y..y011..11 Complement
736 * 0..0100..00 And with the original
738 * (Yes, complementing and adding 1 is just taking the negative on 2's
739 * complement machines, but not on 1's complement ones, and some compilers
740 * complain about negating an unsigned.)
742 return single_1bit_pos64(word & (~word + 1));
748 # define lsbit_pos_uintmax_(word) lsbit_pos64(word)
750 # define lsbit_pos_uintmax_(word) lsbit_pos32(word)
753 PERL_STATIC_INLINE unsigned /* Like above for 32 bit word */
754 Perl_lsbit_pos32(U32 word)
756 /* Find the position (0..31) of the least significant set bit in the input
761 #if defined(PERL_CTZ_32)
762 # define PERL_HAS_FAST_GET_LSB_POS32
764 return (unsigned) PERL_CTZ_32(word);
766 #elif U32SIZE == 4 && defined(_MSC_VER)
767 # define PERL_HAS_FAST_GET_LSB_POS32
771 _BitScanForward(&index, word);
772 return (unsigned)index;
777 return single_1bit_pos32(word & (~word + 1));
784 /* Convert the leading zeros count to the bit position of the first set bit.
785 * This just subtracts from the highest position, 31 or 63. But some compilers
786 * don't optimize this optimally, and so a bit of bit twiddling encourages them
787 * to do the right thing. It turns out that subtracting a smaller non-negative
788 * number 'x' from 2**n-1 for any n is the same as taking the exclusive-or of
789 * the two numbers. To see why, first note that the sum of any number, x, and
790 * its complement, x', is all ones. So all ones minus x is x'. Then note that
791 * the xor of x and all ones is x'. */
792 #define LZC_TO_MSBIT_POS_(size, lzc) ((size##SIZE * CHARBITS - 1) ^ (lzc))
794 #ifdef U64TYPE /* HAS_QUAD not usable outside the core */
796 PERL_STATIC_INLINE unsigned
797 Perl_msbit_pos64(U64 word)
799 /* Find the position (0..63) of the most significant set bit in the input
804 /* If we can determine that the platform has a usable fast method to get
807 # if defined(PERL_CLZ_64)
808 # define PERL_HAS_FAST_GET_MSB_POS64
810 return (unsigned) LZC_TO_MSBIT_POS_(U64, PERL_CLZ_64(word));
812 # elif U64SIZE == 8 && defined(_WIN64) && defined(_MSC_VER)
813 # define PERL_HAS_FAST_GET_MSB_POS64
817 _BitScanReverse64(&index, word);
818 return (unsigned)index;
823 /* Here, we didn't find a fast method for finding the msb. Fall back to
824 * making the msb the only set bit in the word, and use our function that
825 * works on words with a single bit set.
827 * Isolate the msb; http://codeforces.com/blog/entry/10330
829 * Only the most significant set bit matters. Or'ing word with its right
830 * shift of 1 makes that bit and the next one to its right both 1.
831 * Repeating that with the right shift of 2 makes for 4 1-bits in a row.
832 * ... We end with the msb and all to the right being 1. */
837 word |= (word >> 16);
838 word |= (word >> 32);
840 /* Then subtracting the right shift by 1 clears all but the left-most of
841 * the 1 bits, which is our desired result */
844 /* Now we have a single bit set */
845 return single_1bit_pos64(word);
851 # define msbit_pos_uintmax_(word) msbit_pos64(word)
853 # define msbit_pos_uintmax_(word) msbit_pos32(word)
856 PERL_STATIC_INLINE unsigned
857 Perl_msbit_pos32(U32 word)
859 /* Find the position (0..31) of the most significant set bit in the input
864 #if defined(PERL_CLZ_32)
865 # define PERL_HAS_FAST_GET_MSB_POS32
867 return (unsigned) LZC_TO_MSBIT_POS_(U32, PERL_CLZ_32(word));
869 #elif U32SIZE == 4 && defined(_MSC_VER)
870 # define PERL_HAS_FAST_GET_MSB_POS32
874 _BitScanReverse(&index, word);
875 return (unsigned)index;
884 word |= (word >> 16);
886 return single_1bit_pos32(word);
892 #if UVSIZE == U64SIZE
893 # define msbit_pos(word) msbit_pos64(word)
894 # define lsbit_pos(word) lsbit_pos64(word)
895 #elif UVSIZE == U32SIZE
896 # define msbit_pos(word) msbit_pos32(word)
897 # define lsbit_pos(word) lsbit_pos32(word)
900 #ifdef U64TYPE /* HAS_QUAD not usable outside the core */
902 PERL_STATIC_INLINE unsigned
903 Perl_single_1bit_pos64(U64 word)
905 /* Given a 64-bit word known to contain all zero bits except one 1 bit,
906 * find and return the 1's position: 0..63 */
908 # ifdef PERL_CORE /* macro not exported */
909 ASSUME(isPOWER_OF_2(word));
911 ASSUME(word && (word & (word-1)) == 0);
914 /* The only set bit is both the most and least significant bit. If we have
915 * a fast way of finding either one, use that.
917 * It may appear at first glance that those functions call this one, but
918 * they don't if the corresponding #define is set */
920 # ifdef PERL_HAS_FAST_GET_MSB_POS64
922 return msbit_pos64(word);
924 # elif defined(PERL_HAS_FAST_GET_LSB_POS64)
926 return lsbit_pos64(word);
930 /* The position of the only set bit in a word can be quickly calculated
931 * using deBruijn sequences. See for example
932 * https://en.wikipedia.org/wiki/De_Bruijn_sequence */
933 return PL_deBruijn_bitpos_tab64[(word * PERL_deBruijnMagic64_)
934 >> PERL_deBruijnShift64_];
941 PERL_STATIC_INLINE unsigned
942 Perl_single_1bit_pos32(U32 word)
944 /* Given a 32-bit word known to contain all zero bits except one 1 bit,
945 * find and return the 1's position: 0..31 */
947 #ifdef PERL_CORE /* macro not exported */
948 ASSUME(isPOWER_OF_2(word));
950 ASSUME(word && (word & (word-1)) == 0);
952 #ifdef PERL_HAS_FAST_GET_MSB_POS32
954 return msbit_pos32(word);
956 #elif defined(PERL_HAS_FAST_GET_LSB_POS32)
958 return lsbit_pos32(word);
960 /* Unlikely, but possible for the platform to have a wider fast operation but
961 * not a narrower one. But easy enough to handle the case by widening the
962 * parameter size. (Going the other way, emulating 64 bit by two 32 bit ops
963 * would be slower than the deBruijn method.) */
964 #elif defined(PERL_HAS_FAST_GET_MSB_POS64)
966 return msbit_pos64(word);
968 #elif defined(PERL_HAS_FAST_GET_LSB_POS64)
970 return lsbit_pos64(word);
974 return PL_deBruijn_bitpos_tab32[(word * PERL_deBruijnMagic32_)
975 >> PERL_deBruijnShift32_];
982 PERL_STATIC_INLINE unsigned int
983 Perl_variant_byte_number(PERL_UINTMAX_T word)
985 /* This returns the position in a word (0..7) of the first variant byte in
986 * it. This is a helper function. Note that there are no branches */
988 /* Get just the msb bits of each byte */
989 word &= PERL_VARIANTS_WORD_MASK;
991 /* This should only be called if we know there is a variant byte in the
995 # if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678
997 /* Bytes are stored like
998 * Byte8 ... Byte2 Byte1
999 * 63..56...15...8 7...0
1000 * so getting the lsb of the whole modified word is getting the msb of the
1001 * first byte that has its msb set */
1002 word = lsbit_pos_uintmax_(word);
1004 /* Here, word contains the position 7,15,23,...55,63 of that bit. Convert
1006 return (unsigned int) ((word + 1) >> 3) - 1;
1008 # elif BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
1010 /* Bytes are stored like
1011 * Byte1 Byte2 ... Byte8
1012 * 63..56 55..47 ... 7...0
1013 * so getting the msb of the whole modified word is getting the msb of the
1014 * first byte that has its msb set */
1015 word = msbit_pos_uintmax_(word);
1017 /* Here, word contains the position 63,55,...,23,15,7 of that bit. Convert
1019 word = ((word + 1) >> 3) - 1;
1021 /* And invert the result because of the reversed byte order on this
1023 word = CHARBITS - word - 1;
1025 return (unsigned int) word;
1028 # error Unexpected byte order
1034 #if defined(PERL_CORE) || defined(PERL_EXT)
1037 =for apidoc variant_under_utf8_count
1039 This function looks at the sequence of bytes between C<s> and C<e>, which are
1040 assumed to be encoded in ASCII/Latin1, and returns how many of them would
1041 change should the string be translated into UTF-8. Due to the nature of UTF-8,
1042 each of these would occupy two bytes instead of the single one in the input
1043 string. Thus, this function returns the precise number of bytes the string
1044 would expand by when translated to UTF-8.
1046 Unlike most of the other functions that have C<utf8> in their name, the input
1047 to this function is NOT a UTF-8-encoded string. The function name is slightly
1048 I<odd> to emphasize this.
1050 This function is internal to Perl because khw thinks that any XS code that
1051 would want this is probably operating too close to the internals. Presenting a
1052 valid use case could change that.
1055 C<L<perlapi/is_utf8_invariant_string>>
1057 C<L<perlapi/is_utf8_invariant_string_loc>>,
1063 PERL_STATIC_INLINE Size_t
1064 S_variant_under_utf8_count(const U8* const s, const U8* const e)
1069 PERL_ARGS_ASSERT_VARIANT_UNDER_UTF8_COUNT;
1073 /* Test if the string is long enough to use word-at-a-time. (Logic is the
1074 * same as for is_utf8_invariant_string()) */
1075 if ((STRLEN) (e - x) >= PERL_WORDSIZE
1076 + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(x)
1077 - (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK))
1080 /* Process per-byte until reach word boundary. XXX This loop could be
1081 * eliminated if we knew that this platform had fast unaligned reads */
1082 while (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK) {
1083 count += ! UTF8_IS_INVARIANT(*x++);
1086 /* Process per-word as long as we have at least a full word left */
1087 do { /* Commit 03c1e4ab1d6ee9062fb3f94b0ba31db6698724b1 contains an
1088 explanation of how this works */
1089 PERL_UINTMAX_T increment
1090 = ((((* (PERL_UINTMAX_T *) x) & PERL_VARIANTS_WORD_MASK) >> 7)
1091 * PERL_COUNT_MULTIPLIER)
1092 >> ((PERL_WORDSIZE - 1) * CHARBITS);
1093 count += (Size_t) increment;
1095 } while (x + PERL_WORDSIZE <= e);
1100 /* Process per-byte */
1102 if (! UTF8_IS_INVARIANT(*x)) {
1114 /* Keep these around for these files */
1115 #if ! defined(PERL_IN_REGEXEC_C) && ! defined(PERL_IN_UTF8_C)
1116 # undef PERL_WORDSIZE
1117 # undef PERL_COUNT_MULTIPLIER
1118 # undef PERL_WORD_BOUNDARY_MASK
1119 # undef PERL_VARIANTS_WORD_MASK
1123 =for apidoc is_utf8_string
1125 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1126 Perl-extended-UTF-8 string; returns FALSE otherwise. If C<len> is 0, it will
1127 be calculated using C<strlen(s)> (which means if you use this option, that C<s>
1128 can't have embedded C<NUL> characters and has to have a terminating C<NUL>
1129 byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'.
1131 This function considers Perl's extended UTF-8 to be valid. That means that
1132 code points above Unicode, surrogates, and non-character code points are
1133 considered valid by this function. Use C<L</is_strict_utf8_string>>,
1134 C<L</is_c9strict_utf8_string>>, or C<L</is_utf8_string_flags>> to restrict what
1135 code points are considered valid.
1138 C<L</is_utf8_invariant_string>>,
1139 C<L</is_utf8_invariant_string_loc>>,
1140 C<L</is_utf8_string_loc>>,
1141 C<L</is_utf8_string_loclen>>,
1142 C<L</is_utf8_fixed_width_buf_flags>>,
1143 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1144 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1149 #define is_utf8_string(s, len) is_utf8_string_loclen(s, len, NULL, NULL)
1151 #if defined(PERL_CORE) || defined (PERL_EXT)
1154 =for apidoc is_utf8_non_invariant_string
1156 Returns TRUE if L<perlapi/is_utf8_invariant_string> returns FALSE for the first
1157 C<len> bytes of the string C<s>, but they are, nonetheless, legal Perl-extended
1158 UTF-8; otherwise returns FALSE.
1160 A TRUE return means that at least one code point represented by the sequence
1161 either is a wide character not representable as a single byte, or the
1162 representation differs depending on whether the sequence is encoded in UTF-8 or
1166 C<L<perlapi/is_utf8_invariant_string>>,
1167 C<L<perlapi/is_utf8_string>>
1171 This is commonly used to determine if a SV's UTF-8 flag should be turned on.
1172 It generally needn't be if its string is entirely UTF-8 invariant, and it
1173 shouldn't be if it otherwise contains invalid UTF-8.
1175 It is an internal function because khw thinks that XS code shouldn't be working
1176 at this low a level. A valid use case could change that.
1180 PERL_STATIC_INLINE bool
1181 Perl_is_utf8_non_invariant_string(const U8* const s, STRLEN len)
1183 const U8 * first_variant;
1185 PERL_ARGS_ASSERT_IS_UTF8_NON_INVARIANT_STRING;
1187 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1191 return is_utf8_string(first_variant, len - (first_variant - s));
1197 =for apidoc is_strict_utf8_string
1199 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1200 UTF-8-encoded string that is fully interchangeable by any application using
1201 Unicode rules; otherwise it returns FALSE. If C<len> is 0, it will be
1202 calculated using C<strlen(s)> (which means if you use this option, that C<s>
1203 can't have embedded C<NUL> characters and has to have a terminating C<NUL>
1204 byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'.
1206 This function returns FALSE for strings containing any
1207 code points above the Unicode max of 0x10FFFF, surrogate code points, or
1208 non-character code points.
1211 C<L</is_utf8_invariant_string>>,
1212 C<L</is_utf8_invariant_string_loc>>,
1213 C<L</is_utf8_string>>,
1214 C<L</is_utf8_string_flags>>,
1215 C<L</is_utf8_string_loc>>,
1216 C<L</is_utf8_string_loc_flags>>,
1217 C<L</is_utf8_string_loclen>>,
1218 C<L</is_utf8_string_loclen_flags>>,
1219 C<L</is_utf8_fixed_width_buf_flags>>,
1220 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1221 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1222 C<L</is_strict_utf8_string_loc>>,
1223 C<L</is_strict_utf8_string_loclen>>,
1224 C<L</is_c9strict_utf8_string>>,
1225 C<L</is_c9strict_utf8_string_loc>>,
1227 C<L</is_c9strict_utf8_string_loclen>>.
1232 #define is_strict_utf8_string(s, len) is_strict_utf8_string_loclen(s, len, NULL, NULL)
1235 =for apidoc is_c9strict_utf8_string
1237 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1238 UTF-8-encoded string that conforms to
1239 L<Unicode Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>;
1240 otherwise it returns FALSE. If C<len> is 0, it will be calculated using
1241 C<strlen(s)> (which means if you use this option, that C<s> can't have embedded
1242 C<NUL> characters and has to have a terminating C<NUL> byte). Note that all
1243 characters being ASCII constitute 'a valid UTF-8 string'.
1245 This function returns FALSE for strings containing any code points above the
1246 Unicode max of 0x10FFFF or surrogate code points, but accepts non-character
1248 L<Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>.
1251 C<L</is_utf8_invariant_string>>,
1252 C<L</is_utf8_invariant_string_loc>>,
1253 C<L</is_utf8_string>>,
1254 C<L</is_utf8_string_flags>>,
1255 C<L</is_utf8_string_loc>>,
1256 C<L</is_utf8_string_loc_flags>>,
1257 C<L</is_utf8_string_loclen>>,
1258 C<L</is_utf8_string_loclen_flags>>,
1259 C<L</is_utf8_fixed_width_buf_flags>>,
1260 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1261 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1262 C<L</is_strict_utf8_string>>,
1263 C<L</is_strict_utf8_string_loc>>,
1264 C<L</is_strict_utf8_string_loclen>>,
1265 C<L</is_c9strict_utf8_string_loc>>,
1267 C<L</is_c9strict_utf8_string_loclen>>.
1272 #define is_c9strict_utf8_string(s, len) is_c9strict_utf8_string_loclen(s, len, NULL, 0)
1275 =for apidoc is_utf8_string_flags
1277 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1278 UTF-8 string, subject to the restrictions imposed by C<flags>;
1279 returns FALSE otherwise. If C<len> is 0, it will be calculated
1280 using C<strlen(s)> (which means if you use this option, that C<s> can't have
1281 embedded C<NUL> characters and has to have a terminating C<NUL> byte). Note
1282 that all characters being ASCII constitute 'a valid UTF-8 string'.
1284 If C<flags> is 0, this gives the same results as C<L</is_utf8_string>>; if
1285 C<flags> is C<UTF8_DISALLOW_ILLEGAL_INTERCHANGE>, this gives the same results
1286 as C<L</is_strict_utf8_string>>; and if C<flags> is
1287 C<UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE>, this gives the same results as
1288 C<L</is_c9strict_utf8_string>>. Otherwise C<flags> may be any
1289 combination of the C<UTF8_DISALLOW_I<foo>> flags understood by
1290 C<L</utf8n_to_uvchr>>, with the same meanings.
1293 C<L</is_utf8_invariant_string>>,
1294 C<L</is_utf8_invariant_string_loc>>,
1295 C<L</is_utf8_string>>,
1296 C<L</is_utf8_string_loc>>,
1297 C<L</is_utf8_string_loc_flags>>,
1298 C<L</is_utf8_string_loclen>>,
1299 C<L</is_utf8_string_loclen_flags>>,
1300 C<L</is_utf8_fixed_width_buf_flags>>,
1301 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1302 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1303 C<L</is_strict_utf8_string>>,
1304 C<L</is_strict_utf8_string_loc>>,
1305 C<L</is_strict_utf8_string_loclen>>,
1306 C<L</is_c9strict_utf8_string>>,
1307 C<L</is_c9strict_utf8_string_loc>>,
1309 C<L</is_c9strict_utf8_string_loclen>>.
1314 PERL_STATIC_INLINE bool
1315 Perl_is_utf8_string_flags(const U8 *s, STRLEN len, const U32 flags)
1317 const U8 * first_variant;
1319 PERL_ARGS_ASSERT_IS_UTF8_STRING_FLAGS;
1320 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
1321 |UTF8_DISALLOW_PERL_EXTENDED)));
1324 len = strlen((const char *)s);
1328 return is_utf8_string(s, len);
1331 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1332 == UTF8_DISALLOW_ILLEGAL_INTERCHANGE)
1334 return is_strict_utf8_string(s, len);
1337 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1338 == UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE)
1340 return is_c9strict_utf8_string(s, len);
1343 if (! is_utf8_invariant_string_loc(s, len, &first_variant)) {
1344 const U8* const send = s + len;
1345 const U8* x = first_variant;
1348 STRLEN cur_len = isUTF8_CHAR_flags(x, send, flags);
1349 if (UNLIKELY(! cur_len)) {
1361 =for apidoc is_utf8_string_loc
1363 Like C<L</is_utf8_string>> but stores the location of the failure (in the
1364 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1365 "utf8ness success") in the C<ep> pointer.
1367 See also C<L</is_utf8_string_loclen>>.
1372 #define is_utf8_string_loc(s, len, ep) is_utf8_string_loclen(s, len, ep, 0)
1376 =for apidoc is_utf8_string_loclen
1378 Like C<L</is_utf8_string>> but stores the location of the failure (in the
1379 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1380 "utf8ness success") in the C<ep> pointer, and the number of UTF-8
1381 encoded characters in the C<el> pointer.
1383 See also C<L</is_utf8_string_loc>>.
1388 PERL_STATIC_INLINE bool
1389 Perl_is_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
1391 const U8 * first_variant;
1393 PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN;
1396 len = strlen((const char *) s);
1399 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1411 const U8* const send = s + len;
1412 const U8* x = first_variant;
1413 STRLEN outlen = first_variant - s;
1416 const STRLEN cur_len = isUTF8_CHAR(x, send);
1417 if (UNLIKELY(! cur_len)) {
1435 /* The perl core arranges to never call the DFA below without there being at
1436 * least one byte available to look at. This allows the DFA to use a do {}
1437 * while loop which means that calling it with a UTF-8 invariant has a single
1438 * conditional, same as the calling code checking for invariance ahead of time.
1439 * And having the calling code remove that conditional speeds up by that
1440 * conditional, the case where it wasn't invariant. So there's no reason to
1441 * check before caling this.
1443 * But we don't know this for non-core calls, so have to retain the check for
1446 # define PERL_NON_CORE_CHECK_EMPTY(s,e) assert((e) > (s))
1448 # define PERL_NON_CORE_CHECK_EMPTY(s,e) if ((e) <= (s)) return FALSE
1452 * DFA for checking input is valid UTF-8 syntax.
1454 * This uses adaptations of the table and algorithm given in
1455 * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1456 * documentation of the original version. A copyright notice for the original
1457 * version is given at the beginning of this file. The Perl adapations are
1458 * documented at the definition of PL_extended_utf8_dfa_tab[].
1460 * This dfa is fast. There are three exit conditions:
1461 * 1) a well-formed code point, acceptable to the table
1462 * 2) the beginning bytes of an incomplete character, whose completion might
1463 * or might not be acceptable
1464 * 3) unacceptable to the table. Some of the adaptations have certain,
1465 * hopefully less likely to occur, legal inputs be unacceptable to the
1466 * table, so these must be sorted out afterwards.
1468 * This macro is a complete implementation of the code executing the DFA. It
1469 * is passed the input sequence bounds and the table to use, and what to do
1470 * for each of the exit conditions. There are three canned actions, likely to
1471 * be the ones you want:
1472 * DFA_RETURN_SUCCESS_
1473 * DFA_RETURN_FAILURE_
1474 * DFA_GOTO_TEASE_APART_FF_
1476 * You pass a parameter giving the action to take for each of the three
1477 * possible exit conditions:
1479 * 'accept_action' This is executed when the DFA accepts the input.
1480 * DFA_RETURN_SUCCESS_ is the most likely candidate.
1481 * 'reject_action' This is executed when the DFA rejects the input.
1482 * DFA_RETURN_FAILURE_ is a candidate, or 'goto label' where
1483 * you have written code to distinguish the rejecting state
1484 * results. Because it happens in several places, and
1485 * involves #ifdefs, the special action
1486 * DFA_GOTO_TEASE_APART_FF_ is what you want with
1487 * PL_extended_utf8_dfa_tab. On platforms without
1488 * EXTRA_LONG_UTF8, there is no need to tease anything apart,
1489 * so this evaluates to DFA_RETURN_FAILURE_; otherwise you
1490 * need to have a label 'tease_apart_FF' that it will transfer
1492 * 'incomplete_char_action' This is executed when the DFA ran off the end
1493 * before accepting or rejecting the input.
1494 * DFA_RETURN_FAILURE_ is the likely action, but you could
1495 * have a 'goto', or NOOP. In the latter case the DFA drops
1496 * off the end, and you place your code to handle this case
1497 * immediately after it.
1500 #define DFA_RETURN_SUCCESS_ return s - s0
1501 #define DFA_RETURN_FAILURE_ return 0
1502 #ifdef HAS_EXTRA_LONG_UTF8
1503 # define DFA_TEASE_APART_FF_ goto tease_apart_FF
1505 # define DFA_TEASE_APART_FF_ DFA_RETURN_FAILURE_
1508 #define PERL_IS_UTF8_CHAR_DFA(s0, e, dfa_tab, \
1511 incomplete_char_action) \
1513 const U8 * s = s0; \
1514 const U8 * e_ = e; \
1517 PERL_NON_CORE_CHECK_EMPTY(s, e_); \
1520 state = dfa_tab[256 + state + dfa_tab[*s]]; \
1523 if (state == 0) { /* Accepting state */ \
1527 if (UNLIKELY(state == 1)) { /* Rejecting state */ \
1532 /* Here, dropped out of loop before end-of-char */ \
1533 incomplete_char_action; \
1539 =for apidoc isUTF8_CHAR
1541 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
1542 looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl,
1543 that represents some code point; otherwise it evaluates to 0. If non-zero, the
1544 value gives how many bytes starting at C<s> comprise the code point's
1545 representation. Any bytes remaining before C<e>, but beyond the ones needed to
1546 form the first code point in C<s>, are not examined.
1548 The code point can be any that will fit in an IV on this machine, using Perl's
1549 extension to official UTF-8 to represent those higher than the Unicode maximum
1550 of 0x10FFFF. That means that this macro is used to efficiently decide if the
1551 next few bytes in C<s> is legal UTF-8 for a single character.
1553 Use C<L</isSTRICT_UTF8_CHAR>> to restrict the acceptable code points to those
1554 defined by Unicode to be fully interchangeable across applications;
1555 C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum
1556 #9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable
1557 code points; and C<L</isUTF8_CHAR_flags>> for a more customized definition.
1559 Use C<L</is_utf8_string>>, C<L</is_utf8_string_loc>>, and
1560 C<L</is_utf8_string_loclen>> to check entire strings.
1562 Note also that a UTF-8 "invariant" character (i.e. ASCII on non-EBCDIC
1563 machines) is a valid UTF-8 character.
1567 This uses an adaptation of the table and algorithm given in
1568 https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1569 documentation of the original version. A copyright notice for the original
1570 version is given at the beginning of this file. The Perl adapation is
1571 documented at the definition of PL_extended_utf8_dfa_tab[].
1574 PERL_STATIC_INLINE Size_t
1575 Perl_isUTF8_CHAR(const U8 * const s0, const U8 * const e)
1577 PERL_ARGS_ASSERT_ISUTF8_CHAR;
1579 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab,
1580 DFA_RETURN_SUCCESS_,
1581 DFA_TEASE_APART_FF_,
1582 DFA_RETURN_FAILURE_);
1584 /* Here, we didn't return success, but dropped out of the loop. In the
1585 * case of PL_extended_utf8_dfa_tab, this means the input is either
1586 * malformed, or the start byte was FF on a platform that the dfa doesn't
1587 * handle FF's. Call a helper function. */
1589 #ifdef HAS_EXTRA_LONG_UTF8
1593 /* In the case of PL_extended_utf8_dfa_tab, getting here means the input is
1594 * either malformed, or was for the largest possible start byte, which we
1595 * now check, not inline */
1596 if (*s0 != I8_TO_NATIVE_UTF8(0xFF)) {
1600 return is_utf8_FF_helper_(s0, e,
1601 FALSE /* require full, not partial char */
1609 =for apidoc isSTRICT_UTF8_CHAR
1611 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
1612 looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some
1613 Unicode code point completely acceptable for open interchange between all
1614 applications; otherwise it evaluates to 0. If non-zero, the value gives how
1615 many bytes starting at C<s> comprise the code point's representation. Any
1616 bytes remaining before C<e>, but beyond the ones needed to form the first code
1617 point in C<s>, are not examined.
1619 The largest acceptable code point is the Unicode maximum 0x10FFFF, and must not
1620 be a surrogate nor a non-character code point. Thus this excludes any code
1621 point from Perl's extended UTF-8.
1623 This is used to efficiently decide if the next few bytes in C<s> is
1624 legal Unicode-acceptable UTF-8 for a single character.
1626 Use C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum
1627 #9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable
1628 code points; C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8;
1629 and C<L</isUTF8_CHAR_flags>> for a more customized definition.
1631 Use C<L</is_strict_utf8_string>>, C<L</is_strict_utf8_string_loc>>, and
1632 C<L</is_strict_utf8_string_loclen>> to check entire strings.
1636 This uses an adaptation of the tables and algorithm given in
1637 https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1638 documentation of the original version. A copyright notice for the original
1639 version is given at the beginning of this file. The Perl adapation is
1640 documented at the definition of strict_extended_utf8_dfa_tab[].
1644 PERL_STATIC_INLINE Size_t
1645 Perl_isSTRICT_UTF8_CHAR(const U8 * const s0, const U8 * const e)
1647 PERL_ARGS_ASSERT_ISSTRICT_UTF8_CHAR;
1649 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_strict_utf8_dfa_tab,
1650 DFA_RETURN_SUCCESS_,
1652 DFA_RETURN_FAILURE_);
1655 /* Here, we didn't return success, but dropped out of the loop. In the
1656 * case of PL_strict_utf8_dfa_tab, this means the input is either
1657 * malformed, or was for certain Hanguls; handle them specially */
1659 /* The dfa above drops out for incomplete or illegal inputs, and certain
1660 * legal Hanguls; check and return accordingly */
1661 return is_HANGUL_ED_utf8_safe(s0, e);
1666 =for apidoc isC9_STRICT_UTF8_CHAR
1668 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
1669 looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some
1670 Unicode non-surrogate code point; otherwise it evaluates to 0. If non-zero,
1671 the value gives how many bytes starting at C<s> comprise the code point's
1672 representation. Any bytes remaining before C<e>, but beyond the ones needed to
1673 form the first code point in C<s>, are not examined.
1675 The largest acceptable code point is the Unicode maximum 0x10FFFF. This
1676 differs from C<L</isSTRICT_UTF8_CHAR>> only in that it accepts non-character
1677 code points. This corresponds to
1678 L<Unicode Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>.
1679 which said that non-character code points are merely discouraged rather than
1680 completely forbidden in open interchange. See
1681 L<perlunicode/Noncharacter code points>.
1683 Use C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8; and
1684 C<L</isUTF8_CHAR_flags>> for a more customized definition.
1686 Use C<L</is_c9strict_utf8_string>>, C<L</is_c9strict_utf8_string_loc>>, and
1687 C<L</is_c9strict_utf8_string_loclen>> to check entire strings.
1691 This uses an adaptation of the tables and algorithm given in
1692 https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1693 documentation of the original version. A copyright notice for the original
1694 version is given at the beginning of this file. The Perl adapation is
1695 documented at the definition of PL_c9_utf8_dfa_tab[].
1699 PERL_STATIC_INLINE Size_t
1700 Perl_isC9_STRICT_UTF8_CHAR(const U8 * const s0, const U8 * const e)
1702 PERL_ARGS_ASSERT_ISC9_STRICT_UTF8_CHAR;
1704 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_c9_utf8_dfa_tab,
1705 DFA_RETURN_SUCCESS_,
1706 DFA_RETURN_FAILURE_,
1707 DFA_RETURN_FAILURE_);
1712 =for apidoc is_strict_utf8_string_loc
1714 Like C<L</is_strict_utf8_string>> but stores the location of the failure (in the
1715 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1716 "utf8ness success") in the C<ep> pointer.
1718 See also C<L</is_strict_utf8_string_loclen>>.
1723 #define is_strict_utf8_string_loc(s, len, ep) \
1724 is_strict_utf8_string_loclen(s, len, ep, 0)
1728 =for apidoc is_strict_utf8_string_loclen
1730 Like C<L</is_strict_utf8_string>> but stores the location of the failure (in the
1731 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1732 "utf8ness success") in the C<ep> pointer, and the number of UTF-8
1733 encoded characters in the C<el> pointer.
1735 See also C<L</is_strict_utf8_string_loc>>.
1740 PERL_STATIC_INLINE bool
1741 Perl_is_strict_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
1743 const U8 * first_variant;
1745 PERL_ARGS_ASSERT_IS_STRICT_UTF8_STRING_LOCLEN;
1748 len = strlen((const char *) s);
1751 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1763 const U8* const send = s + len;
1764 const U8* x = first_variant;
1765 STRLEN outlen = first_variant - s;
1768 const STRLEN cur_len = isSTRICT_UTF8_CHAR(x, send);
1769 if (UNLIKELY(! cur_len)) {
1789 =for apidoc is_c9strict_utf8_string_loc
1791 Like C<L</is_c9strict_utf8_string>> but stores the location of the failure (in
1792 the case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1793 "utf8ness success") in the C<ep> pointer.
1795 See also C<L</is_c9strict_utf8_string_loclen>>.
1800 #define is_c9strict_utf8_string_loc(s, len, ep) \
1801 is_c9strict_utf8_string_loclen(s, len, ep, 0)
1805 =for apidoc is_c9strict_utf8_string_loclen
1807 Like C<L</is_c9strict_utf8_string>> but stores the location of the failure (in
1808 the case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1809 "utf8ness success") in the C<ep> pointer, and the number of UTF-8 encoded
1810 characters in the C<el> pointer.
1812 See also C<L</is_c9strict_utf8_string_loc>>.
1817 PERL_STATIC_INLINE bool
1818 Perl_is_c9strict_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
1820 const U8 * first_variant;
1822 PERL_ARGS_ASSERT_IS_C9STRICT_UTF8_STRING_LOCLEN;
1825 len = strlen((const char *) s);
1828 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1840 const U8* const send = s + len;
1841 const U8* x = first_variant;
1842 STRLEN outlen = first_variant - s;
1845 const STRLEN cur_len = isC9_STRICT_UTF8_CHAR(x, send);
1846 if (UNLIKELY(! cur_len)) {
1866 =for apidoc is_utf8_string_loc_flags
1868 Like C<L</is_utf8_string_flags>> but stores the location of the failure (in the
1869 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1870 "utf8ness success") in the C<ep> pointer.
1872 See also C<L</is_utf8_string_loclen_flags>>.
1877 #define is_utf8_string_loc_flags(s, len, ep, flags) \
1878 is_utf8_string_loclen_flags(s, len, ep, 0, flags)
1881 /* The above 3 actual functions could have been moved into the more general one
1882 * just below, and made #defines that call it with the right 'flags'. They are
1883 * currently kept separate to increase their chances of getting inlined */
1887 =for apidoc is_utf8_string_loclen_flags
1889 Like C<L</is_utf8_string_flags>> but stores the location of the failure (in the
1890 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1891 "utf8ness success") in the C<ep> pointer, and the number of UTF-8
1892 encoded characters in the C<el> pointer.
1894 See also C<L</is_utf8_string_loc_flags>>.
1899 PERL_STATIC_INLINE bool
1900 Perl_is_utf8_string_loclen_flags(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el, const U32 flags)
1902 const U8 * first_variant;
1904 PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN_FLAGS;
1905 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
1906 |UTF8_DISALLOW_PERL_EXTENDED)));
1909 len = strlen((const char *) s);
1913 return is_utf8_string_loclen(s, len, ep, el);
1916 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1917 == UTF8_DISALLOW_ILLEGAL_INTERCHANGE)
1919 return is_strict_utf8_string_loclen(s, len, ep, el);
1922 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1923 == UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE)
1925 return is_c9strict_utf8_string_loclen(s, len, ep, el);
1928 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1940 const U8* send = s + len;
1941 const U8* x = first_variant;
1942 STRLEN outlen = first_variant - s;
1945 const STRLEN cur_len = isUTF8_CHAR_flags(x, send, flags);
1946 if (UNLIKELY(! cur_len)) {
1965 =for apidoc utf8_distance
1967 Returns the number of UTF-8 characters between the UTF-8 pointers C<a>
1970 WARNING: use only if you *know* that the pointers point inside the
1976 PERL_STATIC_INLINE IV
1977 Perl_utf8_distance(pTHX_ const U8 *a, const U8 *b)
1979 PERL_ARGS_ASSERT_UTF8_DISTANCE;
1981 return (a < b) ? -1 * (IV) utf8_length(a, b) : (IV) utf8_length(b, a);
1985 =for apidoc utf8_hop
1987 Return the UTF-8 pointer C<s> displaced by C<off> characters, either
1988 forward or backward.
1990 WARNING: do not use the following unless you *know* C<off> is within
1991 the UTF-8 data pointed to by C<s> *and* that on entry C<s> is aligned
1992 on the first byte of character or just after the last byte of a character.
1997 PERL_STATIC_INLINE U8 *
1998 Perl_utf8_hop(const U8 *s, SSize_t off)
2000 PERL_ARGS_ASSERT_UTF8_HOP;
2002 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2003 * the bitops (especially ~) can create illegal UTF-8.
2004 * In other words: in Perl UTF-8 is not just for Unicode. */
2013 while (UTF8_IS_CONTINUATION(*s))
2017 GCC_DIAG_IGNORE(-Wcast-qual)
2023 =for apidoc utf8_hop_forward
2025 Return the UTF-8 pointer C<s> displaced by up to C<off> characters,
2028 C<off> must be non-negative.
2030 C<s> must be before or equal to C<end>.
2032 When moving forward it will not move beyond C<end>.
2034 Will not exceed this limit even if the string is not valid "UTF-8".
2039 PERL_STATIC_INLINE U8 *
2040 Perl_utf8_hop_forward(const U8 *s, SSize_t off, const U8 *end)
2042 PERL_ARGS_ASSERT_UTF8_HOP_FORWARD;
2044 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2045 * the bitops (especially ~) can create illegal UTF-8.
2046 * In other words: in Perl UTF-8 is not just for Unicode. */
2052 STRLEN skip = UTF8SKIP(s);
2053 if ((STRLEN)(end - s) <= skip) {
2054 GCC_DIAG_IGNORE(-Wcast-qual)
2061 GCC_DIAG_IGNORE(-Wcast-qual)
2067 =for apidoc utf8_hop_back
2069 Return the UTF-8 pointer C<s> displaced by up to C<off> characters,
2072 C<off> must be non-positive.
2074 C<s> must be after or equal to C<start>.
2076 When moving backward it will not move before C<start>.
2078 Will not exceed this limit even if the string is not valid "UTF-8".
2083 PERL_STATIC_INLINE U8 *
2084 Perl_utf8_hop_back(const U8 *s, SSize_t off, const U8 *start)
2086 PERL_ARGS_ASSERT_UTF8_HOP_BACK;
2088 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2089 * the bitops (especially ~) can create illegal UTF-8.
2090 * In other words: in Perl UTF-8 is not just for Unicode. */
2095 /* Note: if we know that the input is well-formed, we can do per-word
2096 * hop-back. Commit d6ad3b72778369a84a215b498d8d60d5b03aa1af implemented
2097 * that. But it was reverted because doing per-word has some
2098 * start-up/tear-down overhead, so only makes sense if the distance to be
2099 * moved is large, and core perl doesn't currently move more than a few
2100 * characters at a time. You can reinstate it if it does become
2102 while (off++ && s > start) {
2105 } while (UTF8_IS_CONTINUATION(*s) && s > start);
2108 GCC_DIAG_IGNORE(-Wcast-qual)
2114 =for apidoc utf8_hop_safe
2116 Return the UTF-8 pointer C<s> displaced by up to C<off> characters,
2117 either forward or backward.
2119 When moving backward it will not move before C<start>.
2121 When moving forward it will not move beyond C<end>.
2123 Will not exceed those limits even if the string is not valid "UTF-8".
2128 PERL_STATIC_INLINE U8 *
2129 Perl_utf8_hop_safe(const U8 *s, SSize_t off, const U8 *start, const U8 *end)
2131 PERL_ARGS_ASSERT_UTF8_HOP_SAFE;
2133 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2134 * the bitops (especially ~) can create illegal UTF-8.
2135 * In other words: in Perl UTF-8 is not just for Unicode. */
2137 assert(start <= s && s <= end);
2140 return utf8_hop_forward(s, off, end);
2143 return utf8_hop_back(s, off, start);
2149 =for apidoc isUTF8_CHAR_flags
2151 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
2152 looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl,
2153 that represents some code point, subject to the restrictions given by C<flags>;
2154 otherwise it evaluates to 0. If non-zero, the value gives how many bytes
2155 starting at C<s> comprise the code point's representation. Any bytes remaining
2156 before C<e>, but beyond the ones needed to form the first code point in C<s>,
2159 If C<flags> is 0, this gives the same results as C<L</isUTF8_CHAR>>;
2160 if C<flags> is C<UTF8_DISALLOW_ILLEGAL_INTERCHANGE>, this gives the same results
2161 as C<L</isSTRICT_UTF8_CHAR>>;
2162 and if C<flags> is C<UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE>, this gives
2163 the same results as C<L</isC9_STRICT_UTF8_CHAR>>.
2164 Otherwise C<flags> may be any combination of the C<UTF8_DISALLOW_I<foo>> flags
2165 understood by C<L</utf8n_to_uvchr>>, with the same meanings.
2167 The three alternative macros are for the most commonly needed validations; they
2168 are likely to run somewhat faster than this more general one, as they can be
2169 inlined into your code.
2171 Use L</is_utf8_string_flags>, L</is_utf8_string_loc_flags>, and
2172 L</is_utf8_string_loclen_flags> to check entire strings.
2177 PERL_STATIC_INLINE STRLEN
2178 Perl_isUTF8_CHAR_flags(const U8 * const s0, const U8 * const e, const U32 flags)
2180 PERL_ARGS_ASSERT_ISUTF8_CHAR_FLAGS;
2181 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
2182 |UTF8_DISALLOW_PERL_EXTENDED)));
2184 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab,
2186 DFA_TEASE_APART_FF_,
2187 DFA_RETURN_FAILURE_);
2191 return is_utf8_char_helper_(s0, e, flags);
2193 #ifdef HAS_EXTRA_LONG_UTF8
2197 /* In the case of PL_extended_utf8_dfa_tab, getting here means the input is
2198 * either malformed, or was for the largest possible start byte, which
2199 * indicates perl extended UTF-8, well above the Unicode maximum */
2200 if ( *s0 != I8_TO_NATIVE_UTF8(0xFF)
2201 || (flags & (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_PERL_EXTENDED)))
2206 /* Otherwise examine the sequence not inline */
2207 return is_utf8_FF_helper_(s0, e,
2208 FALSE /* require full, not partial char */
2216 =for apidoc is_utf8_valid_partial_char
2218 Returns 0 if the sequence of bytes starting at C<s> and looking no further than
2219 S<C<e - 1>> is the UTF-8 encoding, as extended by Perl, for one or more code
2220 points. Otherwise, it returns 1 if there exists at least one non-empty
2221 sequence of bytes that when appended to sequence C<s>, starting at position
2222 C<e> causes the entire sequence to be the well-formed UTF-8 of some code point;
2223 otherwise returns 0.
2225 In other words this returns TRUE if C<s> points to a partial UTF-8-encoded code
2228 This is useful when a fixed-length buffer is being tested for being well-formed
2229 UTF-8, but the final few bytes in it don't comprise a full character; that is,
2230 it is split somewhere in the middle of the final code point's UTF-8
2231 representation. (Presumably when the buffer is refreshed with the next chunk
2232 of data, the new first bytes will complete the partial code point.) This
2233 function is used to verify that the final bytes in the current buffer are in
2234 fact the legal beginning of some code point, so that if they aren't, the
2235 failure can be signalled without having to wait for the next read.
2239 #define is_utf8_valid_partial_char(s, e) \
2240 is_utf8_valid_partial_char_flags(s, e, 0)
2244 =for apidoc is_utf8_valid_partial_char_flags
2246 Like C<L</is_utf8_valid_partial_char>>, it returns a boolean giving whether
2247 or not the input is a valid UTF-8 encoded partial character, but it takes an
2248 extra parameter, C<flags>, which can further restrict which code points are
2251 If C<flags> is 0, this behaves identically to
2252 C<L</is_utf8_valid_partial_char>>. Otherwise C<flags> can be any combination
2253 of the C<UTF8_DISALLOW_I<foo>> flags accepted by C<L</utf8n_to_uvchr>>. If
2254 there is any sequence of bytes that can complete the input partial character in
2255 such a way that a non-prohibited character is formed, the function returns
2256 TRUE; otherwise FALSE. Non character code points cannot be determined based on
2257 partial character input. But many of the other possible excluded types can be
2258 determined from just the first one or two bytes.
2263 PERL_STATIC_INLINE bool
2264 Perl_is_utf8_valid_partial_char_flags(const U8 * const s0, const U8 * const e, const U32 flags)
2266 PERL_ARGS_ASSERT_IS_UTF8_VALID_PARTIAL_CHAR_FLAGS;
2267 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
2268 |UTF8_DISALLOW_PERL_EXTENDED)));
2270 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab,
2271 DFA_RETURN_FAILURE_,
2272 DFA_TEASE_APART_FF_,
2275 /* The NOOP above causes the DFA to drop down here iff the input was a
2276 * partial character. flags=0 => can return TRUE immediately; otherwise we
2277 * need to check (not inline) if the partial character is the beginning of
2278 * a disallowed one */
2283 return cBOOL(is_utf8_char_helper_(s0, e, flags));
2285 #ifdef HAS_EXTRA_LONG_UTF8
2289 /* Getting here means the input is either malformed, or, in the case of
2290 * PL_extended_utf8_dfa_tab, was for the largest possible start byte. The
2291 * latter case has to be extended UTF-8, so can fail immediately if that is
2294 if ( *s0 != I8_TO_NATIVE_UTF8(0xFF)
2295 || (flags & (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_PERL_EXTENDED)))
2300 return is_utf8_FF_helper_(s0, e,
2301 TRUE /* Require to be a partial character */
2309 =for apidoc is_utf8_fixed_width_buf_flags
2311 Returns TRUE if the fixed-width buffer starting at C<s> with length C<len>
2312 is entirely valid UTF-8, subject to the restrictions given by C<flags>;
2313 otherwise it returns FALSE.
2315 If C<flags> is 0, any well-formed UTF-8, as extended by Perl, is accepted
2316 without restriction. If the final few bytes of the buffer do not form a
2317 complete code point, this will return TRUE anyway, provided that
2318 C<L</is_utf8_valid_partial_char_flags>> returns TRUE for them.
2320 If C<flags> in non-zero, it can be any combination of the
2321 C<UTF8_DISALLOW_I<foo>> flags accepted by C<L</utf8n_to_uvchr>>, and with the
2324 This function differs from C<L</is_utf8_string_flags>> only in that the latter
2325 returns FALSE if the final few bytes of the string don't form a complete code
2330 #define is_utf8_fixed_width_buf_flags(s, len, flags) \
2331 is_utf8_fixed_width_buf_loclen_flags(s, len, 0, 0, flags)
2335 =for apidoc is_utf8_fixed_width_buf_loc_flags
2337 Like C<L</is_utf8_fixed_width_buf_flags>> but stores the location of the
2338 failure in the C<ep> pointer. If the function returns TRUE, C<*ep> will point
2339 to the beginning of any partial character at the end of the buffer; if there is
2340 no partial character C<*ep> will contain C<s>+C<len>.
2342 See also C<L</is_utf8_fixed_width_buf_loclen_flags>>.
2347 #define is_utf8_fixed_width_buf_loc_flags(s, len, loc, flags) \
2348 is_utf8_fixed_width_buf_loclen_flags(s, len, loc, 0, flags)
2352 =for apidoc is_utf8_fixed_width_buf_loclen_flags
2354 Like C<L</is_utf8_fixed_width_buf_loc_flags>> but stores the number of
2355 complete, valid characters found in the C<el> pointer.
2360 PERL_STATIC_INLINE bool
2361 Perl_is_utf8_fixed_width_buf_loclen_flags(const U8 * const s,
2367 const U8 * maybe_partial;
2369 PERL_ARGS_ASSERT_IS_UTF8_FIXED_WIDTH_BUF_LOCLEN_FLAGS;
2372 ep = &maybe_partial;
2375 /* If it's entirely valid, return that; otherwise see if the only error is
2376 * that the final few bytes are for a partial character */
2377 return is_utf8_string_loclen_flags(s, len, ep, el, flags)
2378 || is_utf8_valid_partial_char_flags(*ep, s + len, flags);
2381 PERL_STATIC_INLINE UV
2382 Perl_utf8n_to_uvchr_msgs(const U8 *s,
2389 /* This is the inlined portion of utf8n_to_uvchr_msgs. It handles the
2390 * simple cases, and, if necessary calls a helper function to deal with the
2391 * more complex ones. Almost all well-formed non-problematic code points
2392 * are considered simple, so that it's unlikely that the helper function
2393 * will need to be called.
2395 * This is an adaptation of the tables and algorithm given in
2396 * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides
2397 * comprehensive documentation of the original version. A copyright notice
2398 * for the original version is given at the beginning of this file. The
2399 * Perl adapation is documented at the definition of PL_strict_utf8_dfa_tab[].
2402 const U8 * const s0 = s;
2403 const U8 * send = s0 + curlen;
2407 PERL_ARGS_ASSERT_UTF8N_TO_UVCHR_MSGS;
2409 /* This dfa is fast. If it accepts the input, it was for a well-formed,
2410 * non-problematic code point, which can be returned immediately.
2411 * Otherwise we call a helper function to figure out the more complicated
2414 /* No calls from core pass in an empty string; non-core need a check */
2418 if (curlen == 0) return _utf8n_to_uvchr_msgs_helper(s0, 0, retlen,
2419 flags, errors, msgs);
2422 type = PL_strict_utf8_dfa_tab[*s];
2424 /* The table is structured so that 'type' is 0 iff the input byte is
2425 * represented identically regardless of the UTF-8ness of the string */
2426 if (type == 0) { /* UTF-8 invariants are returned unchanged */
2430 UV state = PL_strict_utf8_dfa_tab[256 + type];
2431 uv = (0xff >> type) & NATIVE_UTF8_TO_I8(*s);
2433 while (++s < send) {
2434 type = PL_strict_utf8_dfa_tab[*s];
2435 state = PL_strict_utf8_dfa_tab[256 + state + type];
2437 uv = UTF8_ACCUMULATE(uv, *s);
2441 uv = UNI_TO_NATIVE(uv);
2446 if (UNLIKELY(state == 1)) {
2451 /* Here is potentially problematic. Use the full mechanism */
2452 return _utf8n_to_uvchr_msgs_helper(s0, curlen, retlen, flags,
2458 *retlen = s - s0 + 1;
2470 PERL_STATIC_INLINE UV
2471 Perl_utf8_to_uvchr_buf_helper(pTHX_ const U8 *s, const U8 *send, STRLEN *retlen)
2473 PERL_ARGS_ASSERT_UTF8_TO_UVCHR_BUF_HELPER;
2477 if (! ckWARN_d(WARN_UTF8)) {
2479 /* EMPTY is not really allowed, and asserts on debugging builds. But
2480 * on non-debugging we have to deal with it, and this causes it to
2481 * return the REPLACEMENT CHARACTER, as the documentation indicates */
2482 return utf8n_to_uvchr(s, send - s, retlen,
2483 (UTF8_ALLOW_ANY | UTF8_ALLOW_EMPTY));
2486 UV ret = utf8n_to_uvchr(s, send - s, retlen, 0);
2487 if (retlen && ret == 0 && (send <= s || *s != '\0')) {
2488 *retlen = (STRLEN) -1;
2495 /* ------------------------------- perl.h ----------------------------- */
2498 =for apidoc_section $utility
2500 =for apidoc is_safe_syscall
2502 Test that the given C<pv> (with length C<len>) doesn't contain any internal
2504 If it does, set C<errno> to C<ENOENT>, optionally warn using the C<syscalls>
2505 category, and return FALSE.
2507 Return TRUE if the name is safe.
2509 C<what> and C<op_name> are used in any warning.
2511 Used by the C<IS_SAFE_SYSCALL()> macro.
2516 PERL_STATIC_INLINE bool
2517 Perl_is_safe_syscall(pTHX_ const char *pv, STRLEN len, const char *what, const char *op_name)
2519 /* While the Windows CE API provides only UCS-16 (or UTF-16) APIs
2520 * perl itself uses xce*() functions which accept 8-bit strings.
2523 PERL_ARGS_ASSERT_IS_SAFE_SYSCALL;
2527 if (UNLIKELY((null_at = (char *)memchr(pv, 0, len-1)) != NULL)) {
2528 SETERRNO(ENOENT, LIB_INVARG);
2529 Perl_ck_warner(aTHX_ packWARN(WARN_SYSCALLS),
2530 "Invalid \\0 character in %s for %s: %s\\0%s",
2531 what, op_name, pv, null_at+1);
2541 Return true if the supplied filename has a newline character
2542 immediately before the first (hopefully only) NUL.
2544 My original look at this incorrectly used the len from SvPV(), but
2545 that's incorrect, since we allow for a NUL in pv[len-1].
2547 So instead, strlen() and work from there.
2549 This allow for the user reading a filename, forgetting to chomp it,
2552 open my $foo, "$file\0";
2558 PERL_STATIC_INLINE bool
2559 S_should_warn_nl(const char *pv)
2563 PERL_ARGS_ASSERT_SHOULD_WARN_NL;
2567 return len > 0 && pv[len-1] == '\n';
2572 #if defined(PERL_IN_PP_C) || defined(PERL_IN_PP_HOT_C)
2574 PERL_STATIC_INLINE bool
2575 S_lossless_NV_to_IV(const NV nv, IV *ivp)
2577 /* This function determines if the input NV 'nv' may be converted without
2578 * loss of data to an IV. If not, it returns FALSE taking no other action.
2579 * But if it is possible, it does the conversion, returning TRUE, and
2580 * storing the converted result in '*ivp' */
2582 PERL_ARGS_ASSERT_LOSSLESS_NV_TO_IV;
2584 # if defined(NAN_COMPARE_BROKEN) && defined(Perl_isnan)
2585 /* Normally any comparison with a NaN returns false; if we can't rely
2586 * on that behaviour, check explicitly */
2587 if (UNLIKELY(Perl_isnan(nv))) {
2592 /* Written this way so that with an always-false NaN comparison we
2594 if (!(LIKELY(nv >= (NV) IV_MIN) && LIKELY(nv < IV_MAX_P1))) {
2598 if ((IV) nv != nv) {
2608 /* ------------------ pp.c, regcomp.c, toke.c, universal.c ------------ */
2610 #if defined(PERL_IN_PP_C) || defined(PERL_IN_REGCOMP_C) || defined(PERL_IN_TOKE_C) || defined(PERL_IN_UNIVERSAL_C)
2612 #define MAX_CHARSET_NAME_LENGTH 2
2614 PERL_STATIC_INLINE const char *
2615 S_get_regex_charset_name(const U32 flags, STRLEN* const lenp)
2617 PERL_ARGS_ASSERT_GET_REGEX_CHARSET_NAME;
2619 /* Returns a string that corresponds to the name of the regex character set
2620 * given by 'flags', and *lenp is set the length of that string, which
2621 * cannot exceed MAX_CHARSET_NAME_LENGTH characters */
2624 switch (get_regex_charset(flags)) {
2625 case REGEX_DEPENDS_CHARSET: return DEPENDS_PAT_MODS;
2626 case REGEX_LOCALE_CHARSET: return LOCALE_PAT_MODS;
2627 case REGEX_UNICODE_CHARSET: return UNICODE_PAT_MODS;
2628 case REGEX_ASCII_RESTRICTED_CHARSET: return ASCII_RESTRICT_PAT_MODS;
2629 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
2631 return ASCII_MORE_RESTRICT_PAT_MODS;
2633 /* The NOT_REACHED; hides an assert() which has a rather complex
2634 * definition in perl.h. */
2635 NOT_REACHED; /* NOTREACHED */
2636 return "?"; /* Unknown */
2643 Return false if any get magic is on the SV other than taint magic.
2647 PERL_STATIC_INLINE bool
2648 Perl_sv_only_taint_gmagic(SV *sv)
2650 MAGIC *mg = SvMAGIC(sv);
2652 PERL_ARGS_ASSERT_SV_ONLY_TAINT_GMAGIC;
2655 if (mg->mg_type != PERL_MAGIC_taint
2656 && !(mg->mg_flags & MGf_GSKIP)
2657 && mg->mg_virtual->svt_get) {
2660 mg = mg->mg_moremagic;
2666 /* ------------------ cop.h ------------------------------------------- */
2668 /* implement GIMME_V() macro */
2670 PERL_STATIC_INLINE U8
2674 U8 gimme = (PL_op->op_flags & OPf_WANT);
2678 cxix = PL_curstackinfo->si_cxsubix;
2680 return PL_curstackinfo->si_type == PERLSI_SORT ? G_SCALAR: G_VOID;
2681 assert(cxstack[cxix].blk_gimme & G_WANT);
2682 return (cxstack[cxix].blk_gimme & G_WANT);
2686 /* Enter a block. Push a new base context and return its address. */
2688 PERL_STATIC_INLINE PERL_CONTEXT *
2689 Perl_cx_pushblock(pTHX_ U8 type, U8 gimme, SV** sp, I32 saveix)
2693 PERL_ARGS_ASSERT_CX_PUSHBLOCK;
2698 cx->blk_gimme = gimme;
2699 cx->blk_oldsaveix = saveix;
2700 cx->blk_oldsp = (I32)(sp - PL_stack_base);
2701 cx->blk_oldcop = PL_curcop;
2702 cx->blk_oldmarksp = (I32)(PL_markstack_ptr - PL_markstack);
2703 cx->blk_oldscopesp = PL_scopestack_ix;
2704 cx->blk_oldpm = PL_curpm;
2705 cx->blk_old_tmpsfloor = PL_tmps_floor;
2707 PL_tmps_floor = PL_tmps_ix;
2708 CX_DEBUG(cx, "PUSH");
2713 /* Exit a block (RETURN and LAST). */
2715 PERL_STATIC_INLINE void
2716 Perl_cx_popblock(pTHX_ PERL_CONTEXT *cx)
2718 PERL_ARGS_ASSERT_CX_POPBLOCK;
2720 CX_DEBUG(cx, "POP");
2721 /* these 3 are common to cx_popblock and cx_topblock */
2722 PL_markstack_ptr = PL_markstack + cx->blk_oldmarksp;
2723 PL_scopestack_ix = cx->blk_oldscopesp;
2724 PL_curpm = cx->blk_oldpm;
2726 /* LEAVE_SCOPE() should have made this true. /(?{})/ cheats
2727 * and leaves a CX entry lying around for repeated use, so
2728 * skip for multicall */ \
2729 assert( (CxTYPE(cx) == CXt_SUB && CxMULTICALL(cx))
2730 || PL_savestack_ix == cx->blk_oldsaveix);
2731 PL_curcop = cx->blk_oldcop;
2732 PL_tmps_floor = cx->blk_old_tmpsfloor;
2735 /* Continue a block elsewhere (e.g. NEXT, REDO, GOTO).
2736 * Whereas cx_popblock() restores the state to the point just before
2737 * cx_pushblock() was called, cx_topblock() restores it to the point just
2738 * *after* cx_pushblock() was called. */
2740 PERL_STATIC_INLINE void
2741 Perl_cx_topblock(pTHX_ PERL_CONTEXT *cx)
2743 PERL_ARGS_ASSERT_CX_TOPBLOCK;
2745 CX_DEBUG(cx, "TOP");
2746 /* these 3 are common to cx_popblock and cx_topblock */
2747 PL_markstack_ptr = PL_markstack + cx->blk_oldmarksp;
2748 PL_scopestack_ix = cx->blk_oldscopesp;
2749 PL_curpm = cx->blk_oldpm;
2751 PL_stack_sp = PL_stack_base + cx->blk_oldsp;
2755 PERL_STATIC_INLINE void
2756 Perl_cx_pushsub(pTHX_ PERL_CONTEXT *cx, CV *cv, OP *retop, bool hasargs)
2758 U8 phlags = CX_PUSHSUB_GET_LVALUE_MASK(Perl_was_lvalue_sub);
2760 PERL_ARGS_ASSERT_CX_PUSHSUB;
2762 PERL_DTRACE_PROBE_ENTRY(cv);
2763 cx->blk_sub.old_cxsubix = PL_curstackinfo->si_cxsubix;
2764 PL_curstackinfo->si_cxsubix = cx - PL_curstackinfo->si_cxstack;
2765 cx->blk_sub.cv = cv;
2766 cx->blk_sub.olddepth = CvDEPTH(cv);
2767 cx->blk_sub.prevcomppad = PL_comppad;
2768 cx->cx_type |= (hasargs) ? CXp_HASARGS : 0;
2769 cx->blk_sub.retop = retop;
2770 SvREFCNT_inc_simple_void_NN(cv);
2771 cx->blk_u16 = PL_op->op_private & (phlags|OPpDEREF);
2775 /* subsets of cx_popsub() */
2777 PERL_STATIC_INLINE void
2778 Perl_cx_popsub_common(pTHX_ PERL_CONTEXT *cx)
2782 PERL_ARGS_ASSERT_CX_POPSUB_COMMON;
2783 assert(CxTYPE(cx) == CXt_SUB);
2785 PL_comppad = cx->blk_sub.prevcomppad;
2786 PL_curpad = LIKELY(PL_comppad) ? AvARRAY(PL_comppad) : NULL;
2787 cv = cx->blk_sub.cv;
2788 CvDEPTH(cv) = cx->blk_sub.olddepth;
2789 cx->blk_sub.cv = NULL;
2791 PL_curstackinfo->si_cxsubix = cx->blk_sub.old_cxsubix;
2795 /* handle the @_ part of leaving a sub */
2797 PERL_STATIC_INLINE void
2798 Perl_cx_popsub_args(pTHX_ PERL_CONTEXT *cx)
2802 PERL_ARGS_ASSERT_CX_POPSUB_ARGS;
2803 assert(CxTYPE(cx) == CXt_SUB);
2804 assert(AvARRAY(MUTABLE_AV(
2805 PadlistARRAY(CvPADLIST(cx->blk_sub.cv))[
2806 CvDEPTH(cx->blk_sub.cv)])) == PL_curpad);
2808 CX_POP_SAVEARRAY(cx);
2809 av = MUTABLE_AV(PAD_SVl(0));
2810 if (UNLIKELY(AvREAL(av)))
2811 /* abandon @_ if it got reified */
2812 clear_defarray(av, 0);
2819 PERL_STATIC_INLINE void
2820 Perl_cx_popsub(pTHX_ PERL_CONTEXT *cx)
2822 PERL_ARGS_ASSERT_CX_POPSUB;
2823 assert(CxTYPE(cx) == CXt_SUB);
2825 PERL_DTRACE_PROBE_RETURN(cx->blk_sub.cv);
2829 cx_popsub_common(cx);
2833 PERL_STATIC_INLINE void
2834 Perl_cx_pushformat(pTHX_ PERL_CONTEXT *cx, CV *cv, OP *retop, GV *gv)
2836 PERL_ARGS_ASSERT_CX_PUSHFORMAT;
2838 cx->blk_format.old_cxsubix = PL_curstackinfo->si_cxsubix;
2839 PL_curstackinfo->si_cxsubix= cx - PL_curstackinfo->si_cxstack;
2840 cx->blk_format.cv = cv;
2841 cx->blk_format.retop = retop;
2842 cx->blk_format.gv = gv;
2843 cx->blk_format.dfoutgv = PL_defoutgv;
2844 cx->blk_format.prevcomppad = PL_comppad;
2847 SvREFCNT_inc_simple_void_NN(cv);
2849 SvREFCNT_inc_void(cx->blk_format.dfoutgv);
2853 PERL_STATIC_INLINE void
2854 Perl_cx_popformat(pTHX_ PERL_CONTEXT *cx)
2859 PERL_ARGS_ASSERT_CX_POPFORMAT;
2860 assert(CxTYPE(cx) == CXt_FORMAT);
2862 dfout = cx->blk_format.dfoutgv;
2864 cx->blk_format.dfoutgv = NULL;
2865 SvREFCNT_dec_NN(dfout);
2867 PL_comppad = cx->blk_format.prevcomppad;
2868 PL_curpad = LIKELY(PL_comppad) ? AvARRAY(PL_comppad) : NULL;
2869 cv = cx->blk_format.cv;
2870 cx->blk_format.cv = NULL;
2872 SvREFCNT_dec_NN(cv);
2873 PL_curstackinfo->si_cxsubix = cx->blk_format.old_cxsubix;
2877 PERL_STATIC_INLINE void
2878 Perl_push_evalortry_common(pTHX_ PERL_CONTEXT *cx, OP *retop, SV *namesv)
2880 cx->blk_eval.retop = retop;
2881 cx->blk_eval.old_namesv = namesv;
2882 cx->blk_eval.old_eval_root = PL_eval_root;
2883 cx->blk_eval.cur_text = PL_parser ? PL_parser->linestr : NULL;
2884 cx->blk_eval.cv = NULL; /* later set by doeval_compile() */
2885 cx->blk_eval.cur_top_env = PL_top_env;
2887 assert(!(PL_in_eval & ~ 0x3F));
2888 assert(!(PL_op->op_type & ~0x1FF));
2889 cx->blk_u16 = (PL_in_eval & 0x3F) | ((U16)PL_op->op_type << 7);
2892 PERL_STATIC_INLINE void
2893 Perl_cx_pusheval(pTHX_ PERL_CONTEXT *cx, OP *retop, SV *namesv)
2895 PERL_ARGS_ASSERT_CX_PUSHEVAL;
2897 Perl_push_evalortry_common(aTHX_ cx, retop, namesv);
2899 cx->blk_eval.old_cxsubix = PL_curstackinfo->si_cxsubix;
2900 PL_curstackinfo->si_cxsubix = cx - PL_curstackinfo->si_cxstack;
2903 PERL_STATIC_INLINE void
2904 Perl_cx_pushtry(pTHX_ PERL_CONTEXT *cx, OP *retop)
2906 PERL_ARGS_ASSERT_CX_PUSHTRY;
2908 Perl_push_evalortry_common(aTHX_ cx, retop, NULL);
2910 /* Don't actually change it, just store the current value so it's restored
2911 * by the common popeval */
2912 cx->blk_eval.old_cxsubix = PL_curstackinfo->si_cxsubix;
2916 PERL_STATIC_INLINE void
2917 Perl_cx_popeval(pTHX_ PERL_CONTEXT *cx)
2921 PERL_ARGS_ASSERT_CX_POPEVAL;
2922 assert(CxTYPE(cx) == CXt_EVAL);
2924 PL_in_eval = CxOLD_IN_EVAL(cx);
2925 assert(!(PL_in_eval & 0xc0));
2926 PL_eval_root = cx->blk_eval.old_eval_root;
2927 sv = cx->blk_eval.cur_text;
2928 if (sv && CxEVAL_TXT_REFCNTED(cx)) {
2929 cx->blk_eval.cur_text = NULL;
2930 SvREFCNT_dec_NN(sv);
2933 sv = cx->blk_eval.old_namesv;
2935 cx->blk_eval.old_namesv = NULL;
2936 SvREFCNT_dec_NN(sv);
2938 PL_curstackinfo->si_cxsubix = cx->blk_eval.old_cxsubix;
2942 /* push a plain loop, i.e.
2944 * while (cond) { block }
2945 * for (init;cond;continue) { block }
2946 * This loop can be last/redo'ed etc.
2949 PERL_STATIC_INLINE void
2950 Perl_cx_pushloop_plain(pTHX_ PERL_CONTEXT *cx)
2952 PERL_ARGS_ASSERT_CX_PUSHLOOP_PLAIN;
2953 cx->blk_loop.my_op = cLOOP;
2957 /* push a true for loop, i.e.
2958 * for var (list) { block }
2961 PERL_STATIC_INLINE void
2962 Perl_cx_pushloop_for(pTHX_ PERL_CONTEXT *cx, void *itervarp, SV* itersave)
2964 PERL_ARGS_ASSERT_CX_PUSHLOOP_FOR;
2966 /* this one line is common with cx_pushloop_plain */
2967 cx->blk_loop.my_op = cLOOP;
2969 cx->blk_loop.itervar_u.svp = (SV**)itervarp;
2970 cx->blk_loop.itersave = itersave;
2972 cx->blk_loop.oldcomppad = PL_comppad;
2977 /* pop all loop types, including plain */
2979 PERL_STATIC_INLINE void
2980 Perl_cx_poploop(pTHX_ PERL_CONTEXT *cx)
2982 PERL_ARGS_ASSERT_CX_POPLOOP;
2984 assert(CxTYPE_is_LOOP(cx));
2985 if ( CxTYPE(cx) == CXt_LOOP_ARY
2986 || CxTYPE(cx) == CXt_LOOP_LAZYSV)
2988 /* Free ary or cur. This assumes that state_u.ary.ary
2989 * aligns with state_u.lazysv.cur. See cx_dup() */
2990 SV *sv = cx->blk_loop.state_u.lazysv.cur;
2991 cx->blk_loop.state_u.lazysv.cur = NULL;
2992 SvREFCNT_dec_NN(sv);
2993 if (CxTYPE(cx) == CXt_LOOP_LAZYSV) {
2994 sv = cx->blk_loop.state_u.lazysv.end;
2995 cx->blk_loop.state_u.lazysv.end = NULL;
2996 SvREFCNT_dec_NN(sv);
2999 if (cx->cx_type & (CXp_FOR_PAD|CXp_FOR_GV)) {
3001 SV **svp = (cx)->blk_loop.itervar_u.svp;
3002 if ((cx->cx_type & CXp_FOR_GV))
3003 svp = &GvSV((GV*)svp);
3005 *svp = cx->blk_loop.itersave;
3006 cx->blk_loop.itersave = NULL;
3007 SvREFCNT_dec(cursv);
3012 PERL_STATIC_INLINE void
3013 Perl_cx_pushwhen(pTHX_ PERL_CONTEXT *cx)
3015 PERL_ARGS_ASSERT_CX_PUSHWHEN;
3017 cx->blk_givwhen.leave_op = cLOGOP->op_other;
3021 PERL_STATIC_INLINE void
3022 Perl_cx_popwhen(pTHX_ PERL_CONTEXT *cx)
3024 PERL_ARGS_ASSERT_CX_POPWHEN;
3025 assert(CxTYPE(cx) == CXt_WHEN);
3027 PERL_UNUSED_ARG(cx);
3028 PERL_UNUSED_CONTEXT;
3029 /* currently NOOP */
3033 PERL_STATIC_INLINE void
3034 Perl_cx_pushgiven(pTHX_ PERL_CONTEXT *cx, SV *orig_defsv)
3036 PERL_ARGS_ASSERT_CX_PUSHGIVEN;
3038 cx->blk_givwhen.leave_op = cLOGOP->op_other;
3039 cx->blk_givwhen.defsv_save = orig_defsv;
3043 PERL_STATIC_INLINE void
3044 Perl_cx_popgiven(pTHX_ PERL_CONTEXT *cx)
3048 PERL_ARGS_ASSERT_CX_POPGIVEN;
3049 assert(CxTYPE(cx) == CXt_GIVEN);
3051 sv = GvSV(PL_defgv);
3052 GvSV(PL_defgv) = cx->blk_givwhen.defsv_save;
3053 cx->blk_givwhen.defsv_save = NULL;
3057 /* ------------------ util.h ------------------------------------------- */
3060 =for apidoc_section $string
3064 Returns true if the leading C<len> bytes of the strings C<s1> and C<s2> are the
3066 case-insensitively; false otherwise. Uppercase and lowercase ASCII range bytes
3067 match themselves and their opposite case counterparts. Non-cased and non-ASCII
3068 range bytes match only themselves.
3073 PERL_STATIC_INLINE I32
3074 Perl_foldEQ(pTHX_ const char *s1, const char *s2, I32 len)
3076 const U8 *a = (const U8 *)s1;
3077 const U8 *b = (const U8 *)s2;
3079 PERL_ARGS_ASSERT_FOLDEQ;
3084 if (*a != *b && *a != PL_fold[*b])
3091 PERL_STATIC_INLINE I32
3092 Perl_foldEQ_latin1(pTHX_ const char *s1, const char *s2, I32 len)
3094 /* Compare non-UTF-8 using Unicode (Latin1) semantics. Works on all folds
3095 * representable without UTF-8, except for LATIN_SMALL_LETTER_SHARP_S, and
3096 * does not check for this. Nor does it check that the strings each have
3097 * at least 'len' characters. */
3099 const U8 *a = (const U8 *)s1;
3100 const U8 *b = (const U8 *)s2;
3102 PERL_ARGS_ASSERT_FOLDEQ_LATIN1;
3107 if (*a != *b && *a != PL_fold_latin1[*b]) {
3116 =for apidoc_section $locale
3117 =for apidoc foldEQ_locale
3119 Returns true if the leading C<len> bytes of the strings C<s1> and C<s2> are the
3120 same case-insensitively in the current locale; false otherwise.
3125 PERL_STATIC_INLINE I32
3126 Perl_foldEQ_locale(pTHX_ const char *s1, const char *s2, I32 len)
3128 const U8 *a = (const U8 *)s1;
3129 const U8 *b = (const U8 *)s2;
3131 PERL_ARGS_ASSERT_FOLDEQ_LOCALE;
3136 if (*a != *b && *a != PL_fold_locale[*b])
3144 =for apidoc_section $string
3145 =for apidoc my_strnlen
3147 The C library C<strnlen> if available, or a Perl implementation of it.
3149 C<my_strnlen()> computes the length of the string, up to C<maxlen>
3150 characters. It will never attempt to address more than C<maxlen>
3151 characters, making it suitable for use with strings that are not
3152 guaranteed to be NUL-terminated.
3156 Description stolen from http://man.openbsd.org/strnlen.3,
3157 implementation stolen from PostgreSQL.
3161 PERL_STATIC_INLINE Size_t
3162 Perl_my_strnlen(const char *str, Size_t maxlen)
3164 const char *end = (char *) memchr(str, '\0', maxlen);
3166 PERL_ARGS_ASSERT_MY_STRNLEN;
3168 if (end == NULL) return maxlen;
3174 #if ! defined (HAS_MEMRCHR) && (defined(PERL_CORE) || defined(PERL_EXT))
3176 PERL_STATIC_INLINE void *
3177 S_my_memrchr(const char * s, const char c, const STRLEN len)
3179 /* memrchr(), since many platforms lack it */
3181 const char * t = s + len - 1;
3183 PERL_ARGS_ASSERT_MY_MEMRCHR;
3197 PERL_STATIC_INLINE char *
3198 Perl_mortal_getenv(const char * str)
3200 /* This implements a (mostly) thread-safe, sequential-call-safe getenv().
3202 * It's (mostly) thread-safe because it uses a mutex to prevent other
3203 * threads (that look at this mutex) from destroying the result before this
3204 * routine has a chance to copy the result to a place that won't be
3205 * destroyed before the caller gets a chance to handle it. That place is a
3206 * mortal SV. khw chose this over SAVEFREEPV because he is under the
3207 * impression that the SV will hang around longer under more circumstances
3209 * The reason it isn't completely thread-safe is that other code could
3210 * simply not pay attention to the mutex. All of the Perl core uses the
3211 * mutex, but it is possible for code from, say XS, to not use this mutex,
3212 * defeating the safety.
3214 * getenv() returns, in some implementations, a pointer to a spot in the
3215 * **environ array, which could be invalidated at any time by this or
3216 * another thread changing the environment. Other implementations copy the
3217 * **environ value to a static buffer, returning a pointer to that. That
3218 * buffer might or might not be invalidated by a getenv() call in another
3219 * thread. If it does get zapped, we need an exclusive lock. Otherwise,
3220 * many getenv() calls can safely be running simultaneously, so a
3221 * many-reader (but no simultaneous writers) lock is ok. There is a
3222 * Configure probe to see if another thread destroys the buffer, and the
3223 * mutex is defined accordingly.
3225 * But in all cases, using the mutex prevents these problems, as long as
3226 * all code uses the same mutex.
3228 * A complication is that this can be called during phases where the
3229 * mortalization process isn't available. These are in interpreter
3230 * destruction or early in construction. khw believes that at these times
3231 * there shouldn't be anything else going on, so plain getenv is safe AS
3232 * LONG AS the caller acts on the return before calling it again. */
3237 PERL_ARGS_ASSERT_MORTAL_GETENV;
3239 /* Can't mortalize without stacks. khw believes that no other threads
3240 * should be running, so no need to lock things, and this may be during a
3241 * phase when locking isn't even available */
3242 if (UNLIKELY(PL_scopestack_ix == 0)) {
3248 /* A major complication arises under PERL_MEM_LOG. When that is active,
3249 * every memory allocation may result in logging, depending on the value of
3250 * ENV{PERL_MEM_LOG} at the moment. That means, as we create the SV for
3251 * saving ENV{foo}'s value (but before saving it), the logging code will
3252 * call us recursively to find out what ENV{PERL_MEM_LOG} is. Without some
3253 * care that could lead to: 1) infinite recursion; or 2) deadlock (trying to
3254 * lock a boolean mutex recursively); 3) destroying the getenv() static
3255 * buffer; or 4) destroying the temporary created by this for the copy
3256 * causes a log entry to be made which could cause a new temporary to be
3257 * created, which will need to be destroyed at some point, leading to an
3260 * The solution adopted here (after some gnashing of teeth) is to detect
3261 * the recursive calls and calls from the logger, and treat them specially.
3262 * Let's say we want to do getenv("foo"). We first find
3263 * getenv(PERL_MEM_LOG) and save it to a fixed-length per-interpreter
3264 * variable, so no temporary is required. Then we do getenv(foo}, and in
3265 * the process of creating a temporary to save it, this function will be
3266 * called recursively to do a getenv(PERL_MEM_LOG). On the recursed call,
3267 * we detect that it is such a call and return our saved value instead of
3268 * locking and doing a new getenv(). This solves all of problems 1), 2),
3269 * and 3). Because all the getenv()s are done while the mutex is locked,
3270 * the state cannot have changed. To solve 4), we don't create a temporary
3271 * when this is called from the logging code. That code disposes of the
3272 * return value while the mutex is still locked.
3274 * The value of getenv(PERL_MEM_LOG) can be anything, but only initial
3275 * digits and 3 particular letters are significant; the rest are ignored by
3276 * the memory logging code. Thus the per-interpreter variable only needs
3277 * to be large enough to save the significant information, the size of
3278 * which is known at compile time. The first byte is extra, reserved for
3279 * flags for our use. To protect against overflowing, only the reserved
3280 * byte, as many digits as don't overflow, and the three letters are
3283 * The reserved byte has two bits:
3284 * 0x1 if set indicates that if we get here, it is a recursive call of
3286 * 0x2 if set indicates that the call is from the logging code.
3288 * If the flag indicates this is a recursive call, just return the stored
3289 * value of PL_mem_log; An empty value gets turned into NULL. */
3290 if (strEQ(str, "PERL_MEM_LOG") && PL_mem_log[0] & 0x1) {
3291 if (PL_mem_log[1] == '\0') {
3294 return PL_mem_log + 1;
3304 /* Here we are in a critical section. As explained above, we do our own
3305 * getenv(PERL_MEM_LOG), saving the result safely. */
3306 ret = getenv("PERL_MEM_LOG");
3307 if (ret == NULL) { /* No logging active */
3309 /* Return that immediately if called from the logging code */
3310 if (PL_mem_log[0] & 0x2) {
3315 PL_mem_log[1] = '\0';
3318 char *mem_log_meat = PL_mem_log + 1; /* first byte reserved */
3320 /* There is nothing to prevent the value of PERL_MEM_LOG from being an
3321 * extremely long string. But we want only a few characters from it.
3322 * PL_mem_log has been made large enough to hold just the ones we need.
3323 * First the file descriptor. */
3324 if (isDIGIT(*ret)) {
3325 const char * s = ret;
3326 if (UNLIKELY(*s == '0')) {
3328 /* Reduce multiple leading zeros to a single one. This is to
3329 * allow the caller to change what to do with leading zeros. */
3330 *mem_log_meat++ = '0';
3337 /* If the input overflows, copy just enough for the result to also
3338 * overflow, plus 1 to make sure */
3339 while (isDIGIT(*s) && s < ret + TYPE_DIGITS(UV) + 1) {
3340 *mem_log_meat++ = *s++;
3344 /* Then each of the four significant characters */
3345 if (strchr(ret, 'm')) {
3346 *mem_log_meat++ = 'm';
3348 if (strchr(ret, 's')) {
3349 *mem_log_meat++ = 's';
3351 if (strchr(ret, 't')) {
3352 *mem_log_meat++ = 't';
3354 if (strchr(ret, 'c')) {
3355 *mem_log_meat++ = 'c';
3357 *mem_log_meat = '\0';
3359 assert(mem_log_meat < PL_mem_log + sizeof(PL_mem_log));
3362 /* If we are being called from the logger, it only needs the significant
3363 * portion of PERL_MEM_LOG, and doesn't need a safe copy */
3364 if (PL_mem_log[0] & 0x2) {
3365 assert(strEQ(str, "PERL_MEM_LOG"));
3367 return PL_mem_log + 1;
3370 /* Here is a generic getenv(). This could be a getenv("PERL_MEM_LOG") that
3371 * is coming from other than the logging code, so it should be treated the
3372 * same as any other getenv(), returning the full value, not just the
3373 * significant part, and having its value saved. Set the flag that
3374 * indicates any call to this routine will be a recursion from here */
3375 PL_mem_log[0] = 0x1;
3379 /* Now get the value of the real desired variable, and save a copy */
3383 ret = SvPVX( newSVpvn_flags(ret, strlen(ret) ,SVs_TEMP) );
3390 /* Clear the buffer */
3391 Zero(PL_mem_log, sizeof(PL_mem_log), char);
3398 PERL_STATIC_INLINE bool
3399 Perl_sv_isbool(pTHX_ const SV *sv)
3401 /* change to the following in 5.37, logically the same but
3402 * more efficient and more future proof */
3404 return (SvBoolFlagsOK(sv) && BOOL_INTERNALS_sv_isbool(sv));
3406 return SvIOK(sv) && SvPOK(sv) && SvIsCOW_static(sv) &&
3407 (SvPVX_const(sv) == PL_Yes || SvPVX_const(sv) == PL_No);
3414 PERL_STATIC_INLINE AV *
3415 Perl_cop_file_avn(pTHX_ const COP *cop) {
3417 PERL_ARGS_ASSERT_COP_FILE_AVN;
3419 const char *file = CopFILE(cop);
3421 GV *gv = gv_fetchfile_flags(file, strlen(file), GVF_NOADD);
3434 PERL_STATIC_INLINE PADNAME *
3435 Perl_padname_refcnt_inc(PADNAME *pn)
3437 PadnameREFCNT(pn)++;
3442 * ex: set ts=8 sts=4 sw=4 et: