3 * Copyright (C) 2012 by Larry Wall and others
5 * You may distribute under the terms of either the GNU General Public
6 * License or the Artistic License, as specified in the README file.
8 * This file contains tables and code adapted from
9 * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which requires this
12 Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
14 Permission is hereby granted, free of charge, to any person obtaining a copy of
15 this software and associated documentation files (the "Software"), to deal in
16 the Software without restriction, including without limitation the rights to
17 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
18 of the Software, and to permit persons to whom the Software is furnished to do
19 so, subject to the following conditions:
21 The above copyright notice and this permission notice shall be included in all
22 copies or substantial portions of the Software.
24 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
29 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * This file is a home for static inline functions that cannot go in other
34 * header files, because they depend on proto.h (included after most other
35 * headers) or struct definitions.
37 * Each section names the header file that the functions "belong" to.
40 /* ------------------------------- av.h ------------------------------- */
43 =for apidoc_section $AV
45 Returns the number of elements in the array C<av>. This is the true length of
46 the array, including any undefined elements. It is always the same as
47 S<C<av_top_index(av) + 1>>.
51 PERL_STATIC_INLINE Size_t
52 Perl_av_count(pTHX_ AV *av)
54 PERL_ARGS_ASSERT_AV_COUNT;
55 assert(SvTYPE(av) == SVt_PVAV);
57 return AvFILL(av) + 1;
60 /* ------------------------------- av.c ------------------------------- */
63 =for apidoc av_store_simple
65 This is a cut-down version of av_store that assumes that the array is
66 very straightforward - no magic, not readonly, and AvREAL - and that
67 C<key> is not negative. This function MUST NOT be used in situations
68 where any of those assumptions may not hold.
70 Stores an SV in an array. The array index is specified as C<key>. It
71 can be dereferenced to get the C<SV*> that was stored there (= C<val>)).
73 Note that the caller is responsible for suitably incrementing the reference
74 count of C<val> before the call.
76 Approximate Perl equivalent: C<splice(@myarray, $key, 1, $val)>.
81 PERL_STATIC_INLINE SV**
82 Perl_av_store_simple(pTHX_ AV *av, SSize_t key, SV *val)
84 PERL_ARGS_ASSERT_AV_STORE_SIMPLE;
85 assert(SvTYPE(av) == SVt_PVAV);
86 assert(!SvMAGICAL(av));
87 assert(!SvREADONLY(av));
91 SV **ary = AvARRAY(av);
93 if (AvFILLp(av) < key) {
94 if (key > AvMAX(av)) {
100 SvREFCNT_dec(ary[key]);
107 =for apidoc av_fetch_simple
109 This is a cut-down version of av_fetch that assumes that the array is
110 very straightforward - no magic, not readonly, and AvREAL - and that
111 C<key> is not negative. This function MUST NOT be used in situations
112 where any of those assumptions may not hold.
114 Returns the SV at the specified index in the array. The C<key> is the
115 index. If lval is true, you are guaranteed to get a real SV back (in case
116 it wasn't real before), which you can then modify. Check that the return
117 value is non-null before dereferencing it to a C<SV*>.
119 The rough perl equivalent is C<$myarray[$key]>.
124 PERL_STATIC_INLINE SV**
125 Perl_av_fetch_simple(pTHX_ AV *av, SSize_t key, I32 lval)
127 PERL_ARGS_ASSERT_AV_FETCH_SIMPLE;
128 assert(SvTYPE(av) == SVt_PVAV);
129 assert(!SvMAGICAL(av));
130 assert(!SvREADONLY(av));
134 if ( (key > AvFILLp(av)) || !AvARRAY(av)[key]) {
135 return lval ? av_store_simple(av,key,newSV(0)) : NULL;
137 return &AvARRAY(av)[key];
141 /* ------------------------------- cv.h ------------------------------- */
144 =for apidoc_section $CV
146 Returns the GV associated with the CV C<sv>, reifying it if necessary.
150 PERL_STATIC_INLINE GV *
151 Perl_CvGV(pTHX_ CV *sv)
153 PERL_ARGS_ASSERT_CVGV;
156 ? Perl_cvgv_from_hek(aTHX_ sv)
157 : ((XPVCV*)MUTABLE_PTR(SvANY(sv)))->xcv_gv_u.xcv_gv;
160 PERL_STATIC_INLINE I32 *
161 Perl_CvDEPTH(const CV * const sv)
163 PERL_ARGS_ASSERT_CVDEPTH;
164 assert(SvTYPE(sv) == SVt_PVCV || SvTYPE(sv) == SVt_PVFM);
166 return &((XPVCV*)SvANY(sv))->xcv_depth;
170 CvPROTO returns the prototype as stored, which is not necessarily what
171 the interpreter should be using. Specifically, the interpreter assumes
172 that spaces have been stripped, which has been the case if the prototype
173 was added by toke.c, but is generally not the case if it was added elsewhere.
174 Since we can't enforce the spacelessness at assignment time, this routine
175 provides a temporary copy at parse time with spaces removed.
176 I<orig> is the start of the original buffer, I<len> is the length of the
177 prototype and will be updated when this returns.
181 PERL_STATIC_INLINE char *
182 S_strip_spaces(pTHX_ const char * orig, STRLEN * const len)
186 tmpsv = newSVpvn_flags(orig, *len, SVs_TEMP);
194 *len = tmps - SvPVX(tmpsv);
199 /* ------------------------------- mg.h ------------------------------- */
201 #if defined(PERL_CORE) || defined(PERL_EXT)
202 /* assumes get-magic and stringification have already occurred */
203 PERL_STATIC_INLINE STRLEN
204 S_MgBYTEPOS(pTHX_ MAGIC *mg, SV *sv, const char *s, STRLEN len)
206 assert(mg->mg_type == PERL_MAGIC_regex_global);
207 assert(mg->mg_len != -1);
208 if (mg->mg_flags & MGf_BYTES || !DO_UTF8(sv))
209 return (STRLEN)mg->mg_len;
211 const STRLEN pos = (STRLEN)mg->mg_len;
212 /* Without this check, we may read past the end of the buffer: */
213 if (pos > sv_or_pv_len_utf8(sv, s, len)) return len+1;
214 return sv_or_pv_pos_u2b(sv, s, pos, NULL);
219 /* ------------------------------- pad.h ------------------------------ */
221 #if defined(PERL_IN_PAD_C) || defined(PERL_IN_OP_C)
222 PERL_STATIC_INLINE bool
223 S_PadnameIN_SCOPE(const PADNAME * const pn, const U32 seq)
225 PERL_ARGS_ASSERT_PADNAMEIN_SCOPE;
227 /* is seq within the range _LOW to _HIGH ?
228 * This is complicated by the fact that PL_cop_seqmax
229 * may have wrapped around at some point */
230 if (COP_SEQ_RANGE_LOW(pn) == PERL_PADSEQ_INTRO)
231 return FALSE; /* not yet introduced */
233 if (COP_SEQ_RANGE_HIGH(pn) == PERL_PADSEQ_INTRO) {
234 /* in compiling scope */
236 (seq > COP_SEQ_RANGE_LOW(pn))
237 ? (seq - COP_SEQ_RANGE_LOW(pn) < (U32_MAX >> 1))
238 : (COP_SEQ_RANGE_LOW(pn) - seq > (U32_MAX >> 1))
243 (COP_SEQ_RANGE_LOW(pn) > COP_SEQ_RANGE_HIGH(pn))
245 ( seq > COP_SEQ_RANGE_LOW(pn)
246 || seq <= COP_SEQ_RANGE_HIGH(pn))
248 : ( seq > COP_SEQ_RANGE_LOW(pn)
249 && seq <= COP_SEQ_RANGE_HIGH(pn))
256 /* ------------------------------- pp.h ------------------------------- */
258 PERL_STATIC_INLINE I32
261 DEBUG_s(DEBUG_v(PerlIO_printf(Perl_debug_log,
262 "MARK top %p %" IVdf "\n",
264 (IV)*PL_markstack_ptr)));
265 return *PL_markstack_ptr;
268 PERL_STATIC_INLINE I32
271 DEBUG_s(DEBUG_v(PerlIO_printf(Perl_debug_log,
272 "MARK pop %p %" IVdf "\n",
273 (PL_markstack_ptr-1),
274 (IV)*(PL_markstack_ptr-1))));
275 assert((PL_markstack_ptr > PL_markstack) || !"MARK underflow");
276 return *PL_markstack_ptr--;
279 /* ----------------------------- regexp.h ----------------------------- */
281 /* PVLVs need to act as a superset of all scalar types - they are basically
282 * PVMGs with a few extra fields.
283 * REGEXPs are first class scalars, but have many fields that can't be copied
286 * Hence we take a different approach - instead of a copy, PVLVs store a pointer
287 * back to the original body. To avoid increasing the size of PVLVs just for the
288 * rare case of REGEXP assignment, this pointer is stored in the memory usually
289 * used for SvLEN(). Hence the check for SVt_PVLV below, and the ? : ternary to
290 * read the pointer from the two possible locations. The macro SvLEN() wraps the
291 * access to the union's member xpvlenu_len, but there is no equivalent macro
292 * for wrapping the union's member xpvlenu_rx, hence the direct reference here.
294 * See commit df6b4bd56551f2d3 for more details. */
296 PERL_STATIC_INLINE struct regexp *
297 Perl_ReANY(const REGEXP * const re)
299 XPV* const p = (XPV*)SvANY(re);
301 PERL_ARGS_ASSERT_REANY;
302 assert(isREGEXP(re));
304 return SvTYPE(re) == SVt_PVLV ? p->xpv_len_u.xpvlenu_rx
305 : (struct regexp *)p;
308 /* ------------------------------- sv.h ------------------------------- */
310 PERL_STATIC_INLINE bool
311 Perl_SvTRUE(pTHX_ SV *sv)
313 PERL_ARGS_ASSERT_SVTRUE;
315 if (UNLIKELY(sv == NULL))
318 return SvTRUE_nomg_NN(sv);
321 PERL_STATIC_INLINE bool
322 Perl_SvTRUE_nomg(pTHX_ SV *sv)
324 PERL_ARGS_ASSERT_SVTRUE_NOMG;
326 if (UNLIKELY(sv == NULL))
328 return SvTRUE_nomg_NN(sv);
331 PERL_STATIC_INLINE bool
332 Perl_SvTRUE_NN(pTHX_ SV *sv)
334 PERL_ARGS_ASSERT_SVTRUE_NN;
337 return SvTRUE_nomg_NN(sv);
340 PERL_STATIC_INLINE bool
341 Perl_SvTRUE_common(pTHX_ SV * sv, const bool sv_2bool_is_fallback)
343 PERL_ARGS_ASSERT_SVTRUE_COMMON;
345 if (UNLIKELY(SvIMMORTAL_INTERP(sv)))
346 return SvIMMORTAL_TRUE(sv);
352 return SvPVXtrue(sv);
355 return SvIVX(sv) != 0; /* casts to bool */
357 if (SvROK(sv) && !(SvOBJECT(SvRV(sv)) && HvAMAGIC(SvSTASH(SvRV(sv)))))
360 if (sv_2bool_is_fallback)
361 return sv_2bool_nomg(sv);
363 return isGV_with_GP(sv);
367 PERL_STATIC_INLINE SV *
368 Perl_SvREFCNT_inc(SV *sv)
370 if (LIKELY(sv != NULL))
374 PERL_STATIC_INLINE SV *
375 Perl_SvREFCNT_inc_NN(SV *sv)
377 PERL_ARGS_ASSERT_SVREFCNT_INC_NN;
382 PERL_STATIC_INLINE void
383 Perl_SvREFCNT_inc_void(SV *sv)
385 if (LIKELY(sv != NULL))
388 PERL_STATIC_INLINE void
389 Perl_SvREFCNT_dec(pTHX_ SV *sv)
391 if (LIKELY(sv != NULL)) {
392 U32 rc = SvREFCNT(sv);
394 SvREFCNT(sv) = rc - 1;
396 Perl_sv_free2(aTHX_ sv, rc);
400 PERL_STATIC_INLINE void
401 Perl_SvREFCNT_dec_NN(pTHX_ SV *sv)
403 U32 rc = SvREFCNT(sv);
405 PERL_ARGS_ASSERT_SVREFCNT_DEC_NN;
408 SvREFCNT(sv) = rc - 1;
410 Perl_sv_free2(aTHX_ sv, rc);
413 PERL_STATIC_INLINE void
414 Perl_SvAMAGIC_on(SV *sv)
416 PERL_ARGS_ASSERT_SVAMAGIC_ON;
419 if (SvOBJECT(SvRV(sv))) HvAMAGIC_on(SvSTASH(SvRV(sv)));
421 PERL_STATIC_INLINE void
422 Perl_SvAMAGIC_off(SV *sv)
424 PERL_ARGS_ASSERT_SVAMAGIC_OFF;
426 if (SvROK(sv) && SvOBJECT(SvRV(sv)))
427 HvAMAGIC_off(SvSTASH(SvRV(sv)));
430 PERL_STATIC_INLINE U32
431 Perl_SvPADSTALE_on(SV *sv)
433 assert(!(SvFLAGS(sv) & SVs_PADTMP));
434 return SvFLAGS(sv) |= SVs_PADSTALE;
436 PERL_STATIC_INLINE U32
437 Perl_SvPADSTALE_off(SV *sv)
439 assert(!(SvFLAGS(sv) & SVs_PADTMP));
440 return SvFLAGS(sv) &= ~SVs_PADSTALE;
442 #if defined(PERL_CORE) || defined (PERL_EXT)
443 PERL_STATIC_INLINE STRLEN
444 S_sv_or_pv_pos_u2b(pTHX_ SV *sv, const char *pv, STRLEN pos, STRLEN *lenp)
446 PERL_ARGS_ASSERT_SV_OR_PV_POS_U2B;
448 U8 *hopped = utf8_hop((U8 *)pv, pos);
449 if (lenp) *lenp = (STRLEN)(utf8_hop(hopped, *lenp) - hopped);
450 return (STRLEN)(hopped - (U8 *)pv);
452 return sv_pos_u2b_flags(sv,pos,lenp,SV_CONST_RETURN);
456 /* ------------------------------- utf8.h ------------------------------- */
459 =for apidoc_section $unicode
462 PERL_STATIC_INLINE void
463 Perl_append_utf8_from_native_byte(const U8 byte, U8** dest)
465 /* Takes an input 'byte' (Latin1 or EBCDIC) and appends it to the UTF-8
466 * encoded string at '*dest', updating '*dest' to include it */
468 PERL_ARGS_ASSERT_APPEND_UTF8_FROM_NATIVE_BYTE;
470 if (NATIVE_BYTE_IS_INVARIANT(byte))
473 *((*dest)++) = UTF8_EIGHT_BIT_HI(byte);
474 *((*dest)++) = UTF8_EIGHT_BIT_LO(byte);
479 =for apidoc valid_utf8_to_uvchr
480 Like C<L<perlapi/utf8_to_uvchr_buf>>, but should only be called when it is
481 known that the next character in the input UTF-8 string C<s> is well-formed
482 (I<e.g.>, it passes C<L<perlapi/isUTF8_CHAR>>. Surrogates, non-character code
483 points, and non-Unicode code points are allowed.
489 PERL_STATIC_INLINE UV
490 Perl_valid_utf8_to_uvchr(const U8 *s, STRLEN *retlen)
492 const UV expectlen = UTF8SKIP(s);
493 const U8* send = s + expectlen;
496 PERL_ARGS_ASSERT_VALID_UTF8_TO_UVCHR;
502 /* An invariant is trivially returned */
503 if (expectlen == 1) {
507 /* Remove the leading bits that indicate the number of bytes, leaving just
508 * the bits that are part of the value */
509 uv = NATIVE_UTF8_TO_I8(uv) & UTF_START_MASK(expectlen);
511 /* Now, loop through the remaining bytes, accumulating each into the
512 * working total as we go. (I khw tried unrolling the loop for up to 4
513 * bytes, but there was no performance improvement) */
514 for (++s; s < send; s++) {
515 uv = UTF8_ACCUMULATE(uv, *s);
518 return UNI_TO_NATIVE(uv);
523 =for apidoc is_utf8_invariant_string
525 Returns TRUE if the first C<len> bytes of the string C<s> are the same
526 regardless of the UTF-8 encoding of the string (or UTF-EBCDIC encoding on
527 EBCDIC machines); otherwise it returns FALSE. That is, it returns TRUE if they
528 are UTF-8 invariant. On ASCII-ish machines, all the ASCII characters and only
529 the ASCII characters fit this definition. On EBCDIC machines, the ASCII-range
530 characters are invariant, but so also are the C1 controls.
532 If C<len> is 0, it will be calculated using C<strlen(s)>, (which means if you
533 use this option, that C<s> can't have embedded C<NUL> characters and has to
534 have a terminating C<NUL> byte).
537 C<L</is_utf8_string>>,
538 C<L</is_utf8_string_flags>>,
539 C<L</is_utf8_string_loc>>,
540 C<L</is_utf8_string_loc_flags>>,
541 C<L</is_utf8_string_loclen>>,
542 C<L</is_utf8_string_loclen_flags>>,
543 C<L</is_utf8_fixed_width_buf_flags>>,
544 C<L</is_utf8_fixed_width_buf_loc_flags>>,
545 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
546 C<L</is_strict_utf8_string>>,
547 C<L</is_strict_utf8_string_loc>>,
548 C<L</is_strict_utf8_string_loclen>>,
549 C<L</is_c9strict_utf8_string>>,
550 C<L</is_c9strict_utf8_string_loc>>,
552 C<L</is_c9strict_utf8_string_loclen>>.
558 #define is_utf8_invariant_string(s, len) \
559 is_utf8_invariant_string_loc(s, len, NULL)
562 =for apidoc is_utf8_invariant_string_loc
564 Like C<L</is_utf8_invariant_string>> but upon failure, stores the location of
565 the first UTF-8 variant character in the C<ep> pointer; if all characters are
566 UTF-8 invariant, this function does not change the contents of C<*ep>.
572 PERL_STATIC_INLINE bool
573 Perl_is_utf8_invariant_string_loc(const U8* const s, STRLEN len, const U8 ** ep)
578 PERL_ARGS_ASSERT_IS_UTF8_INVARIANT_STRING_LOC;
581 len = strlen((const char *)s);
586 /* This looks like 0x010101... */
587 # define PERL_COUNT_MULTIPLIER (~ (UINTMAX_C(0)) / 0xFF)
589 /* This looks like 0x808080... */
590 # define PERL_VARIANTS_WORD_MASK (PERL_COUNT_MULTIPLIER * 0x80)
591 # define PERL_WORDSIZE sizeof(PERL_UINTMAX_T)
592 # define PERL_WORD_BOUNDARY_MASK (PERL_WORDSIZE - 1)
594 /* Evaluates to 0 if 'x' is at a word boundary; otherwise evaluates to 1, by
595 * or'ing together the lowest bits of 'x'. Hopefully the final term gets
596 * optimized out completely on a 32-bit system, and its mask gets optimized out
597 * on a 64-bit system */
598 # define PERL_IS_SUBWORD_ADDR(x) (1 & ( PTR2nat(x) \
599 | ( PTR2nat(x) >> 1) \
601 & PERL_WORD_BOUNDARY_MASK) >> 2))))
605 /* Do the word-at-a-time iff there is at least one usable full word. That
606 * means that after advancing to a word boundary, there still is at least a
607 * full word left. The number of bytes needed to advance is 'wordsize -
608 * offset' unless offset is 0. */
609 if ((STRLEN) (send - x) >= PERL_WORDSIZE
611 /* This term is wordsize if subword; 0 if not */
612 + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(x)
615 - (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK))
618 /* Process per-byte until reach word boundary. XXX This loop could be
619 * eliminated if we knew that this platform had fast unaligned reads */
620 while (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK) {
621 if (! UTF8_IS_INVARIANT(*x)) {
631 /* Here, we know we have at least one full word to process. Process
632 * per-word as long as we have at least a full word left */
634 if ((* (PERL_UINTMAX_T *) x) & PERL_VARIANTS_WORD_MASK) {
636 /* Found a variant. Just return if caller doesn't want its
642 # if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 \
643 || BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
645 *ep = x + variant_byte_number(* (PERL_UINTMAX_T *) x);
646 assert(*ep >= s && *ep < send);
650 # else /* If weird byte order, drop into next loop to do byte-at-a-time
659 } while (x + PERL_WORDSIZE <= send);
662 #endif /* End of ! EBCDIC */
664 /* Process per-byte */
666 if (! UTF8_IS_INVARIANT(*x)) {
680 /* See if the platform has builtins for finding the most/least significant bit,
681 * and which one is right for using on 32 and 64 bit operands */
682 #if (__has_builtin(__builtin_clz) || PERL_GCC_VERSION_GE(3,4,0))
683 # if U32SIZE == INTSIZE
684 # define PERL_CLZ_32 __builtin_clz
686 # if defined(U64TYPE) && U64SIZE == INTSIZE
687 # define PERL_CLZ_64 __builtin_clz
690 #if (__has_builtin(__builtin_ctz) || PERL_GCC_VERSION_GE(3,4,0))
691 # if U32SIZE == INTSIZE
692 # define PERL_CTZ_32 __builtin_ctz
694 # if defined(U64TYPE) && U64SIZE == INTSIZE
695 # define PERL_CTZ_64 __builtin_ctz
699 #if (__has_builtin(__builtin_clzl) || PERL_GCC_VERSION_GE(3,4,0))
700 # if U32SIZE == LONGSIZE && ! defined(PERL_CLZ_32)
701 # define PERL_CLZ_32 __builtin_clzl
703 # if defined(U64TYPE) && U64SIZE == LONGSIZE && ! defined(PERL_CLZ_64)
704 # define PERL_CLZ_64 __builtin_clzl
707 #if (__has_builtin(__builtin_ctzl) || PERL_GCC_VERSION_GE(3,4,0))
708 # if U32SIZE == LONGSIZE && ! defined(PERL_CTZ_32)
709 # define PERL_CTZ_32 __builtin_ctzl
711 # if defined(U64TYPE) && U64SIZE == LONGSIZE && ! defined(PERL_CTZ_64)
712 # define PERL_CTZ_64 __builtin_ctzl
716 #if (__has_builtin(__builtin_clzll) || PERL_GCC_VERSION_GE(3,4,0))
717 # if U32SIZE == LONGLONGSIZE && ! defined(PERL_CLZ_32)
718 # define PERL_CLZ_32 __builtin_clzll
720 # if defined(U64TYPE) && U64SIZE == LONGLONGSIZE && ! defined(PERL_CLZ_64)
721 # define PERL_CLZ_64 __builtin_clzll
724 #if (__has_builtin(__builtin_ctzll) || PERL_GCC_VERSION_GE(3,4,0))
725 # if U32SIZE == LONGLONGSIZE && ! defined(PERL_CTZ_32)
726 # define PERL_CTZ_32 __builtin_ctzll
728 # if defined(U64TYPE) && U64SIZE == LONGLONGSIZE && ! defined(PERL_CTZ_64)
729 # define PERL_CTZ_64 __builtin_ctzll
733 #if defined(_MSC_VER) && _MSC_VER >= 1400
735 # pragma intrinsic(_BitScanForward)
736 # pragma intrinsic(_BitScanReverse)
738 # pragma intrinsic(_BitScanForward64)
739 # pragma intrinsic(_BitScanReverse64)
743 /* The reason there are not checks to see if ffs() and ffsl() are available for
744 * determining the lsb, is because these don't improve on the deBruijn method
745 * fallback, which is just a branchless integer multiply, array element
746 * retrieval, and shift. The others, even if the function call overhead is
747 * optimized out, have to cope with the possibility of the input being all
748 * zeroes, and almost certainly will have conditionals for this eventuality.
749 * khw, at the time of this commit, looked at the source for both gcc and clang
750 * to verify this. (gcc used a method inferior to deBruijn.) */
752 /* Below are functions to find the first, last, or only set bit in a word. On
753 * platforms with 64-bit capability, there is a pair for each operation; the
754 * first taking a 64 bit operand, and the second a 32 bit one. The logic is
755 * the same in each pair, so the second is stripped of most comments. */
757 #ifdef U64TYPE /* HAS_QUAD not usable outside the core */
759 PERL_STATIC_INLINE unsigned
760 Perl_lsbit_pos64(U64 word)
762 /* Find the position (0..63) of the least significant set bit in the input
767 /* If we can determine that the platform has a usable fast method to get
768 * this info, use that */
770 # if defined(PERL_CTZ_64)
771 # define PERL_HAS_FAST_GET_LSB_POS64
773 return (unsigned) PERL_CTZ_64(word);
775 # elif U64SIZE == 8 && defined(_MSC_VER) && _MSC_VER >= 1400
776 # define PERL_HAS_FAST_GET_LSB_POS64
780 _BitScanForward64(&index, word);
781 return (unsigned)index;
786 /* Here, we didn't find a fast method for finding the lsb. Fall back to
787 * making the lsb the only set bit in the word, and use our function that
788 * works on words with a single bit set.
791 * https://stackoverflow.com/questions/757059/position-of-least-significant-bit-that-is-set
793 * The word will look like this, with a rightmost set bit in position 's':
794 * ('x's are don't cares, and 'y's are their complements)
797 * y..y011..11 Complement
799 * 0..0100..00 And with the original
801 * (Yes, complementing and adding 1 is just taking the negative on 2's
802 * complement machines, but not on 1's complement ones, and some compilers
803 * complain about negating an unsigned.)
805 return single_1bit_pos64(word & (~word + 1));
811 # define lsbit_pos_uintmax_(word) lsbit_pos64(word)
813 # define lsbit_pos_uintmax_(word) lsbit_pos32(word)
816 PERL_STATIC_INLINE unsigned /* Like above for 32 bit word */
817 Perl_lsbit_pos32(U32 word)
819 /* Find the position (0..31) of the least significant set bit in the input
824 #if defined(PERL_CTZ_32)
825 # define PERL_HAS_FAST_GET_LSB_POS32
827 return (unsigned) PERL_CTZ_32(word);
829 #elif U32SIZE == 4 && defined(_MSC_VER) && _MSC_VER >= 1400
830 # define PERL_HAS_FAST_GET_LSB_POS32
834 _BitScanForward(&index, word);
835 return (unsigned)index;
840 return single_1bit_pos32(word & (~word + 1));
847 /* Convert the leading zeros count to the bit position of the first set bit.
848 * This just subtracts from the highest position, 31 or 63. But some compilers
849 * don't optimize this optimally, and so a bit of bit twiddling encourages them
850 * to do the right thing. It turns out that subtracting a smaller non-negative
851 * number 'x' from 2**n-1 for any n is the same as taking the exclusive-or of
852 * the two numbers. To see why, first note that the sum of any number, x, and
853 * its complement, x', is all ones. So all ones minus x is x'. Then note that
854 * the xor of x and all ones is x'. */
855 #define LZC_TO_MSBIT_POS_(size, lzc) ((size##SIZE * CHARBITS - 1) ^ (lzc))
857 #ifdef U64TYPE /* HAS_QUAD not usable outside the core */
859 PERL_STATIC_INLINE unsigned
860 Perl_msbit_pos64(U64 word)
862 /* Find the position (0..63) of the most significant set bit in the input
867 /* If we can determine that the platform has a usable fast method to get
870 # if defined(PERL_CLZ_64)
871 # define PERL_HAS_FAST_GET_MSB_POS64
873 return (unsigned) LZC_TO_MSBIT_POS_(U64, PERL_CLZ_64(word));
875 # elif U64SIZE == 8 && defined(_WIN64) && defined(_MSC_VER) && _MSC_VER >= 1400
876 # define PERL_HAS_FAST_GET_MSB_POS64
880 _BitScanReverse64(&index, word);
881 return (unsigned)index;
886 /* Here, we didn't find a fast method for finding the msb. Fall back to
887 * making the msb the only set bit in the word, and use our function that
888 * works on words with a single bit set.
890 * Isolate the msb; http://codeforces.com/blog/entry/10330
892 * Only the most significant set bit matters. Or'ing word with its right
893 * shift of 1 makes that bit and the next one to its right both 1.
894 * Repeating that with the right shift of 2 makes for 4 1-bits in a row.
895 * ... We end with the msb and all to the right being 1. */
900 word |= (word >> 16);
901 word |= (word >> 32);
903 /* Then subtracting the right shift by 1 clears all but the left-most of
904 * the 1 bits, which is our desired result */
907 /* Now we have a single bit set */
908 return single_1bit_pos64(word);
914 # define msbit_pos_uintmax_(word) msbit_pos64(word)
916 # define msbit_pos_uintmax_(word) msbit_pos32(word)
919 PERL_STATIC_INLINE unsigned
920 Perl_msbit_pos32(U32 word)
922 /* Find the position (0..31) of the most significant set bit in the input
927 #if defined(PERL_CLZ_32)
928 # define PERL_HAS_FAST_GET_MSB_POS32
930 return (unsigned) LZC_TO_MSBIT_POS_(U32, PERL_CLZ_32(word));
932 #elif U32SIZE == 4 && defined(_MSC_VER) && _MSC_VER >= 1400
933 # define PERL_HAS_FAST_GET_MSB_POS32
937 _BitScanReverse(&index, word);
938 return (unsigned)index;
947 word |= (word >> 16);
949 return single_1bit_pos32(word);
955 #if UVSIZE == U64SIZE
956 # define msbit_pos(word) msbit_pos64(word)
957 # define lsbit_pos(word) lsbit_pos64(word)
958 #elif UVSIZE == U32SIZE
959 # define msbit_pos(word) msbit_pos32(word)
960 # define lsbit_pos(word) lsbit_pos32(word)
963 #ifdef U64TYPE /* HAS_QUAD not usable outside the core */
965 PERL_STATIC_INLINE unsigned
966 Perl_single_1bit_pos64(U64 word)
968 /* Given a 64-bit word known to contain all zero bits except one 1 bit,
969 * find and return the 1's position: 0..63 */
971 # ifdef PERL_CORE /* macro not exported */
972 ASSUME(isPOWER_OF_2(word));
974 ASSUME(word && (word & (word-1)) == 0);
977 /* The only set bit is both the most and least significant bit. If we have
978 * a fast way of finding either one, use that.
980 * It may appear at first glance that those functions call this one, but
981 * they don't if the corresponding #define is set */
983 # ifdef PERL_HAS_FAST_GET_MSB_POS64
985 return msbit_pos64(word);
987 # elif defined(PERL_HAS_FAST_GET_LSB_POS64)
989 return lsbit_pos64(word);
993 /* The position of the only set bit in a word can be quickly calculated
994 * using deBruijn sequences. See for example
995 * https://en.wikipedia.org/wiki/De_Bruijn_sequence */
996 return PL_deBruijn_bitpos_tab64[(word * PERL_deBruijnMagic64_)
997 >> PERL_deBruijnShift64_];
1004 PERL_STATIC_INLINE unsigned
1005 Perl_single_1bit_pos32(U32 word)
1007 /* Given a 32-bit word known to contain all zero bits except one 1 bit,
1008 * find and return the 1's position: 0..31 */
1010 #ifdef PERL_CORE /* macro not exported */
1011 ASSUME(isPOWER_OF_2(word));
1013 ASSUME(word && (word & (word-1)) == 0);
1015 #ifdef PERL_HAS_FAST_GET_MSB_POS32
1017 return msbit_pos32(word);
1019 #elif defined(PERL_HAS_FAST_GET_LSB_POS32)
1021 return lsbit_pos32(word);
1023 /* Unlikely, but possible for the platform to have a wider fast operation but
1024 * not a narrower one. But easy enough to handle the case by widening the
1025 * parameter size. (Going the other way, emulating 64 bit by two 32 bit ops
1026 * would be slower than the deBruijn method.) */
1027 #elif defined(PERL_HAS_FAST_GET_MSB_POS64)
1029 return msbit_pos64(word);
1031 #elif defined(PERL_HAS_FAST_GET_LSB_POS64)
1033 return lsbit_pos64(word);
1037 return PL_deBruijn_bitpos_tab32[(word * PERL_deBruijnMagic32_)
1038 >> PERL_deBruijnShift32_];
1045 PERL_STATIC_INLINE unsigned int
1046 Perl_variant_byte_number(PERL_UINTMAX_T word)
1048 /* This returns the position in a word (0..7) of the first variant byte in
1049 * it. This is a helper function. Note that there are no branches */
1051 /* Get just the msb bits of each byte */
1052 word &= PERL_VARIANTS_WORD_MASK;
1054 /* This should only be called if we know there is a variant byte in the
1058 # if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678
1060 /* Bytes are stored like
1061 * Byte8 ... Byte2 Byte1
1062 * 63..56...15...8 7...0
1063 * so getting the lsb of the whole modified word is getting the msb of the
1064 * first byte that has its msb set */
1065 word = lsbit_pos_uintmax_(word);
1067 /* Here, word contains the position 7,15,23,...55,63 of that bit. Convert
1069 return (unsigned int) ((word + 1) >> 3) - 1;
1071 # elif BYTEORDER == 0x4321 || BYTEORDER == 0x87654321
1073 /* Bytes are stored like
1074 * Byte1 Byte2 ... Byte8
1075 * 63..56 55..47 ... 7...0
1076 * so getting the msb of the whole modified word is getting the msb of the
1077 * first byte that has its msb set */
1078 word = msbit_pos_uintmax_(word);
1080 /* Here, word contains the position 63,55,...,23,15,7 of that bit. Convert
1082 word = ((word + 1) >> 3) - 1;
1084 /* And invert the result because of the reversed byte order on this
1086 word = CHARBITS - word - 1;
1088 return (unsigned int) word;
1091 # error Unexpected byte order
1097 #if defined(PERL_CORE) || defined(PERL_EXT)
1100 =for apidoc variant_under_utf8_count
1102 This function looks at the sequence of bytes between C<s> and C<e>, which are
1103 assumed to be encoded in ASCII/Latin1, and returns how many of them would
1104 change should the string be translated into UTF-8. Due to the nature of UTF-8,
1105 each of these would occupy two bytes instead of the single one in the input
1106 string. Thus, this function returns the precise number of bytes the string
1107 would expand by when translated to UTF-8.
1109 Unlike most of the other functions that have C<utf8> in their name, the input
1110 to this function is NOT a UTF-8-encoded string. The function name is slightly
1111 I<odd> to emphasize this.
1113 This function is internal to Perl because khw thinks that any XS code that
1114 would want this is probably operating too close to the internals. Presenting a
1115 valid use case could change that.
1118 C<L<perlapi/is_utf8_invariant_string>>
1120 C<L<perlapi/is_utf8_invariant_string_loc>>,
1126 PERL_STATIC_INLINE Size_t
1127 S_variant_under_utf8_count(const U8* const s, const U8* const e)
1132 PERL_ARGS_ASSERT_VARIANT_UNDER_UTF8_COUNT;
1136 /* Test if the string is long enough to use word-at-a-time. (Logic is the
1137 * same as for is_utf8_invariant_string()) */
1138 if ((STRLEN) (e - x) >= PERL_WORDSIZE
1139 + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(x)
1140 - (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK))
1143 /* Process per-byte until reach word boundary. XXX This loop could be
1144 * eliminated if we knew that this platform had fast unaligned reads */
1145 while (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK) {
1146 count += ! UTF8_IS_INVARIANT(*x++);
1149 /* Process per-word as long as we have at least a full word left */
1150 do { /* Commit 03c1e4ab1d6ee9062fb3f94b0ba31db6698724b1 contains an
1151 explanation of how this works */
1152 PERL_UINTMAX_T increment
1153 = ((((* (PERL_UINTMAX_T *) x) & PERL_VARIANTS_WORD_MASK) >> 7)
1154 * PERL_COUNT_MULTIPLIER)
1155 >> ((PERL_WORDSIZE - 1) * CHARBITS);
1156 count += (Size_t) increment;
1158 } while (x + PERL_WORDSIZE <= e);
1163 /* Process per-byte */
1165 if (! UTF8_IS_INVARIANT(*x)) {
1177 #ifndef PERL_IN_REGEXEC_C /* Keep these around for that file */
1178 # undef PERL_WORDSIZE
1179 # undef PERL_COUNT_MULTIPLIER
1180 # undef PERL_WORD_BOUNDARY_MASK
1181 # undef PERL_VARIANTS_WORD_MASK
1185 =for apidoc is_utf8_string
1187 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1188 Perl-extended-UTF-8 string; returns FALSE otherwise. If C<len> is 0, it will
1189 be calculated using C<strlen(s)> (which means if you use this option, that C<s>
1190 can't have embedded C<NUL> characters and has to have a terminating C<NUL>
1191 byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'.
1193 This function considers Perl's extended UTF-8 to be valid. That means that
1194 code points above Unicode, surrogates, and non-character code points are
1195 considered valid by this function. Use C<L</is_strict_utf8_string>>,
1196 C<L</is_c9strict_utf8_string>>, or C<L</is_utf8_string_flags>> to restrict what
1197 code points are considered valid.
1200 C<L</is_utf8_invariant_string>>,
1201 C<L</is_utf8_invariant_string_loc>>,
1202 C<L</is_utf8_string_loc>>,
1203 C<L</is_utf8_string_loclen>>,
1204 C<L</is_utf8_fixed_width_buf_flags>>,
1205 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1206 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1211 #define is_utf8_string(s, len) is_utf8_string_loclen(s, len, NULL, NULL)
1213 #if defined(PERL_CORE) || defined (PERL_EXT)
1216 =for apidoc is_utf8_non_invariant_string
1218 Returns TRUE if L<perlapi/is_utf8_invariant_string> returns FALSE for the first
1219 C<len> bytes of the string C<s>, but they are, nonetheless, legal Perl-extended
1220 UTF-8; otherwise returns FALSE.
1222 A TRUE return means that at least one code point represented by the sequence
1223 either is a wide character not representable as a single byte, or the
1224 representation differs depending on whether the sequence is encoded in UTF-8 or
1228 C<L<perlapi/is_utf8_invariant_string>>,
1229 C<L<perlapi/is_utf8_string>>
1233 This is commonly used to determine if a SV's UTF-8 flag should be turned on.
1234 It generally needn't be if its string is entirely UTF-8 invariant, and it
1235 shouldn't be if it otherwise contains invalid UTF-8.
1237 It is an internal function because khw thinks that XS code shouldn't be working
1238 at this low a level. A valid use case could change that.
1242 PERL_STATIC_INLINE bool
1243 Perl_is_utf8_non_invariant_string(const U8* const s, STRLEN len)
1245 const U8 * first_variant;
1247 PERL_ARGS_ASSERT_IS_UTF8_NON_INVARIANT_STRING;
1249 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1253 return is_utf8_string(first_variant, len - (first_variant - s));
1259 =for apidoc is_strict_utf8_string
1261 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1262 UTF-8-encoded string that is fully interchangeable by any application using
1263 Unicode rules; otherwise it returns FALSE. If C<len> is 0, it will be
1264 calculated using C<strlen(s)> (which means if you use this option, that C<s>
1265 can't have embedded C<NUL> characters and has to have a terminating C<NUL>
1266 byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'.
1268 This function returns FALSE for strings containing any
1269 code points above the Unicode max of 0x10FFFF, surrogate code points, or
1270 non-character code points.
1273 C<L</is_utf8_invariant_string>>,
1274 C<L</is_utf8_invariant_string_loc>>,
1275 C<L</is_utf8_string>>,
1276 C<L</is_utf8_string_flags>>,
1277 C<L</is_utf8_string_loc>>,
1278 C<L</is_utf8_string_loc_flags>>,
1279 C<L</is_utf8_string_loclen>>,
1280 C<L</is_utf8_string_loclen_flags>>,
1281 C<L</is_utf8_fixed_width_buf_flags>>,
1282 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1283 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1284 C<L</is_strict_utf8_string_loc>>,
1285 C<L</is_strict_utf8_string_loclen>>,
1286 C<L</is_c9strict_utf8_string>>,
1287 C<L</is_c9strict_utf8_string_loc>>,
1289 C<L</is_c9strict_utf8_string_loclen>>.
1294 #define is_strict_utf8_string(s, len) is_strict_utf8_string_loclen(s, len, NULL, NULL)
1297 =for apidoc is_c9strict_utf8_string
1299 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1300 UTF-8-encoded string that conforms to
1301 L<Unicode Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>;
1302 otherwise it returns FALSE. If C<len> is 0, it will be calculated using
1303 C<strlen(s)> (which means if you use this option, that C<s> can't have embedded
1304 C<NUL> characters and has to have a terminating C<NUL> byte). Note that all
1305 characters being ASCII constitute 'a valid UTF-8 string'.
1307 This function returns FALSE for strings containing any code points above the
1308 Unicode max of 0x10FFFF or surrogate code points, but accepts non-character
1310 L<Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>.
1313 C<L</is_utf8_invariant_string>>,
1314 C<L</is_utf8_invariant_string_loc>>,
1315 C<L</is_utf8_string>>,
1316 C<L</is_utf8_string_flags>>,
1317 C<L</is_utf8_string_loc>>,
1318 C<L</is_utf8_string_loc_flags>>,
1319 C<L</is_utf8_string_loclen>>,
1320 C<L</is_utf8_string_loclen_flags>>,
1321 C<L</is_utf8_fixed_width_buf_flags>>,
1322 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1323 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1324 C<L</is_strict_utf8_string>>,
1325 C<L</is_strict_utf8_string_loc>>,
1326 C<L</is_strict_utf8_string_loclen>>,
1327 C<L</is_c9strict_utf8_string_loc>>,
1329 C<L</is_c9strict_utf8_string_loclen>>.
1334 #define is_c9strict_utf8_string(s, len) is_c9strict_utf8_string_loclen(s, len, NULL, 0)
1337 =for apidoc is_utf8_string_flags
1339 Returns TRUE if the first C<len> bytes of string C<s> form a valid
1340 UTF-8 string, subject to the restrictions imposed by C<flags>;
1341 returns FALSE otherwise. If C<len> is 0, it will be calculated
1342 using C<strlen(s)> (which means if you use this option, that C<s> can't have
1343 embedded C<NUL> characters and has to have a terminating C<NUL> byte). Note
1344 that all characters being ASCII constitute 'a valid UTF-8 string'.
1346 If C<flags> is 0, this gives the same results as C<L</is_utf8_string>>; if
1347 C<flags> is C<UTF8_DISALLOW_ILLEGAL_INTERCHANGE>, this gives the same results
1348 as C<L</is_strict_utf8_string>>; and if C<flags> is
1349 C<UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE>, this gives the same results as
1350 C<L</is_c9strict_utf8_string>>. Otherwise C<flags> may be any
1351 combination of the C<UTF8_DISALLOW_I<foo>> flags understood by
1352 C<L</utf8n_to_uvchr>>, with the same meanings.
1355 C<L</is_utf8_invariant_string>>,
1356 C<L</is_utf8_invariant_string_loc>>,
1357 C<L</is_utf8_string>>,
1358 C<L</is_utf8_string_loc>>,
1359 C<L</is_utf8_string_loc_flags>>,
1360 C<L</is_utf8_string_loclen>>,
1361 C<L</is_utf8_string_loclen_flags>>,
1362 C<L</is_utf8_fixed_width_buf_flags>>,
1363 C<L</is_utf8_fixed_width_buf_loc_flags>>,
1364 C<L</is_utf8_fixed_width_buf_loclen_flags>>,
1365 C<L</is_strict_utf8_string>>,
1366 C<L</is_strict_utf8_string_loc>>,
1367 C<L</is_strict_utf8_string_loclen>>,
1368 C<L</is_c9strict_utf8_string>>,
1369 C<L</is_c9strict_utf8_string_loc>>,
1371 C<L</is_c9strict_utf8_string_loclen>>.
1376 PERL_STATIC_INLINE bool
1377 Perl_is_utf8_string_flags(const U8 *s, STRLEN len, const U32 flags)
1379 const U8 * first_variant;
1381 PERL_ARGS_ASSERT_IS_UTF8_STRING_FLAGS;
1382 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
1383 |UTF8_DISALLOW_PERL_EXTENDED)));
1386 len = strlen((const char *)s);
1390 return is_utf8_string(s, len);
1393 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1394 == UTF8_DISALLOW_ILLEGAL_INTERCHANGE)
1396 return is_strict_utf8_string(s, len);
1399 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1400 == UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE)
1402 return is_c9strict_utf8_string(s, len);
1405 if (! is_utf8_invariant_string_loc(s, len, &first_variant)) {
1406 const U8* const send = s + len;
1407 const U8* x = first_variant;
1410 STRLEN cur_len = isUTF8_CHAR_flags(x, send, flags);
1411 if (UNLIKELY(! cur_len)) {
1423 =for apidoc is_utf8_string_loc
1425 Like C<L</is_utf8_string>> but stores the location of the failure (in the
1426 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1427 "utf8ness success") in the C<ep> pointer.
1429 See also C<L</is_utf8_string_loclen>>.
1434 #define is_utf8_string_loc(s, len, ep) is_utf8_string_loclen(s, len, ep, 0)
1438 =for apidoc is_utf8_string_loclen
1440 Like C<L</is_utf8_string>> but stores the location of the failure (in the
1441 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1442 "utf8ness success") in the C<ep> pointer, and the number of UTF-8
1443 encoded characters in the C<el> pointer.
1445 See also C<L</is_utf8_string_loc>>.
1450 PERL_STATIC_INLINE bool
1451 Perl_is_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
1453 const U8 * first_variant;
1455 PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN;
1458 len = strlen((const char *) s);
1461 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1473 const U8* const send = s + len;
1474 const U8* x = first_variant;
1475 STRLEN outlen = first_variant - s;
1478 const STRLEN cur_len = isUTF8_CHAR(x, send);
1479 if (UNLIKELY(! cur_len)) {
1497 /* The perl core arranges to never call the DFA below without there being at
1498 * least one byte available to look at. This allows the DFA to use a do {}
1499 * while loop which means that calling it with a UTF-8 invariant has a single
1500 * conditional, same as the calling code checking for invariance ahead of time.
1501 * And having the calling code remove that conditional speeds up by that
1502 * conditional, the case where it wasn't invariant. So there's no reason to
1503 * check before caling this.
1505 * But we don't know this for non-core calls, so have to retain the check for
1508 # define PERL_NON_CORE_CHECK_EMPTY(s,e) assert((e) > (s))
1510 # define PERL_NON_CORE_CHECK_EMPTY(s,e) if ((e) <= (s)) return FALSE
1514 * DFA for checking input is valid UTF-8 syntax.
1516 * This uses adaptations of the table and algorithm given in
1517 * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1518 * documentation of the original version. A copyright notice for the original
1519 * version is given at the beginning of this file. The Perl adapations are
1520 * documented at the definition of PL_extended_utf8_dfa_tab[].
1522 * This dfa is fast. There are three exit conditions:
1523 * 1) a well-formed code point, acceptable to the table
1524 * 2) the beginning bytes of an incomplete character, whose completion might
1525 * or might not be acceptable
1526 * 3) unacceptable to the table. Some of the adaptations have certain,
1527 * hopefully less likely to occur, legal inputs be unacceptable to the
1528 * table, so these must be sorted out afterwards.
1530 * This macro is a complete implementation of the code executing the DFA. It
1531 * is passed the input sequence bounds and the table to use, and what to do
1532 * for each of the exit conditions. There are three canned actions, likely to
1533 * be the ones you want:
1534 * DFA_RETURN_SUCCESS_
1535 * DFA_RETURN_FAILURE_
1536 * DFA_GOTO_TEASE_APART_FF_
1538 * You pass a parameter giving the action to take for each of the three
1539 * possible exit conditions:
1541 * 'accept_action' This is executed when the DFA accepts the input.
1542 * DFA_RETURN_SUCCESS_ is the most likely candidate.
1543 * 'reject_action' This is executed when the DFA rejects the input.
1544 * DFA_RETURN_FAILURE_ is a candidate, or 'goto label' where
1545 * you have written code to distinguish the rejecting state
1546 * results. Because it happens in several places, and
1547 * involves #ifdefs, the special action
1548 * DFA_GOTO_TEASE_APART_FF_ is what you want with
1549 * PL_extended_utf8_dfa_tab. On platforms without
1550 * EXTRA_LONG_UTF8, there is no need to tease anything apart,
1551 * so this evaluates to DFA_RETURN_FAILURE_; otherwise you
1552 * need to have a label 'tease_apart_FF' that it will transfer
1554 * 'incomplete_char_action' This is executed when the DFA ran off the end
1555 * before accepting or rejecting the input.
1556 * DFA_RETURN_FAILURE_ is the likely action, but you could
1557 * have a 'goto', or NOOP. In the latter case the DFA drops
1558 * off the end, and you place your code to handle this case
1559 * immediately after it.
1562 #define DFA_RETURN_SUCCESS_ return s - s0
1563 #define DFA_RETURN_FAILURE_ return 0
1564 #ifdef HAS_EXTRA_LONG_UTF8
1565 # define DFA_TEASE_APART_FF_ goto tease_apart_FF
1567 # define DFA_TEASE_APART_FF_ DFA_RETURN_FAILURE_
1570 #define PERL_IS_UTF8_CHAR_DFA(s0, e, dfa_tab, \
1573 incomplete_char_action) \
1575 const U8 * s = s0; \
1578 PERL_NON_CORE_CHECK_EMPTY(s,e); \
1581 state = dfa_tab[256 + state + dfa_tab[*s]]; \
1584 if (state == 0) { /* Accepting state */ \
1588 if (UNLIKELY(state == 1)) { /* Rejecting state */ \
1593 /* Here, dropped out of loop before end-of-char */ \
1594 incomplete_char_action; \
1600 =for apidoc isUTF8_CHAR
1602 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
1603 looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl,
1604 that represents some code point; otherwise it evaluates to 0. If non-zero, the
1605 value gives how many bytes starting at C<s> comprise the code point's
1606 representation. Any bytes remaining before C<e>, but beyond the ones needed to
1607 form the first code point in C<s>, are not examined.
1609 The code point can be any that will fit in an IV on this machine, using Perl's
1610 extension to official UTF-8 to represent those higher than the Unicode maximum
1611 of 0x10FFFF. That means that this macro is used to efficiently decide if the
1612 next few bytes in C<s> is legal UTF-8 for a single character.
1614 Use C<L</isSTRICT_UTF8_CHAR>> to restrict the acceptable code points to those
1615 defined by Unicode to be fully interchangeable across applications;
1616 C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum
1617 #9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable
1618 code points; and C<L</isUTF8_CHAR_flags>> for a more customized definition.
1620 Use C<L</is_utf8_string>>, C<L</is_utf8_string_loc>>, and
1621 C<L</is_utf8_string_loclen>> to check entire strings.
1623 Note also that a UTF-8 "invariant" character (i.e. ASCII on non-EBCDIC
1624 machines) is a valid UTF-8 character.
1628 This uses an adaptation of the table and algorithm given in
1629 https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1630 documentation of the original version. A copyright notice for the original
1631 version is given at the beginning of this file. The Perl adapation is
1632 documented at the definition of PL_extended_utf8_dfa_tab[].
1635 PERL_STATIC_INLINE Size_t
1636 Perl_isUTF8_CHAR(const U8 * const s0, const U8 * const e)
1638 PERL_ARGS_ASSERT_ISUTF8_CHAR;
1640 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab,
1641 DFA_RETURN_SUCCESS_,
1642 DFA_TEASE_APART_FF_,
1643 DFA_RETURN_FAILURE_);
1645 /* Here, we didn't return success, but dropped out of the loop. In the
1646 * case of PL_extended_utf8_dfa_tab, this means the input is either
1647 * malformed, or the start byte was FF on a platform that the dfa doesn't
1648 * handle FF's. Call a helper function. */
1650 #ifdef HAS_EXTRA_LONG_UTF8
1654 /* In the case of PL_extended_utf8_dfa_tab, getting here means the input is
1655 * either malformed, or was for the largest possible start byte, which we
1656 * now check, not inline */
1657 if (*s0 != I8_TO_NATIVE_UTF8(0xFF)) {
1661 return is_utf8_FF_helper_(s0, e,
1662 FALSE /* require full, not partial char */
1670 =for apidoc isSTRICT_UTF8_CHAR
1672 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
1673 looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some
1674 Unicode code point completely acceptable for open interchange between all
1675 applications; otherwise it evaluates to 0. If non-zero, the value gives how
1676 many bytes starting at C<s> comprise the code point's representation. Any
1677 bytes remaining before C<e>, but beyond the ones needed to form the first code
1678 point in C<s>, are not examined.
1680 The largest acceptable code point is the Unicode maximum 0x10FFFF, and must not
1681 be a surrogate nor a non-character code point. Thus this excludes any code
1682 point from Perl's extended UTF-8.
1684 This is used to efficiently decide if the next few bytes in C<s> is
1685 legal Unicode-acceptable UTF-8 for a single character.
1687 Use C<L</isC9_STRICT_UTF8_CHAR>> to use the L<Unicode Corrigendum
1688 #9|http://www.unicode.org/versions/corrigendum9.html> definition of allowable
1689 code points; C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8;
1690 and C<L</isUTF8_CHAR_flags>> for a more customized definition.
1692 Use C<L</is_strict_utf8_string>>, C<L</is_strict_utf8_string_loc>>, and
1693 C<L</is_strict_utf8_string_loclen>> to check entire strings.
1697 This uses an adaptation of the tables and algorithm given in
1698 https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1699 documentation of the original version. A copyright notice for the original
1700 version is given at the beginning of this file. The Perl adapation is
1701 documented at the definition of strict_extended_utf8_dfa_tab[].
1705 PERL_STATIC_INLINE Size_t
1706 Perl_isSTRICT_UTF8_CHAR(const U8 * const s0, const U8 * const e)
1708 PERL_ARGS_ASSERT_ISSTRICT_UTF8_CHAR;
1710 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_strict_utf8_dfa_tab,
1711 DFA_RETURN_SUCCESS_,
1713 DFA_RETURN_FAILURE_);
1716 /* Here, we didn't return success, but dropped out of the loop. In the
1717 * case of PL_strict_utf8_dfa_tab, this means the input is either
1718 * malformed, or was for certain Hanguls; handle them specially */
1720 /* The dfa above drops out for incomplete or illegal inputs, and certain
1721 * legal Hanguls; check and return accordingly */
1722 return is_HANGUL_ED_utf8_safe(s0, e);
1727 =for apidoc isC9_STRICT_UTF8_CHAR
1729 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
1730 looking no further than S<C<e - 1>> are well-formed UTF-8 that represents some
1731 Unicode non-surrogate code point; otherwise it evaluates to 0. If non-zero,
1732 the value gives how many bytes starting at C<s> comprise the code point's
1733 representation. Any bytes remaining before C<e>, but beyond the ones needed to
1734 form the first code point in C<s>, are not examined.
1736 The largest acceptable code point is the Unicode maximum 0x10FFFF. This
1737 differs from C<L</isSTRICT_UTF8_CHAR>> only in that it accepts non-character
1738 code points. This corresponds to
1739 L<Unicode Corrigendum #9|http://www.unicode.org/versions/corrigendum9.html>.
1740 which said that non-character code points are merely discouraged rather than
1741 completely forbidden in open interchange. See
1742 L<perlunicode/Noncharacter code points>.
1744 Use C<L</isUTF8_CHAR>> to check for Perl's extended UTF-8; and
1745 C<L</isUTF8_CHAR_flags>> for a more customized definition.
1747 Use C<L</is_c9strict_utf8_string>>, C<L</is_c9strict_utf8_string_loc>>, and
1748 C<L</is_c9strict_utf8_string_loclen>> to check entire strings.
1752 This uses an adaptation of the tables and algorithm given in
1753 https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive
1754 documentation of the original version. A copyright notice for the original
1755 version is given at the beginning of this file. The Perl adapation is
1756 documented at the definition of PL_c9_utf8_dfa_tab[].
1760 PERL_STATIC_INLINE Size_t
1761 Perl_isC9_STRICT_UTF8_CHAR(const U8 * const s0, const U8 * const e)
1763 PERL_ARGS_ASSERT_ISC9_STRICT_UTF8_CHAR;
1765 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_c9_utf8_dfa_tab,
1766 DFA_RETURN_SUCCESS_,
1767 DFA_RETURN_FAILURE_,
1768 DFA_RETURN_FAILURE_);
1773 =for apidoc is_strict_utf8_string_loc
1775 Like C<L</is_strict_utf8_string>> but stores the location of the failure (in the
1776 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1777 "utf8ness success") in the C<ep> pointer.
1779 See also C<L</is_strict_utf8_string_loclen>>.
1784 #define is_strict_utf8_string_loc(s, len, ep) \
1785 is_strict_utf8_string_loclen(s, len, ep, 0)
1789 =for apidoc is_strict_utf8_string_loclen
1791 Like C<L</is_strict_utf8_string>> but stores the location of the failure (in the
1792 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1793 "utf8ness success") in the C<ep> pointer, and the number of UTF-8
1794 encoded characters in the C<el> pointer.
1796 See also C<L</is_strict_utf8_string_loc>>.
1801 PERL_STATIC_INLINE bool
1802 Perl_is_strict_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
1804 const U8 * first_variant;
1806 PERL_ARGS_ASSERT_IS_STRICT_UTF8_STRING_LOCLEN;
1809 len = strlen((const char *) s);
1812 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1824 const U8* const send = s + len;
1825 const U8* x = first_variant;
1826 STRLEN outlen = first_variant - s;
1829 const STRLEN cur_len = isSTRICT_UTF8_CHAR(x, send);
1830 if (UNLIKELY(! cur_len)) {
1850 =for apidoc is_c9strict_utf8_string_loc
1852 Like C<L</is_c9strict_utf8_string>> but stores the location of the failure (in
1853 the case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1854 "utf8ness success") in the C<ep> pointer.
1856 See also C<L</is_c9strict_utf8_string_loclen>>.
1861 #define is_c9strict_utf8_string_loc(s, len, ep) \
1862 is_c9strict_utf8_string_loclen(s, len, ep, 0)
1866 =for apidoc is_c9strict_utf8_string_loclen
1868 Like C<L</is_c9strict_utf8_string>> but stores the location of the failure (in
1869 the case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1870 "utf8ness success") in the C<ep> pointer, and the number of UTF-8 encoded
1871 characters in the C<el> pointer.
1873 See also C<L</is_c9strict_utf8_string_loc>>.
1878 PERL_STATIC_INLINE bool
1879 Perl_is_c9strict_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
1881 const U8 * first_variant;
1883 PERL_ARGS_ASSERT_IS_C9STRICT_UTF8_STRING_LOCLEN;
1886 len = strlen((const char *) s);
1889 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
1901 const U8* const send = s + len;
1902 const U8* x = first_variant;
1903 STRLEN outlen = first_variant - s;
1906 const STRLEN cur_len = isC9_STRICT_UTF8_CHAR(x, send);
1907 if (UNLIKELY(! cur_len)) {
1927 =for apidoc is_utf8_string_loc_flags
1929 Like C<L</is_utf8_string_flags>> but stores the location of the failure (in the
1930 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1931 "utf8ness success") in the C<ep> pointer.
1933 See also C<L</is_utf8_string_loclen_flags>>.
1938 #define is_utf8_string_loc_flags(s, len, ep, flags) \
1939 is_utf8_string_loclen_flags(s, len, ep, 0, flags)
1942 /* The above 3 actual functions could have been moved into the more general one
1943 * just below, and made #defines that call it with the right 'flags'. They are
1944 * currently kept separate to increase their chances of getting inlined */
1948 =for apidoc is_utf8_string_loclen_flags
1950 Like C<L</is_utf8_string_flags>> but stores the location of the failure (in the
1951 case of "utf8ness failure") or the location C<s>+C<len> (in the case of
1952 "utf8ness success") in the C<ep> pointer, and the number of UTF-8
1953 encoded characters in the C<el> pointer.
1955 See also C<L</is_utf8_string_loc_flags>>.
1960 PERL_STATIC_INLINE bool
1961 Perl_is_utf8_string_loclen_flags(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el, const U32 flags)
1963 const U8 * first_variant;
1965 PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN_FLAGS;
1966 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
1967 |UTF8_DISALLOW_PERL_EXTENDED)));
1970 len = strlen((const char *) s);
1974 return is_utf8_string_loclen(s, len, ep, el);
1977 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1978 == UTF8_DISALLOW_ILLEGAL_INTERCHANGE)
1980 return is_strict_utf8_string_loclen(s, len, ep, el);
1983 if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED)
1984 == UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE)
1986 return is_c9strict_utf8_string_loclen(s, len, ep, el);
1989 if (is_utf8_invariant_string_loc(s, len, &first_variant)) {
2001 const U8* send = s + len;
2002 const U8* x = first_variant;
2003 STRLEN outlen = first_variant - s;
2006 const STRLEN cur_len = isUTF8_CHAR_flags(x, send, flags);
2007 if (UNLIKELY(! cur_len)) {
2026 =for apidoc utf8_distance
2028 Returns the number of UTF-8 characters between the UTF-8 pointers C<a>
2031 WARNING: use only if you *know* that the pointers point inside the
2037 PERL_STATIC_INLINE IV
2038 Perl_utf8_distance(pTHX_ const U8 *a, const U8 *b)
2040 PERL_ARGS_ASSERT_UTF8_DISTANCE;
2042 return (a < b) ? -1 * (IV) utf8_length(a, b) : (IV) utf8_length(b, a);
2046 =for apidoc utf8_hop
2048 Return the UTF-8 pointer C<s> displaced by C<off> characters, either
2049 forward or backward.
2051 WARNING: do not use the following unless you *know* C<off> is within
2052 the UTF-8 data pointed to by C<s> *and* that on entry C<s> is aligned
2053 on the first byte of character or just after the last byte of a character.
2058 PERL_STATIC_INLINE U8 *
2059 Perl_utf8_hop(const U8 *s, SSize_t off)
2061 PERL_ARGS_ASSERT_UTF8_HOP;
2063 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2064 * the bitops (especially ~) can create illegal UTF-8.
2065 * In other words: in Perl UTF-8 is not just for Unicode. */
2074 while (UTF8_IS_CONTINUATION(*s))
2078 GCC_DIAG_IGNORE(-Wcast-qual)
2084 =for apidoc utf8_hop_forward
2086 Return the UTF-8 pointer C<s> displaced by up to C<off> characters,
2089 C<off> must be non-negative.
2091 C<s> must be before or equal to C<end>.
2093 When moving forward it will not move beyond C<end>.
2095 Will not exceed this limit even if the string is not valid "UTF-8".
2100 PERL_STATIC_INLINE U8 *
2101 Perl_utf8_hop_forward(const U8 *s, SSize_t off, const U8 *end)
2103 PERL_ARGS_ASSERT_UTF8_HOP_FORWARD;
2105 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2106 * the bitops (especially ~) can create illegal UTF-8.
2107 * In other words: in Perl UTF-8 is not just for Unicode. */
2113 STRLEN skip = UTF8SKIP(s);
2114 if ((STRLEN)(end - s) <= skip) {
2115 GCC_DIAG_IGNORE(-Wcast-qual)
2122 GCC_DIAG_IGNORE(-Wcast-qual)
2128 =for apidoc utf8_hop_back
2130 Return the UTF-8 pointer C<s> displaced by up to C<off> characters,
2133 C<off> must be non-positive.
2135 C<s> must be after or equal to C<start>.
2137 When moving backward it will not move before C<start>.
2139 Will not exceed this limit even if the string is not valid "UTF-8".
2144 PERL_STATIC_INLINE U8 *
2145 Perl_utf8_hop_back(const U8 *s, SSize_t off, const U8 *start)
2147 PERL_ARGS_ASSERT_UTF8_HOP_BACK;
2149 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2150 * the bitops (especially ~) can create illegal UTF-8.
2151 * In other words: in Perl UTF-8 is not just for Unicode. */
2156 while (off++ && s > start) {
2159 } while (UTF8_IS_CONTINUATION(*s) && s > start);
2162 GCC_DIAG_IGNORE(-Wcast-qual)
2168 =for apidoc utf8_hop_safe
2170 Return the UTF-8 pointer C<s> displaced by up to C<off> characters,
2171 either forward or backward.
2173 When moving backward it will not move before C<start>.
2175 When moving forward it will not move beyond C<end>.
2177 Will not exceed those limits even if the string is not valid "UTF-8".
2182 PERL_STATIC_INLINE U8 *
2183 Perl_utf8_hop_safe(const U8 *s, SSize_t off, const U8 *start, const U8 *end)
2185 PERL_ARGS_ASSERT_UTF8_HOP_SAFE;
2187 /* Note: cannot use UTF8_IS_...() too eagerly here since e.g
2188 * the bitops (especially ~) can create illegal UTF-8.
2189 * In other words: in Perl UTF-8 is not just for Unicode. */
2191 assert(start <= s && s <= end);
2194 return utf8_hop_forward(s, off, end);
2197 return utf8_hop_back(s, off, start);
2203 =for apidoc isUTF8_CHAR_flags
2205 Evaluates to non-zero if the first few bytes of the string starting at C<s> and
2206 looking no further than S<C<e - 1>> are well-formed UTF-8, as extended by Perl,
2207 that represents some code point, subject to the restrictions given by C<flags>;
2208 otherwise it evaluates to 0. If non-zero, the value gives how many bytes
2209 starting at C<s> comprise the code point's representation. Any bytes remaining
2210 before C<e>, but beyond the ones needed to form the first code point in C<s>,
2213 If C<flags> is 0, this gives the same results as C<L</isUTF8_CHAR>>;
2214 if C<flags> is C<UTF8_DISALLOW_ILLEGAL_INTERCHANGE>, this gives the same results
2215 as C<L</isSTRICT_UTF8_CHAR>>;
2216 and if C<flags> is C<UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE>, this gives
2217 the same results as C<L</isC9_STRICT_UTF8_CHAR>>.
2218 Otherwise C<flags> may be any combination of the C<UTF8_DISALLOW_I<foo>> flags
2219 understood by C<L</utf8n_to_uvchr>>, with the same meanings.
2221 The three alternative macros are for the most commonly needed validations; they
2222 are likely to run somewhat faster than this more general one, as they can be
2223 inlined into your code.
2225 Use L</is_utf8_string_flags>, L</is_utf8_string_loc_flags>, and
2226 L</is_utf8_string_loclen_flags> to check entire strings.
2231 PERL_STATIC_INLINE STRLEN
2232 Perl_isUTF8_CHAR_flags(const U8 * const s0, const U8 * const e, const U32 flags)
2234 PERL_ARGS_ASSERT_ISUTF8_CHAR_FLAGS;
2235 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
2236 |UTF8_DISALLOW_PERL_EXTENDED)));
2238 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab,
2240 DFA_TEASE_APART_FF_,
2241 DFA_RETURN_FAILURE_);
2245 return is_utf8_char_helper_(s0, e, flags);
2247 #ifdef HAS_EXTRA_LONG_UTF8
2251 /* In the case of PL_extended_utf8_dfa_tab, getting here means the input is
2252 * either malformed, or was for the largest possible start byte, which
2253 * indicates perl extended UTF-8, well above the Unicode maximum */
2254 if ( *s0 != I8_TO_NATIVE_UTF8(0xFF)
2255 || (flags & (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_PERL_EXTENDED)))
2260 /* Otherwise examine the sequence not inline */
2261 return is_utf8_FF_helper_(s0, e,
2262 FALSE /* require full, not partial char */
2270 =for apidoc is_utf8_valid_partial_char
2272 Returns 0 if the sequence of bytes starting at C<s> and looking no further than
2273 S<C<e - 1>> is the UTF-8 encoding, as extended by Perl, for one or more code
2274 points. Otherwise, it returns 1 if there exists at least one non-empty
2275 sequence of bytes that when appended to sequence C<s>, starting at position
2276 C<e> causes the entire sequence to be the well-formed UTF-8 of some code point;
2277 otherwise returns 0.
2279 In other words this returns TRUE if C<s> points to a partial UTF-8-encoded code
2282 This is useful when a fixed-length buffer is being tested for being well-formed
2283 UTF-8, but the final few bytes in it don't comprise a full character; that is,
2284 it is split somewhere in the middle of the final code point's UTF-8
2285 representation. (Presumably when the buffer is refreshed with the next chunk
2286 of data, the new first bytes will complete the partial code point.) This
2287 function is used to verify that the final bytes in the current buffer are in
2288 fact the legal beginning of some code point, so that if they aren't, the
2289 failure can be signalled without having to wait for the next read.
2293 #define is_utf8_valid_partial_char(s, e) \
2294 is_utf8_valid_partial_char_flags(s, e, 0)
2298 =for apidoc is_utf8_valid_partial_char_flags
2300 Like C<L</is_utf8_valid_partial_char>>, it returns a boolean giving whether
2301 or not the input is a valid UTF-8 encoded partial character, but it takes an
2302 extra parameter, C<flags>, which can further restrict which code points are
2305 If C<flags> is 0, this behaves identically to
2306 C<L</is_utf8_valid_partial_char>>. Otherwise C<flags> can be any combination
2307 of the C<UTF8_DISALLOW_I<foo>> flags accepted by C<L</utf8n_to_uvchr>>. If
2308 there is any sequence of bytes that can complete the input partial character in
2309 such a way that a non-prohibited character is formed, the function returns
2310 TRUE; otherwise FALSE. Non character code points cannot be determined based on
2311 partial character input. But many of the other possible excluded types can be
2312 determined from just the first one or two bytes.
2317 PERL_STATIC_INLINE bool
2318 Perl_is_utf8_valid_partial_char_flags(const U8 * const s0, const U8 * const e, const U32 flags)
2320 PERL_ARGS_ASSERT_IS_UTF8_VALID_PARTIAL_CHAR_FLAGS;
2321 assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE
2322 |UTF8_DISALLOW_PERL_EXTENDED)));
2324 PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab,
2325 DFA_RETURN_FAILURE_,
2326 DFA_TEASE_APART_FF_,
2329 /* The NOOP above causes the DFA to drop down here iff the input was a
2330 * partial character. flags=0 => can return TRUE immediately; otherwise we
2331 * need to check (not inline) if the partial character is the beginning of
2332 * a disallowed one */
2337 return cBOOL(is_utf8_char_helper_(s0, e, flags));
2339 #ifdef HAS_EXTRA_LONG_UTF8
2343 /* Getting here means the input is either malformed, or, in the case of
2344 * PL_extended_utf8_dfa_tab, was for the largest possible start byte. The
2345 * latter case has to be extended UTF-8, so can fail immediately if that is
2348 if ( *s0 != I8_TO_NATIVE_UTF8(0xFF)
2349 || (flags & (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_PERL_EXTENDED)))
2354 return is_utf8_FF_helper_(s0, e,
2355 TRUE /* Require to be a partial character */
2363 =for apidoc is_utf8_fixed_width_buf_flags
2365 Returns TRUE if the fixed-width buffer starting at C<s> with length C<len>
2366 is entirely valid UTF-8, subject to the restrictions given by C<flags>;
2367 otherwise it returns FALSE.
2369 If C<flags> is 0, any well-formed UTF-8, as extended by Perl, is accepted
2370 without restriction. If the final few bytes of the buffer do not form a
2371 complete code point, this will return TRUE anyway, provided that
2372 C<L</is_utf8_valid_partial_char_flags>> returns TRUE for them.
2374 If C<flags> in non-zero, it can be any combination of the
2375 C<UTF8_DISALLOW_I<foo>> flags accepted by C<L</utf8n_to_uvchr>>, and with the
2378 This function differs from C<L</is_utf8_string_flags>> only in that the latter
2379 returns FALSE if the final few bytes of the string don't form a complete code
2384 #define is_utf8_fixed_width_buf_flags(s, len, flags) \
2385 is_utf8_fixed_width_buf_loclen_flags(s, len, 0, 0, flags)
2389 =for apidoc is_utf8_fixed_width_buf_loc_flags
2391 Like C<L</is_utf8_fixed_width_buf_flags>> but stores the location of the
2392 failure in the C<ep> pointer. If the function returns TRUE, C<*ep> will point
2393 to the beginning of any partial character at the end of the buffer; if there is
2394 no partial character C<*ep> will contain C<s>+C<len>.
2396 See also C<L</is_utf8_fixed_width_buf_loclen_flags>>.
2401 #define is_utf8_fixed_width_buf_loc_flags(s, len, loc, flags) \
2402 is_utf8_fixed_width_buf_loclen_flags(s, len, loc, 0, flags)
2406 =for apidoc is_utf8_fixed_width_buf_loclen_flags
2408 Like C<L</is_utf8_fixed_width_buf_loc_flags>> but stores the number of
2409 complete, valid characters found in the C<el> pointer.
2414 PERL_STATIC_INLINE bool
2415 Perl_is_utf8_fixed_width_buf_loclen_flags(const U8 * const s,
2421 const U8 * maybe_partial;
2423 PERL_ARGS_ASSERT_IS_UTF8_FIXED_WIDTH_BUF_LOCLEN_FLAGS;
2426 ep = &maybe_partial;
2429 /* If it's entirely valid, return that; otherwise see if the only error is
2430 * that the final few bytes are for a partial character */
2431 return is_utf8_string_loclen_flags(s, len, ep, el, flags)
2432 || is_utf8_valid_partial_char_flags(*ep, s + len, flags);
2435 PERL_STATIC_INLINE UV
2436 Perl_utf8n_to_uvchr_msgs(const U8 *s,
2443 /* This is the inlined portion of utf8n_to_uvchr_msgs. It handles the
2444 * simple cases, and, if necessary calls a helper function to deal with the
2445 * more complex ones. Almost all well-formed non-problematic code points
2446 * are considered simple, so that it's unlikely that the helper function
2447 * will need to be called.
2449 * This is an adaptation of the tables and algorithm given in
2450 * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides
2451 * comprehensive documentation of the original version. A copyright notice
2452 * for the original version is given at the beginning of this file. The
2453 * Perl adapation is documented at the definition of PL_strict_utf8_dfa_tab[].
2456 const U8 * const s0 = s;
2457 const U8 * send = s0 + curlen;
2458 UV uv = 0; /* The 0 silences some stupid compilers */
2461 PERL_ARGS_ASSERT_UTF8N_TO_UVCHR_MSGS;
2463 /* This dfa is fast. If it accepts the input, it was for a well-formed,
2464 * non-problematic code point, which can be returned immediately.
2465 * Otherwise we call a helper function to figure out the more complicated
2468 while (s < send && LIKELY(state != 1)) {
2469 UV type = PL_strict_utf8_dfa_tab[*s];
2472 ? ((0xff >> type) & NATIVE_UTF8_TO_I8(*s))
2473 : UTF8_ACCUMULATE(uv, *s);
2474 state = PL_strict_utf8_dfa_tab[256 + state + type];
2482 *retlen = s - s0 + 1;
2491 return UNI_TO_NATIVE(uv);
2494 /* Here is potentially problematic. Use the full mechanism */
2495 return _utf8n_to_uvchr_msgs_helper(s0, curlen, retlen, flags, errors, msgs);
2498 PERL_STATIC_INLINE UV
2499 Perl_utf8_to_uvchr_buf_helper(pTHX_ const U8 *s, const U8 *send, STRLEN *retlen)
2501 PERL_ARGS_ASSERT_UTF8_TO_UVCHR_BUF_HELPER;
2505 if (! ckWARN_d(WARN_UTF8)) {
2507 /* EMPTY is not really allowed, and asserts on debugging builds. But
2508 * on non-debugging we have to deal with it, and this causes it to
2509 * return the REPLACEMENT CHARACTER, as the documentation indicates */
2510 return utf8n_to_uvchr(s, send - s, retlen,
2511 (UTF8_ALLOW_ANY | UTF8_ALLOW_EMPTY));
2514 UV ret = utf8n_to_uvchr(s, send - s, retlen, 0);
2515 if (retlen && ret == 0 && *s != '\0') {
2516 *retlen = (STRLEN) -1;
2523 /* ------------------------------- perl.h ----------------------------- */
2526 =for apidoc_section $utility
2528 =for apidoc is_safe_syscall
2530 Test that the given C<pv> (with length C<len>) doesn't contain any internal
2532 If it does, set C<errno> to C<ENOENT>, optionally warn using the C<syscalls>
2533 category, and return FALSE.
2535 Return TRUE if the name is safe.
2537 C<what> and C<op_name> are used in any warning.
2539 Used by the C<IS_SAFE_SYSCALL()> macro.
2544 PERL_STATIC_INLINE bool
2545 Perl_is_safe_syscall(pTHX_ const char *pv, STRLEN len, const char *what, const char *op_name)
2547 /* While the Windows CE API provides only UCS-16 (or UTF-16) APIs
2548 * perl itself uses xce*() functions which accept 8-bit strings.
2551 PERL_ARGS_ASSERT_IS_SAFE_SYSCALL;
2555 if (UNLIKELY((null_at = (char *)memchr(pv, 0, len-1)) != NULL)) {
2556 SETERRNO(ENOENT, LIB_INVARG);
2557 Perl_ck_warner(aTHX_ packWARN(WARN_SYSCALLS),
2558 "Invalid \\0 character in %s for %s: %s\\0%s",
2559 what, op_name, pv, null_at+1);
2569 Return true if the supplied filename has a newline character
2570 immediately before the first (hopefully only) NUL.
2572 My original look at this incorrectly used the len from SvPV(), but
2573 that's incorrect, since we allow for a NUL in pv[len-1].
2575 So instead, strlen() and work from there.
2577 This allow for the user reading a filename, forgetting to chomp it,
2580 open my $foo, "$file\0";
2586 PERL_STATIC_INLINE bool
2587 S_should_warn_nl(const char *pv)
2591 PERL_ARGS_ASSERT_SHOULD_WARN_NL;
2595 return len > 0 && pv[len-1] == '\n';
2600 #if defined(PERL_IN_PP_C) || defined(PERL_IN_PP_HOT_C)
2602 PERL_STATIC_INLINE bool
2603 S_lossless_NV_to_IV(const NV nv, IV *ivp)
2605 /* This function determines if the input NV 'nv' may be converted without
2606 * loss of data to an IV. If not, it returns FALSE taking no other action.
2607 * But if it is possible, it does the conversion, returning TRUE, and
2608 * storing the converted result in '*ivp' */
2610 PERL_ARGS_ASSERT_LOSSLESS_NV_TO_IV;
2612 # if defined(NAN_COMPARE_BROKEN) && defined(Perl_isnan)
2613 /* Normally any comparison with a NaN returns false; if we can't rely
2614 * on that behaviour, check explicitly */
2615 if (UNLIKELY(Perl_isnan(nv))) {
2620 /* Written this way so that with an always-false NaN comparison we
2622 if (!(LIKELY(nv >= (NV) IV_MIN) && LIKELY(nv < IV_MAX_P1))) {
2626 if ((IV) nv != nv) {
2636 /* ------------------ pp.c, regcomp.c, toke.c, universal.c ------------ */
2638 #if defined(PERL_IN_PP_C) || defined(PERL_IN_REGCOMP_C) || defined(PERL_IN_TOKE_C) || defined(PERL_IN_UNIVERSAL_C)
2640 #define MAX_CHARSET_NAME_LENGTH 2
2642 PERL_STATIC_INLINE const char *
2643 S_get_regex_charset_name(const U32 flags, STRLEN* const lenp)
2645 PERL_ARGS_ASSERT_GET_REGEX_CHARSET_NAME;
2647 /* Returns a string that corresponds to the name of the regex character set
2648 * given by 'flags', and *lenp is set the length of that string, which
2649 * cannot exceed MAX_CHARSET_NAME_LENGTH characters */
2652 switch (get_regex_charset(flags)) {
2653 case REGEX_DEPENDS_CHARSET: return DEPENDS_PAT_MODS;
2654 case REGEX_LOCALE_CHARSET: return LOCALE_PAT_MODS;
2655 case REGEX_UNICODE_CHARSET: return UNICODE_PAT_MODS;
2656 case REGEX_ASCII_RESTRICTED_CHARSET: return ASCII_RESTRICT_PAT_MODS;
2657 case REGEX_ASCII_MORE_RESTRICTED_CHARSET:
2659 return ASCII_MORE_RESTRICT_PAT_MODS;
2661 /* The NOT_REACHED; hides an assert() which has a rather complex
2662 * definition in perl.h. */
2663 NOT_REACHED; /* NOTREACHED */
2664 return "?"; /* Unknown */
2671 Return false if any get magic is on the SV other than taint magic.
2675 PERL_STATIC_INLINE bool
2676 Perl_sv_only_taint_gmagic(SV *sv)
2678 MAGIC *mg = SvMAGIC(sv);
2680 PERL_ARGS_ASSERT_SV_ONLY_TAINT_GMAGIC;
2683 if (mg->mg_type != PERL_MAGIC_taint
2684 && !(mg->mg_flags & MGf_GSKIP)
2685 && mg->mg_virtual->svt_get) {
2688 mg = mg->mg_moremagic;
2694 /* ------------------ cop.h ------------------------------------------- */
2696 /* implement GIMME_V() macro */
2698 PERL_STATIC_INLINE U8
2702 U8 gimme = (PL_op->op_flags & OPf_WANT);
2706 cxix = PL_curstackinfo->si_cxsubix;
2708 return PL_curstackinfo->si_type == PERLSI_SORT ? G_SCALAR: G_VOID;
2709 assert(cxstack[cxix].blk_gimme & G_WANT);
2710 return (cxstack[cxix].blk_gimme & G_WANT);
2714 /* Enter a block. Push a new base context and return its address. */
2716 PERL_STATIC_INLINE PERL_CONTEXT *
2717 Perl_cx_pushblock(pTHX_ U8 type, U8 gimme, SV** sp, I32 saveix)
2721 PERL_ARGS_ASSERT_CX_PUSHBLOCK;
2726 cx->blk_gimme = gimme;
2727 cx->blk_oldsaveix = saveix;
2728 cx->blk_oldsp = (I32)(sp - PL_stack_base);
2729 cx->blk_oldcop = PL_curcop;
2730 cx->blk_oldmarksp = (I32)(PL_markstack_ptr - PL_markstack);
2731 cx->blk_oldscopesp = PL_scopestack_ix;
2732 cx->blk_oldpm = PL_curpm;
2733 cx->blk_old_tmpsfloor = PL_tmps_floor;
2735 PL_tmps_floor = PL_tmps_ix;
2736 CX_DEBUG(cx, "PUSH");
2741 /* Exit a block (RETURN and LAST). */
2743 PERL_STATIC_INLINE void
2744 Perl_cx_popblock(pTHX_ PERL_CONTEXT *cx)
2746 PERL_ARGS_ASSERT_CX_POPBLOCK;
2748 CX_DEBUG(cx, "POP");
2749 /* these 3 are common to cx_popblock and cx_topblock */
2750 PL_markstack_ptr = PL_markstack + cx->blk_oldmarksp;
2751 PL_scopestack_ix = cx->blk_oldscopesp;
2752 PL_curpm = cx->blk_oldpm;
2754 /* LEAVE_SCOPE() should have made this true. /(?{})/ cheats
2755 * and leaves a CX entry lying around for repeated use, so
2756 * skip for multicall */ \
2757 assert( (CxTYPE(cx) == CXt_SUB && CxMULTICALL(cx))
2758 || PL_savestack_ix == cx->blk_oldsaveix);
2759 PL_curcop = cx->blk_oldcop;
2760 PL_tmps_floor = cx->blk_old_tmpsfloor;
2763 /* Continue a block elsewhere (e.g. NEXT, REDO, GOTO).
2764 * Whereas cx_popblock() restores the state to the point just before
2765 * cx_pushblock() was called, cx_topblock() restores it to the point just
2766 * *after* cx_pushblock() was called. */
2768 PERL_STATIC_INLINE void
2769 Perl_cx_topblock(pTHX_ PERL_CONTEXT *cx)
2771 PERL_ARGS_ASSERT_CX_TOPBLOCK;
2773 CX_DEBUG(cx, "TOP");
2774 /* these 3 are common to cx_popblock and cx_topblock */
2775 PL_markstack_ptr = PL_markstack + cx->blk_oldmarksp;
2776 PL_scopestack_ix = cx->blk_oldscopesp;
2777 PL_curpm = cx->blk_oldpm;
2779 PL_stack_sp = PL_stack_base + cx->blk_oldsp;
2783 PERL_STATIC_INLINE void
2784 Perl_cx_pushsub(pTHX_ PERL_CONTEXT *cx, CV *cv, OP *retop, bool hasargs)
2786 U8 phlags = CX_PUSHSUB_GET_LVALUE_MASK(Perl_was_lvalue_sub);
2788 PERL_ARGS_ASSERT_CX_PUSHSUB;
2790 PERL_DTRACE_PROBE_ENTRY(cv);
2791 cx->blk_sub.old_cxsubix = PL_curstackinfo->si_cxsubix;
2792 PL_curstackinfo->si_cxsubix = cx - PL_curstackinfo->si_cxstack;
2793 cx->blk_sub.cv = cv;
2794 cx->blk_sub.olddepth = CvDEPTH(cv);
2795 cx->blk_sub.prevcomppad = PL_comppad;
2796 cx->cx_type |= (hasargs) ? CXp_HASARGS : 0;
2797 cx->blk_sub.retop = retop;
2798 SvREFCNT_inc_simple_void_NN(cv);
2799 cx->blk_u16 = PL_op->op_private & (phlags|OPpDEREF);
2803 /* subsets of cx_popsub() */
2805 PERL_STATIC_INLINE void
2806 Perl_cx_popsub_common(pTHX_ PERL_CONTEXT *cx)
2810 PERL_ARGS_ASSERT_CX_POPSUB_COMMON;
2811 assert(CxTYPE(cx) == CXt_SUB);
2813 PL_comppad = cx->blk_sub.prevcomppad;
2814 PL_curpad = LIKELY(PL_comppad) ? AvARRAY(PL_comppad) : NULL;
2815 cv = cx->blk_sub.cv;
2816 CvDEPTH(cv) = cx->blk_sub.olddepth;
2817 cx->blk_sub.cv = NULL;
2819 PL_curstackinfo->si_cxsubix = cx->blk_sub.old_cxsubix;
2823 /* handle the @_ part of leaving a sub */
2825 PERL_STATIC_INLINE void
2826 Perl_cx_popsub_args(pTHX_ PERL_CONTEXT *cx)
2830 PERL_ARGS_ASSERT_CX_POPSUB_ARGS;
2831 assert(CxTYPE(cx) == CXt_SUB);
2832 assert(AvARRAY(MUTABLE_AV(
2833 PadlistARRAY(CvPADLIST(cx->blk_sub.cv))[
2834 CvDEPTH(cx->blk_sub.cv)])) == PL_curpad);
2836 CX_POP_SAVEARRAY(cx);
2837 av = MUTABLE_AV(PAD_SVl(0));
2838 if (UNLIKELY(AvREAL(av)))
2839 /* abandon @_ if it got reified */
2840 clear_defarray(av, 0);
2847 PERL_STATIC_INLINE void
2848 Perl_cx_popsub(pTHX_ PERL_CONTEXT *cx)
2850 PERL_ARGS_ASSERT_CX_POPSUB;
2851 assert(CxTYPE(cx) == CXt_SUB);
2853 PERL_DTRACE_PROBE_RETURN(cx->blk_sub.cv);
2857 cx_popsub_common(cx);
2861 PERL_STATIC_INLINE void
2862 Perl_cx_pushformat(pTHX_ PERL_CONTEXT *cx, CV *cv, OP *retop, GV *gv)
2864 PERL_ARGS_ASSERT_CX_PUSHFORMAT;
2866 cx->blk_format.old_cxsubix = PL_curstackinfo->si_cxsubix;
2867 PL_curstackinfo->si_cxsubix= cx - PL_curstackinfo->si_cxstack;
2868 cx->blk_format.cv = cv;
2869 cx->blk_format.retop = retop;
2870 cx->blk_format.gv = gv;
2871 cx->blk_format.dfoutgv = PL_defoutgv;
2872 cx->blk_format.prevcomppad = PL_comppad;
2875 SvREFCNT_inc_simple_void_NN(cv);
2877 SvREFCNT_inc_void(cx->blk_format.dfoutgv);
2881 PERL_STATIC_INLINE void
2882 Perl_cx_popformat(pTHX_ PERL_CONTEXT *cx)
2887 PERL_ARGS_ASSERT_CX_POPFORMAT;
2888 assert(CxTYPE(cx) == CXt_FORMAT);
2890 dfout = cx->blk_format.dfoutgv;
2892 cx->blk_format.dfoutgv = NULL;
2893 SvREFCNT_dec_NN(dfout);
2895 PL_comppad = cx->blk_format.prevcomppad;
2896 PL_curpad = LIKELY(PL_comppad) ? AvARRAY(PL_comppad) : NULL;
2897 cv = cx->blk_format.cv;
2898 cx->blk_format.cv = NULL;
2900 SvREFCNT_dec_NN(cv);
2901 PL_curstackinfo->si_cxsubix = cx->blk_format.old_cxsubix;
2905 PERL_STATIC_INLINE void
2906 Perl_push_evalortry_common(pTHX_ PERL_CONTEXT *cx, OP *retop, SV *namesv)
2908 cx->blk_eval.retop = retop;
2909 cx->blk_eval.old_namesv = namesv;
2910 cx->blk_eval.old_eval_root = PL_eval_root;
2911 cx->blk_eval.cur_text = PL_parser ? PL_parser->linestr : NULL;
2912 cx->blk_eval.cv = NULL; /* later set by doeval_compile() */
2913 cx->blk_eval.cur_top_env = PL_top_env;
2915 assert(!(PL_in_eval & ~ 0x3F));
2916 assert(!(PL_op->op_type & ~0x1FF));
2917 cx->blk_u16 = (PL_in_eval & 0x3F) | ((U16)PL_op->op_type << 7);
2920 PERL_STATIC_INLINE void
2921 Perl_cx_pusheval(pTHX_ PERL_CONTEXT *cx, OP *retop, SV *namesv)
2923 PERL_ARGS_ASSERT_CX_PUSHEVAL;
2925 Perl_push_evalortry_common(aTHX_ cx, retop, namesv);
2927 cx->blk_eval.old_cxsubix = PL_curstackinfo->si_cxsubix;
2928 PL_curstackinfo->si_cxsubix = cx - PL_curstackinfo->si_cxstack;
2931 PERL_STATIC_INLINE void
2932 Perl_cx_pushtry(pTHX_ PERL_CONTEXT *cx, OP *retop)
2934 PERL_ARGS_ASSERT_CX_PUSHTRY;
2936 Perl_push_evalortry_common(aTHX_ cx, retop, NULL);
2938 /* Don't actually change it, just store the current value so it's restored
2939 * by the common popeval */
2940 cx->blk_eval.old_cxsubix = PL_curstackinfo->si_cxsubix;
2944 PERL_STATIC_INLINE void
2945 Perl_cx_popeval(pTHX_ PERL_CONTEXT *cx)
2949 PERL_ARGS_ASSERT_CX_POPEVAL;
2950 assert(CxTYPE(cx) == CXt_EVAL);
2952 PL_in_eval = CxOLD_IN_EVAL(cx);
2953 assert(!(PL_in_eval & 0xc0));
2954 PL_eval_root = cx->blk_eval.old_eval_root;
2955 sv = cx->blk_eval.cur_text;
2956 if (sv && CxEVAL_TXT_REFCNTED(cx)) {
2957 cx->blk_eval.cur_text = NULL;
2958 SvREFCNT_dec_NN(sv);
2961 sv = cx->blk_eval.old_namesv;
2963 cx->blk_eval.old_namesv = NULL;
2964 SvREFCNT_dec_NN(sv);
2966 PL_curstackinfo->si_cxsubix = cx->blk_eval.old_cxsubix;
2970 /* push a plain loop, i.e.
2972 * while (cond) { block }
2973 * for (init;cond;continue) { block }
2974 * This loop can be last/redo'ed etc.
2977 PERL_STATIC_INLINE void
2978 Perl_cx_pushloop_plain(pTHX_ PERL_CONTEXT *cx)
2980 PERL_ARGS_ASSERT_CX_PUSHLOOP_PLAIN;
2981 cx->blk_loop.my_op = cLOOP;
2985 /* push a true for loop, i.e.
2986 * for var (list) { block }
2989 PERL_STATIC_INLINE void
2990 Perl_cx_pushloop_for(pTHX_ PERL_CONTEXT *cx, void *itervarp, SV* itersave)
2992 PERL_ARGS_ASSERT_CX_PUSHLOOP_FOR;
2994 /* this one line is common with cx_pushloop_plain */
2995 cx->blk_loop.my_op = cLOOP;
2997 cx->blk_loop.itervar_u.svp = (SV**)itervarp;
2998 cx->blk_loop.itersave = itersave;
3000 cx->blk_loop.oldcomppad = PL_comppad;
3005 /* pop all loop types, including plain */
3007 PERL_STATIC_INLINE void
3008 Perl_cx_poploop(pTHX_ PERL_CONTEXT *cx)
3010 PERL_ARGS_ASSERT_CX_POPLOOP;
3012 assert(CxTYPE_is_LOOP(cx));
3013 if ( CxTYPE(cx) == CXt_LOOP_ARY
3014 || CxTYPE(cx) == CXt_LOOP_LAZYSV)
3016 /* Free ary or cur. This assumes that state_u.ary.ary
3017 * aligns with state_u.lazysv.cur. See cx_dup() */
3018 SV *sv = cx->blk_loop.state_u.lazysv.cur;
3019 cx->blk_loop.state_u.lazysv.cur = NULL;
3020 SvREFCNT_dec_NN(sv);
3021 if (CxTYPE(cx) == CXt_LOOP_LAZYSV) {
3022 sv = cx->blk_loop.state_u.lazysv.end;
3023 cx->blk_loop.state_u.lazysv.end = NULL;
3024 SvREFCNT_dec_NN(sv);
3027 if (cx->cx_type & (CXp_FOR_PAD|CXp_FOR_GV)) {
3029 SV **svp = (cx)->blk_loop.itervar_u.svp;
3030 if ((cx->cx_type & CXp_FOR_GV))
3031 svp = &GvSV((GV*)svp);
3033 *svp = cx->blk_loop.itersave;
3034 cx->blk_loop.itersave = NULL;
3035 SvREFCNT_dec(cursv);
3040 PERL_STATIC_INLINE void
3041 Perl_cx_pushwhen(pTHX_ PERL_CONTEXT *cx)
3043 PERL_ARGS_ASSERT_CX_PUSHWHEN;
3045 cx->blk_givwhen.leave_op = cLOGOP->op_other;
3049 PERL_STATIC_INLINE void
3050 Perl_cx_popwhen(pTHX_ PERL_CONTEXT *cx)
3052 PERL_ARGS_ASSERT_CX_POPWHEN;
3053 assert(CxTYPE(cx) == CXt_WHEN);
3055 PERL_UNUSED_ARG(cx);
3056 PERL_UNUSED_CONTEXT;
3057 /* currently NOOP */
3061 PERL_STATIC_INLINE void
3062 Perl_cx_pushgiven(pTHX_ PERL_CONTEXT *cx, SV *orig_defsv)
3064 PERL_ARGS_ASSERT_CX_PUSHGIVEN;
3066 cx->blk_givwhen.leave_op = cLOGOP->op_other;
3067 cx->blk_givwhen.defsv_save = orig_defsv;
3071 PERL_STATIC_INLINE void
3072 Perl_cx_popgiven(pTHX_ PERL_CONTEXT *cx)
3076 PERL_ARGS_ASSERT_CX_POPGIVEN;
3077 assert(CxTYPE(cx) == CXt_GIVEN);
3079 sv = GvSV(PL_defgv);
3080 GvSV(PL_defgv) = cx->blk_givwhen.defsv_save;
3081 cx->blk_givwhen.defsv_save = NULL;
3085 /* ------------------ util.h ------------------------------------------- */
3088 =for apidoc_section $string
3092 Returns true if the leading C<len> bytes of the strings C<s1> and C<s2> are the
3094 case-insensitively; false otherwise. Uppercase and lowercase ASCII range bytes
3095 match themselves and their opposite case counterparts. Non-cased and non-ASCII
3096 range bytes match only themselves.
3101 PERL_STATIC_INLINE I32
3102 Perl_foldEQ(const char *s1, const char *s2, I32 len)
3104 const U8 *a = (const U8 *)s1;
3105 const U8 *b = (const U8 *)s2;
3107 PERL_ARGS_ASSERT_FOLDEQ;
3112 if (*a != *b && *a != PL_fold[*b])
3119 PERL_STATIC_INLINE I32
3120 Perl_foldEQ_latin1(const char *s1, const char *s2, I32 len)
3122 /* Compare non-UTF-8 using Unicode (Latin1) semantics. Works on all folds
3123 * representable without UTF-8, except for LATIN_SMALL_LETTER_SHARP_S, and
3124 * does not check for this. Nor does it check that the strings each have
3125 * at least 'len' characters. */
3127 const U8 *a = (const U8 *)s1;
3128 const U8 *b = (const U8 *)s2;
3130 PERL_ARGS_ASSERT_FOLDEQ_LATIN1;
3135 if (*a != *b && *a != PL_fold_latin1[*b]) {
3144 =for apidoc_section $locale
3145 =for apidoc foldEQ_locale
3147 Returns true if the leading C<len> bytes of the strings C<s1> and C<s2> are the
3148 same case-insensitively in the current locale; false otherwise.
3153 PERL_STATIC_INLINE I32
3154 Perl_foldEQ_locale(const char *s1, const char *s2, I32 len)
3156 const U8 *a = (const U8 *)s1;
3157 const U8 *b = (const U8 *)s2;
3159 PERL_ARGS_ASSERT_FOLDEQ_LOCALE;
3164 if (*a != *b && *a != PL_fold_locale[*b])
3172 =for apidoc_section $string
3173 =for apidoc my_strnlen
3175 The C library C<strnlen> if available, or a Perl implementation of it.
3177 C<my_strnlen()> computes the length of the string, up to C<maxlen>
3178 characters. It will never attempt to address more than C<maxlen>
3179 characters, making it suitable for use with strings that are not
3180 guaranteed to be NUL-terminated.
3184 Description stolen from http://man.openbsd.org/strnlen.3,
3185 implementation stolen from PostgreSQL.
3189 PERL_STATIC_INLINE Size_t
3190 Perl_my_strnlen(const char *str, Size_t maxlen)
3192 const char *end = (char *) memchr(str, '\0', maxlen);
3194 PERL_ARGS_ASSERT_MY_STRNLEN;
3196 if (end == NULL) return maxlen;
3202 #if ! defined (HAS_MEMRCHR) && (defined(PERL_CORE) || defined(PERL_EXT))
3204 PERL_STATIC_INLINE void *
3205 S_my_memrchr(const char * s, const char c, const STRLEN len)
3207 /* memrchr(), since many platforms lack it */
3209 const char * t = s + len - 1;
3211 PERL_ARGS_ASSERT_MY_MEMRCHR;
3225 PERL_STATIC_INLINE char *
3226 Perl_mortal_getenv(const char * str)
3228 /* This implements a (mostly) thread-safe, sequential-call-safe getenv().
3230 * It's (mostly) thread-safe because it uses a mutex to prevent other
3231 * threads (that look at this mutex) from destroying the result before this
3232 * routine has a chance to copy the result to a place that won't be
3233 * destroyed before the caller gets a chance to handle it. That place is a
3234 * mortal SV. khw chose this over SAVEFREEPV because he is under the
3235 * impression that the SV will hang around longer under more circumstances
3237 * The reason it isn't completely thread-safe is that other code could
3238 * simply not pay attention to the mutex. All of the Perl core uses the
3239 * mutex, but it is possible for code from, say XS, to not use this mutex,
3240 * defeating the safety.
3242 * getenv() returns, in some implementations, a pointer to a spot in the
3243 * **environ array, which could be invalidated at any time by this or
3244 * another thread changing the environment. Other implementations copy the
3245 * **environ value to a static buffer, returning a pointer to that. That
3246 * buffer might or might not be invalidated by a getenv() call in another
3247 * thread. If it does get zapped, we need an exclusive lock. Otherwise,
3248 * many getenv() calls can safely be running simultaneously, so a
3249 * many-reader (but no simultaneous writers) lock is ok. There is a
3250 * Configure probe to see if another thread destroys the buffer, and the
3251 * mutex is defined accordingly.
3253 * But in all cases, using the mutex prevents these problems, as long as
3254 * all code uses the same mutex..
3256 * A complication is that this can be called during phases where the
3257 * mortalization process isn't available. These are in interpreter
3258 * destruction or early in construction. khw believes that at these times
3259 * there shouldn't be anything else going on, so plain getenv is safe AS
3260 * LONG AS the caller acts on the return before calling it again. */
3265 PERL_ARGS_ASSERT_MORTAL_GETENV;
3267 /* Can't mortalize without stacks. khw believes that no other threads
3268 * should be running, so no need to lock things, and this may be during a
3269 * phase when locking isn't even available */
3270 if (UNLIKELY(PL_scopestack_ix == 0)) {
3276 /* A major complication arises under PERL_MEM_LOG. When that is active,
3277 * every memory allocation may result in logging, depending on the value of
3278 * ENV{PERL_MEM_LOG} at the moment. That means, as we create the SV for
3279 * saving ENV{foo}'s value (but before saving it), the logging code will
3280 * call us recursively to find out what ENV{PERL_MEM_LOG} is. Without some
3281 * care that could lead to: 1) infinite recursion; or 2) deadlock (trying to
3282 * lock a boolean mutex recursively); 3) destroying the getenv() static
3283 * buffer; or 4) destroying the temporary created by this for the copy
3284 * causes a log entry to be made which could cause a new temporary to be
3285 * created, which will need to be destroyed at some point, leading to an
3288 * The solution adopted here (after some gnashing of teeth) is to detect
3289 * the recursive calls and calls from the logger, and treat them specially.
3290 * Let's say we want to do getenv("foo"). We first find
3291 * getenv(PERL_MEM_LOG) and save it to a fixed-length per-interpreter
3292 * variable, so no temporary is required. Then we do getenv(foo}, and in
3293 * the process of creating a temporary to save it, this function will be
3294 * called recursively to do a getenv(PERL_MEM_LOG). On the recursed call,
3295 * we detect that it is such a call and return our saved value instead of
3296 * locking and doing a new getenv(). This solves all of problems 1), 2),
3297 * and 3). Because all the getenv()s are done while the mutex is locked,
3298 * the state cannot have changed. To solve 4), we don't create a temporary
3299 * when this is called from the logging code. That code disposes of the
3300 * return value while the mutex is still locked.
3302 * The value of getenv(PERL_MEM_LOG) can be anything, but only initial
3303 * digits and 3 particular letters are significant; the rest are ignored by
3304 * the memory logging code. Thus the per-interpreter variable only needs
3305 * to be large enough to save the significant information, the size of
3306 * which is known at compile time. The first byte is extra, reserved for
3307 * flags for our use. To protect against overflowing, only the reserved
3308 * byte, as many digits as don't overflow, and the three letters are
3311 * The reserved byte has two bits:
3312 * 0x1 if set indicates that if we get here, it is a recursive call of
3314 * 0x2 if set indicates that the call is from the logging code.
3316 * If the flag indicates this is a recursive call, just return the stored
3317 * value of PL_mem_log; An empty value gets turned into NULL. */
3318 if (strEQ(str, "PERL_MEM_LOG") && PL_mem_log[0] & 0x1) {
3319 if (PL_mem_log[1] == '\0') {
3322 return PL_mem_log + 1;
3332 /* Here we are in a critical section. As explained above, we do our own
3333 * getenv(PERL_MEM_LOG), saving the result safely. */
3334 ret = getenv("PERL_MEM_LOG");
3335 if (ret == NULL) { /* No logging active */
3337 /* Return that immediately if called from the logging code */
3338 if (PL_mem_log[0] & 0x2) {
3343 PL_mem_log[1] = '\0';
3346 char *mem_log_meat = PL_mem_log + 1; /* first byte reserved */
3348 /* There is nothing to prevent the value of PERL_MEM_LOG from being an
3349 * extremely long string. But we want only a few characters from it.
3350 * PL_mem_log has been made large enough to hold just the ones we need.
3351 * First the file descriptor. */
3352 if (isDIGIT(*ret)) {
3353 const char * s = ret;
3354 if (UNLIKELY(*s == '0')) {
3356 /* Reduce multiple leading zeros to a single one. This is to
3357 * allow the caller to change what to do with leading zeros. */
3358 *mem_log_meat++ = '0';
3365 /* If the input overflows, copy just enough for the result to also
3366 * overflow, plus 1 to make sure */
3367 while (isDIGIT(*s) && s < ret + TYPE_DIGITS(UV) + 1) {
3368 *mem_log_meat++ = *s++;
3372 /* Then each of the three significant characters */
3373 if (strchr(ret, 'm')) {
3374 *mem_log_meat++ = 'm';
3376 if (strchr(ret, 's')) {
3377 *mem_log_meat++ = 's';
3379 if (strchr(ret, 't')) {
3380 *mem_log_meat++ = 't';
3382 *mem_log_meat = '\0';
3384 assert(mem_log_meat < PL_mem_log + sizeof(PL_mem_log));
3387 /* If we are being called from the logger, it only needs the significant
3388 * portion of PERL_MEM_LOG, and doesn't need a safe copy */
3389 if (PL_mem_log[0] & 0x2) {
3390 assert(strEQ(str, "PERL_MEM_LOG"));
3392 return PL_mem_log + 1;
3395 /* Here is a generic getenv(). This could be a getenv("PERL_MEM_LOG") that
3396 * is coming from other than the logging code, so it should be treated the
3397 * same as any other getenv(), returning the full value, not just the
3398 * significant part, and having its value saved. Set the flag that
3399 * indicates any call to this routine will be a recursion from here */
3400 PL_mem_log[0] = 0x1;
3404 /* Now get the value of the real desired variable, and save a copy */
3408 ret = SvPVX(sv_2mortal(newSVpv(ret, 0)));
3415 /* Clear the buffer */
3416 Zero(PL_mem_log, sizeof(PL_mem_log), char);
3423 PERL_STATIC_INLINE bool
3424 Perl_sv_isbool(pTHX_ const SV *sv)
3426 return SvIOK(sv) && SvPOK(sv) && SvIsCOW_static(sv) &&
3427 (SvPVX_const(sv) == PL_Yes || SvPVX_const(sv) == PL_No);
3431 * ex: set ts=8 sts=4 sw=4 et: