X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/da8c1a98236a9f56df850c47705cb3046d6636aa..4cdca267b2a4aa72f52426806d483ad4fe34de7c:/utf8.h?ds=sidebyside diff --git a/utf8.h b/utf8.h index 5568039..4d2d01b 100644 --- a/utf8.h +++ b/utf8.h @@ -14,8 +14,8 @@ * */ -#ifndef H_UTF8 /* Guard against recursive inclusion */ -#define H_UTF8 1 +#ifndef PERL_UTF8_H_ /* Guard against recursive inclusion */ +#define PERL_UTF8_H_ 1 /* Use UTF-8 as the default script encoding? * Turning this on will break scripts having non-UTF-8 binary @@ -76,10 +76,15 @@ the string is invariant. utf8n_to_uvchr_error(s, len, lenp, flags, 0) #define to_uni_fold(c, p, lenp) _to_uni_fold_flags(c, p, lenp, FOLD_FLAGS_FULL) -#define to_utf8_fold(c, p, lenp) _to_utf8_fold_flags(c, p, lenp, FOLD_FLAGS_FULL) -#define to_utf8_lower(a,b,c) _to_utf8_lower_flags(a,b,c,0) -#define to_utf8_upper(a,b,c) _to_utf8_upper_flags(a,b,c,0) -#define to_utf8_title(a,b,c) _to_utf8_title_flags(a,b,c,0) + +#define to_utf8_fold(s, r, lenr) \ + _to_utf8_fold_flags (s, NULL, r, lenr, FOLD_FLAGS_FULL, __FILE__, __LINE__) +#define to_utf8_lower(s, r, lenr) \ + _to_utf8_lower_flags(s, NULL, r ,lenr, 0, __FILE__, __LINE__) +#define to_utf8_upper(s, r, lenr) \ + _to_utf8_upper_flags(s, NULL, r, lenr, 0, __FILE__, __LINE__) +#define to_utf8_title(s, r, lenr) \ + _to_utf8_title_flags(s, NULL, r, lenr ,0, __FILE__, __LINE__) #define foldEQ_utf8(s1, pe1, l1, u1, s2, pe2, l2, u2) \ foldEQ_utf8_flags(s1, pe1, l1, u1, s2, pe2, l2, u2, 0) @@ -230,13 +235,6 @@ are in the character. */ -/* Anything larger than this will overflow the word if it were converted into a UV */ -#if defined(UV_IS_QUAD) -# define HIGHEST_REPRESENTABLE_UTF8 "\xFF\x80\x8F\xBF\xBF\xBF\xBF\xBF\xBF\xBF\xBF\xBF\xBF" -#else -# define HIGHEST_REPRESENTABLE_UTF8 "\xFE\x83\xBF\xBF\xBF\xBF\xBF" -#endif - /* Is the representation of the Unicode code point 'cp' the same regardless of * being encoded in UTF-8 or not? */ #define OFFUNI_IS_INVARIANT(cp) isASCII(cp) @@ -261,13 +259,15 @@ C is Unicode if above 255; otherwise is platform-native. /* Misleadingly named: is the UTF8-encoded byte 'c' part of a variant sequence * in UTF-8? This is the inverse of UTF8_IS_INVARIANT. The |0 makes sure this * isn't mistakenly called with a ptr argument */ -#define UTF8_IS_CONTINUED(c) (((U8)((c) | 0)) & UTF_CONTINUATION_MARK) +#define UTF8_IS_CONTINUED(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ + ((U8)((c) | 0)) & UTF_CONTINUATION_MARK) /* Is the byte 'c' the first byte of a multi-byte UTF8-8 encoded sequence? * This doesn't catch invariants (they are single-byte). It also excludes the * illegal overlong sequences that begin with C0 and C1. The |0 makes sure * this isn't mistakenly called with a ptr argument */ -#define UTF8_IS_START(c) (((U8)((c) | 0)) >= 0xc2) +#define UTF8_IS_START(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ + ((U8)((c) | 0)) >= 0xc2) /* For use in UTF8_IS_CONTINUATION() below */ #define UTF_IS_CONTINUATION_MASK 0xC0 @@ -275,20 +275,22 @@ C is Unicode if above 255; otherwise is platform-native. /* Is the byte 'c' part of a multi-byte UTF8-8 encoded sequence, and not the * first byte thereof? The |0 makes sure this isn't mistakenly called with a * ptr argument */ -#define UTF8_IS_CONTINUATION(c) \ - ((((U8)((c) | 0)) & UTF_IS_CONTINUATION_MASK) == UTF_CONTINUATION_MARK) +#define UTF8_IS_CONTINUATION(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ + (((U8)((c) | 0)) & UTF_IS_CONTINUATION_MASK) == UTF_CONTINUATION_MARK) /* Is the UTF8-encoded byte 'c' the first byte of a two byte sequence? Use * UTF8_IS_NEXT_CHAR_DOWNGRADEABLE() instead if the input isn't known to * be well-formed. Masking with 0xfe allows the low bit to be 0 or 1; thus * this matches 0xc[23]. The |0 makes sure this isn't mistakenly called with a * ptr argument */ -#define UTF8_IS_DOWNGRADEABLE_START(c) ((((U8)((c) | 0)) & 0xfe) == 0xc2) +#define UTF8_IS_DOWNGRADEABLE_START(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ + (((U8)((c) | 0)) & 0xfe) == 0xc2) /* Is the UTF8-encoded byte 'c' the first byte of a sequence of bytes that * represent a code point > 255? The |0 makes sure this isn't mistakenly * called with a ptr argument */ -#define UTF8_IS_ABOVE_LATIN1(c) (((U8)((c) | 0)) >= 0xc4) +#define UTF8_IS_ABOVE_LATIN1(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ + ((U8)((c) | 0)) >= 0xc4) /* This is the number of low-order bits a continuation byte in a UTF-8 encoded * sequence contributes to the specification of the code point. In the bit @@ -304,7 +306,8 @@ C is Unicode if above 255; otherwise is platform-native. * problematic in some contexts. This allows code that needs to check for * those to to quickly exclude the vast majority of code points it will * encounter */ -#define isUTF8_POSSIBLY_PROBLEMATIC(c) ((U8) c >= 0xED) +#define isUTF8_POSSIBLY_PROBLEMATIC(c) (__ASSERT_(FITS_IN_8_BITS(c)) \ + (U8) c >= 0xED) /* A helper macro for isUTF8_CHAR, so use that one instead of this. This was * generated by regen/regcharclass.pl, and then moved here. Then it was @@ -404,6 +407,8 @@ C is Unicode if above 255; otherwise is platform-native. ( LIKELY( ( ( ( ((const U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\ : LIKELY( ( ( ( 0xF4 == ((const U8*)s)[0] ) && ( ( ((const U8*)s)[1] & 0xF0 ) == 0x80 ) ) && ( ( ((const U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((const U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 ) +#define UNICODE_IS_PERL_EXTENDED(uv) UNLIKELY((UV) (uv) > 0x7FFFFFFF) + #endif /* EBCDIC vs ASCII */ /* 2**UTF_ACCUMULATION_SHIFT - 1 */ @@ -524,14 +529,16 @@ encoded as UTF-8. C is a native (ASCII or EBCDIC) code point if less than * that this is asymmetric on EBCDIC platforms, in that the 'new' parameter is * the UTF-EBCDIC byte, whereas the 'old' parameter is a Unicode (not EBCDIC) * code point in process of being generated */ -#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) \ +#define UTF8_ACCUMULATE(old, new) (__ASSERT_(FITS_IN_8_BITS(new)) \ + ((old) << UTF_ACCUMULATION_SHIFT) \ | ((NATIVE_UTF8_TO_I8((U8)new)) \ & UTF_CONTINUATION_MASK)) /* This works in the face of malformed UTF-8. */ -#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) (UTF8_IS_DOWNGRADEABLE_START(*s) \ - && ( (e) - (s) > 1) \ - && UTF8_IS_CONTINUATION(*((s)+1))) +#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) \ + ( UTF8_IS_DOWNGRADEABLE_START(*(s)) \ + && ( (e) - (s) > 1) \ + && UTF8_IS_CONTINUATION(*((s)+1))) /* Number of bytes a code point occupies in UTF-8. */ #define NATIVE_SKIP(uv) UVCHR_SKIP(uv) @@ -566,8 +573,10 @@ encoded as UTF-8. C is a native (ASCII or EBCDIC) code point if less than * Note that the result can be larger than 255 if the input character is not * downgradable */ #define TWO_BYTE_UTF8_TO_NATIVE(HI, LO) \ - ( __ASSERT_(PL_utf8skip[HI] == 2) \ - __ASSERT_(UTF8_IS_CONTINUATION(LO)) \ + (__ASSERT_(FITS_IN_8_BITS(HI)) \ + __ASSERT_(FITS_IN_8_BITS(LO)) \ + __ASSERT_(PL_utf8skip[HI] == 2) \ + __ASSERT_(UTF8_IS_CONTINUATION(LO)) \ UNI_TO_NATIVE(UTF8_ACCUMULATE((NATIVE_UTF8_TO_I8(HI) & UTF_START_MASK(2)), \ (LO)))) @@ -672,31 +681,37 @@ with a ptr argument. * beginning of a utf8 character. Now that foo_utf8() determines that itself, * no need to do it again here */ -#define isIDFIRST_lazy_if(p,UTF) ((IN_BYTES || !UTF) \ - ? isIDFIRST(*(p)) \ - : isIDFIRST_utf8((const U8*)p)) -#define isWORDCHAR_lazy_if(p,UTF) ((IN_BYTES || (!UTF)) \ - ? isWORDCHAR(*(p)) \ - : isWORDCHAR_utf8((const U8*)p)) -#define isALNUM_lazy_if(p,UTF) isWORDCHAR_lazy_if(p,UTF) +#define isIDFIRST_lazy_if(p,UTF) \ + _is_utf8_FOO(_CC_IDFIRST, (const U8 *) p, "isIDFIRST_lazy_if", \ + "isIDFIRST_lazy_if_safe", \ + cBOOL(UTF && ! IN_BYTES), 0, __FILE__,__LINE__) #define isIDFIRST_lazy_if_safe(p, e, UTF) \ ((IN_BYTES || !UTF) \ ? isIDFIRST(*(p)) \ : isIDFIRST_utf8_safe(p, e)) +#define isWORDCHAR_lazy_if(p,UTF) \ + _is_utf8_FOO(_CC_IDFIRST, (const U8 *) p, "isWORDCHAR_lazy_if", \ + "isWORDCHAR_lazy_if_safe", \ + cBOOL(UTF && ! IN_BYTES), 0, __FILE__,__LINE__) + #define isWORDCHAR_lazy_if_safe(p, e, UTF) \ ((IN_BYTES || !UTF) \ ? isWORDCHAR(*(p)) \ : isWORDCHAR_utf8_safe((U8 *) p, (U8 *) e)) +#define isALNUM_lazy_if(p,UTF) \ + _is_utf8_FOO(_CC_IDFIRST, (const U8 *) p, "isALNUM_lazy_if", \ + "isWORDCHAR_lazy_if_safe", \ + cBOOL(UTF && ! IN_BYTES), 0, __FILE__,__LINE__) #define UTF8_MAXLEN UTF8_MAXBYTES /* A Unicode character can fold to up to 3 characters */ #define UTF8_MAX_FOLD_CHAR_EXPAND 3 -#define IN_BYTES (CopHINTS_get(PL_curcop) & HINT_BYTES) +#define IN_BYTES UNLIKELY(CopHINTS_get(PL_curcop) & HINT_BYTES) /* @@ -715,12 +730,12 @@ case any call to string overloading updates the internal UTF-8 encoding flag. * Is so within 'feature unicode_strings' or 'locale :not_characters', and not * within 'use bytes'. UTF-8 locales are not tested for here, but perhaps * could be */ -#define IN_UNI_8_BIT \ - (((CopHINTS_get(PL_curcop) & (HINT_UNI_8_BIT)) \ - || (CopHINTS_get(PL_curcop) & HINT_LOCALE_PARTIAL \ - /* -1 below is for :not_characters */ \ - && _is_in_locale_category(FALSE, -1))) \ - && ! IN_BYTES) +#define IN_UNI_8_BIT \ + (( ( (CopHINTS_get(PL_curcop) & HINT_UNI_8_BIT)) \ + || ( CopHINTS_get(PL_curcop) & HINT_LOCALE_PARTIAL \ + /* -1 below is for :not_characters */ \ + && _is_in_locale_category(FALSE, -1))) \ + && (! IN_BYTES)) #define UTF8_ALLOW_EMPTY 0x0001 /* Allow a zero length string */ @@ -738,38 +753,53 @@ case any call to string overloading updates the internal UTF-8 encoding flag. #define UTF8_ALLOW_SHORT 0x0008 #define UTF8_GOT_SHORT UTF8_ALLOW_SHORT -/* Overlong sequence; i.e., the code point can be specified in fewer bytes. */ +/* Overlong sequence; i.e., the code point can be specified in fewer bytes. + * First one will convert the overlong to the REPLACEMENT CHARACTER; second + * will return what the overlong evaluates to */ #define UTF8_ALLOW_LONG 0x0010 +#define UTF8_ALLOW_LONG_AND_ITS_VALUE (UTF8_ALLOW_LONG|0x0020) #define UTF8_GOT_LONG UTF8_ALLOW_LONG -/* Currently no way to allow overflow */ -#define UTF8_GOT_OVERFLOW 0x0020 +#define UTF8_ALLOW_OVERFLOW 0x0080 +#define UTF8_GOT_OVERFLOW UTF8_ALLOW_OVERFLOW -#define UTF8_DISALLOW_SURROGATE 0x0040 /* Unicode surrogates */ +#define UTF8_DISALLOW_SURROGATE 0x0100 /* Unicode surrogates */ #define UTF8_GOT_SURROGATE UTF8_DISALLOW_SURROGATE -#define UTF8_WARN_SURROGATE 0x0080 +#define UTF8_WARN_SURROGATE 0x0200 -#define UTF8_DISALLOW_NONCHAR 0x0100 /* Unicode non-character */ +/* Unicode non-character code points */ +#define UTF8_DISALLOW_NONCHAR 0x0400 #define UTF8_GOT_NONCHAR UTF8_DISALLOW_NONCHAR -#define UTF8_WARN_NONCHAR 0x0200 /* code points */ +#define UTF8_WARN_NONCHAR 0x0800 -#define UTF8_DISALLOW_SUPER 0x0400 /* Super-set of Unicode: code */ +/* Super-set of Unicode: code points above the legal max */ +#define UTF8_DISALLOW_SUPER 0x1000 #define UTF8_GOT_SUPER UTF8_DISALLOW_SUPER -#define UTF8_WARN_SUPER 0x0800 /* points above the legal max */ - -/* Code points which never were part of the original UTF-8 standard, which only - * went up to 2 ** 31 - 1. Note that these all overflow a signed 32-bit word, - * The first byte of these code points is FE or FF on ASCII platforms. If the - * first byte is FF, it will overflow a 32-bit word. */ -#define UTF8_DISALLOW_ABOVE_31_BIT 0x1000 -#define UTF8_GOT_ABOVE_31_BIT UTF8_DISALLOW_ABOVE_31_BIT -#define UTF8_WARN_ABOVE_31_BIT 0x2000 - -/* For back compat, these old names are misleading for UTF_EBCDIC */ -#define UTF8_DISALLOW_FE_FF UTF8_DISALLOW_ABOVE_31_BIT -#define UTF8_WARN_FE_FF UTF8_WARN_ABOVE_31_BIT - -#define UTF8_CHECK_ONLY 0x4000 +#define UTF8_WARN_SUPER 0x2000 + +/* The original UTF-8 standard did not define UTF-8 with start bytes of 0xFE or + * 0xFF, though UTF-EBCDIC did. This allowed both versions to represent code + * points up to 2 ** 31 - 1. Perl extends UTF-8 so that 0xFE and 0xFF are + * usable on ASCII platforms, and 0xFF means something different than + * UTF-EBCDIC defines. These changes allow code points of 64 bits (actually + * somewhat more) to be represented on both platforms. But these are Perl + * extensions, and not likely to be interchangeable with other languages. Note + * that on ASCII platforms, FE overflows a signed 32-bit word, and FF an + * unsigned one. */ +#define UTF8_DISALLOW_PERL_EXTENDED 0x4000 +#define UTF8_GOT_PERL_EXTENDED UTF8_DISALLOW_PERL_EXTENDED +#define UTF8_WARN_PERL_EXTENDED 0x8000 + +/* For back compat, these old names are misleading for overlongs and + * UTF_EBCDIC. */ +#define UTF8_DISALLOW_ABOVE_31_BIT UTF8_DISALLOW_PERL_EXTENDED +#define UTF8_GOT_ABOVE_31_BIT UTF8_GOT_PERL_EXTENDED +#define UTF8_WARN_ABOVE_31_BIT UTF8_WARN_PERL_EXTENDED +#define UTF8_DISALLOW_FE_FF UTF8_DISALLOW_PERL_EXTENDED +#define UTF8_WARN_FE_FF UTF8_WARN_PERL_EXTENDED + +#define UTF8_CHECK_ONLY 0x10000 +#define _UTF8_NO_CONFIDENCE_IN_CURLEN 0x20000 /* Internal core use only */ /* For backwards source compatibility. They do nothing, as the default now * includes what they used to mean. The first one's meaning was to allow the @@ -787,12 +817,21 @@ case any call to string overloading updates the internal UTF-8 encoding flag. #define UTF8_WARN_ILLEGAL_INTERCHANGE \ (UTF8_WARN_ILLEGAL_C9_INTERCHANGE|UTF8_WARN_NONCHAR) -#define UTF8_ALLOW_ANY \ - (~( UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_DISALLOW_ABOVE_31_BIT \ - |UTF8_WARN_ILLEGAL_INTERCHANGE|UTF8_WARN_ABOVE_31_BIT)) -#define UTF8_ALLOW_ANYUV UTF8_ALLOW_EMPTY -#define UTF8_ALLOW_DEFAULT (ckWARN(WARN_UTF8) ? 0 : \ - UTF8_ALLOW_ANYUV) +/* This is typically used for code that processes UTF-8 input and doesn't want + * to have to deal with any malformations that might be present. All such will + * be safely replaced by the REPLACEMENT CHARACTER, unless other flags + * overriding this are also present. */ +#define UTF8_ALLOW_ANY ( UTF8_ALLOW_CONTINUATION \ + |UTF8_ALLOW_NON_CONTINUATION \ + |UTF8_ALLOW_SHORT \ + |UTF8_ALLOW_LONG \ + |UTF8_ALLOW_OVERFLOW) + +/* Accept any Perl-extended UTF-8 that evaluates to any UV on the platform, but + * not any malformed. This is the default. (Note that UVs above IV_MAX are + * deprecated. */ +#define UTF8_ALLOW_ANYUV 0 +#define UTF8_ALLOW_DEFAULT UTF8_ALLOW_ANYUV /* =for apidoc Am|bool|UTF8_IS_SURROGATE|const U8 *s|const U8 *e @@ -881,14 +920,16 @@ point's representation. * let's be conservative and do as Unicode says. */ #define PERL_UNICODE_MAX 0x10FFFF -#define UNICODE_WARN_SURROGATE 0x0001 /* UTF-16 surrogates */ -#define UNICODE_WARN_NONCHAR 0x0002 /* Non-char code points */ -#define UNICODE_WARN_SUPER 0x0004 /* Above 0x10FFFF */ -#define UNICODE_WARN_ABOVE_31_BIT 0x0008 /* Above 0x7FFF_FFFF */ -#define UNICODE_DISALLOW_SURROGATE 0x0010 -#define UNICODE_DISALLOW_NONCHAR 0x0020 -#define UNICODE_DISALLOW_SUPER 0x0040 -#define UNICODE_DISALLOW_ABOVE_31_BIT 0x0080 +#define UNICODE_WARN_SURROGATE 0x0001 /* UTF-16 surrogates */ +#define UNICODE_WARN_NONCHAR 0x0002 /* Non-char code points */ +#define UNICODE_WARN_SUPER 0x0004 /* Above 0x10FFFF */ +#define UNICODE_WARN_PERL_EXTENDED 0x0008 /* Above 0x7FFF_FFFF */ +#define UNICODE_WARN_ABOVE_31_BIT UNICODE_WARN_PERL_EXTENDED +#define UNICODE_DISALLOW_SURROGATE 0x0010 +#define UNICODE_DISALLOW_NONCHAR 0x0020 +#define UNICODE_DISALLOW_SUPER 0x0040 +#define UNICODE_DISALLOW_PERL_EXTENDED 0x0080 +#define UNICODE_DISALLOW_ABOVE_31_BIT UNICODE_DISALLOW_PERL_EXTENDED #define UNICODE_WARN_ILLEGAL_C9_INTERCHANGE \ (UNICODE_WARN_SURROGATE|UNICODE_WARN_SUPER) #define UNICODE_WARN_ILLEGAL_INTERCHANGE \ @@ -927,7 +968,6 @@ point's representation. && UNICODE_IS_END_PLANE_NONCHAR_GIVEN_NOT_SUPER(uv))) #define UNICODE_IS_SUPER(uv) ((UV) (uv) > PERL_UNICODE_MAX) -#define UNICODE_IS_ABOVE_31_BIT(uv) ((UV) (uv) > 0x7FFFFFFF) #define LATIN_SMALL_LETTER_SHARP_S LATIN_SMALL_LETTER_SHARP_S_NATIVE #define LATIN_SMALL_LETTER_Y_WITH_DIAERESIS \ @@ -1016,6 +1056,8 @@ is a valid UTF-8 character. : _is_utf8_char_helper(s, e, 0)) #define is_utf8_char_buf(buf, buf_end) isUTF8_CHAR(buf, buf_end) +#define bytes_from_utf8(s, lenp, is_utf8p) \ + bytes_from_utf8_loc(s, lenp, is_utf8p, 0) /* @@ -1136,7 +1178,7 @@ L to check entire strings. * retained solely for backwards compatibility */ #define IS_UTF8_CHAR(p, n) (isUTF8_CHAR(p, (p) + (n)) == n) -#endif /* H_UTF8 */ +#endif /* PERL_UTF8_H_ */ /* * ex: set ts=8 sts=4 sw=4 et: