X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/9495395586e6a655057cb766ed00213037dd06c0..8c0364339f063fa32be742877e8206c8b6203de0:/utf8.h diff --git a/utf8.h b/utf8.h index 3dde45a..affa2d6 100644 --- a/utf8.h +++ b/utf8.h @@ -76,10 +76,15 @@ the string is invariant. utf8n_to_uvchr_error(s, len, lenp, flags, 0) #define to_uni_fold(c, p, lenp) _to_uni_fold_flags(c, p, lenp, FOLD_FLAGS_FULL) -#define to_utf8_fold(c, p, lenp) _to_utf8_fold_flags(c, p, lenp, FOLD_FLAGS_FULL) -#define to_utf8_lower(a,b,c) _to_utf8_lower_flags(a,b,c,0) -#define to_utf8_upper(a,b,c) _to_utf8_upper_flags(a,b,c,0) -#define to_utf8_title(a,b,c) _to_utf8_title_flags(a,b,c,0) + +#define to_utf8_fold(s, r, lenr) \ + _to_utf8_fold_flags (s, NULL, r, lenr, FOLD_FLAGS_FULL, __FILE__, __LINE__) +#define to_utf8_lower(s, r, lenr) \ + _to_utf8_lower_flags(s, NULL, r ,lenr, 0, __FILE__, __LINE__) +#define to_utf8_upper(s, r, lenr) \ + _to_utf8_upper_flags(s, NULL, r, lenr, 0, __FILE__, __LINE__) +#define to_utf8_title(s, r, lenr) \ + _to_utf8_title_flags(s, NULL, r, lenr ,0, __FILE__, __LINE__) #define foldEQ_utf8(s1, pe1, l1, u1, s2, pe2, l2, u2) \ foldEQ_utf8_flags(s1, pe1, l1, u1, s2, pe2, l2, u2, 0) @@ -672,31 +677,37 @@ with a ptr argument. * beginning of a utf8 character. Now that foo_utf8() determines that itself, * no need to do it again here */ -#define isIDFIRST_lazy_if(p,UTF) ((IN_BYTES || !UTF) \ - ? isIDFIRST(*(p)) \ - : isIDFIRST_utf8((const U8*)p)) -#define isWORDCHAR_lazy_if(p,UTF) ((IN_BYTES || (!UTF)) \ - ? isWORDCHAR(*(p)) \ - : isWORDCHAR_utf8((const U8*)p)) -#define isALNUM_lazy_if(p,UTF) isWORDCHAR_lazy_if(p,UTF) +#define isIDFIRST_lazy_if(p,UTF) \ + _is_utf8_FOO(_CC_IDFIRST, (const U8 *) p, "isIDFIRST_lazy_if", \ + "isIDFIRST_lazy_if_safe", \ + cBOOL(UTF && ! IN_BYTES), 0, __FILE__,__LINE__) #define isIDFIRST_lazy_if_safe(p, e, UTF) \ ((IN_BYTES || !UTF) \ ? isIDFIRST(*(p)) \ : isIDFIRST_utf8_safe(p, e)) +#define isWORDCHAR_lazy_if(p,UTF) \ + _is_utf8_FOO(_CC_IDFIRST, (const U8 *) p, "isWORDCHAR_lazy_if", \ + "isWORDCHAR_lazy_if_safe", \ + cBOOL(UTF && ! IN_BYTES), 0, __FILE__,__LINE__) + #define isWORDCHAR_lazy_if_safe(p, e, UTF) \ ((IN_BYTES || !UTF) \ ? isWORDCHAR(*(p)) \ : isWORDCHAR_utf8_safe((U8 *) p, (U8 *) e)) +#define isALNUM_lazy_if(p,UTF) \ + _is_utf8_FOO(_CC_IDFIRST, (const U8 *) p, "isALNUM_lazy_if", \ + "isWORDCHAR_lazy_if_safe", \ + cBOOL(UTF && ! IN_BYTES), 0, __FILE__,__LINE__) #define UTF8_MAXLEN UTF8_MAXBYTES /* A Unicode character can fold to up to 3 characters */ #define UTF8_MAX_FOLD_CHAR_EXPAND 3 -#define IN_BYTES (CopHINTS_get(PL_curcop) & HINT_BYTES) +#define IN_BYTES UNLIKELY(CopHINTS_get(PL_curcop) & HINT_BYTES) /* @@ -715,12 +726,12 @@ case any call to string overloading updates the internal UTF-8 encoding flag. * Is so within 'feature unicode_strings' or 'locale :not_characters', and not * within 'use bytes'. UTF-8 locales are not tested for here, but perhaps * could be */ -#define IN_UNI_8_BIT \ - (((CopHINTS_get(PL_curcop) & (HINT_UNI_8_BIT)) \ - || (CopHINTS_get(PL_curcop) & HINT_LOCALE_PARTIAL \ - /* -1 below is for :not_characters */ \ - && _is_in_locale_category(FALSE, -1))) \ - && ! IN_BYTES) +#define IN_UNI_8_BIT \ + (( ( (CopHINTS_get(PL_curcop) & HINT_UNI_8_BIT)) \ + || ( CopHINTS_get(PL_curcop) & HINT_LOCALE_PARTIAL \ + /* -1 below is for :not_characters */ \ + && _is_in_locale_category(FALSE, -1))) \ + && (! IN_BYTES)) #define UTF8_ALLOW_EMPTY 0x0001 /* Allow a zero length string */ @@ -745,8 +756,8 @@ case any call to string overloading updates the internal UTF-8 encoding flag. #define UTF8_ALLOW_LONG_AND_ITS_VALUE (UTF8_ALLOW_LONG|0x0020) #define UTF8_GOT_LONG UTF8_ALLOW_LONG -/* Currently no way to allow overflow */ -#define UTF8_GOT_OVERFLOW 0x0080 +#define UTF8_ALLOW_OVERFLOW 0x0080 +#define UTF8_GOT_OVERFLOW UTF8_ALLOW_OVERFLOW #define UTF8_DISALLOW_SURROGATE 0x0100 /* Unicode surrogates */ #define UTF8_GOT_SURROGATE UTF8_DISALLOW_SURROGATE @@ -773,6 +784,7 @@ case any call to string overloading updates the internal UTF-8 encoding flag. #define UTF8_WARN_FE_FF UTF8_WARN_ABOVE_31_BIT #define UTF8_CHECK_ONLY 0x10000 +#define _UTF8_NO_CONFIDENCE_IN_CURLEN 0x20000 /* Internal core use only */ /* For backwards source compatibility. They do nothing, as the default now * includes what they used to mean. The first one's meaning was to allow the @@ -790,10 +802,15 @@ case any call to string overloading updates the internal UTF-8 encoding flag. #define UTF8_WARN_ILLEGAL_INTERCHANGE \ (UTF8_WARN_ILLEGAL_C9_INTERCHANGE|UTF8_WARN_NONCHAR) +/* This is typically used for code that processes UTF-8 input and doesn't want + * to have to deal with any malformations that might be present. All such will + * be safely replaced by the REPLACEMENT CHARACTER, unless other flags + * overriding this are also present. */ #define UTF8_ALLOW_ANY ( UTF8_ALLOW_CONTINUATION \ |UTF8_ALLOW_NON_CONTINUATION \ |UTF8_ALLOW_SHORT \ - |UTF8_ALLOW_LONG) + |UTF8_ALLOW_LONG \ + |UTF8_ALLOW_OVERFLOW) /* Accept any Perl-extended UTF-8 that evaluates to any UV on the platform, but * not any malformed. This is the default. (Note that UVs above IV_MAX are