X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/8b563eef9044008579e8e4bac7e31e12a2e54bca..fd8a2e8903d029f115ef570ab7ad92308c9d3726:/utf8.c diff --git a/utf8.c b/utf8.c index 266cb9e..9d3770d 100644 --- a/utf8.c +++ b/utf8.c @@ -88,8 +88,8 @@ Perl_is_ascii_string(const U8 *s, STRLEN len) /* =for apidoc uvuni_to_utf8_flags -Adds the UTF-8 representation of the Unicode codepoint C to the end -of the string C; C should be have at least C free +Adds the UTF-8 representation of the code point C to the end +of the string C; C should have at least C free bytes available. The return value is the pointer to the byte after the end of the new character. In other words, @@ -103,10 +103,31 @@ or, in most cases, d = uvuni_to_utf8_flags(d, uv, 0); -is the recommended Unicode-aware way of saying +This is the recommended Unicode-aware way of saying *(d++) = uv; +This function will convert to UTF-8 (and not warn) even code points that aren't +legal Unicode or are problematic, unless C contains one or more of the +following flags. +If C is a Unicode surrogate code point and UNICODE_WARN_SURROGATE is set, +the function will raise a warning, provided UTF8 warnings are enabled. If instead +UNICODE_DISALLOW_SURROGATE is set, the function will fail and return NULL. +If both flags are set, the function will both warn and return NULL. + +The UNICODE_WARN_NONCHAR and UNICODE_DISALLOW_NONCHAR flags correspondingly +affect how the function handles a Unicode non-character. And, likewise for the +UNICODE_WARN_SUPER and UNICODE_DISALLOW_SUPER flags, and code points that are +above the Unicode maximum of 0x10FFFF. Code points above 0x7FFF_FFFF (which are +even less portable) can be warned and/or disallowed even if other above-Unicode +code points are accepted by the UNICODE_WARN_FE_FF and UNICODE_DISALLOW_FE_FF +flags. + +And finally, the flag UNICODE_WARN_ILLEGAL_INTERCHANGE selects all four of the +above WARN flags; and UNICODE_DISALLOW_ILLEGAL_INTERCHANGE selects all four +DISALLOW flags. + + =cut */ @@ -115,23 +136,39 @@ Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) { PERL_ARGS_ASSERT_UVUNI_TO_UTF8_FLAGS; - if (ckWARN(WARN_UTF8)) { - if (UNICODE_IS_SURROGATE(uv) && - !(flags & UNICODE_ALLOW_SURROGATE)) - Perl_warner(aTHX_ packWARN(WARN_UTF8), "UTF-16 surrogate 0x%04"UVxf, uv); - else if ( - ((uv >= 0xFDD0 && uv <= 0xFDEF && - !(flags & UNICODE_ALLOW_FDD0)) - || - ((uv & 0xFFFE) == 0xFFFE && /* Either FFFE or FFFF. */ - !(flags & UNICODE_ALLOW_FFFF))) && - /* UNICODE_ALLOW_SUPER includes - * FFFEs and FFFFs beyond 0x10FFFF. */ - ((uv <= PERL_UNICODE_MAX) || - !(flags & UNICODE_ALLOW_SUPER)) - ) - Perl_warner(aTHX_ packWARN(WARN_UTF8), - "Unicode non-character 0x%04"UVxf" is illegal for interchange", uv); + if (ckWARN_d(WARN_UTF8)) { + if (UNICODE_IS_SURROGATE(uv)) { + if (flags & UNICODE_WARN_SURROGATE) { + Perl_ck_warner_d(aTHX_ packWARN(WARN_SURROGATE), + "UTF-16 surrogate U+%04"UVXf, uv); + } + if (flags & UNICODE_DISALLOW_SURROGATE) { + return NULL; + } + } + else if (UNICODE_IS_SUPER(uv)) { + if (flags & UNICODE_WARN_SUPER + || (UNICODE_IS_FE_FF(uv) && (flags & UNICODE_WARN_FE_FF))) + { + Perl_ck_warner_d(aTHX_ packWARN(WARN_NON_UNICODE), + "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv); + } + if (flags & UNICODE_DISALLOW_SUPER + || (UNICODE_IS_FE_FF(uv) && (flags & UNICODE_DISALLOW_FE_FF))) + { + return NULL; + } + } + else if (UNICODE_IS_NONCHAR(uv)) { + if (flags & UNICODE_WARN_NONCHAR) { + Perl_ck_warner_d(aTHX_ packWARN(WARN_NONCHAR), + "Unicode non-character U+%04"UVXf" is illegal for open interchange", + uv); + } + if (flags & UNICODE_DISALLOW_NONCHAR) { + return NULL; + } + } } if (UNI_IS_INVARIANT(uv)) { *d++ = (U8)UTF_TO_NATIVE(uv); @@ -428,20 +465,62 @@ Perl_is_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el) =for apidoc utf8n_to_uvuni Bottom level UTF-8 decode routine. -Returns the Unicode code point value of the first character in the string C -which is assumed to be in UTF-8 encoding and no longer than C; -C will be set to the length, in bytes, of that character. - -If C does not point to a well-formed UTF-8 character, the behaviour -is dependent on the value of C: if it contains UTF8_CHECK_ONLY, -it is assumed that the caller will raise a warning, and this function -will silently just set C to C<-1> and return zero. If the -C does not contain UTF8_CHECK_ONLY, warnings about -malformations will be given, C will be set to the expected -length of the UTF-8 character in bytes, and zero will be returned. - -The C can also contain various flags to allow deviations from -the strict UTF-8 encoding (see F). +Returns the code point value of the first character in the string C +which is assumed to be in UTF-8 (or UTF-EBCDIC) encoding and no longer than +C bytes; C will be set to the length, in bytes, of that +character. + +The value of C determines the behavior when C does not point to a +well-formed UTF-8 character. If C is 0, when a malformation is found, +C is set to the expected length of the UTF-8 character in bytes, zero +is returned, and if UTF-8 warnings haven't been lexically disabled, a warning +is raised. + +Various ALLOW flags can be set in C to allow (and not warn on) +individual types of malformations, such as the sequence being overlong (that +is, when there is a shorter sequence that can express the same code point; +overlong sequences are expressly forbidden in the UTF-8 standard due to +potential security issues). Another malformation example is the first byte of +a character not being a legal first byte. See F for the list of such +flags. Of course, the value returned by this function under such conditions is +not reliable. + +The UTF8_CHECK_ONLY flag overrides the behavior when a non-allowed (by other +flags) malformation is found. If this flag is set, the routine assumes that +the caller will raise a warning, and this function will silently just set +C to C<-1> and return zero. + +Certain code points are considered problematic. These are Unicode surrogates, +Unicode non-characters, and code points above the Unicode maximum of 0x10FFF. +By default these are considered regular code points, but certain situations +warrant special handling for them. if C contains +UTF8_DISALLOW_ILLEGAL_INTERCHANGE, all three classes are treated as +malformations and handled as such. The flags UTF8_DISALLOW_SURROGATE, +UTF8_DISALLOW_NONCHAR, and UTF8_DISALLOW_SUPER (meaning above the legal Unicode +maximum) can be set to disallow these categories individually. + +The flags UTF8_WARN_ILLEGAL_INTERCHANGE, UTF8_WARN_SURROGATE, +UTF8_WARN_NONCHAR, and UTF8_WARN_SUPER will cause warning messages to be raised +for their respective categories, but otherwise the code points are considered +valid (not malformations). To get a category to both be treated as a +malformation and raise a warning, specify both the WARN and DISALLOW flags. +(But note that warnings are not raised if lexically disabled nor if +UTF8_CHECK_ONLY is also specified.) + +Very large code points (above 0x7FFF_FFFF) are considered more problematic than +the others that are above the Unicode legal maximum. There are several +reasons, one of which is that the original UTF-8 specification never went above +this number (the current 0x10FFF limit was imposed later). The UTF-8 encoding +on ASCII platforms for these large code point begins with a byte containing +0xFE or 0xFF. The UTF8_DISALLOW_FE_FF flag will cause them to be treated as +malformations, while allowing smaller above-Unicode code points. (Of course +UTF8_DISALLOW_SUPER will treat all above-Unicode code points, including these, +as malformations.) Similarly, UTF8_WARN_FE_FF acts just like the other WARN +flags, but applies just to these code points. + +All other code points corresponding to Unicode characters, including private +use and those yet to be assigned, are never considered malformed and never +warn. Most code should use utf8_to_uvchr() rather than call this directly. @@ -455,25 +534,22 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) const U8 * const s0 = s; UV uv = *s, ouv = 0; STRLEN len = 1; - const bool dowarn = ckWARN_d(WARN_UTF8); + bool dowarn = ckWARN_d(WARN_UTF8); const UV startbyte = *s; STRLEN expectlen = 0; U32 warning = 0; - SV* sv; + SV* sv = NULL; PERL_ARGS_ASSERT_UTF8N_TO_UVUNI; -/* This list is a superset of the UTF8_ALLOW_XXX. BUT it isn't, eg SUPER missing XXX */ +/* This list is a superset of the UTF8_ALLOW_XXX. */ #define UTF8_WARN_EMPTY 1 #define UTF8_WARN_CONTINUATION 2 #define UTF8_WARN_NON_CONTINUATION 3 -#define UTF8_WARN_FE_FF 4 -#define UTF8_WARN_SHORT 5 -#define UTF8_WARN_OVERFLOW 6 -#define UTF8_WARN_SURROGATE 7 -#define UTF8_WARN_LONG 8 -#define UTF8_WARN_FFFF 9 /* Also FFFE. */ +#define UTF8_WARN_SHORT 4 +#define UTF8_WARN_OVERFLOW 5 +#define UTF8_WARN_LONG 6 if (curlen == 0 && !(flags & UTF8_ALLOW_EMPTY)) { @@ -502,10 +578,14 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) #ifdef EBCDIC uv = NATIVE_TO_UTF(uv); #else - if ((uv == 0xfe || uv == 0xff) && - !(flags & UTF8_ALLOW_FE_FF)) { - warning = UTF8_WARN_FE_FF; - goto malformed; + if (uv == 0xfe || uv == 0xff) { + if (flags & (UTF8_WARN_SUPER|UTF8_WARN_FE_FF)) { + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "Code point beginning with byte 0x%02"UVXf" is not Unicode, and not portable", uv)); + flags &= ~UTF8_WARN_SUPER; /* Only warn once on this problem */ + } + if (flags & (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_FE_FF)) { + goto malformed; + } } #endif @@ -535,7 +615,7 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) len--; s++; - ouv = uv; + ouv = uv; /* ouv is the value from the previous iteration */ while (len--) { if (!UTF8_IS_CONTINUATION(*s) && @@ -546,7 +626,8 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) } else uv = UTF8_ACCUMULATE(uv, *s); - if (!(uv > ouv)) { + if (!(uv > ouv)) { /* If the value didn't grow from the previous + iteration, something is horribly wrong */ /* These cannot be allowed. */ if (uv == ouv) { if (expectlen != 13 && !(flags & UTF8_ALLOW_LONG)) { @@ -564,22 +645,47 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) ouv = uv; } - if (UNICODE_IS_SURROGATE(uv) && - !(flags & UTF8_ALLOW_SURROGATE)) { - warning = UTF8_WARN_SURROGATE; - goto malformed; - } else if ((expectlen > (STRLEN)UNISKIP(uv)) && - !(flags & UTF8_ALLOW_LONG)) { + if ((expectlen > (STRLEN)UNISKIP(uv)) && !(flags & UTF8_ALLOW_LONG)) { warning = UTF8_WARN_LONG; goto malformed; - } else if (UNICODE_IS_ILLEGAL(uv) && - !(flags & UTF8_ALLOW_FFFF)) { - warning = UTF8_WARN_FFFF; - goto malformed; + } else if (flags & (UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_WARN_ILLEGAL_INTERCHANGE)) { + if (UNICODE_IS_SURROGATE(uv)) { + if ((flags & (UTF8_WARN_SURROGATE|UTF8_CHECK_ONLY)) == UTF8_WARN_SURROGATE) { + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "UTF-16 surrogate U+%04"UVXf"", uv)); + } + if (flags & UTF8_DISALLOW_SURROGATE) { + goto disallowed; + } + } + else if (UNICODE_IS_NONCHAR(uv)) { + if ((flags & (UTF8_WARN_NONCHAR|UTF8_CHECK_ONLY)) == UTF8_WARN_NONCHAR ) { + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "Unicode non-character U+%04"UVXf" is illegal for open interchange", uv)); + } + if (flags & UTF8_DISALLOW_NONCHAR) { + goto disallowed; + } + } + else if ((uv > PERL_UNICODE_MAX)) { + if ((flags & (UTF8_WARN_SUPER|UTF8_CHECK_ONLY)) == UTF8_WARN_SUPER) { + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv)); + } + if (flags & UTF8_DISALLOW_SUPER) { + goto disallowed; + } + } + + /* Here, this is not considered a malformed character, so drop through + * to return it */ } return uv; +disallowed: /* Is disallowed, but otherwise not malformed. 'sv' will have been + set if there is to be a warning. */ + if (!sv) { + dowarn = 0; + } + malformed: if (flags & UTF8_CHECK_ONLY) { @@ -589,58 +695,48 @@ malformed: } if (dowarn) { - if (warning == UTF8_WARN_FFFF) { - sv = newSVpvs_flags("Unicode non-character ", SVs_TEMP); - Perl_sv_catpvf(aTHX_ sv, "0x%04"UVxf" is illegal for interchange", uv); - } - else { + if (! sv) { sv = newSVpvs_flags("Malformed UTF-8 character ", SVs_TEMP); + } - switch (warning) { - case 0: /* Intentionally empty. */ break; - case UTF8_WARN_EMPTY: - sv_catpvs(sv, "(empty string)"); - break; - case UTF8_WARN_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); - break; - case UTF8_WARN_NON_CONTINUATION: - if (s == s0) - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", - (UV)s[1], startbyte); - else { - const int len = (int)(s-s0); - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", - (UV)s[1], len, len > 1 ? "s" : "", startbyte, (int)expectlen); - } + switch (warning) { + case 0: /* Intentionally empty. */ break; + case UTF8_WARN_EMPTY: + sv_catpvs(sv, "(empty string)"); + break; + case UTF8_WARN_CONTINUATION: + Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); + break; + case UTF8_WARN_NON_CONTINUATION: + if (s == s0) + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", + (UV)s[1], startbyte); + else { + const int len = (int)(s-s0); + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", + (UV)s[1], len, len > 1 ? "s" : "", startbyte, (int)expectlen); + } - break; - case UTF8_WARN_FE_FF: - Perl_sv_catpvf(aTHX_ sv, "(byte 0x%02"UVxf")", uv); - break; - case UTF8_WARN_SHORT: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", - (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte); - expectlen = curlen; /* distance for caller to skip */ - break; - case UTF8_WARN_OVERFLOW: - Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", - ouv, *s, startbyte); - break; - case UTF8_WARN_SURROGATE: - Perl_sv_catpvf(aTHX_ sv, "(UTF-16 surrogate 0x%04"UVxf")", uv); - break; - case UTF8_WARN_LONG: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", - (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); - break; - default: - sv_catpvs(sv, "(unknown reason)"); - break; - } + break; + case UTF8_WARN_SHORT: + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte); + expectlen = curlen; /* distance for caller to skip */ + break; + case UTF8_WARN_OVERFLOW: + Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", + ouv, *s, startbyte); + break; + case UTF8_WARN_LONG: + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); + break; + default: + sv_catpvs(sv, "(unknown reason)"); + break; } - if (warning) { + if (sv) { const char * const s = SvPVX_const(sv); if (PL_op) @@ -660,7 +756,7 @@ malformed: /* =for apidoc utf8_to_uvchr -Returns the native character value of the first character in the string C +Returns the native code point of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the length, in bytes, of that character. @@ -670,13 +766,14 @@ returned and retlen is set, if possible, to -1. =cut */ + UV Perl_utf8_to_uvchr(pTHX_ const U8 *s, STRLEN *retlen) { PERL_ARGS_ASSERT_UTF8_TO_UVCHR; return utf8n_to_uvchr(s, UTF8_MAXBYTES, retlen, - ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* @@ -702,7 +799,7 @@ Perl_utf8_to_uvuni(pTHX_ const U8 *s, STRLEN *retlen) /* Call the low level routine asking for checks */ return Perl_utf8n_to_uvuni(aTHX_ s, UTF8_MAXBYTES, retlen, - ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* @@ -1404,6 +1501,19 @@ Perl_is_utf8_idfirst(pTHX_ const U8 *p) /* The naming is historical. */ } bool +Perl_is_utf8_xidfirst(pTHX_ const U8 *p) /* The naming is historical. */ +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_XIDFIRST; + + if (*p == '_') + return TRUE; + /* is_utf8_idstart would be more logical. */ + return is_utf8_common(p, &PL_utf8_xidstart, "XIdStart"); +} + +bool Perl_is_utf8_idcont(pTHX_ const U8 *p) { dVAR; @@ -1416,6 +1526,18 @@ Perl_is_utf8_idcont(pTHX_ const U8 *p) } bool +Perl_is_utf8_xidcont(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_XIDCONT; + + if (*p == '_') + return TRUE; + return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue"); +} + +bool Perl_is_utf8_alpha(pTHX_ const U8 *p) { dVAR; @@ -1705,6 +1827,28 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, PERL_ARGS_ASSERT_TO_UTF8_CASE; + /* Note that swash_fetch() doesn't output warnings for these because it + * assumes we will */ + if (uv1 >= UNICODE_SURROGATE_FIRST) { + if (uv1 <= UNICODE_SURROGATE_LAST) { + if (ckWARN_d(WARN_SURROGATE)) { + const char* desc = (PL_op) ? OP_DESC(PL_op) : normal; + Perl_warner(aTHX_ packWARN(WARN_SURROGATE), + "Operation \"%s\" returns its argument for UTF-16 surrogate U+%04"UVXf"", desc, uv1); + } + } + else if (UNICODE_IS_SUPER(uv1)) { + if (ckWARN_d(WARN_NON_UNICODE)) { + const char* desc = (PL_op) ? OP_DESC(PL_op) : normal; + Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), + "Operation \"%s\" returns its argument for non-Unicode code point 0x%04"UVXf"", desc, uv1); + } + } + + /* Note that non-characters are perfectly legal, so no warning should + * be given */ + } + uvuni_to_utf8(tmpbuf, uv1); if (!*swashp) /* load on-demand */ @@ -2025,6 +2169,18 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) /* If char is encoded then swatch is for the prefix */ needents = (1 << UTF_ACCUMULATION_SHIFT); off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK; + if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_NON_UNICODE)) { + const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0); + + /* This outputs warnings for binary properties only, assuming that + * to_utf8_case() will output any. Also, surrogates aren't checked + * for, as that would warn on things like /\p{Gc=Cs}/ */ + SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); + if (SvUV(*bitssvp) == 1) { + Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), + "Code point 0x%04"UVXf" is not Unicode, no properties match it; all inverse properties do", code_point); + } + } } /* @@ -2455,7 +2611,7 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) } HV* -Perl__swash_inversion_hash(pTHX_ SV* swash) +Perl__swash_inversion_hash(pTHX_ SV* const swash) { /* Subject to change or removal. For use only in one place in regexec.c @@ -2530,10 +2686,6 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) char* key_end = (char *) uvuni_to_utf8((U8*) key, val); STRLEN key_len = key_end - key; - /* And the value is what the forward mapping is from. */ - char utf8_inverse[UTF8_MAXBYTES+1]; - char *utf8_inverse_end = (char *) uvuni_to_utf8((U8*) utf8_inverse, inverse); - /* Get the list for the map */ if ((listp = hv_fetch(ret, key, key_len, FALSE))) { list = (AV*) *listp; @@ -2552,22 +2704,21 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed"); } entry = *entryp; - if (SvCUR(entry) != key_len) { - continue; - } - if (memEQ(key, SvPVX(entry), key_len)) { + if (SvUV(entry) == val) { found_key = TRUE; break; } } + + /* Make sure there is a mapping to itself on the list */ if (! found_key) { - element = newSVpvn_flags(key, key_len, SVf_UTF8); + element = newSVuv(val); av_push(list, element); } /* Simply add the value to the list */ - element = newSVpvn_flags(utf8_inverse, utf8_inverse_end - utf8_inverse, SVf_UTF8); + element = newSVuv(inverse); av_push(list, element); /* swash_get() increments the value of val for each element in the @@ -2586,10 +2737,85 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) return ret; } +HV* +Perl__swash_to_invlist(pTHX_ SV* const swash) +{ + + /* Subject to change or removal. For use only in one place in regcomp.c */ + + U8 *l, *lend; + char *loc; + STRLEN lcur; + HV *const hv = MUTABLE_HV(SvRV(swash)); + UV elements = 0; /* Number of elements in the inversion list */ + U8 empty[] = ""; + + /* The string containing the main body of the table */ + SV** const listsvp = hv_fetchs(hv, "LIST", FALSE); + SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE); + SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); + + const U8* const typestr = (U8*)SvPV_nolen(*typesvp); + const STRLEN bits = SvUV(*bitssvp); + const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */ + + HV* invlist; + + PERL_ARGS_ASSERT__SWASH_TO_INVLIST; + + /* read $swash->{LIST} */ + if (SvPOK(*listsvp)) { + l = (U8*)SvPV(*listsvp, lcur); + } + else { + /* LIST legitimately doesn't contain a string during compilation phases + * of Perl itself, before the Unicode tables are generated. In this + * case, just fake things up by creating an empty list */ + l = empty; + lcur = 0; + } + loc = (char *) l; + lend = l + lcur; + + /* Scan the input to count the number of lines to preallocate array size + * based on worst possible case, which is each line in the input creates 2 + * elements in the inversion list: 1) the beginning of a range in the list; + * 2) the beginning of a range not in the list. */ + while ((loc = (strchr(loc, '\n'))) != NULL) { + elements += 2; + loc++; + } + + /* If the ending is somehow corrupt and isn't a new line, add another + * element for the final range that isn't in the inversion list */ + if (! (*lend == '\n' || (*lend == '\0' && *(lend - 1) == '\n'))) { + elements++; + } + + invlist = _new_invlist(elements); + + /* Now go through the input again, adding each range to the list */ + while (l < lend) { + UV start, end; + UV val; /* Not used by this function */ + + l = S_swash_scan_list_line(aTHX_ l, lend, &start, &end, &val, + cBOOL(octets), typestr); + + if (l > lend) { + break; + } + + _append_range_to_invlist(invlist, start, end); + } + + return invlist; +} + /* =for apidoc uvchr_to_utf8 -Adds the UTF-8 representation of the Native codepoint C to the end +Adds the UTF-8 representation of the Native code point C to the end of the string C; C should be have at least C free bytes available. The return value is the pointer to the byte after the end of the new character. In other words, @@ -2624,14 +2850,13 @@ Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) /* =for apidoc utf8n_to_uvchr -flags Returns the native character value of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the length, in bytes, of that character. -Allows length and flags to be passed to low level routine. +length and flags are the same as utf8n_to_uvuni(). =cut */ @@ -2649,6 +2874,59 @@ U32 flags) return UNI_TO_NATIVE(uv); } +bool +Perl_check_utf8_print(pTHX_ register const U8* s, const STRLEN len) +{ + /* May change: warns if surrogates, non-character code points, or + * non-Unicode code points are in s which has length len. Returns TRUE if + * none found; FALSE otherwise. The only other validity check is to make + * sure that this won't exceed the string's length */ + + const U8* const e = s + len; + bool ok = TRUE; + + PERL_ARGS_ASSERT_CHECK_UTF8_PRINT; + + while (s < e) { + if (UTF8SKIP(s) > len) { + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), + "%s in %s", unees, PL_op ? OP_DESC(PL_op) : "print"); + return FALSE; + } + if (*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE) { + STRLEN char_len; + if (UTF8_IS_SUPER(s)) { + if (ckWARN_d(WARN_NON_UNICODE)) { + UV uv = utf8_to_uvchr(s, &char_len); + Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), + "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv); + ok = FALSE; + } + } + else if (UTF8_IS_SURROGATE(s)) { + if (ckWARN_d(WARN_SURROGATE)) { + UV uv = utf8_to_uvchr(s, &char_len); + Perl_warner(aTHX_ packWARN(WARN_SURROGATE), + "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv); + ok = FALSE; + } + } + else if + ((UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s)) + && (ckWARN_d(WARN_NONCHAR))) + { + UV uv = utf8_to_uvchr(s, &char_len); + Perl_warner(aTHX_ packWARN(WARN_NONCHAR), + "Unicode non-character U+%04"UVXf" is illegal for open interchange", uv); + ok = FALSE; + } + } + s += UTF8SKIP(s); + } + + return ok; +} + /* =for apidoc pv_uni_display @@ -2789,8 +3067,19 @@ instead of upper/lowercasing both the characters, see http://www.unicode.org/unicode/reports/tr21/ (Case Mappings). =cut */ + +/* A flags parameter has been added which may change, and hence isn't + * externally documented. Currently it is: + * 0 for as-documented above + * FOLDEQ_UTF8_NOMIX_ASCII meaning that if a non-ASCII character folds to an + ASCII one, to not match + * FOLDEQ_UTF8_LOCALE meaning that locale rules are to be used for code + * points below 256; unicode rules for above 255; and + * folds that cross those boundaries are disallowed, + * like the NOMIX_ASCII option + */ I32 -Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2) +Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2, U32 flags) { dVAR; register const U8 *p1 = (const U8*)s1; /* Point to current char */ @@ -2807,7 +3096,7 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons U8 natbuf[2]; /* Holds native 8-bit char converted to utf8; these always fit in 2 bytes */ - PERL_ARGS_ASSERT_FOLDEQ_UTF8; + PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS; if (pe1) { e1 = *(U8**)pe1; @@ -2854,9 +3143,45 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons while (p1 < e1 && p2 < e2) { /* If at the beginning of a new character in s1, get its fold to use - * and the length of the fold */ + * and the length of the fold. (exception: locale rules just get the + * character to a single byte) */ if (n1 == 0) { - if (u1) { + + /* If in locale matching, we use two sets of rules, depending on if + * the code point is above or below 255. Here, we test for and + * handle locale rules */ + if ((flags & FOLDEQ_UTF8_LOCALE) + && (! u1 || UTF8_IS_INVARIANT(*p1) || UTF8_IS_DOWNGRADEABLE_START(*p1))) + { + /* There is no mixing of code points above and below 255. */ + if (u2 && (! UTF8_IS_INVARIANT(*p2) + && ! UTF8_IS_DOWNGRADEABLE_START(*p2))) + { + return 0; + } + + /* We handle locale rules by converting, if necessary, the code + * point to a single byte. */ + if (! u1 || UTF8_IS_INVARIANT(*p1)) { + *foldbuf1 = *p1; + } + else { + *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1)); + } + n1 = 1; + } + else if (isASCII(*p1)) { /* Note, that here won't be both ASCII + and using locale rules */ + + /* If trying to mix non- with ASCII, and not supposed to, fail */ + if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) { + return 0; + } + n1 = 1; + *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are + just lowercased */ + } + else if (u1) { to_utf8_fold(p1, foldbuf1, &n1); } else { /* Not utf8, convert to it first and then get fold */ @@ -2867,7 +3192,38 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons } if (n2 == 0) { /* Same for s2 */ - if (u2) { + if ((flags & FOLDEQ_UTF8_LOCALE) + && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2))) + { + /* Here, the next char in s2 is < 256. We've already worked on + * s1, and if it isn't also < 256, can't match */ + if (u1 && (! UTF8_IS_INVARIANT(*p1) + && ! UTF8_IS_DOWNGRADEABLE_START(*p1))) + { + return 0; + } + if (! u2 || UTF8_IS_INVARIANT(*p2)) { + *foldbuf2 = *p2; + } + else { + *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1)); + } + + /* Use another function to handle locale rules. We've made + * sure that both characters to compare are single bytes */ + if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) { + return 0; + } + n1 = n2 = 0; + } + else if (isASCII(*p2)) { + if (flags && ! isASCII(*p1)) { + return 0; + } + n2 = 1; + *foldbuf2 = toLOWER(*p2); + } + else if (u2) { to_utf8_fold(p2, foldbuf2, &n2); } else { @@ -2877,6 +3233,10 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons f2 = foldbuf2; } + /* Here f1 and f2 point to the beginning of the strings to compare. + * These strings are the folds of the input characters, stored in utf8. + */ + /* While there is more to look for in both folds, see if they * continue to match */ while (n1 && n2) {