X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/949cf4983af707fbd15e422845f4f3df20505f97..a3ba843fbf4d6d259cd1202c59836d0aed492dd3:/utf8.c diff --git a/utf8.c b/utf8.c index 3a382a2..5f0e08c 100644 --- a/utf8.c +++ b/utf8.c @@ -139,7 +139,7 @@ Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) if (ckWARN_d(WARN_UTF8)) { if (UNICODE_IS_SURROGATE(uv)) { if (flags & UNICODE_WARN_SURROGATE) { - Perl_warner(aTHX_ packWARN(WARN_UTF8), + Perl_ck_warner_d(aTHX_ packWARN(WARN_SURROGATE), "UTF-16 surrogate U+%04"UVXf, uv); } if (flags & UNICODE_DISALLOW_SURROGATE) { @@ -150,7 +150,7 @@ Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) if (flags & UNICODE_WARN_SUPER || (UNICODE_IS_FE_FF(uv) && (flags & UNICODE_WARN_FE_FF))) { - Perl_warner(aTHX_ packWARN(WARN_UTF8), + Perl_ck_warner_d(aTHX_ packWARN(WARN_NON_UNICODE), "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv); } if (flags & UNICODE_DISALLOW_SUPER @@ -161,7 +161,7 @@ Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) } else if (UNICODE_IS_NONCHAR(uv)) { if (flags & UNICODE_WARN_NONCHAR) { - Perl_warner(aTHX_ packWARN(WARN_UTF8), + Perl_ck_warner_d(aTHX_ packWARN(WARN_NONCHAR), "Unicode non-character U+%04"UVXf" is illegal for open interchange", uv); } @@ -547,9 +547,9 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) #define UTF8_WARN_EMPTY 1 #define UTF8_WARN_CONTINUATION 2 #define UTF8_WARN_NON_CONTINUATION 3 -#define UTF8_WARN_SHORT 5 -#define UTF8_WARN_OVERFLOW 6 -#define UTF8_WARN_LONG 8 +#define UTF8_WARN_SHORT 4 +#define UTF8_WARN_OVERFLOW 5 +#define UTF8_WARN_LONG 6 if (curlen == 0 && !(flags & UTF8_ALLOW_EMPTY)) { @@ -580,7 +580,7 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) #else if (uv == 0xfe || uv == 0xff) { if (flags & (UTF8_WARN_SUPER|UTF8_WARN_FE_FF)) { - sv = sv_2mortal(newSVpvf_nocontext("Code point beginning with byte 0x%02"UVXf" is not Unicode, and not portable", uv)); + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "Code point beginning with byte 0x%02"UVXf" is not Unicode, and not portable", uv)); flags &= ~UTF8_WARN_SUPER; /* Only warn once on this problem */ } if (flags & (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_FE_FF)) { @@ -651,7 +651,7 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) } else if (flags & (UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_WARN_ILLEGAL_INTERCHANGE)) { if (UNICODE_IS_SURROGATE(uv)) { if ((flags & (UTF8_WARN_SURROGATE|UTF8_CHECK_ONLY)) == UTF8_WARN_SURROGATE) { - sv = sv_2mortal(newSVpvf_nocontext("UTF-16 surrogate U+%04"UVXf"", uv)); + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "UTF-16 surrogate U+%04"UVXf"", uv)); } if (flags & UTF8_DISALLOW_SURROGATE) { goto disallowed; @@ -659,7 +659,7 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) } else if (UNICODE_IS_NONCHAR(uv)) { if ((flags & (UTF8_WARN_NONCHAR|UTF8_CHECK_ONLY)) == UTF8_WARN_NONCHAR ) { - sv = sv_2mortal(newSVpvf_nocontext("Unicode non-character U+%04"UVXf" is illegal for open interchange", uv)); + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "Unicode non-character U+%04"UVXf" is illegal for open interchange", uv)); } if (flags & UTF8_DISALLOW_NONCHAR) { goto disallowed; @@ -667,7 +667,7 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) } else if ((uv > PERL_UNICODE_MAX)) { if ((flags & (UTF8_WARN_SUPER|UTF8_CHECK_ONLY)) == UTF8_WARN_SUPER) { - sv = sv_2mortal(newSVpvf_nocontext("Code point 0x%04"UVXf" is not Unicode, may not be portable", uv)); + sv = sv_2mortal(Perl_newSVpvf(aTHX_ "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv)); } if (flags & UTF8_DISALLOW_SUPER) { goto disallowed; @@ -699,42 +699,42 @@ malformed: sv = newSVpvs_flags("Malformed UTF-8 character ", SVs_TEMP); } - switch (warning) { - case 0: /* Intentionally empty. */ break; - case UTF8_WARN_EMPTY: - sv_catpvs(sv, "(empty string)"); - break; - case UTF8_WARN_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); - break; - case UTF8_WARN_NON_CONTINUATION: - if (s == s0) - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", - (UV)s[1], startbyte); - else { - const int len = (int)(s-s0); - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", - (UV)s[1], len, len > 1 ? "s" : "", startbyte, (int)expectlen); - } + switch (warning) { + case 0: /* Intentionally empty. */ break; + case UTF8_WARN_EMPTY: + sv_catpvs(sv, "(empty string)"); + break; + case UTF8_WARN_CONTINUATION: + Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); + break; + case UTF8_WARN_NON_CONTINUATION: + if (s == s0) + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", + (UV)s[1], startbyte); + else { + const int len = (int)(s-s0); + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", + (UV)s[1], len, len > 1 ? "s" : "", startbyte, (int)expectlen); + } - break; - case UTF8_WARN_SHORT: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", - (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte); - expectlen = curlen; /* distance for caller to skip */ - break; - case UTF8_WARN_OVERFLOW: - Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", - ouv, *s, startbyte); - break; - case UTF8_WARN_LONG: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", - (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); - break; - default: - sv_catpvs(sv, "(unknown reason)"); - break; - } + break; + case UTF8_WARN_SHORT: + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte); + expectlen = curlen; /* distance for caller to skip */ + break; + case UTF8_WARN_OVERFLOW: + Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", + ouv, *s, startbyte); + break; + case UTF8_WARN_LONG: + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); + break; + default: + sv_catpvs(sv, "(unknown reason)"); + break; + } if (sv) { const char * const s = SvPVX_const(sv); @@ -766,13 +766,14 @@ returned and retlen is set, if possible, to -1. =cut */ + UV Perl_utf8_to_uvchr(pTHX_ const U8 *s, STRLEN *retlen) { PERL_ARGS_ASSERT_UTF8_TO_UVCHR; return utf8n_to_uvchr(s, UTF8_MAXBYTES, retlen, - ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* @@ -798,7 +799,7 @@ Perl_utf8_to_uvuni(pTHX_ const U8 *s, STRLEN *retlen) /* Call the low level routine asking for checks */ return Perl_utf8n_to_uvuni(aTHX_ s, UTF8_MAXBYTES, retlen, - ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* @@ -840,7 +841,7 @@ Perl_utf8_length(pTHX_ const U8 *s, const U8 *e) Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s in %s", unees, OP_DESC(PL_op)); else - Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees); + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s", unees); } return len; @@ -952,7 +953,7 @@ Perl_bytes_cmp_utf8(pTHX_ const U8 *b, STRLEN blen, const U8 *u, STRLEN ulen) Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s in %s", unees, OP_DESC(PL_op)); else - Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees); + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s", unees); return -2; /* Really want to return undef :-) */ } } else { @@ -1340,12 +1341,12 @@ Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp) } UV -Perl_to_uni_fold(pTHX_ UV c, U8* p, STRLEN *lenp) +Perl__to_uni_fold_flags(pTHX_ UV c, U8* p, STRLEN *lenp, U8 flags) { - PERL_ARGS_ASSERT_TO_UNI_FOLD; + PERL_ARGS_ASSERT__TO_UNI_FOLD_FLAGS; uvchr_to_utf8(p, c); - return to_utf8_fold(p, p, lenp); + return _to_utf8_fold_flags(p, p, lenp, flags); } /* for now these all assume no locale info available for Unicode > 255 */ @@ -1500,6 +1501,19 @@ Perl_is_utf8_idfirst(pTHX_ const U8 *p) /* The naming is historical. */ } bool +Perl_is_utf8_xidfirst(pTHX_ const U8 *p) /* The naming is historical. */ +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_XIDFIRST; + + if (*p == '_') + return TRUE; + /* is_utf8_idstart would be more logical. */ + return is_utf8_common(p, &PL_utf8_xidstart, "XIdStart"); +} + +bool Perl_is_utf8_idcont(pTHX_ const U8 *p) { dVAR; @@ -1512,6 +1526,18 @@ Perl_is_utf8_idcont(pTHX_ const U8 *p) } bool +Perl_is_utf8_xidcont(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_XIDCONT; + + if (*p == '_') + return TRUE; + return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue"); +} + +bool Perl_is_utf8_alpha(pTHX_ const U8 *p) { dVAR; @@ -1773,7 +1799,7 @@ of the result. The "swashp" is a pointer to the swash to use. -Both the special and normal mappings are stored lib/unicore/To/Foo.pl, +Both the special and normal mappings are stored in lib/unicore/To/Foo.pl, and loaded by SWASHNEW, using lib/utf8_heavy.pl. The special (usually, but not always, a multicharacter mapping), is tried first. @@ -1801,26 +1827,32 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, PERL_ARGS_ASSERT_TO_UTF8_CASE; + /* Note that swash_fetch() doesn't output warnings for these because it + * assumes we will */ + if (uv1 >= UNICODE_SURROGATE_FIRST) { + if (uv1 <= UNICODE_SURROGATE_LAST) { + if (ckWARN_d(WARN_SURROGATE)) { + const char* desc = (PL_op) ? OP_DESC(PL_op) : normal; + Perl_warner(aTHX_ packWARN(WARN_SURROGATE), + "Operation \"%s\" returns its argument for UTF-16 surrogate U+%04"UVXf"", desc, uv1); + } + } + else if (UNICODE_IS_SUPER(uv1)) { + if (ckWARN_d(WARN_NON_UNICODE)) { + const char* desc = (PL_op) ? OP_DESC(PL_op) : normal; + Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), + "Operation \"%s\" returns its argument for non-Unicode code point 0x%04"UVXf"", desc, uv1); + } + } + + /* Note that non-characters are perfectly legal, so no warning should + * be given */ + } + uvuni_to_utf8(tmpbuf, uv1); if (!*swashp) /* load on-demand */ *swashp = swash_init("utf8", normal, &PL_sv_undef, 4, 0); - /* This is the beginnings of a skeleton of code to read the info section - * that is in all the swashes in case we ever want to do that, so one can - * read things whose maps aren't code points, and whose default if missing - * is not to the code point itself. This was just to see if it actually - * worked. Details on what the possibilities are are in perluniprops.pod - HV * const hv = get_hv("utf8::SwashInfo", 0); - if (hv) { - SV **svp; - svp = hv_fetch(hv, (const char*)normal, strlen(normal), FALSE); - const char *s; - - HV * const this_hash = SvRV(*svp); - svp = hv_fetch(this_hash, "type", strlen("type"), FALSE); - s = SvPV_const(*svp, len); - } - }*/ if (special) { /* It might be "special" (sometimes, but not always, @@ -1978,15 +2010,20 @@ The first character of the foldcased version is returned =cut */ +/* Not currently externally documented is 'flags', which currently is non-zero + * if full case folds are to be used; otherwise simple folds */ + UV -Perl_to_utf8_fold(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) +Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags) { + const char *specials = (flags) ? "utf8::ToSpecFold" : NULL; + dVAR; - PERL_ARGS_ASSERT_TO_UTF8_FOLD; + PERL_ARGS_ASSERT__TO_UTF8_FOLD_FLAGS; return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, - &PL_utf8_tofold, "ToFold", "utf8::ToSpecFold"); + &PL_utf8_tofold, "ToFold", specials); } /* Note: @@ -2121,6 +2158,18 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) /* If char is encoded then swatch is for the prefix */ needents = (1 << UTF_ACCUMULATION_SHIFT); off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK; + if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_NON_UNICODE)) { + const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0); + + /* This outputs warnings for binary properties only, assuming that + * to_utf8_case() will output any. Also, surrogates aren't checked + * for, as that would warn on things like /\p{Gc=Cs}/ */ + SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); + if (SvUV(*bitssvp) == 1) { + Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), + "Code point 0x%04"UVXf" is not Unicode, no properties match it; all inverse properties do", code_point); + } + } } /* @@ -2551,10 +2600,14 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) } HV* -Perl__swash_inversion_hash(pTHX_ SV* swash) +Perl__swash_inversion_hash(pTHX_ SV* const swash) { - /* Subject to change or removal. For use only in one place in regexec.c + /* Subject to change or removal. For use only in one place in regcomp.c. + * Can't be used on a property that is subject to user override, as it + * relies on the value of SPECIALS in the swash which would be set by + * utf8_heavy.pl to the hash in the non-overriden file, and hence is not set + * for overridden properties * * Returns a hash which is the inversion and closure of a swash mapping. * For example, consider the input lines: @@ -2570,8 +2623,22 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) * Essentially, for any code point, it gives all the code points that map to * it, or the list of 'froms' for that point. * - * Currently it only looks at the main body of the swash, and ignores any - * additions or deletions from other swashes */ + * Currently it ignores any additions or deletions from other swashes, + * looking at just the main body of the swash, and if there are SPECIALS + * in the swash, at that hash + * + * The specials hash can be extra code points, and most likely consists of + * maps from single code points to multiple ones (each expressed as a string + * of utf8 characters). This function currently returns only 1-1 mappings. + * However consider this possible input in the specials hash: + * "\xEF\xAC\x85" => "\x{0073}\x{0074}", # U+FB05 => 0073 0074 + * "\xEF\xAC\x86" => "\x{0073}\x{0074}", # U+FB06 => 0073 0074 + * + * Both FB05 and FB06 map to the same multi-char sequence, which we don't + * currently handle. But it also means that FB05 and FB06 are equivalent in + * a 1-1 mapping which we should handle, and this relationship may not be in + * the main table. Therefore this function examines all the multi-char + * sequences and adds the 1-1 mappings that come out of that. */ U8 *l, *lend; STRLEN lcur; @@ -2588,6 +2655,7 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) const STRLEN bits = SvUV(*bitssvp); const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */ const UV none = SvUV(*nonesvp); + SV **specials_p = hv_fetchs(hv, "SPECIALS", 0); HV* ret = newHV(); @@ -2599,6 +2667,114 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) (UV)bits); } + if (specials_p) { /* It might be "special" (sometimes, but not always, a + mapping to more than one character */ + + /* Construct an inverse mapping hash for the specials */ + HV * const specials_hv = MUTABLE_HV(SvRV(*specials_p)); + HV * specials_inverse = newHV(); + char *char_from; /* the lhs of the map */ + I32 from_len; /* its byte length */ + char *char_to; /* the rhs of the map */ + I32 to_len; /* its byte length */ + SV *sv_to; /* and in a sv */ + AV* from_list; /* list of things that map to each 'to' */ + + hv_iterinit(specials_hv); + + /* The keys are the characters (in utf8) that map to the corresponding + * utf8 string value. Iterate through the list creating the inverse + * list. */ + while ((sv_to = hv_iternextsv(specials_hv, &char_from, &from_len))) { + SV** listp; + if (! SvPOK(sv_to)) { + Perl_croak(aTHX_ "panic: value returned from hv_iternextsv() unexpectedly is not a string"); + } + /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Found mapping from %"UVXf", First char of to is %"UVXf"\n", utf8_to_uvchr((U8*) char_from, 0), utf8_to_uvchr((U8*) SvPVX(sv_to), 0)));*/ + + /* Each key in the inverse list is a mapped-to value, and the key's + * hash value is a list of the strings (each in utf8) that map to + * it. Those strings are all one character long */ + if ((listp = hv_fetch(specials_inverse, + SvPVX(sv_to), + SvCUR(sv_to), 0))) + { + from_list = (AV*) *listp; + } + else { /* No entry yet for it: create one */ + from_list = newAV(); + if (! hv_store(specials_inverse, + SvPVX(sv_to), + SvCUR(sv_to), + (SV*) from_list, 0)) + { + Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed"); + } + } + + /* Here have the list associated with this 'to' (perhaps newly + * created and empty). Just add to it. Note that we ASSUME that + * the input is guaranteed to not have duplications, so we don't + * check for that. Duplications just slow down execution time. */ + av_push(from_list, newSVpvn_utf8(char_from, from_len, TRUE)); + } + + /* Here, 'specials_inverse' contains the inverse mapping. Go through + * it looking for cases like the FB05/FB06 examples above. There would + * be an entry in the hash like + * 'st' => [ FB05, FB06 ] + * In this example we will create two lists that get stored in the + * returned hash, 'ret': + * FB05 => [ FB05, FB06 ] + * FB06 => [ FB05, FB06 ] + * + * Note that there is nothing to do if the array only has one element. + * (In the normal 1-1 case handled below, we don't have to worry about + * two lists, as everything gets tied to the single list that is + * generated for the single character 'to'. But here, we are omitting + * that list, ('st' in the example), so must have multiple lists.) */ + while ((from_list = (AV *) hv_iternextsv(specials_inverse, + &char_to, &to_len))) + { + if (av_len(from_list) > 0) { + int i; + + /* We iterate over all combinations of i,j to place each code + * point on each list */ + for (i = 0; i <= av_len(from_list); i++) { + int j; + AV* i_list = newAV(); + SV** entryp = av_fetch(from_list, i, FALSE); + if (entryp == NULL) { + Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed"); + } + if (hv_fetch(ret, SvPVX(*entryp), SvCUR(*entryp), FALSE)) { + Perl_croak(aTHX_ "panic: unexpected entry for %s", SvPVX(*entryp)); + } + if (! hv_store(ret, SvPVX(*entryp), SvCUR(*entryp), + (SV*) i_list, FALSE)) + { + Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed"); + } + + /* For debugging: UV u = utf8_to_uvchr((U8*) SvPVX(*entryp), 0);*/ + for (j = 0; j <= av_len(from_list); j++) { + entryp = av_fetch(from_list, j, FALSE); + if (entryp == NULL) { + Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed"); + } + + /* When i==j this adds itself to the list */ + av_push(i_list, newSVuv(utf8_to_uvchr( + (U8*) SvPVX(*entryp), 0))); + /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", utf8_to_uvchr((U8*) SvPVX(*entryp), 0), u));*/ + } + } + } + } + SvREFCNT_dec(specials_inverse); /* done with it */ + } /* End of specials */ + /* read $swash->{LIST} */ l = (U8*)SvPV(*listsvp, lcur); lend = l + lcur; @@ -2616,20 +2792,16 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) /* Each element in the range is to be inverted */ for (inverse = min; inverse <= max; inverse++) { AV* list; - SV* element; SV** listp; IV i; bool found_key = FALSE; + bool found_inverse = FALSE; /* The key is the inverse mapping */ char key[UTF8_MAXBYTES+1]; char* key_end = (char *) uvuni_to_utf8((U8*) key, val); STRLEN key_len = key_end - key; - /* And the value is what the forward mapping is from. */ - char utf8_inverse[UTF8_MAXBYTES+1]; - char *utf8_inverse_end = (char *) uvuni_to_utf8((U8*) utf8_inverse, inverse); - /* Get the list for the map */ if ((listp = hv_fetch(ret, key, key_len, FALSE))) { list = (AV*) *listp; @@ -2641,38 +2813,51 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) } } - for (i = 0; i < av_len(list); i++) { + /* Look through list to see if this inverse mapping already is + * listed, or if there is a mapping to itself already */ + for (i = 0; i <= av_len(list); i++) { SV** entryp = av_fetch(list, i, FALSE); SV* entry; if (entryp == NULL) { Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed"); } entry = *entryp; - if (SvCUR(entry) != key_len) { - continue; - } - if (memEQ(key, SvPVX(entry), key_len)) { + /*DEBUG_U(PerlIO_printf(Perl_debug_log, "list for %"UVXf" contains %"UVXf"\n", val, SvUV(entry)));*/ + if (SvUV(entry) == val) { found_key = TRUE; + } + if (SvUV(entry) == inverse) { + found_inverse = TRUE; + } + + /* No need to continue searching if found everything we are + * looking for */ + if (found_key && found_inverse) { break; } } + + /* Make sure there is a mapping to itself on the list */ if (! found_key) { - element = newSVpvn_flags(key, key_len, SVf_UTF8); - av_push(list, element); + av_push(list, newSVuv(val)); + /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", val, val));*/ } /* Simply add the value to the list */ - element = newSVpvn_flags(utf8_inverse, utf8_inverse_end - utf8_inverse, SVf_UTF8); - av_push(list, element); + if (! found_inverse) { + av_push(list, newSVuv(inverse)); + /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", inverse, val));*/ + } /* swash_get() increments the value of val for each element in the * range. That makes more compact tables possible. You can * express the capitalization, for example, of all consecutive * letters with a single line: 0061\t007A\t0041 This maps 0061 to * 0041, 0062 to 0042, etc. I (khw) have never understood 'none', - * and it's not documented, and perhaps not even currently used, - * but I copied the semantics from swash_get(), just in case */ + * and it's not documented; it appears to be used only in + * implementing tr//; I copied the semantics from swash_get(), just + * in case */ if (!none || val < none) { ++val; } @@ -2682,6 +2867,81 @@ Perl__swash_inversion_hash(pTHX_ SV* swash) return ret; } +HV* +Perl__swash_to_invlist(pTHX_ SV* const swash) +{ + + /* Subject to change or removal. For use only in one place in regcomp.c */ + + U8 *l, *lend; + char *loc; + STRLEN lcur; + HV *const hv = MUTABLE_HV(SvRV(swash)); + UV elements = 0; /* Number of elements in the inversion list */ + U8 empty[] = ""; + + /* The string containing the main body of the table */ + SV** const listsvp = hv_fetchs(hv, "LIST", FALSE); + SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE); + SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); + + const U8* const typestr = (U8*)SvPV_nolen(*typesvp); + const STRLEN bits = SvUV(*bitssvp); + const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */ + + HV* invlist; + + PERL_ARGS_ASSERT__SWASH_TO_INVLIST; + + /* read $swash->{LIST} */ + if (SvPOK(*listsvp)) { + l = (U8*)SvPV(*listsvp, lcur); + } + else { + /* LIST legitimately doesn't contain a string during compilation phases + * of Perl itself, before the Unicode tables are generated. In this + * case, just fake things up by creating an empty list */ + l = empty; + lcur = 0; + } + loc = (char *) l; + lend = l + lcur; + + /* Scan the input to count the number of lines to preallocate array size + * based on worst possible case, which is each line in the input creates 2 + * elements in the inversion list: 1) the beginning of a range in the list; + * 2) the beginning of a range not in the list. */ + while ((loc = (strchr(loc, '\n'))) != NULL) { + elements += 2; + loc++; + } + + /* If the ending is somehow corrupt and isn't a new line, add another + * element for the final range that isn't in the inversion list */ + if (! (*lend == '\n' || (*lend == '\0' && *(lend - 1) == '\n'))) { + elements++; + } + + invlist = _new_invlist(elements); + + /* Now go through the input again, adding each range to the list */ + while (l < lend) { + UV start, end; + UV val; /* Not used by this function */ + + l = S_swash_scan_list_line(aTHX_ l, lend, &start, &end, &val, + cBOOL(octets), typestr); + + if (l > lend) { + break; + } + + _append_range_to_invlist(invlist, start, end); + } + + return invlist; +} + /* =for apidoc uvchr_to_utf8 @@ -2766,22 +3026,27 @@ Perl_check_utf8_print(pTHX_ register const U8* s, const STRLEN len) if (*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE) { STRLEN char_len; if (UTF8_IS_SUPER(s)) { - UV uv = utf8_to_uvchr(s, &char_len); - Perl_warner(aTHX_ packWARN(WARN_UTF8), - "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv); - ok = FALSE; + if (ckWARN_d(WARN_NON_UNICODE)) { + UV uv = utf8_to_uvchr(s, &char_len); + Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), + "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv); + ok = FALSE; + } } else if (UTF8_IS_SURROGATE(s)) { - UV uv = utf8_to_uvchr(s, &char_len); - Perl_warner(aTHX_ packWARN(WARN_UTF8), - "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv); - ok = FALSE; + if (ckWARN_d(WARN_SURROGATE)) { + UV uv = utf8_to_uvchr(s, &char_len); + Perl_warner(aTHX_ packWARN(WARN_SURROGATE), + "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv); + ok = FALSE; + } } else if - (UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s)) + ((UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s)) + && (ckWARN_d(WARN_NONCHAR))) { UV uv = utf8_to_uvchr(s, &char_len); - Perl_warner(aTHX_ packWARN(WARN_UTF8), + Perl_warner(aTHX_ packWARN(WARN_NONCHAR), "Unicode non-character U+%04"UVXf" is illegal for open interchange", uv); ok = FALSE; } @@ -2932,8 +3197,19 @@ instead of upper/lowercasing both the characters, see http://www.unicode.org/unicode/reports/tr21/ (Case Mappings). =cut */ + +/* A flags parameter has been added which may change, and hence isn't + * externally documented. Currently it is: + * 0 for as-documented above + * FOLDEQ_UTF8_NOMIX_ASCII meaning that if a non-ASCII character folds to an + ASCII one, to not match + * FOLDEQ_UTF8_LOCALE meaning that locale rules are to be used for code + * points below 256; unicode rules for above 255; and + * folds that cross those boundaries are disallowed, + * like the NOMIX_ASCII option + */ I32 -Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2) +Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2, U32 flags) { dVAR; register const U8 *p1 = (const U8*)s1; /* Point to current char */ @@ -2950,7 +3226,7 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons U8 natbuf[2]; /* Holds native 8-bit char converted to utf8; these always fit in 2 bytes */ - PERL_ARGS_ASSERT_FOLDEQ_UTF8; + PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS; if (pe1) { e1 = *(U8**)pe1; @@ -2997,9 +3273,45 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons while (p1 < e1 && p2 < e2) { /* If at the beginning of a new character in s1, get its fold to use - * and the length of the fold */ + * and the length of the fold. (exception: locale rules just get the + * character to a single byte) */ if (n1 == 0) { - if (u1) { + + /* If in locale matching, we use two sets of rules, depending on if + * the code point is above or below 255. Here, we test for and + * handle locale rules */ + if ((flags & FOLDEQ_UTF8_LOCALE) + && (! u1 || UTF8_IS_INVARIANT(*p1) || UTF8_IS_DOWNGRADEABLE_START(*p1))) + { + /* There is no mixing of code points above and below 255. */ + if (u2 && (! UTF8_IS_INVARIANT(*p2) + && ! UTF8_IS_DOWNGRADEABLE_START(*p2))) + { + return 0; + } + + /* We handle locale rules by converting, if necessary, the code + * point to a single byte. */ + if (! u1 || UTF8_IS_INVARIANT(*p1)) { + *foldbuf1 = *p1; + } + else { + *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1)); + } + n1 = 1; + } + else if (isASCII(*p1)) { /* Note, that here won't be both ASCII + and using locale rules */ + + /* If trying to mix non- with ASCII, and not supposed to, fail */ + if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) { + return 0; + } + n1 = 1; + *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are + just lowercased */ + } + else if (u1) { to_utf8_fold(p1, foldbuf1, &n1); } else { /* Not utf8, convert to it first and then get fold */ @@ -3010,7 +3322,38 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons } if (n2 == 0) { /* Same for s2 */ - if (u2) { + if ((flags & FOLDEQ_UTF8_LOCALE) + && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2))) + { + /* Here, the next char in s2 is < 256. We've already worked on + * s1, and if it isn't also < 256, can't match */ + if (u1 && (! UTF8_IS_INVARIANT(*p1) + && ! UTF8_IS_DOWNGRADEABLE_START(*p1))) + { + return 0; + } + if (! u2 || UTF8_IS_INVARIANT(*p2)) { + *foldbuf2 = *p2; + } + else { + *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1)); + } + + /* Use another function to handle locale rules. We've made + * sure that both characters to compare are single bytes */ + if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) { + return 0; + } + n1 = n2 = 0; + } + else if (isASCII(*p2)) { + if (flags && ! isASCII(*p1)) { + return 0; + } + n2 = 1; + *foldbuf2 = toLOWER(*p2); + } + else if (u2) { to_utf8_fold(p2, foldbuf2, &n2); } else { @@ -3020,6 +3363,10 @@ Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, cons f2 = foldbuf2; } + /* Here f1 and f2 point to the beginning of the strings to compare. + * These strings are the folds of the input characters, stored in utf8. + */ + /* While there is more to look for in both folds, see if they * continue to match */ while (n1 && n2) {