X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/d28a9254e445aee7212523d9a7ff62ae0a743fec..e1812838584f1ccec086cb1385d37f694942a1f2:/utf8.c diff --git a/utf8.c b/utf8.c index 000b340..bf5a36e 100644 --- a/utf8.c +++ b/utf8.c @@ -49,12 +49,15 @@ within non-zero characters. */ /* -=for apidoc is_ascii_string +=for apidoc is_invariant_string -Returns true if the first C bytes of the string C are the same whether -or not the string is encoded in UTF-8 (or UTF-EBCDIC on EBCDIC machines). That -is, if they are invariant. On ASCII-ish machines, only ASCII characters -fit this definition, hence the function's name. +Returns true iff the first C bytes of the string C are the same +regardless of the UTF-8 encoding of the string (or UTF-EBCDIC encoding on +EBCDIC machines). That is, if they are UTF-8 invariant. On ASCII-ish +machines, all the ASCII characters and only the ASCII characters fit this +definition. On EBCDIC machines, the ASCII-range characters are invariant, but +so also are the C1 controls and C<\c?> (which isn't in the ASCII range on +EBCDIC). If C is 0, it will be calculated using C, (which means if you use this option, that C can't have embedded C characters and has to @@ -66,12 +69,12 @@ See also L(), L(), and L= UNICODE_SURROGATE_FIRST && ckWARN3_d(WARN_SURROGATE, WARN_NON_UNICODE, WARN_NONCHAR)) @@ -130,6 +138,10 @@ Perl_uvoffuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) if (flags & UNICODE_DISALLOW_SUPER || (UNICODE_IS_FE_FF(uv) && (flags & UNICODE_DISALLOW_FE_FF))) { +#ifdef EBCDIC + Perl_die(aTHX_ "Can't represent character for Ox%"UVXf" on this platform", uv); + NOT_REACHED; +#endif return NULL; } } @@ -307,22 +319,6 @@ Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) } /* -=for apidoc is_utf8_char_buf - -This is identical to the macro L. - -=cut */ - -STRLEN -Perl_is_utf8_char_buf(const U8 *buf, const U8* buf_end) -{ - - PERL_ARGS_ASSERT_IS_UTF8_CHAR_BUF; - - return isUTF8_CHAR(buf, buf_end); -} - -/* =for apidoc is_utf8_string Returns true if the first C bytes of string C form a valid @@ -331,7 +327,7 @@ using C (which means if you use this option, that C can't have embedded C characters and has to have a terminating C byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'. -See also L(), L(), and L(). +See also L(), L(), and L(). =cut */ @@ -794,13 +790,13 @@ Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) * is the label . */ -malformed: + malformed: if (sv && ckWARN_d(WARN_UTF8)) { pack_warn = packWARN(WARN_UTF8); } -disallowed: + disallowed: if (flags & UTF8_CHECK_ONLY) { if (retlen) @@ -808,7 +804,7 @@ disallowed: return 0; } -do_warn: + do_warn: if (pack_warn) { /* was initialized to 0, and changed only if warnings are to be raised. */ @@ -1418,7 +1414,7 @@ Perl__to_upper_title_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp, const char S_ return 'S'; default: Perl_croak(aTHX_ "panic: to_upper_title_latin1 did not expect '%c' to map to '%c'", c, LATIN_SMALL_LETTER_Y_WITH_DIAERESIS); - assert(0); /* NOTREACHED */ + NOT_REACHED; /* NOTREACHED */ } } @@ -1588,22 +1584,23 @@ Perl__to_uni_fold_flags(pTHX_ UV c, U8* p, STRLEN *lenp, U8 flags) PERL_ARGS_ASSERT__TO_UNI_FOLD_FLAGS; - /* Tread a UTF-8 locale as not being in locale at all */ - if (IN_UTF8_CTYPE_LOCALE) { - flags &= ~FOLD_FLAGS_LOCALE; + if (flags & FOLD_FLAGS_LOCALE) { + /* Treat a UTF-8 locale as not being in locale at all */ + if (IN_UTF8_CTYPE_LOCALE) { + flags &= ~FOLD_FLAGS_LOCALE; + } + else { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + goto needs_full_generality; + } } if (c < 256) { - UV result = _to_fold_latin1((U8) c, p, lenp, + return _to_fold_latin1((U8) c, p, lenp, flags & (FOLD_FLAGS_FULL | FOLD_FLAGS_NOMIX_ASCII)); - /* It is illegal for the fold to cross the 255/256 boundary under - * locale; in this case return the original */ - return (result > 256 && flags & FOLD_FLAGS_LOCALE) - ? c - : result; } - /* If no special needs, just use the macro */ + /* Here, above 255. If no special needs, just use the macro */ if ( ! (flags & (FOLD_FLAGS_LOCALE|FOLD_FLAGS_NOMIX_ASCII))) { uvchr_to_utf8(p, c); return CALL_FOLD_CASE(p, p, lenp, flags & FOLD_FLAGS_FULL); @@ -1611,6 +1608,8 @@ Perl__to_uni_fold_flags(pTHX_ UV c, U8* p, STRLEN *lenp, U8 flags) else { /* Otherwise, _to_utf8_fold_flags has the intelligence to deal with the special flags. */ U8 utf8_c[UTF8_MAXBYTES + 1]; + + needs_full_generality: uvchr_to_utf8(utf8_c, c); return _to_utf8_fold_flags(utf8_c, p, lenp, flags); } @@ -1688,7 +1687,7 @@ Perl__is_utf8_perl_idstart(pTHX_ const U8 *p) if (! PL_utf8_perl_idstart) { invlist = _new_invlist_C_array(_Perl_IDStart_invlist); } - return is_utf8_common(p, &PL_utf8_perl_idstart, "", invlist); + return is_utf8_common(p, &PL_utf8_perl_idstart, "_Perl_IDStart", invlist); } bool @@ -1711,7 +1710,7 @@ Perl__is_utf8_perl_idcont(pTHX_ const U8 *p) if (! PL_utf8_perl_idcont) { invlist = _new_invlist_C_array(_Perl_IDCont_invlist); } - return is_utf8_common(p, &PL_utf8_perl_idcont, "", invlist); + return is_utf8_common(p, &PL_utf8_perl_idcont, "_Perl_IDCont", invlist); } bool @@ -1899,14 +1898,23 @@ S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* c s += UTF8SKIP(s); } - /* Here, no characters crossed, result is ok as-is */ + /* Here, no characters crossed, result is ok as-is, but we warn. */ + _CHECK_AND_OUTPUT_WIDE_LOCALE_UTF8_MSG(p, p + UTF8SKIP(p)); return result; } -bad_crossing: + bad_crossing: /* Failed, have to return the original */ original = valid_utf8_to_uvchr(p, lenp); + + /* diag_listed_as: Can't do %s("%s") on non-UTF-8 locale; resolved to "%s". */ + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + "Can't do %s(\"\\x{%"UVXf"}\") on non-UTF-8 locale; " + "resolved to \"\\x{%"UVXf"}\".", + OP_DESC(PL_op), + original, + original); Copy(p, ustrp, *lenp, char); return original; } @@ -1929,8 +1937,14 @@ Perl__to_utf8_upper_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags PERL_ARGS_ASSERT__TO_UTF8_UPPER_FLAGS; - if (flags && IN_UTF8_CTYPE_LOCALE) { - flags = FALSE; + if (flags) { + /* Treat a UTF-8 locale as not being in locale at all */ + if (IN_UTF8_CTYPE_LOCALE) { + flags = FALSE; + } + else { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + } } if (UTF8_IS_INVARIANT(*p)) { @@ -1994,8 +2008,14 @@ Perl__to_utf8_title_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags PERL_ARGS_ASSERT__TO_UTF8_TITLE_FLAGS; - if (flags && IN_UTF8_CTYPE_LOCALE) { - flags = FALSE; + if (flags) { + /* Treat a UTF-8 locale as not being in locale at all */ + if (IN_UTF8_CTYPE_LOCALE) { + flags = FALSE; + } + else { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + } } if (UTF8_IS_INVARIANT(*p)) { @@ -2058,8 +2078,14 @@ Perl__to_utf8_lower_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags PERL_ARGS_ASSERT__TO_UTF8_LOWER_FLAGS; - if (flags && IN_UTF8_CTYPE_LOCALE) { - flags = FALSE; + if (flags) { + /* Treat a UTF-8 locale as not being in locale at all */ + if (IN_UTF8_CTYPE_LOCALE) { + flags = FALSE; + } + else { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + } } if (UTF8_IS_INVARIANT(*p)) { @@ -2133,8 +2159,14 @@ Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags) assert(p != ustrp); /* Otherwise overwrites */ - if (flags & FOLD_FLAGS_LOCALE && IN_UTF8_CTYPE_LOCALE) { - flags &= ~FOLD_FLAGS_LOCALE; + if (flags & FOLD_FLAGS_LOCALE) { + /* Treat a UTF-8 locale as not being in locale at all */ + if (IN_UTF8_CTYPE_LOCALE) { + flags &= ~FOLD_FLAGS_LOCALE; + } + else { + _CHECK_AND_WARN_PROBLEMATIC_LOCALE; + } } if (UTF8_IS_INVARIANT(*p)) { @@ -2162,18 +2194,30 @@ Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags) if (flags & FOLD_FLAGS_LOCALE) { +# define CAP_SHARP_S LATIN_CAPITAL_LETTER_SHARP_S_UTF8 +# define LONG_S_T LATIN_SMALL_LIGATURE_LONG_S_T_UTF8 + + const unsigned int cap_sharp_s_len = sizeof(CAP_SHARP_S) - 1; + const unsigned int long_s_t_len = sizeof(LONG_S_T) - 1; + /* Special case these two characters, as what normally gets * returned under locale doesn't work */ - if (UTF8SKIP(p) == sizeof(LATIN_CAPITAL_LETTER_SHARP_S_UTF8) - 1 - && memEQ((char *) p, LATIN_CAPITAL_LETTER_SHARP_S_UTF8, - sizeof(LATIN_CAPITAL_LETTER_SHARP_S_UTF8) - 1)) + if (UTF8SKIP(p) == cap_sharp_s_len + && memEQ((char *) p, CAP_SHARP_S, cap_sharp_s_len)) { + /* diag_listed_as: Can't do %s("%s") on non-UTF-8 locale; resolved to "%s". */ + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + "Can't do fc(\"\\x{1E9E}\") on non-UTF-8 locale; " + "resolved to \"\\x{17F}\\x{17F}\"."); goto return_long_s; } - else if (UTF8SKIP(p) == sizeof(LATIN_SMALL_LIGATURE_LONG_S_T) - 1 - && memEQ((char *) p, LATIN_SMALL_LIGATURE_LONG_S_T_UTF8, - sizeof(LATIN_SMALL_LIGATURE_LONG_S_T_UTF8) - 1)) + else if (UTF8SKIP(p) == long_s_t_len + && memEQ((char *) p, LONG_S_T, long_s_t_len)) { + /* diag_listed_as: Can't do %s("%s") on non-UTF-8 locale; resolved to "%s". */ + Perl_ck_warner(aTHX_ packWARN(WARN_LOCALE), + "Can't do fc(\"\\x{FB05}\") on non-UTF-8 locale; " + "resolved to \"\\x{FB06}\"."); goto return_ligature_st; } return check_locale_boundary_crossing(p, result, ustrp, lenp); @@ -2368,17 +2412,9 @@ Perl__core_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 m ENTER; if ((errsv_save = GvSV(PL_errgv))) SAVEFREESV(errsv_save); GvSV(PL_errgv) = NULL; +#ifndef NO_TAINT_SUPPORT /* It is assumed that callers of this routine are not passing in * any user derived data. */ - /* XXX The following comment is out of date. The - save_re_context() call used to be right after - SAVEHINTS() above, but no longer exists. Does the - errsv_save bit still apply? */ - /* Need to do this after save_re_context() as it will set - * PL_tainted to 1 while saving $1 etc (see the code after getrx: - * in Perl_magic_get). Even line to create errsv_save can turn on - * PL_tainted. */ -#ifndef NO_TAINT_SUPPORT SAVEBOOL(TAINT_get); TAINT_NOT; #endif @@ -3887,8 +3923,18 @@ L (Case Mappings). * FOLDEQ_LOCALE is set iff the rules from the current underlying * locale are to be used. * FOLDEQ_S1_ALREADY_FOLDED s1 has already been folded before calling this - * routine. This allows that step to be skipped. + * routine. This allows that step to be skipped. + * Currently, this requires s1 to be encoded as UTF-8 + * (u1 must be true), which is asserted for. + * FOLDEQ_S1_FOLDS_SANE With either NOMIX_ASCII or LOCALE, no folds may + * cross certain boundaries. Hence, the caller should + * let this function do the folding instead of + * pre-folding. This code contains an assertion to + * that effect. However, if the caller knows what + * it's doing, it can pass this flag to indicate that, + * and the assertion is skipped. * FOLDEQ_S2_ALREADY_FOLDED Similarly. + * FOLDEQ_S2_FOLDS_SANE */ I32 Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const char *s2, char **pe2, UV l2, bool u2, U32 flags) @@ -3904,11 +3950,15 @@ Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const c STRLEN n1 = 0, n2 = 0; /* Number of bytes in current char */ U8 foldbuf1[UTF8_MAXBYTES_CASE+1]; U8 foldbuf2[UTF8_MAXBYTES_CASE+1]; + U8 flags_for_folder = FOLD_FLAGS_FULL; PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS; assert( ! ((flags & (FOLDEQ_UTF8_NOMIX_ASCII | FOLDEQ_LOCALE)) - && (flags & (FOLDEQ_S1_ALREADY_FOLDED | FOLDEQ_S2_ALREADY_FOLDED)))); + && (((flags & FOLDEQ_S1_ALREADY_FOLDED) + && !(flags & FOLDEQ_S1_FOLDS_SANE)) + || ((flags & FOLDEQ_S2_ALREADY_FOLDED) + && !(flags & FOLDEQ_S2_FOLDS_SANE))))); /* The algorithm is to trial the folds without regard to the flags on * the first line of the above assert(), and then see if the result * violates them. This means that the inputs can't be pre-folded to a @@ -3920,8 +3970,13 @@ Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const c * and /iaa matches are most likely to involve code points 0-255, and this * function only under rare conditions gets called for 0-255. */ - if (IN_UTF8_CTYPE_LOCALE) { - flags &= ~FOLDEQ_LOCALE; + if (flags & FOLDEQ_LOCALE) { + if (IN_UTF8_CTYPE_LOCALE) { + flags &= ~FOLDEQ_LOCALE; + } + else { + flags_for_folder |= FOLD_FLAGS_LOCALE; + } } if (pe1) { @@ -3973,98 +4028,59 @@ Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const c while (p1 < e1 && p2 < e2) { /* If at the beginning of a new character in s1, get its fold to use - * and the length of the fold. (exception: locale rules just get the - * character to a single byte) */ + * and the length of the fold. */ if (n1 == 0) { if (flags & FOLDEQ_S1_ALREADY_FOLDED) { f1 = (U8 *) p1; + assert(u1); n1 = UTF8SKIP(f1); } else { - /* If in locale matching, we use two sets of rules, depending - * on if the code point is above or below 255. Here, we test - * for and handle locale rules */ - if ((flags & FOLDEQ_LOCALE) - && (! u1 || ! UTF8_IS_ABOVE_LATIN1(*p1))) - { - /* There is no mixing of code points above and below 255. */ - if (u2 && UTF8_IS_ABOVE_LATIN1(*p2)) { - return 0; - } - - /* We handle locale rules by converting, if necessary, the - * code point to a single byte. */ - if (! u1 || UTF8_IS_INVARIANT(*p1)) { - *foldbuf1 = *p1; - } - else { - *foldbuf1 = TWO_BYTE_UTF8_TO_NATIVE(*p1, *(p1 + 1)); - } - n1 = 1; - } - else if (isASCII(*p1)) { /* Note, that here won't be both - ASCII and using locale rules */ - - /* If trying to mix non- with ASCII, and not supposed to, - * fail */ - if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) { - return 0; - } - n1 = 1; - *foldbuf1 = toFOLD(*p1); - } - else if (u1) { - to_utf8_fold(p1, foldbuf1, &n1); - } - else { /* Not utf8, get utf8 fold */ - to_uni_fold(*p1, foldbuf1, &n1); - } - f1 = foldbuf1; - } + if (isASCII(*p1) && ! (flags & FOLDEQ_LOCALE)) { + + /* We have to forbid mixing ASCII with non-ASCII if the + * flags so indicate. And, we can short circuit having to + * call the general functions for this common ASCII case, + * all of whose non-locale folds are also ASCII, and hence + * UTF-8 invariants, so the UTF8ness of the strings is not + * relevant. */ + if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) { + return 0; + } + n1 = 1; + *foldbuf1 = toFOLD(*p1); + } + else if (u1) { + _to_utf8_fold_flags(p1, foldbuf1, &n1, flags_for_folder); + } + else { /* Not utf8, get utf8 fold */ + _to_uni_fold_flags(*p1, foldbuf1, &n1, flags_for_folder); + } + f1 = foldbuf1; + } } if (n2 == 0) { /* Same for s2 */ if (flags & FOLDEQ_S2_ALREADY_FOLDED) { f2 = (U8 *) p2; + assert(u2); n2 = UTF8SKIP(f2); } else { - if ((flags & FOLDEQ_LOCALE) - && (! u2 || ! UTF8_IS_ABOVE_LATIN1(*p2))) - { - /* Here, the next char in s2 is < 256. We've already - * worked on s1, and if it isn't also < 256, can't match */ - if (u1 && UTF8_IS_ABOVE_LATIN1(*p1)) { - return 0; - } - if (! u2 || UTF8_IS_INVARIANT(*p2)) { - *foldbuf2 = *p2; - } - else { - *foldbuf2 = TWO_BYTE_UTF8_TO_NATIVE(*p2, *(p2 + 1)); - } - - /* Use another function to handle locale rules. We've made - * sure that both characters to compare are single bytes */ - if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) { - return 0; - } - n1 = n2 = 0; - } - else if (isASCII(*p2)) { - if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p1)) { - return 0; - } - n2 = 1; - *foldbuf2 = toFOLD(*p2); - } - else if (u2) { - to_utf8_fold(p2, foldbuf2, &n2); - } - else { - to_uni_fold(*p2, foldbuf2, &n2); - } - f2 = foldbuf2; + if (isASCII(*p2) && ! (flags & FOLDEQ_LOCALE)) { + if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p1)) { + return 0; + } + n2 = 1; + *foldbuf2 = toFOLD(*p2); + } + else if (u2) { + _to_utf8_fold_flags(p2, foldbuf2, &n2, flags_for_folder); + } + else { + _to_uni_fold_flags(*p2, foldbuf2, &n2, flags_for_folder); + } + f2 = foldbuf2; } }