X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/6f207bd3ddac24959aa7f00f2d7a66f116dcc7ed..19c63c21eb37c467819c765287ec580a86cb6162:/utf8.c diff --git a/utf8.c b/utf8.c index ecfbd69..019d49f 100644 --- a/utf8.c +++ b/utf8.c @@ -1,6 +1,6 @@ /* utf8.c * - * Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, + * Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 * by Larry Wall and others * * You may distribute under the terms of either the GNU General Public @@ -9,26 +9,40 @@ */ /* - * 'What a fix!' said Sam. 'That's the one place in all the lands we've ever - * heard of that we don't want to see any closer; and that's the one place - * we're trying to get to! And that's just where we can't get, nohow.' + * 'What a fix!' said Sam. 'That's the one place in all the lands we've ever + * heard of that we don't want to see any closer; and that's the one place + * we're trying to get to! And that's just where we can't get, nohow.' + * + * [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"] * * 'Well do I understand your speech,' he answered in the same language; * 'yet few strangers do so. Why then do you not speak in the Common Tongue, - * as is the custom in the West, if you wish to be answered?' + * as is the custom in the West, if you wish to be answered?' + * --Gandalf, addressing Théoden's door wardens + * + * [p.508 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"] * * ...the travellers perceived that the floor was paved with stones of many * hues; branching runes and strange devices intertwined beneath their feet. + * + * [p.512 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"] */ #include "EXTERN.h" #define PERL_IN_UTF8_C #include "perl.h" +#ifndef EBCDIC +/* Separate prototypes needed because in ASCII systems these are + * usually macros but they still are compiled as code, too. */ +PERL_CALLCONV UV Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags); +PERL_CALLCONV U8* Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv); +#endif + static const char unees[] = "Malformed UTF-8 character (unexpected end of string)"; -/* +/* =head1 Unicode Support This file contains various utility functions for manipulating UTF8-encoded @@ -37,7 +51,40 @@ Unicode characters as a variable number of bytes, in such a way that characters in the ASCII range are unmodified, and a zero byte never appears within non-zero characters. -=for apidoc A|U8 *|uvuni_to_utf8_flags|U8 *d|UV uv|UV flags +=cut +*/ + +/* +=for apidoc is_ascii_string + +Returns true if the first C bytes of the given string are the same whether +or not the string is encoded in UTF-8 (or UTF-EBCDIC on EBCDIC machines). That +is, if they are invariant. On ASCII-ish machines, only ASCII characters +fit this definition, hence the function's name. + +See also is_utf8_string(), is_utf8_string_loclen(), and is_utf8_string_loc(). + +=cut +*/ + +bool +Perl_is_ascii_string(const U8 *s, STRLEN len) +{ + const U8* const send = s + (len ? len : strlen((const char *)s)); + const U8* x = s; + + PERL_ARGS_ASSERT_IS_ASCII_STRING; + + for (; x < send; ++x) { + if (!UTF8_IS_INVARIANT(*x)) + break; + } + + return x == send; +} + +/* +=for apidoc uvuni_to_utf8_flags Adds the UTF-8 representation of the Unicode codepoint C to the end of the string C; C should be have at least C free @@ -64,6 +111,8 @@ is the recommended Unicode-aware way of saying U8 * Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) { + PERL_ARGS_ASSERT_UVUNI_TO_UTF8_FLAGS; + if (ckWARN(WARN_UTF8)) { if (UNICODE_IS_SURROGATE(uv) && !(flags & UNICODE_ALLOW_SURROGATE)) @@ -80,7 +129,7 @@ Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) !(flags & UNICODE_ALLOW_SUPER)) ) Perl_warner(aTHX_ packWARN(WARN_UTF8), - "Unicode character 0x%04"UVxf" is illegal", uv); + "Unicode non-character 0x%04"UVxf" is illegal for interchange", uv); } if (UNI_IS_INVARIANT(uv)) { *d++ = (U8)UTF_TO_NATIVE(uv); @@ -192,6 +241,8 @@ S_is_utf8_char_slow(const U8 *s, const STRLEN len) STRLEN slen; UV uv, ouv; + PERL_ARGS_ASSERT_IS_UTF8_CHAR_SLOW; + if (UTF8_IS_INVARIANT(u)) return 1; @@ -213,7 +264,7 @@ S_is_utf8_char_slow(const U8 *s, const STRLEN len) if (!UTF8_IS_CONTINUATION(*s)) return 0; uv = UTF8_ACCUMULATE(uv, *s); - if (uv < ouv) + if (uv < ouv) return 0; ouv = uv; s++; @@ -226,19 +277,20 @@ S_is_utf8_char_slow(const U8 *s, const STRLEN len) } /* -=for apidoc A|STRLEN|is_utf8_char|const U8 *s +=for apidoc is_utf8_char Tests if some arbitrary number of bytes begins in a valid UTF-8 -character. Note that an INVARIANT (i.e. ASCII) character is a valid -UTF-8 character. The actual number of bytes in the UTF-8 character -will be returned if it is valid, otherwise 0. +character. Note that an INVARIANT (i.e. ASCII on non-EBCDIC machines) +character is a valid UTF-8 character. The actual number of bytes in the UTF-8 +character will be returned if it is valid, otherwise 0. =cut */ STRLEN -Perl_is_utf8_char(pTHX_ const U8 *s) +Perl_is_utf8_char(const U8 *s) { const STRLEN len = UTF8SKIP(s); - PERL_UNUSED_CONTEXT; + + PERL_ARGS_ASSERT_IS_UTF8_CHAR; #ifdef IS_UTF8_CHAR if (IS_UTF8_CHAR_FAST(len)) return IS_UTF8_CHAR(s, len) ? len : 0; @@ -246,29 +298,27 @@ Perl_is_utf8_char(pTHX_ const U8 *s) return is_utf8_char_slow(s, len); } + /* -=for apidoc A|bool|is_utf8_string|const U8 *s|STRLEN len +=for apidoc is_utf8_string Returns true if first C bytes of the given string form a valid UTF-8 string, false otherwise. Note that 'a valid UTF-8 string' does not mean 'a string that contains code points above 0x7F encoded in UTF-8' because a valid ASCII string is a valid UTF-8 string. -See also is_utf8_string_loclen() and is_utf8_string_loc(). +See also is_ascii_string(), is_utf8_string_loclen(), and is_utf8_string_loc(). =cut */ bool -Perl_is_utf8_string(pTHX_ const U8 *s, STRLEN len) +Perl_is_utf8_string(const U8 *s, STRLEN len) { + const U8* const send = s + (len ? len : strlen((const char *)s)); const U8* x = s; - const U8* send; - PERL_UNUSED_CONTEXT; - if (!len) - len = strlen((const char *)s); - send = s + len; + PERL_ARGS_ASSERT_IS_UTF8_STRING; while (x < send) { STRLEN c; @@ -306,7 +356,7 @@ Perl_is_utf8_string(pTHX_ const U8 *s, STRLEN len) /* Implemented as a macro in utf8.h -=for apidoc A|bool|is_utf8_string_loc|const U8 *s|STRLEN len|const U8 **ep +=for apidoc is_utf8_string_loc Like is_utf8_string() but stores the location of the failure (in the case of "utf8ness failure") or the location s+len (in the case of @@ -314,7 +364,7 @@ case of "utf8ness failure") or the location s+len (in the case of See also is_utf8_string_loclen() and is_utf8_string(). -=for apidoc A|bool|is_utf8_string_loclen|const U8 *s|STRLEN len|const U8 **ep|const STRLEN *el +=for apidoc is_utf8_string_loclen Like is_utf8_string() but stores the location of the failure (in the case of "utf8ness failure") or the location s+len (in the case of @@ -327,18 +377,14 @@ See also is_utf8_string_loc() and is_utf8_string(). */ bool -Perl_is_utf8_string_loclen(pTHX_ const U8 *s, STRLEN len, const U8 **ep, STRLEN *el) +Perl_is_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el) { + const U8* const send = s + (len ? len : strlen((const char *)s)); const U8* x = s; - const U8* send; STRLEN c; - PERL_UNUSED_CONTEXT; + STRLEN outlen = 0; - if (!len) - len = strlen((const char *)s); - send = s + len; - if (el) - *el = 0; + PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN; while (x < send) { /* Inline the easy bits of is_utf8_char() here for speed... */ @@ -362,25 +408,24 @@ Perl_is_utf8_string_loclen(pTHX_ const U8 *s, STRLEN len, const U8 **ep, STRLEN goto out; } x += c; - if (el) - (*el)++; + outlen++; } out: + if (el) + *el = outlen; + if (ep) *ep = x; - if (x != send) - return FALSE; - - return TRUE; + return (x == send); } /* -=for apidoc A|UV|utf8n_to_uvuni|const U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags +=for apidoc utf8n_to_uvuni Bottom level UTF-8 decode routine. -Returns the unicode code point value of the first character in the string C +Returns the Unicode code point value of the first character in the string C which is assumed to be in UTF-8 encoding and no longer than C; C will be set to the length, in bytes, of that character. @@ -411,8 +456,11 @@ Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) const UV startbyte = *s; STRLEN expectlen = 0; U32 warning = 0; + SV* sv; + + PERL_ARGS_ASSERT_UTF8N_TO_UVUNI; -/* This list is a superset of the UTF8_ALLOW_XXX. */ +/* This list is a superset of the UTF8_ALLOW_XXX. BUT it isn't, eg SUPER missing XXX */ #define UTF8_WARN_EMPTY 1 #define UTF8_WARN_CONTINUATION 2 @@ -533,57 +581,60 @@ malformed: if (flags & UTF8_CHECK_ONLY) { if (retlen) - *retlen = -1; + *retlen = ((STRLEN) -1); return 0; } if (dowarn) { - SV* const sv = sv_2mortal(newSVpvs("Malformed UTF-8 character ")); + if (warning == UTF8_WARN_FFFF) { + sv = newSVpvs_flags("Unicode non-character ", SVs_TEMP); + Perl_sv_catpvf(aTHX_ sv, "0x%04"UVxf" is illegal for interchange", uv); + } + else { + sv = newSVpvs_flags("Malformed UTF-8 character ", SVs_TEMP); + + switch (warning) { + case 0: /* Intentionally empty. */ break; + case UTF8_WARN_EMPTY: + sv_catpvs(sv, "(empty string)"); + break; + case UTF8_WARN_CONTINUATION: + Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); + break; + case UTF8_WARN_NON_CONTINUATION: + if (s == s0) + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", + (UV)s[1], startbyte); + else { + const int len = (int)(s-s0); + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", + (UV)s[1], len, len > 1 ? "s" : "", startbyte, (int)expectlen); + } - switch (warning) { - case 0: /* Intentionally empty. */ break; - case UTF8_WARN_EMPTY: - sv_catpvs(sv, "(empty string)"); - break; - case UTF8_WARN_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); - break; - case UTF8_WARN_NON_CONTINUATION: - if (s == s0) - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", - (UV)s[1], startbyte); - else { - const int len = (int)(s-s0); - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", - (UV)s[1], len, len > 1 ? "s" : "", startbyte, (int)expectlen); + break; + case UTF8_WARN_FE_FF: + Perl_sv_catpvf(aTHX_ sv, "(byte 0x%02"UVxf")", uv); + break; + case UTF8_WARN_SHORT: + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte); + expectlen = curlen; /* distance for caller to skip */ + break; + case UTF8_WARN_OVERFLOW: + Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", + ouv, *s, startbyte); + break; + case UTF8_WARN_SURROGATE: + Perl_sv_catpvf(aTHX_ sv, "(UTF-16 surrogate 0x%04"UVxf")", uv); + break; + case UTF8_WARN_LONG: + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); + break; + default: + sv_catpvs(sv, "(unknown reason)"); + break; } - - break; - case UTF8_WARN_FE_FF: - Perl_sv_catpvf(aTHX_ sv, "(byte 0x%02"UVxf")", uv); - break; - case UTF8_WARN_SHORT: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", - (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte); - expectlen = curlen; /* distance for caller to skip */ - break; - case UTF8_WARN_OVERFLOW: - Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", - ouv, *s, startbyte); - break; - case UTF8_WARN_SURROGATE: - Perl_sv_catpvf(aTHX_ sv, "(UTF-16 surrogate 0x%04"UVxf")", uv); - break; - case UTF8_WARN_LONG: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", - (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); - break; - case UTF8_WARN_FFFF: - Perl_sv_catpvf(aTHX_ sv, "(character 0x%04"UVxf")", uv); - break; - default: - sv_catpvs(sv, "(unknown reason)"); - break; } if (warning) { @@ -604,7 +655,7 @@ malformed: } /* -=for apidoc A|UV|utf8_to_uvchr|const U8 *s|STRLEN *retlen +=for apidoc utf8_to_uvchr Returns the native character value of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the @@ -619,18 +670,20 @@ returned and retlen is set, if possible, to -1. UV Perl_utf8_to_uvchr(pTHX_ const U8 *s, STRLEN *retlen) { + PERL_ARGS_ASSERT_UTF8_TO_UVCHR; + return utf8n_to_uvchr(s, UTF8_MAXBYTES, retlen, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* -=for apidoc A|UV|utf8_to_uvuni|const U8 *s|STRLEN *retlen +=for apidoc utf8_to_uvuni Returns the Unicode code point of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the length, in bytes, of that character. -This function should only be used when returned UV is considered +This function should only be used when the returned UV is considered an index into the Unicode semantic tables (e.g. swashes). If C does not point to a well-formed UTF-8 character, zero is @@ -642,13 +695,15 @@ returned and retlen is set, if possible, to -1. UV Perl_utf8_to_uvuni(pTHX_ const U8 *s, STRLEN *retlen) { + PERL_ARGS_ASSERT_UTF8_TO_UVUNI; + /* Call the low level routine asking for checks */ return Perl_utf8n_to_uvuni(aTHX_ s, UTF8_MAXBYTES, retlen, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* -=for apidoc A|STRLEN|utf8_length|const U8 *s|const U8 *e +=for apidoc utf8_length Return the length of the UTF-8 char encoded string C in characters. Stops at C (inclusive). If C s> or if the scan would end @@ -663,6 +718,8 @@ Perl_utf8_length(pTHX_ const U8 *s, const U8 *e) dVAR; STRLEN len = 0; + PERL_ARGS_ASSERT_UTF8_LENGTH; + /* Note: cannot use UTF8_IS_...() too eagerly here since e.g. * the bitops (especially ~) can create illegal UTF-8. * In other words: in Perl UTF-8 is not just for Unicode. */ @@ -670,27 +727,28 @@ Perl_utf8_length(pTHX_ const U8 *s, const U8 *e) if (e < s) goto warn_and_return; while (s < e) { - const U8 t = UTF8SKIP(s); - if (e - s < t) { - warn_and_return: - if (ckWARN_d(WARN_UTF8)) { - if (PL_op) - Perl_warner(aTHX_ packWARN(WARN_UTF8), - "%s in %s", unees, OP_DESC(PL_op)); - else - Perl_warner(aTHX_ packWARN(WARN_UTF8), unees); - } - return len; - } - s += t; + if (!UTF8_IS_INVARIANT(*s)) + s += UTF8SKIP(s); + else + s++; len++; } + if (e != s) { + len--; + warn_and_return: + if (PL_op) + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), + "%s in %s", unees, OP_DESC(PL_op)); + else + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees); + } + return len; } /* -=for apidoc A|IV|utf8_distance|const U8 *a|const U8 *b +=for apidoc utf8_distance Returns the number of UTF-8 characters between the UTF-8 pointers C and C. @@ -704,11 +762,13 @@ same UTF-8 buffer. IV Perl_utf8_distance(pTHX_ const U8 *a, const U8 *b) { + PERL_ARGS_ASSERT_UTF8_DISTANCE; + return (a < b) ? -1 * (IV) utf8_length(a, b) : (IV) utf8_length(b, a); } /* -=for apidoc A|U8 *|utf8_hop|U8 *s|I32 off +=for apidoc utf8_hop Return the UTF-8 pointer C displaced by C characters, either forward or backward. @@ -723,6 +783,8 @@ on the first byte of character or just after the last byte of a character. U8 * Perl_utf8_hop(pTHX_ const U8 *s, I32 off) { + PERL_ARGS_ASSERT_UTF8_HOP; + PERL_UNUSED_CONTEXT; /* Note: cannot use UTF8_IS_...() too eagerly here since e.g * the bitops (especially ~) can create illegal UTF-8. @@ -743,13 +805,84 @@ Perl_utf8_hop(pTHX_ const U8 *s, I32 off) } /* -=for apidoc A|U8 *|utf8_to_bytes|U8 *s|STRLEN *len +=for apidoc bytes_cmp_utf8 + +Compares the sequence of characters (stored as octets) in b, blen with the +sequence of characters (stored as UTF-8) in u, ulen. Returns 0 if they are +equal, -1 or -2 if the first string is less than the second string, +1 or +2 +if the first string is greater than the second string. + +-1 or +1 is returned if the shorter string was identical to the start of the +longer string. -2 or +2 is returned if the was a difference between characters +within the strings. + +=cut +*/ + +int +Perl_bytes_cmp_utf8(pTHX_ const U8 *b, STRLEN blen, const U8 *u, STRLEN ulen) +{ + const U8 *const bend = b + blen; + const U8 *const uend = u + ulen; + + PERL_ARGS_ASSERT_BYTES_CMP_UTF8; + + PERL_UNUSED_CONTEXT; + + while (b < bend && u < uend) { + U8 c = *u++; + if (!UTF8_IS_INVARIANT(c)) { + if (UTF8_IS_DOWNGRADEABLE_START(c)) { + if (u < uend) { + U8 c1 = *u++; + if (UTF8_IS_CONTINUATION(c1)) { + c = UTF8_ACCUMULATE(NATIVE_TO_UTF(c), c1); + c = ASCII_TO_NATIVE(c); + } else { + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), + "Malformed UTF-8 character " + "(unexpected non-continuation byte 0x%02x" + ", immediately after start byte 0x%02x)" + /* Dear diag.t, it's in the pod. */ + "%s%s", c1, c, + PL_op ? " in " : "", + PL_op ? OP_DESC(PL_op) : ""); + return -2; + } + } else { + if (PL_op) + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), + "%s in %s", unees, OP_DESC(PL_op)); + else + Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees); + return -2; /* Really want to return undef :-) */ + } + } else { + return -2; + } + } + if (*b != c) { + return *b < c ? -2 : +2; + } + ++b; + } + + if (b == bend && u == uend) + return 0; + + return b < bend ? +1 : -1; +} + +/* +=for apidoc utf8_to_bytes -Converts a string C of length C from UTF-8 into byte encoding. +Converts a string C of length C from UTF-8 into native byte encoding. Unlike C, this over-writes the original string, and updates len to contain the new length. Returns zero on failure, setting C to -1. +If you need a copy of the string, see C. + =cut */ @@ -760,6 +893,8 @@ Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len) U8 * const send = s + *len; U8 *d; + PERL_ARGS_ASSERT_UTF8_TO_BYTES; + /* ensure valid UTF-8 and chars < 256 before updating string */ while (s < send) { U8 c = *s++; @@ -767,7 +902,7 @@ Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len) if (!UTF8_IS_INVARIANT(c) && (!UTF8_IS_DOWNGRADEABLE_START(c) || (s >= send) || !(c = *s++) || !UTF8_IS_CONTINUATION(c))) { - *len = -1; + *len = ((STRLEN) -1); return 0; } } @@ -784,14 +919,15 @@ Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len) } /* -=for apidoc A|U8 *|bytes_from_utf8|const U8 *s|STRLEN *len|bool *is_utf8 +=for apidoc bytes_from_utf8 -Converts a string C of length C from UTF-8 into byte encoding. +Converts a string C of length C from UTF-8 into native byte encoding. Unlike C but like C, returns a pointer to the newly-created string, and updates C to contain the new length. Returns the original string if no conversion occurs, C is unchanged. Do nothing if C points to 0. Sets C to -0 if C is converted or contains all 7bit characters. +0 if C is converted or consisted entirely of characters that are invariant +in utf8 (i.e., US-ASCII on non-EBCDIC machines). =cut */ @@ -804,6 +940,8 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8) const U8 *send; I32 count = 0; + PERL_ARGS_ASSERT_BYTES_FROM_UTF8; + PERL_UNUSED_CONTEXT; if (!*is_utf8) return (U8 *)start; @@ -820,7 +958,7 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8) } } - *is_utf8 = 0; + *is_utf8 = FALSE; Newx(d, (*len) - count + 1, U8); s = start; start = d; @@ -839,13 +977,16 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8) } /* -=for apidoc A|U8 *|bytes_to_utf8|const U8 *s|STRLEN *len +=for apidoc bytes_to_utf8 -Converts a string C of length C from ASCII into UTF-8 encoding. +Converts a string C of length C from the native encoding into UTF-8. Returns a pointer to the newly-created string, and sets C to reflect the new length. -If you want to convert to UTF-8 from other encodings than ASCII, +A NUL character will be written after the end of the string. + +If you want to convert to UTF-8 from encodings other than +the native (Latin1 or EBCDIC), see sv_recode_to_utf8(). =cut @@ -857,6 +998,8 @@ Perl_bytes_to_utf8(pTHX_ const U8 *s, STRLEN *len) const U8 * const send = s + (*len); U8 *d; U8 *dst; + + PERL_ARGS_ASSERT_BYTES_TO_UTF8; PERL_UNUSED_CONTEXT; Newx(d, (*len) * 2 + 1, U8); @@ -888,14 +1031,10 @@ Perl_utf16_to_utf8(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen) U8* pend; U8* dstart = d; - if (bytelen == 1 && p[0] == 0) { /* Be understanding. */ - d[0] = 0; - *newlen = 1; - return d; - } + PERL_ARGS_ASSERT_UTF16_TO_UTF8; if (bytelen & 1) - Perl_croak(aTHX_ "panic: utf16_to_utf8: odd bytelen %"UVf, (UV)bytelen); + Perl_croak(aTHX_ "panic: utf16_to_utf8: odd bytelen %"UVuf, (UV)bytelen); pend = p + bytelen; @@ -903,7 +1042,11 @@ Perl_utf16_to_utf8(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen) UV uv = (p[0] << 8) + p[1]; /* UTF-16BE */ p += 2; if (uv < 0x80) { +#ifdef EBCDIC + *d++ = UNI_TO_NATIVE(uv); +#else *d++ = (U8)uv; +#endif continue; } if (uv < 0x800) { @@ -911,12 +1054,18 @@ Perl_utf16_to_utf8(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen) *d++ = (U8)(( uv & 0x3f) | 0x80); continue; } - if (uv >= 0xd800 && uv < 0xdbff) { /* surrogates */ - UV low = (p[0] << 8) + p[1]; - p += 2; - if (low < 0xdc00 || low >= 0xdfff) + if (uv >= 0xd800 && uv <= 0xdbff) { /* surrogates */ + if (p >= pend) { Perl_croak(aTHX_ "Malformed UTF-16 surrogate"); - uv = ((uv - 0xd800) << 10) + (low - 0xdc00) + 0x10000; + } else { + UV low = (p[0] << 8) + p[1]; + p += 2; + if (low < 0xdc00 || low > 0xdfff) + Perl_croak(aTHX_ "Malformed UTF-16 surrogate"); + uv = ((uv - 0xd800) << 10) + (low - 0xdc00) + 0x10000; + } + } else if (uv >= 0xdc00 && uv <= 0xdfff) { + Perl_croak(aTHX_ "Malformed UTF-16 surrogate"); } if (uv < 0x10000) { *d++ = (U8)(( uv >> 12) | 0xe0); @@ -943,6 +1092,13 @@ Perl_utf16_to_utf8_reversed(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen) { U8* s = (U8*)p; U8* const send = s + bytelen; + + PERL_ARGS_ASSERT_UTF16_TO_UTF8_REVERSED; + + if (bytelen & 1) + Perl_croak(aTHX_ "panic: utf16_to_utf8_reversed: odd bytelen %"UVuf, + (UV)bytelen); + while (s < send) { const U8 tmp = s[0]; s[0] = s[1]; @@ -963,14 +1119,6 @@ Perl_is_uni_alnum(pTHX_ UV c) } bool -Perl_is_uni_alnumc(pTHX_ UV c) -{ - U8 tmpbuf[UTF8_MAXBYTES+1]; - uvchr_to_utf8(tmpbuf, c); - return is_utf8_alnumc(tmpbuf); -} - -bool Perl_is_uni_idfirst(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXBYTES+1]; @@ -1069,6 +1217,8 @@ Perl_is_uni_xdigit(pTHX_ UV c) UV Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp) { + PERL_ARGS_ASSERT_TO_UNI_UPPER; + uvchr_to_utf8(p, c); return to_utf8_upper(p, p, lenp); } @@ -1076,6 +1226,8 @@ Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp) UV Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp) { + PERL_ARGS_ASSERT_TO_UNI_TITLE; + uvchr_to_utf8(p, c); return to_utf8_title(p, p, lenp); } @@ -1083,6 +1235,8 @@ Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp) UV Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp) { + PERL_ARGS_ASSERT_TO_UNI_LOWER; + uvchr_to_utf8(p, c); return to_utf8_lower(p, p, lenp); } @@ -1090,6 +1244,8 @@ Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp) UV Perl_to_uni_fold(pTHX_ UV c, U8* p, STRLEN *lenp) { + PERL_ARGS_ASSERT_TO_UNI_FOLD; + uvchr_to_utf8(p, c); return to_utf8_fold(p, p, lenp); } @@ -1103,12 +1259,6 @@ Perl_is_uni_alnum_lc(pTHX_ UV c) } bool -Perl_is_uni_alnumc_lc(pTHX_ UV c) -{ - return is_uni_alnumc(c); /* XXX no locale support yet */ -} - -bool Perl_is_uni_idfirst_lc(pTHX_ UV c) { return is_uni_idfirst(c); /* XXX no locale support yet */ @@ -1215,6 +1365,9 @@ S_is_utf8_common(pTHX_ const U8 *const p, SV **swash, const char *const swashname) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_COMMON; + if (!is_utf8_char(p)) return FALSE; if (!*swash) @@ -1226,6 +1379,9 @@ bool Perl_is_utf8_alnum(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_ALNUM; + /* NOTE: "IsWord", not "IsAlnum", since Alnum is a true * descendant of isalnum(3), in other words, it doesn't * contain the '_'. --jhi */ @@ -1233,16 +1389,12 @@ Perl_is_utf8_alnum(pTHX_ const U8 *p) } bool -Perl_is_utf8_alnumc(pTHX_ const U8 *p) -{ - dVAR; - return is_utf8_common(p, &PL_utf8_alnumc, "IsAlnumC"); -} - -bool Perl_is_utf8_idfirst(pTHX_ const U8 *p) /* The naming is historical. */ { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_IDFIRST; + if (*p == '_') return TRUE; /* is_utf8_idstart would be more logical. */ @@ -1253,6 +1405,9 @@ bool Perl_is_utf8_idcont(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_IDCONT; + if (*p == '_') return TRUE; return is_utf8_common(p, &PL_utf8_idcont, "IdContinue"); @@ -1262,6 +1417,9 @@ bool Perl_is_utf8_alpha(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_ALPHA; + return is_utf8_common(p, &PL_utf8_alpha, "IsAlpha"); } @@ -1269,6 +1427,9 @@ bool Perl_is_utf8_ascii(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_ASCII; + return is_utf8_common(p, &PL_utf8_ascii, "IsAscii"); } @@ -1276,20 +1437,59 @@ bool Perl_is_utf8_space(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_SPACE; + return is_utf8_common(p, &PL_utf8_space, "IsSpacePerl"); } bool +Perl_is_utf8_perl_space(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_PERL_SPACE; + + return is_utf8_common(p, &PL_utf8_perl_space, "IsPerlSpace"); +} + +bool +Perl_is_utf8_perl_word(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_PERL_WORD; + + return is_utf8_common(p, &PL_utf8_perl_word, "IsPerlWord"); +} + +bool Perl_is_utf8_digit(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_DIGIT; + return is_utf8_common(p, &PL_utf8_digit, "IsDigit"); } bool +Perl_is_utf8_posix_digit(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_POSIX_DIGIT; + + return is_utf8_common(p, &PL_utf8_posix_digit, "IsPosixDigit"); +} + +bool Perl_is_utf8_upper(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_UPPER; + return is_utf8_common(p, &PL_utf8_upper, "IsUppercase"); } @@ -1297,6 +1497,9 @@ bool Perl_is_utf8_lower(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_LOWER; + return is_utf8_common(p, &PL_utf8_lower, "IsLowercase"); } @@ -1304,6 +1507,9 @@ bool Perl_is_utf8_cntrl(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_CNTRL; + return is_utf8_common(p, &PL_utf8_cntrl, "IsCntrl"); } @@ -1311,6 +1517,9 @@ bool Perl_is_utf8_graph(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_GRAPH; + return is_utf8_common(p, &PL_utf8_graph, "IsGraph"); } @@ -1318,6 +1527,9 @@ bool Perl_is_utf8_print(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_PRINT; + return is_utf8_common(p, &PL_utf8_print, "IsPrint"); } @@ -1325,6 +1537,9 @@ bool Perl_is_utf8_punct(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_PUNCT; + return is_utf8_common(p, &PL_utf8_punct, "IsPunct"); } @@ -1332,18 +1547,124 @@ bool Perl_is_utf8_xdigit(pTHX_ const U8 *p) { dVAR; - return is_utf8_common(p, &PL_utf8_xdigit, "Isxdigit"); + + PERL_ARGS_ASSERT_IS_UTF8_XDIGIT; + + return is_utf8_common(p, &PL_utf8_xdigit, "IsXDigit"); } bool Perl_is_utf8_mark(pTHX_ const U8 *p) { dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_MARK; + return is_utf8_common(p, &PL_utf8_mark, "IsM"); } +bool +Perl_is_utf8_X_begin(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_BEGIN; + + return is_utf8_common(p, &PL_utf8_X_begin, "_X_Begin"); +} + +bool +Perl_is_utf8_X_extend(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_EXTEND; + + return is_utf8_common(p, &PL_utf8_X_extend, "_X_Extend"); +} + +bool +Perl_is_utf8_X_prepend(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_PREPEND; + + return is_utf8_common(p, &PL_utf8_X_prepend, "GCB=Prepend"); +} + +bool +Perl_is_utf8_X_non_hangul(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_NON_HANGUL; + + return is_utf8_common(p, &PL_utf8_X_non_hangul, "HST=Not_Applicable"); +} + +bool +Perl_is_utf8_X_L(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_L; + + return is_utf8_common(p, &PL_utf8_X_L, "GCB=L"); +} + +bool +Perl_is_utf8_X_LV(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_LV; + + return is_utf8_common(p, &PL_utf8_X_LV, "GCB=LV"); +} + +bool +Perl_is_utf8_X_LVT(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_LVT; + + return is_utf8_common(p, &PL_utf8_X_LVT, "GCB=LVT"); +} + +bool +Perl_is_utf8_X_T(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_T; + + return is_utf8_common(p, &PL_utf8_X_T, "GCB=T"); +} + +bool +Perl_is_utf8_X_V(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_V; + + return is_utf8_common(p, &PL_utf8_X_V, "GCB=V"); +} + +bool +Perl_is_utf8_X_LV_LVT_V(pTHX_ const U8 *p) +{ + dVAR; + + PERL_ARGS_ASSERT_IS_UTF8_X_LV_LVT_V; + + return is_utf8_common(p, &PL_utf8_X_LV_LVT_V, "_X_LV_LVT_V"); +} + /* -=for apidoc A|UV|to_utf8_case|U8 *p|U8* ustrp|STRLEN *lenp|SV **swash|char *normal|char *special +=for apidoc to_utf8_case The "p" contains the pointer to the UTF-8 string encoding the character that is being converted. @@ -1374,25 +1695,42 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, dVAR; U8 tmpbuf[UTF8_MAXBYTES_CASE+1]; STRLEN len = 0; - const UV uv0 = utf8_to_uvchr(p, NULL); /* The NATIVE_TO_UNI() and UNI_TO_NATIVE() mappings * are necessary in EBCDIC, they are redundant no-ops * in ASCII-ish platforms, and hopefully optimized away. */ const UV uv1 = NATIVE_TO_UNI(uv0); + + PERL_ARGS_ASSERT_TO_UTF8_CASE; + uvuni_to_utf8(tmpbuf, uv1); if (!*swashp) /* load on-demand */ *swashp = swash_init("utf8", normal, &PL_sv_undef, 4, 0); + /* This is the beginnings of a skeleton of code to read the info section + * that is in all the swashes in case we ever want to do that, so one can + * read things whose maps aren't code points, and whose default if missing + * is not to the code point itself. This was just to see if it actually + * worked. Details on what the possibilities are are in perluniprops.pod + HV * const hv = get_hv("utf8::SwashInfo", 0); + if (hv) { + SV **svp; + svp = hv_fetch(hv, (const char*)normal, strlen(normal), FALSE); + const char *s; + + HV * const this_hash = SvRV(*svp); + svp = hv_fetch(this_hash, "type", strlen("type"), FALSE); + s = SvPV_const(*svp, len); + } + }*/ - /* The 0xDF is the only special casing Unicode code point below 0x100. */ - if (special && (uv1 == 0xDF || uv1 > 0xFF)) { + if (special) { /* It might be "special" (sometimes, but not always, * a multicharacter mapping) */ - HV *hv; + HV * const hv = get_hv(special, 0); SV **svp; - if ((hv = get_hv(special, FALSE)) && + if (hv && (svp = hv_fetch(hv, (const char*)tmpbuf, UNISKIP(uv1), FALSE)) && (*svp)) { const char *s; @@ -1446,7 +1784,8 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, } } - if (!len) /* Neither: just copy. */ + if (!len) /* Neither: just copy. In other words, there was no mapping + defined, which means that the code point maps to itself */ len = uvchr_to_utf8(ustrp, uv0) - ustrp; if (lenp) @@ -1456,7 +1795,7 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, } /* -=for apidoc A|UV|to_utf8_upper|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_upper Convert the UTF-8 encoded character at p to its uppercase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1472,12 +1811,15 @@ UV Perl_to_utf8_upper(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { dVAR; + + PERL_ARGS_ASSERT_TO_UTF8_UPPER; + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_toupper, "ToUpper", "utf8::ToSpecUpper"); } /* -=for apidoc A|UV|to_utf8_title|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_title Convert the UTF-8 encoded character at p to its titlecase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1493,12 +1835,15 @@ UV Perl_to_utf8_title(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { dVAR; + + PERL_ARGS_ASSERT_TO_UTF8_TITLE; + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_totitle, "ToTitle", "utf8::ToSpecTitle"); } /* -=for apidoc A|UV|to_utf8_lower|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_lower Convert the UTF-8 encoded character at p to its lowercase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1514,12 +1859,15 @@ UV Perl_to_utf8_lower(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { dVAR; + + PERL_ARGS_ASSERT_TO_UTF8_LOWER; + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_tolower, "ToLower", "utf8::ToSpecLower"); } /* -=for apidoc A|UV|to_utf8_fold|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_fold Convert the UTF-8 encoded character at p to its foldcase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1536,6 +1884,9 @@ UV Perl_to_utf8_fold(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { dVAR; + + PERL_ARGS_ASSERT_TO_UTF8_FOLD; + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_tofold, "ToFold", "utf8::ToSpecFold"); } @@ -1551,17 +1902,17 @@ Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits { dVAR; SV* retval; - SV* const tokenbufsv = sv_newmortal(); dSP; const size_t pkg_len = strlen(pkg); const size_t name_len = strlen(name); - HV * const stash = gv_stashpvn(pkg, pkg_len, FALSE); + HV * const stash = gv_stashpvn(pkg, pkg_len, 0); SV* errsv_save; + PERL_ARGS_ASSERT_SWASH_INIT; + PUSHSTACKi(PERLSI_MAGIC); ENTER; - SAVEI32(PL_hints); - PL_hints = 0; + SAVEHINTS(); save_re_context(); if (!gv_fetchmeth(stash, "SWASHNEW", 8, -1)) { /* demand load utf8 */ ENTER; @@ -1583,18 +1934,12 @@ Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits SPAGAIN; PUSHMARK(SP); EXTEND(SP,5); - PUSHs(sv_2mortal(newSVpvn(pkg, pkg_len))); - PUSHs(sv_2mortal(newSVpvn(name, name_len))); + mPUSHp(pkg, pkg_len); + mPUSHp(name, name_len); PUSHs(listsv); - PUSHs(sv_2mortal(newSViv(minbits))); - PUSHs(sv_2mortal(newSViv(none))); + mPUSHi(minbits); + mPUSHi(none); PUTBACK; - if (IN_PERL_COMPILETIME) { - /* XXX ought to be handled by lex_start */ - SAVEI32(PL_in_my); - PL_in_my = 0; - sv_setpv(tokenbufsv, PL_tokenbuf); - } errsv_save = newSVsv(ERRSV); if (call_method("SWASHNEW", G_SCALAR)) retval = newSVsv(*PL_stack_sp--); @@ -1606,16 +1951,12 @@ Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits LEAVE; POPSTACK; if (IN_PERL_COMPILETIME) { - STRLEN len; - const char* const pv = SvPV_const(tokenbufsv, len); - - Copy(pv, PL_tokenbuf, len+1, char); CopHINTS_set(PL_curcop, PL_hints); } if (!SvROK(retval) || SvTYPE(SvRV(retval)) != SVt_PVHV) { if (SvPOK(retval)) Perl_croak(aTHX_ "Can't find Unicode property definition \"%"SVf"\"", - retval); + SVfARG(retval)); Perl_croak(aTHX_ "SWASHNEW didn't return an HV ref"); } return retval; @@ -1639,7 +1980,7 @@ UV Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) { dVAR; - HV* const hv = (HV*)SvRV(swash); + HV *const hv = MUTABLE_HV(SvRV(swash)); U32 klen; U32 off; STRLEN slen; @@ -1648,7 +1989,9 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) U32 bit; SV *swatch; U8 tmputf8[2]; - UV c = NATIVE_TO_ASCII(*ptr); + const UV c = NATIVE_TO_ASCII(*ptr); + + PERL_ARGS_ASSERT_SWASH_FETCH; if (!do_utf8 && !UNI_IS_INVARIANT(c)) { tmputf8[0] = (U8)UTF8_EIGHT_BIT_HI(c); @@ -1656,7 +1999,7 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) ptr = tmputf8; } /* Given a UTF-X encoded char 0xAA..0xYY,0xZZ - * then the "swatch" is a vec() for al the chars which start + * then the "swatch" is a vec() for all the chars which start * with 0xAA..0xYY * So the key in the hash (klen) is length of encoded char -1 */ @@ -1664,7 +2007,7 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) off = ptr[klen]; if (klen == 0) { - /* If char in invariant then swatch is for all the invariant chars + /* If char is invariant then swatch is for all the invariant chars * In both UTF-8 and UTF-8-MOD that happens to be UTF_CONTINUATION_MARK */ needents = UTF_CONTINUATION_MARK; @@ -1721,7 +2064,8 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) } PL_last_swash_hv = hv; - PL_last_swash_klen = klen; + assert(klen <= sizeof(PL_last_swash_key)); + PL_last_swash_klen = (U8)klen; /* FIXME change interpvar.h? */ PL_last_swash_tmps = (U8 *) tmps; PL_last_swash_slen = slen; @@ -1744,6 +2088,106 @@ Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) return (tmps[off] << 24) + (tmps[off+1] << 16) + (tmps[off+2] << 8) + tmps[off + 3] ; } Perl_croak(aTHX_ "panic: swash_fetch got swatch of unexpected bit width"); + NORETURN_FUNCTION_END; +} + +/* Read a single line of the main body of the swash input text. These are of + * the form: + * 0053 0056 0073 + * where each number is hex. The first two numbers form the minimum and + * maximum of a range, and the third is the value associated with the range. + * Not all swashes should have a third number + * + * On input: l points to the beginning of the line to be examined; it points + * to somewhere in the string of the whole input text, and is + * terminated by a \n or the null string terminator. + * lend points to the null terminator of that string + * wants_value is non-zero if the swash expects a third number + * typestr is the name of the swash's mapping, like 'ToLower' + * On output: *min, *max, and *val are set to the values read from the line. + * returns a pointer just beyond the line examined. If there was no + * valid min number on the line, returns lend+1 + */ + +STATIC U8* +S_swash_scan_list_line(pTHX_ U8* l, U8* const lend, UV* min, UV* max, UV* val, + const bool wants_value, const U8* const typestr) +{ + const int typeto = typestr[0] == 'T' && typestr[1] == 'o'; + STRLEN numlen; /* Length of the number */ + I32 flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX; + + /* nl points to the next \n in the scan */ + U8* const nl = (U8*)memchr(l, '\n', lend - l); + + /* Get the first number on the line: the range minimum */ + numlen = lend - l; + *min = grok_hex((char *)l, &numlen, &flags, NULL); + if (numlen) /* If found a hex number, position past it */ + l += numlen; + else if (nl) { /* Else, go handle next line, if any */ + return nl + 1; /* 1 is length of "\n" */ + } + else { /* Else, no next line */ + return lend + 1; /* to LIST's end at which \n is not found */ + } + + /* The max range value follows, separated by a BLANK */ + if (isBLANK(*l)) { + ++l; + flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX; + numlen = lend - l; + *max = grok_hex((char *)l, &numlen, &flags, NULL); + if (numlen) + l += numlen; + else /* If no value here, it is a single element range */ + *max = *min; + + /* Non-binary tables have a third entry: what the first element of the + * range maps to */ + if (wants_value) { + if (isBLANK(*l)) { + ++l; + flags = PERL_SCAN_SILENT_ILLDIGIT | + PERL_SCAN_DISALLOW_PREFIX; + numlen = lend - l; + *val = grok_hex((char *)l, &numlen, &flags, NULL); + if (numlen) + l += numlen; + else + *val = 0; + } + else { + *val = 0; + if (typeto) { + Perl_croak(aTHX_ "%s: illegal mapping '%s'", + typestr, l); + } + } + } + else + *val = 0; /* bits == 1, then any val should be ignored */ + } + else { /* Nothing following range min, should be single element with no + mapping expected */ + *max = *min; + if (wants_value) { + *val = 0; + if (typeto) { + Perl_croak(aTHX_ "%s: illegal mapping '%s'", typestr, l); + } + } + else + *val = 0; /* bits == 1, then val should be ignored */ + } + + /* Position to next line if any, or EOF */ + if (nl) + l = nl + 1; + else + l = lend; + + return l; } /* Note: @@ -1758,29 +2202,32 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) SV *swatch; U8 *l, *lend, *x, *xend, *s; STRLEN lcur, xcur, scur; + HV *const hv = MUTABLE_HV(SvRV(swash)); - HV* const hv = (HV*)SvRV(swash); + /* The string containing the main body of the table */ SV** const listsvp = hv_fetchs(hv, "LIST", FALSE); + SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE); SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE); SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE); const U8* const typestr = (U8*)SvPV_nolen(*typesvp); - const int typeto = typestr[0] == 'T' && typestr[1] == 'o'; const STRLEN bits = SvUV(*bitssvp); const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */ const UV none = SvUV(*nonesvp); const UV end = start + span; + PERL_ARGS_ASSERT_SWASH_GET; + if (bits != 1 && bits != 8 && bits != 16 && bits != 32) { Perl_croak(aTHX_ "panic: swash_get doesn't expect bits %"UVuf, (UV)bits); } /* create and initialize $swatch */ - swatch = newSVpvs(""); scur = octets ? (span * octets) : (span + 7) / 8; - SvGROW(swatch, scur + 1); + swatch = newSV(scur); + SvPOK_on(swatch); s = (U8*)SvPVX(swatch); if (octets && none) { const U8* const e = s + scur; @@ -1810,79 +2257,19 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) l = (U8*)SvPV(*listsvp, lcur); lend = l + lcur; while (l < lend) { - UV min, max, val, key; - STRLEN numlen; - I32 flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX; - - U8* const nl = (U8*)memchr(l, '\n', lend - l); - - numlen = lend - l; - min = grok_hex((char *)l, &numlen, &flags, NULL); - if (numlen) - l += numlen; - else if (nl) { - l = nl + 1; /* 1 is length of "\n" */ - continue; - } - else { - l = lend; /* to LIST's end at which \n is not found */ + UV min, max, val; + l = S_swash_scan_list_line(aTHX_ l, lend, &min, &max, &val, + cBOOL(octets), typestr); + if (l > lend) { break; } - if (isBLANK(*l)) { - ++l; - flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX; - numlen = lend - l; - max = grok_hex((char *)l, &numlen, &flags, NULL); - if (numlen) - l += numlen; - else - max = min; - - if (octets) { - if (isBLANK(*l)) { - ++l; - flags = PERL_SCAN_SILENT_ILLDIGIT | - PERL_SCAN_DISALLOW_PREFIX; - numlen = lend - l; - val = grok_hex((char *)l, &numlen, &flags, NULL); - if (numlen) - l += numlen; - else - val = 0; - } - else { - val = 0; - if (typeto) { - Perl_croak(aTHX_ "%s: illegal mapping '%s'", - typestr, l); - } - } - } - else - val = 0; /* bits == 1, then val should be ignored */ - } - else { - max = min; - if (octets) { - val = 0; - if (typeto) { - Perl_croak(aTHX_ "%s: illegal mapping '%s'", typestr, l); - } - } - else - val = 0; /* bits == 1, then val should be ignored */ - } - - if (nl) - l = nl + 1; - else - l = lend; - + /* If looking for something beyond this range, go try the next one */ if (max < start) continue; if (octets) { + UV key; if (min < start) { if (!none || val < none) { val += start - min; @@ -1913,6 +2300,7 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) } } else { /* bits == 1, then val should be ignored */ + UV key; if (min < start) min = start; for (key = min; key <= max; key++) { @@ -1938,7 +2326,7 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) U8 *s, *o, *nl; STRLEN slen, olen; - U8 opc = *x++; + const U8 opc = *x++; if (opc == '\n') continue; @@ -1966,7 +2354,7 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) } othersvp = hv_fetch(hv, (char *)namestr, namelen, FALSE); - otherhv = (HV*)SvRV(*othersvp); + otherhv = MUTABLE_HV(SvRV(*othersvp)); otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE); otherbits = (STRLEN)SvUV(*otherbitssvp); if (bits < otherbits) @@ -2008,7 +2396,7 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) else { STRLEN otheroctets = otherbits >> 3; STRLEN offset = 0; - U8* send = s + slen; + U8* const send = s + slen; while (s < send) { UV otherval = 0; @@ -2058,8 +2446,140 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) return swatch; } +HV* +Perl__swash_inversion_hash(pTHX_ SV* swash) +{ + + /* Subject to change or removal. For use only in one place in regexec.c + * + * Returns a hash which is the inversion and closure of a swash mapping. + * For example, consider the input lines: + * 004B 006B + * 004C 006C + * 212A 006B + * + * The returned hash would have two keys, the utf8 for 006B and the utf8 for + * 006C. The value for each key is an array. For 006C, the array would + * have a two elements, the utf8 for itself, and for 004C. For 006B, there + * would be three elements in its array, the utf8 for 006B, 004B and 212A. + * + * Essentially, for any code point, it gives all the code points that map to + * it, or the list of 'froms' for that point. + * + * Currently it only looks at the main body of the swash, and ignores any + * additions or deletions from other swashes */ + + U8 *l, *lend; + STRLEN lcur; + HV *const hv = MUTABLE_HV(SvRV(swash)); + + /* The string containing the main body of the table */ + SV** const listsvp = hv_fetchs(hv, "LIST", FALSE); + + SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE); + SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); + SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE); + /*SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);*/ + const U8* const typestr = (U8*)SvPV_nolen(*typesvp); + const STRLEN bits = SvUV(*bitssvp); + const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */ + const UV none = SvUV(*nonesvp); + + HV* ret = newHV(); + + PERL_ARGS_ASSERT__SWASH_INVERSION_HASH; + + /* Must have at least 8 bits to get the mappings */ + if (bits != 8 && bits != 16 && bits != 32) { + Perl_croak(aTHX_ "panic: swash_inversion_hash doesn't expect bits %"UVuf, + (UV)bits); + } + + /* read $swash->{LIST} */ + l = (U8*)SvPV(*listsvp, lcur); + lend = l + lcur; + + /* Go through each input line */ + while (l < lend) { + UV min, max, val; + UV inverse; + l = S_swash_scan_list_line(aTHX_ l, lend, &min, &max, &val, + cBOOL(octets), typestr); + if (l > lend) { + break; + } + + /* Each element in the range is to be inverted */ + for (inverse = min; inverse <= max; inverse++) { + AV* list; + SV* element; + SV** listp; + IV i; + bool found_key = FALSE; + + /* The key is the inverse mapping */ + char key[UTF8_MAXBYTES+1]; + char* key_end = (char *) uvuni_to_utf8((U8*) key, val); + STRLEN key_len = key_end - key; + + /* And the value is what the forward mapping is from. */ + char utf8_inverse[UTF8_MAXBYTES+1]; + char *utf8_inverse_end = (char *) uvuni_to_utf8((U8*) utf8_inverse, inverse); + + /* Get the list for the map */ + if ((listp = hv_fetch(ret, key, key_len, FALSE))) { + list = (AV*) *listp; + } + else { /* No entry yet for it: create one */ + list = newAV(); + if (! hv_store(ret, key, key_len, (SV*) list, FALSE)) { + Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed"); + } + } + + for (i = 0; i < av_len(list); i++) { + SV** entryp = av_fetch(list, i, FALSE); + SV* entry; + if (entryp == NULL) { + Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed"); + } + entry = *entryp; + if (SvCUR(entry) != key_len) { + continue; + } + if (memEQ(key, SvPVX(entry), key_len)) { + found_key = TRUE; + break; + } + } + if (! found_key) { + element = newSVpvn_flags(key, key_len, SVf_UTF8); + av_push(list, element); + } + + + /* Simply add the value to the list */ + element = newSVpvn_flags(utf8_inverse, utf8_inverse_end - utf8_inverse, SVf_UTF8); + av_push(list, element); + + /* swash_get() increments the value of val for each element in the + * range. That makes more compact tables possible. You can + * express the capitalization, for example, of all consecutive + * letters with a single line: 0061\t007A\t0041 This maps 0061 to + * 0041, 0062 to 0042, etc. I (khw) have never understood 'none', + * and it's not documented, and perhaps not even currently used, + * but I copied the semantics from swash_get(), just in case */ + if (!none || val < none) { + ++val; + } + } + } + + return ret; +} + /* -=for apidoc A|U8 *|uvchr_to_utf8|U8 *d|UV uv +=for apidoc uvchr_to_utf8 Adds the UTF-8 representation of the Native codepoint C to the end of the string C; C should be have at least C free @@ -2081,20 +2601,24 @@ is the recommended wide native character-aware way of saying U8 * Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv) { + PERL_ARGS_ASSERT_UVCHR_TO_UTF8; + return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), 0); } U8 * Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) { + PERL_ARGS_ASSERT_UVCHR_TO_UTF8_FLAGS; + return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), flags); } /* -=for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 +=for apidoc utf8n_to_uvchr flags -Returns the native character value of the first character in the string +Returns the native character value of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the length, in bytes, of that character. @@ -2107,15 +2631,18 @@ Allows length and flags to be passed to low level routine. a real function in case XS code wants it */ UV -Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, +Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) { const UV uv = Perl_utf8n_to_uvuni(aTHX_ s, curlen, retlen, flags); + + PERL_ARGS_ASSERT_UTF8N_TO_UVCHR; + return UNI_TO_NATIVE(uv); } /* -=for apidoc A|char *|pv_uni_display|SV *dsv|U8 *spv|STRLEN len|STRLEN pvlim|UV flags +=for apidoc pv_uni_display Build to the scalar dsv a displayable version of the string spv, length len, the displayable version being at most pvlim bytes long @@ -2137,7 +2664,10 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f int truncated = 0; const char *s, *e; - sv_setpvn(dsv, "", 0); + PERL_ARGS_ASSERT_PV_UNI_DISPLAY; + + sv_setpvs(dsv, ""); + SvUTF8_off(dsv); for (s = (const char *)spv, e = s + len; s < e; s += UTF8SKIP(s)) { UV u; /* This serves double duty as a flag and a character to print after @@ -2169,12 +2699,15 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f default: break; } if (ok) { - Perl_sv_catpvf(aTHX_ dsv, "\\%c", ok); + const char string = ok; + sv_catpvs(dsv, "\\"); + sv_catpvn(dsv, &string, 1); } } /* isPRINT() is the locale-blind version. */ if (!ok && (flags & UNI_DISPLAY_ISPRINT) && isPRINT(c)) { - Perl_sv_catpvf(aTHX_ dsv, "%c", c); + const char string = c; + sv_catpvn(dsv, &string, 1); ok = 1; } } @@ -2183,12 +2716,12 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f } if (truncated) sv_catpvs(dsv, "..."); - + return SvPVX(dsv); } /* -=for apidoc A|char *|sv_uni_display|SV *dsv|SV *ssv|STRLEN pvlim|UV flags +=for apidoc sv_uni_display Build to the scalar dsv a displayable version of the scalar sv, the displayable version being at most pvlim bytes long @@ -2203,29 +2736,45 @@ The pointer to the PV of the dsv is returned. char * Perl_sv_uni_display(pTHX_ SV *dsv, SV *ssv, STRLEN pvlim, UV flags) { + PERL_ARGS_ASSERT_SV_UNI_DISPLAY; + return Perl_pv_uni_display(aTHX_ dsv, (const U8*)SvPVX_const(ssv), SvCUR(ssv), pvlim, flags); } /* -=for apidoc A|I32|ibcmp_utf8|const char *s1|char **pe1|register UV l1|bool u1|const char *s2|char **pe2|register UV l2|bool u2 - -Return true if the strings s1 and s2 differ case-insensitively, false -if not (if they are equal case-insensitively). If u1 is true, the -string s1 is assumed to be in UTF-8-encoded Unicode. If u2 is true, -the string s2 is assumed to be in UTF-8-encoded Unicode. If u1 or u2 -are false, the respective string is assumed to be in native 8-bit -encoding. - -If the pe1 and pe2 are non-NULL, the scanning pointers will be copied -in there (they will point at the beginning of the I character). -If the pointers behind pe1 or pe2 are non-NULL, they are the end -pointers beyond which scanning will not continue under any -circumstances. If the byte lengths l1 and l2 are non-zero, s1+l1 and -s2+l2 will be used as goal end pointers that will also stop the scan, -and which qualify towards defining a successful match: all the scans -that define an explicit length must reach their goal pointers for -a match to succeed). +=for apidoc foldEQ_utf8 + +Returns true if the leading portions of the strings s1 and s2 (either or both +of which may be in UTF-8) are the same case-insensitively; false otherwise. +How far into the strings to compare is determined by other input parameters. + +If u1 is true, the string s1 is assumed to be in UTF-8-encoded Unicode; +otherwise it is assumed to be in native 8-bit encoding. Correspondingly for u2 +with respect to s2. + +If the byte length l1 is non-zero, it says how far into s1 to check for fold +equality. In other words, s1+l1 will be used as a goal to reach. The +scan will not be considered to be a match unless the goal is reached, and +scanning won't continue past that goal. Correspondingly for l2 with respect to +s2. + +If pe1 is non-NULL and the pointer it points to is not NULL, that pointer is +considered an end pointer beyond which scanning of s1 will not continue under +any circumstances. This means that if both l1 and pe1 are specified, and pe1 +is less than s1+l1, the match will never be successful because it can never +get as far as its goal (and in fact is asserted against). Correspondingly for +pe2 with respect to s2. + +At least one of s1 and s2 must have a goal (at least one of l1 and l2 must be +non-zero), and if both do, both have to be +reached for a successful match. Also, if the fold of a character is multiple +characters, all of them must be matched (see tr21 reference below for +'folding'). + +Upon a successful match, if pe1 is non-NULL, +it will be set to point to the beginning of the I character of s1 beyond +what was matched. Correspondingly for pe2 and s2. For case-insensitiveness, the "casefolding" of Unicode is used instead of upper/lowercasing both the characters, see @@ -2233,92 +2782,138 @@ http://www.unicode.org/unicode/reports/tr21/ (Case Mappings). =cut */ I32 -Perl_ibcmp_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2) -{ - dVAR; - register const U8 *p1 = (const U8*)s1; - register const U8 *p2 = (const U8*)s2; - register const U8 *f1 = NULL; - register const U8 *f2 = NULL; - register U8 *e1 = NULL; - register U8 *q1 = NULL; - register U8 *e2 = NULL; - register U8 *q2 = NULL; - STRLEN n1 = 0, n2 = 0; - U8 foldbuf1[UTF8_MAXBYTES_CASE+1]; - U8 foldbuf2[UTF8_MAXBYTES_CASE+1]; - U8 natbuf[1+1]; - STRLEN foldlen1, foldlen2; - bool match; - - if (pe1) - e1 = *(U8**)pe1; - if (e1 == 0 || (l1 && l1 < (UV)(e1 - (const U8*)s1))) - f1 = (const U8*)s1 + l1; - if (pe2) - e2 = *(U8**)pe2; - if (e2 == 0 || (l2 && l2 < (UV)(e2 - (const U8*)s2))) - f2 = (const U8*)s2 + l2; - - if ((e1 == 0 && f1 == 0) || (e2 == 0 && f2 == 0) || (f1 == 0 && f2 == 0)) - return 1; /* mismatch; possible infinite loop or false positive */ - - if (!u1 || !u2) - natbuf[1] = 0; /* Need to terminate the buffer. */ - - while ((e1 == 0 || p1 < e1) && - (f1 == 0 || p1 < f1) && - (e2 == 0 || p2 < e2) && - (f2 == 0 || p2 < f2)) { - if (n1 == 0) { - if (u1) - to_utf8_fold(p1, foldbuf1, &foldlen1); - else { - uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p1))); - to_utf8_fold(natbuf, foldbuf1, &foldlen1); - } - q1 = foldbuf1; - n1 = foldlen1; - } - if (n2 == 0) { - if (u2) - to_utf8_fold(p2, foldbuf2, &foldlen2); - else { - uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p2))); - to_utf8_fold(natbuf, foldbuf2, &foldlen2); - } - q2 = foldbuf2; - n2 = foldlen2; - } - while (n1 && n2) { - if ( UTF8SKIP(q1) != UTF8SKIP(q2) || - (UTF8SKIP(q1) == 1 && *q1 != *q2) || - memNE((char*)q1, (char*)q2, UTF8SKIP(q1)) ) - return 1; /* mismatch */ - n1 -= UTF8SKIP(q1); - q1 += UTF8SKIP(q1); - n2 -= UTF8SKIP(q2); - q2 += UTF8SKIP(q2); - } - if (n1 == 0) - p1 += u1 ? UTF8SKIP(p1) : 1; - if (n2 == 0) - p2 += u2 ? UTF8SKIP(p2) : 1; - - } - - /* A match is defined by all the scans that specified - * an explicit length reaching their final goals. */ - match = (f1 == 0 || p1 == f1) && (f2 == 0 || p2 == f2); - - if (match) { - if (pe1) - *pe1 = (char*)p1; - if (pe2) - *pe2 = (char*)p2; - } - - return match ? 0 : 1; /* 0 match, 1 mismatch */ +Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2) +{ + dVAR; + register const U8 *p1 = (const U8*)s1; /* Point to current char */ + register const U8 *p2 = (const U8*)s2; + register const U8 *g1 = NULL; /* goal for s1 */ + register const U8 *g2 = NULL; + register const U8 *e1 = NULL; /* Don't scan s1 past this */ + register U8 *f1 = NULL; /* Point to current folded */ + register const U8 *e2 = NULL; + register U8 *f2 = NULL; + STRLEN n1 = 0, n2 = 0; /* Number of bytes in current char */ + U8 foldbuf1[UTF8_MAXBYTES_CASE+1]; + U8 foldbuf2[UTF8_MAXBYTES_CASE+1]; + U8 natbuf[2]; /* Holds native 8-bit char converted to utf8; + these always fit in 2 bytes */ + + PERL_ARGS_ASSERT_FOLDEQ_UTF8; + + if (pe1) { + e1 = *(U8**)pe1; + } + + if (l1) { + g1 = (const U8*)s1 + l1; + } + + if (pe2) { + e2 = *(U8**)pe2; + } + + if (l2) { + g2 = (const U8*)s2 + l2; + } + + /* Must have at least one goal */ + assert(g1 || g2); + + if (g1) { + + /* Will never match if goal is out-of-bounds */ + assert(! e1 || e1 >= g1); + + /* Here, there isn't an end pointer, or it is beyond the goal. We + * only go as far as the goal */ + e1 = g1; + } + else { + assert(e1); /* Must have an end for looking at s1 */ + } + + /* Same for goal for s2 */ + if (g2) { + assert(! e2 || e2 >= g2); + e2 = g2; + } + else { + assert(e2); + } + + /* Look through both strings, a character at a time */ + while (p1 < e1 && p2 < e2) { + + /* If at the beginning of a new character in s1, get its fold to use + * and the length of the fold */ + if (n1 == 0) { + if (u1) { + to_utf8_fold(p1, foldbuf1, &n1); + } + else { /* Not utf8, convert to it first and then get fold */ + uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p1))); + to_utf8_fold(natbuf, foldbuf1, &n1); + } + f1 = foldbuf1; + } + + if (n2 == 0) { /* Same for s2 */ + if (u2) { + to_utf8_fold(p2, foldbuf2, &n2); + } + else { + uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p2))); + to_utf8_fold(natbuf, foldbuf2, &n2); + } + f2 = foldbuf2; + } + + /* While there is more to look for in both folds, see if they + * continue to match */ + while (n1 && n2) { + U8 fold_length = UTF8SKIP(f1); + if (fold_length != UTF8SKIP(f2) + || (fold_length == 1 && *f1 != *f2) /* Short circuit memNE + function call for single + character */ + || memNE((char*)f1, (char*)f2, fold_length)) + { + return 0; /* mismatch */ + } + + /* Here, they matched, advance past them */ + n1 -= fold_length; + f1 += fold_length; + n2 -= fold_length; + f2 += fold_length; + } + + /* When reach the end of any fold, advance the input past it */ + if (n1 == 0) { + p1 += u1 ? UTF8SKIP(p1) : 1; + } + if (n2 == 0) { + p2 += u2 ? UTF8SKIP(p2) : 1; + } + } /* End of loop through both strings */ + + /* A match is defined by each scan that specified an explicit length + * reaching its final goal, and the other not having matched a partial + * character (which can happen when the fold of a character is more than one + * character). */ + if (! ((g1 == 0 || p1 == g1) && (g2 == 0 || p2 == g2)) || n1 || n2) { + return 0; + } + + /* Successful match. Set output pointers */ + if (pe1) { + *pe1 = (char*)p1; + } + if (pe2) { + *pe2 = (char*)p2; + } + return 1; } /*