* heard of that we don't want to see any closer; and that's the one place
* we're trying to get to! And that's just where we can't get, nohow.'
*
- * [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"]
+ * [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"]
*
* 'Well do I understand your speech,' he answered in the same language;
* 'yet few strangers do so. Why then do you not speak in the Common Tongue,
* as is the custom in the West, if you wish to be answered?'
- * --Gandalf, addressing Théoden's door wardens
+ * --Gandalf, addressing Théoden's door wardens
*
* [p.508 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"]
*
if (ckWARN_d(WARN_UTF8)) {
if (UNICODE_IS_SURROGATE(uv)) {
if (flags & UNICODE_WARN_SURROGATE) {
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_SURROGATE),
"UTF-16 surrogate U+%04"UVXf, uv);
}
if (flags & UNICODE_DISALLOW_SURROGATE) {
if (flags & UNICODE_WARN_SUPER
|| (UNICODE_IS_FE_FF(uv) && (flags & UNICODE_WARN_FE_FF)))
{
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_NON_UNICODE),
"Code point 0x%04"UVXf" is not Unicode, may not be portable", uv);
}
if (flags & UNICODE_DISALLOW_SUPER
}
else if (UNICODE_IS_NONCHAR(uv)) {
if (flags & UNICODE_WARN_NONCHAR) {
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_NONCHAR),
"Unicode non-character U+%04"UVXf" is illegal for open interchange",
uv);
}
character is a valid UTF-8 character. The actual number of bytes in the UTF-8
character will be returned if it is valid, otherwise 0.
+WARNING: use only if you *know* that C<s> has at least either UTF8_MAXBYTES or
+UTF8SKIP(s) bytes.
+
=cut */
STRLEN
Perl_is_utf8_char(const U8 *s)
Returns true if first C<len> bytes of the given string form a valid
UTF-8 string, false otherwise. If C<len> is 0, it will be calculated
-using C<strlen(s)>. Note that 'a valid UTF-8 string' does not mean 'a
-string that contains code points above 0x7F encoded in UTF-8' because a
-valid ASCII string is a valid UTF-8 string.
+using C<strlen(s)> (which means if you use this option, that C<s> has to have a
+terminating NUL byte). Note that all characters being ASCII constitute 'a
+valid UTF-8 string'.
See also is_ascii_string(), is_utf8_string_loclen(), and is_utf8_string_loc().
PERL_ARGS_ASSERT_IS_UTF8_STRING;
while (x < send) {
- STRLEN c;
/* Inline the easy bits of is_utf8_char() here for speed... */
- if (UTF8_IS_INVARIANT(*x))
- c = 1;
+ if (UTF8_IS_INVARIANT(*x)) {
+ x++;
+ }
else if (!UTF8_IS_START(*x))
- goto out;
+ return FALSE;
else {
/* ... and call is_utf8_char() only if really needed. */
-#ifdef IS_UTF8_CHAR
- c = UTF8SKIP(x);
+ const STRLEN c = UTF8SKIP(x);
+ const U8* const next_char_ptr = x + c;
+
+ if (next_char_ptr > send) {
+ return FALSE;
+ }
+
if (IS_UTF8_CHAR_FAST(c)) {
if (!IS_UTF8_CHAR(x, c))
- c = 0;
+ return FALSE;
}
- else
- c = is_utf8_char_slow(x, c);
-#else
- c = is_utf8_char(x);
-#endif /* #ifdef IS_UTF8_CHAR */
- if (!c)
- goto out;
+ else if (! is_utf8_char_slow(x, c)) {
+ return FALSE;
+ }
+ x = next_char_ptr;
}
- x += c;
}
- out:
- if (x != send)
- return FALSE;
-
return TRUE;
}
PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN;
while (x < send) {
+ const U8* next_char_ptr;
+
/* Inline the easy bits of is_utf8_char() here for speed... */
if (UTF8_IS_INVARIANT(*x))
- c = 1;
+ next_char_ptr = x + 1;
else if (!UTF8_IS_START(*x))
goto out;
else {
/* ... and call is_utf8_char() only if really needed. */
-#ifdef IS_UTF8_CHAR
c = UTF8SKIP(x);
+ next_char_ptr = c + x;
+ if (next_char_ptr > send) {
+ goto out;
+ }
if (IS_UTF8_CHAR_FAST(c)) {
if (!IS_UTF8_CHAR(x, c))
c = 0;
} else
c = is_utf8_char_slow(x, c);
-#else
- c = is_utf8_char(x);
-#endif /* #ifdef IS_UTF8_CHAR */
if (!c)
goto out;
}
- x += c;
+ x = next_char_ptr;
outlen++;
}
Certain code points are considered problematic. These are Unicode surrogates,
Unicode non-characters, and code points above the Unicode maximum of 0x10FFF.
By default these are considered regular code points, but certain situations
-warrant special handling for them. if C<flags> contains
+warrant special handling for them. If C<flags> contains
UTF8_DISALLOW_ILLEGAL_INTERCHANGE, all three classes are treated as
malformations and handled as such. The flags UTF8_DISALLOW_SURROGATE,
UTF8_DISALLOW_NONCHAR, and UTF8_DISALLOW_SUPER (meaning above the legal Unicode
the others that are above the Unicode legal maximum. There are several
reasons, one of which is that the original UTF-8 specification never went above
this number (the current 0x10FFF limit was imposed later). The UTF-8 encoding
-on ASCII platforms for these large code point begins with a byte containing
+on ASCII platforms for these large code points begins with a byte containing
0xFE or 0xFF. The UTF8_DISALLOW_FE_FF flag will cause them to be treated as
malformations, while allowing smaller above-Unicode code points. (Of course
UTF8_DISALLOW_SUPER will treat all above-Unicode code points, including these,
Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8),
"%s in %s", unees, OP_DESC(PL_op));
else
- Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees);
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s", unees);
}
return len;
Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8),
"%s in %s", unees, OP_DESC(PL_op));
else
- Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees);
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s", unees);
return -2; /* Really want to return undef :-) */
}
} else {
=cut
*/
+/* This logic is duplicated in sv_catpvn_flags, so any bug fixes will
+ likewise need duplication. */
+
U8*
Perl_bytes_to_utf8(pTHX_ const U8 *s, STRLEN *len)
{
return utf16_to_utf8(p, d, bytelen, newlen);
}
-/* for now these are all defined (inefficiently) in terms of the utf8 versions */
+/* for now these are all defined (inefficiently) in terms of the utf8 versions.
+ * Note that the macros in handy.h that call these short-circuit calling them
+ * for Latin-1 range inputs */
bool
Perl_is_uni_alnum(pTHX_ UV c)
bool
Perl_is_uni_ascii(pTHX_ UV c)
{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return is_utf8_ascii(tmpbuf);
+ return isASCII(c);
}
bool
bool
Perl_is_uni_cntrl(pTHX_ UV c)
{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return is_utf8_cntrl(tmpbuf);
+ return isCNTRL_L1(c);
}
bool
}
UV
+Perl__to_upper_title_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp, const char S_or_s)
+{
+ /* We have the latin1-range values compiled into the core, so just use
+ * those, converting the result to utf8. The only difference between upper
+ * and title case in this range is that LATIN_SMALL_LETTER_SHARP_S is
+ * either "SS" or "Ss". Which one to use is passed into the routine in
+ * 'S_or_s' to avoid a test */
+
+ UV converted = toUPPER_LATIN1_MOD(c);
+
+ PERL_ARGS_ASSERT__TO_UPPER_TITLE_LATIN1;
+
+ assert(S_or_s == 'S' || S_or_s == 's');
+
+ if (UNI_IS_INVARIANT(converted)) { /* No difference between the two for
+ characters in this range */
+ *p = (U8) converted;
+ *lenp = 1;
+ return converted;
+ }
+
+ /* toUPPER_LATIN1_MOD gives the correct results except for three outliers,
+ * which it maps to one of them, so as to only have to have one check for
+ * it in the main case */
+ if (UNLIKELY(converted == LATIN_SMALL_LETTER_Y_WITH_DIAERESIS)) {
+ switch (c) {
+ case LATIN_SMALL_LETTER_Y_WITH_DIAERESIS:
+ converted = LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS;
+ break;
+ case MICRO_SIGN:
+ converted = GREEK_CAPITAL_LETTER_MU;
+ break;
+ case LATIN_SMALL_LETTER_SHARP_S:
+ *(p)++ = 'S';
+ *p = S_or_s;
+ *lenp = 2;
+ return 'S';
+ default:
+ Perl_croak(aTHX_ "panic: to_upper_title_latin1 did not expect '%c' to map to '%c'", c, LATIN_SMALL_LETTER_Y_WITH_DIAERESIS);
+ /* NOTREACHED */
+ }
+ }
+
+ *(p)++ = UTF8_TWO_BYTE_HI(converted);
+ *p = UTF8_TWO_BYTE_LO(converted);
+ *lenp = 2;
+
+ return converted;
+}
+
+/* Call the function to convert a UTF-8 encoded character to the specified case.
+ * Note that there may be more than one character in the result.
+ * INP is a pointer to the first byte of the input character
+ * OUTP will be set to the first byte of the string of changed characters. It
+ * needs to have space for UTF8_MAXBYTES_CASE+1 bytes
+ * LENP will be set to the length in bytes of the string of changed characters
+ *
+ * The functions return the ordinal of the first character in the string of OUTP */
+#define CALL_UPPER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_toupper, "ToUc", "utf8::ToSpecUc")
+#define CALL_TITLE_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_totitle, "ToTc", "utf8::ToSpecTc")
+#define CALL_LOWER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tolower, "ToLc", "utf8::ToSpecLc")
+
+/* This additionally has the input parameter SPECIALS, which if non-zero will
+ * cause this to use the SPECIALS hash for folding (meaning get full case
+ * folding); otherwise, when zero, this implies a simple case fold */
+#define CALL_FOLD_CASE(INP, OUTP, LENP, SPECIALS) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tofold, "ToCf", (SPECIALS) ? "utf8::ToSpecCf" : NULL)
+
+UV
Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ dVAR;
+
+ /* Convert the Unicode character whose ordinal is c to its uppercase
+ * version and store that in UTF-8 in p and its length in bytes in lenp.
+ * Note that the p needs to be at least UTF8_MAXBYTES_CASE+1 bytes since
+ * the changed version may be longer than the original character.
+ *
+ * The ordinal of the first character of the changed version is returned
+ * (but note, as explained above, that there may be more.) */
+
PERL_ARGS_ASSERT_TO_UNI_UPPER;
+ if (c < 256) {
+ return _to_upper_title_latin1((U8) c, p, lenp, 'S');
+ }
+
uvchr_to_utf8(p, c);
- return to_utf8_upper(p, p, lenp);
+ return CALL_UPPER_CASE(p, p, lenp);
}
UV
Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ dVAR;
+
PERL_ARGS_ASSERT_TO_UNI_TITLE;
+ if (c < 256) {
+ return _to_upper_title_latin1((U8) c, p, lenp, 's');
+ }
+
uvchr_to_utf8(p, c);
- return to_utf8_title(p, p, lenp);
+ return CALL_TITLE_CASE(p, p, lenp);
+}
+
+STATIC U8
+S_to_lower_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp)
+{
+ /* We have the latin1-range values compiled into the core, so just use
+ * those, converting the result to utf8. Since the result is always just
+ * one character, we allow p to be NULL */
+
+ U8 converted = toLOWER_LATIN1(c);
+
+ if (p != NULL) {
+ if (UNI_IS_INVARIANT(converted)) {
+ *p = converted;
+ *lenp = 1;
+ }
+ else {
+ *p = UTF8_TWO_BYTE_HI(converted);
+ *(p+1) = UTF8_TWO_BYTE_LO(converted);
+ *lenp = 2;
+ }
+ }
+ return converted;
}
UV
Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ dVAR;
+
PERL_ARGS_ASSERT_TO_UNI_LOWER;
+ if (c < 256) {
+ return to_lower_latin1((U8) c, p, lenp);
+ }
+
uvchr_to_utf8(p, c);
- return to_utf8_lower(p, p, lenp);
+ return CALL_LOWER_CASE(p, p, lenp);
}
UV
-Perl_to_uni_fold(pTHX_ UV c, U8* p, STRLEN *lenp)
+Perl__to_fold_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp, const bool flags)
{
- PERL_ARGS_ASSERT_TO_UNI_FOLD;
+ /* Corresponds to to_lower_latin1(), flags is TRUE if to use full case
+ * folding */
+
+ UV converted;
+
+ PERL_ARGS_ASSERT__TO_FOLD_LATIN1;
+
+ if (c == MICRO_SIGN) {
+ converted = GREEK_SMALL_LETTER_MU;
+ }
+ else if (flags && c == LATIN_SMALL_LETTER_SHARP_S) {
+ *(p)++ = 's';
+ *p = 's';
+ *lenp = 2;
+ return 's';
+ }
+ else { /* In this range the fold of all other characters is their lower
+ case */
+ converted = toLOWER_LATIN1(c);
+ }
+
+ if (UNI_IS_INVARIANT(converted)) {
+ *p = (U8) converted;
+ *lenp = 1;
+ }
+ else {
+ *(p)++ = UTF8_TWO_BYTE_HI(converted);
+ *p = UTF8_TWO_BYTE_LO(converted);
+ *lenp = 2;
+ }
+
+ return converted;
+}
+
+UV
+Perl__to_uni_fold_flags(pTHX_ UV c, U8* p, STRLEN *lenp, const bool flags)
+{
+
+ /* Not currently externally documented, and subject to change, <flags> is
+ * TRUE iff full folding is to be used */
+
+ PERL_ARGS_ASSERT__TO_UNI_FOLD_FLAGS;
+
+ if (c < 256) {
+ return _to_fold_latin1((U8) c, p, lenp, flags);
+ }
uvchr_to_utf8(p, c);
- return to_utf8_fold(p, p, lenp);
+ return CALL_FOLD_CASE(p, p, lenp, flags);
}
/* for now these all assume no locale info available for Unicode > 255 */
}
bool
-Perl_is_utf8_idcont(pTHX_ const U8 *p)
+Perl_is_utf8_xidfirst(pTHX_ const U8 *p) /* The naming is historical. */
{
dVAR;
- PERL_ARGS_ASSERT_IS_UTF8_IDCONT;
+ PERL_ARGS_ASSERT_IS_UTF8_XIDFIRST;
if (*p == '_')
return TRUE;
+ /* is_utf8_idstart would be more logical. */
+ return is_utf8_common(p, &PL_utf8_xidstart, "XIdStart");
+}
+
+bool
+Perl__is_utf8__perl_idstart(pTHX_ const U8 *p)
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT__IS_UTF8__PERL_IDSTART;
+
+ return is_utf8_common(p, &PL_utf8_perl_idstart, "_Perl_IDStart");
+}
+
+bool
+Perl_is_utf8_idcont(pTHX_ const U8 *p)
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT_IS_UTF8_IDCONT;
+
return is_utf8_common(p, &PL_utf8_idcont, "IdContinue");
}
bool
+Perl_is_utf8_xidcont(pTHX_ const U8 *p)
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT_IS_UTF8_XIDCONT;
+
+ return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue");
+}
+
+bool
Perl_is_utf8_alpha(pTHX_ const U8 *p)
{
dVAR;
PERL_ARGS_ASSERT_IS_UTF8_ASCII;
- return is_utf8_common(p, &PL_utf8_ascii, "IsAscii");
+ /* ASCII characters are the same whether in utf8 or not. So the macro
+ * works on both utf8 and non-utf8 representations. */
+ return isASCII(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_SPACE;
- return is_utf8_common(p, &PL_utf8_space, "IsSpacePerl");
+ return is_utf8_common(p, &PL_utf8_space, "IsXPerlSpace");
}
bool
PERL_ARGS_ASSERT_IS_UTF8_PERL_SPACE;
- return is_utf8_common(p, &PL_utf8_perl_space, "IsPerlSpace");
+ /* Only true if is an ASCII space-like character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isSPACE_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_PERL_WORD;
- return is_utf8_common(p, &PL_utf8_perl_word, "IsPerlWord");
+ /* Only true if is an ASCII word character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isWORDCHAR_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_POSIX_DIGIT;
- return is_utf8_common(p, &PL_utf8_posix_digit, "IsPosixDigit");
+ /* Only true if is an ASCII digit character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isDIGIT_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_CNTRL;
- return is_utf8_common(p, &PL_utf8_cntrl, "IsCntrl");
+ if (isASCII(*p)) {
+ return isCNTRL_A(*p);
+ }
+
+ /* All controls are in Latin1 */
+ if (! UTF8_IS_DOWNGRADEABLE_START(*p)) {
+ return 0;
+ }
+ return isCNTRL_L1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
}
bool
The "swashp" is a pointer to the swash to use.
-Both the special and normal mappings are stored lib/unicore/To/Foo.pl,
+Both the special and normal mappings are stored in lib/unicore/To/Foo.pl,
and loaded by SWASHNEW, using lib/utf8_heavy.pl. The special (usually,
but not always, a multicharacter mapping), is tried first.
/* Note that swash_fetch() doesn't output warnings for these because it
* assumes we will */
- if (uv1 >= UNICODE_SURROGATE_FIRST && ckWARN_d(WARN_UTF8)) {
+ if (uv1 >= UNICODE_SURROGATE_FIRST) {
if (uv1 <= UNICODE_SURROGATE_LAST) {
- const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Operation \"%s\" returns its argument for UTF-16 surrogate U+%04"UVXf"", desc, uv1);
+ if (ckWARN_d(WARN_SURROGATE)) {
+ const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
+ Perl_warner(aTHX_ packWARN(WARN_SURROGATE),
+ "Operation \"%s\" returns its argument for UTF-16 surrogate U+%04"UVXf"", desc, uv1);
+ }
}
else if (UNICODE_IS_SUPER(uv1)) {
- const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Operation \"%s\" returns its argument for non-Unicode code point 0x%04"UVXf"", desc, uv1);
+ if (ckWARN_d(WARN_NON_UNICODE)) {
+ const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
+ "Operation \"%s\" returns its argument for non-Unicode code point 0x%04"UVXf"", desc, uv1);
+ }
}
/* Note that non-characters are perfectly legal, so no warning should
if (!*swashp) /* load on-demand */
*swashp = swash_init("utf8", normal, &PL_sv_undef, 4, 0);
- /* This is the beginnings of a skeleton of code to read the info section
- * that is in all the swashes in case we ever want to do that, so one can
- * read things whose maps aren't code points, and whose default if missing
- * is not to the code point itself. This was just to see if it actually
- * worked. Details on what the possibilities are are in perluniprops.pod
- HV * const hv = get_hv("utf8::SwashInfo", 0);
- if (hv) {
- SV **svp;
- svp = hv_fetch(hv, (const char*)normal, strlen(normal), FALSE);
- const char *s;
-
- HV * const this_hash = SvRV(*svp);
- svp = hv_fetch(this_hash, "type", strlen("type"), FALSE);
- s = SvPV_const(*svp, len);
- }
- }*/
if (special) {
/* It might be "special" (sometimes, but not always,
return len ? utf8_to_uvchr(ustrp, 0) : 0;
}
+STATIC UV
+S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* const ustrp, STRLEN *lenp)
+{
+ /* This is called when changing the case of a utf8-encoded character above
+ * the Latin1 range, and the operation is in locale. If the result
+ * contains a character that crosses the 255/256 boundary, disallow the
+ * change, and return the original code point. See L<perlfunc/lc> for why;
+ *
+ * p points to the original string whose case was changed
+ * result the code point of the first character in the changed-case string
+ * ustrp points to the changed-case string (<result> represents its first char)
+ * lenp points to the length of <ustrp> */
+
+ UV original; /* To store the first code point of <p> */
+
+ PERL_ARGS_ASSERT_CHECK_LOCALE_BOUNDARY_CROSSING;
+
+ assert(! UTF8_IS_INVARIANT(*p) && ! UTF8_IS_DOWNGRADEABLE_START(*p));
+
+ /* We know immediately if the first character in the string crosses the
+ * boundary, so can skip */
+ if (result > 255) {
+
+ /* Look at every character in the result; if any cross the
+ * boundary, the whole thing is disallowed */
+ U8* s = ustrp + UTF8SKIP(ustrp);
+ U8* e = ustrp + *lenp;
+ while (s < e) {
+ if (UTF8_IS_INVARIANT(*s) || UTF8_IS_DOWNGRADEABLE_START(*s))
+ {
+ goto bad_crossing;
+ }
+ s += UTF8SKIP(s);
+ }
+
+ /* Here, no characters crossed, result is ok as-is */
+ return result;
+ }
+
+bad_crossing:
+
+ /* Failed, have to return the original */
+ original = utf8_to_uvchr(p, lenp);
+ Copy(p, ustrp, *lenp, char);
+ return original;
+}
+
/*
=for apidoc to_utf8_upper
=cut */
+/* Not currently externally documented, and subject to change:
+ * <flags> is set iff locale semantics are to be used for code points < 256
+ * <tainted_ptr> if non-null, *tainted_ptr will be set TRUE iff locale rules
+ * were used in the calculation; otherwise unchanged. */
+
UV
-Perl_to_utf8_upper(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp)
+Perl__to_utf8_upper_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, const bool flags, bool* tainted_ptr)
{
dVAR;
- PERL_ARGS_ASSERT_TO_UTF8_UPPER;
+ UV result;
+
+ PERL_ARGS_ASSERT__TO_UTF8_UPPER_FLAGS;
+
+ if (UTF8_IS_INVARIANT(*p)) {
+ if (flags) {
+ result = toUPPER_LC(*p);
+ }
+ else {
+ return _to_upper_title_latin1(*p, ustrp, lenp, 'S');
+ }
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ if (flags) {
+ result = toUPPER_LC(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
+ }
+ else {
+ return _to_upper_title_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp, 'S');
+ }
+ }
+ else { /* utf8, ord above 255 */
+ result = CALL_UPPER_CASE(p, ustrp, lenp);
+
+ if (flags) {
+ result = check_locale_boundary_crossing(p, result, ustrp, lenp);
+ }
+ return result;
+ }
+
+ /* Here, used locale rules. Convert back to utf8 */
+ if (UTF8_IS_INVARIANT(result)) {
+ *ustrp = (U8) result;
+ *lenp = 1;
+ }
+ else {
+ *ustrp = UTF8_EIGHT_BIT_HI(result);
+ *(ustrp + 1) = UTF8_EIGHT_BIT_LO(result);
+ *lenp = 2;
+ }
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_toupper, "ToUpper", "utf8::ToSpecUpper");
+ if (tainted_ptr) {
+ *tainted_ptr = TRUE;
+ }
+ return result;
}
/*
=cut */
+/* Not currently externally documented, and subject to change:
+ * <flags> is set iff locale semantics are to be used for code points < 256
+ * Since titlecase is not defined in POSIX, uppercase is used instead
+ * for these/
+ * <tainted_ptr> if non-null, *tainted_ptr will be set TRUE iff locale rules
+ * were used in the calculation; otherwise unchanged. */
+
UV
-Perl_to_utf8_title(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp)
+Perl__to_utf8_title_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, const bool flags, bool* tainted_ptr)
{
dVAR;
- PERL_ARGS_ASSERT_TO_UTF8_TITLE;
+ UV result;
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_totitle, "ToTitle", "utf8::ToSpecTitle");
+ PERL_ARGS_ASSERT__TO_UTF8_TITLE_FLAGS;
+
+ if (UTF8_IS_INVARIANT(*p)) {
+ if (flags) {
+ result = toUPPER_LC(*p);
+ }
+ else {
+ return _to_upper_title_latin1(*p, ustrp, lenp, 's');
+ }
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ if (flags) {
+ result = toUPPER_LC(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
+ }
+ else {
+ return _to_upper_title_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp, 's');
+ }
+ }
+ else { /* utf8, ord above 255 */
+ result = CALL_TITLE_CASE(p, ustrp, lenp);
+
+ if (flags) {
+ result = check_locale_boundary_crossing(p, result, ustrp, lenp);
+ }
+ return result;
+ }
+
+ /* Here, used locale rules. Convert back to utf8 */
+ if (UTF8_IS_INVARIANT(result)) {
+ *ustrp = (U8) result;
+ *lenp = 1;
+ }
+ else {
+ *ustrp = UTF8_EIGHT_BIT_HI(result);
+ *(ustrp + 1) = UTF8_EIGHT_BIT_LO(result);
+ *lenp = 2;
+ }
+
+ if (tainted_ptr) {
+ *tainted_ptr = TRUE;
+ }
+ return result;
}
/*
=cut */
+/* Not currently externally documented, and subject to change:
+ * <flags> is set iff locale semantics are to be used for code points < 256
+ * <tainted_ptr> if non-null, *tainted_ptr will be set TRUE iff locale rules
+ * were used in the calculation; otherwise unchanged. */
+
UV
-Perl_to_utf8_lower(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp)
+Perl__to_utf8_lower_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, const bool flags, bool* tainted_ptr)
{
+ UV result;
+
dVAR;
- PERL_ARGS_ASSERT_TO_UTF8_LOWER;
+ PERL_ARGS_ASSERT__TO_UTF8_LOWER_FLAGS;
+
+ if (UTF8_IS_INVARIANT(*p)) {
+ if (flags) {
+ result = toLOWER_LC(*p);
+ }
+ else {
+ return to_lower_latin1(*p, ustrp, lenp);
+ }
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ if (flags) {
+ result = toLOWER_LC(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
+ }
+ else {
+ return to_lower_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp);
+ }
+ }
+ else { /* utf8, ord above 255 */
+ result = CALL_LOWER_CASE(p, ustrp, lenp);
+
+ if (flags) {
+ result = check_locale_boundary_crossing(p, result, ustrp, lenp);
+ }
+
+ return result;
+ }
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_tolower, "ToLower", "utf8::ToSpecLower");
+ /* Here, used locale rules. Convert back to utf8 */
+ if (UTF8_IS_INVARIANT(result)) {
+ *ustrp = (U8) result;
+ *lenp = 1;
+ }
+ else {
+ *ustrp = UTF8_EIGHT_BIT_HI(result);
+ *(ustrp + 1) = UTF8_EIGHT_BIT_LO(result);
+ *lenp = 2;
+ }
+
+ if (tainted_ptr) {
+ *tainted_ptr = TRUE;
+ }
+ return result;
}
/*
=cut */
+/* Not currently externally documented, and subject to change,
+ * in <flags>
+ * bit FOLD_FLAGS_LOCALE is set iff locale semantics are to be used for code
+ * points < 256. Since foldcase is not defined in
+ * POSIX, lowercase is used instead
+ * bit FOLD_FLAGS_FULL is set iff full case folds are to be used;
+ * otherwise simple folds
+ * <tainted_ptr> if non-null, *tainted_ptr will be set TRUE iff locale rules
+ * were used in the calculation; otherwise unchanged. */
+
UV
-Perl_to_utf8_fold(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp)
+Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags, bool* tainted_ptr)
{
dVAR;
- PERL_ARGS_ASSERT_TO_UTF8_FOLD;
+ UV result;
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_tofold, "ToFold", "utf8::ToSpecFold");
+ PERL_ARGS_ASSERT__TO_UTF8_FOLD_FLAGS;
+
+ if (UTF8_IS_INVARIANT(*p)) {
+ if (flags & FOLD_FLAGS_LOCALE) {
+ result = toLOWER_LC(*p);
+ }
+ else {
+ return _to_fold_latin1(*p, ustrp, lenp,
+ cBOOL(flags & FOLD_FLAGS_FULL));
+ }
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ if (flags & FOLD_FLAGS_LOCALE) {
+ result = toLOWER_LC(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
+ }
+ else {
+ return _to_fold_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp, cBOOL(flags & FOLD_FLAGS_FULL));
+ }
+ }
+ else { /* utf8, ord above 255 */
+ result = CALL_FOLD_CASE(p, ustrp, lenp, flags);
+
+ if ((flags & FOLD_FLAGS_LOCALE)) {
+ result = check_locale_boundary_crossing(p, result, ustrp, lenp);
+ }
+
+ return result;
+ }
+
+ /* Here, used locale rules. Convert back to utf8 */
+ if (UTF8_IS_INVARIANT(result)) {
+ *ustrp = (U8) result;
+ *lenp = 1;
+ }
+ else {
+ *ustrp = UTF8_EIGHT_BIT_HI(result);
+ *(ustrp + 1) = UTF8_EIGHT_BIT_LO(result);
+ *lenp = 2;
+ }
+
+ if (tainted_ptr) {
+ *tainted_ptr = TRUE;
+ }
+ return result;
}
/* Note:
- * A "swash" is a swatch hash.
- * A "swatch" is a bit vector generated by utf8.c:S_swash_get().
+ * Returns a "swash" which is a hash described in utf8.c:Perl_swash_fetch().
* C<pkg> is a pointer to a package name for SWASHNEW, should be "utf8".
* For other parameters, see utf8::SWASHNEW in lib/utf8_heavy.pl.
*/
+
SV*
Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits, I32 none)
{
+ PERL_ARGS_ASSERT_SWASH_INIT;
+
+ /* Returns a copy of a swash initiated by the called function. This is the
+ * public interface, and returning a copy prevents others from doing
+ * mischief on the original */
+
+ return newSVsv(_core_swash_init(pkg, name, listsv, minbits, none, FALSE, NULL, FALSE));
+}
+
+SV*
+Perl__core_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits, I32 none, bool return_if_undef, SV* invlist, bool passed_in_invlist_has_user_defined_property)
+{
+ /* Initialize and return a swash, creating it if necessary. It does this
+ * by calling utf8_heavy.pl in the general case.
+ *
+ * This interface should only be used by functions that won't destroy or
+ * adversely change the swash, as doing so affects all other uses of the
+ * swash in the program; the general public should use 'Perl_swash_init'
+ * instead.
+ *
+ * pkg is the name of the package that <name> should be in.
+ * name is the name of the swash to find. Typically it is a Unicode
+ * property name, including user-defined ones
+ * listsv is a string to initialize the swash with. It must be of the form
+ * documented as the subroutine return value in
+ * L<perlunicode/User-Defined Character Properties>
+ * minbits is the number of bits required to represent each data element.
+ * It is '1' for binary properties.
+ * none I (khw) do not understand this one, but it is used only in tr///.
+ * return_if_undef is TRUE if the routine shouldn't croak if it can't find
+ * the requested property
+ * invlist is an inversion list to initialize the swash with (or NULL)
+ * has_user_defined_property is TRUE if <invlist> has some component that
+ * came from a user-defined property
+ *
+ * Thus there are three possible inputs to find the swash: <name>,
+ * <listsv>, and <invlist>. At least one must be specified. The result
+ * will be the union of the specified ones, although <listsv>'s various
+ * actions can intersect, etc. what <name> gives.
+ *
+ * <invlist> is only valid for binary properties */
+
dVAR;
- SV* retval;
- dSP;
- const size_t pkg_len = strlen(pkg);
- const size_t name_len = strlen(name);
- HV * const stash = gv_stashpvn(pkg, pkg_len, 0);
- SV* errsv_save;
- GV *method;
+ SV* retval = &PL_sv_undef;
- PERL_ARGS_ASSERT_SWASH_INIT;
+ assert(listsv != &PL_sv_undef || strNE(name, "") || invlist);
+ assert(! invlist || minbits == 1);
+
+ /* If data was passed in to go out to utf8_heavy to find the swash of, do
+ * so */
+ if (listsv != &PL_sv_undef || strNE(name, "")) {
+ dSP;
+ const size_t pkg_len = strlen(pkg);
+ const size_t name_len = strlen(name);
+ HV * const stash = gv_stashpvn(pkg, pkg_len, 0);
+ SV* errsv_save;
+ GV *method;
- PUSHSTACKi(PERLSI_MAGIC);
- ENTER;
- SAVEHINTS();
- save_re_context();
- method = gv_fetchmeth(stash, "SWASHNEW", 8, -1);
- if (!method) { /* demand load utf8 */
+ PERL_ARGS_ASSERT__CORE_SWASH_INIT;
+
+ PUSHSTACKi(PERLSI_MAGIC);
ENTER;
+ SAVEHINTS();
+ save_re_context();
+ if (PL_parser && PL_parser->error_count)
+ SAVEI8(PL_parser->error_count), PL_parser->error_count = 0;
+ method = gv_fetchmeth(stash, "SWASHNEW", 8, -1);
+ if (!method) { /* demand load utf8 */
+ ENTER;
+ errsv_save = newSVsv(ERRSV);
+ /* It is assumed that callers of this routine are not passing in
+ * any user derived data. */
+ /* Need to do this after save_re_context() as it will set
+ * PL_tainted to 1 while saving $1 etc (see the code after getrx:
+ * in Perl_magic_get). Even line to create errsv_save can turn on
+ * PL_tainted. */
+ SAVEBOOL(PL_tainted);
+ PL_tainted = 0;
+ Perl_load_module(aTHX_ PERL_LOADMOD_NOIMPORT, newSVpvn(pkg,pkg_len),
+ NULL);
+ if (!SvTRUE(ERRSV))
+ sv_setsv(ERRSV, errsv_save);
+ SvREFCNT_dec(errsv_save);
+ LEAVE;
+ }
+ SPAGAIN;
+ PUSHMARK(SP);
+ EXTEND(SP,5);
+ mPUSHp(pkg, pkg_len);
+ mPUSHp(name, name_len);
+ PUSHs(listsv);
+ mPUSHi(minbits);
+ mPUSHi(none);
+ PUTBACK;
errsv_save = newSVsv(ERRSV);
- /* It is assumed that callers of this routine are not passing in any
- user derived data. */
- /* Need to do this after save_re_context() as it will set PL_tainted to
- 1 while saving $1 etc (see the code after getrx: in Perl_magic_get).
- Even line to create errsv_save can turn on PL_tainted. */
- SAVEBOOL(PL_tainted);
- PL_tainted = 0;
- Perl_load_module(aTHX_ PERL_LOADMOD_NOIMPORT, newSVpvn(pkg,pkg_len),
- NULL);
+ /* If we already have a pointer to the method, no need to use
+ * call_method() to repeat the lookup. */
+ if (method ? call_sv(MUTABLE_SV(method), G_SCALAR)
+ : call_sv(newSVpvs_flags("SWASHNEW", SVs_TEMP), G_SCALAR | G_METHOD))
+ {
+ retval = *PL_stack_sp--;
+ SvREFCNT_inc(retval);
+ }
if (!SvTRUE(ERRSV))
sv_setsv(ERRSV, errsv_save);
SvREFCNT_dec(errsv_save);
LEAVE;
+ POPSTACK;
+ if (IN_PERL_COMPILETIME) {
+ CopHINTS_set(PL_curcop, PL_hints);
+ }
+ if (!SvROK(retval) || SvTYPE(SvRV(retval)) != SVt_PVHV) {
+ if (SvPOK(retval))
+
+ /* If caller wants to handle missing properties, let them */
+ if (return_if_undef) {
+ return NULL;
+ }
+ Perl_croak(aTHX_
+ "Can't find Unicode property definition \"%"SVf"\"",
+ SVfARG(retval));
+ Perl_croak(aTHX_ "SWASHNEW didn't return an HV ref");
+ }
+ } /* End of calling the module to find the swash */
+
+ /* Make sure there is an inversion list for binary properties */
+ if (minbits == 1) {
+ SV** swash_invlistsvp = NULL;
+ SV* swash_invlist = NULL;
+ bool invlist_in_swash_is_valid = FALSE;
+ HV* swash_hv = NULL;
+
+ /* If this operation fetched a swash, get its already existing
+ * inversion list or create one for it */
+ if (retval != &PL_sv_undef) {
+ swash_hv = MUTABLE_HV(SvRV(retval));
+
+ swash_invlistsvp = hv_fetchs(swash_hv, "INVLIST", FALSE);
+ if (swash_invlistsvp) {
+ swash_invlist = *swash_invlistsvp;
+ invlist_in_swash_is_valid = TRUE;
+ }
+ else {
+ swash_invlist = _swash_to_invlist(retval);
+ }
+ }
+
+ /* If an inversion list was passed in, have to include it */
+ if (invlist) {
+
+ /* Any fetched swash will by now have an inversion list in it;
+ * otherwise <swash_invlist> will be NULL, indicating that we
+ * didn't fetch a swash */
+ if (swash_invlist) {
+
+ /* Add the passed-in inversion list, which invalidates the one
+ * already stored in the swash */
+ invlist_in_swash_is_valid = FALSE;
+ _invlist_union(invlist, swash_invlist, &swash_invlist);
+ }
+ else {
+
+ /* Here, there is no swash already. Set up a minimal one */
+ swash_hv = newHV();
+ retval = newRV_inc(MUTABLE_SV(swash_hv));
+ swash_invlist = invlist;
+ }
+
+ if (passed_in_invlist_has_user_defined_property) {
+ if (! hv_stores(swash_hv, "USER_DEFINED", newSVuv(1))) {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+ }
+ }
+
+ /* Here, we have computed the union of all the passed-in data. It may
+ * be that there was an inversion list in the swash which didn't get
+ * touched; otherwise save the one computed one */
+ if (! invlist_in_swash_is_valid) {
+ if (! hv_stores(MUTABLE_HV(SvRV(retval)), "INVLIST", swash_invlist))
+ {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+ }
}
- SPAGAIN;
- PUSHMARK(SP);
- EXTEND(SP,5);
- mPUSHp(pkg, pkg_len);
- mPUSHp(name, name_len);
- PUSHs(listsv);
- mPUSHi(minbits);
- mPUSHi(none);
- PUTBACK;
- errsv_save = newSVsv(ERRSV);
- /* If we already have a pointer to the method, no need to use call_method()
- to repeat the lookup. */
- if (method ? call_sv(MUTABLE_SV(method), G_SCALAR)
- : call_sv(newSVpvs_flags("SWASHNEW", SVs_TEMP), G_SCALAR | G_METHOD))
- retval = newSVsv(*PL_stack_sp--);
- else
- retval = &PL_sv_undef;
- if (!SvTRUE(ERRSV))
- sv_setsv(ERRSV, errsv_save);
- SvREFCNT_dec(errsv_save);
- LEAVE;
- POPSTACK;
- if (IN_PERL_COMPILETIME) {
- CopHINTS_set(PL_curcop, PL_hints);
- }
- if (!SvROK(retval) || SvTYPE(SvRV(retval)) != SVt_PVHV) {
- if (SvPOK(retval))
- Perl_croak(aTHX_ "Can't find Unicode property definition \"%"SVf"\"",
- SVfARG(retval));
- Perl_croak(aTHX_ "SWASHNEW didn't return an HV ref");
- }
+
return retval;
}
* the lower-level routine, and it is similarly broken for returning
* multiple values. --jhi
* For those, you should use to_utf8_case() instead */
-/* Now SWASHGET is recasted into S_swash_get in this file. */
+/* Now SWASHGET is recasted into S_swatch_get in this file. */
/* Note:
* Returns the value of property/mapping C<swash> for the first character
* of the string C<ptr>. If C<do_utf8> is true, the string C<ptr> is
* assumed to be in utf8. If C<do_utf8> is false, the string C<ptr> is
* assumed to be in native 8-bit encoding. Caches the swatch in C<swash>.
+ *
+ * A "swash" is a hash which contains initially the keys/values set up by
+ * SWASHNEW. The purpose is to be able to completely represent a Unicode
+ * property for all possible code points. Things are stored in a compact form
+ * (see utf8_heavy.pl) so that calculation is required to find the actual
+ * property value for a given code point. As code points are looked up, new
+ * key/value pairs are added to the hash, so that the calculation doesn't have
+ * to ever be re-done. Further, each calculation is done, not just for the
+ * desired one, but for a whole block of code points adjacent to that one.
+ * For binary properties on ASCII machines, the block is usually for 64 code
+ * points, starting with a code point evenly divisible by 64. Thus if the
+ * property value for code point 257 is requested, the code goes out and
+ * calculates the property values for all 64 code points between 256 and 319,
+ * and stores these as a single 64-bit long bit vector, called a "swatch",
+ * under the key for code point 256. The key is the UTF-8 encoding for code
+ * point 256, minus the final byte. Thus, if the length of the UTF-8 encoding
+ * for a code point is 13 bytes, the key will be 12 bytes long. If the value
+ * for code point 258 is then requested, this code realizes that it would be
+ * stored under the key for 256, and would find that value and extract the
+ * relevant bit, offset from 256.
+ *
+ * Non-binary properties are stored in as many bits as necessary to represent
+ * their values (32 currently, though the code is more general than that), not
+ * as single bits, but the principal is the same: the value for each key is a
+ * vector that encompasses the property values for all code points whose UTF-8
+ * representations are represented by the key. That is, for all code points
+ * whose UTF-8 representations are length N bytes, and the key is the first N-1
+ * bytes of that.
*/
UV
Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8)
PERL_ARGS_ASSERT_SWASH_FETCH;
+ /* Convert to utf8 if not already */
if (!do_utf8 && !UNI_IS_INVARIANT(c)) {
tmputf8[0] = (U8)UTF8_EIGHT_BIT_HI(c);
tmputf8[1] = (U8)UTF8_EIGHT_BIT_LO(c);
/* If char is encoded then swatch is for the prefix */
needents = (1 << UTF_ACCUMULATION_SHIFT);
off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK;
- if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_UTF8)) {
- const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0);
-
- /* This outputs warnings for binary properties only, assuming that
- * to_utf8_case() will output any. Also, surrogates aren't checked
- * for, as that would warn on things like /\p{Gc=Cs}/ */
- SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
- if (SvUV(*bitssvp) == 1) {
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Code point 0x%04"UVXf" is not Unicode, no properties match it; all inverse properties do", code_point);
- }
- }
}
/*
/* Try our second-level swatch cache, kept in a hash. */
SV** svp = hv_fetch(hv, (const char*)ptr, klen, FALSE);
- /* If not cached, generate it via swash_get */
+ /* If not cached, generate it via swatch_get */
if (!svp || !SvPOK(*svp)
|| !(tmps = (const U8*)SvPV_const(*svp, slen))) {
/* We use utf8n_to_uvuni() as we want an index into
const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0,
ckWARN(WARN_UTF8) ?
0 : UTF8_ALLOW_ANY);
- swatch = swash_get(swash,
+ swatch = swatch_get(swash,
/* On EBCDIC & ~(0xA0-1) isn't a useful thing to do */
- (klen) ? (code_point & ~(needents - 1)) : 0,
+ (klen) ? (code_point & ~((UV)needents - 1)) : 0,
needents);
if (IN_PERL_COMPILETIME)
if (!svp || !(tmps = (U8*)SvPV(*svp, slen))
|| (slen << 3) < needents)
- Perl_croak(aTHX_ "panic: swash_fetch got improper swatch");
+ Perl_croak(aTHX_ "panic: swash_fetch got improper swatch, "
+ "svp=%p, tmps=%p, slen=%"UVuf", needents=%"UVuf,
+ svp, tmps, (UV)slen, (UV)needents);
}
PL_last_swash_hv = hv;
Copy(ptr, PL_last_swash_key, klen, U8);
}
+ if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_NON_UNICODE)) {
+ SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+
+ /* This outputs warnings for binary properties only, assuming that
+ * to_utf8_case() will output any for non-binary. Also, surrogates
+ * aren't checked for, as that would warn on things like /\p{Gc=Cs}/ */
+
+ if (! bitssvp || SvUV(*bitssvp) == 1) {
+ /* User-defined properties can silently match above-Unicode */
+ SV** const user_defined_svp = hv_fetchs(hv, "USER_DEFINED", FALSE);
+ if (! user_defined_svp || ! SvUV(*user_defined_svp)) {
+ const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0);
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
+ "Code point 0x%04"UVXf" is not Unicode, all \\p{} matches fail; all \\P{} matches succeed", code_point);
+ }
+ }
+ }
+
switch ((int)((slen << 3) / needents)) {
case 1:
bit = 1 << (off & 7);
off <<= 2;
return (tmps[off] << 24) + (tmps[off+1] << 16) + (tmps[off+2] << 8) + tmps[off + 3] ;
}
- Perl_croak(aTHX_ "panic: swash_fetch got swatch of unexpected bit width");
+ Perl_croak(aTHX_ "panic: swash_fetch got swatch of unexpected bit width, "
+ "slen=%"UVuf", needents=%"UVuf, (UV)slen, (UV)needents);
NORETURN_FUNCTION_END;
}
{
const int typeto = typestr[0] == 'T' && typestr[1] == 'o';
STRLEN numlen; /* Length of the number */
- I32 flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX;
+ I32 flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
/* nl points to the next \n in the scan */
U8* const nl = (U8*)memchr(l, '\n', lend - l);
/* The max range value follows, separated by a BLANK */
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX;
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
numlen = lend - l;
*max = grok_hex((char *)l, &numlen, &flags, NULL);
if (numlen)
if (wants_value) {
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT |
- PERL_SCAN_DISALLOW_PREFIX;
+
+ /* The ToLc, etc table mappings are not in hex, and must be
+ * corrected by adding the code point to them */
+ if (typeto) {
+ char *after_strtol = (char *) lend;
+ *val = Strtol((char *)l, &after_strtol, 10);
+ l = (U8 *) after_strtol;
+ *val += *min;
+ }
+ else { /* Other tables are in hex, and are the correct result
+ without tweaking */
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
numlen = lend - l;
*val = grok_hex((char *)l, &numlen, &flags, NULL);
if (numlen)
l += numlen;
else
*val = 0;
+ }
}
else {
*val = 0;
if (typeto) {
+ /* diag_listed_as: To%s: illegal mapping '%s' */
Perl_croak(aTHX_ "%s: illegal mapping '%s'",
typestr, l);
}
if (wants_value) {
*val = 0;
if (typeto) {
+ /* diag_listed_as: To%s: illegal mapping '%s' */
Perl_croak(aTHX_ "%s: illegal mapping '%s'", typestr, l);
}
}
* Should be used via swash_fetch, which will cache the swatch in C<swash>.
*/
STATIC SV*
-S_swash_get(pTHX_ SV* swash, UV start, UV span)
+S_swatch_get(pTHX_ SV* swash, UV start, UV span)
{
SV *swatch;
- U8 *l, *lend, *x, *xend, *s;
+ U8 *l, *lend, *x, *xend, *s, *send;
STRLEN lcur, xcur, scur;
HV *const hv = MUTABLE_HV(SvRV(swash));
+ SV** const invlistsvp = hv_fetchs(hv, "INVLIST", FALSE);
+
+ SV** listsvp = NULL; /* The string containing the main body of the table */
+ SV** extssvp = NULL;
+ SV** invert_it_svp = NULL;
+ U8* typestr = NULL;
+ STRLEN bits;
+ STRLEN octets; /* if bits == 1, then octets == 0 */
+ UV none;
+ UV end = start + span;
+
+ if (invlistsvp == NULL) {
+ SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+ SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE);
+ SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
+ extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ listsvp = hv_fetchs(hv, "LIST", FALSE);
+ invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
+
+ bits = SvUV(*bitssvp);
+ none = SvUV(*nonesvp);
+ typestr = (U8*)SvPV_nolen(*typesvp);
+ }
+ else {
+ bits = 1;
+ none = 0;
+ }
+ octets = bits >> 3; /* if bits == 1, then octets == 0 */
- /* The string containing the main body of the table */
- SV** const listsvp = hv_fetchs(hv, "LIST", FALSE);
-
- SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
- SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
- SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE);
- SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
- const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
- const STRLEN bits = SvUV(*bitssvp);
- const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
- const UV none = SvUV(*nonesvp);
- const UV end = start + span;
-
- PERL_ARGS_ASSERT_SWASH_GET;
+ PERL_ARGS_ASSERT_SWATCH_GET;
if (bits != 1 && bits != 8 && bits != 16 && bits != 32) {
- Perl_croak(aTHX_ "panic: swash_get doesn't expect bits %"UVuf,
+ Perl_croak(aTHX_ "panic: swatch_get doesn't expect bits %"UVuf,
(UV)bits);
}
+ /* If overflowed, use the max possible */
+ if (end < start) {
+ end = UV_MAX;
+ span = end - start;
+ }
+
/* create and initialize $swatch */
scur = octets ? (span * octets) : (span + 7) / 8;
swatch = newSV(scur);
SvCUR_set(swatch, scur);
s = (U8*)SvPVX(swatch);
+ if (invlistsvp) { /* If has an inversion list set up use that */
+ _invlist_populate_swatch(*invlistsvp, start, end, s);
+ return swatch;
+ }
+
/* read $swash->{LIST} */
l = (U8*)SvPV(*listsvp, lcur);
lend = l + lcur;
while (l < lend) {
- UV min, max, val;
+ UV min, max, val, upper;
l = S_swash_scan_list_line(aTHX_ l, lend, &min, &max, &val,
cBOOL(octets), typestr);
if (l > lend) {
if (max < start)
continue;
+ /* <end> is generally 1 beyond where we want to set things, but at the
+ * platform's infinity, where we can't go any higher, we want to
+ * include the code point at <end> */
+ upper = (max < end)
+ ? max
+ : (max != UV_MAX || end != UV_MAX)
+ ? end - 1
+ : end;
+
if (octets) {
UV key;
if (min < start) {
}
min = start;
}
- for (key = min; key <= max; key++) {
+ for (key = min; key <= upper; key++) {
STRLEN offset;
- if (key >= end)
- goto go_out_list;
/* offset must be non-negative (start <= min <= key < end) */
offset = octets * (key - start);
if (bits == 8)
UV key;
if (min < start)
min = start;
- for (key = min; key <= max; key++) {
+
+ for (key = min; key <= upper; key++) {
const STRLEN offset = (STRLEN)(key - start);
- if (key >= end)
- goto go_out_list;
s[offset >> 3] |= 1 << (offset & 7);
}
}
} /* while */
- go_out_list:
- /* read $swash->{EXTRAS} */
+ /* Invert if the data says it should be. Assumes that bits == 1 */
+ if (invert_it_svp && SvUV(*invert_it_svp)) {
+
+ /* Unicode properties should come with all bits above PERL_UNICODE_MAX
+ * be 0, and their inversion should also be 0, as we don't succeed any
+ * Unicode property matches for non-Unicode code points */
+ if (start <= PERL_UNICODE_MAX) {
+
+ /* The code below assumes that we never cross the
+ * Unicode/above-Unicode boundary in a range, as otherwise we would
+ * have to figure out where to stop flipping the bits. Since this
+ * boundary is divisible by a large power of 2, and swatches comes
+ * in small powers of 2, this should be a valid assumption */
+ assert(start + span - 1 <= PERL_UNICODE_MAX);
+
+ send = s + scur;
+ while (s < send) {
+ *s = ~(*s);
+ s++;
+ }
+ }
+ }
+
+ /* read $swash->{EXTRAS}
+ * This code also copied to swash_to_invlist() below */
x = (U8*)SvPV(*extssvp, xcur);
xend = x + xcur;
while (x < xend) {
otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE);
otherbits = (STRLEN)SvUV(*otherbitssvp);
if (bits < otherbits)
- Perl_croak(aTHX_ "panic: swash_get found swatch size mismatch");
+ Perl_croak(aTHX_ "panic: swatch_get found swatch size mismatch, "
+ "bits=%"UVuf", otherbits=%"UVuf, (UV)bits, (UV)otherbits);
/* The "other" swatch must be destroyed after. */
- other = swash_get(*othersvp, start, span);
+ other = swatch_get(*othersvp, start, span);
o = (U8*)SvPV(other, olen);
if (!olen)
- Perl_croak(aTHX_ "panic: swash_get got improper swatch");
+ Perl_croak(aTHX_ "panic: swatch_get got improper swatch");
s = (U8*)SvPV(swatch, slen);
if (bits == 1 && otherbits == 1) {
if (slen != olen)
- Perl_croak(aTHX_ "panic: swash_get found swatch length mismatch");
+ Perl_croak(aTHX_ "panic: swatch_get found swatch length "
+ "mismatch, slen=%"UVuf", olen=%"UVuf,
+ (UV)slen, (UV)olen);
switch (opc) {
case '+':
}
HV*
-Perl__swash_inversion_hash(pTHX_ SV* swash)
+Perl__swash_inversion_hash(pTHX_ SV* const swash)
{
- /* Subject to change or removal. For use only in one place in regexec.c
+ /* Subject to change or removal. For use only in one place in regcomp.c.
+ * Can't be used on a property that is subject to user override, as it
+ * relies on the value of SPECIALS in the swash which would be set by
+ * utf8_heavy.pl to the hash in the non-overriden file, and hence is not set
+ * for overridden properties
*
* Returns a hash which is the inversion and closure of a swash mapping.
* For example, consider the input lines:
* Essentially, for any code point, it gives all the code points that map to
* it, or the list of 'froms' for that point.
*
- * Currently it only looks at the main body of the swash, and ignores any
- * additions or deletions from other swashes */
+ * Currently it ignores any additions or deletions from other swashes,
+ * looking at just the main body of the swash, and if there are SPECIALS
+ * in the swash, at that hash
+ *
+ * The specials hash can be extra code points, and most likely consists of
+ * maps from single code points to multiple ones (each expressed as a string
+ * of utf8 characters). This function currently returns only 1-1 mappings.
+ * However consider this possible input in the specials hash:
+ * "\xEF\xAC\x85" => "\x{0073}\x{0074}", # U+FB05 => 0073 0074
+ * "\xEF\xAC\x86" => "\x{0073}\x{0074}", # U+FB06 => 0073 0074
+ *
+ * Both FB05 and FB06 map to the same multi-char sequence, which we don't
+ * currently handle. But it also means that FB05 and FB06 are equivalent in
+ * a 1-1 mapping which we should handle, and this relationship may not be in
+ * the main table. Therefore this function examines all the multi-char
+ * sequences and adds the 1-1 mappings that come out of that. */
U8 *l, *lend;
STRLEN lcur;
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
const UV none = SvUV(*nonesvp);
+ SV **specials_p = hv_fetchs(hv, "SPECIALS", 0);
HV* ret = newHV();
(UV)bits);
}
+ if (specials_p) { /* It might be "special" (sometimes, but not always, a
+ mapping to more than one character */
+
+ /* Construct an inverse mapping hash for the specials */
+ HV * const specials_hv = MUTABLE_HV(SvRV(*specials_p));
+ HV * specials_inverse = newHV();
+ char *char_from; /* the lhs of the map */
+ I32 from_len; /* its byte length */
+ char *char_to; /* the rhs of the map */
+ I32 to_len; /* its byte length */
+ SV *sv_to; /* and in a sv */
+ AV* from_list; /* list of things that map to each 'to' */
+
+ hv_iterinit(specials_hv);
+
+ /* The keys are the characters (in utf8) that map to the corresponding
+ * utf8 string value. Iterate through the list creating the inverse
+ * list. */
+ while ((sv_to = hv_iternextsv(specials_hv, &char_from, &from_len))) {
+ SV** listp;
+ if (! SvPOK(sv_to)) {
+ Perl_croak(aTHX_ "panic: value returned from hv_iternextsv() "
+ "unexpectedly is not a string, flags=%lu",
+ (unsigned long)SvFLAGS(sv_to));
+ }
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Found mapping from %"UVXf", First char of to is %"UVXf"\n", utf8_to_uvchr((U8*) char_from, 0), utf8_to_uvchr((U8*) SvPVX(sv_to), 0)));*/
+
+ /* Each key in the inverse list is a mapped-to value, and the key's
+ * hash value is a list of the strings (each in utf8) that map to
+ * it. Those strings are all one character long */
+ if ((listp = hv_fetch(specials_inverse,
+ SvPVX(sv_to),
+ SvCUR(sv_to), 0)))
+ {
+ from_list = (AV*) *listp;
+ }
+ else { /* No entry yet for it: create one */
+ from_list = newAV();
+ if (! hv_store(specials_inverse,
+ SvPVX(sv_to),
+ SvCUR(sv_to),
+ (SV*) from_list, 0))
+ {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+ }
+
+ /* Here have the list associated with this 'to' (perhaps newly
+ * created and empty). Just add to it. Note that we ASSUME that
+ * the input is guaranteed to not have duplications, so we don't
+ * check for that. Duplications just slow down execution time. */
+ av_push(from_list, newSVpvn_utf8(char_from, from_len, TRUE));
+ }
+
+ /* Here, 'specials_inverse' contains the inverse mapping. Go through
+ * it looking for cases like the FB05/FB06 examples above. There would
+ * be an entry in the hash like
+ * 'st' => [ FB05, FB06 ]
+ * In this example we will create two lists that get stored in the
+ * returned hash, 'ret':
+ * FB05 => [ FB05, FB06 ]
+ * FB06 => [ FB05, FB06 ]
+ *
+ * Note that there is nothing to do if the array only has one element.
+ * (In the normal 1-1 case handled below, we don't have to worry about
+ * two lists, as everything gets tied to the single list that is
+ * generated for the single character 'to'. But here, we are omitting
+ * that list, ('st' in the example), so must have multiple lists.) */
+ while ((from_list = (AV *) hv_iternextsv(specials_inverse,
+ &char_to, &to_len)))
+ {
+ if (av_len(from_list) > 0) {
+ int i;
+
+ /* We iterate over all combinations of i,j to place each code
+ * point on each list */
+ for (i = 0; i <= av_len(from_list); i++) {
+ int j;
+ AV* i_list = newAV();
+ SV** entryp = av_fetch(from_list, i, FALSE);
+ if (entryp == NULL) {
+ Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
+ }
+ if (hv_fetch(ret, SvPVX(*entryp), SvCUR(*entryp), FALSE)) {
+ Perl_croak(aTHX_ "panic: unexpected entry for %s", SvPVX(*entryp));
+ }
+ if (! hv_store(ret, SvPVX(*entryp), SvCUR(*entryp),
+ (SV*) i_list, FALSE))
+ {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+
+ /* For debugging: UV u = utf8_to_uvchr((U8*) SvPVX(*entryp), 0);*/
+ for (j = 0; j <= av_len(from_list); j++) {
+ entryp = av_fetch(from_list, j, FALSE);
+ if (entryp == NULL) {
+ Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
+ }
+
+ /* When i==j this adds itself to the list */
+ av_push(i_list, newSVuv(utf8_to_uvchr(
+ (U8*) SvPVX(*entryp), 0)));
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", utf8_to_uvchr((U8*) SvPVX(*entryp), 0), u));*/
+ }
+ }
+ }
+ }
+ SvREFCNT_dec(specials_inverse); /* done with it */
+ } /* End of specials */
+
/* read $swash->{LIST} */
l = (U8*)SvPV(*listsvp, lcur);
lend = l + lcur;
/* Each element in the range is to be inverted */
for (inverse = min; inverse <= max; inverse++) {
AV* list;
- SV* element;
SV** listp;
IV i;
bool found_key = FALSE;
+ bool found_inverse = FALSE;
/* The key is the inverse mapping */
char key[UTF8_MAXBYTES+1];
char* key_end = (char *) uvuni_to_utf8((U8*) key, val);
STRLEN key_len = key_end - key;
- /* And the value is what the forward mapping is from. */
- char utf8_inverse[UTF8_MAXBYTES+1];
- char *utf8_inverse_end = (char *) uvuni_to_utf8((U8*) utf8_inverse, inverse);
-
/* Get the list for the map */
if ((listp = hv_fetch(ret, key, key_len, FALSE))) {
list = (AV*) *listp;
}
}
- for (i = 0; i < av_len(list); i++) {
+ /* Look through list to see if this inverse mapping already is
+ * listed, or if there is a mapping to itself already */
+ for (i = 0; i <= av_len(list); i++) {
SV** entryp = av_fetch(list, i, FALSE);
SV* entry;
if (entryp == NULL) {
Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
}
entry = *entryp;
- if (SvCUR(entry) != key_len) {
- continue;
- }
- if (memEQ(key, SvPVX(entry), key_len)) {
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "list for %"UVXf" contains %"UVXf"\n", val, SvUV(entry)));*/
+ if (SvUV(entry) == val) {
found_key = TRUE;
+ }
+ if (SvUV(entry) == inverse) {
+ found_inverse = TRUE;
+ }
+
+ /* No need to continue searching if found everything we are
+ * looking for */
+ if (found_key && found_inverse) {
break;
}
}
+
+ /* Make sure there is a mapping to itself on the list */
if (! found_key) {
- element = newSVpvn_flags(key, key_len, SVf_UTF8);
- av_push(list, element);
+ av_push(list, newSVuv(val));
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", val, val));*/
}
/* Simply add the value to the list */
- element = newSVpvn_flags(utf8_inverse, utf8_inverse_end - utf8_inverse, SVf_UTF8);
- av_push(list, element);
+ if (! found_inverse) {
+ av_push(list, newSVuv(inverse));
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", inverse, val));*/
+ }
- /* swash_get() increments the value of val for each element in the
+ /* swatch_get() increments the value of val for each element in the
* range. That makes more compact tables possible. You can
* express the capitalization, for example, of all consecutive
* letters with a single line: 0061\t007A\t0041 This maps 0061 to
* 0041, 0062 to 0042, etc. I (khw) have never understood 'none',
- * and it's not documented, and perhaps not even currently used,
- * but I copied the semantics from swash_get(), just in case */
+ * and it's not documented; it appears to be used only in
+ * implementing tr//; I copied the semantics from swatch_get(), just
+ * in case */
if (!none || val < none) {
++val;
}
return ret;
}
+SV*
+Perl__swash_to_invlist(pTHX_ SV* const swash)
+{
+
+ /* Subject to change or removal. For use only in one place in regcomp.c */
+
+ U8 *l, *lend;
+ char *loc;
+ STRLEN lcur;
+ HV *const hv = MUTABLE_HV(SvRV(swash));
+ UV elements = 0; /* Number of elements in the inversion list */
+ U8 empty[] = "";
+
+ /* The string containing the main body of the table */
+ SV** const listsvp = hv_fetchs(hv, "LIST", FALSE);
+ SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
+ SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+ SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ SV** const invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
+
+ const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
+ const STRLEN bits = SvUV(*bitssvp);
+ const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
+ U8 *x, *xend;
+ STRLEN xcur;
+
+ SV* invlist;
+
+ PERL_ARGS_ASSERT__SWASH_TO_INVLIST;
+
+ /* read $swash->{LIST} */
+ if (SvPOK(*listsvp)) {
+ l = (U8*)SvPV(*listsvp, lcur);
+ }
+ else {
+ /* LIST legitimately doesn't contain a string during compilation phases
+ * of Perl itself, before the Unicode tables are generated. In this
+ * case, just fake things up by creating an empty list */
+ l = empty;
+ lcur = 0;
+ }
+ loc = (char *) l;
+ lend = l + lcur;
+
+ /* Scan the input to count the number of lines to preallocate array size
+ * based on worst possible case, which is each line in the input creates 2
+ * elements in the inversion list: 1) the beginning of a range in the list;
+ * 2) the beginning of a range not in the list. */
+ while ((loc = (strchr(loc, '\n'))) != NULL) {
+ elements += 2;
+ loc++;
+ }
+
+ /* If the ending is somehow corrupt and isn't a new line, add another
+ * element for the final range that isn't in the inversion list */
+ if (! (*lend == '\n'
+ || (*lend == '\0' && (lcur == 0 || *(lend - 1) == '\n'))))
+ {
+ elements++;
+ }
+
+ invlist = _new_invlist(elements);
+
+ /* Now go through the input again, adding each range to the list */
+ while (l < lend) {
+ UV start, end;
+ UV val; /* Not used by this function */
+
+ l = S_swash_scan_list_line(aTHX_ l, lend, &start, &end, &val,
+ cBOOL(octets), typestr);
+
+ if (l > lend) {
+ break;
+ }
+
+ _append_range_to_invlist(invlist, start, end);
+ }
+
+ /* Invert if the data says it should be */
+ if (invert_it_svp && SvUV(*invert_it_svp)) {
+ _invlist_invert_prop(invlist);
+ }
+
+ /* This code is copied from swatch_get()
+ * read $swash->{EXTRAS} */
+ x = (U8*)SvPV(*extssvp, xcur);
+ xend = x + xcur;
+ while (x < xend) {
+ STRLEN namelen;
+ U8 *namestr;
+ SV** othersvp;
+ HV* otherhv;
+ STRLEN otherbits;
+ SV **otherbitssvp, *other;
+ U8 *nl;
+
+ const U8 opc = *x++;
+ if (opc == '\n')
+ continue;
+
+ nl = (U8*)memchr(x, '\n', xend - x);
+
+ if (opc != '-' && opc != '+' && opc != '!' && opc != '&') {
+ if (nl) {
+ x = nl + 1; /* 1 is length of "\n" */
+ continue;
+ }
+ else {
+ x = xend; /* to EXTRAS' end at which \n is not found */
+ break;
+ }
+ }
+
+ namestr = x;
+ if (nl) {
+ namelen = nl - namestr;
+ x = nl + 1;
+ }
+ else {
+ namelen = xend - namestr;
+ x = xend;
+ }
+
+ othersvp = hv_fetch(hv, (char *)namestr, namelen, FALSE);
+ otherhv = MUTABLE_HV(SvRV(*othersvp));
+ otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE);
+ otherbits = (STRLEN)SvUV(*otherbitssvp);
+
+ if (bits != otherbits || bits != 1) {
+ Perl_croak(aTHX_ "panic: _swash_to_invlist only operates on boolean "
+ "properties, bits=%"UVuf", otherbits=%"UVuf,
+ (UV)bits, (UV)otherbits);
+ }
+
+ /* The "other" swatch must be destroyed after. */
+ other = _swash_to_invlist((SV *)*othersvp);
+
+ /* End of code copied from swatch_get() */
+ switch (opc) {
+ case '+':
+ _invlist_union(invlist, other, &invlist);
+ break;
+ case '!':
+ _invlist_invert(other);
+ _invlist_union(invlist, other, &invlist);
+ break;
+ case '-':
+ _invlist_subtract(invlist, other, &invlist);
+ break;
+ case '&':
+ _invlist_intersection(invlist, other, &invlist);
+ break;
+ default:
+ break;
+ }
+ sv_free(other); /* through with it! */
+ }
+
+ return invlist;
+}
+
/*
=for apidoc uvchr_to_utf8
Adds the UTF-8 representation of the Native code point C<uv> to the end
-of the string C<d>; C<d> should be have at least C<UTF8_MAXBYTES+1> free
+of the string C<d>; C<d> should have at least C<UTF8_MAXBYTES+1> free
bytes available. The return value is the pointer to the byte after the
end of the new character. In other words,
Perl_check_utf8_print(pTHX_ register const U8* s, const STRLEN len)
{
/* May change: warns if surrogates, non-character code points, or
- * non-Unicode code points are in s which has length len. Returns TRUE if
- * none found; FALSE otherwise. The only other validity check is to make
- * sure that this won't exceed the string's length */
+ * non-Unicode code points are in s which has length len bytes. Returns
+ * TRUE if none found; FALSE otherwise. The only other validity check is
+ * to make sure that this won't exceed the string's length */
const U8* const e = s + len;
bool ok = TRUE;
"%s in %s", unees, PL_op ? OP_DESC(PL_op) : "print");
return FALSE;
}
- if (*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE) {
+ if (UNLIKELY(*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE)) {
STRLEN char_len;
if (UTF8_IS_SUPER(s)) {
- UV uv = utf8_to_uvchr(s, &char_len);
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv);
- ok = FALSE;
+ if (ckWARN_d(WARN_NON_UNICODE)) {
+ UV uv = utf8_to_uvchr(s, &char_len);
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
+ "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv);
+ ok = FALSE;
+ }
}
else if (UTF8_IS_SURROGATE(s)) {
- UV uv = utf8_to_uvchr(s, &char_len);
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv);
- ok = FALSE;
+ if (ckWARN_d(WARN_SURROGATE)) {
+ UV uv = utf8_to_uvchr(s, &char_len);
+ Perl_warner(aTHX_ packWARN(WARN_SURROGATE),
+ "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv);
+ ok = FALSE;
+ }
}
else if
- (UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s))
+ ((UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s))
+ && (ckWARN_d(WARN_NONCHAR)))
{
UV uv = utf8_to_uvchr(s, &char_len);
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_warner(aTHX_ packWARN(WARN_NONCHAR),
"Unicode non-character U+%04"UVXf" is illegal for open interchange", uv);
ok = FALSE;
}
http://www.unicode.org/unicode/reports/tr21/ (Case Mappings).
=cut */
+
+/* A flags parameter has been added which may change, and hence isn't
+ * externally documented. Currently it is:
+ * 0 for as-documented above
+ * FOLDEQ_UTF8_NOMIX_ASCII meaning that if a non-ASCII character folds to an
+ ASCII one, to not match
+ * FOLDEQ_UTF8_LOCALE meaning that locale rules are to be used for code
+ * points below 256; unicode rules for above 255; and
+ * folds that cross those boundaries are disallowed,
+ * like the NOMIX_ASCII option
+ * FOLDEQ_S1_ALREADY_FOLDED s1 has already been folded before calling this
+ * routine. This allows that step to be skipped.
+ * FOLDEQ_S2_ALREADY_FOLDED Similarly.
+ */
I32
-Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2)
+Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2, U32 flags)
{
dVAR;
register const U8 *p1 = (const U8*)s1; /* Point to current char */
STRLEN n1 = 0, n2 = 0; /* Number of bytes in current char */
U8 foldbuf1[UTF8_MAXBYTES_CASE+1];
U8 foldbuf2[UTF8_MAXBYTES_CASE+1];
- U8 natbuf[2]; /* Holds native 8-bit char converted to utf8;
- these always fit in 2 bytes */
- PERL_ARGS_ASSERT_FOLDEQ_UTF8;
+ PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS;
+
+ /* The algorithm requires that input with the flags on the first line of
+ * the assert not be pre-folded. */
+ assert( ! ((flags & (FOLDEQ_UTF8_NOMIX_ASCII | FOLDEQ_UTF8_LOCALE))
+ && (flags & (FOLDEQ_S1_ALREADY_FOLDED | FOLDEQ_S2_ALREADY_FOLDED))));
if (pe1) {
e1 = *(U8**)pe1;
assert(e2);
}
+ /* If both operands are already folded, we could just do a memEQ on the
+ * whole strings at once, but it would be better if the caller realized
+ * this and didn't even call us */
+
/* Look through both strings, a character at a time */
while (p1 < e1 && p2 < e2) {
/* If at the beginning of a new character in s1, get its fold to use
- * and the length of the fold */
+ * and the length of the fold. (exception: locale rules just get the
+ * character to a single byte) */
if (n1 == 0) {
- if (u1) {
- to_utf8_fold(p1, foldbuf1, &n1);
- }
- else { /* Not utf8, convert to it first and then get fold */
- uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p1)));
- to_utf8_fold(natbuf, foldbuf1, &n1);
- }
- f1 = foldbuf1;
+ if (flags & FOLDEQ_S1_ALREADY_FOLDED) {
+ f1 = (U8 *) p1;
+ n1 = UTF8SKIP(f1);
+ }
+
+ else {
+ /* If in locale matching, we use two sets of rules, depending
+ * on if the code point is above or below 255. Here, we test
+ * for and handle locale rules */
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u1 || UTF8_IS_INVARIANT(*p1)
+ || UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ {
+ /* There is no mixing of code points above and below 255. */
+ if (u2 && (! UTF8_IS_INVARIANT(*p2)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ {
+ return 0;
+ }
+
+ /* We handle locale rules by converting, if necessary, the
+ * code point to a single byte. */
+ if (! u1 || UTF8_IS_INVARIANT(*p1)) {
+ *foldbuf1 = *p1;
+ }
+ else {
+ *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1));
+ }
+ n1 = 1;
+ }
+ else if (isASCII(*p1)) { /* Note, that here won't be both
+ ASCII and using locale rules */
+
+ /* If trying to mix non- with ASCII, and not supposed to,
+ * fail */
+ if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) {
+ return 0;
+ }
+ n1 = 1;
+ *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are
+ just lowercased */
+ }
+ else if (u1) {
+ to_utf8_fold(p1, foldbuf1, &n1);
+ }
+ else { /* Not utf8, get utf8 fold */
+ to_uni_fold(NATIVE_TO_UNI(*p1), foldbuf1, &n1);
+ }
+ f1 = foldbuf1;
+ }
}
if (n2 == 0) { /* Same for s2 */
- if (u2) {
- to_utf8_fold(p2, foldbuf2, &n2);
- }
- else {
- uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p2)));
- to_utf8_fold(natbuf, foldbuf2, &n2);
- }
- f2 = foldbuf2;
+ if (flags & FOLDEQ_S2_ALREADY_FOLDED) {
+ f2 = (U8 *) p2;
+ n2 = UTF8SKIP(f2);
+ }
+ else {
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ {
+ /* Here, the next char in s2 is < 256. We've already
+ * worked on s1, and if it isn't also < 256, can't match */
+ if (u1 && (! UTF8_IS_INVARIANT(*p1)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ {
+ return 0;
+ }
+ if (! u2 || UTF8_IS_INVARIANT(*p2)) {
+ *foldbuf2 = *p2;
+ }
+ else {
+ *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1));
+ }
+
+ /* Use another function to handle locale rules. We've made
+ * sure that both characters to compare are single bytes */
+ if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) {
+ return 0;
+ }
+ n1 = n2 = 0;
+ }
+ else if (isASCII(*p2)) {
+ if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p1)) {
+ return 0;
+ }
+ n2 = 1;
+ *foldbuf2 = toLOWER(*p2);
+ }
+ else if (u2) {
+ to_utf8_fold(p2, foldbuf2, &n2);
+ }
+ else {
+ to_uni_fold(NATIVE_TO_UNI(*p2), foldbuf2, &n2);
+ }
+ f2 = foldbuf2;
+ }
}
+ /* Here f1 and f2 point to the beginning of the strings to compare.
+ * These strings are the folds of the next character from each input
+ * string, stored in utf8. */
+
/* While there is more to look for in both folds, see if they
* continue to match */
while (n1 && n2) {
if (fold_length != UTF8SKIP(f2)
|| (fold_length == 1 && *f1 != *f2) /* Short circuit memNE
function call for single
- character */
+ byte */
|| memNE((char*)f1, (char*)f2, fold_length))
{
return 0; /* mismatch */
https://perl5.git.perl.org / perl5.git / blobdiff