character is a valid UTF-8 character. The actual number of bytes in the UTF-8
character will be returned if it is valid, otherwise 0.
+WARNING: use only if you *know* that C<s> has at least either UTF8_MAXBYTES or
+UTF8SKIP(s) bytes.
+
=cut */
STRLEN
Perl_is_utf8_char(const U8 *s)
Returns true if first C<len> bytes of the given string form a valid
UTF-8 string, false otherwise. If C<len> is 0, it will be calculated
-using C<strlen(s)>. Note that 'a valid UTF-8 string' does not mean 'a
-string that contains code points above 0x7F encoded in UTF-8' because a
-valid ASCII string is a valid UTF-8 string.
+using C<strlen(s)> (which means if you use this option, that C<s> has to have a
+terminating NUL byte). Note that all characters being ASCII constitute 'a
+valid UTF-8 string'.
See also is_ascii_string(), is_utf8_string_loclen(), and is_utf8_string_loc().
PERL_ARGS_ASSERT_IS_UTF8_STRING;
while (x < send) {
- STRLEN c;
/* Inline the easy bits of is_utf8_char() here for speed... */
- if (UTF8_IS_INVARIANT(*x))
- c = 1;
+ if (UTF8_IS_INVARIANT(*x)) {
+ x++;
+ }
else if (!UTF8_IS_START(*x))
- goto out;
+ return FALSE;
else {
/* ... and call is_utf8_char() only if really needed. */
-#ifdef IS_UTF8_CHAR
- c = UTF8SKIP(x);
+ const STRLEN c = UTF8SKIP(x);
+ const U8* const next_char_ptr = x + c;
+
+ if (next_char_ptr > send) {
+ return FALSE;
+ }
+
if (IS_UTF8_CHAR_FAST(c)) {
if (!IS_UTF8_CHAR(x, c))
- c = 0;
+ return FALSE;
}
- else
- c = is_utf8_char_slow(x, c);
-#else
- c = is_utf8_char(x);
-#endif /* #ifdef IS_UTF8_CHAR */
- if (!c)
- goto out;
+ else if (! is_utf8_char_slow(x, c)) {
+ return FALSE;
+ }
+ x = next_char_ptr;
}
- x += c;
}
- out:
- if (x != send)
- return FALSE;
-
return TRUE;
}
PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN;
while (x < send) {
+ const U8* next_char_ptr;
+
/* Inline the easy bits of is_utf8_char() here for speed... */
if (UTF8_IS_INVARIANT(*x))
- c = 1;
+ next_char_ptr = x + 1;
else if (!UTF8_IS_START(*x))
goto out;
else {
/* ... and call is_utf8_char() only if really needed. */
-#ifdef IS_UTF8_CHAR
c = UTF8SKIP(x);
+ next_char_ptr = c + x;
+ if (next_char_ptr > send) {
+ goto out;
+ }
if (IS_UTF8_CHAR_FAST(c)) {
if (!IS_UTF8_CHAR(x, c))
c = 0;
} else
c = is_utf8_char_slow(x, c);
-#else
- c = is_utf8_char(x);
-#endif /* #ifdef IS_UTF8_CHAR */
if (!c)
goto out;
}
- x += c;
+ x = next_char_ptr;
outlen++;
}
Certain code points are considered problematic. These are Unicode surrogates,
Unicode non-characters, and code points above the Unicode maximum of 0x10FFF.
By default these are considered regular code points, but certain situations
-warrant special handling for them. if C<flags> contains
+warrant special handling for them. If C<flags> contains
UTF8_DISALLOW_ILLEGAL_INTERCHANGE, all three classes are treated as
malformations and handled as such. The flags UTF8_DISALLOW_SURROGATE,
UTF8_DISALLOW_NONCHAR, and UTF8_DISALLOW_SUPER (meaning above the legal Unicode
the others that are above the Unicode legal maximum. There are several
reasons, one of which is that the original UTF-8 specification never went above
this number (the current 0x10FFF limit was imposed later). The UTF-8 encoding
-on ASCII platforms for these large code point begins with a byte containing
+on ASCII platforms for these large code points begins with a byte containing
0xFE or 0xFF. The UTF8_DISALLOW_FE_FF flag will cause them to be treated as
malformations, while allowing smaller above-Unicode code points. (Of course
UTF8_DISALLOW_SUPER will treat all above-Unicode code points, including these,
=cut
*/
+/* This logic is duplicated in sv_catpvn_flags, so any bug fixes will
+ likewise need duplication. */
+
U8*
Perl_bytes_to_utf8(pTHX_ const U8 *s, STRLEN *len)
{
return utf16_to_utf8(p, d, bytelen, newlen);
}
-/* for now these are all defined (inefficiently) in terms of the utf8 versions */
+/* for now these are all defined (inefficiently) in terms of the utf8 versions.
+ * Note that the macros in handy.h that call these short-circuit calling them
+ * for Latin-1 range inputs */
bool
Perl_is_uni_alnum(pTHX_ UV c)
bool
Perl_is_uni_ascii(pTHX_ UV c)
{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return is_utf8_ascii(tmpbuf);
+ return isASCII(c);
}
bool
bool
Perl_is_uni_cntrl(pTHX_ UV c)
{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return is_utf8_cntrl(tmpbuf);
+ return isCNTRL_L1(c);
}
bool
}
UV
+Perl__to_upper_title_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp, const char S_or_s)
+{
+ /* We have the latin1-range values compiled into the core, so just use
+ * those, converting the result to utf8. The only difference between upper
+ * and title case in this range is that LATIN_SMALL_LETTER_SHARP_S is
+ * either "SS" or "Ss". Which one to use is passed into the routine in
+ * 'S_or_s' to avoid a test */
+
+ UV converted = toUPPER_LATIN1_MOD(c);
+
+ PERL_ARGS_ASSERT__TO_UPPER_TITLE_LATIN1;
+
+ assert(S_or_s == 'S' || S_or_s == 's');
+
+ if (UNI_IS_INVARIANT(converted)) { /* No difference between the two for
+ characters in this range */
+ *p = (U8) converted;
+ *lenp = 1;
+ return converted;
+ }
+
+ /* toUPPER_LATIN1_MOD gives the correct results except for three outliers,
+ * which it maps to one of them, so as to only have to have one check for
+ * it in the main case */
+ if (UNLIKELY(converted == LATIN_SMALL_LETTER_Y_WITH_DIAERESIS)) {
+ switch (c) {
+ case LATIN_SMALL_LETTER_Y_WITH_DIAERESIS:
+ converted = LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS;
+ break;
+ case MICRO_SIGN:
+ converted = GREEK_CAPITAL_LETTER_MU;
+ break;
+ case LATIN_SMALL_LETTER_SHARP_S:
+ *(p)++ = 'S';
+ *p = S_or_s;
+ *lenp = 2;
+ return 'S';
+ default:
+ Perl_croak(aTHX_ "panic: to_upper_title_latin1 did not expect '%c' to map to '%c'", c, LATIN_SMALL_LETTER_Y_WITH_DIAERESIS);
+ /* NOTREACHED */
+ }
+ }
+
+ *(p)++ = UTF8_TWO_BYTE_HI(converted);
+ *p = UTF8_TWO_BYTE_LO(converted);
+ *lenp = 2;
+
+ return converted;
+}
+
+/* Call the function to convert a UTF-8 encoded character to the specified case.
+ * Note that there may be more than one character in the result.
+ * INP is a pointer to the first byte of the input character
+ * OUTP will be set to the first byte of the string of changed characters. It
+ * needs to have space for UTF8_MAXBYTES_CASE+1 bytes
+ * LENP will be set to the length in bytes of the string of changed characters
+ *
+ * The functions return the ordinal of the first character in the string of OUTP */
+#define CALL_UPPER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_toupper, "ToUc", "utf8::ToSpecUpper")
+#define CALL_TITLE_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_totitle, "ToTc", "utf8::ToSpecTitle")
+#define CALL_LOWER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tolower, "ToLc", "utf8::ToSpecLower")
+
+/* This additionally has the input parameter SPECIALS, which if non-zero will
+ * cause this to use the SPECIALS hash for folding (meaning get full case
+ * folding); otherwise, when zero, this implies a simple case fold */
+#define CALL_FOLD_CASE(INP, OUTP, LENP, SPECIALS) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tofold, "ToCf", (SPECIALS) ? "utf8::ToSpecFold" : NULL)
+
+UV
Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ dVAR;
+
+ /* Convert the Unicode character whose ordinal is c to its uppercase
+ * version and store that in UTF-8 in p and its length in bytes in lenp.
+ * Note that the p needs to be at least UTF8_MAXBYTES_CASE+1 bytes since
+ * the changed version may be longer than the original character.
+ *
+ * The ordinal of the first character of the changed version is returned
+ * (but note, as explained above, that there may be more.) */
+
PERL_ARGS_ASSERT_TO_UNI_UPPER;
+ if (c < 256) {
+ return _to_upper_title_latin1((U8) c, p, lenp, 'S');
+ }
+
uvchr_to_utf8(p, c);
- return to_utf8_upper(p, p, lenp);
+ return CALL_UPPER_CASE(p, p, lenp);
}
UV
Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ dVAR;
+
PERL_ARGS_ASSERT_TO_UNI_TITLE;
+ if (c < 256) {
+ return _to_upper_title_latin1((U8) c, p, lenp, 's');
+ }
+
uvchr_to_utf8(p, c);
- return to_utf8_title(p, p, lenp);
+ return CALL_TITLE_CASE(p, p, lenp);
+}
+
+STATIC U8
+S_to_lower_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp)
+{
+ /* We have the latin1-range values compiled into the core, so just use
+ * those, converting the result to utf8. Since the result is always just
+ * one character, we allow p to be NULL */
+
+ U8 converted = toLOWER_LATIN1(c);
+
+ if (p != NULL) {
+ if (UNI_IS_INVARIANT(converted)) {
+ *p = converted;
+ *lenp = 1;
+ }
+ else {
+ *p = UTF8_TWO_BYTE_HI(converted);
+ *(p+1) = UTF8_TWO_BYTE_LO(converted);
+ *lenp = 2;
+ }
+ }
+ return converted;
}
UV
Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ dVAR;
+
PERL_ARGS_ASSERT_TO_UNI_LOWER;
+ if (c < 256) {
+ return to_lower_latin1((U8) c, p, lenp);
+ }
+
uvchr_to_utf8(p, c);
- return to_utf8_lower(p, p, lenp);
+ return CALL_LOWER_CASE(p, p, lenp);
+}
+
+UV
+Perl__to_fold_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp, const U8 flags)
+{
+ UV converted;
+
+ PERL_ARGS_ASSERT__TO_FOLD_LATIN1;
+
+ if (c == MICRO_SIGN) {
+ converted = GREEK_SMALL_LETTER_MU;
+ }
+ else if (flags && c == LATIN_SMALL_LETTER_SHARP_S) {
+ *(p)++ = 's';
+ *p = 's';
+ *lenp = 2;
+ return 's';
+ }
+ else { /* In this range the fold of all other characters is their lower
+ case */
+ converted = toLOWER_LATIN1(c);
+ }
+
+ if (UNI_IS_INVARIANT(converted)) {
+ *p = (U8) converted;
+ *lenp = 1;
+ }
+ else {
+ *(p)++ = UTF8_TWO_BYTE_HI(converted);
+ *p = UTF8_TWO_BYTE_LO(converted);
+ *lenp = 2;
+ }
+
+ return converted;
}
UV
{
PERL_ARGS_ASSERT__TO_UNI_FOLD_FLAGS;
+ if (c < 256) {
+ return _to_fold_latin1((U8) c, p, lenp, flags);
+ }
+
uvchr_to_utf8(p, c);
- return _to_utf8_fold_flags(p, p, lenp, flags);
+ return CALL_FOLD_CASE(p, p, lenp, flags);
}
/* for now these all assume no locale info available for Unicode > 255 */
}
bool
+Perl__is_utf8__perl_idstart(pTHX_ const U8 *p)
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT__IS_UTF8__PERL_IDSTART;
+
+ return is_utf8_common(p, &PL_utf8_perl_idstart, "_Perl_IDStart");
+}
+
+bool
Perl_is_utf8_idcont(pTHX_ const U8 *p)
{
dVAR;
PERL_ARGS_ASSERT_IS_UTF8_IDCONT;
- if (*p == '_')
- return TRUE;
return is_utf8_common(p, &PL_utf8_idcont, "IdContinue");
}
PERL_ARGS_ASSERT_IS_UTF8_XIDCONT;
- if (*p == '_')
- return TRUE;
return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue");
}
PERL_ARGS_ASSERT_IS_UTF8_ASCII;
- return is_utf8_common(p, &PL_utf8_ascii, "IsAscii");
+ /* ASCII characters are the same whether in utf8 or not. So the macro
+ * works on both utf8 and non-utf8 representations. */
+ return isASCII(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_SPACE;
- return is_utf8_common(p, &PL_utf8_space, "IsSpacePerl");
+ return is_utf8_common(p, &PL_utf8_space, "IsXPerlSpace");
}
bool
PERL_ARGS_ASSERT_IS_UTF8_PERL_SPACE;
- return is_utf8_common(p, &PL_utf8_perl_space, "IsPerlSpace");
+ /* Only true if is an ASCII space-like character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isSPACE_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_PERL_WORD;
- return is_utf8_common(p, &PL_utf8_perl_word, "IsPerlWord");
+ /* Only true if is an ASCII word character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isWORDCHAR_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_POSIX_DIGIT;
- return is_utf8_common(p, &PL_utf8_posix_digit, "IsPosixDigit");
+ /* Only true if is an ASCII digit character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isDIGIT_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_CNTRL;
- return is_utf8_common(p, &PL_utf8_cntrl, "IsCntrl");
+ if (isASCII(*p)) {
+ return isCNTRL_A(*p);
+ }
+
+ /* All controls are in Latin1 */
+ if (! UTF8_IS_DOWNGRADEABLE_START(*p)) {
+ return 0;
+ }
+ return isCNTRL_L1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
}
bool
PERL_ARGS_ASSERT_TO_UTF8_UPPER;
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_toupper, "ToUpper", "utf8::ToSpecUpper");
+ if (UTF8_IS_INVARIANT(*p)) {
+ return _to_upper_title_latin1(*p, ustrp, lenp, 'S');
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ return _to_upper_title_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp, 'S');
+ }
+
+ return CALL_UPPER_CASE(p, ustrp, lenp);
}
/*
PERL_ARGS_ASSERT_TO_UTF8_TITLE;
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_totitle, "ToTitle", "utf8::ToSpecTitle");
+ if (UTF8_IS_INVARIANT(*p)) {
+ return _to_upper_title_latin1(*p, ustrp, lenp, 's');
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ return _to_upper_title_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp, 's');
+ }
+
+ return CALL_TITLE_CASE(p, ustrp, lenp);
}
/*
PERL_ARGS_ASSERT_TO_UTF8_LOWER;
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_tolower, "ToLower", "utf8::ToSpecLower");
+ if (UTF8_IS_INVARIANT(*p)) {
+ return to_lower_latin1(*p, ustrp, lenp);
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ return to_lower_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)), ustrp, lenp);
+ }
+
+ return CALL_LOWER_CASE(p, ustrp, lenp);
}
/*
UV
Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags)
{
- const char *specials = (flags) ? "utf8::ToSpecFold" : NULL;
-
dVAR;
PERL_ARGS_ASSERT__TO_UTF8_FOLD_FLAGS;
- return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_tofold, "ToFold", specials);
+ if (UTF8_IS_INVARIANT(*p)) {
+ return _to_fold_latin1(*p, ustrp, lenp, flags);
+ }
+ else if UTF8_IS_DOWNGRADEABLE_START(*p) {
+ return _to_fold_latin1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)),
+ ustrp, lenp, flags);
+ }
+
+ return CALL_FOLD_CASE(p, ustrp, lenp, flags);
}
/* Note:
- * A "swash" is a swatch hash.
- * A "swatch" is a bit vector generated by utf8.c:S_swash_get().
+ * Returns a "swash" which is a hash described in utf8.c:S_swash_fetch().
* C<pkg> is a pointer to a package name for SWASHNEW, should be "utf8".
* For other parameters, see utf8::SWASHNEW in lib/utf8_heavy.pl.
*/
* of the string C<ptr>. If C<do_utf8> is true, the string C<ptr> is
* assumed to be in utf8. If C<do_utf8> is false, the string C<ptr> is
* assumed to be in native 8-bit encoding. Caches the swatch in C<swash>.
+ *
+ * A "swash" is a hash which contains initially the keys/values set up by
+ * SWASHNEW. The purpose is to be able to completely represent a Unicode
+ * property for all possible code points. Things are stored in a compact form
+ * (see utf8_heavy.pl) so that calculation is required to find the actual
+ * property value for a given code point. As code points are looked up, new
+ * key/value pairs are added to the hash, so that the calculation doesn't have
+ * to ever be re-done. Further, each calculation is done, not just for the
+ * desired one, but for a whole block of code points adjacent to that one.
+ * For binary properties on ASCII machines, the block is usually for 64 code
+ * points, starting with a code point evenly divisible by 64. Thus if the
+ * property value for code point 257 is requested, the code goes out and
+ * calculates the property values for all 64 code points between 256 and 319,
+ * and stores these as a single 64-bit long bit vector, called a "swatch",
+ * under the key for code point 256. The key is the UTF-8 encoding for code
+ * point 256, minus the final byte. Thus, if the length of the UTF-8 encoding
+ * for a code point is 13 bytes, the key will be 12 bytes long. If the value
+ * for code point 258 is then requested, this code realizes that it would be
+ * stored under the key for 256, and would find that value and extract the
+ * relevant bit, offset from 256.
+ *
+ * Non-binary properties are stored in as many bits as necessary to represent
+ * their values (32 currently, though the code is more general than that), not
+ * as single bits, but the principal is the same: the value for each key is a
+ * vector that encompasses the property values for all code points whose UTF-8
+ * representations are represented by the key. That is, for all code points
+ * whose UTF-8 representations are length N bytes, and the key is the first N-1
+ * bytes of that.
*/
UV
Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8)
/* If char is encoded then swatch is for the prefix */
needents = (1 << UTF_ACCUMULATION_SHIFT);
off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK;
- if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_NON_UNICODE)) {
- const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0);
-
- /* This outputs warnings for binary properties only, assuming that
- * to_utf8_case() will output any. Also, surrogates aren't checked
- * for, as that would warn on things like /\p{Gc=Cs}/ */
- SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
- if (SvUV(*bitssvp) == 1) {
- Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
- "Code point 0x%04"UVXf" is not Unicode, no properties match it; all inverse properties do", code_point);
- }
- }
}
/*
0 : UTF8_ALLOW_ANY);
swatch = swash_get(swash,
/* On EBCDIC & ~(0xA0-1) isn't a useful thing to do */
- (klen) ? (code_point & ~(needents - 1)) : 0,
+ (klen) ? (code_point & ~((UV)needents - 1)) : 0,
needents);
if (IN_PERL_COMPILETIME)
Copy(ptr, PL_last_swash_key, klen, U8);
}
+ if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_NON_UNICODE)) {
+ SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+
+ /* This outputs warnings for binary properties only, assuming that
+ * to_utf8_case() will output any for non-binary. Also, surrogates
+ * aren't checked for, as that would warn on things like /\p{Gc=Cs}/ */
+
+ if (SvUV(*bitssvp) == 1) {
+ /* User-defined properties can silently match above-Unicode */
+ SV** const user_defined_svp = hv_fetchs(hv, "USER_DEFINED", FALSE);
+ if (! user_defined_svp || ! SvUV(*user_defined_svp)) {
+ const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0);
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
+ "Code point 0x%04"UVXf" is not Unicode, all \\p{} matches fail; all \\P{} matches succeed", code_point);
+ }
+ }
+ }
+
switch ((int)((slen << 3) / needents)) {
case 1:
bit = 1 << (off & 7);
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
const UV none = SvUV(*nonesvp);
- const UV end = start + span;
+ UV end = start + span;
PERL_ARGS_ASSERT_SWASH_GET;
(UV)bits);
}
+ /* If overflowed, use the max possible */
+ if (end < start) {
+ end = UV_MAX;
+ span = end - start;
+ }
+
/* create and initialize $swatch */
scur = octets ? (span * octets) : (span + 7) / 8;
swatch = newSV(scur);
SvCUR_set(swatch, scur);
s = (U8*)SvPVX(swatch);
- /* read $swash->{LIST} */
+ /* read $swash->{LIST}. XXX Note that this is a linear scan through a
+ * sorted list. A binary search would be much more efficient */
l = (U8*)SvPV(*listsvp, lcur);
lend = l + lcur;
while (l < lend) {
STRLEN offset;
if (key >= end)
goto go_out_list;
+ /* XXX If it should ever happen (very unlikely) that we would
+ * want a non-binary result for the code point at UV_MAX,
+ * special handling would need to be inserted here, as is done
+ * below for the binary case */
/* offset must be non-negative (start <= min <= key < end) */
offset = octets * (key - start);
if (bits == 8)
UV key;
if (min < start)
min = start;
+
+ /* Special case when the upper-end is the highest possible code
+ * point representable on the platform. Otherwise, the code below
+ * exits before setting this bit. Done here to avoid testing for
+ * this extremely unlikely possibility in the loop */
+ if (UNLIKELY(end == UV_MAX && max == UV_MAX)) {
+ const STRLEN offset = (STRLEN)(max - start);
+ s[offset >> 3] |= 1 << (offset & 7);
+ }
for (key = min; key <= max; key++) {
const STRLEN offset = (STRLEN)(key - start);
if (key >= end)
} /* while */
go_out_list:
- /* Invert if the data says it should be */
+ /* Invert if the data says it should be. Assumes that bits == 1 */
if (invert_it_svp && SvUV(*invert_it_svp)) {
- send = s + scur;
- while (s < send) {
- *s = ~(*s);
- s++;
+
+ /* Unicode properties should come with all bits above PERL_UNICODE_MAX
+ * be 0, and their inversion should also be 0, as we don't succeed any
+ * Unicode property matches for non-Unicode code points */
+ if (start <= PERL_UNICODE_MAX) {
+
+ /* The code below assumes that we never cross the
+ * Unicode/above-Unicode boundary in a range, as otherwise we would
+ * have to figure out where to stop flipping the bits. Since this
+ * boundary is divisible by a large power of 2, and swatches comes
+ * in small powers of 2, this should be a valid assumption */
+ assert(start + span - 1 <= PERL_UNICODE_MAX);
+
+ send = s + scur;
+ while (s < send) {
+ *s = ~(*s);
+ s++;
+ }
}
}
/* Invert if the data says it should be */
if (invert_it_svp && SvUV(*invert_it_svp)) {
- _invlist_invert(invlist);
+ _invlist_invert_prop(invlist);
}
/* This code is copied from swash_get()
Perl_check_utf8_print(pTHX_ register const U8* s, const STRLEN len)
{
/* May change: warns if surrogates, non-character code points, or
- * non-Unicode code points are in s which has length len. Returns TRUE if
- * none found; FALSE otherwise. The only other validity check is to make
- * sure that this won't exceed the string's length */
+ * non-Unicode code points are in s which has length len bytes. Returns
+ * TRUE if none found; FALSE otherwise. The only other validity check is
+ * to make sure that this won't exceed the string's length */
const U8* const e = s + len;
bool ok = TRUE;
"%s in %s", unees, PL_op ? OP_DESC(PL_op) : "print");
return FALSE;
}
- if (*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE) {
+ if (UNLIKELY(*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE)) {
STRLEN char_len;
if (UTF8_IS_SUPER(s)) {
if (ckWARN_d(WARN_NON_UNICODE)) {
* points below 256; unicode rules for above 255; and
* folds that cross those boundaries are disallowed,
* like the NOMIX_ASCII option
+ * FOLDEQ_S1_ALREADY_FOLDED s1 has already been folded before calling this
+ * routine. This allows that step to be skipped.
+ * FOLDEQ_S2_ALREADY_FOLDED Similarly.
*/
I32
Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2, U32 flags)
STRLEN n1 = 0, n2 = 0; /* Number of bytes in current char */
U8 foldbuf1[UTF8_MAXBYTES_CASE+1];
U8 foldbuf2[UTF8_MAXBYTES_CASE+1];
- U8 natbuf[2]; /* Holds native 8-bit char converted to utf8;
- these always fit in 2 bytes */
PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS;
+ /* The algorithm requires that input with the flags on the first line of
+ * the assert not be pre-folded. */
+ assert( ! ((flags & (FOLDEQ_UTF8_NOMIX_ASCII | FOLDEQ_UTF8_LOCALE))
+ && (flags & (FOLDEQ_S1_ALREADY_FOLDED | FOLDEQ_S2_ALREADY_FOLDED))));
+
if (pe1) {
e1 = *(U8**)pe1;
}
assert(e2);
}
+ /* If both operands are already folded, we could just do a memEQ on the
+ * whole strings at once, but it would be better if the caller realized
+ * this and didn't even call us */
+
/* Look through both strings, a character at a time */
while (p1 < e1 && p2 < e2) {
* and the length of the fold. (exception: locale rules just get the
* character to a single byte) */
if (n1 == 0) {
+ if (flags & FOLDEQ_S1_ALREADY_FOLDED) {
+ f1 = (U8 *) p1;
+ n1 = UTF8SKIP(f1);
/* If in locale matching, we use two sets of rules, depending on if
* the code point is above or below 255. Here, we test for and
* handle locale rules */
- if ((flags & FOLDEQ_UTF8_LOCALE)
- && (! u1 || UTF8_IS_INVARIANT(*p1) || UTF8_IS_DOWNGRADEABLE_START(*p1)))
- {
- /* There is no mixing of code points above and below 255. */
- if (u2 && (! UTF8_IS_INVARIANT(*p2)
- && ! UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ }
+ else {
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u1 || UTF8_IS_INVARIANT(*p1)
+ || UTF8_IS_DOWNGRADEABLE_START(*p1)))
{
- return 0;
- }
+ /* There is no mixing of code points above and below 255. */
+ if (u2 && (! UTF8_IS_INVARIANT(*p2)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ {
+ return 0;
+ }
- /* We handle locale rules by converting, if necessary, the code
- * point to a single byte. */
- if (! u1 || UTF8_IS_INVARIANT(*p1)) {
- *foldbuf1 = *p1;
+ /* We handle locale rules by converting, if necessary, the
+ * code point to a single byte. */
+ if (! u1 || UTF8_IS_INVARIANT(*p1)) {
+ *foldbuf1 = *p1;
+ }
+ else {
+ *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1));
+ }
+ n1 = 1;
}
- else {
- *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1));
+ else if (isASCII(*p1)) { /* Note, that here won't be
+ both ASCII and using locale
+ rules */
+
+ /* If trying to mix non- with ASCII, and not supposed to,
+ * fail */
+ if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) {
+ return 0;
+ }
+ n1 = 1;
+ *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are
+ just lowercased */
}
- n1 = 1;
- }
- else if (isASCII(*p1)) { /* Note, that here won't be both ASCII
- and using locale rules */
-
- /* If trying to mix non- with ASCII, and not supposed to, fail */
- if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) {
- return 0;
+ else if (u1) {
+ to_utf8_fold(p1, foldbuf1, &n1);
+ }
+ else { /* Not utf8, get utf8 fold */
+ to_uni_fold(NATIVE_TO_UNI(*p1), foldbuf1, &n1);
}
- n1 = 1;
- *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are
- just lowercased */
+ f1 = foldbuf1;
}
- else if (u1) {
- to_utf8_fold(p1, foldbuf1, &n1);
- }
- else { /* Not utf8, convert to it first and then get fold */
- uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p1)));
- to_utf8_fold(natbuf, foldbuf1, &n1);
- }
- f1 = foldbuf1;
}
if (n2 == 0) { /* Same for s2 */
- if ((flags & FOLDEQ_UTF8_LOCALE)
- && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2)))
- {
- /* Here, the next char in s2 is < 256. We've already worked on
- * s1, and if it isn't also < 256, can't match */
- if (u1 && (! UTF8_IS_INVARIANT(*p1)
- && ! UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ if (flags & FOLDEQ_S2_ALREADY_FOLDED) {
+ f2 = (U8 *) p2;
+ n2 = UTF8SKIP(f2);
+ }
+ else {
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2)))
{
- return 0;
- }
- if (! u2 || UTF8_IS_INVARIANT(*p2)) {
- *foldbuf2 = *p2;
+ /* Here, the next char in s2 is < 256. We've already
+ * worked on s1, and if it isn't also < 256, can't match */
+ if (u1 && (! UTF8_IS_INVARIANT(*p1)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ {
+ return 0;
+ }
+ if (! u2 || UTF8_IS_INVARIANT(*p2)) {
+ *foldbuf2 = *p2;
+ }
+ else {
+ *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1));
+ }
+
+ /* Use another function to handle locale rules. We've made
+ * sure that both characters to compare are single bytes */
+ if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) {
+ return 0;
+ }
+ n1 = n2 = 0;
}
- else {
- *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1));
+ else if (isASCII(*p2)) {
+ if (flags && ! isASCII(*p1)) {
+ return 0;
+ }
+ n2 = 1;
+ *foldbuf2 = toLOWER(*p2);
}
-
- /* Use another function to handle locale rules. We've made
- * sure that both characters to compare are single bytes */
- if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) {
- return 0;
+ else if (u2) {
+ to_utf8_fold(p2, foldbuf2, &n2);
}
- n1 = n2 = 0;
- }
- else if (isASCII(*p2)) {
- if (flags && ! isASCII(*p1)) {
- return 0;
+ else {
+ to_uni_fold(NATIVE_TO_UNI(*p2), foldbuf2, &n2);
}
- n2 = 1;
- *foldbuf2 = toLOWER(*p2);
+ f2 = foldbuf2;
}
- else if (u2) {
- to_utf8_fold(p2, foldbuf2, &n2);
- }
- else {
- uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p2)));
- to_utf8_fold(natbuf, foldbuf2, &n2);
- }
- f2 = foldbuf2;
}
/* Here f1 and f2 point to the beginning of the strings to compare.
- * These strings are the folds of the input characters, stored in utf8.
- */
+ * These strings are the folds of the next character from each input
+ * string, stored in utf8. */
/* While there is more to look for in both folds, see if they
* continue to match */
https://perl5.git.perl.org / perl5.git / blobdiff