* heard of that we don't want to see any closer; and that's the one place
* we're trying to get to! And that's just where we can't get, nohow.'
*
- * [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"]
+ * [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"]
*
* 'Well do I understand your speech,' he answered in the same language;
* 'yet few strangers do so. Why then do you not speak in the Common Tongue,
* as is the custom in the West, if you wish to be answered?'
- * --Gandalf, addressing Théoden's door wardens
+ * --Gandalf, addressing Théoden's door wardens
*
* [p.508 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"]
*
=cut
*/
+/* This logic is duplicated in sv_catpvn_flags, so any bug fixes will
+ likewise need duplication. */
+
U8*
Perl_bytes_to_utf8(pTHX_ const U8 *s, STRLEN *len)
{
return utf16_to_utf8(p, d, bytelen, newlen);
}
-/* for now these are all defined (inefficiently) in terms of the utf8 versions */
+/* for now these are all defined (inefficiently) in terms of the utf8 versions.
+ * Note that the macros in handy.h that call these short-circuit calling them
+ * for Latin-1 range inputs */
bool
Perl_is_uni_alnum(pTHX_ UV c)
bool
Perl_is_uni_ascii(pTHX_ UV c)
{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return is_utf8_ascii(tmpbuf);
+ return isASCII(c);
}
bool
bool
Perl_is_uni_cntrl(pTHX_ UV c)
{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return is_utf8_cntrl(tmpbuf);
+ return isCNTRL_L1(c);
}
bool
return is_utf8_xdigit(tmpbuf);
}
+
UV
Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp)
{
+ /* Convert the Unicode character whose ordinal is c to its uppercase
+ * version and store that in UTF-8 in p and its length in bytes in lenp.
+ * Note that the p needs to be at least UTF8_MAXBYTES_CASE+1 bytes since
+ * the changed version may be longer than the original character.
+ *
+ * The ordinal of the first character of the changed version is returned
+ * (but note, as explained above, that there may be more.) */
+
PERL_ARGS_ASSERT_TO_UNI_UPPER;
uvchr_to_utf8(p, c);
{
PERL_ARGS_ASSERT_TO_UNI_LOWER;
- uvchr_to_utf8(p, c);
- return to_utf8_lower(p, p, lenp);
+ if (c > 255) {
+ uvchr_to_utf8(p, c);
+ return to_utf8_lower(p, p, lenp);
+ }
+
+ /* We have the latin1-range values compiled into the core, so just use
+ * those, converting the result to utf8 */
+ c = toLOWER_LATIN1(c);
+ if (UNI_IS_INVARIANT(c)) {
+ *p = c;
+ *lenp = 1;
+ }
+ else {
+ *p = UTF8_TWO_BYTE_HI(c);
+ *(p+1) = UTF8_TWO_BYTE_LO(c);
+ *lenp = 2;
+ }
+ return c;
}
UV
}
bool
+Perl__is_utf8__perl_idstart(pTHX_ const U8 *p)
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT__IS_UTF8__PERL_IDSTART;
+
+ return is_utf8_common(p, &PL_utf8_perl_idstart, "_Perl_IDStart");
+}
+
+bool
Perl_is_utf8_idcont(pTHX_ const U8 *p)
{
dVAR;
PERL_ARGS_ASSERT_IS_UTF8_IDCONT;
- if (*p == '_')
- return TRUE;
return is_utf8_common(p, &PL_utf8_idcont, "IdContinue");
}
PERL_ARGS_ASSERT_IS_UTF8_XIDCONT;
- if (*p == '_')
- return TRUE;
return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue");
}
PERL_ARGS_ASSERT_IS_UTF8_ASCII;
- return is_utf8_common(p, &PL_utf8_ascii, "IsAscii");
+ /* ASCII characters are the same whether in utf8 or not. So the macro
+ * works on both utf8 and non-utf8 representations. */
+ return isASCII(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_SPACE;
- return is_utf8_common(p, &PL_utf8_space, "IsSpacePerl");
+ return is_utf8_common(p, &PL_utf8_space, "IsXPerlSpace");
}
bool
PERL_ARGS_ASSERT_IS_UTF8_PERL_SPACE;
- return is_utf8_common(p, &PL_utf8_perl_space, "IsPerlSpace");
+ /* Only true if is an ASCII space-like character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isSPACE_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_PERL_WORD;
- return is_utf8_common(p, &PL_utf8_perl_word, "IsPerlWord");
+ /* Only true if is an ASCII word character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isWORDCHAR_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_POSIX_DIGIT;
- return is_utf8_common(p, &PL_utf8_posix_digit, "IsPosixDigit");
+ /* Only true if is an ASCII digit character, and ASCII is invariant
+ * under utf8, so can just use the macro */
+ return isDIGIT_A(*p);
}
bool
PERL_ARGS_ASSERT_IS_UTF8_CNTRL;
- return is_utf8_common(p, &PL_utf8_cntrl, "IsCntrl");
+ if (isASCII(*p)) {
+ return isCNTRL_A(*p);
+ }
+
+ /* All controls are in Latin1 */
+ if (! UTF8_IS_DOWNGRADEABLE_START(*p)) {
+ return 0;
+ }
+ return isCNTRL_L1(TWO_BYTE_UTF8_TO_UNI(*p, *(p+1)));
}
bool
ENTER;
SAVEHINTS();
save_re_context();
+ if (PL_parser && PL_parser->error_count)
+ SAVEI8(PL_parser->error_count), PL_parser->error_count = 0;
method = gv_fetchmeth(stash, "SWASHNEW", 8, -1);
if (!method) { /* demand load utf8 */
ENTER;
const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0);
/* This outputs warnings for binary properties only, assuming that
- * to_utf8_case() will output any. Also, surrogates aren't checked
- * for, as that would warn on things like /\p{Gc=Cs}/ */
+ * to_utf8_case() will output any for non-binary. Also, surrogates
+ * aren't checked for, as that would warn on things like
+ * /\p{Gc=Cs}/ */
SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
if (SvUV(*bitssvp) == 1) {
Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
- "Code point 0x%04"UVXf" is not Unicode, no properties match it; all inverse properties do", code_point);
+ "Code point 0x%04"UVXf" is not Unicode, all \\p{} matches fail; all \\P{} matches succeed", code_point);
}
}
}
{
const int typeto = typestr[0] == 'T' && typestr[1] == 'o';
STRLEN numlen; /* Length of the number */
- I32 flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX;
+ I32 flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
/* nl points to the next \n in the scan */
U8* const nl = (U8*)memchr(l, '\n', lend - l);
/* The max range value follows, separated by a BLANK */
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX;
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
numlen = lend - l;
*max = grok_hex((char *)l, &numlen, &flags, NULL);
if (numlen)
if (wants_value) {
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT |
- PERL_SCAN_DISALLOW_PREFIX;
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
numlen = lend - l;
*val = grok_hex((char *)l, &numlen, &flags, NULL);
if (numlen)
S_swash_get(pTHX_ SV* swash, UV start, UV span)
{
SV *swatch;
- U8 *l, *lend, *x, *xend, *s;
+ U8 *l, *lend, *x, *xend, *s, *send;
STRLEN lcur, xcur, scur;
HV *const hv = MUTABLE_HV(SvRV(swash));
SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE);
SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ SV** const invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
} /* while */
go_out_list:
- /* read $swash->{EXTRAS} */
+ /* Invert if the data says it should be. Assumes that bits == 1 */
+ if (invert_it_svp && SvUV(*invert_it_svp)) {
+
+ /* Unicode properties should come with all bits above PERL_UNICODE_MAX
+ * be 0, and their inversion should also be 0, as we don't succeed any
+ * Unicode property matches for non-Unicode code points */
+ if (start <= PERL_UNICODE_MAX) {
+
+ /* The code below assumes that we never cross the
+ * Unicode/above-Unicode boundary in a range, as otherwise we would
+ * have to figure out where to stop flipping the bits. Since this
+ * boundary is divisible by a large power of 2, and swatches comes
+ * in small powers of 2, this should be a valid assumption */
+ assert(start + span - 1 <= PERL_UNICODE_MAX);
+
+ send = s + scur;
+ while (s < send) {
+ *s = ~(*s);
+ s++;
+ }
+ }
+ }
+
+ /* read $swash->{EXTRAS}
+ * This code also copied to swash_to_invlist() below */
x = (U8*)SvPV(*extssvp, xcur);
xend = x + xcur;
while (x < xend) {
* express the capitalization, for example, of all consecutive
* letters with a single line: 0061\t007A\t0041 This maps 0061 to
* 0041, 0062 to 0042, etc. I (khw) have never understood 'none',
- * and it's not documented, and perhaps not even currently used,
- * but I copied the semantics from swash_get(), just in case */
+ * and it's not documented; it appears to be used only in
+ * implementing tr//; I copied the semantics from swash_get(), just
+ * in case */
if (!none || val < none) {
++val;
}
return ret;
}
-HV*
+SV*
Perl__swash_to_invlist(pTHX_ SV* const swash)
{
SV** const listsvp = hv_fetchs(hv, "LIST", FALSE);
SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+ SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ SV** const invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
+ U8 *x, *xend;
+ STRLEN xcur;
- HV* invlist;
+ SV* invlist;
PERL_ARGS_ASSERT__SWASH_TO_INVLIST;
_append_range_to_invlist(invlist, start, end);
}
+ /* Invert if the data says it should be */
+ if (invert_it_svp && SvUV(*invert_it_svp)) {
+ _invlist_invert_prop(invlist);
+ }
+
+ /* This code is copied from swash_get()
+ * read $swash->{EXTRAS} */
+ x = (U8*)SvPV(*extssvp, xcur);
+ xend = x + xcur;
+ while (x < xend) {
+ STRLEN namelen;
+ U8 *namestr;
+ SV** othersvp;
+ HV* otherhv;
+ STRLEN otherbits;
+ SV **otherbitssvp, *other;
+ U8 *nl;
+
+ const U8 opc = *x++;
+ if (opc == '\n')
+ continue;
+
+ nl = (U8*)memchr(x, '\n', xend - x);
+
+ if (opc != '-' && opc != '+' && opc != '!' && opc != '&') {
+ if (nl) {
+ x = nl + 1; /* 1 is length of "\n" */
+ continue;
+ }
+ else {
+ x = xend; /* to EXTRAS' end at which \n is not found */
+ break;
+ }
+ }
+
+ namestr = x;
+ if (nl) {
+ namelen = nl - namestr;
+ x = nl + 1;
+ }
+ else {
+ namelen = xend - namestr;
+ x = xend;
+ }
+
+ othersvp = hv_fetch(hv, (char *)namestr, namelen, FALSE);
+ otherhv = MUTABLE_HV(SvRV(*othersvp));
+ otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE);
+ otherbits = (STRLEN)SvUV(*otherbitssvp);
+
+ if (bits != otherbits || bits != 1) {
+ Perl_croak(aTHX_ "panic: _swash_to_invlist only operates on boolean properties");
+ }
+
+ /* The "other" swatch must be destroyed after. */
+ other = _swash_to_invlist((SV *)*othersvp);
+
+ /* End of code copied from swash_get() */
+ switch (opc) {
+ case '+':
+ _invlist_union(invlist, other, &invlist);
+ break;
+ case '!':
+ _invlist_invert(other);
+ _invlist_union(invlist, other, &invlist);
+ break;
+ case '-':
+ _invlist_subtract(invlist, other, &invlist);
+ break;
+ case '&':
+ _invlist_intersection(invlist, other, &invlist);
+ break;
+ default:
+ break;
+ }
+ sv_free(other); /* through with it! */
+ }
+
return invlist;
}
* points below 256; unicode rules for above 255; and
* folds that cross those boundaries are disallowed,
* like the NOMIX_ASCII option
+ * FOLDEQ_S1_ALREADY_FOLDED s1 has already been folded before calling this
+ * routine. This allows that step to be skipped.
+ * FOLDEQ_S2_ALREADY_FOLDED Similarly.
*/
I32
Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2, U32 flags)
PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS;
+ /* The algorithm requires that input with the flags on the first line of
+ * the assert not be pre-folded. */
+ assert( ! ((flags & (FOLDEQ_UTF8_NOMIX_ASCII | FOLDEQ_UTF8_LOCALE))
+ && (flags & (FOLDEQ_S1_ALREADY_FOLDED | FOLDEQ_S2_ALREADY_FOLDED))));
+
if (pe1) {
e1 = *(U8**)pe1;
}
assert(e2);
}
+ /* If both operands are already folded, we could just do a memEQ on the
+ * whole strings at once, but it would be better if the caller realized
+ * this and didn't even call us */
+
/* Look through both strings, a character at a time */
while (p1 < e1 && p2 < e2) {
* and the length of the fold. (exception: locale rules just get the
* character to a single byte) */
if (n1 == 0) {
+ if (flags & FOLDEQ_S1_ALREADY_FOLDED) {
+ f1 = (U8 *) p1;
+ n1 = UTF8SKIP(f1);
/* If in locale matching, we use two sets of rules, depending on if
* the code point is above or below 255. Here, we test for and
* handle locale rules */
- if ((flags & FOLDEQ_UTF8_LOCALE)
- && (! u1 || UTF8_IS_INVARIANT(*p1) || UTF8_IS_DOWNGRADEABLE_START(*p1)))
- {
- /* There is no mixing of code points above and below 255. */
- if (u2 && (! UTF8_IS_INVARIANT(*p2)
- && ! UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ }
+ else {
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u1 || UTF8_IS_INVARIANT(*p1)
+ || UTF8_IS_DOWNGRADEABLE_START(*p1)))
{
- return 0;
- }
+ /* There is no mixing of code points above and below 255. */
+ if (u2 && (! UTF8_IS_INVARIANT(*p2)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ {
+ return 0;
+ }
- /* We handle locale rules by converting, if necessary, the code
- * point to a single byte. */
- if (! u1 || UTF8_IS_INVARIANT(*p1)) {
- *foldbuf1 = *p1;
+ /* We handle locale rules by converting, if necessary, the
+ * code point to a single byte. */
+ if (! u1 || UTF8_IS_INVARIANT(*p1)) {
+ *foldbuf1 = *p1;
+ }
+ else {
+ *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1));
+ }
+ n1 = 1;
}
- else {
- *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1));
+ else if (isASCII(*p1)) { /* Note, that here won't be
+ both ASCII and using locale
+ rules */
+
+ /* If trying to mix non- with ASCII, and not supposed to,
+ * fail */
+ if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) {
+ return 0;
+ }
+ n1 = 1;
+ *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are
+ just lowercased */
}
- n1 = 1;
- }
- else if (isASCII(*p1)) { /* Note, that here won't be both ASCII
- and using locale rules */
-
- /* If trying to mix non- with ASCII, and not supposed to, fail */
- if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) {
- return 0;
+ else if (u1) {
+ to_utf8_fold(p1, foldbuf1, &n1);
}
- n1 = 1;
- *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are
- just lowercased */
+ else { /* Not utf8, convert to it first and then get fold */
+ uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p1)));
+ to_utf8_fold(natbuf, foldbuf1, &n1);
+ }
+ f1 = foldbuf1;
}
- else if (u1) {
- to_utf8_fold(p1, foldbuf1, &n1);
- }
- else { /* Not utf8, convert to it first and then get fold */
- uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p1)));
- to_utf8_fold(natbuf, foldbuf1, &n1);
- }
- f1 = foldbuf1;
}
if (n2 == 0) { /* Same for s2 */
- if ((flags & FOLDEQ_UTF8_LOCALE)
- && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2)))
- {
- /* Here, the next char in s2 is < 256. We've already worked on
- * s1, and if it isn't also < 256, can't match */
- if (u1 && (! UTF8_IS_INVARIANT(*p1)
- && ! UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ if (flags & FOLDEQ_S2_ALREADY_FOLDED) {
+ f2 = (U8 *) p2;
+ n2 = UTF8SKIP(f2);
+ }
+ else {
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2)))
{
- return 0;
- }
- if (! u2 || UTF8_IS_INVARIANT(*p2)) {
- *foldbuf2 = *p2;
+ /* Here, the next char in s2 is < 256. We've already
+ * worked on s1, and if it isn't also < 256, can't match */
+ if (u1 && (! UTF8_IS_INVARIANT(*p1)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ {
+ return 0;
+ }
+ if (! u2 || UTF8_IS_INVARIANT(*p2)) {
+ *foldbuf2 = *p2;
+ }
+ else {
+ *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1));
+ }
+
+ /* Use another function to handle locale rules. We've made
+ * sure that both characters to compare are single bytes */
+ if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) {
+ return 0;
+ }
+ n1 = n2 = 0;
}
- else {
- *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1));
+ else if (isASCII(*p2)) {
+ if (flags && ! isASCII(*p1)) {
+ return 0;
+ }
+ n2 = 1;
+ *foldbuf2 = toLOWER(*p2);
}
-
- /* Use another function to handle locale rules. We've made
- * sure that both characters to compare are single bytes */
- if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) {
- return 0;
+ else if (u2) {
+ to_utf8_fold(p2, foldbuf2, &n2);
}
- n1 = n2 = 0;
- }
- else if (isASCII(*p2)) {
- if (flags && ! isASCII(*p1)) {
- return 0;
+ else {
+ uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p2)));
+ to_utf8_fold(natbuf, foldbuf2, &n2);
}
- n2 = 1;
- *foldbuf2 = toLOWER(*p2);
+ f2 = foldbuf2;
}
- else if (u2) {
- to_utf8_fold(p2, foldbuf2, &n2);
- }
- else {
- uvuni_to_utf8(natbuf, (UV) NATIVE_TO_UNI(((UV)*p2)));
- to_utf8_fold(natbuf, foldbuf2, &n2);
- }
- f2 = foldbuf2;
}
/* Here f1 and f2 point to the beginning of the strings to compare.
- * These strings are the folds of the input characters, stored in utf8.
- */
+ * These strings are the folds of the next character from each input
+ * string, stored in utf8. */
/* While there is more to look for in both folds, see if they
* continue to match */