if (ckWARN_d(WARN_UTF8)) {
if (UNICODE_IS_SURROGATE(uv)) {
if (flags & UNICODE_WARN_SURROGATE) {
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_SURROGATE),
"UTF-16 surrogate U+%04"UVXf, uv);
}
if (flags & UNICODE_DISALLOW_SURROGATE) {
if (flags & UNICODE_WARN_SUPER
|| (UNICODE_IS_FE_FF(uv) && (flags & UNICODE_WARN_FE_FF)))
{
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_NON_UNICODE),
"Code point 0x%04"UVXf" is not Unicode, may not be portable", uv);
}
if (flags & UNICODE_DISALLOW_SUPER
}
else if (UNICODE_IS_NONCHAR(uv)) {
if (flags & UNICODE_WARN_NONCHAR) {
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_NONCHAR),
"Unicode non-character U+%04"UVXf" is illegal for open interchange",
uv);
}
Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8),
"%s in %s", unees, OP_DESC(PL_op));
else
- Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees);
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s", unees);
}
return len;
Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8),
"%s in %s", unees, OP_DESC(PL_op));
else
- Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), unees);
+ Perl_ck_warner_d(aTHX_ packWARN(WARN_UTF8), "%s", unees);
return -2; /* Really want to return undef :-) */
}
} else {
}
UV
-Perl_to_uni_fold(pTHX_ UV c, U8* p, STRLEN *lenp)
+Perl__to_uni_fold_flags(pTHX_ UV c, U8* p, STRLEN *lenp, U8 flags)
{
- PERL_ARGS_ASSERT_TO_UNI_FOLD;
+ PERL_ARGS_ASSERT__TO_UNI_FOLD_FLAGS;
uvchr_to_utf8(p, c);
- return to_utf8_fold(p, p, lenp);
+ return _to_utf8_fold_flags(p, p, lenp, flags);
}
/* for now these all assume no locale info available for Unicode > 255 */
}
bool
+Perl_is_utf8_xidfirst(pTHX_ const U8 *p) /* The naming is historical. */
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT_IS_UTF8_XIDFIRST;
+
+ if (*p == '_')
+ return TRUE;
+ /* is_utf8_idstart would be more logical. */
+ return is_utf8_common(p, &PL_utf8_xidstart, "XIdStart");
+}
+
+bool
Perl_is_utf8_idcont(pTHX_ const U8 *p)
{
dVAR;
}
bool
+Perl_is_utf8_xidcont(pTHX_ const U8 *p)
+{
+ dVAR;
+
+ PERL_ARGS_ASSERT_IS_UTF8_XIDCONT;
+
+ if (*p == '_')
+ return TRUE;
+ return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue");
+}
+
+bool
Perl_is_utf8_alpha(pTHX_ const U8 *p)
{
dVAR;
The "swashp" is a pointer to the swash to use.
-Both the special and normal mappings are stored lib/unicore/To/Foo.pl,
+Both the special and normal mappings are stored in lib/unicore/To/Foo.pl,
and loaded by SWASHNEW, using lib/utf8_heavy.pl. The special (usually,
but not always, a multicharacter mapping), is tried first.
/* Note that swash_fetch() doesn't output warnings for these because it
* assumes we will */
- if (uv1 >= UNICODE_SURROGATE_FIRST && ckWARN_d(WARN_UTF8)) {
+ if (uv1 >= UNICODE_SURROGATE_FIRST) {
if (uv1 <= UNICODE_SURROGATE_LAST) {
- const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Operation \"%s\" returns its argument for UTF-16 surrogate U+%04"UVXf"", desc, uv1);
+ if (ckWARN_d(WARN_SURROGATE)) {
+ const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
+ Perl_warner(aTHX_ packWARN(WARN_SURROGATE),
+ "Operation \"%s\" returns its argument for UTF-16 surrogate U+%04"UVXf"", desc, uv1);
+ }
}
else if (UNICODE_IS_SUPER(uv1)) {
- const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Operation \"%s\" returns its argument for non-Unicode code point 0x%04"UVXf"", desc, uv1);
+ if (ckWARN_d(WARN_NON_UNICODE)) {
+ const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
+ "Operation \"%s\" returns its argument for non-Unicode code point 0x%04"UVXf"", desc, uv1);
+ }
}
/* Note that non-characters are perfectly legal, so no warning should
if (!*swashp) /* load on-demand */
*swashp = swash_init("utf8", normal, &PL_sv_undef, 4, 0);
- /* This is the beginnings of a skeleton of code to read the info section
- * that is in all the swashes in case we ever want to do that, so one can
- * read things whose maps aren't code points, and whose default if missing
- * is not to the code point itself. This was just to see if it actually
- * worked. Details on what the possibilities are are in perluniprops.pod
- HV * const hv = get_hv("utf8::SwashInfo", 0);
- if (hv) {
- SV **svp;
- svp = hv_fetch(hv, (const char*)normal, strlen(normal), FALSE);
- const char *s;
-
- HV * const this_hash = SvRV(*svp);
- svp = hv_fetch(this_hash, "type", strlen("type"), FALSE);
- s = SvPV_const(*svp, len);
- }
- }*/
if (special) {
/* It might be "special" (sometimes, but not always,
=cut */
+/* Not currently externally documented is 'flags', which currently is non-zero
+ * if full case folds are to be used; otherwise simple folds */
+
UV
-Perl_to_utf8_fold(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp)
+Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags)
{
+ const char *specials = (flags) ? "utf8::ToSpecFold" : NULL;
+
dVAR;
- PERL_ARGS_ASSERT_TO_UTF8_FOLD;
+ PERL_ARGS_ASSERT__TO_UTF8_FOLD_FLAGS;
return Perl_to_utf8_case(aTHX_ p, ustrp, lenp,
- &PL_utf8_tofold, "ToFold", "utf8::ToSpecFold");
+ &PL_utf8_tofold, "ToFold", specials);
}
/* Note:
/* If char is encoded then swatch is for the prefix */
needents = (1 << UTF_ACCUMULATION_SHIFT);
off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK;
- if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_UTF8)) {
+ if (UTF8_IS_SUPER(ptr) && ckWARN_d(WARN_NON_UNICODE)) {
const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, 0);
/* This outputs warnings for binary properties only, assuming that
* for, as that would warn on things like /\p{Gc=Cs}/ */
SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
if (SvUV(*bitssvp) == 1) {
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
"Code point 0x%04"UVXf" is not Unicode, no properties match it; all inverse properties do", code_point);
}
}
{
const int typeto = typestr[0] == 'T' && typestr[1] == 'o';
STRLEN numlen; /* Length of the number */
- I32 flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX;
+ I32 flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
/* nl points to the next \n in the scan */
U8* const nl = (U8*)memchr(l, '\n', lend - l);
/* The max range value follows, separated by a BLANK */
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX;
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
numlen = lend - l;
*max = grok_hex((char *)l, &numlen, &flags, NULL);
if (numlen)
if (wants_value) {
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT |
- PERL_SCAN_DISALLOW_PREFIX;
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
numlen = lend - l;
*val = grok_hex((char *)l, &numlen, &flags, NULL);
if (numlen)
S_swash_get(pTHX_ SV* swash, UV start, UV span)
{
SV *swatch;
- U8 *l, *lend, *x, *xend, *s;
+ U8 *l, *lend, *x, *xend, *s, *send;
STRLEN lcur, xcur, scur;
HV *const hv = MUTABLE_HV(SvRV(swash));
SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE);
SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ SV** const invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
} /* while */
go_out_list:
- /* read $swash->{EXTRAS} */
+ /* Invert if the data says it should be */
+ if (invert_it_svp && SvUV(*invert_it_svp)) {
+ send = s + scur;
+ while (s < send) {
+ *s = ~(*s);
+ s++;
+ }
+ }
+
+ /* read $swash->{EXTRAS}
+ * This code also copied to swash_to_invlist() below */
x = (U8*)SvPV(*extssvp, xcur);
xend = x + xcur;
while (x < xend) {
Perl__swash_inversion_hash(pTHX_ SV* const swash)
{
- /* Subject to change or removal. For use only in one place in regexec.c
+ /* Subject to change or removal. For use only in one place in regcomp.c.
+ * Can't be used on a property that is subject to user override, as it
+ * relies on the value of SPECIALS in the swash which would be set by
+ * utf8_heavy.pl to the hash in the non-overriden file, and hence is not set
+ * for overridden properties
*
* Returns a hash which is the inversion and closure of a swash mapping.
* For example, consider the input lines:
* Essentially, for any code point, it gives all the code points that map to
* it, or the list of 'froms' for that point.
*
- * Currently it only looks at the main body of the swash, and ignores any
- * additions or deletions from other swashes */
+ * Currently it ignores any additions or deletions from other swashes,
+ * looking at just the main body of the swash, and if there are SPECIALS
+ * in the swash, at that hash
+ *
+ * The specials hash can be extra code points, and most likely consists of
+ * maps from single code points to multiple ones (each expressed as a string
+ * of utf8 characters). This function currently returns only 1-1 mappings.
+ * However consider this possible input in the specials hash:
+ * "\xEF\xAC\x85" => "\x{0073}\x{0074}", # U+FB05 => 0073 0074
+ * "\xEF\xAC\x86" => "\x{0073}\x{0074}", # U+FB06 => 0073 0074
+ *
+ * Both FB05 and FB06 map to the same multi-char sequence, which we don't
+ * currently handle. But it also means that FB05 and FB06 are equivalent in
+ * a 1-1 mapping which we should handle, and this relationship may not be in
+ * the main table. Therefore this function examines all the multi-char
+ * sequences and adds the 1-1 mappings that come out of that. */
U8 *l, *lend;
STRLEN lcur;
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
const UV none = SvUV(*nonesvp);
+ SV **specials_p = hv_fetchs(hv, "SPECIALS", 0);
HV* ret = newHV();
(UV)bits);
}
+ if (specials_p) { /* It might be "special" (sometimes, but not always, a
+ mapping to more than one character */
+
+ /* Construct an inverse mapping hash for the specials */
+ HV * const specials_hv = MUTABLE_HV(SvRV(*specials_p));
+ HV * specials_inverse = newHV();
+ char *char_from; /* the lhs of the map */
+ I32 from_len; /* its byte length */
+ char *char_to; /* the rhs of the map */
+ I32 to_len; /* its byte length */
+ SV *sv_to; /* and in a sv */
+ AV* from_list; /* list of things that map to each 'to' */
+
+ hv_iterinit(specials_hv);
+
+ /* The keys are the characters (in utf8) that map to the corresponding
+ * utf8 string value. Iterate through the list creating the inverse
+ * list. */
+ while ((sv_to = hv_iternextsv(specials_hv, &char_from, &from_len))) {
+ SV** listp;
+ if (! SvPOK(sv_to)) {
+ Perl_croak(aTHX_ "panic: value returned from hv_iternextsv() unexpectedly is not a string");
+ }
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Found mapping from %"UVXf", First char of to is %"UVXf"\n", utf8_to_uvchr((U8*) char_from, 0), utf8_to_uvchr((U8*) SvPVX(sv_to), 0)));*/
+
+ /* Each key in the inverse list is a mapped-to value, and the key's
+ * hash value is a list of the strings (each in utf8) that map to
+ * it. Those strings are all one character long */
+ if ((listp = hv_fetch(specials_inverse,
+ SvPVX(sv_to),
+ SvCUR(sv_to), 0)))
+ {
+ from_list = (AV*) *listp;
+ }
+ else { /* No entry yet for it: create one */
+ from_list = newAV();
+ if (! hv_store(specials_inverse,
+ SvPVX(sv_to),
+ SvCUR(sv_to),
+ (SV*) from_list, 0))
+ {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+ }
+
+ /* Here have the list associated with this 'to' (perhaps newly
+ * created and empty). Just add to it. Note that we ASSUME that
+ * the input is guaranteed to not have duplications, so we don't
+ * check for that. Duplications just slow down execution time. */
+ av_push(from_list, newSVpvn_utf8(char_from, from_len, TRUE));
+ }
+
+ /* Here, 'specials_inverse' contains the inverse mapping. Go through
+ * it looking for cases like the FB05/FB06 examples above. There would
+ * be an entry in the hash like
+ * 'st' => [ FB05, FB06 ]
+ * In this example we will create two lists that get stored in the
+ * returned hash, 'ret':
+ * FB05 => [ FB05, FB06 ]
+ * FB06 => [ FB05, FB06 ]
+ *
+ * Note that there is nothing to do if the array only has one element.
+ * (In the normal 1-1 case handled below, we don't have to worry about
+ * two lists, as everything gets tied to the single list that is
+ * generated for the single character 'to'. But here, we are omitting
+ * that list, ('st' in the example), so must have multiple lists.) */
+ while ((from_list = (AV *) hv_iternextsv(specials_inverse,
+ &char_to, &to_len)))
+ {
+ if (av_len(from_list) > 0) {
+ int i;
+
+ /* We iterate over all combinations of i,j to place each code
+ * point on each list */
+ for (i = 0; i <= av_len(from_list); i++) {
+ int j;
+ AV* i_list = newAV();
+ SV** entryp = av_fetch(from_list, i, FALSE);
+ if (entryp == NULL) {
+ Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
+ }
+ if (hv_fetch(ret, SvPVX(*entryp), SvCUR(*entryp), FALSE)) {
+ Perl_croak(aTHX_ "panic: unexpected entry for %s", SvPVX(*entryp));
+ }
+ if (! hv_store(ret, SvPVX(*entryp), SvCUR(*entryp),
+ (SV*) i_list, FALSE))
+ {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+
+ /* For debugging: UV u = utf8_to_uvchr((U8*) SvPVX(*entryp), 0);*/
+ for (j = 0; j <= av_len(from_list); j++) {
+ entryp = av_fetch(from_list, j, FALSE);
+ if (entryp == NULL) {
+ Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
+ }
+
+ /* When i==j this adds itself to the list */
+ av_push(i_list, newSVuv(utf8_to_uvchr(
+ (U8*) SvPVX(*entryp), 0)));
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", utf8_to_uvchr((U8*) SvPVX(*entryp), 0), u));*/
+ }
+ }
+ }
+ }
+ SvREFCNT_dec(specials_inverse); /* done with it */
+ } /* End of specials */
+
/* read $swash->{LIST} */
l = (U8*)SvPV(*listsvp, lcur);
lend = l + lcur;
/* Each element in the range is to be inverted */
for (inverse = min; inverse <= max; inverse++) {
AV* list;
- SV* element;
SV** listp;
IV i;
bool found_key = FALSE;
+ bool found_inverse = FALSE;
/* The key is the inverse mapping */
char key[UTF8_MAXBYTES+1];
}
}
- for (i = 0; i < av_len(list); i++) {
+ /* Look through list to see if this inverse mapping already is
+ * listed, or if there is a mapping to itself already */
+ for (i = 0; i <= av_len(list); i++) {
SV** entryp = av_fetch(list, i, FALSE);
SV* entry;
if (entryp == NULL) {
Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
}
entry = *entryp;
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "list for %"UVXf" contains %"UVXf"\n", val, SvUV(entry)));*/
if (SvUV(entry) == val) {
found_key = TRUE;
+ }
+ if (SvUV(entry) == inverse) {
+ found_inverse = TRUE;
+ }
+
+ /* No need to continue searching if found everything we are
+ * looking for */
+ if (found_key && found_inverse) {
break;
}
}
/* Make sure there is a mapping to itself on the list */
if (! found_key) {
- element = newSVuv(val);
- av_push(list, element);
+ av_push(list, newSVuv(val));
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", val, val));*/
}
/* Simply add the value to the list */
- element = newSVuv(inverse);
- av_push(list, element);
+ if (! found_inverse) {
+ av_push(list, newSVuv(inverse));
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", inverse, val));*/
+ }
/* swash_get() increments the value of val for each element in the
* range. That makes more compact tables possible. You can
* express the capitalization, for example, of all consecutive
* letters with a single line: 0061\t007A\t0041 This maps 0061 to
* 0041, 0062 to 0042, etc. I (khw) have never understood 'none',
- * and it's not documented, and perhaps not even currently used,
- * but I copied the semantics from swash_get(), just in case */
+ * and it's not documented; it appears to be used only in
+ * implementing tr//; I copied the semantics from swash_get(), just
+ * in case */
if (!none || val < none) {
++val;
}
return ret;
}
-HV*
+SV*
Perl__swash_to_invlist(pTHX_ SV* const swash)
{
STRLEN lcur;
HV *const hv = MUTABLE_HV(SvRV(swash));
UV elements = 0; /* Number of elements in the inversion list */
+ U8 empty[] = "";
/* The string containing the main body of the table */
SV** const listsvp = hv_fetchs(hv, "LIST", FALSE);
SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+ SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ SV** const invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
const STRLEN bits = SvUV(*bitssvp);
const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
+ U8 *x, *xend;
+ STRLEN xcur;
- HV* invlist;
+ SV* invlist;
PERL_ARGS_ASSERT__SWASH_TO_INVLIST;
/* read $swash->{LIST} */
- l = (U8*)SvPV(*listsvp, lcur);
+ if (SvPOK(*listsvp)) {
+ l = (U8*)SvPV(*listsvp, lcur);
+ }
+ else {
+ /* LIST legitimately doesn't contain a string during compilation phases
+ * of Perl itself, before the Unicode tables are generated. In this
+ * case, just fake things up by creating an empty list */
+ l = empty;
+ lcur = 0;
+ }
loc = (char *) l;
lend = l + lcur;
_append_range_to_invlist(invlist, start, end);
}
+ /* Invert if the data says it should be */
+ if (invert_it_svp && SvUV(*invert_it_svp)) {
+ _invlist_invert(invlist);
+ }
+
+ /* This code is copied from swash_get()
+ * read $swash->{EXTRAS} */
+ x = (U8*)SvPV(*extssvp, xcur);
+ xend = x + xcur;
+ while (x < xend) {
+ STRLEN namelen;
+ U8 *namestr;
+ SV** othersvp;
+ HV* otherhv;
+ STRLEN otherbits;
+ SV **otherbitssvp, *other;
+ U8 *nl;
+
+ const U8 opc = *x++;
+ if (opc == '\n')
+ continue;
+
+ nl = (U8*)memchr(x, '\n', xend - x);
+
+ if (opc != '-' && opc != '+' && opc != '!' && opc != '&') {
+ if (nl) {
+ x = nl + 1; /* 1 is length of "\n" */
+ continue;
+ }
+ else {
+ x = xend; /* to EXTRAS' end at which \n is not found */
+ break;
+ }
+ }
+
+ namestr = x;
+ if (nl) {
+ namelen = nl - namestr;
+ x = nl + 1;
+ }
+ else {
+ namelen = xend - namestr;
+ x = xend;
+ }
+
+ othersvp = hv_fetch(hv, (char *)namestr, namelen, FALSE);
+ otherhv = MUTABLE_HV(SvRV(*othersvp));
+ otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE);
+ otherbits = (STRLEN)SvUV(*otherbitssvp);
+
+ if (bits != otherbits || bits != 1) {
+ Perl_croak(aTHX_ "panic: _swash_to_invlist only operates on boolean properties");
+ }
+
+ /* The "other" swatch must be destroyed after. */
+ other = _swash_to_invlist((SV *)*othersvp);
+
+ /* End of code copied from swash_get() */
+ switch (opc) {
+ case '+':
+ _invlist_union(invlist, other, &invlist);
+ break;
+ case '!':
+ _invlist_invert(other);
+ _invlist_union(invlist, other, &invlist);
+ break;
+ case '-':
+ _invlist_subtract(invlist, other, &invlist);
+ break;
+ case '&':
+ _invlist_intersection(invlist, other, &invlist);
+ break;
+ default:
+ break;
+ }
+ sv_free(other); /* through with it! */
+ }
+
return invlist;
}
if (*s >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE) {
STRLEN char_len;
if (UTF8_IS_SUPER(s)) {
- UV uv = utf8_to_uvchr(s, &char_len);
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv);
- ok = FALSE;
+ if (ckWARN_d(WARN_NON_UNICODE)) {
+ UV uv = utf8_to_uvchr(s, &char_len);
+ Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE),
+ "Code point 0x%04"UVXf" is not Unicode, may not be portable", uv);
+ ok = FALSE;
+ }
}
else if (UTF8_IS_SURROGATE(s)) {
- UV uv = utf8_to_uvchr(s, &char_len);
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv);
- ok = FALSE;
+ if (ckWARN_d(WARN_SURROGATE)) {
+ UV uv = utf8_to_uvchr(s, &char_len);
+ Perl_warner(aTHX_ packWARN(WARN_SURROGATE),
+ "Unicode surrogate U+%04"UVXf" is illegal in UTF-8", uv);
+ ok = FALSE;
+ }
}
else if
- (UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s))
+ ((UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s))
+ && (ckWARN_d(WARN_NONCHAR)))
{
UV uv = utf8_to_uvchr(s, &char_len);
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
+ Perl_warner(aTHX_ packWARN(WARN_NONCHAR),
"Unicode non-character U+%04"UVXf" is illegal for open interchange", uv);
ok = FALSE;
}
http://www.unicode.org/unicode/reports/tr21/ (Case Mappings).
=cut */
+
+/* A flags parameter has been added which may change, and hence isn't
+ * externally documented. Currently it is:
+ * 0 for as-documented above
+ * FOLDEQ_UTF8_NOMIX_ASCII meaning that if a non-ASCII character folds to an
+ ASCII one, to not match
+ * FOLDEQ_UTF8_LOCALE meaning that locale rules are to be used for code
+ * points below 256; unicode rules for above 255; and
+ * folds that cross those boundaries are disallowed,
+ * like the NOMIX_ASCII option
+ */
I32
-Perl_foldEQ_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2)
+Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2, U32 flags)
{
dVAR;
register const U8 *p1 = (const U8*)s1; /* Point to current char */
U8 natbuf[2]; /* Holds native 8-bit char converted to utf8;
these always fit in 2 bytes */
- PERL_ARGS_ASSERT_FOLDEQ_UTF8;
+ PERL_ARGS_ASSERT_FOLDEQ_UTF8_FLAGS;
if (pe1) {
e1 = *(U8**)pe1;
while (p1 < e1 && p2 < e2) {
/* If at the beginning of a new character in s1, get its fold to use
- * and the length of the fold */
+ * and the length of the fold. (exception: locale rules just get the
+ * character to a single byte) */
if (n1 == 0) {
- if (u1) {
+
+ /* If in locale matching, we use two sets of rules, depending on if
+ * the code point is above or below 255. Here, we test for and
+ * handle locale rules */
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u1 || UTF8_IS_INVARIANT(*p1) || UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ {
+ /* There is no mixing of code points above and below 255. */
+ if (u2 && (! UTF8_IS_INVARIANT(*p2)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ {
+ return 0;
+ }
+
+ /* We handle locale rules by converting, if necessary, the code
+ * point to a single byte. */
+ if (! u1 || UTF8_IS_INVARIANT(*p1)) {
+ *foldbuf1 = *p1;
+ }
+ else {
+ *foldbuf1 = TWO_BYTE_UTF8_TO_UNI(*p1, *(p1 + 1));
+ }
+ n1 = 1;
+ }
+ else if (isASCII(*p1)) { /* Note, that here won't be both ASCII
+ and using locale rules */
+
+ /* If trying to mix non- with ASCII, and not supposed to, fail */
+ if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p2)) {
+ return 0;
+ }
+ n1 = 1;
+ *foldbuf1 = toLOWER(*p1); /* Folds in the ASCII range are
+ just lowercased */
+ }
+ else if (u1) {
to_utf8_fold(p1, foldbuf1, &n1);
}
else { /* Not utf8, convert to it first and then get fold */
}
if (n2 == 0) { /* Same for s2 */
- if (u2) {
+ if ((flags & FOLDEQ_UTF8_LOCALE)
+ && (! u2 || UTF8_IS_INVARIANT(*p2) || UTF8_IS_DOWNGRADEABLE_START(*p2)))
+ {
+ /* Here, the next char in s2 is < 256. We've already worked on
+ * s1, and if it isn't also < 256, can't match */
+ if (u1 && (! UTF8_IS_INVARIANT(*p1)
+ && ! UTF8_IS_DOWNGRADEABLE_START(*p1)))
+ {
+ return 0;
+ }
+ if (! u2 || UTF8_IS_INVARIANT(*p2)) {
+ *foldbuf2 = *p2;
+ }
+ else {
+ *foldbuf2 = TWO_BYTE_UTF8_TO_UNI(*p2, *(p2 + 1));
+ }
+
+ /* Use another function to handle locale rules. We've made
+ * sure that both characters to compare are single bytes */
+ if (! foldEQ_locale((char *) f1, (char *) foldbuf2, 1)) {
+ return 0;
+ }
+ n1 = n2 = 0;
+ }
+ else if (isASCII(*p2)) {
+ if (flags && ! isASCII(*p1)) {
+ return 0;
+ }
+ n2 = 1;
+ *foldbuf2 = toLOWER(*p2);
+ }
+ else if (u2) {
to_utf8_fold(p2, foldbuf2, &n2);
}
else {
f2 = foldbuf2;
}
+ /* Here f1 and f2 point to the beginning of the strings to compare.
+ * These strings are the folds of the input characters, stored in utf8.
+ */
+
/* While there is more to look for in both folds, see if they
* continue to match */
while (n1 && n2) {
https://perl5.git.perl.org / perl5.git / blobdiff