/*
-Tests if some arbitrary number of bytes begins in a valid UTF-8
+Tests if the first C<len> bytes of string C<s> form a valid UTF-8
character. Note that an INVARIANT (i.e. ASCII) character is a valid
-UTF-8 character. The actual number of bytes in the UTF-8 character
+UTF-8 character. The number of bytes in the UTF-8 character
will be returned if it is valid, otherwise 0.
This is the "slow" version as opposed to the "fast" version which is
PERL_ARGS_ASSERT_IS_UTF8_CHAR_SLOW;
if (UTF8_IS_INVARIANT(u))
- return 1;
+ return len == 1;
if (!UTF8_IS_START(u))
return 0;
}
/*
+=for apidoc is_utf8_char_buf
+
+Returns the number of bytes that comprise the first UTF-8 encoded character in
+buffer C<buf>. C<buf_end> should point to one position beyond the end of the
+buffer. 0 is returned if C<buf> does not point to a complete, valid UTF-8
+encoded character.
+
+Note that an INVARIANT character (i.e. ASCII on non-EBCDIC
+machines) is a valid UTF-8 character.
+
+=cut */
+
+STRLEN
+Perl_is_utf8_char_buf(const U8 *buf, const U8* buf_end)
+{
+
+ STRLEN len;
+
+ PERL_ARGS_ASSERT_IS_UTF8_CHAR_BUF;
+
+ if (buf_end <= buf) {
+ return 0;
+ }
+
+ len = buf_end - buf;
+ if (len > UTF8SKIP(buf)) {
+ len = UTF8SKIP(buf);
+ }
+
+#ifdef IS_UTF8_CHAR
+ if (IS_UTF8_CHAR_FAST(len))
+ return IS_UTF8_CHAR(buf, len) ? len : 0;
+#endif /* #ifdef IS_UTF8_CHAR */
+ return is_utf8_char_slow(buf, len);
+}
+
+/*
=for apidoc is_utf8_char
+DEPRECATED!
+
Tests if some arbitrary number of bytes begins in a valid UTF-8
character. Note that an INVARIANT (i.e. ASCII on non-EBCDIC machines)
character is a valid UTF-8 character. The actual number of bytes in the UTF-8
character will be returned if it is valid, otherwise 0.
-WARNING: use only if you *know* that C<s> has at least either UTF8_MAXBYTES or
-UTF8SKIP(s) bytes.
+This function is deprecated due to the possibility that malformed input could
+cause reading beyond the end of the input buffer. Use C<is_utf8_char_buf>
+instead.
=cut */
+
STRLEN
Perl_is_utf8_char(const U8 *s)
{
- const STRLEN len = UTF8SKIP(s);
-
PERL_ARGS_ASSERT_IS_UTF8_CHAR;
-#ifdef IS_UTF8_CHAR
- if (IS_UTF8_CHAR_FAST(len))
- return IS_UTF8_CHAR(s, len) ? len : 0;
-#endif /* #ifdef IS_UTF8_CHAR */
- return is_utf8_char_slow(s, len);
+
+ /* Assumes we have enough space, which is why this is deprecated */
+ return is_utf8_char_buf(s, s + UTF8SKIP(s));
}
* LENP will be set to the length in bytes of the string of changed characters
*
* The functions return the ordinal of the first character in the string of OUTP */
-#define CALL_UPPER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_toupper, "ToUc", "utf8::ToSpecUpper")
-#define CALL_TITLE_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_totitle, "ToTc", "utf8::ToSpecTitle")
-#define CALL_LOWER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tolower, "ToLc", "utf8::ToSpecLower")
+#define CALL_UPPER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_toupper, "ToUc", "utf8::ToSpecUc")
+#define CALL_TITLE_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_totitle, "ToTc", "utf8::ToSpecTc")
+#define CALL_LOWER_CASE(INP, OUTP, LENP) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tolower, "ToLc", "utf8::ToSpecLc")
/* This additionally has the input parameter SPECIALS, which if non-zero will
* cause this to use the SPECIALS hash for folding (meaning get full case
* folding); otherwise, when zero, this implies a simple case fold */
-#define CALL_FOLD_CASE(INP, OUTP, LENP, SPECIALS) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tofold, "ToCf", (SPECIALS) ? "utf8::ToSpecFold" : NULL)
+#define CALL_FOLD_CASE(INP, OUTP, LENP, SPECIALS) Perl_to_utf8_case(aTHX_ INP, OUTP, LENP, &PL_utf8_tofold, "ToCf", (SPECIALS) ? "utf8::ToSpecCf" : NULL)
UV
Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp)
return CALL_FOLD_CASE(p, p, lenp, flags);
}
-/* for now these all assume no locale info available for Unicode > 255 */
+/* for now these all assume no locale info available for Unicode > 255; and
+ * the corresponding macros in handy.h (like isALNUM_LC_uvchr) should have been
+ * called instead, so that these don't get called for < 255 */
bool
Perl_is_uni_alnum_lc(pTHX_ UV c)
S_is_utf8_common(pTHX_ const U8 *const p, SV **swash,
const char *const swashname)
{
+ /* returns a boolean giving whether or not the UTF8-encoded character that
+ * starts at <p> is in the swash indicated by <swashname>. <swash>
+ * contains a pointer to where the swash indicated by <swashname>
+ * is to be stored; which this routine will do, so that future calls will
+ * look at <*swash> and only generate a swash if it is not null
+ *
+ * Note that it is assumed that the buffer length of <p> is enough to
+ * contain all the bytes that comprise the character. Thus, <*p> should
+ * have been checked before this call for mal-formedness enough to assure
+ * that. */
+
dVAR;
PERL_ARGS_ASSERT_IS_UTF8_COMMON;
- if (!is_utf8_char(p))
+ /* The API should have included a length for the UTF-8 character in <p>,
+ * but it doesn't. We therefor assume that p has been validated at least
+ * as far as there being enough bytes available in it to accommodate the
+ * character without reading beyond the end, and pass that number on to the
+ * validating routine */
+ if (!is_utf8_char_buf(p, p + UTF8SKIP(p)))
return FALSE;
if (!*swash)
*swash = swash_init("utf8", swashname, &PL_sv_undef, 1, 0);
return is_utf8_common(p, &PL_utf8_X_LV_LVT_V, "_X_LV_LVT_V");
}
+bool
+Perl__is_utf8_quotemeta(pTHX_ const U8 *p)
+{
+ /* For exclusive use of pp_quotemeta() */
+
+ dVAR;
+
+ PERL_ARGS_ASSERT__IS_UTF8_QUOTEMETA;
+
+ return is_utf8_common(p, &PL_utf8_quotemeta, "_Perl_Quotemeta");
+}
+
/*
=for apidoc to_utf8_case
}
/* Note:
- * Returns a "swash" which is a hash described in utf8.c:S_swash_fetch().
+ * Returns a "swash" which is a hash described in utf8.c:Perl_swash_fetch().
* C<pkg> is a pointer to a package name for SWASHNEW, should be "utf8".
* For other parameters, see utf8::SWASHNEW in lib/utf8_heavy.pl.
*/
+
SV*
Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits, I32 none)
{
+ PERL_ARGS_ASSERT_SWASH_INIT;
+
+ /* Returns a copy of a swash initiated by the called function. This is the
+ * public interface, and returning a copy prevents others from doing
+ * mischief on the original */
+
+ return newSVsv(_core_swash_init(pkg, name, listsv, minbits, none, FALSE, NULL, FALSE));
+}
+
+SV*
+Perl__core_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits, I32 none, bool return_if_undef, SV* invlist, bool passed_in_invlist_has_user_defined_property)
+{
+ /* Initialize and return a swash, creating it if necessary. It does this
+ * by calling utf8_heavy.pl in the general case.
+ *
+ * This interface should only be used by functions that won't destroy or
+ * adversely change the swash, as doing so affects all other uses of the
+ * swash in the program; the general public should use 'Perl_swash_init'
+ * instead.
+ *
+ * pkg is the name of the package that <name> should be in.
+ * name is the name of the swash to find. Typically it is a Unicode
+ * property name, including user-defined ones
+ * listsv is a string to initialize the swash with. It must be of the form
+ * documented as the subroutine return value in
+ * L<perlunicode/User-Defined Character Properties>
+ * minbits is the number of bits required to represent each data element.
+ * It is '1' for binary properties.
+ * none I (khw) do not understand this one, but it is used only in tr///.
+ * return_if_undef is TRUE if the routine shouldn't croak if it can't find
+ * the requested property
+ * invlist is an inversion list to initialize the swash with (or NULL)
+ * has_user_defined_property is TRUE if <invlist> has some component that
+ * came from a user-defined property
+ *
+ * Thus there are three possible inputs to find the swash: <name>,
+ * <listsv>, and <invlist>. At least one must be specified. The result
+ * will be the union of the specified ones, although <listsv>'s various
+ * actions can intersect, etc. what <name> gives.
+ *
+ * <invlist> is only valid for binary properties */
+
dVAR;
- SV* retval;
- dSP;
- const size_t pkg_len = strlen(pkg);
- const size_t name_len = strlen(name);
- HV * const stash = gv_stashpvn(pkg, pkg_len, 0);
- SV* errsv_save;
- GV *method;
+ SV* retval = &PL_sv_undef;
- PERL_ARGS_ASSERT_SWASH_INIT;
+ assert(listsv != &PL_sv_undef || strNE(name, "") || invlist);
+ assert(! invlist || minbits == 1);
+
+ /* If data was passed in to go out to utf8_heavy to find the swash of, do
+ * so */
+ if (listsv != &PL_sv_undef || strNE(name, "")) {
+ dSP;
+ const size_t pkg_len = strlen(pkg);
+ const size_t name_len = strlen(name);
+ HV * const stash = gv_stashpvn(pkg, pkg_len, 0);
+ SV* errsv_save;
+ GV *method;
- PUSHSTACKi(PERLSI_MAGIC);
- ENTER;
- SAVEHINTS();
- save_re_context();
- if (PL_parser && PL_parser->error_count)
- SAVEI8(PL_parser->error_count), PL_parser->error_count = 0;
- method = gv_fetchmeth(stash, "SWASHNEW", 8, -1);
- if (!method) { /* demand load utf8 */
+ PERL_ARGS_ASSERT__CORE_SWASH_INIT;
+
+ PUSHSTACKi(PERLSI_MAGIC);
ENTER;
+ SAVEHINTS();
+ save_re_context();
+ if (PL_parser && PL_parser->error_count)
+ SAVEI8(PL_parser->error_count), PL_parser->error_count = 0;
+ method = gv_fetchmeth(stash, "SWASHNEW", 8, -1);
+ if (!method) { /* demand load utf8 */
+ ENTER;
+ errsv_save = newSVsv(ERRSV);
+ /* It is assumed that callers of this routine are not passing in
+ * any user derived data. */
+ /* Need to do this after save_re_context() as it will set
+ * PL_tainted to 1 while saving $1 etc (see the code after getrx:
+ * in Perl_magic_get). Even line to create errsv_save can turn on
+ * PL_tainted. */
+ SAVEBOOL(PL_tainted);
+ PL_tainted = 0;
+ Perl_load_module(aTHX_ PERL_LOADMOD_NOIMPORT, newSVpvn(pkg,pkg_len),
+ NULL);
+ if (!SvTRUE(ERRSV))
+ sv_setsv(ERRSV, errsv_save);
+ SvREFCNT_dec(errsv_save);
+ LEAVE;
+ }
+ SPAGAIN;
+ PUSHMARK(SP);
+ EXTEND(SP,5);
+ mPUSHp(pkg, pkg_len);
+ mPUSHp(name, name_len);
+ PUSHs(listsv);
+ mPUSHi(minbits);
+ mPUSHi(none);
+ PUTBACK;
errsv_save = newSVsv(ERRSV);
- /* It is assumed that callers of this routine are not passing in any
- user derived data. */
- /* Need to do this after save_re_context() as it will set PL_tainted to
- 1 while saving $1 etc (see the code after getrx: in Perl_magic_get).
- Even line to create errsv_save can turn on PL_tainted. */
- SAVEBOOL(PL_tainted);
- PL_tainted = 0;
- Perl_load_module(aTHX_ PERL_LOADMOD_NOIMPORT, newSVpvn(pkg,pkg_len),
- NULL);
+ /* If we already have a pointer to the method, no need to use
+ * call_method() to repeat the lookup. */
+ if (method ? call_sv(MUTABLE_SV(method), G_SCALAR)
+ : call_sv(newSVpvs_flags("SWASHNEW", SVs_TEMP), G_SCALAR | G_METHOD))
+ {
+ retval = *PL_stack_sp--;
+ SvREFCNT_inc(retval);
+ }
if (!SvTRUE(ERRSV))
sv_setsv(ERRSV, errsv_save);
SvREFCNT_dec(errsv_save);
LEAVE;
+ POPSTACK;
+ if (IN_PERL_COMPILETIME) {
+ CopHINTS_set(PL_curcop, PL_hints);
+ }
+ if (!SvROK(retval) || SvTYPE(SvRV(retval)) != SVt_PVHV) {
+ if (SvPOK(retval))
+
+ /* If caller wants to handle missing properties, let them */
+ if (return_if_undef) {
+ return NULL;
+ }
+ Perl_croak(aTHX_
+ "Can't find Unicode property definition \"%"SVf"\"",
+ SVfARG(retval));
+ Perl_croak(aTHX_ "SWASHNEW didn't return an HV ref");
+ }
+ } /* End of calling the module to find the swash */
+
+ /* Make sure there is an inversion list for binary properties */
+ if (minbits == 1) {
+ SV** swash_invlistsvp = NULL;
+ SV* swash_invlist = NULL;
+ bool invlist_in_swash_is_valid = FALSE;
+ HV* swash_hv = NULL;
+
+ /* If this operation fetched a swash, get its already existing
+ * inversion list or create one for it */
+ if (retval != &PL_sv_undef) {
+ swash_hv = MUTABLE_HV(SvRV(retval));
+
+ swash_invlistsvp = hv_fetchs(swash_hv, "INVLIST", FALSE);
+ if (swash_invlistsvp) {
+ swash_invlist = *swash_invlistsvp;
+ invlist_in_swash_is_valid = TRUE;
+ }
+ else {
+ swash_invlist = _swash_to_invlist(retval);
+ }
+ }
+
+ /* If an inversion list was passed in, have to include it */
+ if (invlist) {
+
+ /* Any fetched swash will by now have an inversion list in it;
+ * otherwise <swash_invlist> will be NULL, indicating that we
+ * didn't fetch a swash */
+ if (swash_invlist) {
+
+ /* Add the passed-in inversion list, which invalidates the one
+ * already stored in the swash */
+ invlist_in_swash_is_valid = FALSE;
+ _invlist_union(invlist, swash_invlist, &swash_invlist);
+ }
+ else {
+
+ /* Here, there is no swash already. Set up a minimal one */
+ swash_hv = newHV();
+ retval = newRV_inc(MUTABLE_SV(swash_hv));
+ swash_invlist = invlist;
+ }
+
+ if (passed_in_invlist_has_user_defined_property) {
+ if (! hv_stores(swash_hv, "USER_DEFINED", newSVuv(1))) {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+ }
+ }
+
+ /* Here, we have computed the union of all the passed-in data. It may
+ * be that there was an inversion list in the swash which didn't get
+ * touched; otherwise save the one computed one */
+ if (! invlist_in_swash_is_valid) {
+ if (! hv_stores(MUTABLE_HV(SvRV(retval)), "INVLIST", swash_invlist))
+ {
+ Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
+ }
+ }
}
- SPAGAIN;
- PUSHMARK(SP);
- EXTEND(SP,5);
- mPUSHp(pkg, pkg_len);
- mPUSHp(name, name_len);
- PUSHs(listsv);
- mPUSHi(minbits);
- mPUSHi(none);
- PUTBACK;
- errsv_save = newSVsv(ERRSV);
- /* If we already have a pointer to the method, no need to use call_method()
- to repeat the lookup. */
- if (method ? call_sv(MUTABLE_SV(method), G_SCALAR)
- : call_sv(newSVpvs_flags("SWASHNEW", SVs_TEMP), G_SCALAR | G_METHOD))
- retval = newSVsv(*PL_stack_sp--);
- else
- retval = &PL_sv_undef;
- if (!SvTRUE(ERRSV))
- sv_setsv(ERRSV, errsv_save);
- SvREFCNT_dec(errsv_save);
- LEAVE;
- POPSTACK;
- if (IN_PERL_COMPILETIME) {
- CopHINTS_set(PL_curcop, PL_hints);
- }
- if (!SvROK(retval) || SvTYPE(SvRV(retval)) != SVt_PVHV) {
- if (SvPOK(retval))
- Perl_croak(aTHX_ "Can't find Unicode property definition \"%"SVf"\"",
- SVfARG(retval));
- Perl_croak(aTHX_ "SWASHNEW didn't return an HV ref");
- }
+
return retval;
}
* the lower-level routine, and it is similarly broken for returning
* multiple values. --jhi
* For those, you should use to_utf8_case() instead */
-/* Now SWASHGET is recasted into S_swash_get in this file. */
+/* Now SWASHGET is recasted into S_swatch_get in this file. */
/* Note:
* Returns the value of property/mapping C<swash> for the first character
PERL_ARGS_ASSERT_SWASH_FETCH;
+ /* Convert to utf8 if not already */
if (!do_utf8 && !UNI_IS_INVARIANT(c)) {
tmputf8[0] = (U8)UTF8_EIGHT_BIT_HI(c);
tmputf8[1] = (U8)UTF8_EIGHT_BIT_LO(c);
/* Try our second-level swatch cache, kept in a hash. */
SV** svp = hv_fetch(hv, (const char*)ptr, klen, FALSE);
- /* If not cached, generate it via swash_get */
+ /* If not cached, generate it via swatch_get */
if (!svp || !SvPOK(*svp)
|| !(tmps = (const U8*)SvPV_const(*svp, slen))) {
/* We use utf8n_to_uvuni() as we want an index into
const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0,
ckWARN(WARN_UTF8) ?
0 : UTF8_ALLOW_ANY);
- swatch = swash_get(swash,
+ swatch = swatch_get(swash,
/* On EBCDIC & ~(0xA0-1) isn't a useful thing to do */
(klen) ? (code_point & ~((UV)needents - 1)) : 0,
needents);
if (!svp || !(tmps = (U8*)SvPV(*svp, slen))
|| (slen << 3) < needents)
- Perl_croak(aTHX_ "panic: swash_fetch got improper swatch");
+ Perl_croak(aTHX_ "panic: swash_fetch got improper swatch, "
+ "svp=%p, tmps=%p, slen=%"UVuf", needents=%"UVuf,
+ svp, tmps, (UV)slen, (UV)needents);
}
PL_last_swash_hv = hv;
* to_utf8_case() will output any for non-binary. Also, surrogates
* aren't checked for, as that would warn on things like /\p{Gc=Cs}/ */
- if (SvUV(*bitssvp) == 1) {
+ if (! bitssvp || SvUV(*bitssvp) == 1) {
/* User-defined properties can silently match above-Unicode */
SV** const user_defined_svp = hv_fetchs(hv, "USER_DEFINED", FALSE);
if (! user_defined_svp || ! SvUV(*user_defined_svp)) {
off <<= 2;
return (tmps[off] << 24) + (tmps[off+1] << 16) + (tmps[off+2] << 8) + tmps[off + 3] ;
}
- Perl_croak(aTHX_ "panic: swash_fetch got swatch of unexpected bit width");
+ Perl_croak(aTHX_ "panic: swash_fetch got swatch of unexpected bit width, "
+ "slen=%"UVuf", needents=%"UVuf, (UV)slen, (UV)needents);
NORETURN_FUNCTION_END;
}
if (wants_value) {
if (isBLANK(*l)) {
++l;
- flags = PERL_SCAN_SILENT_ILLDIGIT
- | PERL_SCAN_DISALLOW_PREFIX
- | PERL_SCAN_SILENT_NON_PORTABLE;
- numlen = lend - l;
- *val = grok_hex((char *)l, &numlen, &flags, NULL);
- if (numlen)
- l += numlen;
- else
- *val = 0;
+
+ /* The ToLc, etc table mappings are not in hex, and must be
+ * corrected by adding the code point to them */
+ if (typeto) {
+ char *after_strtol = (char *) lend;
+ *val = Strtol((char *)l, &after_strtol, 10);
+ l = (U8 *) after_strtol;
+ }
+ else { /* Other tables are in hex, and are the correct result
+ without tweaking */
+ flags = PERL_SCAN_SILENT_ILLDIGIT
+ | PERL_SCAN_DISALLOW_PREFIX
+ | PERL_SCAN_SILENT_NON_PORTABLE;
+ numlen = lend - l;
+ *val = grok_hex((char *)l, &numlen, &flags, NULL);
+ if (numlen)
+ l += numlen;
+ else
+ *val = 0;
+ }
}
else {
*val = 0;
if (typeto) {
+ /* diag_listed_as: To%s: illegal mapping '%s' */
Perl_croak(aTHX_ "%s: illegal mapping '%s'",
typestr, l);
}
if (wants_value) {
*val = 0;
if (typeto) {
+ /* diag_listed_as: To%s: illegal mapping '%s' */
Perl_croak(aTHX_ "%s: illegal mapping '%s'", typestr, l);
}
}
* Should be used via swash_fetch, which will cache the swatch in C<swash>.
*/
STATIC SV*
-S_swash_get(pTHX_ SV* swash, UV start, UV span)
+S_swatch_get(pTHX_ SV* swash, UV start, UV span)
{
SV *swatch;
U8 *l, *lend, *x, *xend, *s, *send;
STRLEN lcur, xcur, scur;
HV *const hv = MUTABLE_HV(SvRV(swash));
+ SV** const invlistsvp = hv_fetchs(hv, "INVLIST", FALSE);
+
+ SV** listsvp = NULL; /* The string containing the main body of the table */
+ SV** extssvp = NULL;
+ SV** invert_it_svp = NULL;
+ U8* typestr = NULL;
+ STRLEN bits;
+ STRLEN octets; /* if bits == 1, then octets == 0 */
+ UV none;
+ UV end = start + span;
+
+ if (invlistsvp == NULL) {
+ SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
+ SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE);
+ SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
+ extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
+ listsvp = hv_fetchs(hv, "LIST", FALSE);
+ invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
+
+ bits = SvUV(*bitssvp);
+ none = SvUV(*nonesvp);
+ typestr = (U8*)SvPV_nolen(*typesvp);
+ }
+ else {
+ bits = 1;
+ none = 0;
+ }
+ octets = bits >> 3; /* if bits == 1, then octets == 0 */
- /* The string containing the main body of the table */
- SV** const listsvp = hv_fetchs(hv, "LIST", FALSE);
-
- SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE);
- SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE);
- SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE);
- SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE);
- SV** const invert_it_svp = hv_fetchs(hv, "INVERT_IT", FALSE);
- const U8* const typestr = (U8*)SvPV_nolen(*typesvp);
- const STRLEN bits = SvUV(*bitssvp);
- const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */
- const UV none = SvUV(*nonesvp);
- UV end = start + span;
-
- PERL_ARGS_ASSERT_SWASH_GET;
+ PERL_ARGS_ASSERT_SWATCH_GET;
if (bits != 1 && bits != 8 && bits != 16 && bits != 32) {
- Perl_croak(aTHX_ "panic: swash_get doesn't expect bits %"UVuf,
+ Perl_croak(aTHX_ "panic: swatch_get doesn't expect bits %"UVuf,
(UV)bits);
}
SvCUR_set(swatch, scur);
s = (U8*)SvPVX(swatch);
- /* read $swash->{LIST}. XXX Note that this is a linear scan through a
- * sorted list. A binary search would be much more efficient */
+ if (invlistsvp) { /* If has an inversion list set up use that */
+ _invlist_populate_swatch(*invlistsvp, start, end, s);
+ return swatch;
+ }
+
+ /* read $swash->{LIST} */
l = (U8*)SvPV(*listsvp, lcur);
lend = l + lcur;
while (l < lend) {
- UV min, max, val;
+ UV min, max, val, upper;
l = S_swash_scan_list_line(aTHX_ l, lend, &min, &max, &val,
cBOOL(octets), typestr);
if (l > lend) {
if (max < start)
continue;
+ /* <end> is generally 1 beyond where we want to set things, but at the
+ * platform's infinity, where we can't go any higher, we want to
+ * include the code point at <end> */
+ upper = (max < end)
+ ? max
+ : (max != UV_MAX || end != UV_MAX)
+ ? end - 1
+ : end;
+
if (octets) {
UV key;
if (min < start) {
}
min = start;
}
- for (key = min; key <= max; key++) {
+ for (key = min; key <= upper; key++) {
STRLEN offset;
- if (key >= end)
- goto go_out_list;
- /* XXX If it should ever happen (very unlikely) that we would
- * want a non-binary result for the code point at UV_MAX,
- * special handling would need to be inserted here, as is done
- * below for the binary case */
/* offset must be non-negative (start <= min <= key < end) */
offset = octets * (key - start);
if (bits == 8)
if (min < start)
min = start;
- /* Special case when the upper-end is the highest possible code
- * point representable on the platform. Otherwise, the code below
- * exits before setting this bit. Done here to avoid testing for
- * this extremely unlikely possibility in the loop */
- if (UNLIKELY(end == UV_MAX && max == UV_MAX)) {
- const STRLEN offset = (STRLEN)(max - start);
- s[offset >> 3] |= 1 << (offset & 7);
- }
- for (key = min; key <= max; key++) {
+ for (key = min; key <= upper; key++) {
const STRLEN offset = (STRLEN)(key - start);
- if (key >= end)
- goto go_out_list;
s[offset >> 3] |= 1 << (offset & 7);
}
}
} /* while */
- go_out_list:
/* Invert if the data says it should be. Assumes that bits == 1 */
if (invert_it_svp && SvUV(*invert_it_svp)) {
otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE);
otherbits = (STRLEN)SvUV(*otherbitssvp);
if (bits < otherbits)
- Perl_croak(aTHX_ "panic: swash_get found swatch size mismatch");
+ Perl_croak(aTHX_ "panic: swatch_get found swatch size mismatch, "
+ "bits=%"UVuf", otherbits=%"UVuf, (UV)bits, (UV)otherbits);
/* The "other" swatch must be destroyed after. */
- other = swash_get(*othersvp, start, span);
+ other = swatch_get(*othersvp, start, span);
o = (U8*)SvPV(other, olen);
if (!olen)
- Perl_croak(aTHX_ "panic: swash_get got improper swatch");
+ Perl_croak(aTHX_ "panic: swatch_get got improper swatch");
s = (U8*)SvPV(swatch, slen);
if (bits == 1 && otherbits == 1) {
if (slen != olen)
- Perl_croak(aTHX_ "panic: swash_get found swatch length mismatch");
+ Perl_croak(aTHX_ "panic: swatch_get found swatch length "
+ "mismatch, slen=%"UVuf", olen=%"UVuf,
+ (UV)slen, (UV)olen);
switch (opc) {
case '+':
while ((sv_to = hv_iternextsv(specials_hv, &char_from, &from_len))) {
SV** listp;
if (! SvPOK(sv_to)) {
- Perl_croak(aTHX_ "panic: value returned from hv_iternextsv() unexpectedly is not a string");
+ Perl_croak(aTHX_ "panic: value returned from hv_iternextsv() "
+ "unexpectedly is not a string, flags=%lu",
+ (unsigned long)SvFLAGS(sv_to));
}
/*DEBUG_U(PerlIO_printf(Perl_debug_log, "Found mapping from %"UVXf", First char of to is %"UVXf"\n", utf8_to_uvchr((U8*) char_from, 0), utf8_to_uvchr((U8*) SvPVX(sv_to), 0)));*/
/*DEBUG_U(PerlIO_printf(Perl_debug_log, "Adding %"UVXf" to list for %"UVXf"\n", inverse, val));*/
}
- /* swash_get() increments the value of val for each element in the
+ /* swatch_get() increments the value of val for each element in the
* range. That makes more compact tables possible. You can
* express the capitalization, for example, of all consecutive
* letters with a single line: 0061\t007A\t0041 This maps 0061 to
* 0041, 0062 to 0042, etc. I (khw) have never understood 'none',
* and it's not documented; it appears to be used only in
- * implementing tr//; I copied the semantics from swash_get(), just
+ * implementing tr//; I copied the semantics from swatch_get(), just
* in case */
if (!none || val < none) {
++val;
/* If the ending is somehow corrupt and isn't a new line, add another
* element for the final range that isn't in the inversion list */
- if (! (*lend == '\n' || (*lend == '\0' && *(lend - 1) == '\n'))) {
+ if (! (*lend == '\n'
+ || (*lend == '\0' && (lcur == 0 || *(lend - 1) == '\n'))))
+ {
elements++;
}
_invlist_invert_prop(invlist);
}
- /* This code is copied from swash_get()
+ /* This code is copied from swatch_get()
* read $swash->{EXTRAS} */
x = (U8*)SvPV(*extssvp, xcur);
xend = x + xcur;
otherbits = (STRLEN)SvUV(*otherbitssvp);
if (bits != otherbits || bits != 1) {
- Perl_croak(aTHX_ "panic: _swash_to_invlist only operates on boolean properties");
+ Perl_croak(aTHX_ "panic: _swash_to_invlist only operates on boolean "
+ "properties, bits=%"UVuf", otherbits=%"UVuf,
+ (UV)bits, (UV)otherbits);
}
/* The "other" swatch must be destroyed after. */
other = _swash_to_invlist((SV *)*othersvp);
- /* End of code copied from swash_get() */
+ /* End of code copied from swatch_get() */
switch (opc) {
case '+':
_invlist_union(invlist, other, &invlist);
=for apidoc uvchr_to_utf8
Adds the UTF-8 representation of the Native code point C<uv> to the end
-of the string C<d>; C<d> should be have at least C<UTF8_MAXBYTES+1> free
+of the string C<d>; C<d> should have at least C<UTF8_MAXBYTES+1> free
bytes available. The return value is the pointer to the byte after the
end of the new character. In other words,
if (flags & FOLDEQ_S1_ALREADY_FOLDED) {
f1 = (U8 *) p1;
n1 = UTF8SKIP(f1);
-
- /* If in locale matching, we use two sets of rules, depending on if
- * the code point is above or below 255. Here, we test for and
- * handle locale rules */
}
+
else {
+ /* If in locale matching, we use two sets of rules, depending
+ * on if the code point is above or below 255. Here, we test
+ * for and handle locale rules */
if ((flags & FOLDEQ_UTF8_LOCALE)
&& (! u1 || UTF8_IS_INVARIANT(*p1)
|| UTF8_IS_DOWNGRADEABLE_START(*p1)))
}
n1 = 1;
}
- else if (isASCII(*p1)) { /* Note, that here won't be
- both ASCII and using locale
- rules */
+ else if (isASCII(*p1)) { /* Note, that here won't be both
+ ASCII and using locale rules */
/* If trying to mix non- with ASCII, and not supposed to,
* fail */
n1 = n2 = 0;
}
else if (isASCII(*p2)) {
- if (flags && ! isASCII(*p1)) {
+ if ((flags & FOLDEQ_UTF8_NOMIX_ASCII) && ! isASCII(*p1)) {
return 0;
}
n2 = 1;
if (fold_length != UTF8SKIP(f2)
|| (fold_length == 1 && *f1 != *f2) /* Short circuit memNE
function call for single
- character */
+ byte */
|| memNE((char*)f1, (char*)f2, fold_length))
{
return 0; /* mismatch */
https://perl5.git.perl.org / perl5.git / blobdiff