/*
=head1 Unicode Support
-
-This file contains various utility functions for manipulating UTF8-encoded
+These are various utility functions for manipulating UTF8-encoded
strings. For the uninitiated, this is a method of representing arbitrary
Unicode characters as a variable number of bytes, in such a way that
characters in the ASCII range are unmodified, and a zero byte never appears
/*
=for apidoc is_utf8_char_buf
-This is identical to the macro isUTF8_CHAR.
+This is identical to the macro L</isUTF8_CHAR>.
=cut */
}
/*
-=for apidoc is_utf8_char
-
-Tests if some arbitrary number of bytes begins in a valid UTF-8
-character. Note that an INVARIANT (i.e. ASCII on non-EBCDIC machines)
-character is a valid UTF-8 character. The actual number of bytes in the UTF-8
-character will be returned if it is valid, otherwise 0.
-
-This function is deprecated due to the possibility that malformed input could
-cause reading beyond the end of the input buffer. Use L</is_utf8_char_buf>
-instead.
-
-=cut */
-
-STRLEN
-Perl_is_utf8_char(const U8 *s)
-{
- PERL_ARGS_ASSERT_IS_UTF8_CHAR;
-
- /* Assumes we have enough space, which is why this is deprecated */
- return isUTF8_CHAR(s, s + UTF8SKIP(s));
-}
-
-
-/*
=for apidoc is_utf8_string
Returns true if the first C<len> bytes of string C<s> form a valid
UV
Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags)
{
- dVAR;
const U8 * const s0 = s;
U8 overflow_byte = '\0'; /* Save byte in case of overflow */
U8 * send;
UV uv = *s;
PERL_ARGS_ASSERT_VALID_UTF8_TO_UVCHR;
+ PERL_UNUSED_CONTEXT;
if (retlen) {
*retlen = expectlen;
}
/*
-=for apidoc utf8_to_uvchr
-
-Returns the native code point of the first character in the string C<s>
-which is assumed to be in UTF-8 encoding; C<retlen> will be set to the
-length, in bytes, of that character.
-
-Some, but not all, UTF-8 malformations are detected, and in fact, some
-malformed input could cause reading beyond the end of the input buffer, which
-is why this function is deprecated. Use L</utf8_to_uvchr_buf> instead.
-
-If C<s> points to one of the detected malformations, and UTF8 warnings are
-enabled, zero is returned and C<*retlen> is set (if C<retlen> isn't
-NULL) to -1. If those warnings are off, the computed value if well-defined (or
-the Unicode REPLACEMENT CHARACTER, if not) is silently returned, and C<*retlen>
-is set (if C<retlen> isn't NULL) so that (S<C<s> + C<*retlen>>) is the
-next possible position in C<s> that could begin a non-malformed character.
-See L</utf8n_to_uvchr> for details on when the REPLACEMENT CHARACTER is returned.
-
-=cut
-*/
-
-UV
-Perl_utf8_to_uvchr(pTHX_ const U8 *s, STRLEN *retlen)
-{
- PERL_ARGS_ASSERT_UTF8_TO_UVCHR;
-
- return utf8_to_uvchr_buf(s, s + UTF8_MAXBYTES, retlen);
-}
-
-/*
=for apidoc utf8_to_uvuni_buf
Only in very rare circumstances should code need to be dealing in Unicode
ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY));
}
-/* DEPRECATED!
- * Like L</utf8_to_uvuni_buf>(), but should only be called when it is known that
- * there are no malformations in the input UTF-8 string C<s>. Surrogates,
- * non-character code points, and non-Unicode code points are allowed */
-
-UV
-Perl_valid_utf8_to_uvuni(pTHX_ const U8 *s, STRLEN *retlen)
-{
- PERL_ARGS_ASSERT_VALID_UTF8_TO_UVUNI;
-
- return NATIVE_TO_UNI(valid_utf8_to_uvchr(s, retlen));
-}
-
-/*
-=for apidoc utf8_to_uvuni
-
-Returns the Unicode code point of the first character in the string C<s>
-which is assumed to be in UTF-8 encoding; C<retlen> will be set to the
-length, in bytes, of that character.
-
-Some, but not all, UTF-8 malformations are detected, and in fact, some
-malformed input could cause reading beyond the end of the input buffer, which
-is one reason why this function is deprecated. The other is that only in
-extremely limited circumstances should the Unicode versus native code point be
-of any interest to you. See L</utf8_to_uvuni_buf> for alternatives.
-
-If C<s> points to one of the detected malformations, and UTF8 warnings are
-enabled, zero is returned and C<*retlen> is set (if C<retlen> doesn't point to
-NULL) to -1. If those warnings are off, the computed value if well-defined (or
-the Unicode REPLACEMENT CHARACTER, if not) is silently returned, and C<*retlen>
-is set (if C<retlen> isn't NULL) so that (S<C<s> + C<*retlen>>) is the
-next possible position in C<s> that could begin a non-malformed character.
-See L</utf8n_to_uvchr> for details on when the REPLACEMENT CHARACTER is returned.
-
-=cut
-*/
-
-UV
-Perl_utf8_to_uvuni(pTHX_ const U8 *s, STRLEN *retlen)
-{
- PERL_ARGS_ASSERT_UTF8_TO_UVUNI;
-
- return NATIVE_TO_UNI(valid_utf8_to_uvchr(s, retlen));
-}
-
/*
=for apidoc utf8_length
STRLEN
Perl_utf8_length(pTHX_ const U8 *s, const U8 *e)
{
- dVAR;
STRLEN len = 0;
PERL_ARGS_ASSERT_UTF8_LENGTH;
*/
U8 *
-Perl_utf8_hop(pTHX_ const U8 *s, I32 off)
+Perl_utf8_hop(const U8 *s, I32 off)
{
PERL_ARGS_ASSERT_UTF8_HOP;
- PERL_UNUSED_CONTEXT;
/* Note: cannot use UTF8_IS_...() too eagerly here since e.g
* the bitops (especially ~) can create illegal UTF-8.
* In other words: in Perl UTF-8 is not just for Unicode. */
PERL_ARGS_ASSERT_BYTES_CMP_UTF8;
- PERL_UNUSED_CONTEXT;
-
while (b < bend && u < uend) {
U8 c = *u++;
if (!UTF8_IS_INVARIANT(c)) {
U8 *d;
PERL_ARGS_ASSERT_UTF8_TO_BYTES;
+ PERL_UNUSED_CONTEXT;
/* ensure valid UTF-8 and chars < 256 before updating string */
while (s < send) {
I32 count = 0;
PERL_ARGS_ASSERT_BYTES_FROM_UTF8;
-
PERL_UNUSED_CONTEXT;
if (!*is_utf8)
return (U8 *)start;
/* Internal function so we can deprecate the external one, and call
this one from other deprecated functions in this file */
-PERL_STATIC_INLINE bool
-S_is_utf8_idfirst(pTHX_ const U8 *p)
+bool
+Perl__is_utf8_idstart(pTHX_ const U8 *p)
{
- dVAR;
+ PERL_ARGS_ASSERT__IS_UTF8_IDSTART;
if (*p == '_')
return TRUE;
- /* is_utf8_idstart would be more logical. */
return is_utf8_common(p, &PL_utf8_idstart, "IdStart", NULL);
}
bool
-Perl_is_uni_idfirst(pTHX_ UV c)
-{
- U8 tmpbuf[UTF8_MAXBYTES+1];
- uvchr_to_utf8(tmpbuf, c);
- return S_is_utf8_idfirst(aTHX_ tmpbuf);
-}
-
-bool
Perl__is_uni_perl_idcont(pTHX_ UV c)
{
U8 tmpbuf[UTF8_MAXBYTES+1];
UV
Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp)
{
- dVAR;
-
/* Convert the Unicode character whose ordinal is <c> to its uppercase
* version and store that in UTF-8 in <p> and its length in bytes in <lenp>.
* Note that the <p> needs to be at least UTF8_MAXBYTES_CASE+1 bytes since
UV
Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp)
{
- dVAR;
-
PERL_ARGS_ASSERT_TO_UNI_TITLE;
if (c < 256) {
}
STATIC U8
-S_to_lower_latin1(pTHX_ const U8 c, U8* p, STRLEN *lenp)
+S_to_lower_latin1(const U8 c, U8* p, STRLEN *lenp)
{
/* We have the latin1-range values compiled into the core, so just use
* those, converting the result to utf8. Since the result is always just
*lenp = 1;
}
else {
- *p = UTF8_TWO_BYTE_HI(converted);
- *(p+1) = UTF8_TWO_BYTE_LO(converted);
+ /* Result is known to always be < 256, so can use the EIGHT_BIT
+ * macros */
+ *p = UTF8_EIGHT_BIT_HI(converted);
+ *(p+1) = UTF8_EIGHT_BIT_LO(converted);
*lenp = 2;
}
}
UV
Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp)
{
- dVAR;
-
PERL_ARGS_ASSERT_TO_UNI_LOWER;
if (c < 256) {
UV converted;
PERL_ARGS_ASSERT__TO_FOLD_LATIN1;
+ PERL_UNUSED_CONTEXT;
assert (! (flags & FOLD_FLAGS_LOCALE));
* have been checked before this call for mal-formedness enough to assure
* that. */
- dVAR;
-
PERL_ARGS_ASSERT_IS_UTF8_COMMON;
/* The API should have included a length for the UTF-8 character in <p>,
bool
Perl__is_utf8_FOO(pTHX_ const U8 classnum, const U8 *p)
{
- dVAR;
-
PERL_ARGS_ASSERT__IS_UTF8_FOO;
assert(classnum < _FIRST_NON_SWASH_CC);
}
bool
-Perl_is_utf8_idfirst(pTHX_ const U8 *p) /* The naming is historical. */
+Perl__is_utf8_perl_idstart(pTHX_ const U8 *p)
{
- dVAR;
+ SV* invlist = NULL;
- PERL_ARGS_ASSERT_IS_UTF8_IDFIRST;
+ PERL_ARGS_ASSERT__IS_UTF8_PERL_IDSTART;
- return S_is_utf8_idfirst(aTHX_ p);
+ if (! PL_utf8_perl_idstart) {
+ invlist = _new_invlist_C_array(_Perl_IDStart_invlist);
+ }
+ return is_utf8_common(p, &PL_utf8_perl_idstart, "", invlist);
}
bool
-Perl_is_utf8_xidfirst(pTHX_ const U8 *p) /* The naming is historical. */
+Perl__is_utf8_xidstart(pTHX_ const U8 *p)
{
- dVAR;
-
- PERL_ARGS_ASSERT_IS_UTF8_XIDFIRST;
+ PERL_ARGS_ASSERT__IS_UTF8_XIDSTART;
if (*p == '_')
return TRUE;
- /* is_utf8_idstart would be more logical. */
return is_utf8_common(p, &PL_utf8_xidstart, "XIdStart", NULL);
}
bool
-Perl__is_utf8_perl_idstart(pTHX_ const U8 *p)
-{
- dVAR;
- SV* invlist = NULL;
-
- PERL_ARGS_ASSERT__IS_UTF8_PERL_IDSTART;
-
- if (! PL_utf8_perl_idstart) {
- invlist = _new_invlist_C_array(_Perl_IDStart_invlist);
- }
- return is_utf8_common(p, &PL_utf8_perl_idstart, "", invlist);
-}
-
-bool
Perl__is_utf8_perl_idcont(pTHX_ const U8 *p)
{
- dVAR;
SV* invlist = NULL;
PERL_ARGS_ASSERT__IS_UTF8_PERL_IDCONT;
return is_utf8_common(p, &PL_utf8_perl_idcont, "", invlist);
}
-
bool
-Perl_is_utf8_idcont(pTHX_ const U8 *p)
+Perl__is_utf8_idcont(pTHX_ const U8 *p)
{
- dVAR;
-
- PERL_ARGS_ASSERT_IS_UTF8_IDCONT;
+ PERL_ARGS_ASSERT__IS_UTF8_IDCONT;
return is_utf8_common(p, &PL_utf8_idcont, "IdContinue", NULL);
}
bool
-Perl_is_utf8_xidcont(pTHX_ const U8 *p)
+Perl__is_utf8_xidcont(pTHX_ const U8 *p)
{
- dVAR;
-
- PERL_ARGS_ASSERT_IS_UTF8_XIDCONT;
+ PERL_ARGS_ASSERT__IS_UTF8_XIDCONT;
return is_utf8_common(p, &PL_utf8_idcont, "XIdContinue", NULL);
}
bool
Perl__is_utf8_mark(pTHX_ const U8 *p)
{
- dVAR;
-
PERL_ARGS_ASSERT__IS_UTF8_MARK;
return is_utf8_common(p, &PL_utf8_mark, "IsM", NULL);
Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp,
SV **swashp, const char *normal, const char *special)
{
- dVAR;
STRLEN len = 0;
const UV uv1 = valid_utf8_to_uvchr(p, NULL);
UV
Perl__to_utf8_upper_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags)
{
- dVAR;
-
UV result;
PERL_ARGS_ASSERT__TO_UTF8_UPPER_FLAGS;
UV
Perl__to_utf8_title_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags)
{
- dVAR;
-
UV result;
PERL_ARGS_ASSERT__TO_UTF8_TITLE_FLAGS;
{
UV result;
- dVAR;
-
PERL_ARGS_ASSERT__TO_UTF8_LOWER_FLAGS;
if (flags && IN_UTF8_CTYPE_LOCALE) {
UV
Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags)
{
- dVAR;
-
UV result;
PERL_ARGS_ASSERT__TO_UTF8_FOLD_FLAGS;
*
* <invlist> is only valid for binary properties */
- dVAR;
SV* retval = &PL_sv_undef;
HV* swash_hv = NULL;
const int invlist_swash_boundary =
Perl_croak(aTHX_
"Can't find Unicode property definition \"%"SVf"\"",
SVfARG(retval));
- Perl_croak(aTHX_ "SWASHNEW didn't return an HV ref");
+ NOT_REACHED; /* NOTREACHED */
}
} /* End of calling the module to find the swash */
UV
Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8)
{
- dVAR;
HV *const hv = MUTABLE_HV(SvRV(swash));
U32 klen;
U32 off;
for (i = 0; i <= av_tindex(list); i++) {
SV** entryp = av_fetch(list, i, FALSE);
SV* entry;
+ UV uv;
if (entryp == NULL) {
Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
}
entry = *entryp;
- /*DEBUG_U(PerlIO_printf(Perl_debug_log, "list for %"UVXf" contains %"UVXf"\n", val, SvUV(entry)));*/
- if (SvUV(entry) == val) {
+ uv = SvUV(entry);
+ /*DEBUG_U(PerlIO_printf(Perl_debug_log, "list for %"UVXf" contains %"UVXf"\n", val, uv));*/
+ if (uv == val) {
found_key = TRUE;
}
- if (SvUV(entry) == inverse) {
+ if (uv == inverse) {
found_inverse = TRUE;
}
lend = l + lcur;
if (*l == 'V') { /* Inversion list format */
- char *after_strtol = (char *) lend;
+ const char *after_atou = (char *) lend;
UV element0;
UV* other_elements_ptr;
/* The first number is a count of the rest */
l++;
- elements = Strtoul((char *)l, &after_strtol, 10);
+ elements = grok_atou((const char *)l, &after_atou);
if (elements == 0) {
invlist = _new_invlist(0);
}
else {
- l = (U8 *) after_strtol;
+ while (isSPACE(*l)) l++;
+ l = (U8 *) after_atou;
/* Get the 0th element, which is needed to setup the inversion list */
- element0 = (UV) Strtoul((char *)l, &after_strtol, 10);
- l = (U8 *) after_strtol;
+ while (isSPACE(*l)) l++;
+ element0 = (UV) grok_atou((const char *)l, &after_atou);
+ l = (U8 *) after_atou;
invlist = _setup_canned_invlist(elements, element0, &other_elements_ptr);
elements--;
if (l > lend) {
Perl_croak(aTHX_ "panic: Expecting %"UVuf" more elements than available", elements);
}
- *other_elements_ptr++ = (UV) Strtoul((char *)l, &after_strtol, 10);
- l = (U8 *) after_strtol;
+ while (isSPACE(*l)) l++;
+ *other_elements_ptr++ = (UV) grok_atou((const char *)l, &after_atou);
+ l = (U8 *) after_atou;
}
}
}
I32
Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const char *s2, char **pe2, UV l2, bool u2, U32 flags)
{
- dVAR;
const U8 *p1 = (const U8*)s1; /* Point to current char */
const U8 *p2 = (const U8*)s2;
const U8 *g1 = NULL; /* goal for s1 */
return 1;
}
-/* XXX The next four functions should likely be moved to mathoms.c once all
+/* XXX The next two functions should likely be moved to mathoms.c once all
* occurrences of them are removed from the core; some cpan-upstream modules
* still use them */