*/
UV
-Perl_utf8n_to_uvchr(pTHX_ const U8 *s,
- STRLEN curlen,
- STRLEN *retlen,
- const U32 flags)
+Perl_utf8n_to_uvchr(const U8 *s,
+ STRLEN curlen,
+ STRLEN *retlen,
+ const U32 flags)
{
PERL_ARGS_ASSERT_UTF8N_TO_UVCHR;
*/
UV
-Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
+Perl_utf8n_to_uvchr_error(const U8 *s,
STRLEN curlen,
STRLEN *retlen,
const U32 flags,
*/
UV
-Perl_utf8n_to_uvchr_msgs(pTHX_ const U8 *s,
+Perl__utf8n_to_uvchr_msgs_helper(const U8 *s,
STRLEN curlen,
STRLEN *retlen,
const U32 flags,
{
const U8 * const s0 = s;
const U8 * send = s0 + curlen;
- U32 possible_problems = 0; /* A bit is set here for each potential problem
- found as we go along */
- UV uv = (UV) -1;
- STRLEN expectlen = 0; /* How long should this sequence be?
- (initialized to silence compilers' wrong
- warning) */
- STRLEN avail_len = 0; /* When input is too short, gives what that is */
- U32 discard_errors = 0; /* Used to save branches when 'errors' is NULL;
- this gets set and discarded */
+ U32 possible_problems; /* A bit is set here for each potential problem
+ found as we go along */
+ UV uv;
+ STRLEN expectlen; /* How long should this sequence be? */
+ STRLEN avail_len; /* When input is too short, gives what that is */
+ U32 discard_errors; /* Used to save branches when 'errors' is NULL; this
+ gets set and discarded */
/* The below are used only if there is both an overlong malformation and a
* too short one. Otherwise the first two are set to 's0' and 'send', and
* the third not used at all */
- U8 * adjusted_s0 = (U8 *) s0;
+ U8 * adjusted_s0;
U8 temp_char_buf[UTF8_MAXBYTES + 1]; /* Used to avoid a Newx in this
routine; see [perl #130921] */
- UV uv_so_far = 0; /* (Initialized to silence compilers' wrong warning) */
+ UV uv_so_far;
+ dTHX;
+
+ PERL_ARGS_ASSERT__UTF8N_TO_UVCHR_MSGS_HELPER;
+
+ /* Here, is one of: a) malformed; b) a problematic code point (surrogate,
+ * non-unicode, or nonchar); or c) on ASCII platforms, one of the Hangul
+ * syllables that the dfa doesn't properly handle. Quickly dispose of the
+ * final case. */
+
+#ifndef EBCDIC
+
+ /* Each of the affected Hanguls starts with \xED */
+
+ if (is_HANGUL_ED_utf8_safe(s0, send)) {
+ if (retlen) {
+ *retlen = 3;
+ }
+ if (errors) {
+ *errors = 0;
+ }
+ if (msgs) {
+ *msgs = NULL;
+ }
+
+ return ((0xED & UTF_START_MASK(3)) << (2 * UTF_ACCUMULATION_SHIFT))
+ | ((s0[1] & UTF_CONTINUATION_MASK) << UTF_ACCUMULATION_SHIFT)
+ | (s0[2] & UTF_CONTINUATION_MASK);
+ }
- UV state = 0;
+#endif
- PERL_ARGS_ASSERT_UTF8N_TO_UVCHR_MSGS;
+ /* In conjunction with the exhaustive tests that can be enabled in
+ * APItest/t/utf8_warn_base.pl, this can make sure the dfa does precisely
+ * what it is intended to do, and that no flaws in it are masked by
+ * dropping down and executing the code below
+ assert(! isUTF8_CHAR(s0, send)
+ || UTF8_IS_SURROGATE(s0, send)
+ || UTF8_IS_SUPER(s0, send)
+ || UTF8_IS_NONCHAR(s0,send));
+ */
+
+ s = s0;
+ uv = *s0;
+ possible_problems = 0;
+ expectlen = 0;
+ avail_len = 0;
+ discard_errors = 0;
+ adjusted_s0 = (U8 *) s0;
+ uv_so_far = 0;
if (errors) {
*errors = 0;
*retlen = expectlen;
}
- /* An invariant is trivially well-formed */
- if (UTF8_IS_INVARIANT(*s0)) {
- return *s0;
- }
-
- /* Measurements show that this dfa is somewhat faster than the regular code
- * below, so use it first, dropping down for the non-normal cases. */
-
-#define PERL_UTF8_DECODE_REJECT 1
-
- while (s < send && LIKELY(state != PERL_UTF8_DECODE_REJECT)) {
- UV type = perl_extended_utf8_dfa_tab[*s];
-
- uv = (state == 0)
- ? ((0xff >> type) & NATIVE_UTF8_TO_I8(*s))
- : UTF8_ACCUMULATE(uv, *s);
- state = perl_extended_utf8_dfa_tab[256 + state + type];
-
- if (state == 0) {
-
- /* If this could be a code point that the flags don't allow (the first
- * surrogate is the first such possible one), delve further, but we already
- * have calculated 'uv' */
- if ( (flags & (UTF8_DISALLOW_ILLEGAL_INTERCHANGE
- |UTF8_DISALLOW_PERL_EXTENDED
- |UTF8_WARN_ILLEGAL_INTERCHANGE
- |UTF8_WARN_PERL_EXTENDED))
- && uv >= UNICODE_SURROGATE_FIRST)
- {
- curlen = s + 1 - s0;
- goto got_uv;
- }
-
- return UNI_TO_NATIVE(uv);
- }
-
- s++;
- }
-
- /* Here, is some sort of failure. Use the full mechanism */
-
- uv = *s0;
-
/* A continuation character can't start a valid sequence */
if (UNLIKELY(UTF8_IS_CONTINUATION(uv))) {
possible_problems |= UTF8_GOT_CONTINUATION;
}
}
- got_uv:
-
/* Here, we have found all the possible problems, except for when the input
* is for a problematic code point not allowed by the input parameters. */
return _to_upper_title_latin1((U8) c, p, lenp, 'S');
}
- uvchr_to_utf8(p, c);
- return CALL_UPPER_CASE(c, p, p, lenp);
+ return CALL_UPPER_CASE(c, NULL, p, lenp);
}
UV
return _to_upper_title_latin1((U8) c, p, lenp, 's');
}
- uvchr_to_utf8(p, c);
- return CALL_TITLE_CASE(c, p, p, lenp);
+ return CALL_TITLE_CASE(c, NULL, p, lenp);
}
STATIC U8
return to_lower_latin1((U8) c, p, lenp, 0 /* 0 is a dummy arg */ );
}
- uvchr_to_utf8(p, c);
- return CALL_LOWER_CASE(c, p, p, lenp);
+ return CALL_LOWER_CASE(c, NULL, p, lenp);
}
UV
/* Here, above 255. If no special needs, just use the macro */
if ( ! (flags & (FOLD_FLAGS_LOCALE|FOLD_FLAGS_NOMIX_ASCII))) {
- uvchr_to_utf8(p, c);
- return CALL_FOLD_CASE(c, p, p, lenp, flags & FOLD_FLAGS_FULL);
+ return CALL_FOLD_CASE(c, NULL, p, lenp, flags & FOLD_FLAGS_FULL);
}
else { /* Otherwise, _toFOLD_utf8_flags has the intelligence to deal with
the special flags. */
{
PERL_ARGS_ASSERT__IS_UTF8_MARK;
- return is_utf8_common(p, &PL_utf8_mark, "IsM", NULL);
+ return is_utf8_common(p, NULL, "IsM", PL_utf8_mark);
}
STATIC UV
/* Here, there was no mapping defined, which means that the code point maps
* to itself. Return the inputs */
cases_to_self:
- len = UTF8SKIP(p);
- if (p != ustrp) { /* Don't copy onto itself */
- Copy(p, ustrp, len, U8);
+ if (p) {
+ len = UTF8SKIP(p);
+ if (p != ustrp) { /* Don't copy onto itself */
+ Copy(p, ustrp, len, U8);
+ }
+ *lenp = len;
+ }
+ else {
+ *lenp = uvchr_to_utf8(ustrp, uv1) - ustrp;
}
-
- if (lenp)
- *lenp = len;
return uv1;
PL_utf8_tofold = _new_invlist_C_array(Case_Folding_invlist);
PL_utf8_tosimplefold = _new_invlist_C_array(Simple_Case_Folding_invlist);
PL_utf8_foldclosures = _new_invlist_C_array(_Perl_IVCF_invlist);
+ PL_utf8_mark = _new_invlist_C_array(PL_uni_prop_ptrs[PL_M]);
}
SV *
https://perl5.git.perl.org / perl5.git / blobdiff