This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
utf8.c: A byte count should be Size_t, not I32
[perl5.git] / utf8.c
diff --git a/utf8.c b/utf8.c
index 5fca6f7..3ded4a6 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -37,7 +37,8 @@ static const char malformed_text[] = "Malformed UTF-8 character";
 static const char unees[] =
                         "Malformed UTF-8 character (unexpected end of string)";
 static const char cp_above_legal_max[] =
- "Use of code point 0x%" UVXf " is deprecated; the permissible max is 0x%" UVXf;
+      "Use of code point 0x%" UVXf " is not allowed; "
+      "the permissible max is 0x%" UVXf;
 
 #define MAX_NON_DEPRECATED_CP ((UV) (IV_MAX))
 
@@ -78,7 +79,7 @@ Perl__force_out_malformed_utf8_message(pTHX_
     PERL_ARGS_ASSERT__FORCE_OUT_MALFORMED_UTF8_MESSAGE;
 
     ENTER;
-    SAVESPTR(PL_dowarn);
+    SAVEI8(PL_dowarn);
     SAVESPTR(PL_curcop);
 
     PL_dowarn = G_WARN_ALL_ON|G_WARN_ON;
@@ -146,7 +147,7 @@ For details, see the description for L</uvchr_to_utf8_flags>.
 #define MASK    UTF_CONTINUATION_MASK
 
 U8 *
-Perl_uvoffuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags)
+Perl_uvoffuni_to_utf8_flags(pTHX_ U8 *d, UV uv, const UV flags)
 {
     PERL_ARGS_ASSERT_UVOFFUNI_TO_UTF8_FLAGS;
 
@@ -198,11 +199,8 @@ Perl_uvoffuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags)
      * performance hit on these high EBCDIC code points. */
 
     if (UNLIKELY(UNICODE_IS_SUPER(uv))) {
-        if (   UNLIKELY(uv > MAX_NON_DEPRECATED_CP)
-            && ckWARN_d(WARN_DEPRECATED))
-        {
-            Perl_warner(aTHX_ packWARN(WARN_DEPRECATED),
-                        cp_above_legal_max, uv, MAX_NON_DEPRECATED_CP);
+        if (UNLIKELY(uv > MAX_NON_DEPRECATED_CP)) {
+            Perl_croak(aTHX_ cp_above_legal_max, uv, MAX_NON_DEPRECATED_CP);
         }
         if (   (flags & UNICODE_WARN_SUPER)
             || (   UNICODE_IS_ABOVE_31_BIT(uv)
@@ -754,11 +752,15 @@ Perl__is_utf8_char_helper(const U8 * const s, const U8 * e, const U32 flags)
     return UTF8SKIP(s);
 }
 
-STATIC char *
-S__byte_dump_string(pTHX_ const U8 * s, const STRLEN len)
+char *
+Perl__byte_dump_string(pTHX_ const U8 * s, const STRLEN len, const bool format)
 {
     /* Returns a mortalized C string that is a displayable copy of the 'len'
-     * bytes starting at 's', each in a \xXY format. */
+     * bytes starting at 's'.  'format' gives how to display each byte.
+     * Currently, there are only two formats, so it is currently a bool:
+     *      0   \xab
+     *      1    ab         (that is a space between two hex digit bytes)
+     */
 
     const STRLEN output_len = 4 * len + 1;  /* 4 bytes per each input, plus a
                                                trailing NUL */
@@ -776,8 +778,13 @@ S__byte_dump_string(pTHX_ const U8 * s, const STRLEN len)
         const unsigned high_nibble = (*s & 0xF0) >> 4;
         const unsigned low_nibble =  (*s & 0x0F);
 
-        *d++ = '\\';
-        *d++ = 'x';
+        if (format) {
+            *d++ = ' ';
+        }
+        else {
+            *d++ = '\\';
+            *d++ = 'x';
+        }
 
         if (high_nibble < 10) {
             *d++ = high_nibble + '0';
@@ -827,7 +834,7 @@ S_unexpected_non_continuation_text(pTHX_ const U8 * const s,
     return Perl_form(aTHX_ "%s: %s (unexpected non-continuation byte 0x%02x,"
                            " %s after start byte 0x%02x; need %d bytes, got %d)",
                            malformed_text,
-                           _byte_dump_string(s, print_len),
+                           _byte_dump_string(s, print_len, 0),
                            *(s + non_cont_byte_pos),
                            where,
                            *s,
@@ -1070,6 +1077,8 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
     U8 * adjusted_s0 = (U8 *) s0;
     U8 * adjusted_send = NULL;  /* (Initialized to silence compilers' wrong
                                    warning) */
+    U8 temp_char_buf[UTF8_MAXBYTES + 1]; /* Used to avoid a Newx in this
+                                            routine; see [perl #130921] */
     UV uv_so_far = 0;   /* (Initialized to silence compilers' wrong warning) */
 
     PERL_ARGS_ASSERT_UTF8N_TO_UVCHR_ERROR;
@@ -1179,14 +1188,6 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
     /* Save how many bytes were actually in the character */
     curlen = s - s0;
 
-    /* A convenience macro that matches either of the too-short conditions.  */
-#   define UTF8_GOT_TOO_SHORT (UTF8_GOT_SHORT|UTF8_GOT_NON_CONTINUATION)
-
-    if (UNLIKELY(possible_problems & UTF8_GOT_TOO_SHORT)) {
-        uv_so_far = uv;
-        uv = UNICODE_REPLACEMENT;
-    }
-
     /* Note that there are two types of too-short malformation.  One is when
      * there is actual wrong data before the normal termination of the
      * sequence.  The other is that the sequence wasn't complete before the end
@@ -1194,7 +1195,15 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
      * This means that we were passed data for a partial character, but it is
      * valid as far as we saw.  The other is definitely invalid.  This
      * distinction could be important to a caller, so the two types are kept
-     * separate. */
+     * separate.
+     *
+     * A convenience macro that matches either of the too-short conditions.  */
+#   define UTF8_GOT_TOO_SHORT (UTF8_GOT_SHORT|UTF8_GOT_NON_CONTINUATION)
+
+    if (UNLIKELY(possible_problems & UTF8_GOT_TOO_SHORT)) {
+        uv_so_far = uv;
+        uv = UNICODE_REPLACEMENT;
+    }
 
     /* Check for overflow */
     if (UNLIKELY(does_utf8_overflow(s0, send))) {
@@ -1236,10 +1245,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                                      I8_TO_NATIVE_UTF8(UTF_CONTINUATION_MARK));
             }
 
-            Newx(adjusted_s0, OFFUNISKIP(min_uv) + 1, U8);
-            SAVEFREEPV((U8 *) adjusted_s0);    /* Needed because we may not get
-                                                  to free it ourselves if
-                                                  warnings are made fatal */
+            adjusted_s0 = temp_char_buf;
             adjusted_send = uvoffuni_to_utf8_flags(adjusted_s0, min_uv, 0);
         }
     }
@@ -1401,7 +1407,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                         if (pack_warn) {
                             message = Perl_form(aTHX_ "%s: %s (overflows)",
                                             malformed_text,
-                                            _byte_dump_string(s0, send - s0));
+                                            _byte_dump_string(s0, send - s0, 0));
                         }
                     }
                 }
@@ -1437,7 +1443,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                                 "%s: %s (unexpected continuation byte 0x%02x,"
                                 " with no preceding start byte)",
                                 malformed_text,
-                                _byte_dump_string(s0, 1), *s0);
+                                _byte_dump_string(s0, 1, 0), *s0);
                     }
                 }
             }
@@ -1452,7 +1458,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                         message = Perl_form(aTHX_
                                 "%s: %s (too short; %d byte%s available, need %d)",
                                 malformed_text,
-                                _byte_dump_string(s0, send - s0),
+                                _byte_dump_string(s0, send - s0, 0),
                                 (int)avail_len,
                                 avail_len == 1 ? "" : "s",
                                 (int)expectlen);
@@ -1516,8 +1522,8 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                                     " should be represented with a"
                                     " different, shorter sequence)",
                                     malformed_text,
-                                    _byte_dump_string(s0, send - s0),
-                                    _byte_dump_string(s0, curlen));
+                                    _byte_dump_string(s0, send - s0, 0),
+                                    _byte_dump_string(s0, curlen, 0));
                         }
                         else {
                             U8 tmpbuf[UTF8_MAXBYTES+1];
@@ -1527,8 +1533,8 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                                 "%s: %s (overlong; instead use %s to represent"
                                 " U+%0*" UVXf ")",
                                 malformed_text,
-                                _byte_dump_string(s0, send - s0),
-                                _byte_dump_string(tmpbuf, e - tmpbuf),
+                                _byte_dump_string(s0, send - s0, 0),
+                                _byte_dump_string(tmpbuf, e - tmpbuf, 0),
                                 ((uv < 256) ? 2 : 4), /* Field width of 2 for
                                                          small code points */
                                 uv);
@@ -1553,7 +1559,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                             message = Perl_form(aTHX_
                                     "UTF-16 surrogate (any UTF-8 sequence that"
                                     " starts with \"%s\" is for a surrogate)",
-                                    _byte_dump_string(s0, curlen));
+                                    _byte_dump_string(s0, curlen, 0));
                         }
                         else {
                             message = Perl_form(aTHX_
@@ -1583,7 +1589,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                                     "Any UTF-8 sequence that starts with"
                                     " \"%s\" is for a non-Unicode code point,"
                                     " may not be portable",
-                                    _byte_dump_string(s0, curlen));
+                                    _byte_dump_string(s0, curlen, 0));
                         }
                         else {
                             message = Perl_form(aTHX_
@@ -1622,7 +1628,7 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                                         "Any UTF-8 sequence that starts with"
                                         " \"%s\" is for a non-Unicode code"
                                         " point, and is not portable",
-                                        _byte_dump_string(s0, curlen));
+                                        _byte_dump_string(s0, curlen, 0));
                         }
                         else {
                             message = Perl_form(aTHX_
@@ -1655,12 +1661,9 @@ Perl_utf8n_to_uvchr_error(pTHX_ const U8 *s,
                  * where 'uv' is not valid. */
                 if (   ! (orig_problems
                                     & (UTF8_GOT_TOO_SHORT|UTF8_GOT_OVERFLOW))
-                    && UNLIKELY(uv > MAX_NON_DEPRECATED_CP)
-                    && ckWARN_d(WARN_DEPRECATED))
-                {
-                    message = Perl_form(aTHX_ cp_above_legal_max,
-                                              uv, MAX_NON_DEPRECATED_CP);
-                    pack_warn = packWARN(WARN_DEPRECATED);
+                    && UNLIKELY(uv > MAX_NON_DEPRECATED_CP)) {
+                    Perl_croak(aTHX_ cp_above_legal_max, uv,
+                                     MAX_NON_DEPRECATED_CP);
                 }
             }
             else if (possible_problems & UTF8_GOT_NONCHAR) {
@@ -1748,6 +1751,8 @@ Also implemented as a macro in utf8.h
 UV
 Perl_utf8_to_uvchr_buf(pTHX_ const U8 *s, const U8 *send, STRLEN *retlen)
 {
+    PERL_ARGS_ASSERT_UTF8_TO_UVCHR_BUF;
+
     assert(s < send);
 
     return utf8n_to_uvchr(s, send - s, retlen,
@@ -1952,13 +1957,20 @@ Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len)
 /*
 =for apidoc bytes_from_utf8
 
-Converts a string C<s> of length C<len> from UTF-8 into native byte encoding.
-Unlike L</utf8_to_bytes> but like L</bytes_to_utf8>, returns a pointer to
-the newly-created string, and updates C<len> to contain the new
-length.  Returns the original string if no conversion occurs, C<len>
-is unchanged.  Do nothing if C<is_utf8> points to 0.  Sets C<is_utf8> to
-0 if C<s> is converted or consisted entirely of characters that are invariant
-in UTF-8 (i.e., US-ASCII on non-EBCDIC machines).
+Converts a potentially UTF-8 encoded string C<s> of length C<len> into native
+byte encoding.  On input, the boolean C<*is_utf8> gives whether or not C<s> is
+actually encoded in UTF-8.
+
+Unlike L</utf8_to_bytes> but like L</bytes_to_utf8>, this is non-destructive of
+the input string.
+
+Do nothing if C<*is_utf8> is 0, or if there are code points in the string
+not expressible in native byte encoding.  In these cases, C<*is_utf8> and
+C<*len> are unchanged, and the return value is the original C<s>.
+
+Otherwise, C<*is_utf8> is set to 0, and the return value is a pointer to a
+newly created string containing a downgraded copy of C<s>, and whose length is
+returned in C<*len>, updated.
 
 =cut
 */
@@ -1969,7 +1981,7 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8)
     U8 *d;
     const U8 *start = s;
     const U8 *send;
-    I32 count = 0;
+    Size_t count = 0;
 
     PERL_ARGS_ASSERT_BYTES_FROM_UTF8;
     PERL_UNUSED_CONTEXT;
@@ -2428,13 +2440,13 @@ Perl__to_uni_fold_flags(pTHX_ UV c, U8* p, STRLEN *lenp, U8 flags)
        uvchr_to_utf8(p, c);
        return CALL_FOLD_CASE(c, p, p, lenp, flags & FOLD_FLAGS_FULL);
     }
-    else {  /* Otherwise, _to_utf8_fold_flags has the intelligence to deal with
+    else {  /* Otherwise, _toFOLD_utf8_flags has the intelligence to deal with
               the special flags. */
        U8 utf8_c[UTF8_MAXBYTES + 1];
 
       needs_full_generality:
        uvchr_to_utf8(utf8_c, c);
-       return _to_utf8_fold_flags(utf8_c, p, lenp, flags);
+       return _toFOLD_utf8_flags(utf8_c, utf8_c + sizeof(utf8_c), p, lenp, flags);
     }
 }
 
@@ -2541,11 +2553,20 @@ S_warn_on_first_deprecated_use(pTHX_ const char * const name,
                Perl_croak(aTHX_ "panic: hv_store() unexpectedly failed");
             }
 
+            if (instr(file, "mathoms.c")) {
+                Perl_warner(aTHX_ WARN_DEPRECATED,
+                            "In %s, line %d, starting in Perl v5.30, %s()"
+                            " will be removed.  Avoid this message by"
+                            " converting to use %s().\n",
+                            file, line, name, alternative);
+            }
+            else {
                 Perl_warner(aTHX_ WARN_DEPRECATED,
                             "In %s, line %d, starting in Perl v5.30, %s() will"
                             " require an additional parameter.  Avoid this"
                             " message by converting to use %s().\n",
                             file, line, name, alternative);
+            }
         }
     }
 }
@@ -2565,7 +2586,6 @@ Perl__is_utf8_FOO(pTHX_       U8   classnum,
     warn_on_first_deprecated_use(name, alternative, use_locale, file, line);
 
     if (use_utf8 && UTF8_IS_ABOVE_LATIN1(*p)) {
-        SV * invlist;
 
         switch (classnum) {
             case _CC_WORDCHAR:
@@ -2598,14 +2618,18 @@ Perl__is_utf8_FOO(pTHX_       U8   classnum,
                 return is_VERTWS_high(p);
             case _CC_IDFIRST:
                 if (! PL_utf8_perl_idstart) {
-                    invlist = _new_invlist_C_array(_Perl_IDStart_invlist);
+                    PL_utf8_perl_idstart
+                                = _new_invlist_C_array(_Perl_IDStart_invlist);
                 }
-                return is_utf8_common(p, &PL_utf8_perl_idstart, "_Perl_IDStart", invlist);
+                return is_utf8_common(p, &PL_utf8_perl_idstart,
+                                      "_Perl_IDStart", NULL);
             case _CC_IDCONT:
                 if (! PL_utf8_perl_idcont) {
-                    invlist = _new_invlist_C_array(_Perl_IDCont_invlist);
+                    PL_utf8_perl_idcont
+                                = _new_invlist_C_array(_Perl_IDCont_invlist);
                 }
-                return is_utf8_common(p, &PL_utf8_perl_idcont, "_Perl_IDCont", invlist);
+                return is_utf8_common(p, &PL_utf8_perl_idcont,
+                                      "_Perl_IDCont", NULL);
         }
     }
 
@@ -2715,50 +2739,6 @@ Perl__is_utf8_mark(pTHX_ const U8 *p)
     return is_utf8_common(p, &PL_utf8_mark, "IsM", NULL);
 }
 
-/*
-=for apidoc to_utf8_case
-
-Instead use the appropriate one of L</toUPPER_utf8>,
-L</toTITLE_utf8>,
-L</toLOWER_utf8>,
-or L</toFOLD_utf8>.
-
-C<p> contains the pointer to the UTF-8 string encoding
-the character that is being converted.  This routine assumes that the character
-at C<p> is well-formed.
-
-C<ustrp> is a pointer to the character buffer to put the
-conversion result to.  C<lenp> is a pointer to the length
-of the result.
-
-C<swashp> is a pointer to the swash to use.
-
-Both the special and normal mappings are stored in F<lib/unicore/To/Foo.pl>,
-and loaded by C<SWASHNEW>, using F<lib/utf8_heavy.pl>.  C<special> (usually,
-but not always, a multicharacter mapping), is tried first.
-
-C<special> is a string, normally C<NULL> or C<"">.  C<NULL> means to not use
-any special mappings; C<""> means to use the special mappings.  Values other
-than these two are treated as the name of the hash containing the special
-mappings, like C<"utf8::ToSpecLower">.
-
-C<normal> is a string like C<"ToLower"> which means the swash
-C<%utf8::ToLower>.
-
-Code points above the platform's C<IV_MAX> will raise a deprecation warning,
-unless those are turned off.
-
-=cut */
-
-UV
-Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp,
-                       SV **swashp, const char *normal, const char *special)
-{
-    PERL_ARGS_ASSERT_TO_UTF8_CASE;
-
-    return _to_utf8_case(valid_utf8_to_uvchr(p, NULL), p, ustrp, lenp, swashp, normal, special);
-}
-
     /* change namve uv1 to 'from' */
 STATIC UV
 S__to_utf8_case(pTHX_ const UV uv1, const U8 *p, U8* ustrp, STRLEN *lenp,
@@ -2833,11 +2813,9 @@ S__to_utf8_case(pTHX_ const UV uv1, const U8 *p, U8* ustrp, STRLEN *lenp,
                 }
 
                 if (UNLIKELY(UNICODE_IS_SUPER(uv1))) {
-                    if (   UNLIKELY(uv1 > MAX_NON_DEPRECATED_CP)
-                        && ckWARN_d(WARN_DEPRECATED))
-                    {
-                        Perl_warner(aTHX_ packWARN(WARN_DEPRECATED),
-                                cp_above_legal_max, uv1, MAX_NON_DEPRECATED_CP);
+                    if (UNLIKELY(uv1 > MAX_NON_DEPRECATED_CP)) {
+                        Perl_croak(aTHX_ cp_above_legal_max, uv1,
+                                         MAX_NON_DEPRECATED_CP);
                     }
                     if (ckWARN_d(WARN_NON_UNICODE)) {
                         const char* desc = (PL_op) ? OP_DESC(PL_op) : normal;
@@ -2989,6 +2967,84 @@ S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* c
     return original;
 }
 
+STATIC U32
+S_check_and_deprecate(pTHX_ const U8 *p,
+                            const U8 **e,
+                            const unsigned int type,    /* See below */
+                            const bool use_locale,      /* Is this a 'LC_'
+                                                           macro call? */
+                            const char * const file,
+                            const unsigned line)
+{
+    /* This is a temporary function to deprecate the unsafe calls to the case
+     * changing macros and functions.  It keeps all the special stuff in just
+     * one place.
+     *
+     * It updates *e with the pointer to the end of the input string.  If using
+     * the old-style macros, *e is NULL on input, and so this function assumes
+     * the input string is long enough to hold the entire UTF-8 sequence, and
+     * sets *e accordingly, but it then returns a flag to pass the
+     * utf8n_to_uvchr(), to tell it that this size is a guess, and to avoid
+     * using the full length if possible.
+     *
+     * It also does the assert that *e > p when *e is not NULL.  This should be
+     * migrated to the callers when this function gets deleted.
+     *
+     * The 'type' parameter is used for the caller to specify which case
+     * changing function this is called from: */
+
+#       define DEPRECATE_TO_UPPER 0
+#       define DEPRECATE_TO_TITLE 1
+#       define DEPRECATE_TO_LOWER 2
+#       define DEPRECATE_TO_FOLD  3
+
+    U32 utf8n_flags = 0;
+    const char * name;
+    const char * alternative;
+
+    PERL_ARGS_ASSERT_CHECK_AND_DEPRECATE;
+
+    if (*e == NULL) {
+        utf8n_flags = _UTF8_NO_CONFIDENCE_IN_CURLEN;
+        *e = p + UTF8SKIP(p);
+
+        /* For mathoms.c calls, we use the function name we know is stored
+         * there.  It could be part of a larger path */
+        if (type == DEPRECATE_TO_UPPER) {
+            name = instr(file, "mathoms.c")
+                   ? "to_utf8_upper"
+                   : "toUPPER_utf8";
+            alternative = "toUPPER_utf8_safe";
+        }
+        else if (type == DEPRECATE_TO_TITLE) {
+            name = instr(file, "mathoms.c")
+                   ? "to_utf8_title"
+                   : "toTITLE_utf8";
+            alternative = "toTITLE_utf8_safe";
+        }
+        else if (type == DEPRECATE_TO_LOWER) {
+            name = instr(file, "mathoms.c")
+                   ? "to_utf8_lower"
+                   : "toLOWER_utf8";
+            alternative = "toLOWER_utf8_safe";
+        }
+        else if (type == DEPRECATE_TO_FOLD) {
+            name = instr(file, "mathoms.c")
+                   ? "to_utf8_fold"
+                   : "toFOLD_utf8";
+            alternative = "toFOLD_utf8_safe";
+        }
+        else Perl_croak(aTHX_ "panic: Unexpected case change type");
+
+        warn_on_first_deprecated_use(name, alternative, use_locale, file, line);
+    }
+    else {
+        assert (p < *e);
+    }
+
+    return utf8n_flags;
+}
+
 /* The process for changing the case is essentially the same for the four case
  * change types, except there are complications for folding.  Otherwise the
  * difference is only which case to change to.  To make sure that they all do
@@ -3019,6 +3075,7 @@ S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* c
  * going on. */
 #define CASE_CHANGE_BODY_START(locale_flags, LC_L1_change_macro, L1_func,    \
                                L1_func_extra_param)                          \
+                                                                             \
     if (flags & (locale_flags)) {                                            \
         /* Treat a UTF-8 locale as not being in locale at all */             \
         if (IN_UTF8_CTYPE_LOCALE) {                                          \
@@ -3037,7 +3094,7 @@ S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* c
             return L1_func(*p, ustrp, lenp, L1_func_extra_param);            \
         }                                                                    \
     }                                                                        \
-    else if UTF8_IS_DOWNGRADEABLE_START(*p) {                                \
+    else if UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(p, e) {                          \
         if (flags & (locale_flags)) {                                        \
             result = LC_L1_change_macro(EIGHT_BIT_UTF8_TO_NATIVE(*p,         \
                                                                  *(p+1)));   \
@@ -3047,8 +3104,13 @@ S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* c
                            ustrp, lenp,  L1_func_extra_param);               \
         }                                                                    \
     }                                                                        \
-    else {  /* malformed UTF-8 */                                            \
-        result = valid_utf8_to_uvchr(p, NULL);                               \
+    else {  /* malformed UTF-8 or ord above 255 */                           \
+        STRLEN len_result;                                                   \
+        result = utf8n_to_uvchr(p, e - p, &len_result, UTF8_CHECK_ONLY);     \
+        if (len_result == (STRLEN) -1) {                                     \
+            _force_out_malformed_utf8_message(p, e, utf8n_flags,             \
+                                                            1 /* Die */ );   \
+        }
 
 #define CASE_CHANGE_BODY_END(locale_flags, change_macro)                     \
         result = change_macro(result, p, ustrp, lenp);                       \
@@ -3075,7 +3137,7 @@ S_check_locale_boundary_crossing(pTHX_ const U8* const p, const UV result, U8* c
 /*
 =for apidoc to_utf8_upper
 
-Instead use L</toUPPER_utf8>.
+Instead use L</toUPPER_utf8_safe>.
 
 =cut */
 
@@ -3084,9 +3146,17 @@ Instead use L</toUPPER_utf8>.
  *         be used. */
 
 UV
-Perl__to_utf8_upper_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags)
+Perl__to_utf8_upper_flags(pTHX_ const U8 *p,
+                                const U8 *e,
+                                U8* ustrp,
+                                STRLEN *lenp,
+                                bool flags,
+                                const char * const file,
+                                const int line)
 {
     UV result;
+    const U32 utf8n_flags = check_and_deprecate(p, &e, DEPRECATE_TO_UPPER,
+                                                cBOOL(flags), file, line);
 
     PERL_ARGS_ASSERT__TO_UTF8_UPPER_FLAGS;
 
@@ -3099,7 +3169,7 @@ Perl__to_utf8_upper_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags
 /*
 =for apidoc to_utf8_title
 
-Instead use L</toTITLE_utf8>.
+Instead use L</toTITLE_utf8_safe>.
 
 =cut */
 
@@ -3110,9 +3180,17 @@ Instead use L</toTITLE_utf8>.
  */
 
 UV
-Perl__to_utf8_title_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags)
+Perl__to_utf8_title_flags(pTHX_ const U8 *p,
+                                const U8 *e,
+                                U8* ustrp,
+                                STRLEN *lenp,
+                                bool flags,
+                                const char * const file,
+                                const int line)
 {
     UV result;
+    const U32 utf8n_flags = check_and_deprecate(p, &e, DEPRECATE_TO_TITLE,
+                                                cBOOL(flags), file, line);
 
     PERL_ARGS_ASSERT__TO_UTF8_TITLE_FLAGS;
 
@@ -3124,7 +3202,7 @@ Perl__to_utf8_title_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags
 /*
 =for apidoc to_utf8_lower
 
-Instead use L</toLOWER_utf8>.
+Instead use L</toLOWER_utf8_safe>.
 
 =cut */
 
@@ -3134,9 +3212,17 @@ Instead use L</toLOWER_utf8>.
  */
 
 UV
-Perl__to_utf8_lower_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags)
+Perl__to_utf8_lower_flags(pTHX_ const U8 *p,
+                                const U8 *e,
+                                U8* ustrp,
+                                STRLEN *lenp,
+                                bool flags,
+                                const char * const file,
+                                const int line)
 {
     UV result;
+    const U32 utf8n_flags = check_and_deprecate(p, &e, DEPRECATE_TO_LOWER,
+                                                cBOOL(flags), file, line);
 
     PERL_ARGS_ASSERT__TO_UTF8_LOWER_FLAGS;
 
@@ -3147,7 +3233,7 @@ Perl__to_utf8_lower_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, bool flags
 /*
 =for apidoc to_utf8_fold
 
-Instead use L</toFOLD_utf8>.
+Instead use L</toFOLD_utf8_safe>.
 
 =cut */
 
@@ -3162,9 +3248,17 @@ Instead use L</toFOLD_utf8>.
  */
 
 UV
-Perl__to_utf8_fold_flags(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, U8 flags)
+Perl__to_utf8_fold_flags(pTHX_ const U8 *p,
+                               const U8 *e,
+                               U8* ustrp,
+                               STRLEN *lenp,
+                               U8 flags,
+                               const char * const file,
+                               const int line)
 {
     UV result;
+    const U32 utf8n_flags = check_and_deprecate(p, &e, DEPRECATE_TO_FOLD,
+                                                cBOOL(flags), file, line);
 
     PERL_ARGS_ASSERT__TO_UTF8_FOLD_FLAGS;
 
@@ -4401,12 +4495,12 @@ Perl__swash_inversion_hash(pTHX_ SV* const swash)
        while ((from_list = (AV *) hv_iternextsv(specials_inverse,
                                                 &char_to, &to_len)))
        {
-           if (av_tindex_nomg(from_list) > 0) {
+           if (av_tindex_skip_len_mg(from_list) > 0) {
                SSize_t i;
 
                /* We iterate over all combinations of i,j to place each code
                 * point on each list */
-               for (i = 0; i <= av_tindex_nomg(from_list); i++) {
+               for (i = 0; i <= av_tindex_skip_len_mg(from_list); i++) {
                    SSize_t j;
                    AV* i_list = newAV();
                    SV** entryp = av_fetch(from_list, i, FALSE);
@@ -4423,7 +4517,7 @@ Perl__swash_inversion_hash(pTHX_ SV* const swash)
                    }
 
                    /* For DEBUG_U: UV u = valid_utf8_to_uvchr((U8*) SvPVX(*entryp), 0);*/
-                   for (j = 0; j <= av_tindex_nomg(from_list); j++) {
+                   for (j = 0; j <= av_tindex_skip_len_mg(from_list); j++) {
                        entryp = av_fetch(from_list, j, FALSE);
                        if (entryp == NULL) {
                            Perl_croak(aTHX_ "panic: av_fetch() unexpectedly failed");
@@ -4499,7 +4593,7 @@ Perl__swash_inversion_hash(pTHX_ SV* const swash)
 
            /* Look through list to see if this inverse mapping already is
             * listed, or if there is a mapping to itself already */
-           for (i = 0; i <= av_tindex_nomg(list); i++) {
+           for (i = 0; i <= av_tindex_skip_len_mg(list); i++) {
                SV** entryp = av_fetch(list, i, FALSE);
                SV* entry;
                UV uv;
@@ -4626,7 +4720,6 @@ Perl__swash_to_invlist(pTHX_ SV* const swash)
             invlist = _new_invlist(0);
         }
         else {
-            while (isSPACE(*l)) l++;
             l = (U8 *) after_atou;
 
             /* Get the 0th element, which is needed to setup the inversion list */
@@ -5151,7 +5244,7 @@ Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const c
                     *foldbuf1 = toFOLD(*p1);
                 }
                 else if (u1) {
-                    _to_utf8_fold_flags(p1, foldbuf1, &n1, flags_for_folder);
+                    _toFOLD_utf8_flags(p1, e1, foldbuf1, &n1, flags_for_folder);
                 }
                 else {  /* Not UTF-8, get UTF-8 fold */
                     _to_uni_fold_flags(*p1, foldbuf1, &n1, flags_for_folder);
@@ -5175,7 +5268,7 @@ Perl_foldEQ_utf8_flags(pTHX_ const char *s1, char **pe1, UV l1, bool u1, const c
                     *foldbuf2 = toFOLD(*p2);
                 }
                 else if (u2) {
-                    _to_utf8_fold_flags(p2, foldbuf2, &n2, flags_for_folder);
+                    _toFOLD_utf8_flags(p2, e2, foldbuf2, &n2, flags_for_folder);
                 }
                 else {
                     _to_uni_fold_flags(*p2, foldbuf2, &n2, flags_for_folder);