X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/57f0e7e230d864f5b78d28bb89545ef671c101a0..9fc1b446946500eed6f668b6c529a4688d6866c2:/utf8.h

diff --git a/utf8.h b/utf8.h
index 7472de1..5be352e 100644
--- a/utf8.h
+++ b/utf8.h
@@ -171,8 +171,11 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #define UTF8_IS_CONTINUATION(c)		((((U8)c) & 0xC0) == 0x80)
 #define UTF8_IS_CONTINUED(c) 		(((U8)c) &  0x80)
 
-/* Masking with 0xfe allows low bit to be 0 or 1; thus this matches 0xc[23] */
-#define UTF8_IS_DOWNGRADEABLE_START(c)	(((U8)c & 0xfe) == 0xc2)
+/* Use UTF8_IS_NEXT_CHAR_DOWNGRADEABLE() instead if the input isn't known to
+ * be well-formed.  Masking with 0xfe allows low bit to be 0 or 1; thus this
+ * matches 0xc[23]. */
+#define UTF8_IS_DOWNGRADEABLE_START(c)	(((U8)(c) & 0xfe) == 0xc2)
+
 #define UTF8_IS_ABOVE_LATIN1(c)	((U8)(c) >= 0xc4)
 
 #define UTF_START_MARK(len) (((len) >  7) ? 0xFF : (0xFE << (7-(len))))
@@ -243,6 +246,11 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #define UTF8_ACCUMULATE(old, new)	(((old) << UTF_ACCUMULATION_SHIFT)     \
                                         | (((U8)new) & UTF_CONTINUATION_MASK))
 
+/* This works in the face of malformed UTF-8. */
+#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) (UTF8_IS_DOWNGRADEABLE_START(*s) \
+                                               && ( (e) - (s) > 1)             \
+                                               && UTF8_IS_CONTINUATION(*((s)+1)))
+
 /* Convert a two (not one) byte utf8 character to a unicode code point value.
  * Needs just one iteration of accumulate.  Should not be used unless it is
  * known that the two bytes are legal: 1) two-byte start, and 2) continuation.
@@ -367,11 +375,12 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
  * U+110001: \xF4\x90\x80\x81	\xF9\xA2\xA0\xA0\xA1
  */
 #ifdef EBCDIC /* Both versions assume well-formed UTF8 */
-#   define UTF8_IS_SUPER(s)  (NATIVE_TO_I8(*(s)) >= 0xF9                       \
-      && (NATIVE_TO_I8(*(s)) > 0xF9) || (NATIVE_TO_I8(*((s)) + 1 >= 0xA2)))
+#   define UTF8_IS_SUPER(s)  (NATIVE_TO_I8(* (U8*) (s)) >= 0xF9                 \
+                              && (NATIVE_TO_I8(* (U8*) (s)) > 0xF9              \
+                                  || (NATIVE_TO_I8(* (U8*) ((s)) + 1 >= 0xA2))))
 #else
-#   define UTF8_IS_SUPER(s)  (*(s) >= 0xF4                                      \
-					&& (*(s) > 0xF4 || (*((s) + 1) >= 0x90)))
+#   define UTF8_IS_SUPER(s) (*(U8*) (s) >= 0xF4                                 \
+                            && (*(U8*) (s) > 0xF4 || (*((U8*) (s) + 1) >= 0x90)))
 #endif
 
 /* These are now machine generated, and the 'given' clause is no longer
@@ -468,6 +477,11 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
  * takes on the order of 10 minutes to generate, and is never going to change.
  * The EBCDIC equivalent hasn't been commented out in regcharclass.pl, so it
  * should generate and run the correct stuff */
+/*
+	UTF8_CHAR: Matches utf8 from 1 to 4 bytes
+
+	0x0 - 0x1FFFFF
+*/
 /*** GENERATED CODE ***/
 #define is_UTF8_CHAR_utf8_safe(s,e)                                         \
 ( ((e)-(s) > 3) ?                                                           \
@@ -480,23 +494,17 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 	( ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
     : ( 0xF0 == ((U8*)s)[0] ) ?                                             \
 	( ( ( ( 0x90 <= ((U8*)s)[1] && ((U8*)s)[1] <= 0xBF ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
-    : ( 0xF1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xF7 ) ?                      \
-	( ( ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
-    : 0 )                                                                   \
+    : ( ( ( ( 0xF1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xF7 ) && ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
 : ((e)-(s) > 2) ?                                                           \
     ( ( ( ((U8*)s)[0] & 0x80 ) == 0x00 ) ? 1                                \
     : ( 0xC2 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xDF ) ?                      \
 	( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 )                      \
     : ( 0xE0 == ((U8*)s)[0] ) ?                                             \
 	( ( ( ( ((U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-    : ( 0xE1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xEF ) ?                      \
-	( ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
-    : 0 )                                                                   \
+    : ( ( ( 0xE1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xEF ) && ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
 : ((e)-(s) > 1) ?                                                           \
     ( ( ( ((U8*)s)[0] & 0x80 ) == 0x00 ) ? 1                                \
-    : ( 0xC2 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xDF ) ?                      \
-	( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 )                      \
-    : 0 )                                                                   \
+    : ( ( 0xC2 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xDF ) && ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ) ? 2 : 0 )\
 : ((e)-(s) > 0) ?                                                           \
     ( ( ((U8*)s)[0] & 0x80 ) == 0x00 )                                      \
 : 0 )