regen/mk_PL_charclass.pl can test that l1_char_class_tab.h is up to date.

[perl5.git] / utf8.h

diff --git a/utf8.h b/utf8.h
index 636ac8a..9fa9e90 100644 (file)
--- a/utf8.h
+++ b/utf8.h
@@ -118,7 +118,6 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 
 */
 
-
 #define UNI_IS_INVARIANT(c)            (((UV)c) <  0x80)
 /* Note that C0 and C1 are invalid in legal UTF8, so the lower bound of the
  * below might ought to be C2 */
@@ -225,24 +224,59 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
  * SpecialCasing.txt. */
 #define UTF8_MAXBYTES_CASE     6
 
+/* A Unicode character can fold to up to 3 characters */
+#define UTF8_MAX_FOLD_CHAR_EXPAND 3
+
 #define IN_BYTES (CopHINTS_get(PL_curcop) & HINT_BYTES)
 #define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTES)
 #define IN_UNI_8_BIT ( (CopHINTS_get(PL_curcop) & HINT_UNI_8_BIT) \
                        && ! IN_LOCALE_RUNTIME && ! IN_BYTES)
 
-#define UTF8_ALLOW_EMPTY               0x0001
+#define UTF8_ALLOW_EMPTY               0x0001  /* Allow a zero length string */
+
+/* Allow first byte to be a continuation byte */
 #define UTF8_ALLOW_CONTINUATION                0x0002
+
+/* Allow second... bytes to be non-continuation bytes */
 #define UTF8_ALLOW_NON_CONTINUATION    0x0004
-#define UTF8_ALLOW_FE_FF               0x0008 /* Allow FE or FF start bytes, \
-                                                 yields above 0x7fffFFFF = 31 bits */
-#define UTF8_ALLOW_SHORT               0x0010 /* expecting more bytes */
-#define UTF8_ALLOW_SURROGATE           0x0020
-#define UTF8_ALLOW_FFFF                        0x0040 /* Allow UNICODE_ILLEGAL */
-#define UTF8_ALLOW_LONG                        0x0080 /* expecting fewer bytes */
-#define UTF8_ALLOW_ANYUV               (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\
-                                        UTF8_ALLOW_SURROGATE|UTF8_ALLOW_FFFF)
-#define UTF8_ALLOW_ANY                 0x00FF
-#define UTF8_CHECK_ONLY                        0x0200
+
+/* expecting more bytes than were available in the string */
+#define UTF8_ALLOW_SHORT               0x0008
+
+/* Overlong sequence; i.e., the code point can be specified in fewer bytes. */
+#define UTF8_ALLOW_LONG                 0x0010
+
+#define UTF8_DISALLOW_SURROGATE                0x0020  /* Unicode surrogates */
+#define UTF8_WARN_SURROGATE            0x0040
+
+#define UTF8_DISALLOW_NONCHAR           0x0080 /* Unicode non-character */
+#define UTF8_WARN_NONCHAR               0x0100 /*  code points */
+
+#define UTF8_DISALLOW_SUPER            0x0200  /* Super-set of Unicode: code */
+#define UTF8_WARN_SUPER                        0x0400  /* points above the legal max */
+
+/* Code points which never were part of the original UTF-8 standard, the first
+ * byte of which is a FE or FF on ASCII platforms. */
+#define UTF8_DISALLOW_FE_FF            0x0800
+#define UTF8_WARN_FE_FF                        0x1000
+
+#define UTF8_CHECK_ONLY                        0x2000
+
+/* For backwards source compatibility.  They do nothing, as the default now
+ * includes what they used to mean.  The first one's meaning was to allow the
+ * just the single non-character 0xFFFF */
+#define UTF8_ALLOW_FFFF 0
+#define UTF8_ALLOW_SURROGATE 0
+
+#define UTF8_DISALLOW_ILLEGAL_INTERCHANGE \
+           (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_NONCHAR|UTF8_DISALLOW_SURROGATE)
+#define UTF8_WARN_ILLEGAL_INTERCHANGE \
+                       (UTF8_WARN_SUPER|UTF8_WARN_NONCHAR|UTF8_WARN_SURROGATE)
+#define UTF8_ALLOW_ANY \
+           (~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_WARN_ILLEGAL_INTERCHANGE))
+#define UTF8_ALLOW_ANYUV                                                        \
+         (UTF8_ALLOW_EMPTY                                                      \
+         & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_WARN_ILLEGAL_INTERCHANGE))
 #define UTF8_ALLOW_DEFAULT             (ckWARN(WARN_UTF8) ? 0 : \
                                         UTF8_ALLOW_ANYUV)
 
@@ -341,23 +375,33 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #define UNICODE_SURROGATE_LAST         0xDFFF
 #define UNICODE_REPLACEMENT            0xFFFD
 #define UNICODE_BYTE_ORDER_MARK                0xFEFF
-#define UNICODE_ILLEGAL                        0xFFFF
 
 /* Though our UTF-8 encoding can go beyond this,
- * let's be conservative and do as Unicode 5.1 says. */
+ * let's be conservative and do as Unicode says. */
 #define PERL_UNICODE_MAX       0x10FFFF
 
-#define UNICODE_ALLOW_SURROGATE 0x0001 /* Allow UTF-16 surrogates (EVIL) */
-#define UNICODE_ALLOW_FDD0     0x0002  /* Allow the U+FDD0...U+FDEF */
-#define UNICODE_ALLOW_FFFF     0x0004  /* Allow U+FFF[EF], U+1FFF[EF], ... */
-#define UNICODE_ALLOW_SUPER    0x0008  /* Allow past 0x10FFFF */
-#define UNICODE_ALLOW_ANY      0x000F
+#define UNICODE_WARN_SURROGATE     0x0001      /* UTF-16 surrogates */
+#define UNICODE_WARN_NONCHAR       0x0002      /* Non-char code points */
+#define UNICODE_WARN_SUPER         0x0004      /* Above 0x10FFFF */
+#define UNICODE_WARN_FE_FF         0x0008      /* Above 0x10FFFF */
+#define UNICODE_DISALLOW_SURROGATE 0x0010
+#define UNICODE_DISALLOW_NONCHAR   0x0020
+#define UNICODE_DISALLOW_SUPER     0x0040
+#define UNICODE_DISALLOW_FE_FF     0x0080
+#define UNICODE_WARN_ILLEGAL_INTERCHANGE \
+    (UNICODE_WARN_SURROGATE|UNICODE_WARN_NONCHAR|UNICODE_WARN_SUPER)
+#define UNICODE_DISALLOW_ILLEGAL_INTERCHANGE \
+    (UNICODE_DISALLOW_SURROGATE|UNICODE_DISALLOW_NONCHAR|UNICODE_DISALLOW_SUPER)
+
+/* For backward source compatibility, as are now the default */
+#define UNICODE_ALLOW_SURROGATE 0
+#define UNICODE_ALLOW_SUPER    0
+#define UNICODE_ALLOW_ANY      0
 
 #define UNICODE_IS_SURROGATE(c)                ((c) >= UNICODE_SURROGATE_FIRST && \
                                         (c) <= UNICODE_SURROGATE_LAST)
 #define UNICODE_IS_REPLACEMENT(c)      ((c) == UNICODE_REPLACEMENT)
 #define UNICODE_IS_BYTE_ORDER_MARK(c)  ((c) == UNICODE_BYTE_ORDER_MARK)
-#define UNICODE_IS_ILLEGAL(c)          ((c) == UNICODE_ILLEGAL)
 #define UNICODE_IS_NONCHAR(c)          ((c >= 0xFDD0 && c <= 0xFDEF) \
                        /* The other noncharacters end in FFFE or FFFF, which  \
                         * the mask below catches both of, but beyond the last \