Extract _is_full().

[perl5.git] / utf8.h

diff --git a/utf8.h b/utf8.h
index bf8251a..3fbc28a 100644 (file)
--- a/utf8.h
+++ b/utf8.h
@@ -8,6 +8,9 @@
  *
  */
 
+#ifndef H_UTF8      /* Guard against recursive inclusion */
+#define H_UTF8 1
+
 /* Use UTF-8 as the default script encoding?
  * Turning this on will break scripts having non-UTF-8 binary
  * data (such as Latin-1) in string literals. */
@@ -168,9 +171,13 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #define UTF8_IS_CONTINUATION(c)                ((((U8)c) & 0xC0) == 0x80)
 #define UTF8_IS_CONTINUED(c)           (((U8)c) &  0x80)
 
-/* Masking with 0xfe allows low bit to be 0 or 1; thus this matches 0xc[23] */
+/* Use UTF8_IS_NEXT_CHAR_DOWNGRADEABLE() instead if the input isn't known to
+ * be well-formed.  Masking with 0xfe allows low bit to be 0 or 1; thus this
+ * matches 0xc[23]. */
 #define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfe) == 0xc2)
 
+#define UTF8_IS_ABOVE_LATIN1(c)        ((U8)(c) >= 0xc4)
+
 #define UTF_START_MARK(len) (((len) >  7) ? 0xFF : (0xFE << (7-(len))))
 
 /* Masks out the initial one bits in a start byte, leaving the real data ones.
@@ -208,6 +215,25 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
                      (uv) < 0x80000000     ? 6 : 7 )
 #endif
 
+/* How wide can a single UTF-8 encoded character become in bytes. */
+/* NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8
+ * is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be
+ * expressed with 4 bytes.  However, Perl thinks of UTF-8 as a way to encode
+ * non-negative integers in a binary format, even those above Unicode */
+#define UTF8_MAXBYTES 13
+
+/* The maximum number of UTF-8 bytes a single Unicode character can
+ * uppercase/lowercase/fold into.  Unicode guarantees that the maximum
+ * expansion is 3 characters.  On ASCIIish platforms, the highest Unicode
+ * character occupies 4 bytes, therefore this number would be 12, but this is
+ * smaller than the maximum width a single above-Unicode character can occupy,
+ * so use that instead */
+#if UTF8_MAXBYTES < 12
+#error UTF8_MAXBYTES must be at least 12
+#endif
+
+#define UTF8_MAXBYTES_CASE     UTF8_MAXBYTES
+
 #endif /* EBCDIC vs ASCII */
 
 /* Rest of these are attributes of Unicode and perl's internals rather than the
@@ -220,6 +246,11 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #define UTF8_ACCUMULATE(old, new)      (((old) << UTF_ACCUMULATION_SHIFT)     \
                                         | (((U8)new) & UTF_CONTINUATION_MASK))
 
+/* This works in the face of malformed UTF-8. */
+#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) (UTF8_IS_DOWNGRADEABLE_START(*s) \
+                                               && ( (e) - (s) > 1)             \
+                                               && UTF8_IS_CONTINUATION(*((s)+1)))
+
 /* Convert a two (not one) byte utf8 character to a unicode code point value.
  * Needs just one iteration of accumulate.  Should not be used unless it is
  * known that the two bytes are legal: 1) two-byte start, and 2) continuation.
@@ -269,23 +300,8 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #define isIDFIRST_lazy(p)      isIDFIRST_lazy_if(p,1)
 #define isALNUM_lazy(p)                isALNUM_lazy_if(p,1)
 
-#define UTF8_MAXBYTES 13
-/* How wide can a single UTF-8 encoded character become in bytes.
- * NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8
- * since UTF-8 is an encoding of Unicode and given Unicode's current
- * upper limit only four bytes is possible.  Perl thinks of UTF-8
- * as a way to encode non-negative integers in a binary format. */
 #define UTF8_MAXLEN UTF8_MAXBYTES
 
-/* The maximum number of UTF-8 bytes a single Unicode character can
- * uppercase/lowercase/fold into; this number depends on the Unicode
- * version.  An example of maximal expansion is the U+03B0 which
- * uppercases to U+03C5 U+0308 U+0301.  The Unicode databases that
- * tell these things are UnicodeData.txt, CaseFolding.txt, and
- * SpecialCasing.txt.  The value is 6 for strict Unicode characters, but it has
- * to be as big as Perl allows for a single character */
-#define UTF8_MAXBYTES_CASE     UTF8_MAXBYTES
-
 /* A Unicode character can fold to up to 3 characters */
 #define UTF8_MAX_FOLD_CHAR_EXPAND 3
 
@@ -509,6 +525,8 @@ Perl's extended UTF-8 means we can have start bytes up to FF.
 #   define IS_UTF8_CHAR_FAST(n) ((n) <= 4)
 #endif
 
+#endif /* H_UTF8 */
+
 /*
  * Local variables:
  * c-indentation-style: bsd