*
*/
+#ifndef H_UTF8 /* Guard against recursive inclusion */
+#define H_UTF8 1
+
/* Use UTF-8 as the default script encoding?
* Turning this on will break scripts having non-UTF-8 binary
* data (such as Latin-1) in string literals. */
#define UTF8_IS_CONTINUATION(c) ((((U8)c) & 0xC0) == 0x80)
#define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80)
-/* Masking with 0xfe allows low bit to be 0 or 1; thus this matches 0xc[23] */
+/* Use UTF8_IS_NEXT_CHAR_DOWNGRADEABLE() instead if the input isn't known to
+ * be well-formed. Masking with 0xfe allows low bit to be 0 or 1; thus this
+ * matches 0xc[23]. */
#define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfe) == 0xc2)
+#define UTF8_IS_ABOVE_LATIN1(c) ((U8)(c) >= 0xc4)
+
#define UTF_START_MARK(len) (((len) > 7) ? 0xFF : (0xFE << (7-(len))))
/* Masks out the initial one bits in a start byte, leaving the real data ones.
(uv) < 0x80000000 ? 6 : 7 )
#endif
+/* How wide can a single UTF-8 encoded character become in bytes. */
+/* NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8
+ * is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be
+ * expressed with 4 bytes. However, Perl thinks of UTF-8 as a way to encode
+ * non-negative integers in a binary format, even those above Unicode */
+#define UTF8_MAXBYTES 13
+
+/* The maximum number of UTF-8 bytes a single Unicode character can
+ * uppercase/lowercase/fold into. Unicode guarantees that the maximum
+ * expansion is 3 characters. On ASCIIish platforms, the highest Unicode
+ * character occupies 4 bytes, therefore this number would be 12, but this is
+ * smaller than the maximum width a single above-Unicode character can occupy,
+ * so use that instead */
+#if UTF8_MAXBYTES < 12
+#error UTF8_MAXBYTES must be at least 12
+#endif
+
+#define UTF8_MAXBYTES_CASE UTF8_MAXBYTES
+
#endif /* EBCDIC vs ASCII */
/* Rest of these are attributes of Unicode and perl's internals rather than the
#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) \
| (((U8)new) & UTF_CONTINUATION_MASK))
+/* This works in the face of malformed UTF-8. */
+#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) (UTF8_IS_DOWNGRADEABLE_START(*s) \
+ && ( (e) - (s) > 1) \
+ && UTF8_IS_CONTINUATION(*((s)+1)))
+
/* Convert a two (not one) byte utf8 character to a unicode code point value.
* Needs just one iteration of accumulate. Should not be used unless it is
* known that the two bytes are legal: 1) two-byte start, and 2) continuation.
#define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1)
#define isALNUM_lazy(p) isALNUM_lazy_if(p,1)
-#define UTF8_MAXBYTES 13
-/* How wide can a single UTF-8 encoded character become in bytes.
- * NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8
- * since UTF-8 is an encoding of Unicode and given Unicode's current
- * upper limit only four bytes is possible. Perl thinks of UTF-8
- * as a way to encode non-negative integers in a binary format. */
#define UTF8_MAXLEN UTF8_MAXBYTES
-/* The maximum number of UTF-8 bytes a single Unicode character can
- * uppercase/lowercase/fold into; this number depends on the Unicode
- * version. An example of maximal expansion is the U+03B0 which
- * uppercases to U+03C5 U+0308 U+0301. The Unicode databases that
- * tell these things are UnicodeData.txt, CaseFolding.txt, and
- * SpecialCasing.txt. The value is 6 for strict Unicode characters, but it has
- * to be as big as Perl allows for a single character */
-#define UTF8_MAXBYTES_CASE UTF8_MAXBYTES
-
/* A Unicode character can fold to up to 3 characters */
#define UTF8_MAX_FOLD_CHAR_EXPAND 3
toLOWER((input)[1]) == 's')
#define SHARP_S_SKIP 2
-#ifndef EBCDIC
/* If you want to exclude surrogates, and beyond legal Unicode, see the blame
* log for earlier versions which gave details for these */
-# define IS_UTF8_CHAR_1(p) \
- ((p)[0] <= 0x7F)
-# define IS_UTF8_CHAR_2(p) \
- ((p)[0] >= 0xC2 && (p)[0] <= 0xDF && \
- (p)[1] >= 0x80 && (p)[1] <= 0xBF)
-# define IS_UTF8_CHAR_3a(p) \
- ((p)[0] == 0xE0 && \
- (p)[1] >= 0xA0 && (p)[1] <= 0xBF && \
- (p)[2] >= 0x80 && (p)[2] <= 0xBF)
-# define IS_UTF8_CHAR_3b(p) \
- ((p)[0] >= 0xE1 && (p)[0] <= 0xEF && \
- (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
- (p)[2] >= 0x80 && (p)[2] <= 0xBF)
-# define IS_UTF8_CHAR_4a(p) \
- ((p)[0] == 0xF0 && \
- (p)[1] >= 0x90 && (p)[1] <= 0xBF && \
- (p)[2] >= 0x80 && (p)[2] <= 0xBF && \
- (p)[3] >= 0x80 && (p)[3] <= 0xBF)
-/* The 0xF7 allows us to go to 0x1fffff (0x200000 would
- * require five bytes). Not doing any further code points
- * since that is not needed (and that would not be strict
- * UTF-8, anyway). The "slow path" in Perl_is_utf8_char()
- * will take care of the "extended UTF-8". */
-# define IS_UTF8_CHAR_4b(p) \
- ((p)[0] >= 0xF1 && (p)[0] <= 0xF7 && \
- (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
- (p)[2] >= 0x80 && (p)[2] <= 0xBF && \
- (p)[3] >= 0x80 && (p)[3] <= 0xBF)
-
-# define IS_UTF8_CHAR_3(p) \
- (IS_UTF8_CHAR_3a(p) || \
- IS_UTF8_CHAR_3b(p))
-# define IS_UTF8_CHAR_4(p) \
- (IS_UTF8_CHAR_4a(p) || \
- IS_UTF8_CHAR_4b(p))
+
+#ifndef EBCDIC
+/* This was generated by regen/regcharclass.pl, and then moved here. The lines
+ * that generated it were then commented out. This was done solely because it
+ * takes on the order of 10 minutes to generate, and is never going to change.
+ * The EBCDIC equivalent hasn't been commented out in regcharclass.pl, so it
+ * should generate and run the correct stuff */
+/*** GENERATED CODE ***/
+#define is_UTF8_CHAR_utf8_safe(s,e) \
+( ((e)-(s) > 3) ? \
+ ( ( ( ((U8*)s)[0] & 0x80 ) == 0x00 ) ? 1 \
+ : ( 0xC2 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xDF ) ? \
+ ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \
+ : ( 0xE0 == ((U8*)s)[0] ) ? \
+ ( ( ( ( ((U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
+ : ( 0xE1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xEF ) ? \
+ ( ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
+ : ( 0xF0 == ((U8*)s)[0] ) ? \
+ ( ( ( ( 0x90 <= ((U8*)s)[1] && ((U8*)s)[1] <= 0xBF ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
+ : ( 0xF1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xF7 ) ? \
+ ( ( ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) && ( ( ((U8*)s)[3] & 0xC0 ) == 0x80 ) ) ? 4 : 0 )\
+ : 0 ) \
+: ((e)-(s) > 2) ? \
+ ( ( ( ((U8*)s)[0] & 0x80 ) == 0x00 ) ? 1 \
+ : ( 0xC2 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xDF ) ? \
+ ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \
+ : ( 0xE0 == ((U8*)s)[0] ) ? \
+ ( ( ( ( ((U8*)s)[1] & 0xE0 ) == 0xA0 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
+ : ( 0xE1 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xEF ) ? \
+ ( ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) && ( ( ((U8*)s)[2] & 0xC0 ) == 0x80 ) ) ? 3 : 0 )\
+ : 0 ) \
+: ((e)-(s) > 1) ? \
+ ( ( ( ((U8*)s)[0] & 0x80 ) == 0x00 ) ? 1 \
+ : ( 0xC2 <= ((U8*)s)[0] && ((U8*)s)[0] <= 0xDF ) ? \
+ ( ( ( ((U8*)s)[1] & 0xC0 ) == 0x80 ) ? 2 : 0 ) \
+ : 0 ) \
+: ((e)-(s) > 0) ? \
+ ( ( ((U8*)s)[0] & 0x80 ) == 0x00 ) \
+: 0 )
+#endif
/* IS_UTF8_CHAR(p) is strictly speaking wrong (not UTF-8) because it
* (1) allows UTF-8 encoded UTF-16 surrogates
* (2) it allows code points past U+10FFFF.
* The Perl_is_utf8_char() full "slow" code will handle the Perl
* "extended UTF-8". */
-# define IS_UTF8_CHAR(p, n) \
- ((n) == 1 ? IS_UTF8_CHAR_1(p) : \
- (n) == 2 ? IS_UTF8_CHAR_2(p) : \
- (n) == 3 ? IS_UTF8_CHAR_3(p) : \
- (n) == 4 ? IS_UTF8_CHAR_4(p) : 0)
-
-# define IS_UTF8_CHAR_FAST(n) ((n) <= 4)
-
-#else /* EBCDIC */
-
-/* This is an attempt to port IS_UTF8_CHAR to EBCDIC based on eyeballing.
- * untested. If want to exclude surrogates and above-Unicode, see the
- * definitions for UTF8_IS_SURROGATE and UTF8_IS_SUPER */
-# define IS_UTF8_CHAR_1(p) \
- (NATIVE_TO_ASCII((p)[0]) <= 0x9F)
-# define IS_UTF8_CHAR_2(p) \
- (NATIVE_TO_I8((p)[0]) >= 0xC5 && NATIVE_TO_I8((p)[0]) <= 0xDF && \
- NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF)
-# define IS_UTF8_CHAR_3(p) \
- (NATIVE_TO_I8((p)[0]) == 0xE1 && NATIVE_TO_I8((p)[1]) <= 0xEF && \
- NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF)
-# define IS_UTF8_CHAR_4a(p) \
- (NATIVE_TO_I8((p)[0]) == 0xF0 && \
- NATIVE_TO_I8((p)[1]) >= 0xB0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
- NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
-# define IS_UTF8_CHAR_4b(p) \
- (NATIVE_TO_I8((p)[0]) >= 0xF1 && NATIVE_TO_I8((p)[0]) <= 0xF7 && \
- NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
- NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
-# define IS_UTF8_CHAR_5a(p) \
- (NATIVE_TO_I8((p)[0]) == 0xF8 && \
- NATIVE_TO_I8((p)[1]) >= 0xA8 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
- NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
-# define IS_UTF8_CHAR_5b(p) \
- (NATIVE_TO_I8((p)[0]) >= 0xF9 && NATIVE_TO_I8((p)[1]) <= 0xFB && \
- NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
- NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
- NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
-
-# define IS_UTF8_CHAR_4(p) \
- (IS_UTF8_CHAR_4a(p) || \
- IS_UTF8_CHAR_4b(p))
-# define IS_UTF8_CHAR_5(p) \
- (IS_UTF8_CHAR_5a(p) || \
- IS_UTF8_CHAR_5b(p))
-# define IS_UTF8_CHAR(p, n) \
- ((n) == 1 ? IS_UTF8_CHAR_1(p) : \
- (n) == 2 ? IS_UTF8_CHAR_2(p) : \
- (n) == 3 ? IS_UTF8_CHAR_3(p) : \
- (n) == 4 ? IS_UTF8_CHAR_4(p) : \
- (n) == 5 ? IS_UTF8_CHAR_5(p) : 0)
+#define IS_UTF8_CHAR(p, n) (is_UTF8_CHAR_utf8_safe(p, (p) + (n)) == n)
+/* regen/regcharclass.pl generates is_UTF8_CHAR_utf8_safe() macros for up to
+ * these number of bytes. So this has to be coordinated with it */
+#ifdef EBCDIC
# define IS_UTF8_CHAR_FAST(n) ((n) <= 5)
+#else
+# define IS_UTF8_CHAR_FAST(n) ((n) <= 4)
+#endif
-#endif /* IS_UTF8_CHAR() for UTF-8 */
+#endif /* H_UTF8 */
/*
* Local variables:
https://perl5.git.perl.org / perl5.git / blobdiff