* http://www.unicode.org/unicode/reports/tr16
*
* To summarize, the way it works is:
- * To convert an EBCDIC character to UTF-EBCDIC:
- * 1) convert to Unicode. The table in the generated file 'ebcdic_tables.h'
- * that does this for EBCDIC bytes is PL_e2a (with inverse PL_a2e). The
- * 'a' stands for ASCII platform, meaning latin1.
+ * To convert an EBCDIC code point to UTF-EBCDIC:
+ * 1) convert to Unicode. No conversion is necesary for code points above
+ * 255, as Unicode and EBCDIC are identical in this range. For smaller
+ * code points, the conversion is done by lookup in the PL_e2a table (with
+ * inverse PL_a2e) in the generated file 'ebcdic_tables.h'. The 'a'
+ * stands for ASCII platform, meaning 0-255 Unicode.
* 2) convert that to a utf8-like string called I8 ('I' stands for
* intermediate) with variant characters occupying multiple bytes. This
* step is similar to the utf8-creating step from Unicode, but the details
* invariant byte starts with 0 starts with 0 or 100
* continuation byte starts with 10 starts with 101
* start byte same in both: if the code point requires N bytes,
- * then the leading N bits are 1, followed by a 0. (No
- * trailing 0 for the very largest possible allocation
- * in I8, far beyond the current Unicode standard's
- * max, as shown in the comment later in this file.)
+ * then the leading N bits are 1, followed by a 0. If
+ * all 8 bits in the first byte are 1, the code point
+ * will occupy 14 bytes (compared to 13 in Perl's
+ * extended UTF-8). This is incompatible with what
+ * tr16 implies should be the representation of code
+ * points 2**30 and above, but allows Perl to be able
+ * to represent all code points that fit in a 64-bit
+ * word in either our extended UTF-EBCDIC or UTF-8.
* 3) Use the algorithm in tr16 to convert each byte from step 2 into
* final UTF-EBCDIC. This is done by table lookup from a table
* constructed from the algorithm, reproduced in ebcdic_tables.h as
* PL_utf2e, with its inverse being PL_e2utf. They are constructed so that
* all EBCDIC invariants remain invariant, but no others do, and the first
* byte of a variant will always have its upper bit set. But note that
- * the upper bit of some invariants is also 1.
+ * the upper bit of some invariants is also 1. The table also is designed
+ * so that lexically comparing two UTF-EBCDIC-variant characters yields
+ * the Unicode code point order. (To get native code point order, one has
+ * to convert the latin1-range characters to their native code point
+ * value.)
*
* For example, the ordinal value of 'A' is 193 in EBCDIC, and also is 193 in
* UTF-EBCDIC. Step 1) converts it to 65, Step 2 leaves it at 65, and Step 3
/* EBCDIC-happy ways of converting native code to UTF-8 */
-#define NATIVE_TO_LATIN1(ch) PL_e2a[(U8)(ch)]
-#define LATIN1_TO_NATIVE(ch) PL_a2e[(U8)(ch)]
+/* Use these when ch is known to be < 256 */
+#define NATIVE_TO_LATIN1(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) PL_e2a[(U8)(ch)])
+#define LATIN1_TO_NATIVE(ch) (__ASSERT_(FITS_IN_8_BITS(ch)) PL_a2e[(U8)(ch)])
-#define NATIVE_UTF8_TO_I8(ch) PL_e2utf[(U8)(ch)]
-#define I8_TO_NATIVE_UTF8(ch) PL_utf2e[(U8)(ch)]
+/* Use these on bytes */
+#define NATIVE_UTF8_TO_I8(b) (__ASSERT_(FITS_IN_8_BITS(b)) PL_e2utf[(U8)(b)])
+#define I8_TO_NATIVE_UTF8(b) (__ASSERT_(FITS_IN_8_BITS(b)) PL_utf2e[(U8)(b)])
/* Transforms in wide UV chars */
-#define NATIVE_TO_UNI(ch) (((ch) > 255) ? (ch) : NATIVE_TO_LATIN1(ch))
-#define UNI_TO_NATIVE(ch) (((ch) > 255) ? (ch) : LATIN1_TO_NATIVE(ch))
+#define NATIVE_TO_UNI(ch) (FITS_IN_8_BITS(ch) ? NATIVE_TO_LATIN1(ch) : (UV) (ch))
+#define UNI_TO_NATIVE(ch) (FITS_IN_8_BITS(ch) ? LATIN1_TO_NATIVE(ch) : (UV) (ch))
+
+/* How wide can a single UTF-8 encoded character become in bytes. */
+/* NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8
+ * is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be
+ * expressed with 5 bytes. However, Perl thinks of UTF-8 as a way to encode
+ * non-negative integers in a binary format, even those above Unicode. 14 is
+ * the smallest number that covers 2**64
+ *
+ * WARNING: This number must be in sync with the value in
+ * regen/charset_translations.pl. */
+#define UTF8_MAXBYTES 14
/*
- The following table is adapted from tr16, it shows I8 encoding of Unicode code points.
+ The following table is adapted from tr16, it shows the I8 encoding of Unicode code points.
- Unicode U32 Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th byte
+ Unicode U32 Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th Byte
U+0000..U+007F 000000000xxxxxxx 0xxxxxxx
U+0080..U+009F 00000000100xxxxx 100xxxxx
U+00A0..U+03FF 000000yyyyyxxxxx 110yyyyy 101xxxxx
U+4000..U+3FFFF 0wwwzzzzzyyyyyxxxxx 11110www 101zzzzz 101yyyyy 101xxxxx
U+40000..U+3FFFFF 0vvwwwwwzzzzzyyyyyxxxxx 111110vv 101wwwww 101zzzzz 101yyyyy 101xxxxx
U+400000..U+3FFFFFF 0uvvvvvwwwwwzzzzzyyyyyxxxxx 1111110u 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx
- U+4000000..U+7FFFFFFF 0tuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 1111111t 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx
+ U+4000000..U+3FFFFFFF 00uuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 11111110 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx
- Note: The I8 transformation is valid for UCS-4 values X'0' to
- X'7FFFFFFF' (the full extent of ISO/IEC 10646 coding space).
+Beyond this, Perl uses an incompatible extension, similar to the one used in
+regular UTF-8. There are now 14 bytes. A full 32 bits of information thus looks like this:
+ 1st Byte 2nd-7th 8th Byte 9th Byte 10th B 11th B 12th B 13th B 14th B
+U+40000000..U+FFFFFFFF ttuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 11111111 10100000 101000tt 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx
- */
+For 32-bit words, the 2nd through 7th bytes effectively function as leading
+zeros. Above 32 bits, these fill up, with each byte yielding 5 bits of
+information, so that with 13 continuation bytes, we can handle 65 bits, just
+above what a 64 bit word can hold */
-/* Input is a true Unicode (not-native) code point */
-#define OFFUNISKIP(uv) ( (uv) < 0xA0 ? 1 : \
- (uv) < 0x400 ? 2 : \
- (uv) < 0x4000 ? 3 : \
- (uv) < 0x40000 ? 4 : \
- (uv) < 0x400000 ? 5 : \
- (uv) < 0x4000000 ? 6 : 7 )
+/* This is a fundamental property of UTF-EBCDIC */
+#define OFFUNI_IS_INVARIANT(c) (((UV)(c)) < 0xA0)
-#define UNI_IS_INVARIANT(c) (((UV)(c)) < 0xA0)
+/* It turns out that on EBCDIC platforms, the invariants are the characters
+ * that have ASCII equivalents, plus the C1 controls. Since the C0 controls
+ * and DELETE are ASCII, this is the same as: (isASCII(uv) || isCNTRL_L1(uv))
+ * */
+#define UVCHR_IS_INVARIANT(uv) cBOOL(FITS_IN_8_BITS(uv) \
+ && (PL_charclass[(U8) (uv)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL))))
-/* UTF-EBCDIC semantic macros - transform back into I8 and then compare
+/* UTF-EBCDIC semantic macros - We used to transform back into I8 and then
+ * compare, but now only have to do a single lookup by using a bit in
+ * l1_char_class_tab.h.
* Comments as to the meaning of each are given at their corresponding utf8.h
* definitions. */
-#define UTF8_IS_START(c) (NATIVE_UTF8_TO_I8(c) >= 0xC5 \
- && NATIVE_UTF8_TO_I8(c) != 0xE0)
-#define UTF8_IS_CONTINUATION(c) ((NATIVE_UTF8_TO_I8(c) & 0xE0) == 0xA0)
-#define UTF8_IS_CONTINUED(c) (NATIVE_UTF8_TO_I8(c) >= 0xA0)
+#define UTF8_IS_START(c) _generic_isCC(c, _CC_UTF8_IS_START)
-#define UTF8_IS_DOWNGRADEABLE_START(c) (NATIVE_UTF8_TO_I8(c) >= 0xC5 \
- && NATIVE_UTF8_TO_I8(c) <= 0xC7)
-/* Saying it this way adds a runtime test, but removes 2 run-time lookups */
-/*#define UTF8_IS_DOWNGRADEABLE_START(c) ((c) == I8_TO_NATIVE_UTF8(0xC5) \
- || (c) == I8_TO_NATIVE_UTF8(0xC6) \
- || (c) == I8_TO_NATIVE_UTF8(0xC7))
-*/
-#define UTF8_IS_ABOVE_LATIN1(c) (NATIVE_UTF8_TO_I8(c) >= 0xC8)
+#define UTF_IS_CONTINUATION_MASK 0xE0
-/* Can't exceed 7 on EBCDIC platforms */
-#define UTF_START_MARK(len) (0xFF & (0xFE << (7-(len))))
+#define UTF8_IS_CONTINUATION(c) _generic_isCC(c, _CC_UTF8_IS_CONTINUATION)
-#define UTF_START_MASK(len) (((len) >= 6) ? 0x01 : (0x1F >> ((len)-2)))
-#define UTF_CONTINUATION_MARK 0xA0
-#define UTF_CONTINUATION_MASK ((U8)0x1f)
-#define UTF_ACCUMULATION_SHIFT 5
+/* The above instead could be written as this:
+#define UTF8_IS_CONTINUATION(c) \
+ (((NATIVE_UTF8_TO_I8(c) & UTF_IS_CONTINUATION_MASK) \
+ == UTF_CONTINUATION_MARK)
+ */
-/* How wide can a single UTF-8 encoded character become in bytes. */
-/* NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 since UTF-8
- * is an encoding of Unicode, and Unicode's upper limit, 0x10FFFF, can be
- * expressed with 5 bytes. However, Perl thinks of UTF-8 as a way to encode
- * non-negative integers in a binary format, even those above Unicode */
-#define UTF8_MAXBYTES 7
+/* Equivalent to ! UVCHR_IS_INVARIANT(c) */
+#define UTF8_IS_CONTINUED(c) cBOOL(FITS_IN_8_BITS(c) \
+ && ! (PL_charclass[(U8) (c)] & (_CC_mask(_CC_ASCII) | _CC_mask(_CC_CNTRL))))
+
+#define UTF8_IS_DOWNGRADEABLE_START(c) _generic_isCC(c, \
+ _CC_UTF8_IS_DOWNGRADEABLE_START)
-/* The maximum number of UTF-8 bytes a single Unicode character can
- * uppercase/lowercase/fold into. Unicode guarantees that the maximum
- * expansion is 3 characters. On EBCDIC platforms, the highest Unicode
- * character occupies 5 bytes, therefore this number is 15 */
-#define UTF8_MAXBYTES_CASE 15
+/* Equivalent to (UTF8_IS_START(c) && ! UTF8_IS_DOWNGRADEABLE_START(c))
+ * Makes sure that the START bit is set and the DOWNGRADEABLE bit isn't */
+#define UTF8_IS_ABOVE_LATIN1(c) cBOOL(FITS_IN_8_BITS(c) \
+ && ((PL_charclass[(U8) (c)] & ( _CC_mask(_CC_UTF8_IS_START) \
+ |_CC_mask(_CC_UTF8_IS_DOWNGRADEABLE_START))) \
+ == _CC_mask(_CC_UTF8_IS_START)))
+
+#define isUTF8_POSSIBLY_PROBLEMATIC(c) \
+ _generic_isCC(c, _CC_UTF8_START_BYTE_IS_FOR_AT_LEAST_SURROGATE)
+
+#define UTF_CONTINUATION_MARK 0xA0
+#define UTF_ACCUMULATION_SHIFT 5
/* ^? is defined to be APC on EBCDIC systems. See the definition of toCTRL()
* for more */
#define QUESTION_MARK_CTRL LATIN1_TO_NATIVE(0x9F)
-#define MAX_UTF8_TWO_BYTE 0x3FF
-
/*
* ex: set ts=8 sts=4 sw=4 et:
*/
https://perl5.git.perl.org / perl5.git / blobdiff