X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/375122d71fe65bb12f4aac78313fa2d23fd511f3..3fb2f3ac5c8ed8e4fdad7a9dbbce30a09d4f93f7:/utf8.h diff --git a/utf8.h b/utf8.h index a5312ca..5ffee5b 100644 --- a/utf8.h +++ b/utf8.h @@ -1,6 +1,6 @@ /* utf8.h * - * Copyright (c) 1998-2002, Larry Wall + * Copyright (C) 2000, 2001, 2002, 2005, 2006 by Larry Wall and others * * You may distribute under the terms of either the GNU General Public * License or the Artistic License, as specified in the README file. @@ -8,7 +8,7 @@ */ /* Use UTF-8 as the default script encoding? - * Turning this on will break scripts having non-UTF8 binary + * Turning this on will break scripts having non-UTF-8 binary * data (such as Latin-1) in string literals. */ #ifdef USE_UTF8_SCRIPTS # define USE_UTF8_IN_NAMES (!IN_BYTES) @@ -16,12 +16,17 @@ # define USE_UTF8_IN_NAMES (PL_hints & HINT_UTF8) #endif +/* Source backward compatibility. */ +#define uvuni_to_utf8(d, uv) uvuni_to_utf8_flags(d, uv, 0) +#define is_utf8_string_loc(s, len, ep) is_utf8_string_loclen(s, len, ep, 0) + #ifdef EBCDIC /* The equivalent of these macros but implementing UTF-EBCDIC are in the following header file: */ #include "utfebcdic.h" + #else START_EXTERN_C @@ -42,7 +47,7 @@ EXTCONST unsigned char PL_utf8skip[]; #endif END_EXTERN_C -#define UTF8SKIP(s) PL_utf8skip[*(U8*)s] +#define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)] /* Native character to iso-8859-1 */ #define NATIVE_TO_ASCII(ch) (ch) @@ -58,8 +63,8 @@ END_EXTERN_C #define ASCII_TO_NEED(enc,ch) (ch) /* As there are no translations avoid the function wrapper */ -#define Perl_utf8n_to_uvchr Perl_utf8n_to_uvuni -#define Perl_uvchr_to_utf8 Perl_uvuni_to_utf8 +#define utf8n_to_uvchr utf8n_to_uvuni +#define uvchr_to_utf8 uvuni_to_utf8 /* @@ -80,6 +85,10 @@ END_EXTERN_C Note the A0..BF in U+0800..U+0FFF, the 80..9F in U+D000...U+D7FF, the 90..BF in U+10000..U+3FFFF, and the 80...8F in U+100000..U+10FFFF. +The "gaps" are caused by legal UTF-8 avoiding non-shortest encodings: +it is technically possible to UTF-8-encode a single code point in different +ways, but that is explicitly forbidden, and the shortest possible encoding +should always be used (and that is what Perl does). */ @@ -141,14 +150,14 @@ encoded character. * Note: we try to be careful never to call the isXXX_utf8() functions * unless we're pretty sure we've seen the beginning of a UTF-8 character * (that is, the two high bits are set). Otherwise we risk loading in the - * heavy-duty SWASHINIT and SWASHGET routines unnecessarily. + * heavy-duty swash_init and swash_fetch routines unnecessarily. */ -#define isIDFIRST_lazy_if(p,c) ((IN_BYTES || (!c || (*((U8*)p) < 0xc0))) \ +#define isIDFIRST_lazy_if(p,c) ((IN_BYTES || (!c || (*((const U8*)p) < 0xc0))) \ ? isIDFIRST(*(p)) \ - : isIDFIRST_utf8((U8*)p)) -#define isALNUM_lazy_if(p,c) ((IN_BYTES || (!c || (*((U8*)p) < 0xc0))) \ + : isIDFIRST_utf8((const U8*)p)) +#define isALNUM_lazy_if(p,c) ((IN_BYTES || (!c || (*((const U8*)p) < 0xc0))) \ ? isALNUM(*(p)) \ - : isALNUM_utf8((U8*)p)) + : isALNUM_utf8((const U8*)p)) #endif /* EBCDIC vs ASCII */ @@ -158,54 +167,65 @@ encoded character. #define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1) #define isALNUM_lazy(p) isALNUM_lazy_if(p,1) -/* how wide can a single UTF8 encoded character become */ -#define UTF8_MAXLEN 13 -/* how wide a character can become when upper/lowercased */ -#define UTF8_MAXLEN_UCLC_MULT 3 -#define UTF8_MAXLEN_UCLC (UTF8_MAXLEN*UTF8_MAXLEN_UCLC_MULT) -/* how wide a character can become when casefolded */ -#define UTF8_MAXLEN_FOLD_MULT 3 -#define UTF8_MAXLEN_FOLD (UTF8_MAXLEN*UTF8_MAXLEN_FOLD_MULT) - -#define IN_BYTES (PL_curcop->op_private & HINT_BYTES) +#define UTF8_MAXBYTES 13 +/* How wide can a single UTF-8 encoded character become in bytes. + * NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8 + * since UTF-8 is an encoding of Unicode and given Unicode's current + * upper limit only four bytes is possible. Perl thinks of UTF-8 + * as a way to encode non-negative integers in a binary format. */ +#define UTF8_MAXLEN UTF8_MAXBYTES + +#define UTF8_MAXLEN_UCLC 3 /* Obsolete, do not use. */ +#define UTF8_MAXLEN_UCLC_MULT 39 /* Obsolete, do not use. */ +#define UTF8_MAXLEN_FOLD 3 /* Obsolete, do not use. */ +#define UTF8_MAXLEN_FOLD_MULT 39 /* Obsolete, do not use. */ + +/* The maximum number of UTF-8 bytes a single Unicode character can + * uppercase/lowercase/fold into; this number depends on the Unicode + * version. An example of maximal expansion is the U+03B0 which + * uppercases to U+03C5 U+0308 U+0301. The Unicode databases that + * tell these things are UnicodeDatabase.txt, CaseFolding.txt, and + * SpecialCasing.txt. */ +#define UTF8_MAXBYTES_CASE 6 + +#define IN_BYTES (CopHINTS_get(PL_curcop) & HINT_BYTES) #define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTES) #define UTF8_ALLOW_EMPTY 0x0001 #define UTF8_ALLOW_CONTINUATION 0x0002 #define UTF8_ALLOW_NON_CONTINUATION 0x0004 -#define UTF8_ALLOW_FE_FF 0x0008 +#define UTF8_ALLOW_FE_FF 0x0008 /* Allow above 0x7fffFFFF */ #define UTF8_ALLOW_SHORT 0x0010 #define UTF8_ALLOW_SURROGATE 0x0020 -#define UTF8_ALLOW_BOM 0x0040 -#define UTF8_ALLOW_FFFF 0x0080 -#define UTF8_ALLOW_LONG 0x0100 +#define UTF8_ALLOW_FFFF 0x0040 /* Allow UNICODE_ILLEGAL */ +#define UTF8_ALLOW_LONG 0x0080 #define UTF8_ALLOW_ANYUV (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\ - UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\ - UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG) -#define UTF8_ALLOW_ANY 0x00ff + UTF8_ALLOW_SURROGATE|UTF8_ALLOW_FFFF) +#define UTF8_ALLOW_ANY 0x00FF #define UTF8_CHECK_ONLY 0x0200 +#define UTF8_ALLOW_DEFAULT (ckWARN(WARN_UTF8) ? 0 : \ + UTF8_ALLOW_ANYUV) -#define UNICODE_SURROGATE_FIRST 0xd800 -#define UNICODE_SURROGATE_LAST 0xdfff -#define UNICODE_REPLACEMENT 0xfffd -#define UNICODE_BYTER_ORDER_MARK 0xfffe -#define UNICODE_ILLEGAL 0xffff +#define UNICODE_SURROGATE_FIRST 0xD800 +#define UNICODE_SURROGATE_LAST 0xDFFF +#define UNICODE_REPLACEMENT 0xFFFD +#define UNICODE_BYTE_ORDER_MARK 0xFEFF +#define UNICODE_ILLEGAL 0xFFFF /* Though our UTF-8 encoding can go beyond this, - * let's be conservative. */ + * let's be conservative and do as Unicode 3.2 says. */ #define PERL_UNICODE_MAX 0x10FFFF #define UNICODE_ALLOW_SURROGATE 0x0001 /* Allow UTF-16 surrogates (EVIL) */ #define UNICODE_ALLOW_FDD0 0x0002 /* Allow the U+FDD0...U+FDEF */ -#define UNICODE_ALLOW_FFFE 0x0004 /* Allow 0xFFFE, 0x1FFFE, ... */ -#define UNICODE_ALLOW_FFFF 0x0008 /* Allow 0xFFFE, 0x1FFFE, ... */ -#define UNICODE_ALLOW_SUPER 0x0010 /* Allow past 10xFFFF */ -#define UNICODE_ALLOW_ANY 0xFFFF +#define UNICODE_ALLOW_FFFF 0x0004 /* Allow U+FFF[EF], U+1FFF[EF], ... */ +#define UNICODE_ALLOW_SUPER 0x0008 /* Allow past 0x10FFFF */ +#define UNICODE_ALLOW_ANY 0x000F #define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \ (c) <= UNICODE_SURROGATE_LAST) #define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACEMENT) -#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTER_ORDER_MARK) +#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTE_ORDER_MARK) #define UNICODE_IS_ILLEGAL(c) ((c) == UNICODE_ILLEGAL) #ifdef HAS_QUAD @@ -244,3 +264,79 @@ encoded character. toLOWER((input)[1]) == 's') #endif #define SHARP_S_SKIP 2 + +#ifdef EBCDIC +/* IS_UTF8_CHAR() is not ported to EBCDIC */ +#else +#define IS_UTF8_CHAR_1(p) \ + ((p)[0] <= 0x7F) +#define IS_UTF8_CHAR_2(p) \ + ((p)[0] >= 0xC2 && (p)[0] <= 0xDF && \ + (p)[1] >= 0x80 && (p)[1] <= 0xBF) +#define IS_UTF8_CHAR_3a(p) \ + ((p)[0] == 0xE0 && \ + (p)[1] >= 0xA0 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF) +#define IS_UTF8_CHAR_3b(p) \ + ((p)[0] >= 0xE1 && (p)[0] <= 0xEC && \ + (p)[1] >= 0x80 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF) +#define IS_UTF8_CHAR_3c(p) \ + ((p)[0] == 0xED && \ + (p)[1] >= 0x80 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF) +/* In IS_UTF8_CHAR_3c(p) one could use + * (p)[1] >= 0x80 && (p)[1] <= 0x9F + * if one wanted to exclude surrogates. */ +#define IS_UTF8_CHAR_3d(p) \ + ((p)[0] >= 0xEE && (p)[0] <= 0xEF && \ + (p)[1] >= 0x80 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF) +#define IS_UTF8_CHAR_4a(p) \ + ((p)[0] == 0xF0 && \ + (p)[1] >= 0x90 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF && \ + (p)[3] >= 0x80 && (p)[3] <= 0xBF) +#define IS_UTF8_CHAR_4b(p) \ + ((p)[0] >= 0xF1 && (p)[0] <= 0xF3 && \ + (p)[1] >= 0x80 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF && \ + (p)[3] >= 0x80 && (p)[3] <= 0xBF) +/* In IS_UTF8_CHAR_4c(p) one could use + * (p)[0] == 0xF4 + * if one wanted to stop at the Unicode limit U+10FFFF. + * The 0xF7 allows us to go to 0x1fffff (0x200000 would + * require five bytes). Not doing any further code points + * since that is not needed (and that would not be strict + * UTF-8, anyway). The "slow path" in Perl_is_utf8_char() + * will take care of the "extended UTF-8". */ +#define IS_UTF8_CHAR_4c(p) \ + ((p)[0] == 0xF4 && (p)[0] <= 0xF7 && \ + (p)[1] >= 0x80 && (p)[1] <= 0xBF && \ + (p)[2] >= 0x80 && (p)[2] <= 0xBF && \ + (p)[3] >= 0x80 && (p)[3] <= 0xBF) + +#define IS_UTF8_CHAR_3(p) \ + (IS_UTF8_CHAR_3a(p) || \ + IS_UTF8_CHAR_3b(p) || \ + IS_UTF8_CHAR_3c(p) || \ + IS_UTF8_CHAR_3d(p)) +#define IS_UTF8_CHAR_4(p) \ + (IS_UTF8_CHAR_4a(p) || \ + IS_UTF8_CHAR_4b(p) || \ + IS_UTF8_CHAR_4c(p)) + +/* IS_UTF8_CHAR(p) is strictly speaking wrong (not UTF-8) because it + * (1) allows UTF-8 encoded UTF-16 surrogates + * (2) it allows code points past U+10FFFF. + * The Perl_is_utf8_char() full "slow" code will handle the Perl + * "extended UTF-8". */ +#define IS_UTF8_CHAR(p, n) \ + ((n) == 1 ? IS_UTF8_CHAR_1(p) : \ + (n) == 2 ? IS_UTF8_CHAR_2(p) : \ + (n) == 3 ? IS_UTF8_CHAR_3(p) : \ + (n) == 4 ? IS_UTF8_CHAR_4(p) : 0) + +#define IS_UTF8_CHAR_FAST(n) ((n) <= 4) + +#endif /* IS_UTF8_CHAR() for UTF-8 */