This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Minor naming change UTF8_IS_ASCII => UTF8_IS_INVARIANT
[perl5.git] / utf8.h
CommitLineData
a0ed51b3
LW
1/* utf8.h
2 *
bc89e66f 3 * Copyright (c) 1998-2001, Larry Wall
a0ed51b3
LW
4 *
5 * You may distribute under the terms of either the GNU General Public
6 * License or the Artistic License, as specified in the README file.
7 *
8 */
9
73c4f7a1
GS
10START_EXTERN_C
11
a0ed51b3 12#ifdef DOINIT
6f06b55f 13EXTCONST unsigned char PL_utf8skip[] = {
a0ed51b3
LW
141,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
151,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
161,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
171,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
181,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
191,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
202,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* scripts */
3c77ea2b
GS
213,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6, /* cjk etc. */
227,13, /* Perl extended (not UTF-8). Up to 72bit allowed (64-bit + reserved). */
a0ed51b3
LW
23};
24#else
6f06b55f 25EXTCONST unsigned char PL_utf8skip[];
a0ed51b3
LW
26#endif
27
73c4f7a1
GS
28END_EXTERN_C
29
aa6ffa16
GS
30#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */
31
fcc8fcf6 32/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */
5bc28da9 33#define IN_BYTE (PL_curcop->op_private & HINT_BYTE)
03cfe0ae
NIS
34#ifdef USE_BYTES_DOWNGRADES
35#define DO_UTF8(sv) (SvUTF8(sv) && !(IN_BYTE && sv_utf8_downgrade(sv,0)))
36#else
7e2040f0 37#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE)
03cfe0ae 38#endif
a0ed51b3 39
a0dbb045
JH
40#define UTF8_ALLOW_EMPTY 0x0001
41#define UTF8_ALLOW_CONTINUATION 0x0002
42#define UTF8_ALLOW_NON_CONTINUATION 0x0004
43#define UTF8_ALLOW_FE_FF 0x0008
44#define UTF8_ALLOW_SHORT 0x0010
45#define UTF8_ALLOW_SURROGATE 0x0020
46#define UTF8_ALLOW_BOM 0x0040
47#define UTF8_ALLOW_FFFF 0x0080
48#define UTF8_ALLOW_LONG 0x0100
49#define UTF8_ALLOW_ANYUV (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\
50 UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\
51 UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG)
fcc8fcf6
JH
52#define UTF8_ALLOW_ANY 0x00ff
53#define UTF8_CHECK_ONLY 0x0100
54
421a8bf2
JH
55#define UNICODE_SURROGATE_FIRST 0xd800
56#define UNICODE_SURROGATE_LAST 0xdfff
57#define UNICODE_REPLACEMENT 0xfffd
58#define UNICODE_BYTER_ORDER_MARK 0xfffe
59#define UNICODE_ILLEGAL 0xffff
60
61#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \
62 (c) <= UNICODE_SURROGATE_LAST)
63#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACMENT)
64#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTER_ORDER_MARK)
65#define UNICODE_IS_ILLEGAL(c) ((c) == UNICODE_ILLEGAL)
66
6f06b55f 67#define UTF8SKIP(s) PL_utf8skip[*(U8*)s]
7e2040f0 68
d7578b48
JH
69#define UTF8_QUAD_MAX UINT64_C(0x1000000000)
70
877d9f0d 71/*
9041c2e3 72
877d9f0d
JH
73 The following table is from Unicode 3.1.
74
75 Code Points 1st Byte 2nd Byte 3rd Byte 4th Byte
76
77 U+0000..U+007F 00..7F   
78 U+0080..U+07FF C2..DF 80..BF   
79 U+0800..U+0FFF E0 A0..BF 80..BF  
80 U+1000..U+FFFF E1..EF 80..BF 80..BF  
81 U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
82 U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
83 U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
84
85 */
86
c512ce4f
JH
87#define UTF8_IS_ASCII(c) (((U8)c) < 0x80)
88#define UTF8_IS_START(c) (((U8)c) >= 0xc0 && (((U8)c) <= 0xfd))
89#define UTF8_IS_CONTINUATION(c) (((U8)c) >= 0x80 && (((U8)c) <= 0xbf))
90#define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80)
db42d148 91#define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfc) == 0xc0)
8850bf83 92
c512ce4f 93#define UTF8_CONTINUATION_MASK ((U8)0x3f)
8850bf83 94#define UTF8_ACCUMULATION_SHIFT 6
7df053ec 95#define UTF8_ACCUMULATE(old, new) ((old) << UTF8_ACCUMULATION_SHIFT | (((U8)new) & UTF8_CONTINUATION_MASK))
421a8bf2 96
7df053ec
JH
97#define UTF8_EIGHT_BIT_HI(c) ( (((U8)(c))>>6) |0xc0)
98#define UTF8_EIGHT_BIT_LO(c) (((((U8)(c)) )&0x3f)|0x80)
c512ce4f 99
1d68d6cd 100#ifdef HAS_QUAD
5bbb0b5a 101#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
1d68d6cd
SC
102 (uv) < 0x800 ? 2 : \
103 (uv) < 0x10000 ? 3 : \
104 (uv) < 0x200000 ? 4 : \
105 (uv) < 0x4000000 ? 5 : \
106 (uv) < 0x80000000 ? 6 : \
9041c2e3 107 (uv) < UTF8_QUAD_MAX ? 7 : 13 )
1d68d6cd
SC
108#else
109/* No, I'm not even going to *TRY* putting #ifdef inside a #define */
5bbb0b5a 110#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
1d68d6cd
SC
111 (uv) < 0x800 ? 2 : \
112 (uv) < 0x10000 ? 3 : \
113 (uv) < 0x200000 ? 4 : \
114 (uv) < 0x4000000 ? 5 : \
115 (uv) < 0x80000000 ? 6 : 7 )
116#endif
117
ba210ebe 118
7e2040f0
GS
119/*
120 * Note: we try to be careful never to call the isXXX_utf8() functions
121 * unless we're pretty sure we've seen the beginning of a UTF-8 character
122 * (that is, the two high bits are set). Otherwise we risk loading in the
123 * heavy-duty SWASHINIT and SWASHGET routines unnecessarily.
124 */
5b154c98
JH
125#ifdef EBCDIC
126#define isIDFIRST_lazy_if(p,c) isIDFIRST(*(p))
127#define isALNUM_lazy_if(p,c) isALNUM(*(p))
128#else
129#define isIDFIRST_lazy_if(p,c) ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \
7e2040f0
GS
130 ? isIDFIRST(*(p)) \
131 : isIDFIRST_utf8((U8*)p))
5b154c98 132#define isALNUM_lazy_if(p,c) ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \
7e2040f0
GS
133 ? isALNUM(*(p)) \
134 : isALNUM_utf8((U8*)p))
5b154c98 135#endif
7e2040f0
GS
136#define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1)
137#define isALNUM_lazy(p) isALNUM_lazy_if(p,1)
3bd709b1 138
9041c2e3 139/* EBCDIC-happy ways of converting native code to UTF8 */
3bd709b1
PP
140
141#ifdef EBCDIC
db42d148
NIS
142#define NATIVE_TO_ASCII(ch) PL_e2a[(ch)&255]
143#define ASCII_TO_NATIVE(ch) PL_a2e[(ch)&255]
144#define NATIVE_TO_UNI(ch) (((ch) > 255) ? (ch) : (UV) PL_e2a[(ch)])
145#define UNI_TO_NATIVE(ch) (((ch) > 255) ? (ch) : (UV) PL_a2e[(ch)])
146#define NATIVE_TO_NEED(enc,ch) ((enc) ? NATIVE_TO_ASCII(ch) : (ch))
147#define ASCII_TO_NEED(enc,ch) ((enc) ? (ch) : ASCII_TO_NATIVE(ch))
3bd709b1 148#else
db42d148
NIS
149#define NATIVE_TO_ASCII(ch) (ch)
150#define ASCII_TO_NATIVE(ch) (ch)
151#define UNI_TO_NATIVE(ch) (ch)
152#define NATIVE_TO_UNI(ch) (ch)
153#define NATIVE_TO_NEED(enc,ch) (ch)
154#define ASCII_TO_NEED(enc,ch) (ch)
3bd709b1
PP
155#endif
156
d742c382 157#define UTF8_IS_INVARIANT(c) UTF8_IS_ASCII(c)