This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
utf8.h: Comment improvementes, white-space
[perl5.git] / utf8.h
CommitLineData
a0ed51b3
LW
1/* utf8.h
2 *
2eee27d7
SS
3 * Copyright (C) 2000, 2001, 2002, 2005, 2006, 2007, 2009,
4 * 2010, 2011 by Larry Wall and others
a0ed51b3
LW
5 *
6 * You may distribute under the terms of either the GNU General Public
7 * License or the Artistic License, as specified in the README file.
8 *
9 */
10
39e02b42 11/* Use UTF-8 as the default script encoding?
1e54db1a 12 * Turning this on will break scripts having non-UTF-8 binary
39e02b42
JH
13 * data (such as Latin-1) in string literals. */
14#ifdef USE_UTF8_SCRIPTS
15# define USE_UTF8_IN_NAMES (!IN_BYTES)
16#else
17# define USE_UTF8_IN_NAMES (PL_hints & HINT_UTF8)
18#endif
19
051a06d4
KW
20/* For to_utf8_fold_flags, q.v. */
21#define FOLD_FLAGS_LOCALE 0x1
22#define FOLD_FLAGS_FULL 0x2
23
36bb2ab6 24#define to_uni_fold(c, p, lenp) _to_uni_fold_flags(c, p, lenp, 1)
051a06d4
KW
25#define to_utf8_fold(c, p, lenp) _to_utf8_fold_flags(c, p, lenp, \
26 FOLD_FLAGS_FULL, NULL)
27#define to_utf8_lower(a,b,c) _to_utf8_lower_flags(a,b,c,0, NULL)
28#define to_utf8_upper(a,b,c) _to_utf8_upper_flags(a,b,c,0, NULL)
29#define to_utf8_title(a,b,c) _to_utf8_title_flags(a,b,c,0, NULL)
36bb2ab6 30
fd7cb289
RGS
31/* Source backward compatibility. */
32#define uvuni_to_utf8(d, uv) uvuni_to_utf8_flags(d, uv, 0)
33#define is_utf8_string_loc(s, len, ep) is_utf8_string_loclen(s, len, ep, 0)
34
eda9cac1
KW
35#define foldEQ_utf8(s1, pe1, l1, u1, s2, pe2, l2, u2) \
36 foldEQ_utf8_flags(s1, pe1, l1, u1, s2, pe2, l2, u2, 0)
a33c29bc 37#define FOLDEQ_UTF8_NOMIX_ASCII (1 << 0)
5e64d0fa 38#define FOLDEQ_UTF8_LOCALE (1 << 1)
18f762c3
KW
39#define FOLDEQ_S1_ALREADY_FOLDED (1 << 2)
40#define FOLDEQ_S2_ALREADY_FOLDED (1 << 3)
a33c29bc 41
e6226b18
KW
42/*
43=for apidoc ibcmp_utf8
44
45This is a synonym for (! foldEQ_utf8())
46
47=cut
48*/
49#define ibcmp_utf8(s1, pe1, l1, u1, s2, pe2, l2, u2) \
50 cBOOL(! foldEQ_utf8(s1, pe1, l1, u1, s2, pe2, l2, u2))
51
1d72bdf6
NIS
52#ifdef EBCDIC
53/* The equivalent of these macros but implementing UTF-EBCDIC
54 are in the following header file:
55 */
56
57#include "utfebcdic.h"
fd7cb289 58
d06134e5 59#else /* ! EBCDIC */
73c4f7a1
GS
60START_EXTERN_C
61
a0ed51b3 62#ifdef DOINIT
6f06b55f 63EXTCONST unsigned char PL_utf8skip[] = {
b2635aa8
KW
64/* 0x00 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
65/* 0x10 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
66/* 0x20 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
67/* 0x30 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
68/* 0x40 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
69/* 0x50 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
70/* 0x60 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
71/* 0x70 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
72/* 0x80 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus: continuation byte */
73/* 0x90 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus: continuation byte */
74/* 0xA0 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus: continuation byte */
75/* 0xB0 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus: continuation byte */
76/* 0xC0 */ 2,2, /* overlong */
77/* 0xC2 */ 2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* U+0080 to U+03FF */
78/* 0xD0 */ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* U+0400 to U+07FF */
79/* 0xE0 */ 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, /* U+0800 to U+FFFF */
80/* 0xF0 */ 4,4,4,4,4,4,4,4,5,5,5,5,6,6, /* above BMP to 2**31 - 1 */
81/* 0xFE */ 7,13, /* Perl extended (never was official UTF-8). Up to 72bit
82 allowed (64-bit + reserved). */
a0ed51b3
LW
83};
84#else
6f06b55f 85EXTCONST unsigned char PL_utf8skip[];
a0ed51b3
LW
86#endif
87
73c4f7a1 88END_EXTERN_C
7e2040f0 89
1d72bdf6
NIS
90/* Native character to iso-8859-1 */
91#define NATIVE_TO_ASCII(ch) (ch)
92#define ASCII_TO_NATIVE(ch) (ch)
93/* Transform after encoding */
94#define NATIVE_TO_UTF(ch) (ch)
95#define UTF_TO_NATIVE(ch) (ch)
96/* Transforms in wide UV chars */
97#define UNI_TO_NATIVE(ch) (ch)
98#define NATIVE_TO_UNI(ch) (ch)
99/* Transforms in invariant space */
100#define NATIVE_TO_NEED(enc,ch) (ch)
101#define ASCII_TO_NEED(enc,ch) (ch)
d7578b48 102
d06134e5 103/* As there are no translations, avoid the function wrapper */
1754c1a1
NC
104#define utf8n_to_uvchr utf8n_to_uvuni
105#define uvchr_to_utf8 uvuni_to_utf8
2b9d42f0 106
877d9f0d 107/*
9041c2e3 108
8c007b5a 109 The following table is from Unicode 3.2.
877d9f0d
JH
110
111 Code Points 1st Byte 2nd Byte 3rd Byte 4th Byte
112
375122d7 113 U+0000..U+007F 00..7F
e1b711da 114 U+0080..U+07FF * C2..DF 80..BF
37e2e78e 115 U+0800..U+0FFF E0 * A0..BF 80..BF
375122d7 116 U+1000..U+CFFF E1..EC 80..BF 80..BF
e1b711da 117 U+D000..U+D7FF ED 80..9F 80..BF
37e2e78e 118 U+D800..U+DFFF +++++++ utf16 surrogates, not legal utf8 +++++++
375122d7 119 U+E000..U+FFFF EE..EF 80..BF 80..BF
37e2e78e 120 U+10000..U+3FFFF F0 * 90..BF 80..BF 80..BF
877d9f0d
JH
121 U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
122 U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
b2635aa8
KW
123 Below are non-Unicode code points
124 U+110000..U+13FFFF F4 90..BF 80..BF 80..BF
125 U+110000..U+1FFFFF F5..F7 80..BF 80..BF 80..BF
126 U+200000: F8.. * 88..BF 80..BF 80..BF 80..BF
877d9f0d 127
e1b711da 128Note the gaps before several of the byte entries above marked by '*'. These are
37e2e78e
KW
129caused by legal UTF-8 avoiding non-shortest encodings: it is technically
130possible to UTF-8-encode a single code point in different ways, but that is
131explicitly forbidden, and the shortest possible encoding should always be used
132(and that is what Perl does).
8c007b5a 133
877d9f0d
JH
134 */
135
8c007b5a
JH
136/*
137 Another way to look at it, as bits:
138
b2635aa8 139 Code Points 1st Byte 2nd Byte 3rd Byte 4th Byte
8c007b5a 140
b2635aa8
KW
141 0aaa aaaa 0aaa aaaa
142 0000 0bbb bbaa aaaa 110b bbbb 10aa aaaa
143 cccc bbbb bbaa aaaa 1110 cccc 10bb bbbb 10aa aaaa
144 00 000d ddcc cccc bbbb bbaa aaaa 1111 0ddd 10cc cccc 10bb bbbb 10aa aaaa
8c007b5a
JH
145
146As you can see, the continuation bytes all begin with C<10>, and the
e1b711da 147leading bits of the start byte tell how many bytes there are in the
8c007b5a
JH
148encoded character.
149
65ab9279
TC
150Perl's extended UTF-8 means we can have start bytes up to FF.
151
8c007b5a
JH
152*/
153
c4d5f83a 154#define UNI_IS_INVARIANT(c) (((UV)c) < 0x80)
0447e8df 155#define UTF8_IS_START(c) (((U8)c) >= 0xc2)
c512ce4f
JH
156#define UTF8_IS_CONTINUATION(c) (((U8)c) >= 0x80 && (((U8)c) <= 0xbf))
157#define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80)
0ae1fa71
CH
158
159/* Masking with 0xfe allows low bit to be 0 or 1; thus this matches 0xc[23] */
160#define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfe) == 0xc2)
8850bf83 161
22901f30 162#define UTF_START_MARK(len) (((len) > 7) ? 0xFF : (0xFE << (7-(len))))
b2635aa8
KW
163
164/* Masks out the initial one bits in a start byte, leaving the real data ones.
165 * Doesn't work on an invariant byte */
22901f30 166#define UTF_START_MASK(len) (((len) >= 7) ? 0x00 : (0x1F >> ((len)-2)))
1d72bdf6
NIS
167
168#define UTF_CONTINUATION_MARK 0x80
169#define UTF_ACCUMULATION_SHIFT 6
b2635aa8
KW
170
171/* 2**UTF_ACCUMULATION_SHIFT - 1 */
1d72bdf6 172#define UTF_CONTINUATION_MASK ((U8)0x3f)
c512ce4f 173
eb83ed87
KW
174/* This sets the UTF_CONTINUATION_MASK in the upper bits of a word. If a value
175 * is anded with it, and the result is non-zero, then using the original value
176 * in UTF8_ACCUMULATE will overflow, shifting bits off the left */
177#define UTF_ACCUMULATION_OVERFLOW_MASK \
178 (((UV) UTF_CONTINUATION_MASK) << ((sizeof(UV) * CHARBITS) - UTF_ACCUMULATION_SHIFT))
179
1d68d6cd 180#ifdef HAS_QUAD
5bbb0b5a 181#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
1d68d6cd
SC
182 (uv) < 0x800 ? 2 : \
183 (uv) < 0x10000 ? 3 : \
184 (uv) < 0x200000 ? 4 : \
185 (uv) < 0x4000000 ? 5 : \
186 (uv) < 0x80000000 ? 6 : \
9041c2e3 187 (uv) < UTF8_QUAD_MAX ? 7 : 13 )
1d68d6cd
SC
188#else
189/* No, I'm not even going to *TRY* putting #ifdef inside a #define */
5bbb0b5a 190#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
1d68d6cd
SC
191 (uv) < 0x800 ? 2 : \
192 (uv) < 0x10000 ? 3 : \
193 (uv) < 0x200000 ? 4 : \
194 (uv) < 0x4000000 ? 5 : \
195 (uv) < 0x80000000 ? 6 : 7 )
196#endif
197
d06134e5
KW
198#endif /* EBCDIC vs ASCII */
199
200/* Rest of these are attributes of Unicode and perl's internals rather than the
201 * encoding, or happen to be the same in both ASCII and EBCDIC (at least at
202 * this level; the macros that some of these call may have different
203 * definitions in the two encodings */
204
205#define NATIVE8_TO_UNI(ch) NATIVE_TO_ASCII(ch) /* a clearer synonym */
206
207#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) | (((U8)new) & UTF_CONTINUATION_MASK))
208
2950f2a7
KW
209/* Convert a two (not one) byte utf8 character to a unicode code point value.
210 * Needs just one iteration of accumulate. Should not be used unless it is
211 * known that the two bytes are legal: 1) two-byte start, and 2) continuation.
212 * Note that the result can be larger than 255 if the input character is not
213 * downgradable */
214#define TWO_BYTE_UTF8_TO_UNI(HI, LO) \
215 UTF8_ACCUMULATE((NATIVE_TO_UTF(HI) & UTF_START_MASK(2)), \
216 NATIVE_TO_UTF(LO))
217
d06134e5
KW
218#define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)]
219
220#define UTF8_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_UTF(c))
221#define NATIVE_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE8_TO_UNI(c))
222
223#define MAX_PORTABLE_UTF8_TWO_BYTE 0x3FF /* constrained by EBCDIC */
224
225/* The macros in the next sets are used to generate the two utf8 or utfebcdic
226 * bytes from an ordinal that is known to fit into two bytes; it must be less
227 * than 0x3FF to work across both encodings. */
228/* Nocast allows these to be used in the case label of a switch statement */
0072151e 229#define UTF8_TWO_BYTE_HI_nocast(c) UTF_TO_NATIVE(((c) >> UTF_ACCUMULATION_SHIFT) | (0xFF & UTF_START_MARK(2)))
14c28784 230#define UTF8_TWO_BYTE_LO_nocast(c) UTF_TO_NATIVE(((c) & UTF_CONTINUATION_MASK) | UTF_CONTINUATION_MARK)
d06134e5
KW
231
232#define UTF8_TWO_BYTE_HI(c) ((U8) (UTF8_TWO_BYTE_HI_nocast(c)))
233#define UTF8_TWO_BYTE_LO(c) ((U8) (UTF8_TWO_BYTE_LO_nocast(c)))
234
235/* This name is used when the source is a single byte */
236#define UTF8_EIGHT_BIT_HI(c) UTF8_TWO_BYTE_HI((U8)(c))
237#define UTF8_EIGHT_BIT_LO(c) UTF8_TWO_BYTE_LO((U8)(c))
238
7e2040f0 239/*
e3036cf4 240 * 'UTF' is whether or not p is encoded in UTF8. The names 'foo_lazy_if' stem
20df05f4
KW
241 * from an earlier version of these macros in which they didn't call the
242 * foo_utf8() macros (i.e. were 'lazy') unless they decided that *p is the
243 * beginning of a utf8 character. Now that foo_utf8() determines that itself,
244 * no need to do it again here
7e2040f0 245 */
e3036cf4
KW
246#define isIDFIRST_lazy_if(p,UTF) ((IN_BYTES || !UTF ) \
247 ? isIDFIRST(*(p)) \
248 : isIDFIRST_utf8((const U8*)p))
249#define isALNUM_lazy_if(p,UTF) ((IN_BYTES || (!UTF )) \
250 ? isALNUM(*(p)) \
251 : isALNUM_utf8((const U8*)p))
1d72bdf6 252
7e2040f0
GS
253#define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1)
254#define isALNUM_lazy(p) isALNUM_lazy_if(p,1)
3bd709b1 255
89ebb4a3
JH
256#define UTF8_MAXBYTES 13
257/* How wide can a single UTF-8 encoded character become in bytes.
258 * NOTE: Strictly speaking Perl's UTF-8 should not be called UTF-8
259 * since UTF-8 is an encoding of Unicode and given Unicode's current
260 * upper limit only four bytes is possible. Perl thinks of UTF-8
261 * as a way to encode non-negative integers in a binary format. */
262#define UTF8_MAXLEN UTF8_MAXBYTES
263
89ebb4a3
JH
264/* The maximum number of UTF-8 bytes a single Unicode character can
265 * uppercase/lowercase/fold into; this number depends on the Unicode
266 * version. An example of maximal expansion is the U+03B0 which
267 * uppercases to U+03C5 U+0308 U+0301. The Unicode databases that
d06134e5 268 * tell these things are UnicodeData.txt, CaseFolding.txt, and
88d45ddf
KW
269 * SpecialCasing.txt. The value is 6 for strict Unicode characters, but it has
270 * to be as big as Perl allows for a single character */
271#define UTF8_MAXBYTES_CASE UTF8_MAXBYTES
3bd709b1 272
8cb75cc8
KW
273/* A Unicode character can fold to up to 3 characters */
274#define UTF8_MAX_FOLD_CHAR_EXPAND 3
275
a98fe34d 276#define IN_BYTES (CopHINTS_get(PL_curcop) & HINT_BYTES)
0064a8a9 277#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTES)
66cbab2c 278#define IN_UNI_8_BIT \
b36bf33f
KW
279 (CopHINTS_get(PL_curcop) & (HINT_UNI_8_BIT|HINT_LOCALE_NOT_CHARS) \
280 && ! IN_LOCALE_RUNTIME && ! IN_BYTES)
281
1d72bdf6 282
c76687c5
KW
283#define UTF8_ALLOW_EMPTY 0x0001 /* Allow a zero length string */
284
285/* Allow first byte to be a continuation byte */
1d72bdf6 286#define UTF8_ALLOW_CONTINUATION 0x0002
c76687c5
KW
287
288/* Allow second... bytes to be non-continuation bytes */
1d72bdf6 289#define UTF8_ALLOW_NON_CONTINUATION 0x0004
949cf498
KW
290
291/* expecting more bytes than were available in the string */
292#define UTF8_ALLOW_SHORT 0x0008
293
294/* Overlong sequence; i.e., the code point can be specified in fewer bytes. */
295#define UTF8_ALLOW_LONG 0x0010
296
297#define UTF8_DISALLOW_SURROGATE 0x0020 /* Unicode surrogates */
298#define UTF8_WARN_SURROGATE 0x0040
299
300#define UTF8_DISALLOW_NONCHAR 0x0080 /* Unicode non-character */
301#define UTF8_WARN_NONCHAR 0x0100 /* code points */
302
303#define UTF8_DISALLOW_SUPER 0x0200 /* Super-set of Unicode: code */
304#define UTF8_WARN_SUPER 0x0400 /* points above the legal max */
305
306/* Code points which never were part of the original UTF-8 standard, the first
307 * byte of which is a FE or FF on ASCII platforms. */
308#define UTF8_DISALLOW_FE_FF 0x0800
309#define UTF8_WARN_FE_FF 0x1000
310
311#define UTF8_CHECK_ONLY 0x2000
312
313/* For backwards source compatibility. They do nothing, as the default now
314 * includes what they used to mean. The first one's meaning was to allow the
315 * just the single non-character 0xFFFF */
316#define UTF8_ALLOW_FFFF 0
317#define UTF8_ALLOW_SURROGATE 0
318
33d9abfb 319#define UTF8_DISALLOW_ILLEGAL_INTERCHANGE (UTF8_DISALLOW_SUPER|UTF8_DISALLOW_NONCHAR|UTF8_DISALLOW_SURROGATE|UTF8_DISALLOW_FE_FF)
949cf498 320#define UTF8_WARN_ILLEGAL_INTERCHANGE \
33d9abfb 321 (UTF8_WARN_SUPER|UTF8_WARN_NONCHAR|UTF8_WARN_SURROGATE|UTF8_WARN_FE_FF)
949cf498
KW
322#define UTF8_ALLOW_ANY \
323 (~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_WARN_ILLEGAL_INTERCHANGE))
324#define UTF8_ALLOW_ANYUV \
325 (UTF8_ALLOW_EMPTY \
326 & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE|UTF8_WARN_ILLEGAL_INTERCHANGE))
9f7f3913
ST
327#define UTF8_ALLOW_DEFAULT (ckWARN(WARN_UTF8) ? 0 : \
328 UTF8_ALLOW_ANYUV)
1d72bdf6 329
7131f24d
KW
330/* Surrogates, non-character code points and above-Unicode code points are
331 * problematic in some contexts. This allows code that needs to check for
332 * those to to quickly exclude the vast majority of code points it will
333 * encounter */
334#ifdef EBCDIC
335# define UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE UTF_TO_NATIVE(0xF1)
336#else
337# define UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE 0xED
338#endif
339
340/* ASCII EBCDIC I8
341 * U+D7FF: \xED\x9F\xBF \xF1\xB5\xBF\xBF last before surrogates
342 * U+D800: \xED\xA0\x80 \xF1\xB6\xA0\xA0 1st surrogate
343 * U+DFFF: \xED\xBF\xBF \xF1\xB7\xBF\xBF final surrogate
344 * U+E000: \xEE\x80\x80 \xF1\xB8\xA0\xA0 next after surrogates
345 */
346#ifdef EBCDIC /* Both versions assume well-formed UTF8 */
b2635aa8
KW
347# define UTF8_IS_SURROGATE(s) (*(s) == UTF_TO_NATIVE(0xF1) \
348 && ((*((s) +1) == UTF_TO_NATIVE(0xB6)) \
349 || *((s) + 1) == UTF_TO_NATIVE(0xB7)))
7131f24d
KW
350#else
351# define UTF8_IS_SURROGATE(s) (*(s) == 0xED && *((s) + 1) >= 0xA0)
352#endif
353
354/* ASCII EBCDIC I8
355 * U+10FFFF: \xF4\x8F\xBF\xBF \xF9\xA1\xBF\xBF\xBF max legal Unicode
356 * U+110000: \xF4\x90\x80\x80 \xF9\xA2\xA0\xA0\xA0
357 * U+110001: \xF4\x90\x80\x81 \xF9\xA2\xA0\xA0\xA1
358 */
359#ifdef EBCDIC /* Both versions assume well-formed UTF8 */
360# define UTF8_IS_SUPER(s) (*(s) >= UTF_TO_NATIVE(0xF9) \
361 && (*(s) > UTF_TO_NATIVE(0xF9) || (*((s) + 1) >= UTF_TO_NATIVE(0xA2))))
362#else
363# define UTF8_IS_SUPER(s) (*(s) >= 0xF4 \
364 && (*(s) > 0xF4 || (*((s) + 1) >= 0x90)))
365#endif
366
367/* ASCII EBCDIC I8
368 * U+FDCF: \xEF\xB7\x8F \xF1\xBF\xAE\xAF last before non-char block
369 * U+FDD0: \xEF\xB7\x90 \xF1\xBF\xAE\xB0 first non-char in block
370 * U+FDEF: \xEF\xB7\xAF \xF1\xBF\xAF\xAF last non-char in block
371 * U+FDF0: \xEF\xB7\xB0 \xF1\xBF\xAF\xB0 first after non-char block
372 * U+FFFF: \xEF\xBF\xBF \xF1\xBF\xBF\xBF
373 * U+1FFFF: \xF0\x9F\xBF\xBF \xF3\xBF\xBF\xBF
374 * U+2FFFF: \xF0\xAF\xBF\xBF \xF5\xBF\xBF\xBF
375 * U+3FFFF: \xF0\xBF\xBF\xBF \xF7\xBF\xBF\xBF
376 * U+4FFFF: \xF1\x8F\xBF\xBF \xF8\xA9\xBF\xBF\xBF
377 * U+5FFFF: \xF1\x9F\xBF\xBF \xF8\xAB\xBF\xBF\xBF
378 * U+6FFFF: \xF1\xAF\xBF\xBF \xF8\xAD\xBF\xBF\xBF
379 * U+7FFFF: \xF1\xBF\xBF\xBF \xF8\xAF\xBF\xBF\xBF
380 * U+8FFFF: \xF2\x8F\xBF\xBF \xF8\xB1\xBF\xBF\xBF
381 * U+9FFFF: \xF2\x9F\xBF\xBF \xF8\xB3\xBF\xBF\xBF
382 * U+AFFFF: \xF2\xAF\xBF\xBF \xF8\xB5\xBF\xBF\xBF
383 * U+BFFFF: \xF2\xBF\xBF\xBF \xF8\xB7\xBF\xBF\xBF
384 * U+CFFFF: \xF3\x8F\xBF\xBF \xF8\xB9\xBF\xBF\xBF
385 * U+DFFFF: \xF3\x9F\xBF\xBF \xF8\xBB\xBF\xBF\xBF
386 * U+EFFFF: \xF3\xAF\xBF\xBF \xF8\xBD\xBF\xBF\xBF
387 * U+FFFFF: \xF3\xBF\xBF\xBF \xF8\xBF\xBF\xBF\xBF
388 * U+10FFFF: \xF4\x8F\xBF\xBF \xF9\xA1\xBF\xBF\xBF
389 */
390#define UTF8_IS_NONCHAR_(s) ( \
391 *(s) >= UTF8_FIRST_PROBLEMATIC_CODE_POINT_FIRST_BYTE \
392 && ! UTF8_IS_SUPER(s) \
393 && UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_FIRST_PROBLEMATIC(s) \
394
395#ifdef EBCDIC /* Both versions assume well-formed UTF8 */
396# define UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s) \
397 ((*(s) == UTF_TO_NATIVE(0xF1) \
398 && (*((s) + 1) == UTF_TO_NATIVE(0xBF) \
399 && ((*((s) + 2) == UTF_TO_NATIVE(0xAE) \
400 && *((s) + 3) >= UTF_TO_NATIVE(0xB0)) \
401 || (*((s) + 2) == UTF_TO_NATIVE(0xAF) \
402 && *((s) + 3) <= UTF_TO_NATIVE(0xAF))))) \
403 || (UTF8SKIP(*(s)) > 3 \
404 /* (These were all derived by inspection and experimentation with an */ \
405 /* editor) The next line checks the next to final byte in the char */ \
406 && *((s) + UTF8SKIP(*(s)) - 2) == UTF_TO_NATIVE(0xBF) \
407 && *((s) + UTF8SKIP(*(s)) - 3) == UTF_TO_NATIVE(0xBF) \
408 && (NATIVE_TO_UTF(*((s) + UTF8SKIP(*(s)) - 4)) & 0x81) == 0x81 \
409 && (NATIVE_TO_UTF(*((s) + UTF8SKIP(*(s)) - 1)) & 0xBE) == 0XBE))
410#else
411# define UTF8_IS_NONCHAR_GIVEN_THAT_NON_SUPER_AND_GE_PROBLEMATIC(s) \
412 ((*(s) == 0xEF \
413 && ((*((s) + 1) == 0xB7 && (*((s) + 2) >= 0x90 && (*((s) + 2) <= 0xAF)))\
414 /* Gets U+FFF[EF] */ \
415 || (*((s) + 1) == 0xBF && ((*((s) + 2) & 0xBE) == 0xBE)))) \
416 || ((*((s) + 2) == 0xBF \
417 && (*((s) + 3) & 0xBE) == 0xBE \
418 /* Excludes things like U+10FFE = \xF0\x90\xBF\xBE */ \
419 && (*((s) + 1) & 0x8F) == 0x8F)))
420#endif
421
c867b360
JH
422#define UNICODE_SURROGATE_FIRST 0xD800
423#define UNICODE_SURROGATE_LAST 0xDFFF
424#define UNICODE_REPLACEMENT 0xFFFD
425#define UNICODE_BYTE_ORDER_MARK 0xFEFF
1d72bdf6 426
b851fbc1 427/* Though our UTF-8 encoding can go beyond this,
c76687c5 428 * let's be conservative and do as Unicode says. */
b851fbc1
JH
429#define PERL_UNICODE_MAX 0x10FFFF
430
949cf498
KW
431#define UNICODE_WARN_SURROGATE 0x0001 /* UTF-16 surrogates */
432#define UNICODE_WARN_NONCHAR 0x0002 /* Non-char code points */
433#define UNICODE_WARN_SUPER 0x0004 /* Above 0x10FFFF */
434#define UNICODE_WARN_FE_FF 0x0008 /* Above 0x10FFFF */
435#define UNICODE_DISALLOW_SURROGATE 0x0010
436#define UNICODE_DISALLOW_NONCHAR 0x0020
437#define UNICODE_DISALLOW_SUPER 0x0040
438#define UNICODE_DISALLOW_FE_FF 0x0080
439#define UNICODE_WARN_ILLEGAL_INTERCHANGE \
440 (UNICODE_WARN_SURROGATE|UNICODE_WARN_NONCHAR|UNICODE_WARN_SUPER)
441#define UNICODE_DISALLOW_ILLEGAL_INTERCHANGE \
442 (UNICODE_DISALLOW_SURROGATE|UNICODE_DISALLOW_NONCHAR|UNICODE_DISALLOW_SUPER)
443
444/* For backward source compatibility, as are now the default */
445#define UNICODE_ALLOW_SURROGATE 0
446#define UNICODE_ALLOW_SUPER 0
447#define UNICODE_ALLOW_ANY 0
b851fbc1 448
1d72bdf6
NIS
449#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \
450 (c) <= UNICODE_SURROGATE_LAST)
a10ec373 451#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACEMENT)
872c91ae 452#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTE_ORDER_MARK)
7131f24d
KW
453#define UNICODE_IS_NONCHAR(c) ((c >= 0xFDD0 && c <= 0xFDEF) \
454 /* The other noncharacters end in FFFE or FFFF, which \
455 * the mask below catches both of, but beyond the last \
456 * official unicode code point, they aren't \
457 * noncharacters, since those aren't Unicode \
458 * characters at all */ \
459 || ((((c & 0xFFFE) == 0xFFFE)) && ! UNICODE_IS_SUPER(c)))
460#define UNICODE_IS_SUPER(c) ((c) > PERL_UNICODE_MAX)
461#define UNICODE_IS_FE_FF(c) ((c) > 0x7FFFFFFF)
1d72bdf6 462
f067b878
NA
463#ifdef HAS_QUAD
464# define UTF8_QUAD_MAX UINT64_C(0x1000000000)
465#endif
3bd709b1 466
09091399
JH
467#define UNICODE_GREEK_CAPITAL_LETTER_SIGMA 0x03A3
468#define UNICODE_GREEK_SMALL_LETTER_FINAL_SIGMA 0x03C2
469#define UNICODE_GREEK_SMALL_LETTER_SIGMA 0x03C3
9dcbe121 470#define GREEK_SMALL_LETTER_MU 0x03BC
78a0d3cc
KW
471#define GREEK_CAPITAL_LETTER_MU 0x039C /* Upper and title case of MICRON */
472#define LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS 0x0178 /* Also is title case */
97298f37 473#define LATIN_CAPITAL_LETTER_SHARP_S 0x1E9E
09091399 474
9e55ce06 475#define UNI_DISPLAY_ISPRINT 0x0001
c728cb41
JH
476#define UNI_DISPLAY_BACKSLASH 0x0002
477#define UNI_DISPLAY_QQ (UNI_DISPLAY_ISPRINT|UNI_DISPLAY_BACKSLASH)
478#define UNI_DISPLAY_REGEX (UNI_DISPLAY_ISPRINT|UNI_DISPLAY_BACKSLASH)
9e55ce06 479
5cd46e1f
KW
480#ifndef EBCDIC
481# define LATIN_SMALL_LETTER_SHARP_S 0x00DF
482# define LATIN_SMALL_LETTER_Y_WITH_DIAERESIS 0x00FF
483# define MICRO_SIGN 0x00B5
f508a607
KW
484# define LATIN_CAPITAL_LETTER_A_WITH_RING_ABOVE 0x00C5
485# define LATIN_SMALL_LETTER_A_WITH_RING_ABOVE 0x00E5
5cd46e1f
KW
486#endif
487
488#define ANYOF_FOLD_SHARP_S(node, input, end) \
489 (ANYOF_BITMAP_TEST(node, LATIN_SMALL_LETTER_SHARP_S) && \
137165a6 490 (ANYOF_NONBITMAP(node)) && \
39065660 491 (ANYOF_FLAGS(node) & ANYOF_LOC_NONBITMAP_FOLD) && \
07b6858f
JH
492 ((end) > (input) + 1) && \
493 toLOWER((input)[0]) == 's' && \
494 toLOWER((input)[1]) == 's')
ebc501f0 495#define SHARP_S_SKIP 2
3b0fc154 496
2f454f11 497#ifndef EBCDIC
c78f6c49 498# define IS_UTF8_CHAR_1(p) \
3b0fc154 499 ((p)[0] <= 0x7F)
c78f6c49 500# define IS_UTF8_CHAR_2(p) \
3b0fc154
JH
501 ((p)[0] >= 0xC2 && (p)[0] <= 0xDF && \
502 (p)[1] >= 0x80 && (p)[1] <= 0xBF)
c78f6c49 503# define IS_UTF8_CHAR_3a(p) \
3b0fc154
JH
504 ((p)[0] == 0xE0 && \
505 (p)[1] >= 0xA0 && (p)[1] <= 0xBF && \
506 (p)[2] >= 0x80 && (p)[2] <= 0xBF)
c78f6c49 507# define IS_UTF8_CHAR_3b(p) \
3b0fc154
JH
508 ((p)[0] >= 0xE1 && (p)[0] <= 0xEC && \
509 (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
510 (p)[2] >= 0x80 && (p)[2] <= 0xBF)
c78f6c49 511# define IS_UTF8_CHAR_3c(p) \
3b0fc154
JH
512 ((p)[0] == 0xED && \
513 (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
514 (p)[2] >= 0x80 && (p)[2] <= 0xBF)
c78f6c49
KW
515 /* In IS_UTF8_CHAR_3c(p) one could use
516 * (p)[1] >= 0x80 && (p)[1] <= 0x9F
517 * if one wanted to exclude surrogates. */
518# define IS_UTF8_CHAR_3d(p) \
3b0fc154
JH
519 ((p)[0] >= 0xEE && (p)[0] <= 0xEF && \
520 (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
521 (p)[2] >= 0x80 && (p)[2] <= 0xBF)
c78f6c49 522# define IS_UTF8_CHAR_4a(p) \
3b0fc154
JH
523 ((p)[0] == 0xF0 && \
524 (p)[1] >= 0x90 && (p)[1] <= 0xBF && \
525 (p)[2] >= 0x80 && (p)[2] <= 0xBF && \
526 (p)[3] >= 0x80 && (p)[3] <= 0xBF)
c78f6c49 527# define IS_UTF8_CHAR_4b(p) \
3b0fc154
JH
528 ((p)[0] >= 0xF1 && (p)[0] <= 0xF3 && \
529 (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
530 (p)[2] >= 0x80 && (p)[2] <= 0xBF && \
531 (p)[3] >= 0x80 && (p)[3] <= 0xBF)
532/* In IS_UTF8_CHAR_4c(p) one could use
533 * (p)[0] == 0xF4
534 * if one wanted to stop at the Unicode limit U+10FFFF.
535 * The 0xF7 allows us to go to 0x1fffff (0x200000 would
536 * require five bytes). Not doing any further code points
537 * since that is not needed (and that would not be strict
538 * UTF-8, anyway). The "slow path" in Perl_is_utf8_char()
539 * will take care of the "extended UTF-8". */
c78f6c49 540# define IS_UTF8_CHAR_4c(p) \
a18d6e6e 541 ((p)[0] >= 0xF4 && (p)[0] <= 0xF7 && \
3b0fc154
JH
542 (p)[1] >= 0x80 && (p)[1] <= 0xBF && \
543 (p)[2] >= 0x80 && (p)[2] <= 0xBF && \
544 (p)[3] >= 0x80 && (p)[3] <= 0xBF)
545
c78f6c49 546# define IS_UTF8_CHAR_3(p) \
3b0fc154
JH
547 (IS_UTF8_CHAR_3a(p) || \
548 IS_UTF8_CHAR_3b(p) || \
549 IS_UTF8_CHAR_3c(p) || \
550 IS_UTF8_CHAR_3d(p))
c78f6c49 551# define IS_UTF8_CHAR_4(p) \
3b0fc154
JH
552 (IS_UTF8_CHAR_4a(p) || \
553 IS_UTF8_CHAR_4b(p) || \
554 IS_UTF8_CHAR_4c(p))
555
556/* IS_UTF8_CHAR(p) is strictly speaking wrong (not UTF-8) because it
557 * (1) allows UTF-8 encoded UTF-16 surrogates
558 * (2) it allows code points past U+10FFFF.
559 * The Perl_is_utf8_char() full "slow" code will handle the Perl
560 * "extended UTF-8". */
c78f6c49 561# define IS_UTF8_CHAR(p, n) \
3b0fc154
JH
562 ((n) == 1 ? IS_UTF8_CHAR_1(p) : \
563 (n) == 2 ? IS_UTF8_CHAR_2(p) : \
564 (n) == 3 ? IS_UTF8_CHAR_3(p) : \
565 (n) == 4 ? IS_UTF8_CHAR_4(p) : 0)
566
c78f6c49 567# define IS_UTF8_CHAR_FAST(n) ((n) <= 4)
768c67ee 568
2f454f11
KW
569#else /* EBCDIC */
570
571/* This is an attempt to port IS_UTF8_CHAR to EBCDIC based on eyeballing.
572 * untested. If want to exclude surrogates and above-Unicode, see the
573 * definitions for UTF8_IS_SURROGATE and UTF8_IS_SUPER */
574# define IS_UTF8_CHAR_1(p) \
575 (NATIVE_TO_ASCII((p)[0]) <= 0x9F)
576# define IS_UTF8_CHAR_2(p) \
577 (NATIVE_TO_I8((p)[0]) >= 0xC5 && NATIVE_TO_I8((p)[0]) <= 0xDF && \
578 NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF)
579# define IS_UTF8_CHAR_3(p) \
580 (NATIVE_TO_I8((p)[0]) == 0xE1 && NATIVE_TO_I8((p)[1]) <= 0xEF && \
581 NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
582 NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF)
583# define IS_UTF8_CHAR_4a(p) \
584 (NATIVE_TO_I8((p)[0]) == 0xF0 && \
585 NATIVE_TO_I8((p)[1]) >= 0xB0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
586 NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
587 NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
588# define IS_UTF8_CHAR_4b(p) \
589 (NATIVE_TO_I8((p)[0]) >= 0xF1 && NATIVE_TO_I8((p)[0]) <= 0xF7 && \
590 NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
591 NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
592 NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
593# define IS_UTF8_CHAR_5a(p) \
594 (NATIVE_TO_I8((p)[0]) == 0xF8 && \
595 NATIVE_TO_I8((p)[1]) >= 0xA8 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
596 NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
597 NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
598 NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
599# define IS_UTF8_CHAR_5b(p) \
600 (NATIVE_TO_I8((p)[0]) >= 0xF9 && NATIVE_TO_I8((p)[1]) <= 0xFB && \
601 NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
602 NATIVE_TO_I8((p)[1]) >= 0xA0 && NATIVE_TO_I8((p)[1]) <= 0xBF && \
603 NATIVE_TO_I8((p)[2]) >= 0xA0 && NATIVE_TO_I8((p)[2]) <= 0xBF && \
604 NATIVE_TO_I8((p)[3]) >= 0xA0 && NATIVE_TO_I8((p)[3]) <= 0xBF)
605
606# define IS_UTF8_CHAR_4(p) \
607 (IS_UTF8_CHAR_4a(p) || \
608 IS_UTF8_CHAR_4b(p))
609# define IS_UTF8_CHAR_5(p) \
610 (IS_UTF8_CHAR_5a(p) || \
611 IS_UTF8_CHAR_5b(p))
612# define IS_UTF8_CHAR(p, n) \
613 ((n) == 1 ? IS_UTF8_CHAR_1(p) : \
614 (n) == 2 ? IS_UTF8_CHAR_2(p) : \
615 (n) == 3 ? IS_UTF8_CHAR_3(p) : \
616 (n) == 4 ? IS_UTF8_CHAR_4(p) : \
617 (n) == 5 ? IS_UTF8_CHAR_5(p) : 0)
618
619# define IS_UTF8_CHAR_FAST(n) ((n) <= 5)
620
77263263 621#endif /* IS_UTF8_CHAR() for UTF-8 */
e9a8c099
MHM
622
623/*
624 * Local variables:
625 * c-indentation-style: bsd
626 * c-basic-offset: 4
627 * indent-tabs-mode: t
628 * End:
629 *
630 * ex: set ts=8 sts=4 sw=4 noet:
631 */