[perl5.git] / utf8.h

/*    utf8.h
 *
 *    Copyright (c) 1998-2000, Larry Wall
 *
 *    You may distribute under the terms of either the GNU General Public
 *    License or the Artistic License, as specified in the README file.
 *
 */

START_EXTERN_C

#ifdef DOINIT
EXTCONST unsigned char PL_utf8skip[] = {
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* scripts */
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6,	 /* cjk etc. */
7,13, /* Perl extended (not UTF-8).  Up to 72bit allowed (64-bit + reserved). */
};
#else
EXTCONST unsigned char PL_utf8skip[];
#endif

END_EXTERN_C

#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */

/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */
#define IN_BYTE (PL_curcop->op_private & HINT_BYTE)
#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE)

#define UTF8_ALLOW_EMPTY		0x0001
#define UTF8_ALLOW_CONTINUATION		0x0002
#define UTF8_ALLOW_NON_CONTINUATION	0x0004
#define UTF8_ALLOW_FE_FF		0x0008
#define UTF8_ALLOW_SHORT		0x0010
#define UTF8_ALLOW_SURROGATE		0x0020
#define UTF8_ALLOW_BOM			0x0040
#define UTF8_ALLOW_FFFF			0x0080
#define UTF8_ALLOW_LONG			0x0100
#define UTF8_ALLOW_ANYUV		(UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\
					 UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\
					 UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG)
#define UTF8_ALLOW_ANY			0x00ff
#define UTF8_CHECK_ONLY			0x0100

#define UNICODE_SURROGATE_FIRST		0xd800
#define UNICODE_SURROGATE_LAST		0xdfff
#define UNICODE_REPLACEMENT		0xfffd
#define UNICODE_BYTER_ORDER_MARK	0xfffe
#define UNICODE_ILLEGAL			0xffff

#define UNICODE_IS_SURROGATE(c)		((c) >= UNICODE_SURROGATE_FIRST && \
					 (c) <= UNICODE_SURROGATE_LAST)
#define UNICODE_IS_REPLACEMENT(c)	((c) == UNICODE_REPLACMENT)
#define UNICODE_IS_BYTE_ORDER_MARK(c)	((c) == UNICODE_BYTER_ORDER_MARK)
#define UNICODE_IS_ILLEGAL(c)		((c) == UNICODE_ILLEGAL)

#define UTF8SKIP(s) PL_utf8skip[*(U8*)s]

#define UTF8_QUAD_MAX	UINT64_C(0x1000000000)

#define UTF8_IS_ASCII(c) 		(((U8)c) <  0x80)
#define UTF8_IS_START(c)		(((U8)c) >= 0xc0 && (((U8)c) <= 0xfd))
#define UTF8_IS_CONTINUATION(c)		(((U8)c) >= 0x80 && (((U8)c) <= 0xbf))
#define UTF8_IS_CONTINUED(c) 		(((U8)c) &  0x80)

#define UTF8_CONTINUATION_MASK		((U8)0x3f)
#define UTF8_ACCUMULATION_SHIFT		6
#define UTF8_ACCUMULATE(old, new)	((old) << UTF8_ACCUMULATION_SHIFT | (((U8)new) & UTF8_CONTINUATION_MASK))

#define UTF8_EIGHT_BIT_HI(c)	( (((U8)(c))>>6)      |0xc0)
#define UTF8_EIGHT_BIT_LO(c)	(((((U8)(c))   )&0x3f)|0x80)

#ifdef HAS_QUAD
#define UNISKIP(uv) ( (uv) < 0x80           ? 1 : \
		      (uv) < 0x800          ? 2 : \
		      (uv) < 0x10000        ? 3 : \
		      (uv) < 0x200000       ? 4 : \
		      (uv) < 0x4000000      ? 5 : \
		      (uv) < 0x80000000     ? 6 : \
                      (uv) < UTF8_QUAD_MAX ? 7 : 13 ) 
#else
/* No, I'm not even going to *TRY* putting #ifdef inside a #define */
#define UNISKIP(uv) ( (uv) < 0x80           ? 1 : \
		      (uv) < 0x800          ? 2 : \
		      (uv) < 0x10000        ? 3 : \
		      (uv) < 0x200000       ? 4 : \
		      (uv) < 0x4000000      ? 5 : \
		      (uv) < 0x80000000     ? 6 : 7 )
#endif


/*
 * Note: we try to be careful never to call the isXXX_utf8() functions
 * unless we're pretty sure we've seen the beginning of a UTF-8 character
 * (that is, the two high bits are set).  Otherwise we risk loading in the
 * heavy-duty SWASHINIT and SWASHGET routines unnecessarily.
 */
#ifdef EBCDIC
#define isIDFIRST_lazy_if(p,c) isIDFIRST(*(p))
#define isALNUM_lazy_if(p,c)   isALNUM(*(p))
#else
#define isIDFIRST_lazy_if(p,c) ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \
				? isIDFIRST(*(p)) \
				: isIDFIRST_utf8((U8*)p))
#define isALNUM_lazy_if(p,c)   ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \
				? isALNUM(*(p)) \
				: isALNUM_utf8((U8*)p))
#endif
#define isIDFIRST_lazy(p)	isIDFIRST_lazy_if(p,1)
#define isALNUM_lazy(p)		isALNUM_lazy_if(p,1)
Commit	Line	Data
a0ed51b3 LW	1	/* utf8.h
a0ed51b3 LW	2	*
3818b22b	3	* Copyright (c) 1998-2000, Larry Wall
a0ed51b3 LW	4	*
	5	* You may distribute under the terms of either the GNU General Public
	6	* License or the Artistic License, as specified in the README file.
	7	*
	8	*/
	9
73c4f7a1 GS	10	START_EXTERN_C
73c4f7a1 GS	11
a0ed51b3	12	#ifdef DOINIT
6f06b55f	13	EXTCONST unsigned char PL_utf8skip[] = {
a0ed51b3 LW	14	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
	15	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
	16	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
	17	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
	18	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
	19	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
	20	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* scripts */
3c77ea2b GS	21	3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6, /* cjk etc. */
3c77ea2b GS	22	7,13, /* Perl extended (not UTF-8). Up to 72bit allowed (64-bit + reserved). */
a0ed51b3 LW	23	};
a0ed51b3 LW	24	#else
6f06b55f	25	EXTCONST unsigned char PL_utf8skip[];
a0ed51b3 LW	26	#endif
a0ed51b3 LW	27
73c4f7a1 GS	28	END_EXTERN_C
73c4f7a1 GS	29
aa6ffa16 GS	30	#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */
aa6ffa16 GS	31
a4bf32d5	32	/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */
5bc28da9	33	#define IN_BYTE (PL_curcop->op_private & HINT_BYTE)
7e2040f0	34	#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE)
a0ed51b3	35
0da47da8 JH	36	#define UTF8_ALLOW_EMPTY 0x0001
	37	#define UTF8_ALLOW_CONTINUATION 0x0002
	38	#define UTF8_ALLOW_NON_CONTINUATION 0x0004
	39	#define UTF8_ALLOW_FE_FF 0x0008
	40	#define UTF8_ALLOW_SHORT 0x0010
	41	#define UTF8_ALLOW_SURROGATE 0x0020
	42	#define UTF8_ALLOW_BOM 0x0040
	43	#define UTF8_ALLOW_FFFF 0x0080
	44	#define UTF8_ALLOW_LONG 0x0100
	45	#define UTF8_ALLOW_ANYUV (UTF8_ALLOW_EMPTY\|UTF8_ALLOW_FE_FF\|\
	46	UTF8_ALLOW_SURROGATE\|UTF8_ALLOW_BOM\|\
	47	UTF8_ALLOW_FFFF\|UTF8_ALLOW_LONG)
a4bf32d5 A	48	#define UTF8_ALLOW_ANY 0x00ff
	49	#define UTF8_CHECK_ONLY 0x0100
	50
feb4a48f JH	51	#define UNICODE_SURROGATE_FIRST 0xd800
	52	#define UNICODE_SURROGATE_LAST 0xdfff
	53	#define UNICODE_REPLACEMENT 0xfffd
	54	#define UNICODE_BYTER_ORDER_MARK 0xfffe
	55	#define UNICODE_ILLEGAL 0xffff
	56
	57	#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \
	58	(c) <= UNICODE_SURROGATE_LAST)
	59	#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACMENT)
	60	#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTER_ORDER_MARK)
	61	#define UNICODE_IS_ILLEGAL(c) ((c) == UNICODE_ILLEGAL)
	62
6f06b55f	63	#define UTF8SKIP(s) PL_utf8skip[(U8)s]
7e2040f0	64
da27fb41 JH	65	#define UTF8_QUAD_MAX UINT64_C(0x1000000000)
da27fb41 JH	66
1145892d JH	67	#define UTF8_IS_ASCII(c) (((U8)c) < 0x80)
	68	#define UTF8_IS_START(c) (((U8)c) >= 0xc0 && (((U8)c) <= 0xfd))
	69	#define UTF8_IS_CONTINUATION(c) (((U8)c) >= 0x80 && (((U8)c) <= 0xbf))
	70	#define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80)
feb4a48f	71
1145892d	72	#define UTF8_CONTINUATION_MASK ((U8)0x3f)
feb4a48f	73	#define UTF8_ACCUMULATION_SHIFT 6
1145892d JH	74	#define UTF8_ACCUMULATE(old, new) ((old) << UTF8_ACCUMULATION_SHIFT \| (((U8)new) & UTF8_CONTINUATION_MASK))
	75
	76	#define UTF8_EIGHT_BIT_HI(c) ( (((U8)(c))>>6) \|0xc0)
	77	#define UTF8_EIGHT_BIT_LO(c) (((((U8)(c)) )&0x3f)\|0x80)
feb4a48f	78
e2439105	79	#ifdef HAS_QUAD
a4bf32d5	80	#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
e2439105 GS	81	(uv) < 0x800 ? 2 : \
	82	(uv) < 0x10000 ? 3 : \
	83	(uv) < 0x200000 ? 4 : \
	84	(uv) < 0x4000000 ? 5 : \
	85	(uv) < 0x80000000 ? 6 : \
da27fb41	86	(uv) < UTF8_QUAD_MAX ? 7 : 13 )
e2439105 GS	87	#else
e2439105 GS	88	/* No, I'm not even going to TRY putting #ifdef inside a #define */
a4bf32d5	89	#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
e2439105 GS	90	(uv) < 0x800 ? 2 : \
	91	(uv) < 0x10000 ? 3 : \
	92	(uv) < 0x200000 ? 4 : \
	93	(uv) < 0x4000000 ? 5 : \
	94	(uv) < 0x80000000 ? 6 : 7 )
	95	#endif
	96
a4bf32d5	97
7e2040f0 GS	98	/*
	99	* Note: we try to be careful never to call the isXXX_utf8() functions
	100	* unless we're pretty sure we've seen the beginning of a UTF-8 character
	101	* (that is, the two high bits are set). Otherwise we risk loading in the
	102	* heavy-duty SWASHINIT and SWASHGET routines unnecessarily.
	103	*/
72d299db GS	104	#ifdef EBCDIC
	105	#define isIDFIRST_lazy_if(p,c) isIDFIRST(*(p))
	106	#define isALNUM_lazy_if(p,c) isALNUM(*(p))
	107	#else
	108	#define isIDFIRST_lazy_if(p,c) ((IN_BYTE \|\| (!c \|\| (((U8)p) < 0xc0))) \
7e2040f0 GS	109	? isIDFIRST(*(p)) \
7e2040f0 GS	110	: isIDFIRST_utf8((U8*)p))
72d299db	111	#define isALNUM_lazy_if(p,c) ((IN_BYTE \|\| (!c \|\| (((U8)p) < 0xc0))) \
7e2040f0 GS	112	? isALNUM(*(p)) \
7e2040f0 GS	113	: isALNUM_utf8((U8*)p))
72d299db	114	#endif
7e2040f0 GS	115	#define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1)
7e2040f0 GS	116	#define isALNUM_lazy(p) isALNUM_lazy_if(p,1)