# define USE_UTF8_IN_NAMES (PL_hints & HINT_UTF8)
#endif
+#include "regcharclass.h"
+#include "unicode_constants.h"
+
/* For to_utf8_fold_flags, q.v. */
#define FOLD_FLAGS_LOCALE 0x1
#define FOLD_FLAGS_FULL 0x2
#define _CORE_SWASH_INIT_RETURN_IF_UNDEF 0x2
#define _CORE_SWASH_INIT_ACCEPT_INVLIST 0x4
+#define uvchr_to_utf8(a,b) uvchr_to_utf8_flags(a,b,0)
+#define uvchr_to_utf8_flags(d,uv,flags) \
+ uvoffuni_to_utf8_flags(d,NATIVE_TO_UNI(uv),flags)
+#define utf8_to_uvchr_buf(s, e, lenp) \
+ utf8n_to_uvchr(s, (U8*)(e) - (U8*)(s), lenp, \
+ ckWARN_d(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY)
+
#define to_uni_fold(c, p, lenp) _to_uni_fold_flags(c, p, lenp, FOLD_FLAGS_FULL)
#define to_utf8_fold(c, p, lenp) _to_utf8_fold_flags(c, p, lenp, \
FOLD_FLAGS_FULL, NULL)
END_EXTERN_C
-#include "regcharclass.h"
-#include "unicode_constants.h"
-
/* Native character to/from iso-8859-1. Are the identity functions on ASCII
* platforms */
#define NATIVE_TO_LATIN1(ch) (ch)
#define UNI_TO_NATIVE(ch) (ch)
#define NATIVE_TO_UNI(ch) (ch)
-/* As there are no translations, avoid the function wrapper */
-#define utf8n_to_uvchr utf8n_to_uvoffuni
-#define uvchr_to_utf8(a,b) uvoffuni_to_utf8_flags(a,b,0)
-
/*
The following table is from Unicode 3.2.
/* This defines the 1-bits that are to be in the first byte of a multi-byte
* UTF-8 encoded character that give the number of bytes that comprise the
- * character.
- * */
-#define UTF_START_MARK(len) (((len) > 7) ? 0xFF : (0xFE << (7-(len))))
+ * character. 'len' is the number of bytes in the multi-byte sequence. */
+#define UTF_START_MARK(len) (((len) > 7) ? 0xFF : (0xFF & (0xFE << (7-(len)))))
/* Masks out the initial one bits in a start byte, leaving the real data ones.
- * Doesn't work on an invariant byte */
+ * Doesn't work on an invariant byte. 'len' is the number of bytes in the
+ * multi-byte sequence that comprises the character. */
#define UTF_START_MASK(len) (((len) >= 7) ? 0x00 : (0x1F >> ((len)-2)))
/* This defines the bits that are to be in the continuation bytes of a multi-byte
(((UV) UTF_CONTINUATION_MASK) << ((sizeof(UV) * CHARBITS) \
- UTF_ACCUMULATION_SHIFT))
-#ifdef HAS_QUAD
+#if UVSIZE >= 8
+# define UTF8_QUAD_MAX UINT64_C(0x1000000000)
+
/* Input is a true Unicode (not-native) code point */
#define OFFUNISKIP(uv) ( (uv) < 0x80 ? 1 : \
(uv) < 0x800 ? 2 : \
#error UTF8_MAXBYTES must be at least 12
#endif
+#define MAX_UTF8_TWO_BYTE 0x7FF
+
#define UTF8_MAXBYTES_CASE UTF8_MAXBYTES
#endif /* EBCDIC vs ASCII */
/* Adds a UTF8 continuation byte 'new' of information to a running total code
* point 'old' of all the continuation bytes so far. This is designed to be
- * used in a loop to convert from UTF-8 to the code point represented */
-#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) \
- | (((U8)new) & UTF_CONTINUATION_MASK))
+ * used in a loop to convert from UTF-8 to the code point represented. Note
+ * that this is asymmetric on EBCDIC platforms, in that the 'new' parameter is
+ * the UTF-EBCDIC byte, whereas the 'old' parameter is a Unicode (not EBCDIC)
+ * code point in process of being generated */
+#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) \
+ | ((NATIVE_UTF8_TO_I8((U8)new)) \
+ & UTF_CONTINUATION_MASK))
/* This works in the face of malformed UTF-8. */
#define UTF8_IS_NEXT_CHAR_DOWNGRADEABLE(s, e) (UTF8_IS_DOWNGRADEABLE_START(*s) \
* downgradable */
#define TWO_BYTE_UTF8_TO_NATIVE(HI, LO) \
UNI_TO_NATIVE(UTF8_ACCUMULATE((NATIVE_UTF8_TO_I8(HI) & UTF_START_MASK(2)), \
- NATIVE_UTF8_TO_I8(LO)))
+ (LO)))
/* Should never be used, and be deprecated */
#define TWO_BYTE_UTF8_TO_UNI(HI, LO) NATIVE_TO_UNI(TWO_BYTE_UTF8_TO_NATIVE(HI, LO))
* UTF-8 encoded string) */
#define UTF8_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_UTF8_TO_I8(c))
-#define NATIVE_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_LATIN1(c))
+/* Like the above, but its name implies a non-UTF8 input, and is implemented
+ * differently (for no particular reason) */
+#define NATIVE_BYTE_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_LATIN1(c))
+
+/* Like the above, but accepts any UV as input */
+#define UVCHR_IS_INVARIANT(uv) UNI_IS_INVARIANT(NATIVE_TO_UNI(uv))
#define MAX_PORTABLE_UTF8_TWO_BYTE 0x3FF /* constrained by EBCDIC */
* however this doesn't won't work for ebcdic, and should be avoided. Use
* regen/unicode_constants instead */
#define UTF8_TWO_BYTE_HI_nocast(c) I8_TO_NATIVE_UTF8((NATIVE_TO_UNI(c) \
- >> UTF_ACCUMULATION_SHIFT) | (0xFF & UTF_START_MARK(2)))
+ >> UTF_ACCUMULATION_SHIFT) | UTF_START_MARK(2))
#define UTF8_TWO_BYTE_LO_nocast(c) I8_TO_NATIVE_UTF8((NATIVE_TO_UNI(c) \
& UTF_CONTINUATION_MASK) \
| UTF_CONTINUATION_MARK)
#define UTF8_TWO_BYTE_HI(c) ((U8) (UTF8_TWO_BYTE_HI_nocast(c)))
#define UTF8_TWO_BYTE_LO(c) ((U8) (UTF8_TWO_BYTE_LO_nocast(c)))
-/* This name is used when the source is a single byte. For EBCDIC these could
- * be more efficiently written; the reason is that things above 0xFF have to be
- * special-cased, which is done by the EBCDIC version of NATIVE_TO_UNI() */
-#define UTF8_EIGHT_BIT_HI(c) UTF8_TWO_BYTE_HI((U8)(c))
-#define UTF8_EIGHT_BIT_LO(c) UTF8_TWO_BYTE_LO((U8)(c))
+/* This name is used when the source is a single byte (input not checked).
+ * These expand identically to the TWO_BYTE versions on ASCII platforms, but
+ * use to/from LATIN1 instead of UNI, which on EBCDIC eliminates tests */
+#define UTF8_EIGHT_BIT_HI(c) I8_TO_NATIVE_UTF8((NATIVE_TO_LATIN1(c) \
+ >> UTF_ACCUMULATION_SHIFT) \
+ | UTF_START_MARK(2))
+#define UTF8_EIGHT_BIT_LO(c) I8_TO_NATIVE_UTF8((NATIVE_TO_LATIN1(c) \
+ & UTF_CONTINUATION_MASK) \
+ | UTF_CONTINUATION_MARK)
/* This is illegal in any well-formed UTF-8 in both EBCDIC and ASCII
* as it is only in overlongs. */
#ifdef EBCDIC /* Both versions assume well-formed UTF8 */
# define UTF8_IS_SUPER(s) (NATIVE_UTF8_TO_I8(* (U8*) (s)) >= 0xF9 \
&& (NATIVE_UTF8_TO_I8(* (U8*) (s)) > 0xF9 \
- || (NATIVE_UTF8_TO_I8(* (U8*) ((s)) + 1 >= 0xA2))))
+ || (NATIVE_UTF8_TO_I8(* ((U8*) (s) + 1)) >= 0xA2)))
#else
# define UTF8_IS_SUPER(s) (*(U8*) (s) >= 0xF4 \
&& (*(U8*) (s) > 0xF4 || (*((U8*) (s) + 1) >= 0x90)))
#define UNICODE_IS_SUPER(c) ((c) > PERL_UNICODE_MAX)
#define UNICODE_IS_FE_FF(c) ((c) > 0x7FFFFFFF)
-#ifdef HAS_QUAD
-# define UTF8_QUAD_MAX UINT64_C(0x1000000000)
-#endif
-
#define LATIN_SMALL_LETTER_SHARP_S LATIN_SMALL_LETTER_SHARP_S_NATIVE
#define LATIN_SMALL_LETTER_Y_WITH_DIAERESIS \
LATIN_SMALL_LETTER_Y_WITH_DIAERESIS_NATIVE
#define UNICODE_GREEK_SMALL_LETTER_FINAL_SIGMA 0x03C2
#define UNICODE_GREEK_SMALL_LETTER_SIGMA 0x03C3
#define GREEK_SMALL_LETTER_MU 0x03BC
-#define GREEK_CAPITAL_LETTER_MU 0x039C /* Upper and title case of MICRON */
-#define LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS 0x0178 /* Also is title case */
-#define LATIN_CAPITAL_LETTER_SHARP_S 0x1E9E
-#define LATIN_SMALL_LETTER_LONG_S 0x017F
-#define KELVIN_SIGN 0x212A
-#define ANGSTROM_SIGN 0x212B
+#define GREEK_CAPITAL_LETTER_MU 0x039C /* Upper and title case
+ of MICRON */
+#define LATIN_CAPITAL_LETTER_Y_WITH_DIAERESIS 0x0178 /* Also is title case */
+#define LATIN_CAPITAL_LETTER_SHARP_S 0x1E9E
+#define LATIN_SMALL_LETTER_LONG_S 0x017F
+#define LATIN_SMALL_LIGATURE_LONG_S_T 0xFB05
+#define LATIN_SMALL_LIGATURE_ST 0xFB06
+#define KELVIN_SIGN 0x212A
+#define ANGSTROM_SIGN 0x212B
#define UNI_DISPLAY_ISPRINT 0x0001
#define UNI_DISPLAY_BACKSLASH 0x0002
https://perl5.git.perl.org / perl5.git / blobdiff