Commit | Line | Data |
---|---|---|
1d72bdf6 NIS |
1 | /* utfebcdic.h |
2 | * | |
83706693 RGS |
3 | * Copyright (C) 2001, 2002, 2003, 2005, 2006, 2007, 2009 by Larry Wall, |
4 | * Nick Ing-Simmons, and others | |
1d72bdf6 NIS |
5 | * |
6 | * You may distribute under the terms of either the GNU General Public | |
7 | * License or the Artistic License, as specified in the README file. | |
8 | * | |
9 | * Macros to implement UTF-EBCDIC as perl's internal encoding | |
10 | * Taken from version 7.1 of Unicode Techical Report #16: | |
11 | * http://www.unicode.org/unicode/reports/tr16 | |
fe749c9a KW |
12 | * |
13 | * To summarize, the way it works is: | |
14 | * To convert an EBCDIC character to UTF-EBCDIC: | |
15 | * 1) convert to Unicode. The table in this file that does this is for | |
16 | * EBCDIC bytes is PL_e2a (with inverse PLa2e). The 'a' stands for | |
17 | * ASCIIish, meaning latin1. | |
18 | * 2) convert that to a utf8-like string called I8 with variant characters | |
19 | * occupying multiple bytes. This step is similar to the utf8-creating | |
20 | * step from Unicode, but the details are different. There is a chart | |
21 | * about the bit patterns in a comment later in this file. But | |
22 | * essentially here are the differences: | |
23 | * UTF8 I8 | |
24 | * invariant byte starts with 0 starts with 0 or 100 | |
25 | * continuation byte starts with 10 starts with 101 | |
26 | * start byte same in both: if the code point requires N bytes, | |
27 | * then the leading N bits are 1, followed by a 0. (No | |
28 | * trailing 0 for the very largest possible allocation | |
29 | * in I8, far beyond the current Unicode standard's | |
30 | * max, as shown in the comment later in this file.) | |
31 | * 3) Use the table published in tr16 to convert each byte from step 2 into | |
b3ab6785 | 32 | * final UTF-EBCDIC. The table in this file is PL_utf2e, and its inverse |
fe749c9a KW |
33 | * is PL_e2utf. They are constructed so that all EBCDIC invariants remain |
34 | * invariant, but no others do. For example, the ordinal value of 'A' is | |
35 | * 193 in EBCDIC, and also is 193 in UTF-EBCDIC. Step 1) converts it to | |
36 | * 65, Step 2 leaves it at 65, and Step 3 converts it back to 193. As an | |
37 | * example of how a variant character works, take LATIN SMALL LETTER Y | |
38 | * WITH DIAERESIS, which is typicially 0xDF in EBCDIC. Step 1 converts it | |
39 | * to the Unicode value, 0xFF. Step 2 converts that to two bytes = | |
cbbb00c6 KW |
40 | * 11000111 10111111 = C7 BF, and Step 3 converts those to 0x8B 0x73. The |
41 | * table is constructed so that the first bytes of a variant will always | |
42 | * have its upper bit set (at least in the encodings that Perl recognizes, | |
43 | * and probably all). | |
fe749c9a KW |
44 | * |
45 | * If you're starting from Unicode, skip step 1. For UTF-EBCDIC to straight | |
46 | * EBCDIC, reverse the steps. | |
47 | * | |
48 | * The EBCDIC invariants have been chosen to be those characters whose Unicode | |
49 | * equivalents have ordinal numbers less than 160, that is the same characters | |
50 | * that are expressible in ASCII, plus the C1 controls. So there are 160 | |
51 | * invariants instead of the 128 in UTF-8. (My guess is that this is because | |
52 | * the C1 control NEL (and maybe others) is important in IBM.) | |
53 | * | |
54 | * The purpose of Step 3 is to make the encoding be invariant for the chosen | |
55 | * characters. This messes up the convenient patterns found in step 2, so | |
56 | * generally, one has to undo step 3 into a temporary to use them. However, | |
57 | * a "shadow", or parallel table, PL_utf8skip, has been constructed so that for | |
58 | * each byte, it says how long the sequence is if that byte were to begin it | |
59 | * | |
60 | * There are actually 3 slightly different UTF-EBCDIC encodings in this file, | |
61 | * one for each of the code pages recognized by Perl. That means that there | |
62 | * are actually three different sets of tables, one for each code page. (If | |
63 | * Perl is compiled on platforms using other EBCDIC code pages, it may not | |
64 | * compile, or silently mistake it for one of the three.) | |
65 | * | |
66 | * EBCDIC characters above 0xFF are the same as Unicode in Perl's | |
67 | * implementation of all 3 encodings, so for those Step 1 is trivial. | |
68 | * | |
69 | * (Note that the entries for invariant characters are necessarily the same in | |
70 | * PL_e2a and PLe2f, and the same for their inverses.) | |
71 | * | |
72 | * UTF-EBCDIC strings are the same length or longer than UTF-8 representations | |
73 | * of the same string. The maximum code point representable as 2 bytes in | |
74 | * UTF-EBCDIC is 0x3FFF, instead of 0x7FFF in UTF-8. | |
1d72bdf6 NIS |
75 | */ |
76 | ||
77 | START_EXTERN_C | |
78 | ||
79 | #ifdef DOINIT | |
80 | /* Indexed by encoded byte this table gives the length of the sequence. | |
81 | Adapted from the shadow flags table in tr16. | |
c4d5f83a NIS |
82 | The entries marked 9 in tr6 are continuation bytes and are marked |
83 | as length 1 here so that we can recover. | |
1d72bdf6 | 84 | */ |
f5e1abaf | 85 | #if '^' == 95 /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */ |
1d72bdf6 NIS |
86 | EXTCONST unsigned char PL_utf8skip[] = { |
87 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
88 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
89 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
90 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
c4d5f83a NIS |
91 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, |
92 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
93 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
94 | 1,1,1,1,2,2,2,2,2,1,1,1,1,1,1,1, | |
1d72bdf6 NIS |
95 | 2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, |
96 | 2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, | |
97 | 2,1,1,1,1,1,1,1,1,1,2,2,2,1,2,2, | |
98 | 2,2,2,2,2,2,2,3,3,3,3,3,3,1,3,3, | |
99 | 1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3, | |
100 | 1,1,1,1,1,1,1,1,1,1,3,3,4,4,4,4, | |
101 | 1,4,1,1,1,1,1,1,1,1,4,4,4,5,5,5, | |
102 | 1,1,1,1,1,1,1,1,1,1,5,6,6,7,7,1 | |
103 | }; | |
f5e1abaf JH |
104 | #endif |
105 | ||
106 | #if '^' == 106 /* if defined(_OSD_POSIX) POSIX-BC */ | |
107 | unsigned char PL_utf8skip[] = { | |
108 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
109 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
110 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
111 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
112 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
113 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
114 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
115 | 1,1,1,1,2,2,2,2,2,3,1,1,1,1,1,1, | |
116 | 2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, | |
117 | 2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, | |
118 | 2,3,1,1,1,1,1,1,1,1,2,2,2,3,2,2, | |
119 | 1,2,2,2,2,2,2,3,3,3,2,1,1,1,3,3, | |
120 | 4,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3, | |
121 | 1,1,1,1,1,1,1,1,1,1,3,3,4,6,4,4, | |
122 | 7,4,1,1,1,1,1,1,1,1,4,4,4,5,5,5, | |
123 | 1,1,1,1,1,1,1,1,1,1,5,1,6,1,7,1 | |
124 | }; | |
125 | #endif | |
126 | ||
127 | #if '^' == 176 /* if defined(??) (OS/400?) 037 */ | |
128 | unsigned char PL_utf8skip[] = { | |
129 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
130 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
131 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
132 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
133 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
134 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2, | |
135 | 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, | |
136 | 1,1,1,1,2,2,2,2,2,1,1,1,1,1,1,1, | |
137 | 2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, | |
138 | 2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, | |
139 | 2,1,1,1,1,1,1,1,1,1,2,2,2,3,2,2, | |
140 | 1,2,2,2,2,2,2,3,3,3,1,1,3,3,3,3, | |
141 | 1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3, | |
142 | 1,1,1,1,1,1,1,1,1,1,3,3,4,4,4,4, | |
143 | 1,4,1,1,1,1,1,1,1,1,4,4,4,5,5,5, | |
144 | 1,1,1,1,1,1,1,1,1,1,5,6,6,7,7,1 | |
145 | }; | |
146 | #endif | |
1d72bdf6 | 147 | |
fe749c9a KW |
148 | /* Transform tables from tr16 applied after encoding to render encoding EBCDIC |
149 | * like, meaning that all the invariants are actually invariant, eg, that 'A' | |
150 | * remains 'A' */ | |
1d72bdf6 | 151 | |
f5e1abaf | 152 | #if '^' == 95 /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */ |
1d72bdf6 NIS |
153 | EXTCONST unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-1047) */ |
154 | 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
155 | 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, | |
156 | 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, | |
157 | 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, | |
158 | 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, | |
159 | 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D, | |
160 | 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, | |
161 | 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, | |
162 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, | |
163 | 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, | |
164 | 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, | |
165 | 0x57, 0x58, 0x59, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x70, 0x71, 0x72, 0x73, | |
166 | 0x74, 0x75, 0x76, 0x77, 0x78, 0x80, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x90, 0x9A, 0x9B, 0x9C, | |
167 | 0x9D, 0x9E, 0x9F, 0xA0, 0xAA, 0xAB, 0xAC, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, | |
168 | 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBE, 0xBF, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xDA, 0xDB, | |
cbbb00c6 | 169 | 0xDC, 0xDD, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE |
1d72bdf6 NIS |
170 | }; |
171 | ||
f5e1abaf | 172 | EXTCONST unsigned char PL_e2utf[] = { /* EBCDIC (IBM-1047) to UTF-8-mod */ |
1d72bdf6 NIS |
173 | 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, |
174 | 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, | |
175 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, | |
176 | 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, | |
177 | 0x20, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, | |
178 | 0x26, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, | |
179 | 0x2D, 0x2F, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, | |
180 | 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, | |
181 | 0xC5, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, | |
182 | 0xCC, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, | |
183 | 0xD3, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xD4, 0xD5, 0xD6, 0x5B, 0xD7, 0xD8, | |
184 | 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0x5D, 0xE6, 0xE7, | |
185 | 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, | |
186 | 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, | |
187 | 0x5C, 0xF4, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, | |
cbbb00c6 | 188 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x9F |
1d72bdf6 | 189 | }; |
f5e1abaf JH |
190 | #endif /* 1047 */ |
191 | ||
192 | #if '^' == 106 /* if defined(_OSD_POSIX) POSIX-BC */ | |
193 | unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (POSIX-BC) */ | |
194 | 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
195 | 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, | |
196 | 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, | |
197 | 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, | |
198 | 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, | |
199 | 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBB, 0xBC, 0xBD, 0x6A, 0x6D, | |
200 | 0x4A, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, | |
201 | 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xFB, 0x4F, 0xFD, 0xFF, 0x07, | |
202 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, | |
203 | 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0x5F, | |
204 | 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xB0, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, | |
205 | 0x57, 0x58, 0x59, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD0, 0x70, 0x71, 0x72, 0x73, | |
206 | 0x74, 0x75, 0x76, 0x77, 0x78, 0x80, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x90, 0x9A, 0x9B, 0x9C, | |
207 | 0x9D, 0x9E, 0x9F, 0xA0, 0xAA, 0xAB, 0xAC, 0xAE, 0xAF, 0xBA, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, | |
208 | 0xB7, 0xB8, 0xB9, 0xAD, 0x79, 0xA1, 0xBE, 0xBF, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xDA, 0xDB, | |
cbbb00c6 | 209 | 0xDC, 0xC0, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xDD, 0xFC, 0xE0, 0xFE |
f5e1abaf JH |
210 | }; |
211 | ||
212 | unsigned char PL_e2utf[] = { /* EBCDIC (POSIX-BC) to UTF-8-mod */ | |
213 | 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
214 | 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, | |
215 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, | |
216 | 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, | |
217 | 0x20, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0x60, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, | |
218 | 0x26, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x9F, | |
219 | 0x2D, 0x2F, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0x5E, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, | |
220 | 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xE4, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, | |
221 | 0xC5, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, | |
222 | 0xCC, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, | |
223 | 0xD3, 0xE5, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xD4, 0xD5, 0xD6, 0xE3, 0xD7, 0xD8, | |
224 | 0xA9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xD9, 0x5B, 0x5C, 0x5D, 0xE6, 0xE7, | |
225 | 0xF1, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, | |
226 | 0xBB, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xEE, 0xEF, 0xF0, 0xFC, 0xF2, 0xF3, | |
227 | 0xFE, 0xF4, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, | |
cbbb00c6 | 228 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xFB, 0x7B, 0xFD, 0x7D, 0xFF, 0x7E |
f5e1abaf JH |
229 | }; |
230 | #endif /* POSIX-BC */ | |
231 | ||
232 | #if '^' == 176 /* if defined(??) (OS/400?) 037 */ | |
233 | unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-037) */ | |
234 | 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
235 | 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, | |
236 | 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, | |
237 | 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, | |
238 | 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, | |
239 | 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D, | |
240 | 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, | |
241 | 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, | |
242 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x15, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, | |
243 | 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, | |
244 | 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, | |
245 | 0x57, 0x58, 0x59, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x70, 0x71, 0x72, 0x73, | |
246 | 0x74, 0x75, 0x76, 0x77, 0x78, 0x80, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x90, 0x9A, 0x9B, 0x9C, | |
247 | 0x9D, 0x9E, 0x9F, 0xA0, 0xAA, 0xAB, 0xAC, 0xAE, 0xAF, 0x5F, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, | |
248 | 0xB7, 0xB8, 0xB9, 0xAD, 0xBD, 0xBC, 0xBE, 0xBF, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xDA, 0xDB, | |
cbbb00c6 | 249 | 0xDC, 0xDD, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE |
f5e1abaf JH |
250 | }; |
251 | ||
252 | unsigned char PL_e2utf[] = { /* EBCDIC (IBM-037) to UTF-8-mod */ | |
253 | 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
254 | 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, | |
255 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, | |
256 | 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, | |
257 | 0x20, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, | |
258 | 0x26, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0xD9, | |
259 | 0x2D, 0x2F, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, | |
260 | 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, | |
261 | 0xC5, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, | |
262 | 0xCC, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, | |
263 | 0xD3, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xD4, 0xD5, 0xD6, 0xE3, 0xD7, 0xD8, | |
264 | 0x5E, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0x5B, 0x5D, 0xE5, 0xE4, 0xE6, 0xE7, | |
265 | 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, | |
266 | 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, | |
267 | 0x5C, 0xF4, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, | |
cbbb00c6 | 268 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x9F |
f5e1abaf JH |
269 | }; |
270 | #endif /* 037 */ | |
1d72bdf6 NIS |
271 | |
272 | /* These tables moved from perl.h and converted to hex. | |
273 | They map platfrom code page from/to bottom 256 codes of Unicode (i.e. iso-8859-1). | |
274 | */ | |
275 | ||
276 | #if '^' == 95 /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */ | |
277 | EXTCONST unsigned char PL_a2e[] = { /* ASCII (iso-8859-1) to EBCDIC (IBM-1047) */ | |
278 | 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
279 | 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, | |
280 | 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, | |
281 | 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, | |
282 | 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, | |
283 | 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D, | |
284 | 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, | |
285 | 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, | |
286 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, | |
287 | 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, | |
288 | 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBB, 0xB4, 0x9A, 0x8A, 0xB0, 0xCA, 0xAF, 0xBC, | |
289 | 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, | |
290 | 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, | |
291 | 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xBA, 0xAE, 0x59, | |
292 | 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, | |
293 | 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF | |
294 | }; | |
295 | ||
431fc773 | 296 | EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (IBM-1047) to ASCII (iso-8859-1) */ |
1d72bdf6 NIS |
297 | 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, |
298 | 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, | |
299 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, | |
300 | 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, | |
301 | 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, | |
302 | 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, | |
303 | 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, | |
304 | 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, | |
305 | 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, | |
306 | 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, | |
307 | 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0x5B, 0xDE, 0xAE, | |
308 | 0xAC, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xDD, 0xA8, 0xAF, 0x5D, 0xB4, 0xD7, | |
309 | 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, | |
310 | 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, | |
311 | 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, | |
312 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F | |
313 | }; | |
314 | #endif /* 1047 */ | |
315 | ||
316 | #if '^' == 106 /* if defined(_OSD_POSIX) POSIX-BC */ | |
317 | EXTCONST unsigned char PL_a2e[] = { /* ASCII (ISO8859-1) to EBCDIC (POSIX-BC) */ | |
318 | 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
319 | 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, | |
320 | 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, | |
321 | 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, | |
322 | 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, | |
323 | 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBB, 0xBC, 0xBD, 0x6A, 0x6D, | |
324 | 0x4A, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, | |
325 | 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xFB, 0x4F, 0xFD, 0xFF, 0x07, | |
326 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, | |
327 | 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0x5F, | |
328 | 0x41, 0xAA, 0xB0, 0xB1, 0x9F, 0xB2, 0xD0, 0xB5, 0x79, 0xB4, 0x9A, 0x8A, 0xBA, 0xCA, 0xAF, 0xA1, | |
329 | 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, | |
330 | 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, | |
331 | 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xE0, 0xFE, 0xDD, 0xFC, 0xAD, 0xAE, 0x59, | |
332 | 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, | |
333 | 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xC0, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF | |
334 | }; | |
335 | ||
336 | EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (POSIX-BC) to ASCII (ISO8859-1) */ | |
337 | 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
338 | 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, | |
339 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, | |
340 | 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, | |
341 | 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0x60, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, | |
342 | 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x9F, | |
343 | 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0x5E, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, | |
344 | 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0xA8, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, | |
345 | 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, | |
346 | 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, | |
347 | 0xB5, 0xAF, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE, | |
348 | 0xA2, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xAC, 0x5B, 0x5C, 0x5D, 0xB4, 0xD7, | |
349 | 0xF9, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, | |
350 | 0xA6, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xDB, 0xFA, 0xFF, | |
351 | 0xD9, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, | |
352 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0x7B, 0xDC, 0x7D, 0xDA, 0x7E | |
353 | }; | |
354 | #endif /* POSIX-BC */ | |
f5e1abaf | 355 | |
1d72bdf6 NIS |
356 | #if '^' == 176 /* if defined(??) (OS/400?) 037 */ |
357 | EXTCONST unsigned char PL_a2e[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-037) */ | |
358 | 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
359 | 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, | |
360 | 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, | |
361 | 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, | |
362 | 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, | |
363 | 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D, | |
364 | 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, | |
365 | 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, | |
366 | 0x20, 0x21, 0x22, 0x23, 0x24, 0x15, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, | |
367 | 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, | |
368 | 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBD, 0xB4, 0x9A, 0x8A, 0x5F, 0xCA, 0xAF, 0xBC, | |
369 | 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, | |
370 | 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, | |
371 | 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xAD, 0xAE, 0x59, | |
372 | 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, | |
373 | 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF | |
374 | }; | |
375 | ||
376 | EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (IBM-037) to ASCII (ISO8859-1) */ | |
377 | 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, | |
378 | 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, | |
379 | 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, | |
380 | 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, | |
381 | 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, | |
382 | 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0xAC, | |
383 | 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, | |
384 | 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, | |
385 | 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, | |
386 | 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, | |
387 | 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE, | |
388 | 0x5E, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0x5B, 0x5D, 0xAF, 0xA8, 0xB4, 0xD7, | |
389 | 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, | |
390 | 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, | |
391 | 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, | |
392 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F | |
393 | }; | |
394 | #endif /* 037 */ | |
395 | ||
396 | #else | |
397 | EXTCONST unsigned char PL_utf8skip[]; | |
398 | EXTCONST unsigned char PL_e2utf[]; | |
399 | EXTCONST unsigned char PL_utf2e[]; | |
400 | EXTCONST unsigned char PL_e2a[]; | |
401 | EXTCONST unsigned char PL_a2e[]; | |
402 | #endif | |
403 | ||
404 | END_EXTERN_C | |
405 | ||
66af24e1 | 406 | #define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)] |
1d72bdf6 | 407 | |
1e54db1a | 408 | /* EBCDIC-happy ways of converting native code to UTF-8 */ |
1d72bdf6 NIS |
409 | |
410 | /* Native to iso-8859-1 */ | |
411 | #define NATIVE_TO_ASCII(ch) PL_e2a[(U8)(ch)] | |
b3ab6785 | 412 | #define NATIVE8_TO_UNI(ch) NATIVE_TO_ASCII(ch) /* synonym */ |
1d72bdf6 NIS |
413 | #define ASCII_TO_NATIVE(ch) PL_a2e[(U8)(ch)] |
414 | /* Transform after encoding */ | |
415 | #define NATIVE_TO_UTF(ch) PL_e2utf[(U8)(ch)] | |
416 | #define UTF_TO_NATIVE(ch) PL_utf2e[(U8)(ch)] | |
417 | /* Transform in wide UV char space */ | |
418 | #define NATIVE_TO_UNI(ch) (((ch) > 255) ? (ch) : NATIVE_TO_ASCII(ch)) | |
c4d5f83a | 419 | #define UNI_TO_NATIVE(ch) (((ch) > 255) ? (ch) : ASCII_TO_NATIVE(ch)) |
1d72bdf6 NIS |
420 | /* Transform in invariant..byte space */ |
421 | #define NATIVE_TO_NEED(enc,ch) ((enc) ? UTF_TO_NATIVE(NATIVE_TO_ASCII(ch)) : (ch)) | |
422 | #define ASCII_TO_NEED(enc,ch) ((enc) ? UTF_TO_NATIVE(ch) : ASCII_TO_NATIVE(ch)) | |
423 | ||
424 | /* | |
425 | * Note: we should try and be careful never to call the isXXX_utf8() functions | |
426 | * unless we're pretty sure we've seen the beginning of a UTF-EBCDIC character | |
979f2922 TS |
427 | * Otherwise we risk loading in the heavy-duty swash_init and swash_fetch |
428 | * routines unnecessarily. | |
1d72bdf6 NIS |
429 | */ |
430 | ||
0064a8a9 | 431 | #define isIDFIRST_lazy_if(p,c) ((IN_BYTES || (!c || UTF8_IS_INVARIANT(*p))) \ |
1d72bdf6 | 432 | ? isIDFIRST(*(p)) \ |
e1ec3a88 | 433 | : isIDFIRST_utf8((const U8*)p)) |
0064a8a9 | 434 | #define isALNUM_lazy_if(p,c) ((IN_BYTES || (!c || UTF8_IS_INVARIANT(*p))) \ |
1d72bdf6 | 435 | ? isALNUM(*(p)) \ |
e1ec3a88 | 436 | : isALNUM_utf8((const U8*)p)) |
1d72bdf6 NIS |
437 | |
438 | /* | |
439 | The following table is adapted from tr16, it shows UTF-8-mod encoding of Unicode code points. | |
440 | ||
441 | Unicode Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th byte | |
442 | U+0000..U+007F 000000000xxxxxxx 0xxxxxxx | |
443 | U+0080..U+009F 00000000100xxxxx 100xxxxx | |
444 | U+00A0..U+00FF 00000000yyyxxxxx 11000yyy 101xxxxx | |
445 | ||
446 | U+00A0..U+03FF 000000yyyyyxxxxx 110yyyyy 101xxxxx | |
447 | U+0400..U+3FFF 00zzzzyyyyyxxxxx 1110zzzz 101yyyyy 101xxxxx | |
448 | U+4000..U+3FFFF 0wwwzzzzzyyyyyxxxxx 11110www 101zzzzz 101yyyyy 101xxxxx | |
449 | U+40000..U+3FFFFF 0vvwwwwwzzzzzyyyyyxxxxx 111110vv 101wwwww 101zzzzz 101yyyyy 101xxxxx | |
450 | U+400000..U+3FFFFFF 0uvvvvvwwwwwzzzzzyyyyyxxxxx 1111110u 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx | |
451 | U+4000000..U+7FFFFFFF 0tuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 1111111t 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx | |
452 | ||
453 | Note: The UTF-8-Mod transformation is valid for UCS-4 values X'0' to | |
454 | X'7FFFFFFF' (the full extent of ISO/IEC 10646 coding space). | |
455 | ||
456 | */ | |
457 | ||
458 | #define UNISKIP(uv) ( (uv) < 0xA0 ? 1 : \ | |
459 | (uv) < 0x400 ? 2 : \ | |
460 | (uv) < 0x4000 ? 3 : \ | |
461 | (uv) < 0x40000 ? 4 : \ | |
462 | (uv) < 0x400000 ? 5 : \ | |
463 | (uv) < 0x4000000 ? 6 : 7 ) | |
464 | ||
c4d5f83a NIS |
465 | |
466 | #define UNI_IS_INVARIANT(c) ((c) < 0xA0) | |
1d72bdf6 | 467 | /* UTF-EBCDIC sematic macros - transform back into UTF-8-Mod and then compare */ |
b3ab6785 | 468 | #define NATIVE_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE8_TO_UNI(c)) |
c4d5f83a | 469 | #define UTF8_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_UTF(c)) |
1d72bdf6 | 470 | #define UTF8_IS_START(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xE0) != 0xA0) |
2b9d42f0 | 471 | #define UTF8_IS_CONTINUATION(c) ((NATIVE_TO_UTF(c) & 0xE0) == 0xA0) |
1d72bdf6 NIS |
472 | #define UTF8_IS_CONTINUED(c) (NATIVE_TO_UTF(c) >= 0xA0) |
473 | #define UTF8_IS_DOWNGRADEABLE_START(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xF8) == 0xC0) | |
474 | ||
22901f30 RGS |
475 | #define UTF_START_MARK(len) (((len) > 7) ? 0xFF : ((U8)(0xFE << (7-(len))))) |
476 | #define UTF_START_MASK(len) (((len) >= 6) ? 0x01 : (0x1F >> ((len)-2))) | |
1d72bdf6 NIS |
477 | #define UTF_CONTINUATION_MARK 0xA0 |
478 | #define UTF_CONTINUATION_MASK ((U8)0x1f) | |
479 | #define UTF_ACCUMULATION_SHIFT 5 | |
480 | ||
75383841 | 481 | #define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT)|(NATIVE_TO_UTF(new) & UTF_CONTINUATION_MASK)) |
1d72bdf6 NIS |
482 | |
483 | /* UTF-EBCDIC encode a downgradeable value */ | |
484 | #define UTF8_EIGHT_BIT_HI(c) UTF_TO_NATIVE((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2)) | |
485 | #define UTF8_EIGHT_BIT_LO(c) UTF_TO_NATIVE(((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK) | |
486 | ||
e9a8c099 MHM |
487 | /* |
488 | * Local variables: | |
489 | * c-indentation-style: bsd | |
490 | * c-basic-offset: 4 | |
491 | * indent-tabs-mode: t | |
492 | * End: | |
493 | * | |
494 | * ex: set ts=8 sts=4 sw=4 noet: | |
495 | */ |