Commit | Line | Data |
---|---|---|
a9917092 JH |
1 | #!./perl |
2 | ||
3 | BEGIN { | |
4 | chdir 't' if -d 't'; | |
5 | @INC = '../lib'; | |
6 | } | |
7 | ||
ffc61ed2 JH |
8 | no utf8; # this test contains raw 8-bit data on purpose; don't switch to \x{} |
9 | ||
a9917092 JH |
10 | print "1..78\n"; |
11 | ||
12 | my $test = 1; | |
13 | ||
14 | # This table is based on Markus Kuhn's UTF-8 Decode Stress Tester, | |
15 | # http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt, | |
16 | # version dated 2000-09-02. | |
17 | ||
18 | # Note the \0 instead of a raw zero byte in 2.1.1: for example | |
19 | # GNU patch v2.1 has "issues" with raw zero bytes. | |
20 | ||
21 | my @MK = split(/\n/, <<__EOMK__); | |
22 | 1 Correct UTF-8 | |
23 | 1.1.1 y "κόσμε" - 11 ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5 5 | |
24 | 2 Boundary conditions | |
25 | 2.1 First possible sequence of certain length | |
26 | 2.1.1 y "\0" 0 1 00 1 | |
27 | 2.1.2 y "\80" 80 2 c2:80 1 | |
28 | 2.1.3 y "ࠀ" 800 3 e0:a0:80 1 | |
29 | 2.1.4 y "𐀀" 10000 4 f0:90:80:80 1 | |
30 | 2.1.5 y "" 200000 5 f8:88:80:80:80 1 | |
31 | 2.1.6 y "" 4000000 6 fc:84:80:80:80:80 1 | |
32 | 2.2 Last possible sequence of certain length | |
33 | 2.2.1 y "\7f" 7f 1 7f 1 | |
34 | 2.2.2 y "߿" 7ff 2 df:bf 1 | |
35 | # The ffff is illegal unless UTF8_ALLOW_FFFF | |
36 | 2.2.3 n "" ffff 3 ef:bf:bf 1 character 0xffff | |
37 | 2.2.4 y "" 1fffff 4 f7:bf:bf:bf 1 | |
38 | 2.2.5 y "" 3ffffff 5 fb:bf:bf:bf:bf 1 | |
39 | 2.2.6 y "" 7fffffff 6 fd:bf:bf:bf:bf:bf 1 | |
40 | 2.3 Other boundary conditions | |
41 | 2.3.1 y "" d7ff 3 ed:9f:bf 1 | |
42 | 2.3.2 y "" e000 3 ee:80:80 1 | |
43 | 2.3.3 y "�" fffd 3 ef:bf:bd 1 | |
44 | 2.3.4 y "" 10ffff 4 f4:8f:bf:bf 1 | |
45 | 2.3.5 y "" 110000 4 f4:90:80:80 1 | |
46 | 3 Malformed sequences | |
47 | 3.1 Unexpected continuation bytes | |
48 | 3.1.1 n "\80" - 1 80 - unexpected continuation byte 0x80 | |
49 | 3.1.2 n "¿" - 1 bf - unexpected continuation byte 0xbf | |
50 | 3.1.3 n "\80¿" - 2 80:bf - unexpected continuation byte 0x80 | |
51 | 3.1.4 n "\80¿\80" - 3 80:bf:80 - unexpected continuation byte 0x80 | |
52 | 3.1.5 n "\80¿\80¿" - 4 80:bf:80:bf - unexpected continuation byte 0x80 | |
53 | 3.1.6 n "\80¿\80¿\80" - 5 80:bf:80:bf:80 - unexpected continuation byte 0x80 | |
54 | 3.1.7 n "\80¿\80¿\80¿" - 6 80:bf:80:bf:80:bf - unexpected continuation byte 0x80 | |
55 | 3.1.8 n "\80¿\80¿\80¿\80" - 7 80:bf:80:bf:80:bf:80 - unexpected continuation byte 0x80 | |
56 | 3.1.9 n "\80\81\82\83\84\85\86\87\88\89\8a\8b\8c\8d\8e\8f\90\91\92\93\94\95\96\97\98\99\9a\9b\9c\9d\9e\9f ¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿" - 64 80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf - unexpected continuation byte 0x80 | |
57 | 3.2 Lonely start characters | |
421a8bf2 JH |
58 | 3.2.1 n "À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß " - 64 c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 - unexpected non-continuation byte 0x20 after start byte 0xc0 |
59 | 3.2.2 n "à á â ã ä å æ ç è é ê ë ì í î ï " - 32 e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 - unexpected non-continuation byte 0x20 after start byte 0xe0 | |
60 | 3.2.3 n "ð ñ ò ó ô õ ö ÷ " - 16 f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 - unexpected non-continuation byte 0x20 after start byte 0xf0 | |
61 | 3.2.4 n "ø ù ú û " - 8 f8:20:f9:20:fa:20:fb:20 - unexpected non-continuation byte 0x20 after start byte 0xf8 | |
62 | 3.2.5 n "ü ý " - 4 fc:20:fd:20 - unexpected non-continuation byte 0x20 after start byte 0xfc | |
a9917092 JH |
63 | 3.3 Sequences with last continuation byte missing |
64 | 3.3.1 n "À" - 1 c0 - 1 byte, need 2 | |
65 | 3.3.2 n "à\80" - 2 e0:80 - 2 bytes, need 3 | |
66 | 3.3.3 n "ð\80\80" - 3 f0:80:80 - 3 bytes, need 4 | |
67 | 3.3.4 n "ø\80\80\80" - 4 f8:80:80:80 - 4 bytes, need 5 | |
68 | 3.3.5 n "ü\80\80\80\80" - 5 fc:80:80:80:80 - 5 bytes, need 6 | |
69 | 3.3.6 n "ß" - 1 df - 1 byte, need 2 | |
70 | 3.3.7 n "ï¿" - 2 ef:bf - 2 bytes, need 3 | |
71 | 3.3.8 n "÷¿¿" - 3 f7:bf:bf - 3 bytes, need 4 | |
72 | 3.3.9 n "û¿¿¿" - 4 fb:bf:bf:bf - 4 bytes, need 5 | |
73 | 3.3.10 n "ý¿¿¿¿" - 5 fd:bf:bf:bf:bf - 5 bytes, need 6 | |
74 | 3.4 Concatenation of incomplete sequences | |
421a8bf2 | 75 | 3.4.1 n "Àà\80ð\80\80ø\80\80\80ü\80\80\80\80ßï¿÷¿¿û¿¿¿ý¿¿¿¿" - 30 c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf - unexpected non-continuation byte 0xe0 after start byte 0xc0 |
a9917092 JH |
76 | 3.5 Impossible bytes |
77 | 3.5.1 n "þ" - 1 fe - byte 0xfe | |
78 | 3.5.2 n "ÿ" - 1 ff - byte 0xff | |
79 | 3.5.3 n "þþÿÿ" - 4 fe:fe:ff:ff - byte 0xfe | |
80 | 4 Overlong sequences | |
81 | 4.1 Examples of an overlong ASCII character | |
82 | 4.1.1 n "À¯" - 2 c0:af - 2 bytes, need 1 | |
83 | 4.1.2 n "à\80¯" - 3 e0:80:af - 3 bytes, need 1 | |
84 | 4.1.3 n "ð\80\80¯" - 4 f0:80:80:af - 4 bytes, need 1 | |
85 | 4.1.4 n "ø\80\80\80¯" - 5 f8:80:80:80:af - 5 bytes, need 1 | |
86 | 4.1.5 n "ü\80\80\80\80¯" - 6 fc:80:80:80:80:af - 6 bytes, need 1 | |
87 | 4.2 Maximum overlong sequences | |
88 | 4.2.1 n "Á¿" - 2 c1:bf - 2 bytes, need 1 | |
89 | 4.2.2 n "à\9f¿" - 3 e0:9f:bf - 3 bytes, need 2 | |
90 | 4.2.3 n "ð\8f¿¿" - 4 f0:8f:bf:bf - 4 bytes, need 3 | |
91 | 4.2.4 n "ø\87¿¿¿" - 5 f8:87:bf:bf:bf - 5 bytes, need 4 | |
92 | 4.2.5 n "ü\83¿¿¿¿" - 6 fc:83:bf:bf:bf:bf - 6 bytes, need 5 | |
93 | 4.3 Overlong representation of the NUL character | |
94 | 4.3.1 n "À\80" - 2 c0:80 - 2 bytes, need 1 | |
95 | 4.3.2 n "à\80\80" - 3 e0:80:80 - 3 bytes, need 1 | |
96 | 4.3.3 n "ð\80\80\80" - 4 f0:80:80:80 - 4 bytes, need 1 | |
97 | 4.3.4 n "ø\80\80\80\80" - 5 f8:80:80:80:80 - 5 bytes, need 1 | |
98 | 4.3.5 n "ü\80\80\80\80\80" - 6 fc:80:80:80:80:80 - 6 bytes, need 1 | |
99 | 5 Illegal code positions | |
100 | 5.1 Single UTF-16 surrogates | |
101 | 5.1.1 n "" - 3 ed:a0:80 - UTF-16 surrogate 0xd800 | |
102 | 5.1.2 n "" - 3 ed:ad:bf - UTF-16 surrogate 0xdb7f | |
103 | 5.1.3 n "" - 3 ed:ae:80 - UTF-16 surrogate 0xdb80 | |
104 | 5.1.4 n "" - 3 ed:af:bf - UTF-16 surrogate 0xdbff | |
105 | 5.1.5 n "" - 3 ed:b0:80 - UTF-16 surrogate 0xdc00 | |
106 | 5.1.6 n "" - 3 ed:be:80 - UTF-16 surrogate 0xdf80 | |
107 | 5.1.7 n "" - 3 ed:bf:bf - UTF-16 surrogate 0xdfff | |
108 | 5.2 Paired UTF-16 surrogates | |
109 | 5.2.1 n "" - 6 ed:a0:80:ed:b0:80 - UTF-16 surrogate 0xd800 | |
110 | 5.2.2 n "" - 6 ed:a0:80:ed:bf:bf - UTF-16 surrogate 0xd800 | |
111 | 5.2.3 n "" - 6 ed:ad:bf:ed:b0:80 - UTF-16 surrogate 0xdb7f | |
112 | 5.2.4 n "" - 6 ed:ad:bf:ed:bf:bf - UTF-16 surrogate 0xdb7f | |
113 | 5.2.5 n "" - 6 ed:ae:80:ed:b0:80 - UTF-16 surrogate 0xdb80 | |
114 | 5.2.6 n "" - 6 ed:ae:80:ed:bf:bf - UTF-16 surrogate 0xdb80 | |
115 | 5.2.7 n "" - 6 ed:af:bf:ed:b0:80 - UTF-16 surrogate 0xdbff | |
116 | 5.2.8 n "" - 6 ed:af:bf:ed:bf:bf - UTF-16 surrogate 0xdbff | |
117 | 5.3 Other illegal code positions | |
118 | 5.3.1 n "" - 3 ef:bf:be - byte order mark 0xfffe | |
119 | # The ffff is illegal unless UTF8_ALLOW_FFFF | |
120 | 5.3.2 n "" - 3 ef:bf:bf - character 0xffff | |
121 | __EOMK__ | |
122 | ||
123 | # 104..181 | |
124 | { | |
125 | my $WARNCNT; | |
126 | my $id; | |
127 | ||
128 | local $SIG{__WARN__} = | |
129 | sub { | |
421a8bf2 | 130 | print "# $id: @_"; |
a9917092 JH |
131 | $WARNCNT++; |
132 | $WARNMSG = "@_"; | |
133 | }; | |
134 | ||
135 | sub moan { | |
136 | print "$id: @_"; | |
137 | } | |
138 | ||
139 | sub test_unpack_U { | |
140 | $WARNCNT = 0; | |
141 | $WARNMSG = ""; | |
142 | unpack('U*', $_[0]); | |
143 | } | |
144 | ||
145 | for (@MK) { | |
146 | if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) { | |
147 | # print "# $_\n"; | |
148 | } elsif (/^(\d+\.\d+\.\d+[bu]?)\s+([yn])\s+"(.+)"\s+([0-9a-f]{1,8}|-)\s+(\d+)\s+([0-9a-f]{2}(?::[0-9a-f]{2})*)(?:\s+((?:\d+|-)(?:\s+(.+))?))?$/) { | |
149 | $id = $1; | |
150 | my ($okay, $bytes, $Unicode, $byteslen, $hex, $charslen, $error) = | |
151 | ($2, $3, $4, $5, $6, $7, $8); | |
152 | my @hex = split(/:/, $hex); | |
153 | unless (@hex == $byteslen) { | |
154 | my $nhex = @hex; | |
155 | moan "amount of hex ($nhex) not equal to byteslen ($byteslen)\n"; | |
156 | } | |
157 | { | |
158 | use bytes; | |
159 | my $bytesbyteslen = length($bytes); | |
160 | unless ($bytesbyteslen == $byteslen) { | |
161 | moan "bytes length() ($bytesbyteslen) not equal to $byteslen\n"; | |
162 | } | |
163 | } | |
164 | if ($okay eq 'y') { | |
165 | test_unpack_U($bytes); | |
166 | if ($WARNCNT) { | |
167 | moan "unpack('U*') false negative\n"; | |
168 | print "not "; | |
169 | } | |
170 | } elsif ($okay eq 'n') { | |
171 | test_unpack_U($bytes); | |
172 | if ($WARNCNT == 0 || ($error ne '' && $WARNMSG !~ /$error/)) { | |
173 | moan "unpack('U*') false positive\n"; | |
174 | print "not "; | |
175 | } | |
176 | } | |
177 | print "ok $test\n"; | |
178 | $test++; | |
179 | } else { | |
180 | moan "unknown format\n"; | |
181 | } | |
182 | } | |
183 | } |