This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
In utf8decode.t, remove the \x sequence strings of bytes
[perl5.git] / t / op / utf8decode.t
CommitLineData
a9917092
JH
1#!./perl
2
3BEGIN {
4 chdir 't' if -d 't';
5 @INC = '../lib';
680218c4 6 require './test.pl';
daf0f78e
NIS
7}
8
9{
10 my $wide = v256;
11 use bytes;
ffbc6a93
JH
12 my $ordwide = ord($wide);
13 printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide;
680218c4
NC
14 skip_all('UTF-EBCDIC (not UTF-8) used here') if $ordwide == 140;
15
16 if ($ordwide != 196) {
ffbc6a93
JH
17 printf "# v256 starts with 0x%02x\n", $ordwide;
18 }
a9917092
JH
19}
20
3b0e0cb6 21no utf8;
ffc61ed2 22
a9917092
JH
23# This table is based on Markus Kuhn's UTF-8 Decode Stress Tester,
24# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt,
daf0f78e 25# version dated 2000-09-02.
a9917092 26
a9917092
JH
27my @MK = split(/\n/, <<__EOMK__);
281 Correct UTF-8
bb6a3342 291.1.1 y - 11 ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5 5
daf0f78e 302 Boundary conditions
a9917092 312.1 First possible sequence of certain length
bb6a3342
NC
322.1.1 y 0 1 00 1
332.1.2 y 80 2 c2:80 1
342.1.3 y 800 3 e0:a0:80 1
352.1.4 y 10000 4 f0:90:80:80 1
362.1.5 y 200000 5 f8:88:80:80:80 1
372.1.6 y 4000000 6 fc:84:80:80:80:80 1
a9917092 382.2 Last possible sequence of certain length
bb6a3342
NC
392.2.1 y 7f 1 7f 1
402.2.2 y 7ff 2 df:bf 1
8567041c 41# The ffff is legal by default since 872c91ae155f6880
bb6a3342
NC
422.2.3 y ffff 3 ef:bf:bf 1 character 0xffff
432.2.4 y 1fffff 4 f7:bf:bf:bf 1
442.2.5 y 3ffffff 5 fb:bf:bf:bf:bf 1
452.2.6 y 7fffffff 6 fd:bf:bf:bf:bf:bf 1
a9917092 462.3 Other boundary conditions
bb6a3342
NC
472.3.1 y d7ff 3 ed:9f:bf 1
482.3.2 y e000 3 ee:80:80 1
492.3.3 y fffd 3 ef:bf:bd 1
502.3.4 y 10ffff 4 f4:8f:bf:bf 1
512.3.5 y 110000 4 f4:90:80:80 1
a9917092
JH
523 Malformed sequences
533.1 Unexpected continuation bytes
bb6a3342
NC
543.1.1 n - 1 80 - unexpected continuation byte 0x80
553.1.2 n - 1 bf - unexpected continuation byte 0xbf
563.1.3 n - 2 80:bf - unexpected continuation byte 0x80
573.1.4 n - 3 80:bf:80 - unexpected continuation byte 0x80
583.1.5 n - 4 80:bf:80:bf - unexpected continuation byte 0x80
593.1.6 n - 5 80:bf:80:bf:80 - unexpected continuation byte 0x80
603.1.7 n - 6 80:bf:80:bf:80:bf - unexpected continuation byte 0x80
613.1.8 n - 7 80:bf:80:bf:80:bf:80 - unexpected continuation byte 0x80
623.1.9 n - 64 80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf - unexpected continuation byte 0x80
a9917092 633.2 Lonely start characters
bb6a3342
NC
643.2.1 n - 64 c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xc0
653.2.2 n - 32 e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xe0
663.2.3 n - 16 f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf0
673.2.4 n - 8 f8:20:f9:20:fa:20:fb:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf8
683.2.5 n - 4 fc:20:fd:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xfc
a9917092 693.3 Sequences with last continuation byte missing
bb6a3342
NC
703.3.1 n - 1 c0 - 1 byte, need 2
713.3.2 n - 2 e0:80 - 2 bytes, need 3
723.3.3 n - 3 f0:80:80 - 3 bytes, need 4
733.3.4 n - 4 f8:80:80:80 - 4 bytes, need 5
743.3.5 n - 5 fc:80:80:80:80 - 5 bytes, need 6
753.3.6 n - 1 df - 1 byte, need 2
763.3.7 n - 2 ef:bf - 2 bytes, need 3
773.3.8 n - 3 f7:bf:bf - 3 bytes, need 4
783.3.9 n - 4 fb:bf:bf:bf - 4 bytes, need 5
793.3.10 n - 5 fd:bf:bf:bf:bf - 5 bytes, need 6
a9917092 803.4 Concatenation of incomplete sequences
bb6a3342 813.4.1 n - 30 c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf - unexpected non-continuation byte 0xe0, immediately after start byte 0xc0
a9917092 823.5 Impossible bytes
bb6a3342
NC
833.5.1 n - 1 fe - byte 0xfe
843.5.2 n - 1 ff - byte 0xff
853.5.3 n - 4 fe:fe:ff:ff - byte 0xfe
a9917092
JH
864 Overlong sequences
874.1 Examples of an overlong ASCII character
bb6a3342
NC
884.1.1 n - 2 c0:af - 2 bytes, need 1
894.1.2 n - 3 e0:80:af - 3 bytes, need 1
904.1.3 n - 4 f0:80:80:af - 4 bytes, need 1
914.1.4 n - 5 f8:80:80:80:af - 5 bytes, need 1
924.1.5 n - 6 fc:80:80:80:80:af - 6 bytes, need 1
a9917092 934.2 Maximum overlong sequences
bb6a3342
NC
944.2.1 n - 2 c1:bf - 2 bytes, need 1
954.2.2 n - 3 e0:9f:bf - 3 bytes, need 2
964.2.3 n - 4 f0:8f:bf:bf - 4 bytes, need 3
974.2.4 n - 5 f8:87:bf:bf:bf - 5 bytes, need 4
984.2.5 n - 6 fc:83:bf:bf:bf:bf - 6 bytes, need 5
a9917092 994.3 Overlong representation of the NUL character
bb6a3342
NC
1004.3.1 n - 2 c0:80 - 2 bytes, need 1
1014.3.2 n - 3 e0:80:80 - 3 bytes, need 1
1024.3.3 n - 4 f0:80:80:80 - 4 bytes, need 1
1034.3.4 n - 5 f8:80:80:80:80 - 5 bytes, need 1
1044.3.5 n - 6 fc:80:80:80:80:80 - 6 bytes, need 1
a9917092
JH
1055 Illegal code positions
1065.1 Single UTF-16 surrogates
bb6a3342
NC
1075.1.1 y - 3 ed:a0:80 - UTF-16 surrogate 0xd800
1085.1.2 y - 3 ed:ad:bf - UTF-16 surrogate 0xdb7f
1095.1.3 y - 3 ed:ae:80 - UTF-16 surrogate 0xdb80
1105.1.4 y - 3 ed:af:bf - UTF-16 surrogate 0xdbff
1115.1.5 y - 3 ed:b0:80 - UTF-16 surrogate 0xdc00
1125.1.6 y - 3 ed:be:80 - UTF-16 surrogate 0xdf80
1135.1.7 y - 3 ed:bf:bf - UTF-16 surrogate 0xdfff
a9917092 1145.2 Paired UTF-16 surrogates
bb6a3342
NC
1155.2.1 y - 6 ed:a0:80:ed:b0:80 - UTF-16 surrogate 0xd800
1165.2.2 y - 6 ed:a0:80:ed:bf:bf - UTF-16 surrogate 0xd800
1175.2.3 y - 6 ed:ad:bf:ed:b0:80 - UTF-16 surrogate 0xdb7f
1185.2.4 y - 6 ed:ad:bf:ed:bf:bf - UTF-16 surrogate 0xdb7f
1195.2.5 y - 6 ed:ae:80:ed:b0:80 - UTF-16 surrogate 0xdb80
1205.2.6 y - 6 ed:ae:80:ed:bf:bf - UTF-16 surrogate 0xdb80
1215.2.7 y - 6 ed:af:bf:ed:b0:80 - UTF-16 surrogate 0xdbff
1225.2.8 y - 6 ed:af:bf:ed:bf:bf - UTF-16 surrogate 0xdbff
a9917092 1235.3 Other illegal code positions
bb6a3342 1245.3.1 y - 3 ef:bf:be - byte order mark 0xfffe
8567041c 125# The ffff is legal by default since 872c91ae155f6880
bb6a3342 1265.3.2 y - 3 ef:bf:bf - character 0xffff
a9917092
JH
127__EOMK__
128
129# 104..181
130{
a9917092
JH
131 my $id;
132
35bcd338
JH
133 local $SIG{__WARN__} = sub {
134 print "# $id: @_";
8567041c 135 $@ .= "@_";
35bcd338 136 };
a9917092 137
35bcd338
JH
138 sub warn_unpack_U {
139 $@ = '';
8567041c 140 my @null = unpack('C0U*', $_[0]);
35bcd338 141 return $@;
a9917092
JH
142 }
143
144 for (@MK) {
145 if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) {
146 # print "# $_\n";
bb6a3342 147 } elsif (/^(\d+\.\d+\.\d+[bu]?)\s+([yn])\s+([0-9a-f]{1,8}|-)\s+(\d+)\s+([0-9a-f]{2}(?::[0-9a-f]{2})*)(?:\s+((?:\d+|-)(?:\s+(.+))?))?$/) {
a9917092 148 $id = $1;
bb6a3342
NC
149 my ($okay, $Unicode, $byteslen, $hex, $charslen, $experr) =
150 ($2, $3, $4, $5, $6, $7);
a9917092 151 my @hex = split(/:/, $hex);
680218c4 152 is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen');
bb6a3342
NC
153 my $octets = join '', map {chr hex $_} @hex;
154 is(length $octets, $byteslen, 'Number of octets tallies with byteslen');
155 my $warn = warn_unpack_U($octets);
a9917092 156 if ($okay eq 'y') {
680218c4
NC
157 is($warn, '', "No warnings expected for $id");
158 } elsif ($okay ne 'n') {
159 is($okay, 'n', "Confused test description for $id");
160 } elsif($experr) {
161 like($warn, qr/$experr/, "Expected warning for $id");
162 } else {
163 isnt($warn, '', "Expect a warning for $id");
a9917092 164 }
a9917092 165 } else {
680218c4 166 fail("unknown format '$_'");
a9917092
JH
167 }
168 }
169}
680218c4
NC
170
171done_testing();