This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
In utf8decode.t, move the test data from a heredoc to <DATA>
[perl5.git] / t / op / utf8decode.t
1 #!./perl
2
3 BEGIN {
4     chdir 't' if -d 't';
5     @INC = '../lib';
6     require './test.pl';
7 }
8
9 {
10     my $wide = v256;
11     use bytes;
12     my $ordwide = ord($wide);
13     printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide;
14     skip_all('UTF-EBCDIC (not UTF-8) used here') if $ordwide == 140;
15
16     if ($ordwide != 196) {
17         printf "# v256 starts with 0x%02x\n", $ordwide;
18     }
19 }
20
21 no utf8;
22
23 my $id;
24
25 local $SIG{__WARN__} = sub {
26     print "# $id: @_";
27     $@ .= "@_";
28 };
29
30 sub warn_unpack_U {
31     $@ = '';
32     my @null = unpack('C0U*', $_[0]);
33     return $@;
34 }
35
36 foreach (<DATA>) {
37     if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) {
38         # print "# $_\n";
39     } elsif (/^(\d+\.\d+\.\d+[bu]?)\s+([yn])\s+([0-9a-f]{1,8}|-)\s+(\d+)\s+([0-9a-f]{2}(?::[0-9a-f]{2})*)(?:\s+((?:\d+|-)(?:\s+(.+))?))?$/) {
40         $id = $1;
41         my ($okay, $Unicode, $byteslen, $hex, $charslen, $experr) =
42             ($2, $3, $4, $5, $6, $7);
43         my @hex = split(/:/, $hex);
44         is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen');
45         my $octets = join '', map {chr hex $_} @hex;
46         is(length $octets, $byteslen, 'Number of octets tallies with byteslen');
47         my $warn = warn_unpack_U($octets);
48         if ($okay eq 'y') {
49             is($warn, '', "No warnings expected for $id");
50         } elsif ($okay ne 'n') {
51             is($okay, 'n', "Confused test description for $id");
52         } elsif($experr) {
53             like($warn, qr/$experr/, "Expected warning for $id");
54         } else {
55             isnt($warn, '', "Expect a warning for $id");
56         }
57     } else {
58         fail("unknown format '$_'");
59     }
60 }
61
62 done_testing();
63
64 # This table is based on Markus Kuhn's UTF-8 Decode Stress Tester,
65 # http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt,
66 # version dated 2000-09-02.
67
68 __DATA__
69 1       Correct UTF-8
70 1.1.1 y -               11      ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5        5
71 2       Boundary conditions
72 2.1     First possible sequence of certain length
73 2.1.1 y 0               1       00      1
74 2.1.2 y 80              2       c2:80   1
75 2.1.3 y 800             3       e0:a0:80        1
76 2.1.4 y 10000           4       f0:90:80:80     1
77 2.1.5 y 200000          5       f8:88:80:80:80  1
78 2.1.6 y 4000000         6       fc:84:80:80:80:80       1
79 2.2     Last possible sequence of certain length
80 2.2.1 y 7f              1       7f      1
81 2.2.2 y 7ff             2       df:bf   1
82 # The ffff is legal by default since 872c91ae155f6880
83 2.2.3 y ffff            3       ef:bf:bf        1       character 0xffff
84 2.2.4 y 1fffff          4       f7:bf:bf:bf     1
85 2.2.5 y 3ffffff         5       fb:bf:bf:bf:bf  1
86 2.2.6 y 7fffffff        6       fd:bf:bf:bf:bf:bf       1
87 2.3     Other boundary conditions
88 2.3.1 y d7ff            3       ed:9f:bf        1
89 2.3.2 y e000            3       ee:80:80        1
90 2.3.3 y fffd            3       ef:bf:bd        1
91 2.3.4 y 10ffff          4       f4:8f:bf:bf     1
92 2.3.5 y 110000          4       f4:90:80:80     1
93 3       Malformed sequences
94 3.1     Unexpected continuation bytes
95 3.1.1 n -               1       80      -       unexpected continuation byte 0x80
96 3.1.2 n -               1       bf      -       unexpected continuation byte 0xbf
97 3.1.3 n -               2       80:bf   -       unexpected continuation byte 0x80
98 3.1.4 n -               3       80:bf:80        -       unexpected continuation byte 0x80
99 3.1.5 n -               4       80:bf:80:bf     -       unexpected continuation byte 0x80
100 3.1.6 n -               5       80:bf:80:bf:80  -       unexpected continuation byte 0x80
101 3.1.7 n -               6       80:bf:80:bf:80:bf       -       unexpected continuation byte 0x80
102 3.1.8 n -               7       80:bf:80:bf:80:bf:80    -       unexpected continuation byte 0x80
103 3.1.9 n -       64      80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf -       unexpected continuation byte 0x80
104 3.2     Lonely start characters
105 3.2.1 n -       64      c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xc0
106 3.2.2 n -       32      e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xe0
107 3.2.3 n -       16      f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xf0
108 3.2.4 n -       8       f8:20:f9:20:fa:20:fb:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xf8
109 3.2.5 n -       4       fc:20:fd:20     -       unexpected non-continuation byte 0x20, immediately after start byte 0xfc
110 3.3     Sequences with last continuation byte missing
111 3.3.1 n -       1       c0      -       1 byte, need 2
112 3.3.2 n -       2       e0:80   -       2 bytes, need 3
113 3.3.3 n -       3       f0:80:80        -       3 bytes, need 4
114 3.3.4 n -       4       f8:80:80:80     -       4 bytes, need 5
115 3.3.5 n -       5       fc:80:80:80:80  -       5 bytes, need 6
116 3.3.6 n -       1       df      -       1 byte, need 2
117 3.3.7 n -       2       ef:bf   -       2 bytes, need 3
118 3.3.8 n -       3       f7:bf:bf        -       3 bytes, need 4
119 3.3.9 n -       4       fb:bf:bf:bf     -       4 bytes, need 5
120 3.3.10 n -      5       fd:bf:bf:bf:bf  -       5 bytes, need 6
121 3.4     Concatenation of incomplete sequences
122 3.4.1 n -       30      c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf       -       unexpected non-continuation byte 0xe0, immediately after start byte 0xc0
123 3.5     Impossible bytes
124 3.5.1 n -       1       fe      -       byte 0xfe
125 3.5.2 n -       1       ff      -       byte 0xff
126 3.5.3 n -       4       fe:fe:ff:ff     -       byte 0xfe
127 4       Overlong sequences
128 4.1     Examples of an overlong ASCII character
129 4.1.1 n -       2       c0:af   -       2 bytes, need 1
130 4.1.2 n -       3       e0:80:af        -       3 bytes, need 1
131 4.1.3 n -       4       f0:80:80:af     -       4 bytes, need 1
132 4.1.4 n -       5       f8:80:80:80:af  -       5 bytes, need 1
133 4.1.5 n -       6       fc:80:80:80:80:af       -       6 bytes, need 1
134 4.2     Maximum overlong sequences
135 4.2.1 n -       2       c1:bf   -       2 bytes, need 1
136 4.2.2 n -       3       e0:9f:bf        -       3 bytes, need 2
137 4.2.3 n -       4       f0:8f:bf:bf     -       4 bytes, need 3
138 4.2.4 n -       5       f8:87:bf:bf:bf  -       5 bytes, need 4
139 4.2.5 n -       6       fc:83:bf:bf:bf:bf       -       6 bytes, need 5
140 4.3     Overlong representation of the NUL character
141 4.3.1 n -       2       c0:80   -       2 bytes, need 1
142 4.3.2 n -       3       e0:80:80        -       3 bytes, need 1
143 4.3.3 n -       4       f0:80:80:80     -       4 bytes, need 1
144 4.3.4 n -       5       f8:80:80:80:80  -       5 bytes, need 1
145 4.3.5 n -       6       fc:80:80:80:80:80       -       6 bytes, need 1
146 5       Illegal code positions
147 5.1     Single UTF-16 surrogates
148 5.1.1 y -       3       ed:a0:80        -       UTF-16 surrogate 0xd800
149 5.1.2 y -       3       ed:ad:bf        -       UTF-16 surrogate 0xdb7f
150 5.1.3 y -       3       ed:ae:80        -       UTF-16 surrogate 0xdb80
151 5.1.4 y -       3       ed:af:bf        -       UTF-16 surrogate 0xdbff
152 5.1.5 y -       3       ed:b0:80        -       UTF-16 surrogate 0xdc00
153 5.1.6 y -       3       ed:be:80        -       UTF-16 surrogate 0xdf80
154 5.1.7 y -       3       ed:bf:bf        -       UTF-16 surrogate 0xdfff
155 5.2     Paired UTF-16 surrogates
156 5.2.1 y -       6       ed:a0:80:ed:b0:80       -       UTF-16 surrogate 0xd800
157 5.2.2 y -       6       ed:a0:80:ed:bf:bf       -       UTF-16 surrogate 0xd800
158 5.2.3 y -       6       ed:ad:bf:ed:b0:80       -       UTF-16 surrogate 0xdb7f
159 5.2.4 y -       6       ed:ad:bf:ed:bf:bf       -       UTF-16 surrogate 0xdb7f
160 5.2.5 y -       6       ed:ae:80:ed:b0:80       -       UTF-16 surrogate 0xdb80
161 5.2.6 y -       6       ed:ae:80:ed:bf:bf       -       UTF-16 surrogate 0xdb80
162 5.2.7 y -       6       ed:af:bf:ed:b0:80       -       UTF-16 surrogate 0xdbff
163 5.2.8 y -       6       ed:af:bf:ed:bf:bf       -       UTF-16 surrogate 0xdbff
164 5.3     Other illegal code positions
165 5.3.1 y -       3       ef:bf:be        -       byte order mark 0xfffe
166 # The ffff is legal by default since 872c91ae155f6880
167 5.3.2 y -       3       ef:bf:bf        -       character 0xffff