t/op/utf8decode.t

   1 #!./perl
   2
   3 BEGIN {
   4     chdir 't' if -d 't';
   5     require './test.pl';
   6     set_up_inc('../lib');
   7 }
   8
   9 $|=1;
  10
  11 {
  12     my $wide = v256;
  13     use bytes;
  14     my $ordwide = ord($wide);
  15     printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide;
  16     skip_all('UTF-8-centric tests (not valid for UTF-EBCDIC)') if $ordwide == 140;
  17     # This could be ported to EBCDIC, but a lot of trouble.
  18     # ext/XS-APItest/t/utf8.t contains comprehensive tests for both platforms
  19
  20     if ($ordwide != 196) {
  21         printf "# v256 starts with 0x%02x\n", $ordwide;
  22     }
  23 }
  24
  25 no utf8;
  26
  27 my $is64bit = length sprintf("%x", ~0) > 8;
  28
  29 foreach (<DATA>) {
  30     if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) {
  31         # print "# $_\n";
  32     } elsif (my ($id, $okay, $Unicode, $byteslen, $hex, $charslen, $experr)
  33              = /^(\d+\.\d+\.\d+[bu]?)   # ID
  34                 \s+(y|n|N-?\d+(?:,\d+)?)  # expect to pass or fail
  35                                           # 'n' means expect one diagnostic
  36                                           # 'N\d+'     means expect this
  37                                           #            number of diagnostics
  38                                           # 'N\d+,\d+' means expect the first
  39                                           #            number of diagnostics
  40                                           #            on a 32-bit system; the
  41                                           #            second number on a
  42                                           #            64-bit one
  43                 \s+([0-9a-f]{1,8}(?:,[0-9a-f]{1,8})*|-) # Unicode characters
  44                 \s+(\d+)                # number of octets
  45                 \s+([0-9a-f]{2}(?::[0-9a-f]{2})*)       # octets in hex
  46                 \s+(\d+|-)              # number of characters
  47                 (?:\s+(.+))?            # expected error (or comment)
  48                 $/x) {
  49         my @hex = split(/:/, $hex);
  50         is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen');
  51         my $octets = join '', map {chr hex $_} @hex;
  52         is(length $octets, $byteslen, 'Number of octets tallies with byteslen');
  53         if ($okay eq 'y') {
  54             my @chars = map {hex $_} split ',', $Unicode;
  55             is(scalar @chars, $charslen, 'Amount of hex tallies with charslen');
  56             my @got;
  57             warning_is(sub {@got = unpack 'C0U*', $octets}, undef,
  58                        "No warnings expected for $id");
  59             is("@got", "@chars", 'Got expected Unicode characters');
  60         } elsif ($okay eq 'n') {
  61             isnt($experr, '', "Expected warning for $id provided");
  62             warnings_like(sub {unpack 'C0U*', $octets}, [qr/$experr/],
  63                          "Only expected warning for $id");
  64         } elsif ($okay !~ /^N-?(\d+)(?:,(\d+))?/) {
  65             is($okay, 'n', "Confused test description for $id");
  66         } else {
  67             my $expect32 = $1;
  68             my $expect64 = $2 // $expect32;
  69             my $expect = ($is64bit) ? $expect64 : $expect32;
  70             my @warnings;
  71
  72             {
  73                 local $SIG{__WARN__} = sub {
  74                     print "# $id: @_";
  75                     push @warnings, "@_";
  76                 };
  77                 unpack 'C0U*', $octets;
  78             }
  79
  80             unless (is(scalar @warnings, $expect, "Expected number of warnings for $id seen")) {
  81                 note(join "", "Got:\n", @warnings);
  82             }
  83             isnt($experr, '', "Expected first warning for $id provided");
  84
  85             my $message;
  86             my $after = "";
  87             if ($expect64 < $expect32 && ! $is64bit) {
  88                 # This is needed for code points above IV_MAX
  89                 #if (       substr($octets, 0, 1) gt "\xfe"
  90                 #    || (   substr($octets, 0, 1) eq "\xfe"
  91                 #        && length $octets > 1
  92                 #        && substr($octets, 1, 1) le "\xbf"
  93                 #        && substr($octets, 1, 1) ge "\x80"))
  94                 #{
  95                     like($warnings[0], qr/overflow/, "overflow warning for $id seen");
  96                     shift @warnings;
  97                     $after .= "overflow";
  98                 #}
  99             }
 100
 101             # The data below assumes that if there is both a 'short' and
 102             # 'non-continuation' malformation, the latter has precedence.  But
 103             # that has changed, and rather than mess with the data, this works
 104             # around that.
 105             if (   @warnings > 1
 106                 && $warnings[0] =~ /short/
 107                 && $warnings[1] =~ /unexpected non-continuation/)
 108             {
 109                 $after .= " and " if $after;
 110                 $after .= "short";
 111                 shift @warnings;
 112             }
 113             $after = "after $after " if $after;
 114
 115             like($warnings[0], qr/$experr/, "Expected first warning ${after}for $id seen");
 116             local $::TODO;
 117             if ($expect < 0) {
 118                 $expect = -$expect;
 119                 $::TODO = "Markus Kuhn states that $expect invalid sequences should be signalled";
 120             }
 121
 122         }
 123     } else {
 124         fail("unknown format '$_'");
 125     }
 126 }
 127
 128 done_testing();
 129
 130 # This table is based on Markus Kuhn's UTF-8 Decode Stress Tester,
 131 # http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt,
 132 # version dated 2015-08-28.
 133 #
 134 # See the code that parses these lines for comments as to the column meanings
 135
 136 __DATA__
 137 1       Correct UTF-8
 138 1.1.1 y 3ba,1f79,3c3,3bc,3b5    11      ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5        5
 139 2       Boundary conditions
 140 2.1     First possible sequence of certain length
 141 2.1.1 y 0               1       00      1
 142 2.1.2 y 80              2       c2:80   1
 143 2.1.3 y 800             3       e0:a0:80        1
 144 2.1.4 y 10000           4       f0:90:80:80     1
 145 2.1.5 y 200000          5       f8:88:80:80:80  1
 146 2.1.6 y 4000000         6       fc:84:80:80:80:80       1
 147 2.2     Last possible sequence of certain length
 148 2.2.1 y 7f              1       7f      1
 149 2.2.2 y 7ff             2       df:bf   1
 150 # The ffff is legal by default since 872c91ae155f6880
 151 2.2.3 y ffff            3       ef:bf:bf        1       character 0xffff
 152 2.2.4 y 1fffff          4       f7:bf:bf:bf     1
 153 2.2.5 y 3ffffff         5       fb:bf:bf:bf:bf  1
 154 2.2.6 y 7fffffff        6       fd:bf:bf:bf:bf:bf       1
 155 2.3     Other boundary conditions
 156 2.3.1 y d7ff            3       ed:9f:bf        1
 157 2.3.2 y e000            3       ee:80:80        1
 158 2.3.3 y fffd            3       ef:bf:bd        1
 159 2.3.4 y 10ffff          4       f4:8f:bf:bf     1
 160 2.3.5 y 110000          4       f4:90:80:80     1
 161 3       Malformed sequences
 162 3.1     Unexpected continuation bytes
 163 3.1.1 n -               1       80      -       unexpected continuation byte 0x80
 164 3.1.2 n -               1       bf      -       unexpected continuation byte 0xbf
 165 3.1.3 N2 -              2       80:bf   -       unexpected continuation byte 0x80
 166 3.1.4 N3 -              3       80:bf:80        -       unexpected continuation byte 0x80
 167 3.1.5 N4 -              4       80:bf:80:bf     -       unexpected continuation byte 0x80
 168 3.1.6 N5 -              5       80:bf:80:bf:80  -       unexpected continuation byte 0x80
 169 3.1.7 N6 -              6       80:bf:80:bf:80:bf       -       unexpected continuation byte 0x80
 170 3.1.8 N7 -              7       80:bf:80:bf:80:bf:80    -       unexpected continuation byte 0x80
 171 3.1.9 N64 -     64      80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf -       unexpected continuation byte 0x80
 172 3.2     Lonely start characters
 173 3.2.1 N34 -     64      c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xc0
 174 3.2.2 N17 -     32      e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xe0
 175 3.2.3 N9 -      16      f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xf0
 176 3.2.4 N6 -      8       f8:20:f9:20:fa:20:fb:20 -       unexpected non-continuation byte 0x20, immediately after start byte 0xf8
 177 3.2.5 N4 -      4       fc:20:fd:20     -       unexpected non-continuation byte 0x20, immediately after start byte 0xfc
 178 3.3     Sequences with last continuation byte missing
 179 3.3.1 N2 -      1       c0      -       1 byte available, need 2
 180 3.3.2 N2 -      2       e0:80   -       2 bytes available, need 3
 181 3.3.3 N2 -      3       f0:80:80        -       3 bytes available, need 4
 182 3.3.4 N2 -      4       f8:80:80:80     -       4 bytes available, need 5
 183 3.3.5 N2 -      5       fc:80:80:80:80  -       5 bytes available, need 6
 184 3.3.6 n -       1       df      -       1 byte available, need 2
 185 3.3.7 n -       2       ef:bf   -       2 bytes available, need 3
 186 3.3.8 n -       3       f7:bf:bf        -       3 bytes available, need 4
 187 3.3.9 n -       4       fb:bf:bf:bf     -       4 bytes available, need 5
 188 3.3.10 n -      5       fd:bf:bf:bf:bf  -       5 bytes available, need 6
 189 3.4     Concatenation of incomplete sequences
 190 3.4.1 N15 -     30      c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf       -       unexpected non-continuation byte 0xe0, immediately after start byte 0xc0
 191 3.5     Impossible bytes (but not with Perl's extended UTF-8)
 192 3.5.1 n -       1       fe      -       1 byte available, need 7
 193 3.5.2 n -       1       ff      -       1 byte available, need 13
 194 3.5.3 N7 -      4       fe:fe:ff:ff     -       byte 0xfe
 195 4       Overlong sequences
 196 4.1     Examples of an overlong ASCII character
 197 4.1.1 n -       2       c0:af   -       overlong
 198 4.1.2 n -       3       e0:80:af        -       overlong
 199 4.1.3 n -       4       f0:80:80:af     -       overlong
 200 4.1.4 n -       5       f8:80:80:80:af  -       overlong
 201 4.1.5 n -       6       fc:80:80:80:80:af       -       overlong
 202 4.2     Maximum overlong sequences
 203 4.2.1 n -       2       c1:bf   -       overlong
 204 4.2.2 n -       3       e0:9f:bf        -       overlong
 205 4.2.3 n -       4       f0:8f:bf:bf     -       overlong
 206 4.2.4 n -       5       f8:87:bf:bf:bf  -       overlong
 207 4.2.5 n -       6       fc:83:bf:bf:bf:bf       -       overlong
 208 4.3     Overlong representation of the NUL character
 209 4.3.1 n -       2       c0:80   -       overlong
 210 4.3.2 n -       3       e0:80:80        -       overlong
 211 4.3.3 n -       4       f0:80:80:80     -       overlong
 212 4.3.4 n -       5       f8:80:80:80:80  -       overlong
 213 4.3.5 n -       6       fc:80:80:80:80:80       -       overlong
 214 5       Illegal code positions
 215 5.1     Single UTF-16 surrogates
 216 5.1.1 y d800    3       ed:a0:80        1       UTF-16 surrogate 0xd800
 217 5.1.2 y db7f    3       ed:ad:bf        1       UTF-16 surrogate 0xdb7f
 218 5.1.3 y db80    3       ed:ae:80        1       UTF-16 surrogate 0xdb80
 219 5.1.4 y dbff    3       ed:af:bf        1       UTF-16 surrogate 0xdbff
 220 5.1.5 y dc00    3       ed:b0:80        1       UTF-16 surrogate 0xdc00
 221 5.1.6 y df80    3       ed:be:80        1       UTF-16 surrogate 0xdf80
 222 5.1.7 y dfff    3       ed:bf:bf        1       UTF-16 surrogate 0xdfff
 223 5.2     Paired UTF-16 surrogates
 224 5.2.1 y d800,dc00       6       ed:a0:80:ed:b0:80       2       UTF-16 surrogates 0xd800, dc00
 225 5.2.2 y d800,dfff       6       ed:a0:80:ed:bf:bf       2       UTF-16 surrogates 0xd800, dfff
 226 5.2.3 y db7f,dc00       6       ed:ad:bf:ed:b0:80       2       UTF-16 surrogates 0xdb7f, dc00
 227 5.2.4 y db7f,dfff       6       ed:ad:bf:ed:bf:bf       2       UTF-16 surrogates 0xdb7f, dfff
 228 5.2.5 y db80,dc00       6       ed:ae:80:ed:b0:80       2       UTF-16 surrogates 0xdb80, dc00
 229 5.2.6 y db80,dfff       6       ed:ae:80:ed:bf:bf       2       UTF-16 surrogates 0xdb80, dfff
 230 5.2.7 y dbff,dc00       6       ed:af:bf:ed:b0:80       2       UTF-16 surrogates 0xdbff, dc00
 231 5.2.8 y dbff,dfff       6       ed:af:bf:ed:bf:bf       2       UTF-16 surrogates 0xdbff, dfff
 232 5.3     Other illegal code positions
 233 5.3.1 y fffe    3       ef:bf:be        1       byte order mark 0xfffe
 234 # The ffff is legal by default since 872c91ae155f6880
 235 5.3.2 y ffff    3       ef:bf:bf        1       non-character 0xffff
 236 5.3.3 y fdd0    3       ef:b7:90        1       non-character 0xfdd0
 237 5.3.3 y fdd1    3       ef:b7:91        1       non-character 0xfdd1
 238 5.3.3 y fdd2    3       ef:b7:92        1       non-character 0xfdd2
 239 5.3.3 y fdd3    3       ef:b7:93        1       non-character 0xfdd3
 240 5.3.3 y fdd4    3       ef:b7:94        1       non-character 0xfdd4
 241 5.3.3 y fdd5    3       ef:b7:95        1       non-character 0xfdd5
 242 5.3.3 y fdd6    3       ef:b7:96        1       non-character 0xfdd6
 243 5.3.3 y fdd7    3       ef:b7:97        1       non-character 0xfdd7
 244 5.3.3 y fdd8    3       ef:b7:98        1       non-character 0xfdd8
 245 5.3.3 y fdd9    3       ef:b7:99        1       non-character 0xfdd9
 246 5.3.3 y fdda    3       ef:b7:9a        1       non-character 0xfdda
 247 5.3.3 y fddb    3       ef:b7:9b        1       non-character 0xfddb
 248 5.3.3 y fddc    3       ef:b7:9c        1       non-character 0xfddc
 249 5.3.3 y fddd    3       ef:b7:9d        1       non-character 0xfddd
 250 5.3.3 y fdde    3       ef:b7:9e        1       non-character 0xfdde
 251 5.3.3 y fddf    3       ef:b7:9f        1       non-character 0xfddf
 252 5.3.3 y fde0    3       ef:b7:a0        1       non-character 0xfde0
 253 5.3.3 y fde1    3       ef:b7:a1        1       non-character 0xfde1
 254 5.3.3 y fde2    3       ef:b7:a2        1       non-character 0xfde2
 255 5.3.3 y fde3    3       ef:b7:a3        1       non-character 0xfde3
 256 5.3.3 y fde4    3       ef:b7:a4        1       non-character 0xfde4
 257 5.3.3 y fde5    3       ef:b7:a5        1       non-character 0xfde5
 258 5.3.3 y fde6    3       ef:b7:a6        1       non-character 0xfde6
 259 5.3.3 y fde7    3       ef:b7:a7        1       non-character 0xfde7
 260 5.3.3 y fde8    3       ef:b7:a8        1       non-character 0xfde8
 261 5.3.3 y fde9    3       ef:b7:a9        1       non-character 0xfde9
 262 5.3.3 y fdea    3       ef:b7:aa        1       non-character 0xfdea
 263 5.3.3 y fdeb    3       ef:b7:ab        1       non-character 0xfdeb
 264 5.3.3 y fdec    3       ef:b7:ac        1       non-character 0xfdec
 265 5.3.3 y fded    3       ef:b7:ad        1       non-character 0xfded
 266 5.3.3 y fdee    3       ef:b7:ae        1       non-character 0xfdee
 267 5.3.3 y fdef    3       ef:b7:af        1       non-character 0xfdef
 268 5.3.4 y 1fffe   4       f0:9f:bf:be     1       non-character 0x1fffe
 269 5.3.4 y 1ffff   4       f0:9f:bf:bf     1       non-character 0x1ffff
 270 5.3.4 y 2fffe   4       f0:af:bf:be     1       non-character 0x2fffe
 271 5.3.4 y 2ffff   4       f0:af:bf:bf     1       non-character 0x2ffff
 272 5.3.4 y 3fffe   4       f0:bf:bf:be     1       non-character 0x3fffe
 273 5.3.4 y 3ffff   4       f0:bf:bf:bf     1       non-character 0x3ffff
 274 5.3.4 y 4fffe   4       f1:8f:bf:be     1       non-character 0x4fffe
 275 5.3.4 y 4ffff   4       f1:8f:bf:bf     1       non-character 0x4ffff
 276 5.3.4 y 5fffe   4       f1:9f:bf:be     1       non-character 0x5fffe
 277 5.3.4 y 5ffff   4       f1:9f:bf:bf     1       non-character 0x5ffff
 278 5.3.4 y 6fffe   4       f1:af:bf:be     1       non-character 0x6fffe
 279 5.3.4 y 6ffff   4       f1:af:bf:bf     1       non-character 0x6ffff
 280 5.3.4 y 7fffe   4       f1:bf:bf:be     1       non-character 0x7fffe
 281 5.3.4 y 7ffff   4       f1:bf:bf:bf     1       non-character 0x7ffff
 282 5.3.4 y 8fffe   4       f2:8f:bf:be     1       non-character 0x8fffe
 283 5.3.4 y 8ffff   4       f2:8f:bf:bf     1       non-character 0x8ffff
 284 5.3.4 y 9fffe   4       f2:9f:bf:be     1       non-character 0x9fffe
 285 5.3.4 y 9ffff   4       f2:9f:bf:bf     1       non-character 0x9ffff
 286 5.3.4 y afffe   4       f2:af:bf:be     1       non-character 0xafffe
 287 5.3.4 y affff   4       f2:af:bf:bf     1       non-character 0xaffff
 288 5.3.4 y bfffe   4       f2:bf:bf:be     1       non-character 0xbfffe
 289 5.3.4 y bffff   4       f2:bf:bf:bf     1       non-character 0xbffff
 290 5.3.4 y cfffe   4       f3:8f:bf:be     1       non-character 0xcfffe
 291 5.3.4 y cffff   4       f3:8f:bf:bf     1       non-character 0xcffff
 292 5.3.4 y dfffe   4       f3:9f:bf:be     1       non-character 0xdfffe
 293 5.3.4 y dffff   4       f3:9f:bf:bf     1       non-character 0xdffff
 294 5.3.4 y efffe   4       f3:af:bf:be     1       non-character 0xefffe
 295 5.3.4 y effff   4       f3:af:bf:bf     1       non-character 0xeffff
 296 5.3.4 y ffffe   4       f3:bf:bf:be     1       non-character 0xffffe
 297 5.3.4 y fffff   4       f3:bf:bf:bf     1       non-character 0xfffff
 298 5.3.4 y 10fffe  4       f4:8f:bf:be     1       non-character 0x10fffe
 299 5.3.4 y 10ffff  4       f4:8f:bf:bf     1       non-character 0x10ffff