[perl5.git] / t / op / utf8decode.t

#!./perl

BEGIN {
    chdir 't' if -d 't';
    require './test.pl';
    set_up_inc('../lib');
}

$|=1;

{
    my $wide = v256;
    use bytes;
    my $ordwide = ord($wide);
    printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide;
    skip_all('UTF-8-centric tests (not valid for UTF-EBCDIC)') if $ordwide == 140;
    # This could be ported to EBCDIC, but a lot of trouble.
    # ext/XS-APItest/t/utf8.t contains comprehensive tests for both platforms

    if ($ordwide != 196) {
	printf "# v256 starts with 0x%02x\n", $ordwide;
    }
}

no utf8;

my $is64bit = length sprintf("%x", ~0) > 8;

foreach (<DATA>) {
    if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) {
	# print "# $_\n";
    } elsif (my ($id, $okay, $Unicode, $byteslen, $hex, $charslen, $experr)
	     = /^(\d+\.\d+\.\d+[bu]?)   # ID
		\s+(y|n|N-?\d+(?:,\d+)?)  # expect to pass or fail
                                          # 'n' means expect one diagnostic
                                          # 'N\d+'     means expect this
                                          #            number of diagnostics
                                          # 'N\d+,\d+' means expect the first
                                          #            number of diagnostics
                                          #            on a 32-bit system; the
                                          #            second number on a
                                          #            64-bit one
                \s+([0-9a-f]{1,8}(?:,[0-9a-f]{1,8})*|-) # Unicode characters
                \s+(\d+)                # number of octets
                \s+([0-9a-f]{2}(?::[0-9a-f]{2})*)       # octets in hex
                \s+(\d+|-)              # number of characters
                (?:\s+(.+))?            # expected error (or comment)
                $/x) {
	my @hex = split(/:/, $hex);
	is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen');
	my $octets = join '', map {chr hex $_} @hex;
	is(length $octets, $byteslen, 'Number of octets tallies with byteslen');
	if ($okay eq 'y') {
	    my @chars = map {hex $_} split ',', $Unicode;
	    is(scalar @chars, $charslen, 'Amount of hex tallies with charslen');
	    my @got;
	    warning_is(sub {@got = unpack 'C0U*', $octets}, undef,
		       "No warnings expected for $id");
	    is("@got", "@chars", 'Got expected Unicode characters');
	} elsif ($okay eq 'n') {
	    isnt($experr, '', "Expected warning for $id provided");
	    warnings_like(sub {unpack 'C0U*', $octets}, [qr/$experr/],
			 "Only expected warning for $id");
	} elsif ($okay !~ /^N-?(\d+)(?:,(\d+))?/) {
	    is($okay, 'n', "Confused test description for $id");
	} else {
	    my $expect32 = $1;
            my $expect64 = $2 // $expect32;
            my $expect = ($is64bit) ? $expect64 : $expect32;
	    my @warnings;

	    {
		local $SIG{__WARN__} = sub {
		    print "# $id: @_";
		    push @warnings, "@_";
		};
		unpack 'C0U*', $octets;
	    }

	    unless (is(scalar @warnings, $expect, "Expected number of warnings for $id seen")) {
                note(join "", "Got:\n", @warnings);
            }
	    isnt($experr, '', "Expected first warning for $id provided");

            my $message;
            my $after = "";
            if ($expect64 != $expect32 && ! $is64bit) {
                like($warnings[0], qr/overflow/, "overflow warning for $id seen");
                shift @warnings;
                $after .= "overflow";
            }

            # The data below assumes that if there is both a 'short' and
            # 'non-continuation' malformation, the latter has precedence.  But
            # that has changed, and rather than mess with the data, this works
            # around that.
            if (   @warnings > 1
                && $warnings[0] =~ /short/
                && $warnings[1] =~ /unexpected non-continuation/)
            {
                $after .= " and " if $after;
                $after .= "short";
                shift @warnings;
            }
            $after = "after $after " if $after;

	    like($warnings[0], qr/$experr/, "Expected first warning ${after}for $id seen");
	    local $::TODO;
	    if ($expect < 0) {
		$expect = -$expect;
		$::TODO = "Markus Kuhn states that $expect invalid sequences should be signalled";
	    }

	}
    } else {
	fail("unknown format '$_'");
    }
}

done_testing();

# This table is based on Markus Kuhn's UTF-8 Decode Stress Tester,
# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt,
# version dated 2015-08-28.
#
# See the code that parses these lines for comments as to the column meanings

__DATA__
1	Correct UTF-8
1.1.1 y 3ba,1f79,3c3,3bc,3b5	11	ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5	5
2	Boundary conditions
2.1	First possible sequence of certain length
2.1.1 y 0		1	00	1
2.1.2 y 80		2	c2:80	1
2.1.3 y 800		3	e0:a0:80	1
2.1.4 y 10000		4	f0:90:80:80	1
2.1.5 y 200000		5	f8:88:80:80:80	1
2.1.6 y 4000000		6	fc:84:80:80:80:80	1
2.2	Last possible sequence of certain length
2.2.1 y 7f		1	7f	1
2.2.2 y 7ff		2	df:bf	1
# The ffff is legal by default since 872c91ae155f6880
2.2.3 y ffff		3	ef:bf:bf	1	character 0xffff
2.2.4 y 1fffff		4	f7:bf:bf:bf	1
2.2.5 y 3ffffff		5	fb:bf:bf:bf:bf	1
2.2.6 y 7fffffff	6	fd:bf:bf:bf:bf:bf	1
2.3	Other boundary conditions
2.3.1 y d7ff		3	ed:9f:bf	1
2.3.2 y e000		3	ee:80:80	1
2.3.3 y fffd		3	ef:bf:bd	1
2.3.4 y 10ffff		4	f4:8f:bf:bf	1
2.3.5 y 110000		4	f4:90:80:80	1
3	Malformed sequences
3.1	Unexpected continuation bytes
3.1.1 n -		1	80	-	unexpected continuation byte 0x80
3.1.2 n -		1	bf	-	unexpected continuation byte 0xbf
3.1.3 N2 -		2	80:bf	-	unexpected continuation byte 0x80
3.1.4 N3 -		3	80:bf:80	-	unexpected continuation byte 0x80
3.1.5 N4 -		4	80:bf:80:bf	-	unexpected continuation byte 0x80
3.1.6 N5 -		5	80:bf:80:bf:80	-	unexpected continuation byte 0x80
3.1.7 N6 -		6	80:bf:80:bf:80:bf	-	unexpected continuation byte 0x80
3.1.8 N7 -		7	80:bf:80:bf:80:bf:80	-	unexpected continuation byte 0x80
3.1.9 N64 -	64	80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf	-	unexpected continuation byte 0x80
3.2	Lonely start characters
3.2.1 N34 -	64 	c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20	-	unexpected non-continuation byte 0x20, immediately after start byte 0xc0
3.2.2 N17 -	32	e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20	-	unexpected non-continuation byte 0x20, immediately after start byte 0xe0
3.2.3 N9 -	16	f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20	-	unexpected non-continuation byte 0x20, immediately after start byte 0xf0
3.2.4 N6 -	8	f8:20:f9:20:fa:20:fb:20	-	unexpected non-continuation byte 0x20, immediately after start byte 0xf8
3.2.5 N4 -	4	fc:20:fd:20	-	unexpected non-continuation byte 0x20, immediately after start byte 0xfc
3.3	Sequences with last continuation byte missing
3.3.1 N2 -	1	c0	-	1 byte available, need 2
3.3.2 N2 -	2	e0:80	-	2 bytes available, need 3
3.3.3 N2 -	3	f0:80:80	-	3 bytes available, need 4
3.3.4 N2 -	4	f8:80:80:80	-	4 bytes available, need 5
3.3.5 N2 -	5	fc:80:80:80:80	-	5 bytes available, need 6
3.3.6 n -	1	df	-	1 byte available, need 2
3.3.7 n -	2	ef:bf	-	2 bytes available, need 3
3.3.8 n -	3	f7:bf:bf	-	3 bytes available, need 4
3.3.9 n -	4	fb:bf:bf:bf	-	4 bytes available, need 5
3.3.10 n -	5	fd:bf:bf:bf:bf	-	5 bytes available, need 6
3.4	Concatenation of incomplete sequences
3.4.1 N15 -	30	c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf	-	unexpected non-continuation byte 0xe0, immediately after start byte 0xc0
3.5	Impossible bytes (but not with Perl's extended UTF-8)
3.5.1 n -	1	fe	-	1 byte available, need 7
3.5.2 N2,1 -	1	ff	-	1 byte available, need 13
3.5.3 N11,8 -	4	fe:fe:ff:ff	-	byte 0xfe
4	Overlong sequences
4.1	Examples of an overlong ASCII character
4.1.1 n -	2	c0:af	-	overlong
4.1.2 n -	3	e0:80:af	-	overlong
4.1.3 n -	4	f0:80:80:af	-	overlong
4.1.4 n -	5	f8:80:80:80:af	-	overlong
4.1.5 n -	6	fc:80:80:80:80:af	-	overlong
4.2	Maximum overlong sequences
4.2.1 n -	2	c1:bf	-	overlong
4.2.2 n -	3	e0:9f:bf	-	overlong
4.2.3 n -	4	f0:8f:bf:bf	-	overlong
4.2.4 n -	5	f8:87:bf:bf:bf	-	overlong
4.2.5 n -	6	fc:83:bf:bf:bf:bf	-	overlong
4.3	Overlong representation of the NUL character
4.3.1 n -	2	c0:80	-	overlong
4.3.2 n -	3	e0:80:80	-	overlong
4.3.3 n -	4	f0:80:80:80	-	overlong
4.3.4 n -	5	f8:80:80:80:80	-	overlong
4.3.5 n -	6	fc:80:80:80:80:80	-	overlong
5	Illegal code positions
5.1	Single UTF-16 surrogates
5.1.1 y d800	3	ed:a0:80	1	UTF-16 surrogate 0xd800
5.1.2 y db7f	3	ed:ad:bf	1	UTF-16 surrogate 0xdb7f
5.1.3 y db80	3	ed:ae:80	1	UTF-16 surrogate 0xdb80
5.1.4 y dbff	3	ed:af:bf	1	UTF-16 surrogate 0xdbff
5.1.5 y dc00	3	ed:b0:80	1	UTF-16 surrogate 0xdc00
5.1.6 y df80	3	ed:be:80	1	UTF-16 surrogate 0xdf80
5.1.7 y dfff	3	ed:bf:bf	1	UTF-16 surrogate 0xdfff
5.2	Paired UTF-16 surrogates
5.2.1 y d800,dc00	6	ed:a0:80:ed:b0:80	2	UTF-16 surrogates 0xd800, dc00
5.2.2 y d800,dfff	6	ed:a0:80:ed:bf:bf	2	UTF-16 surrogates 0xd800, dfff
5.2.3 y db7f,dc00	6	ed:ad:bf:ed:b0:80	2	UTF-16 surrogates 0xdb7f, dc00
5.2.4 y db7f,dfff	6	ed:ad:bf:ed:bf:bf	2	UTF-16 surrogates 0xdb7f, dfff
5.2.5 y db80,dc00	6	ed:ae:80:ed:b0:80	2	UTF-16 surrogates 0xdb80, dc00
5.2.6 y db80,dfff	6	ed:ae:80:ed:bf:bf	2	UTF-16 surrogates 0xdb80, dfff
5.2.7 y dbff,dc00	6	ed:af:bf:ed:b0:80	2	UTF-16 surrogates 0xdbff, dc00
5.2.8 y dbff,dfff	6	ed:af:bf:ed:bf:bf	2	UTF-16 surrogates 0xdbff, dfff
5.3	Other illegal code positions
5.3.1 y fffe	3	ef:bf:be	1	byte order mark 0xfffe
# The ffff is legal by default since 872c91ae155f6880
5.3.2 y ffff	3	ef:bf:bf	1	non-character 0xffff
5.3.3 y fdd0	3	ef:b7:90	1	non-character 0xfdd0
5.3.3 y	fdd1	3	ef:b7:91	1	non-character 0xfdd1
5.3.3 y	fdd2	3	ef:b7:92	1	non-character 0xfdd2
5.3.3 y	fdd3	3	ef:b7:93	1	non-character 0xfdd3
5.3.3 y	fdd4	3	ef:b7:94	1	non-character 0xfdd4
5.3.3 y	fdd5	3	ef:b7:95	1	non-character 0xfdd5
5.3.3 y	fdd6	3	ef:b7:96	1	non-character 0xfdd6
5.3.3 y	fdd7	3	ef:b7:97	1	non-character 0xfdd7
5.3.3 y	fdd8	3	ef:b7:98	1	non-character 0xfdd8
5.3.3 y	fdd9	3	ef:b7:99	1	non-character 0xfdd9
5.3.3 y	fdda	3	ef:b7:9a	1	non-character 0xfdda
5.3.3 y	fddb	3	ef:b7:9b	1	non-character 0xfddb
5.3.3 y	fddc	3	ef:b7:9c	1	non-character 0xfddc
5.3.3 y	fddd	3	ef:b7:9d	1	non-character 0xfddd
5.3.3 y	fdde	3	ef:b7:9e	1	non-character 0xfdde
5.3.3 y	fddf	3	ef:b7:9f	1	non-character 0xfddf
5.3.3 y	fde0	3	ef:b7:a0	1	non-character 0xfde0
5.3.3 y	fde1	3	ef:b7:a1	1	non-character 0xfde1
5.3.3 y	fde2	3	ef:b7:a2	1	non-character 0xfde2
5.3.3 y	fde3	3	ef:b7:a3	1	non-character 0xfde3
5.3.3 y	fde4	3	ef:b7:a4	1	non-character 0xfde4
5.3.3 y	fde5	3	ef:b7:a5	1	non-character 0xfde5
5.3.3 y	fde6	3	ef:b7:a6	1	non-character 0xfde6
5.3.3 y	fde7	3	ef:b7:a7	1	non-character 0xfde7
5.3.3 y	fde8	3	ef:b7:a8	1	non-character 0xfde8
5.3.3 y	fde9	3	ef:b7:a9	1	non-character 0xfde9
5.3.3 y	fdea	3	ef:b7:aa	1	non-character 0xfdea
5.3.3 y	fdeb	3	ef:b7:ab	1	non-character 0xfdeb
5.3.3 y	fdec	3	ef:b7:ac	1	non-character 0xfdec
5.3.3 y	fded	3	ef:b7:ad	1	non-character 0xfded
5.3.3 y	fdee	3	ef:b7:ae	1	non-character 0xfdee
5.3.3 y	fdef	3	ef:b7:af	1	non-character 0xfdef
5.3.4 y 1fffe	4	f0:9f:bf:be	1	non-character 0x1fffe
5.3.4 y 1ffff	4	f0:9f:bf:bf	1	non-character 0x1ffff
5.3.4 y 2fffe	4	f0:af:bf:be	1	non-character 0x2fffe
5.3.4 y 2ffff	4	f0:af:bf:bf	1	non-character 0x2ffff
5.3.4 y 3fffe	4	f0:bf:bf:be	1	non-character 0x3fffe
5.3.4 y 3ffff	4	f0:bf:bf:bf	1	non-character 0x3ffff
5.3.4 y 4fffe	4	f1:8f:bf:be	1	non-character 0x4fffe
5.3.4 y 4ffff	4	f1:8f:bf:bf	1	non-character 0x4ffff
5.3.4 y 5fffe	4	f1:9f:bf:be	1	non-character 0x5fffe
5.3.4 y 5ffff	4	f1:9f:bf:bf	1	non-character 0x5ffff
5.3.4 y 6fffe	4	f1:af:bf:be	1	non-character 0x6fffe
5.3.4 y 6ffff	4	f1:af:bf:bf	1	non-character 0x6ffff
5.3.4 y 7fffe	4	f1:bf:bf:be	1	non-character 0x7fffe
5.3.4 y 7ffff	4	f1:bf:bf:bf	1	non-character 0x7ffff
5.3.4 y 8fffe	4	f2:8f:bf:be	1	non-character 0x8fffe
5.3.4 y 8ffff	4	f2:8f:bf:bf	1	non-character 0x8ffff
5.3.4 y 9fffe	4	f2:9f:bf:be	1	non-character 0x9fffe
5.3.4 y 9ffff	4	f2:9f:bf:bf	1	non-character 0x9ffff
5.3.4 y afffe	4	f2:af:bf:be	1	non-character 0xafffe
5.3.4 y affff	4	f2:af:bf:bf	1	non-character 0xaffff
5.3.4 y bfffe	4	f2:bf:bf:be	1	non-character 0xbfffe
5.3.4 y bffff	4	f2:bf:bf:bf	1	non-character 0xbffff
5.3.4 y cfffe	4	f3:8f:bf:be	1	non-character 0xcfffe
5.3.4 y cffff	4	f3:8f:bf:bf	1	non-character 0xcffff
5.3.4 y dfffe	4	f3:9f:bf:be	1	non-character 0xdfffe
5.3.4 y dffff	4	f3:9f:bf:bf	1	non-character 0xdffff
5.3.4 y efffe	4	f3:af:bf:be	1	non-character 0xefffe
5.3.4 y effff	4	f3:af:bf:bf	1	non-character 0xeffff
5.3.4 y ffffe	4	f3:bf:bf:be	1	non-character 0xffffe
5.3.4 y fffff	4	f3:bf:bf:bf	1	non-character 0xfffff
5.3.4 y 10fffe	4	f4:8f:bf:be	1	non-character 0x10fffe
5.3.4 y 10ffff	4	f4:8f:bf:bf	1	non-character 0x10ffff
Commit	Line	Data
a9917092 JH	1	#!./perl
	2
	3	BEGIN {
	4	chdir 't' if -d 't';
680218c4	5	require './test.pl';
624c42e2	6	set_up_inc('../lib');
daf0f78e NIS	7	}
daf0f78e NIS	8
a74a9a49 KW	9	$\|=1;
a74a9a49 KW	10
daf0f78e NIS	11	{
	12	my $wide = v256;
	13	use bytes;
ffbc6a93 JH	14	my $ordwide = ord($wide);
ffbc6a93 JH	15	printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide;
26c2b24e	16	skip_all('UTF-8-centric tests (not valid for UTF-EBCDIC)') if $ordwide == 140;
aadb82e0 KW	17	# This could be ported to EBCDIC, but a lot of trouble.
aadb82e0 KW	18	# ext/XS-APItest/t/utf8.t contains comprehensive tests for both platforms
680218c4 NC	19
680218c4 NC	20	if ($ordwide != 196) {
ffbc6a93 JH	21	printf "# v256 starts with 0x%02x\n", $ordwide;
ffbc6a93 JH	22	}
a9917092 JH	23	}
a9917092 JH	24
3b0e0cb6	25	no utf8;
ffc61ed2	26
aadb82e0 KW	27	my $is64bit = length sprintf("%x", ~0) > 8;
aadb82e0 KW	28
e1a67460 NC	29	foreach (<DATA>) {
	30	if (/^(?:\d+(?:\.\d+)?)\s/ \|\| /^#/) {
	31	# print "# $_\n";
deaabfbb NC	32	} elsif (my ($id, $okay, $Unicode, $byteslen, $hex, $charslen, $experr)
deaabfbb NC	33	= /^(\d+\.\d+\.\d+[bu]?) # ID
aadb82e0 KW	34	\s+(y\|n\|N-?\d+(?:,\d+)?) # expect to pass or fail
	35	# 'n' means expect one diagnostic
	36	# 'N\d+' means expect this
	37	# number of diagnostics
	38	# 'N\d+,\d+' means expect the first
	39	# number of diagnostics
	40	# on a 32-bit system; the
	41	# second number on a
	42	# 64-bit one
deaabfbb NC	43	\s+([0-9a-f]{1,8}(?:,[0-9a-f]{1,8})*\|-) # Unicode characters
	44	\s+(\d+) # number of octets
	45	\s+([0-9a-f]{2}(?::[0-9a-f]{2})*) # octets in hex
	46	\s+(\d+\|-) # number of characters
	47	(?:\s+(.+))? # expected error (or comment)
	48	$/x) {
e1a67460 NC	49	my @hex = split(/:/, $hex);
	50	is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen');
	51	my $octets = join '', map {chr hex $_} @hex;
	52	is(length $octets, $byteslen, 'Number of octets tallies with byteslen');
e1a67460	53	if ($okay eq 'y') {
04f3c608 NC	54	my @chars = map {hex $_} split ',', $Unicode;
	55	is(scalar @chars, $charslen, 'Amount of hex tallies with charslen');
	56	my @got;
	57	warning_is(sub {@got = unpack 'C0U*', $octets}, undef,
c03ceae6	58	"No warnings expected for $id");
04f3c608	59	is("@got", "@chars", 'Got expected Unicode characters');
f96a66c2 NC	60	} elsif ($okay eq 'n') {
	61	isnt($experr, '', "Expected warning for $id provided");
	62	warnings_like(sub {unpack 'C0U*', $octets}, [qr/$experr/],
	63	"Only expected warning for $id");
aadb82e0	64	} elsif ($okay !~ /^N-?(\d+)(?:,(\d+))?/) {
e1a67460	65	is($okay, 'n', "Confused test description for $id");
e1a67460	66	} else {
aadb82e0 KW	67	my $expect32 = $1;
	68	my $expect64 = $2 // $expect32;
	69	my $expect = ($is64bit) ? $expect64 : $expect32;
f96a66c2	70	my @warnings;
c03ceae6 NC	71
	72	{
	73	local $SIG{__WARN__} = sub {
	74	print "# $id: @_";
f96a66c2	75	push @warnings, "@_";
c03ceae6 NC	76	};
	77	unpack 'C0U*', $octets;
	78	}
	79
aadb82e0 KW	80	unless (is(scalar @warnings, $expect, "Expected number of warnings for $id seen")) {
	81	note(join "", "Got:\n", @warnings);
	82	}
f96a66c2	83	isnt($experr, '', "Expected first warning for $id provided");
aadb82e0 KW	84
aadb82e0 KW	85	my $message;
e308b348	86	my $after = "";
aadb82e0 KW	87	if ($expect64 != $expect32 && ! $is64bit) {
	88	like($warnings[0], qr/overflow/, "overflow warning for $id seen");
	89	shift @warnings;
e308b348	90	$after .= "overflow";
aadb82e0	91	}
e308b348 KW	92
	93	# The data below assumes that if there is both a 'short' and
	94	# 'non-continuation' malformation, the latter has precedence. But
	95	# that has changed, and rather than mess with the data, this works
	96	# around that.
	97	if ( @warnings > 1
	98	&& $warnings[0] =~ /short/
	99	&& $warnings[1] =~ /unexpected non-continuation/)
	100	{
	101	$after .= " and " if $after;
	102	$after .= "short";
	103	shift @warnings;
aadb82e0	104	}
e308b348 KW	105	$after = "after $after " if $after;
	106
	107	like($warnings[0], qr/$experr/, "Expected first warning ${after}for $id seen");
f96a66c2 NC	108	local $::TODO;
	109	if ($expect < 0) {
	110	$expect = -$expect;
	111	$::TODO = "Markus Kuhn states that $expect invalid sequences should be signalled";
	112	}
a74a9a49	113
e1a67460 NC	114	}
	115	} else {
	116	fail("unknown format '$_'");
	117	}
	118	}
	119
	120	done_testing();
	121
a9917092 JH	122	# This table is based on Markus Kuhn's UTF-8 Decode Stress Tester,
a9917092 JH	123	# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt,
c5278643	124	# version dated 2015-08-28.
aadb82e0 KW	125	#
aadb82e0 KW	126	# See the code that parses these lines for comments as to the column meanings
a9917092	127
e1a67460	128	__DATA__
a9917092	129	1 Correct UTF-8
04f3c608	130	1.1.1 y 3ba,1f79,3c3,3bc,3b5 11 ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5 5
daf0f78e	131	2 Boundary conditions
a9917092	132	2.1 First possible sequence of certain length
bb6a3342 NC	133	2.1.1 y 0 1 00 1
	134	2.1.2 y 80 2 c2:80 1
	135	2.1.3 y 800 3 e0:a0:80 1
	136	2.1.4 y 10000 4 f0:90:80:80 1
	137	2.1.5 y 200000 5 f8:88:80:80:80 1
	138	2.1.6 y 4000000 6 fc:84:80:80:80:80 1
a9917092	139	2.2 Last possible sequence of certain length
bb6a3342 NC	140	2.2.1 y 7f 1 7f 1
bb6a3342 NC	141	2.2.2 y 7ff 2 df:bf 1
8567041c	142	# The ffff is legal by default since 872c91ae155f6880
bb6a3342 NC	143	2.2.3 y ffff 3 ef:bf:bf 1 character 0xffff
	144	2.2.4 y 1fffff 4 f7:bf:bf:bf 1
	145	2.2.5 y 3ffffff 5 fb:bf:bf:bf:bf 1
	146	2.2.6 y 7fffffff 6 fd:bf:bf:bf:bf:bf 1
a9917092	147	2.3 Other boundary conditions
bb6a3342 NC	148	2.3.1 y d7ff 3 ed:9f:bf 1
	149	2.3.2 y e000 3 ee:80:80 1
	150	2.3.3 y fffd 3 ef:bf:bd 1
	151	2.3.4 y 10ffff 4 f4:8f:bf:bf 1
	152	2.3.5 y 110000 4 f4:90:80:80 1
a9917092 JH	153	3 Malformed sequences
a9917092 JH	154	3.1 Unexpected continuation bytes
bb6a3342 NC	155	3.1.1 n - 1 80 - unexpected continuation byte 0x80
bb6a3342 NC	156	3.1.2 n - 1 bf - unexpected continuation byte 0xbf
f96a66c2 NC	157	3.1.3 N2 - 2 80:bf - unexpected continuation byte 0x80
	158	3.1.4 N3 - 3 80:bf:80 - unexpected continuation byte 0x80
	159	3.1.5 N4 - 4 80:bf:80:bf - unexpected continuation byte 0x80
	160	3.1.6 N5 - 5 80:bf:80:bf:80 - unexpected continuation byte 0x80
	161	3.1.7 N6 - 6 80:bf:80:bf:80:bf - unexpected continuation byte 0x80
	162	3.1.8 N7 - 7 80:bf:80:bf:80:bf:80 - unexpected continuation byte 0x80
	163	3.1.9 N64 - 64 80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf - unexpected continuation byte 0x80
a9917092	164	3.2 Lonely start characters
2b5e7bc2	165	3.2.1 N34 - 64 c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xc0
e308b348 KW	166	3.2.2 N17 - 32 e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xe0
	167	3.2.3 N9 - 16 f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf0
	168	3.2.4 N6 - 8 f8:20:f9:20:fa:20:fb:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf8
	169	3.2.5 N4 - 4 fc:20:fd:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xfc
a9917092	170	3.3 Sequences with last continuation byte missing
9a6c9c81 KW	171	3.3.1 N2 - 1 c0 - 1 byte available, need 2
	172	3.3.2 N2 - 2 e0:80 - 2 bytes available, need 3
	173	3.3.3 N2 - 3 f0:80:80 - 3 bytes available, need 4
	174	3.3.4 N2 - 4 f8:80:80:80 - 4 bytes available, need 5
	175	3.3.5 N2 - 5 fc:80:80:80:80 - 5 bytes available, need 6
	176	3.3.6 n - 1 df - 1 byte available, need 2
	177	3.3.7 n - 2 ef:bf - 2 bytes available, need 3
	178	3.3.8 n - 3 f7:bf:bf - 3 bytes available, need 4
	179	3.3.9 n - 4 fb:bf:bf:bf - 4 bytes available, need 5
	180	3.3.10 n - 5 fd:bf:bf:bf:bf - 5 bytes available, need 6
a9917092	181	3.4 Concatenation of incomplete sequences
2b5e7bc2	182	3.4.1 N15 - 30 c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf - unexpected non-continuation byte 0xe0, immediately after start byte 0xc0
7cf8d05d	183	3.5 Impossible bytes (but not with Perl's extended UTF-8)
9a6c9c81 KW	184	3.5.1 n - 1 fe - 1 byte available, need 7
9a6c9c81 KW	185	3.5.2 N2,1 - 1 ff - 1 byte available, need 13
e308b348	186	3.5.3 N11,8 - 4 fe:fe:ff:ff - byte 0xfe
a9917092 JH	187	4 Overlong sequences
a9917092 JH	188	4.1 Examples of an overlong ASCII character
7cf8d05d KW	189	4.1.1 n - 2 c0:af - overlong
	190	4.1.2 n - 3 e0:80:af - overlong
	191	4.1.3 n - 4 f0:80:80:af - overlong
	192	4.1.4 n - 5 f8:80:80:80:af - overlong
	193	4.1.5 n - 6 fc:80:80:80:80:af - overlong
a9917092	194	4.2 Maximum overlong sequences
7cf8d05d KW	195	4.2.1 n - 2 c1:bf - overlong
	196	4.2.2 n - 3 e0:9f:bf - overlong
	197	4.2.3 n - 4 f0:8f:bf:bf - overlong
	198	4.2.4 n - 5 f8:87:bf:bf:bf - overlong
	199	4.2.5 n - 6 fc:83:bf:bf:bf:bf - overlong
a9917092	200	4.3 Overlong representation of the NUL character
7cf8d05d KW	201	4.3.1 n - 2 c0:80 - overlong
	202	4.3.2 n - 3 e0:80:80 - overlong
	203	4.3.3 n - 4 f0:80:80:80 - overlong
	204	4.3.4 n - 5 f8:80:80:80:80 - overlong
	205	4.3.5 n - 6 fc:80:80:80:80:80 - overlong
a9917092 JH	206	5 Illegal code positions
a9917092 JH	207	5.1 Single UTF-16 surrogates
04f3c608 NC	208	5.1.1 y d800 3 ed:a0:80 1 UTF-16 surrogate 0xd800
	209	5.1.2 y db7f 3 ed:ad:bf 1 UTF-16 surrogate 0xdb7f
	210	5.1.3 y db80 3 ed:ae:80 1 UTF-16 surrogate 0xdb80
	211	5.1.4 y dbff 3 ed:af:bf 1 UTF-16 surrogate 0xdbff
	212	5.1.5 y dc00 3 ed:b0:80 1 UTF-16 surrogate 0xdc00
	213	5.1.6 y df80 3 ed:be:80 1 UTF-16 surrogate 0xdf80
	214	5.1.7 y dfff 3 ed:bf:bf 1 UTF-16 surrogate 0xdfff
a9917092	215	5.2 Paired UTF-16 surrogates
04f3c608 NC	216	5.2.1 y d800,dc00 6 ed:a0:80:ed:b0:80 2 UTF-16 surrogates 0xd800, dc00
	217	5.2.2 y d800,dfff 6 ed:a0:80:ed:bf:bf 2 UTF-16 surrogates 0xd800, dfff
	218	5.2.3 y db7f,dc00 6 ed:ad:bf:ed:b0:80 2 UTF-16 surrogates 0xdb7f, dc00
	219	5.2.4 y db7f,dfff 6 ed:ad:bf:ed:bf:bf 2 UTF-16 surrogates 0xdb7f, dfff
	220	5.2.5 y db80,dc00 6 ed:ae:80:ed:b0:80 2 UTF-16 surrogates 0xdb80, dc00
	221	5.2.6 y db80,dfff 6 ed:ae:80:ed:bf:bf 2 UTF-16 surrogates 0xdb80, dfff
	222	5.2.7 y dbff,dc00 6 ed:af:bf:ed:b0:80 2 UTF-16 surrogates 0xdbff, dc00
	223	5.2.8 y dbff,dfff 6 ed:af:bf:ed:bf:bf 2 UTF-16 surrogates 0xdbff, dfff
a9917092	224	5.3 Other illegal code positions
04f3c608	225	5.3.1 y fffe 3 ef:bf:be 1 byte order mark 0xfffe
8567041c	226	# The ffff is legal by default since 872c91ae155f6880
c5278643 KW	227	5.3.2 y ffff 3 ef:bf:bf 1 non-character 0xffff
	228	5.3.3 y fdd0 3 ef:b7:90 1 non-character 0xfdd0
	229	5.3.3 y fdd1 3 ef:b7:91 1 non-character 0xfdd1
	230	5.3.3 y fdd2 3 ef:b7:92 1 non-character 0xfdd2
	231	5.3.3 y fdd3 3 ef:b7:93 1 non-character 0xfdd3
	232	5.3.3 y fdd4 3 ef:b7:94 1 non-character 0xfdd4
	233	5.3.3 y fdd5 3 ef:b7:95 1 non-character 0xfdd5
	234	5.3.3 y fdd6 3 ef:b7:96 1 non-character 0xfdd6
	235	5.3.3 y fdd7 3 ef:b7:97 1 non-character 0xfdd7
	236	5.3.3 y fdd8 3 ef:b7:98 1 non-character 0xfdd8
	237	5.3.3 y fdd9 3 ef:b7:99 1 non-character 0xfdd9
	238	5.3.3 y fdda 3 ef:b7:9a 1 non-character 0xfdda
	239	5.3.3 y fddb 3 ef:b7:9b 1 non-character 0xfddb
	240	5.3.3 y fddc 3 ef:b7:9c 1 non-character 0xfddc
	241	5.3.3 y fddd 3 ef:b7:9d 1 non-character 0xfddd
	242	5.3.3 y fdde 3 ef:b7:9e 1 non-character 0xfdde
	243	5.3.3 y fddf 3 ef:b7:9f 1 non-character 0xfddf
	244	5.3.3 y fde0 3 ef:b7:a0 1 non-character 0xfde0
	245	5.3.3 y fde1 3 ef:b7:a1 1 non-character 0xfde1
	246	5.3.3 y fde2 3 ef:b7:a2 1 non-character 0xfde2
	247	5.3.3 y fde3 3 ef:b7:a3 1 non-character 0xfde3
	248	5.3.3 y fde4 3 ef:b7:a4 1 non-character 0xfde4
	249	5.3.3 y fde5 3 ef:b7:a5 1 non-character 0xfde5
	250	5.3.3 y fde6 3 ef:b7:a6 1 non-character 0xfde6
	251	5.3.3 y fde7 3 ef:b7:a7 1 non-character 0xfde7
	252	5.3.3 y fde8 3 ef:b7:a8 1 non-character 0xfde8
	253	5.3.3 y fde9 3 ef:b7:a9 1 non-character 0xfde9
	254	5.3.3 y fdea 3 ef:b7:aa 1 non-character 0xfdea
	255	5.3.3 y fdeb 3 ef:b7:ab 1 non-character 0xfdeb
	256	5.3.3 y fdec 3 ef:b7:ac 1 non-character 0xfdec
	257	5.3.3 y fded 3 ef:b7:ad 1 non-character 0xfded
	258	5.3.3 y fdee 3 ef:b7:ae 1 non-character 0xfdee
	259	5.3.3 y fdef 3 ef:b7:af 1 non-character 0xfdef
	260	5.3.4 y 1fffe 4 f0:9f:bf:be 1 non-character 0x1fffe
	261	5.3.4 y 1ffff 4 f0:9f:bf:bf 1 non-character 0x1ffff
	262	5.3.4 y 2fffe 4 f0:af:bf:be 1 non-character 0x2fffe
	263	5.3.4 y 2ffff 4 f0:af:bf:bf 1 non-character 0x2ffff
	264	5.3.4 y 3fffe 4 f0:bf:bf:be 1 non-character 0x3fffe
	265	5.3.4 y 3ffff 4 f0:bf:bf:bf 1 non-character 0x3ffff
	266	5.3.4 y 4fffe 4 f1:8f:bf:be 1 non-character 0x4fffe
	267	5.3.4 y 4ffff 4 f1:8f:bf:bf 1 non-character 0x4ffff
	268	5.3.4 y 5fffe 4 f1:9f:bf:be 1 non-character 0x5fffe
	269	5.3.4 y 5ffff 4 f1:9f:bf:bf 1 non-character 0x5ffff
	270	5.3.4 y 6fffe 4 f1:af:bf:be 1 non-character 0x6fffe
	271	5.3.4 y 6ffff 4 f1:af:bf:bf 1 non-character 0x6ffff
	272	5.3.4 y 7fffe 4 f1:bf:bf:be 1 non-character 0x7fffe
	273	5.3.4 y 7ffff 4 f1:bf:bf:bf 1 non-character 0x7ffff
	274	5.3.4 y 8fffe 4 f2:8f:bf:be 1 non-character 0x8fffe
	275	5.3.4 y 8ffff 4 f2:8f:bf:bf 1 non-character 0x8ffff
	276	5.3.4 y 9fffe 4 f2:9f:bf:be 1 non-character 0x9fffe
	277	5.3.4 y 9ffff 4 f2:9f:bf:bf 1 non-character 0x9ffff
	278	5.3.4 y afffe 4 f2:af:bf:be 1 non-character 0xafffe
	279	5.3.4 y affff 4 f2:af:bf:bf 1 non-character 0xaffff
	280	5.3.4 y bfffe 4 f2:bf:bf:be 1 non-character 0xbfffe
	281	5.3.4 y bffff 4 f2:bf:bf:bf 1 non-character 0xbffff
	282	5.3.4 y cfffe 4 f3:8f:bf:be 1 non-character 0xcfffe
	283	5.3.4 y cffff 4 f3:8f:bf:bf 1 non-character 0xcffff
	284	5.3.4 y dfffe 4 f3:9f:bf:be 1 non-character 0xdfffe
	285	5.3.4 y dffff 4 f3:9f:bf:bf 1 non-character 0xdffff
	286	5.3.4 y efffe 4 f3:af:bf:be 1 non-character 0xefffe
	287	5.3.4 y effff 4 f3:af:bf:bf 1 non-character 0xeffff
	288	5.3.4 y ffffe 4 f3:bf:bf:be 1 non-character 0xffffe
	289	5.3.4 y fffff 4 f3:bf:bf:bf 1 non-character 0xfffff
	290	5.3.4 y 10fffe 4 f4:8f:bf:be 1 non-character 0x10fffe
291	5.3.4 y 10ffff 4 f4:8f:bf:bf 1 non-character 0x10ffff