[perl5.git] / lib / Test / Harness / t / strap-analyze.t

#!/usr/bin/perl -w

BEGIN {
    if( $ENV{PERL_CORE} ) {
        chdir 't';
        @INC = ('../lib', 'lib');
    }
    else {
        unshift @INC, 't/lib';
    }
}

my $SAMPLE_TESTS = $ENV{PERL_CORE} ? 'lib/sample-tests' : 't/sample-tests';

use strict;
use Test::More;

my $IsVMS = $^O eq 'VMS';

# VMS uses native, not POSIX, exit codes.
my $die_exit = $IsVMS ? 44 : 1;

# We can only predict that the wait status should be zero or not.
my $wait_non_zero = 1;

my %samples = (
   combined   => {
                  passing     => 0,

                  'exit'      => 0,
                  'wait'      => 0,

                  max         => 10,
                  seen        => 10,

                  'ok'        => 8,
                  'todo'      => 2,
                  'skip'      => 1,
                  bonus       => 1,

                  details     => [ { 'ok' => 1, actual_ok => 1 },
                                   { 'ok' => 1, actual_ok => 1,
                                     name => 'basset hounds got long ears',
                                   },
                                   { 'ok' => 0, actual_ok => 0,
                                     name => 'all hell broke lose',
                                   },
                                   { 'ok' => 1, actual_ok => 1,
                                     type => 'todo'
                                   },
                                   { 'ok' => 1, actual_ok => 1 },
                                   { 'ok' => 1, actual_ok => 1 },
                                   { 'ok' => 1, actual_ok => 1,
                                     type   => 'skip',
                                     reason => 'contract negociations'
                                   },
                                   { 'ok' => 1, actual_ok => 1 },
                                   { 'ok' => 0, actual_ok => 0 },
                                   { 'ok' => 1, actual_ok => 0,
                                     type   => 'todo' 
                                   },
                                 ]
                       },

   descriptive      => {
                        passing     => 1,

                        'wait'      => 0,
                        'exit'      => 0,

                        max         => 5,
                        seen        => 5,

                        'ok'          => 5,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ { 'ok' => 1, actual_ok => 1,
                                           name => 'Interlock activated'
                                         },
                                         { 'ok' => 1, actual_ok => 1,
                                           name => 'Megathrusters are go',
                                         },
                                         { 'ok' => 1, actual_ok => 1,
                                           name => 'Head formed',
                                         },
                                         { 'ok' => 1, actual_ok => 1,
                                           name => 'Blazing sword formed'
                                         },
                                         { 'ok' => 1, actual_ok => 1,
                                           name => 'Robeast destroyed'
                                         },
                                       ],
                       },

   duplicates       => {
                        passing     => 0,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 10,
                        seen        => 11,

                        'ok'          => 11,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 10
                                       ],
                       },

   head_end         => {
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 4,
                        seen        => 4,

                        'ok'        => 4,
                        'todo'      => 0,
                        'skip'      => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
                                       ],
                       },

   lone_not_bug     => {
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 4,
                        seen        => 4,

                        'ok'        => 4,
                        'todo'      => 0,
                        'skip'      => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
                                       ],
                       },

   head_fail           => {
                           passing  => 0,

                           'exit'   => 0,
                           'wait'   => 0,

                           max      => 4,
                           seen     => 4,

                           'ok'     => 3,
                           'todo'   => 0,
                           'skip'   => 0,
                           bonus    => 0,

                           details  => [ { 'ok' => 1, actual_ok => 1 },
                                         { 'ok' => 0, actual_ok => 0 },
                                         ({ 'ok'=> 1, actual_ok => 1 }) x 2
                                       ],
                          },

   simple           => {
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 5,
                        seen        => 5,

                        'ok'          => 5,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 5
                                       ]
                       },

   simple_fail      => {
                        passing     => 0,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 5,
                        seen        => 5,

                        'ok'          => 3,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ { 'ok' => 1, actual_ok => 1 },
                                         { 'ok' => 0, actual_ok => 0 },
                                         { 'ok' => 1, actual_ok => 1 },
                                         { 'ok' => 1, actual_ok => 1 },
                                         { 'ok' => 0, actual_ok => 0 },
                                       ]
                       },

   'skip'             => {
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 5,
                        seen        => 5,

                        'ok'          => 5,
                        'todo'        => 0,
                        'skip'        => 1,
                        bonus       => 0,

                        details     => [ { 'ok' => 1, actual_ok => 1 },
                                         { 'ok'   => 1, actual_ok => 1,
                                           type   => 'skip',
                                           reason => 'rain delay',
                                         },
                                         ({ 'ok' => 1, actual_ok => 1 }) x 3
                                       ]
                       },

   skip_all           => {
                          passing   => 1,

                          'exit'    => 0,
                          'wait'    => 0,

                          max       => 0,
                          seen      => 0,
                          skip_all  => 'rope',

                          'ok'      => 0,
                          'todo'    => 0,
                          'skip'    => 0,
                          bonus     => 0,

                          details   => [],
                         },

   'todo'             => {
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 5,
                        seen        => 5,

                        'ok'          => 5,
                        'todo'        => 2,
                        'skip'        => 0,
                        bonus       => 1,

                        details     => [ { 'ok' => 1, actual_ok => 1 },
                                         { 'ok' => 1, actual_ok => 1,
                                           type => 'todo' },
                                         { 'ok' => 1, actual_ok => 0,
                                           type => 'todo' },
                                         ({ 'ok' => 1, actual_ok => 1 }) x 2
                                       ],
                       },
   taint            => {
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 1,
                        seen        => 1,

                        'ok'          => 1,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ { 'ok' => 1, actual_ok => 1,
                                           name => '- -T honored'
                                         },
                                       ],
                       },
   vms_nit          => {
                        passing     => 0,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 2,
                        seen        => 2,

                        'ok'          => 1,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ { 'ok' => 0, actual_ok => 0 },
                                         { 'ok' => 1, actual_ok => 1 },
                                       ],
                       },
   'die'            => {
                        passing     => 0,

                        'exit'      => $die_exit,
                        'wait'      => $wait_non_zero,

                        max         => 0,
                        seen        => 0,

                        'ok'        => 0,
                        'todo'      => 0,
                        'skip'      => 0,
                        bonus       => 0,

                        details     => []
                       },

   die_head_end     => {
                        passing     => 0,

                        'exit'      => $die_exit,
                        'wait'      => $wait_non_zero,

                        max         => 0,
                        seen        => 4,

                        'ok'        => 4,
                        'todo'      => 0,
                        'skip'      => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
                                       ],
                       },

   die_last_minute  => {
                        passing     => 0,

                        'exit'      => $die_exit,
                        'wait'      => $wait_non_zero,

                        max         => 4,
                        seen        => 4,

                        'ok'        => 4,
                        'todo'      => 0,
                        'skip'      => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
                                       ],
                       },

   bignum           => {
                        passing     => 0,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 2,
                        seen        => 4,

                        'ok'          => 4,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ { 'ok' => 1, actual_ok => 1 },
                                         { 'ok' => 1, actual_ok => 1 },
                                       ]
                       },

   'shbang_misparse' =>{
                        passing     => 1,

                        'exit'      => 0,
                        'wait'      => 0,

                        max         => 2,
                        seen        => 2,

                        'ok'          => 2,
                        'todo'        => 0,
                        'skip'        => 0,
                        bonus       => 0,

                        details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 2 ]
                       },
);

plan tests => (keys(%samples) * 3) + 1;

use_ok('Test::Harness::Straps');

$SIG{__WARN__} = sub { 
    warn @_ unless $_[0] =~ /^Enourmous test number/ ||
                   $_[0] =~ /^Can't detailize/
};
while( my($test, $expect) = each %samples ) {
    for (0..$#{$expect->{details}}) {
        $expect->{details}[$_]{type} = ''
            unless exists $expect->{details}[$_]{type};
        $expect->{details}[$_]{name} = ''
            unless exists $expect->{details}[$_]{name};
        $expect->{details}[$_]{reason} = ''
            unless exists $expect->{details}[$_]{reason};
    }

    my $strap = Test::Harness::Straps->new;
    my %results = $strap->analyze_file("$SAMPLE_TESTS/$test");

    is_deeply($results{details}, $expect->{details}, "$test details" );

    delete $expect->{details};
    delete $results{details};

    # We can only check if it's zero or non-zero.
    is( !!$results{'wait'}, !!$expect->{'wait'}, 'wait status' );
    delete $results{'wait'};
    delete $expect->{'wait'};

    is_deeply(\%results, $expect, "  the rest $test" );
}
Commit	Line	Data
13287dd5 MS	1	#!/usr/bin/perl -w
	2
	3	BEGIN {
	4	if( $ENV{PERL_CORE} ) {
	5	chdir 't';
	6	@INC = ('../lib', 'lib');
	7	}
	8	else {
	9	unshift @INC, 't/lib';
	10	}
	11	}
	12
	13	my $SAMPLE_TESTS = $ENV{PERL_CORE} ? 'lib/sample-tests' : 't/sample-tests';
	14
	15	use strict;
6e5a998b	16	use Test::More;
13287dd5 MS	17
	18	my $IsVMS = $^O eq 'VMS';
	19
f0008e52 MS	20	# VMS uses native, not POSIX, exit codes.
f0008e52 MS	21	my $die_exit = $IsVMS ? 44 : 1;
c0bb2de7 MS	22
	23	# We can only predict that the wait status should be zero or not.
	24	my $wait_non_zero = 1;
f0008e52	25
13287dd5 MS	26	my %samples = (
	27	combined => {
	28	passing => 0,
	29
356733da MS	30	'exit' => 0,
	31	'wait' => 0,
	32
13287dd5 MS	33	max => 10,
	34	seen => 10,
	35
	36	'ok' => 8,
	37	'todo' => 2,
	38	'skip' => 1,
	39	bonus => 1,
	40
	41	details => [ { 'ok' => 1, actual_ok => 1 },
	42	{ 'ok' => 1, actual_ok => 1,
	43	name => 'basset hounds got long ears',
	44	},
	45	{ 'ok' => 0, actual_ok => 0,
	46	name => 'all hell broke lose',
	47	},
	48	{ 'ok' => 1, actual_ok => 1,
	49	type => 'todo'
	50	},
	51	{ 'ok' => 1, actual_ok => 1 },
	52	{ 'ok' => 1, actual_ok => 1 },
	53	{ 'ok' => 1, actual_ok => 1,
	54	type => 'skip',
	55	reason => 'contract negociations'
	56	},
	57	{ 'ok' => 1, actual_ok => 1 },
	58	{ 'ok' => 0, actual_ok => 0 },
	59	{ 'ok' => 1, actual_ok => 0,
	60	type => 'todo'
	61	},
	62	]
	63	},
	64
	65	descriptive => {
	66	passing => 1,
	67
356733da MS	68	'wait' => 0,
	69	'exit' => 0,
	70
13287dd5 MS	71	max => 5,
	72	seen => 5,
	73
	74	'ok' => 5,
	75	'todo' => 0,
	76	'skip' => 0,
	77	bonus => 0,
	78
	79	details => [ { 'ok' => 1, actual_ok => 1,
	80	name => 'Interlock activated'
	81	},
	82	{ 'ok' => 1, actual_ok => 1,
	83	name => 'Megathrusters are go',
	84	},
	85	{ 'ok' => 1, actual_ok => 1,
	86	name => 'Head formed',
	87	},
	88	{ 'ok' => 1, actual_ok => 1,
	89	name => 'Blazing sword formed'
	90	},
	91	{ 'ok' => 1, actual_ok => 1,
	92	name => 'Robeast destroyed'
	93	},
	94	],
	95	},
	96
	97	duplicates => {
	98	passing => 0,
	99
356733da MS	100	'exit' => 0,
	101	'wait' => 0,
	102
13287dd5 MS	103	max => 10,
	104	seen => 11,
	105
	106	'ok' => 11,
	107	'todo' => 0,
	108	'skip' => 0,
	109	bonus => 0,
	110
	111	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 10
	112	],
	113	},
	114
	115	head_end => {
	116	passing => 1,
	117
356733da MS	118	'exit' => 0,
	119	'wait' => 0,
	120
13287dd5 MS	121	max => 4,
	122	seen => 4,
	123
	124	'ok' => 4,
	125	'todo' => 0,
	126	'skip' => 0,
	127	bonus => 0,
	128
	129	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
	130	],
	131	},
	132
	133	lone_not_bug => {
	134	passing => 1,
	135
356733da MS	136	'exit' => 0,
	137	'wait' => 0,
	138
13287dd5 MS	139	max => 4,
	140	seen => 4,
	141
	142	'ok' => 4,
	143	'todo' => 0,
	144	'skip' => 0,
	145	bonus => 0,
	146
	147	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
	148	],
	149	},
	150
	151	head_fail => {
	152	passing => 0,
	153
356733da MS	154	'exit' => 0,
	155	'wait' => 0,
	156
13287dd5 MS	157	max => 4,
	158	seen => 4,
	159
	160	'ok' => 3,
	161	'todo' => 0,
	162	'skip' => 0,
	163	bonus => 0,
	164
	165	details => [ { 'ok' => 1, actual_ok => 1 },
	166	{ 'ok' => 0, actual_ok => 0 },
	167	({ 'ok'=> 1, actual_ok => 1 }) x 2
	168	],
	169	},
308957f5	170
13287dd5 MS	171	simple => {
	172	passing => 1,
	173
356733da MS	174	'exit' => 0,
	175	'wait' => 0,
	176
13287dd5 MS	177	max => 5,
13287dd5 MS	178	seen => 5,
308957f5	179
13287dd5 MS	180	'ok' => 5,
	181	'todo' => 0,
	182	'skip' => 0,
	183	bonus => 0,
308957f5	184
13287dd5 MS	185	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 5
	186	]
	187	},
	188
	189	simple_fail => {
	190	passing => 0,
	191
356733da MS	192	'exit' => 0,
	193	'wait' => 0,
	194
13287dd5 MS	195	max => 5,
13287dd5 MS	196	seen => 5,
308957f5	197
13287dd5 MS	198	'ok' => 3,
	199	'todo' => 0,
	200	'skip' => 0,
	201	bonus => 0,
308957f5	202
13287dd5 MS	203	details => [ { 'ok' => 1, actual_ok => 1 },
	204	{ 'ok' => 0, actual_ok => 0 },
	205	{ 'ok' => 1, actual_ok => 1 },
	206	{ 'ok' => 1, actual_ok => 1 },
	207	{ 'ok' => 0, actual_ok => 0 },
	208	]
	209	},
	210
	211	'skip' => {
	212	passing => 1,
	213
356733da MS	214	'exit' => 0,
	215	'wait' => 0,
	216
13287dd5 MS	217	max => 5,
	218	seen => 5,
	219
	220	'ok' => 5,
	221	'todo' => 0,
	222	'skip' => 1,
	223	bonus => 0,
308957f5	224
13287dd5 MS	225	details => [ { 'ok' => 1, actual_ok => 1 },
	226	{ 'ok' => 1, actual_ok => 1,
	227	type => 'skip',
	228	reason => 'rain delay',
	229	},
	230	({ 'ok' => 1, actual_ok => 1 }) x 3
	231	]
	232	},
	233
	234	skip_all => {
	235	passing => 1,
	236
356733da MS	237	'exit' => 0,
	238	'wait' => 0,
	239
13287dd5 MS	240	max => 0,
	241	seen => 0,
	242	skip_all => 'rope',
	243
	244	'ok' => 0,
	245	'todo' => 0,
	246	'skip' => 0,
	247	bonus => 0,
308957f5	248
13287dd5 MS	249	details => [],
	250	},
	251
	252	'todo' => {
	253	passing => 1,
	254
356733da MS	255	'exit' => 0,
	256	'wait' => 0,
	257
13287dd5 MS	258	max => 5,
13287dd5 MS	259	seen => 5,
308957f5	260
13287dd5 MS	261	'ok' => 5,
	262	'todo' => 2,
	263	'skip' => 0,
	264	bonus => 1,
	265
	266	details => [ { 'ok' => 1, actual_ok => 1 },
	267	{ 'ok' => 1, actual_ok => 1,
	268	type => 'todo' },
	269	{ 'ok' => 1, actual_ok => 0,
	270	type => 'todo' },
	271	({ 'ok' => 1, actual_ok => 1 }) x 2
	272	],
	273	},
	274	taint => {
	275	passing => 1,
	276
356733da MS	277	'exit' => 0,
	278	'wait' => 0,
	279
13287dd5 MS	280	max => 1,
	281	seen => 1,
	282
	283	'ok' => 1,
	284	'todo' => 0,
	285	'skip' => 0,
	286	bonus => 0,
	287
	288	details => [ { 'ok' => 1, actual_ok => 1,
	289	name => '- -T honored'
	290	},
	291	],
	292	},
	293	vms_nit => {
	294	passing => 0,
	295
356733da MS	296	'exit' => 0,
	297	'wait' => 0,
	298
13287dd5 MS	299	max => 2,
	300	seen => 2,
	301
	302	'ok' => 1,
	303	'todo' => 0,
	304	'skip' => 0,
	305	bonus => 0,
	306
	307	details => [ { 'ok' => 0, actual_ok => 0 },
	308	{ 'ok' => 1, actual_ok => 1 },
	309	],
356733da MS	310	},
	311	'die' => {
	312	passing => 0,
	313
f0008e52	314	'exit' => $die_exit,
c0bb2de7	315	'wait' => $wait_non_zero,
356733da MS	316
	317	max => 0,
	318	seen => 0,
	319
	320	'ok' => 0,
	321	'todo' => 0,
	322	'skip' => 0,
	323	bonus => 0,
	324
	325	details => []
	326	},
	327
	328	die_head_end => {
	329	passing => 0,
	330
f0008e52	331	'exit' => $die_exit,
c0bb2de7	332	'wait' => $wait_non_zero,
356733da MS	333
	334	max => 0,
	335	seen => 4,
	336
	337	'ok' => 4,
	338	'todo' => 0,
	339	'skip' => 0,
	340	bonus => 0,
	341
	342	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
	343	],
	344	},
	345
	346	die_last_minute => {
	347	passing => 0,
	348
f0008e52	349	'exit' => $die_exit,
c0bb2de7	350	'wait' => $wait_non_zero,
356733da MS	351
	352	max => 4,
	353	seen => 4,
	354
	355	'ok' => 4,
	356	'todo' => 0,
	357	'skip' => 0,
	358	bonus => 0,
	359
	360	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
	361	],
	362	},
	363
	364	bignum => {
	365	passing => 0,
	366
	367	'exit' => 0,
	368	'wait' => 0,
	369
	370	max => 2,
	371	seen => 4,
308957f5	372
356733da MS	373	'ok' => 4,
	374	'todo' => 0,
	375	'skip' => 0,
	376	bonus => 0,
308957f5	377
356733da MS	378	details => [ { 'ok' => 1, actual_ok => 1 },
	379	{ 'ok' => 1, actual_ok => 1 },
	380	]
	381	},
6e5a998b MS	382
	383	'shbang_misparse' =>{
	384	passing => 1,
	385
	386	'exit' => 0,
	387	'wait' => 0,
	388
	389	max => 2,
	390	seen => 2,
	391
	392	'ok' => 2,
	393	'todo' => 0,
	394	'skip' => 0,
	395	bonus => 0,
	396
	397	details => [ ({ 'ok' => 1, actual_ok => 1 }) x 2 ]
	398	},
13287dd5 MS	399	);
13287dd5 MS	400
c0bb2de7	401	plan tests => (keys(%samples) * 3) + 1;
6e5a998b MS	402
	403	use_ok('Test::Harness::Straps');
	404
356733da MS	405	$SIG{__WARN__} = sub {
	406	warn @_ unless $_[0] =~ /^Enourmous test number/ \|\|
	407	$_[0] =~ /^Can't detailize/
	408	};
13287dd5	409	while( my($test, $expect) = each %samples ) {
308957f5 JH	410	for (0..$#{$expect->{details}}) {
	411	$expect->{details}[$_]{type} = ''
	412	unless exists $expect->{details}[$_]{type};
	413	$expect->{details}[$_]{name} = ''
	414	unless exists $expect->{details}[$_]{name};
	415	$expect->{details}[$_]{reason} = ''
	416	unless exists $expect->{details}[$_]{reason};
	417	}
	418
13287dd5 MS	419	my $strap = Test::Harness::Straps->new;
13287dd5 MS	420	my %results = $strap->analyze_file("$SAMPLE_TESTS/$test");
308957f5	421
356733da	422	is_deeply($results{details}, $expect->{details}, "$test details" );
13287dd5 MS	423
	424	delete $expect->{details};
	425	delete $results{details};
c0bb2de7 MS	426
	427	# We can only check if it's zero or non-zero.
	428	is( !!$results{'wait'}, !!$expect->{'wait'}, 'wait status' );
	429	delete $results{'wait'};
	430	delete $expect->{'wait'};
	431
f0008e52	432	is_deeply(\%results, $expect, " the rest $test" );
13287dd5	433	}