This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
06c0966e6ebcaef7746159abac3ff81915b14b20
[perl5.git] / lib / Test / Harness / t / strap-analyze.t
1 #!/usr/bin/perl -w
2
3 BEGIN {
4     if( $ENV{PERL_CORE} ) {
5         chdir 't';
6         @INC = ('../lib', 'lib');
7     }
8     else {
9         unshift @INC, 't/lib';
10     }
11 }
12
13 use strict;
14 use Test::More;
15 use File::Spec;
16
17 my $Curdir = File::Spec->curdir;
18 my $SAMPLE_TESTS = $ENV{PERL_CORE}
19                     ? File::Spec->catdir($Curdir, 'lib', 'sample-tests')
20                     : File::Spec->catdir($Curdir, 't',   'sample-tests');
21
22
23 my $IsMacPerl = $^O eq 'MacOS';
24 my $IsVMS     = $^O eq 'VMS';
25
26 # VMS uses native, not POSIX, exit codes.
27 my $die_exit = $IsVMS ? 44 : 1;
28
29 # We can only predict that the wait status should be zero or not.
30 my $wait_non_zero = 1;
31
32 my %samples = (
33    combined   => {
34                   passing     => 0,
35
36                   'exit'      => 0,
37                   'wait'      => 0,
38
39                   max         => 10,
40                   seen        => 10,
41
42                   'ok'        => 8,
43                   'todo'      => 2,
44                   'skip'      => 1,
45                   bonus       => 1,
46
47                   details     => [ { 'ok' => 1, actual_ok => 1 },
48                                    { 'ok' => 1, actual_ok => 1,
49                                      name => 'basset hounds got long ears',
50                                    },
51                                    { 'ok' => 0, actual_ok => 0,
52                                      name => 'all hell broke lose',
53                                    },
54                                    { 'ok' => 1, actual_ok => 1,
55                                      type => 'todo'
56                                    },
57                                    { 'ok' => 1, actual_ok => 1 },
58                                    { 'ok' => 1, actual_ok => 1 },
59                                    { 'ok' => 1, actual_ok => 1,
60                                      type   => 'skip',
61                                      reason => 'contract negociations'
62                                    },
63                                    { 'ok' => 1, actual_ok => 1 },
64                                    { 'ok' => 0, actual_ok => 0 },
65                                    { 'ok' => 1, actual_ok => 0,
66                                      type   => 'todo' 
67                                    },
68                                  ]
69                        },
70
71    descriptive      => {
72                         passing     => 1,
73
74                         'wait'      => 0,
75                         'exit'      => 0,
76
77                         max         => 5,
78                         seen        => 5,
79
80                         'ok'          => 5,
81                         'todo'        => 0,
82                         'skip'        => 0,
83                         bonus       => 0,
84
85                         details     => [ { 'ok' => 1, actual_ok => 1,
86                                            name => 'Interlock activated'
87                                          },
88                                          { 'ok' => 1, actual_ok => 1,
89                                            name => 'Megathrusters are go',
90                                          },
91                                          { 'ok' => 1, actual_ok => 1,
92                                            name => 'Head formed',
93                                          },
94                                          { 'ok' => 1, actual_ok => 1,
95                                            name => 'Blazing sword formed'
96                                          },
97                                          { 'ok' => 1, actual_ok => 1,
98                                            name => 'Robeast destroyed'
99                                          },
100                                        ],
101                        },
102
103    duplicates       => {
104                         passing     => 0,
105
106                         'exit'      => 0,
107                         'wait'      => 0,
108
109                         max         => 10,
110                         seen        => 11,
111
112                         'ok'          => 11,
113                         'todo'        => 0,
114                         'skip'        => 0,
115                         bonus       => 0,
116
117                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 10
118                                        ],
119                        },
120
121    head_end         => {
122                         passing     => 1,
123
124                         'exit'      => 0,
125                         'wait'      => 0,
126
127                         max         => 4,
128                         seen        => 4,
129
130                         'ok'        => 4,
131                         'todo'      => 0,
132                         'skip'      => 0,
133                         bonus       => 0,
134
135                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
136                                        ],
137                        },
138
139    lone_not_bug     => {
140                         passing     => 1,
141
142                         'exit'      => 0,
143                         'wait'      => 0,
144
145                         max         => 4,
146                         seen        => 4,
147
148                         'ok'        => 4,
149                         'todo'      => 0,
150                         'skip'      => 0,
151                         bonus       => 0,
152
153                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
154                                        ],
155                        },
156
157    head_fail           => {
158                            passing  => 0,
159
160                            'exit'   => 0,
161                            'wait'   => 0,
162
163                            max      => 4,
164                            seen     => 4,
165
166                            'ok'     => 3,
167                            'todo'   => 0,
168                            'skip'   => 0,
169                            bonus    => 0,
170
171                            details  => [ { 'ok' => 1, actual_ok => 1 },
172                                          { 'ok' => 0, actual_ok => 0 },
173                                          ({ 'ok'=> 1, actual_ok => 1 }) x 2
174                                        ],
175                           },
176
177    simple           => {
178                         passing     => 1,
179
180                         'exit'      => 0,
181                         'wait'      => 0,
182
183                         max         => 5,
184                         seen        => 5,
185
186                         'ok'          => 5,
187                         'todo'        => 0,
188                         'skip'        => 0,
189                         bonus       => 0,
190
191                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 5
192                                        ]
193                        },
194
195    simple_fail      => {
196                         passing     => 0,
197
198                         'exit'      => 0,
199                         'wait'      => 0,
200
201                         max         => 5,
202                         seen        => 5,
203
204                         'ok'          => 3,
205                         'todo'        => 0,
206                         'skip'        => 0,
207                         bonus       => 0,
208
209                         details     => [ { 'ok' => 1, actual_ok => 1 },
210                                          { 'ok' => 0, actual_ok => 0 },
211                                          { 'ok' => 1, actual_ok => 1 },
212                                          { 'ok' => 1, actual_ok => 1 },
213                                          { 'ok' => 0, actual_ok => 0 },
214                                        ]
215                        },
216
217    'skip'             => {
218                         passing     => 1,
219
220                         'exit'      => 0,
221                         'wait'      => 0,
222
223                         max         => 5,
224                         seen        => 5,
225
226                         'ok'          => 5,
227                         'todo'        => 0,
228                         'skip'        => 1,
229                         bonus       => 0,
230
231                         details     => [ { 'ok' => 1, actual_ok => 1 },
232                                          { 'ok'   => 1, actual_ok => 1,
233                                            type   => 'skip',
234                                            reason => 'rain delay',
235                                          },
236                                          ({ 'ok' => 1, actual_ok => 1 }) x 3
237                                        ]
238                        },
239
240    'skip_nomsg'     => {
241                         passing     => 1,
242
243                         'exit'      => 0,
244                         'wait'      => 0,
245
246                         max         => 1,
247                         seen        => 1,
248
249                         'ok'          => 1,
250                         'todo'        => 0,
251                         'skip'        => 1,
252                         bonus       => 0,
253
254                         details     => [ { 'ok'   => 1, actual_ok => 1,
255                                            type   => 'skip',
256                                            reason => '',
257                                          },
258                                        ]
259                        },
260
261    skipall           => {
262                           passing   => 1,
263
264                           'exit'    => 0,
265                           'wait'    => 0,
266
267                           max       => 0,
268                           seen      => 0,
269                           skip_all  => 'rope',
270
271                           'ok'      => 0,
272                           'todo'    => 0,
273                           'skip'    => 0,
274                           bonus     => 0,
275
276                           details   => [],
277                          },
278
279    skipall_nomsg    => {
280                           passing   => 1,
281
282                           'exit'    => 0,
283                           'wait'    => 0,
284
285                           max       => 0,
286                           seen      => 0,
287
288                           'ok'      => 0,
289                           'todo'    => 0,
290                           'skip'    => 0,
291                           bonus     => 0,
292
293                           details   => [],
294                          },
295
296    'todo'             => {
297                         passing     => 1,
298
299                         'exit'      => 0,
300                         'wait'      => 0,
301
302                         max         => 5,
303                         seen        => 5,
304
305                         'ok'          => 5,
306                         'todo'        => 2,
307                         'skip'        => 0,
308                         bonus       => 1,
309
310                         details     => [ { 'ok' => 1, actual_ok => 1 },
311                                          { 'ok' => 1, actual_ok => 1,
312                                            type => 'todo' },
313                                          { 'ok' => 1, actual_ok => 0,
314                                            type => 'todo' },
315                                          ({ 'ok' => 1, actual_ok => 1 }) x 2
316                                        ],
317                        },
318    taint            => {
319                         passing     => 1,
320
321                         'exit'      => 0,
322                         'wait'      => 0,
323
324                         max         => 1,
325                         seen        => 1,
326
327                         'ok'          => 1,
328                         'todo'        => 0,
329                         'skip'        => 0,
330                         bonus       => 0,
331
332                         details     => [ { 'ok' => 1, actual_ok => 1,
333                                            name => '- -T honored'
334                                          },
335                                        ],
336                        },
337    vms_nit          => {
338                         passing     => 0,
339
340                         'exit'      => 0,
341                         'wait'      => 0,
342
343                         max         => 2,
344                         seen        => 2,
345
346                         'ok'          => 1,
347                         'todo'        => 0,
348                         'skip'        => 0,
349                         bonus       => 0,
350
351                         details     => [ { 'ok' => 0, actual_ok => 0 },
352                                          { 'ok' => 1, actual_ok => 1 },
353                                        ],
354                        },
355    'die'            => {
356                         passing     => 0,
357
358                         'exit'      => $die_exit,
359                         'wait'      => $wait_non_zero,
360
361                         max         => 0,
362                         seen        => 0,
363
364                         'ok'        => 0,
365                         'todo'      => 0,
366                         'skip'      => 0,
367                         bonus       => 0,
368
369                         details     => []
370                        },
371
372    die_head_end     => {
373                         passing     => 0,
374
375                         'exit'      => $die_exit,
376                         'wait'      => $wait_non_zero,
377
378                         max         => 0,
379                         seen        => 4,
380
381                         'ok'        => 4,
382                         'todo'      => 0,
383                         'skip'      => 0,
384                         bonus       => 0,
385
386                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
387                                        ],
388                        },
389
390    die_last_minute  => {
391                         passing     => 0,
392
393                         'exit'      => $die_exit,
394                         'wait'      => $wait_non_zero,
395
396                         max         => 4,
397                         seen        => 4,
398
399                         'ok'        => 4,
400                         'todo'      => 0,
401                         'skip'      => 0,
402                         bonus       => 0,
403
404                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 4
405                                        ],
406                        },
407
408    bignum           => {
409                         passing     => 0,
410
411                         'exit'      => 0,
412                         'wait'      => 0,
413
414                         max         => 2,
415                         seen        => 4,
416
417                         'ok'          => 4,
418                         'todo'        => 0,
419                         'skip'        => 0,
420                         bonus       => 0,
421
422                         details     => [ { 'ok' => 1, actual_ok => 1 },
423                                          { 'ok' => 1, actual_ok => 1 },
424                                        ]
425                        },
426
427    'shbang_misparse' =>{
428                         passing     => 1,
429
430                         'exit'      => 0,
431                         'wait'      => 0,
432
433                         max         => 2,
434                         seen        => 2,
435
436                         'ok'          => 2,
437                         'todo'        => 0,
438                         'skip'        => 0,
439                         bonus       => 0,
440
441                         details     => [ ({ 'ok' => 1, actual_ok => 1 }) x 2 ]
442                        },
443 );
444
445 plan tests => (keys(%samples) * 4) + 3;
446
447 use_ok('Test::Harness::Straps');
448
449 $SIG{__WARN__} = sub { 
450     warn @_ unless $_[0] =~ /^Enormous test number/ ||
451                    $_[0] =~ /^Can't detailize/
452 };
453 while( my($test, $expect) = each %samples ) {
454     for (0..$#{$expect->{details}}) {
455         $expect->{details}[$_]{type} = ''
456             unless exists $expect->{details}[$_]{type};
457         $expect->{details}[$_]{name} = ''
458             unless exists $expect->{details}[$_]{name};
459         $expect->{details}[$_]{reason} = ''
460             unless exists $expect->{details}[$_]{reason};
461     }
462
463     my $test_path = File::Spec->catfile($SAMPLE_TESTS, $test);
464     my $strap = Test::Harness::Straps->new;
465     my %results = $strap->analyze_file($test_path);
466
467     is_deeply($results{details}, $expect->{details}, "$test details" );
468
469     delete $expect->{details};
470     delete $results{details};
471
472     SKIP: {
473         skip '$? unreliable in MacPerl', 2 if $IsMacPerl;
474
475         # We can only check if it's zero or non-zero.
476         is( !!$results{'wait'}, !!$expect->{'wait'}, 'wait status' );
477         delete $results{'wait'};
478         delete $expect->{'wait'};
479
480         # Have to check the exit status seperately so we can skip it
481         # in MacPerl.
482         is( $results{'exit'}, $expect->{'exit'} );
483         delete $results{'exit'};
484         delete $expect->{'exit'};
485     }
486
487     is_deeply(\%results, $expect, "  the rest $test" );
488 }
489
490
491 my $strap = Test::Harness::Straps->new;
492 ok( !$strap->analyze_file('I_dont_exist') );
493 is( $strap->{error}, "I_dont_exist does not exist" );