lib/Test/Harness/t/test-harness.t

   1 #!/usr/bin/perl -w
   2
   3 BEGIN {
   4     if( $ENV{PERL_CORE} ) {
   5         chdir 't';
   6         @INC = ('../lib', 'lib');
   7     }
   8     else {
   9         unshift @INC, 't/lib';
  10     }
  11 }
  12
  13 use strict;
  14 use File::Spec;
  15
  16 my $Curdir = File::Spec->curdir;
  17 my $SAMPLE_TESTS = $ENV{PERL_CORE}
  18                     ? File::Spec->catdir($Curdir, 'lib', 'sample-tests')
  19                     : File::Spec->catdir($Curdir, 't',   'sample-tests');
  20
  21
  22 # For shutting up Test::Harness.
  23 # Has to work on 5.004 which doesn't have Tie::StdHandle.
  24 package My::Dev::Null;
  25
  26 sub WRITE  {}
  27 sub PRINT  {}
  28 sub PRINTF {}
  29 sub TIEHANDLE {
  30     my $class = shift;
  31     my $fh    = do { local *HANDLE;  \*HANDLE };
  32     return bless $fh, $class;
  33 }
  34 sub READ {}
  35 sub READLINE {}
  36 sub GETC {}
  37
  38
  39 package main;
  40
  41 use Test::More;
  42
  43 my $IsMacPerl = $^O eq 'MacOS';
  44 my $IsVMS     = $^O eq 'VMS';
  45
  46 # VMS uses native, not POSIX, exit codes.
  47 # MacPerl's exit codes are broken.
  48 my $die_estat = $IsVMS     ? 44 :
  49                 $IsMacPerl ? 0  :
  50                              1;
  51
  52 my %samples = (
  53             simple            => {
  54                                   total => {
  55                                             bonus      => 0,
  56                                             max        => 5,
  57                                             'ok'       => 5,
  58                                             files      => 1,
  59                                             bad        => 0,
  60                                             good       => 1,
  61                                             tests      => 1,
  62                                             sub_skipped=> 0,
  63                                             'todo'     => 0,
  64                                             skipped    => 0,
  65                                            },
  66                                   failed => { },
  67                                   all_ok => 1,
  68                                  },
  69             simple_fail      => {
  70                                  total => {
  71                                            bonus       => 0,
  72                                            max         => 5,
  73                                            'ok'        => 3,
  74                                            files       => 1,
  75                                            bad         => 1,
  76                                            good        => 0,
  77                                            tests       => 1,
  78                                            sub_skipped => 0,
  79                                            'todo'      => 0,
  80                                            skipped     => 0,
  81                                           },
  82                                  failed => {
  83                                             canon      => '2 5',
  84                                            },
  85                                  all_ok => 0,
  86                                 },
  87             descriptive       => {
  88                                   total => {
  89                                             bonus      => 0,
  90                                             max        => 5,
  91                                             'ok'       => 5,
  92                                             files      => 1,
  93                                             bad        => 0,
  94                                             good       => 1,
  95                                             tests      => 1,
  96                                             sub_skipped=> 0,
  97                                             'todo'     => 0,
  98                                             skipped    => 0,
  99                                            },
 100                                   failed => { },
 101                                   all_ok => 1,
 102                                  },
 103             no_nums           => {
 104                                   total => {
 105                                             bonus      => 0,
 106                                             max        => 5,
 107                                             'ok'       => 4,
 108                                             files      => 1,
 109                                             bad        => 1,
 110                                             good       => 0,
 111                                             tests      => 1,
 112                                             sub_skipped=> 0,
 113                                             'todo'     => 0,
 114                                             skipped    => 0,
 115                                            },
 116                                   failed => {
 117                                              canon     => '3',
 118                                             },
 119                                   all_ok => 0,
 120                                  },
 121             'todo'            => {
 122                                   total => {
 123                                             bonus      => 1,
 124                                             max        => 5,
 125                                             'ok'       => 5,
 126                                             files      => 1,
 127                                             bad        => 0,
 128                                             good       => 1,
 129                                             tests      => 1,
 130                                             sub_skipped=> 0,
 131                                             'todo'     => 2,
 132                                             skipped    => 0,
 133                                            },
 134                                   failed => { },
 135                                   all_ok => 1,
 136                                  },
 137             todo_inline       => {
 138                                   total => {
 139                                             bonus       => 1,
 140                                             max         => 3,
 141                                             'ok'        => 3,
 142                                             files       => 1,
 143                                             bad         => 0,
 144                                             good        => 1,
 145                                             tests       => 1,
 146                                             sub_skipped => 0,
 147                                             'todo'      => 2,
 148                                             skipped     => 0,
 149                                            },
 150                                   failed => { },
 151                                   all_ok => 1,
 152                                  },
 153             'skip'            => {
 154                                   total => {
 155                                             bonus      => 0,
 156                                             max        => 5,
 157                                             'ok'       => 5,
 158                                             files      => 1,
 159                                             bad        => 0,
 160                                             good       => 1,
 161                                             tests      => 1,
 162                                             sub_skipped=> 1,
 163                                             'todo'     => 0,
 164                                             skipped    => 0,
 165                                            },
 166                                   failed => { },
 167                                   all_ok => 1,
 168                                  },
 169             'skip_nomsg'      => {
 170                                   total => {
 171                                             bonus      => 0,
 172                                             max        => 1,
 173                                             'ok'       => 1,
 174                                             files      => 1,
 175                                             bad        => 0,
 176                                             good       => 1,
 177                                             tests      => 1,
 178                                             sub_skipped=> 1,
 179                                             'todo'     => 0,
 180                                             skipped    => 0,
 181                                            },
 182                                   failed => { },
 183                                   all_ok => 1,
 184                                  },
 185             bailout           => 0,
 186             combined          => {
 187                                   total => {
 188                                             bonus      => 1,
 189                                             max        => 10,
 190                                             'ok'       => 8,
 191                                             files      => 1,
 192                                             bad        => 1,
 193                                             good       => 0,
 194                                             tests      => 1,
 195                                             sub_skipped=> 1,
 196                                             'todo'     => 2,
 197                                             skipped    => 0
 198                                            },
 199                                   failed => {
 200                                              canon     => '3 9',
 201                                             },
 202                                   all_ok => 0,
 203                                  },
 204             duplicates        => {
 205                                   total => {
 206                                             bonus      => 0,
 207                                             max        => 10,
 208                                             'ok'       => 11,
 209                                             files      => 1,
 210                                             bad        => 1,
 211                                             good       => 0,
 212                                             tests      => 1,
 213                                             sub_skipped=> 0,
 214                                             'todo'     => 0,
 215                                             skipped    => 0,
 216                                            },
 217                                   failed => {
 218                                              canon     => '??',
 219                                             },
 220                                   all_ok => 0,
 221                                  },
 222             head_end          => {
 223                                   total => {
 224                                             bonus      => 0,
 225                                             max        => 4,
 226                                             'ok'       => 4,
 227                                             files      => 1,
 228                                             bad        => 0,
 229                                             good       => 1,
 230                                             tests      => 1,
 231                                             sub_skipped=> 0,
 232                                             'todo'     => 0,
 233                                             skipped    => 0,
 234                                            },
 235                                   failed => { },
 236                                   all_ok => 1,
 237                                  },
 238             head_fail         => {
 239                                   total => {
 240                                             bonus      => 0,
 241                                             max        => 4,
 242                                             'ok'       => 3,
 243                                             files      => 1,
 244                                             bad        => 1,
 245                                             good       => 0,
 246                                             tests      => 1,
 247                                             sub_skipped=> 0,
 248                                             'todo'     => 0,
 249                                             skipped    => 0,
 250                                            },
 251                                   failed => {
 252                                              canon      => '2',
 253                                             },
 254                                   all_ok => 0,
 255                                  },
 256             no_output        => {
 257                                  total => {
 258                                            bonus       => 0,
 259                                            max         => 0,
 260                                            'ok'        => 0,
 261                                            files       => 1,
 262                                            bad         => 1,
 263                                            good        => 0,
 264                                            tests       => 1,
 265                                            sub_skipped => 0,
 266                                            'todo'      => 0,
 267                                            skipped     => 0,
 268                                           },
 269                                  failed => {
 270                                            },
 271                                  all_ok => 0,
 272                                 },
 273             skipall          => {
 274                                   total => {
 275                                             bonus      => 0,
 276                                             max        => 0,
 277                                             'ok'       => 0,
 278                                             files      => 1,
 279                                             bad        => 0,
 280                                             good       => 1,
 281                                             tests      => 1,
 282                                             sub_skipped=> 0,
 283                                             'todo'     => 0,
 284                                             skipped    => 1,
 285                                            },
 286                                   failed => { },
 287                                   all_ok => 1,
 288                                  },
 289             skipall_nomsg   => {
 290                                   total => {
 291                                             bonus      => 0,
 292                                             max        => 0,
 293                                             'ok'       => 0,
 294                                             files      => 1,
 295                                             bad        => 0,
 296                                             good       => 1,
 297                                             tests      => 1,
 298                                             sub_skipped=> 0,
 299                                             'todo'     => 0,
 300                                             skipped    => 1,
 301                                            },
 302                                   failed => { },
 303                                   all_ok => 1,
 304                                  },
 305             with_comments     => {
 306                                   total => {
 307                                             bonus      => 2,
 308                                             max        => 5,
 309                                             'ok'       => 5,
 310                                             files      => 1,
 311                                             bad        => 0,
 312                                             good       => 1,
 313                                             tests      => 1,
 314                                             sub_skipped=> 0,
 315                                             'todo'     => 4,
 316                                             skipped    => 0,
 317                                            },
 318                                   failed => { },
 319                                   all_ok => 1,
 320                                  },
 321             taint             => {
 322                                   total => {
 323                                             bonus      => 0,
 324                                             max        => 1,
 325                                             'ok'       => 1,
 326                                             files      => 1,
 327                                             bad        => 0,
 328                                             good       => 1,
 329                                             tests      => 1,
 330                                             sub_skipped=> 0,
 331                                             'todo'     => 0,
 332                                             skipped    => 0,
 333                                            },
 334                                   failed => { },
 335                                   all_ok => 1,
 336                                  },
 337
 338             'die'             => {
 339                                   total => {
 340                                             bonus      => 0,
 341                                             max        => 0,
 342                                             'ok'       => 0,
 343                                             files      => 1,
 344                                             bad        => 1,
 345                                             good       => 0,
 346                                             tests      => 1,
 347                                             sub_skipped=> 0,
 348                                             'todo'     => 0,
 349                                             skipped    => 0,
 350                                            },
 351                                   failed => {
 352                                              estat      => $die_estat,
 353                                              max        => '??',
 354                                              failed     => '??',
 355                                              canon      => '??',
 356                                             },
 357                                   all_ok => 0,
 358                                  },
 359
 360             die_head_end      => {
 361                                   total => {
 362                                             bonus      => 0,
 363                                             max        => 0,
 364                                             'ok'       => 4,
 365                                             files      => 1,
 366                                             bad        => 1,
 367                                             good       => 0,
 368                                             tests      => 1,
 369                                             sub_skipped=> 0,
 370                                             'todo'     => 0,
 371                                             skipped    => 0,
 372                                            },
 373                                   failed => {
 374                                              estat      => $die_estat,
 375                                              max        => '??',
 376                                              failed     => '??',
 377                                              canon      => '??',
 378                                             },
 379                                   all_ok => 0,
 380                                  },
 381
 382             die_last_minute   => {
 383                                   total => {
 384                                             bonus      => 0,
 385                                             max        => 4,
 386                                             'ok'       => 4,
 387                                             files      => 1,
 388                                             bad        => 1,
 389                                             good       => 0,
 390                                             tests      => 1,
 391                                             sub_skipped=> 0,
 392                                             'todo'     => 0,
 393                                             skipped    => 0,
 394                                            },
 395                                   failed => {
 396                                              estat      => $die_estat,
 397                                              max        => 4,
 398                                              failed     => 0,
 399                                              canon      => '??',
 400                                             },
 401                                   all_ok => 0,
 402                                  },
 403             bignum            => {
 404                                   total => {
 405                                             bonus      => 0,
 406                                             max        => 2,
 407                                             'ok'       => 4,
 408                                             files      => 1,
 409                                             bad        => 1,
 410                                             good       => 0,
 411                                             tests      => 1,
 412                                             sub_skipped=> 0,
 413                                             'todo'     => 0,
 414                                             skipped    => 0,
 415                                            },
 416                                   failed => {
 417                                              canon      => '??',
 418                                             },
 419                                   all_ok => 0,
 420                                  },
 421             'shbang_misparse' => {
 422                                   total => {
 423                                             bonus      => 0,
 424                                             max        => 2,
 425                                             'ok'       => 2,
 426                                             files      => 1,
 427                                             bad        => 0,
 428                                             good       => 1,
 429                                             tests      => 1,
 430                                             sub_skipped=> 0,
 431                                             'todo'     => 0,
 432                                             skipped    => 0,
 433                                            },
 434                                   failed => { },
 435                                   all_ok => 1,
 436                                  },
 437             too_many         => {
 438                                  total => {
 439                                            bonus       => 0,
 440                                            max         => 3,
 441                                            'ok'        => 7,
 442                                            files       => 1,
 443                                            bad         => 1,
 444                                            good        => 0,
 445                                            tests       => 1,
 446                                            sub_skipped => 0,
 447                                            'todo'      => 0,
 448                                            skipped     => 0,
 449                                           },
 450                                  failed => {
 451                                             canon      => '4-7',
 452                                            },
 453                                  all_ok => 0,
 454                                 },
 455             switches         => {
 456                                   total => {
 457                                             bonus      => 0,
 458                                             max        => 1,
 459                                             'ok'       => 1,
 460                                             files      => 1,
 461                                             bad        => 0,
 462                                             good       => 1,
 463                                             tests      => 1,
 464                                             sub_skipped=> 0,
 465                                             'todo'     => 0,
 466                                             skipped    => 0,
 467                                            },
 468                                   failed => { },
 469                                   all_ok => 1,
 470                                  },
 471            );
 472
 473 plan tests => (keys(%samples) * 8) + 1;
 474
 475 use_ok('Test::Harness');
 476 use Test::Harness; # So that we don't get "used only once" warnings on the next line
 477 $Test::Harness::Switches = '"-Mstrict"';
 478
 479 tie *NULL, 'My::Dev::Null' or die $!;
 480
 481 while (my($test, $expect) = each %samples) {
 482     # _run_all_tests() runs the tests but skips the formatting.
 483     my($totals, $failed);
 484     my $warning = '';
 485     my $test_path = File::Spec->catfile($SAMPLE_TESTS, $test);
 486
 487     eval {
 488         select NULL;    # _run_all_tests() isn't as quiet as it should be.
 489         local $SIG{__WARN__} = sub { $warning .= join '', @_; };
 490         ($totals, $failed) =
 491           Test::Harness::_run_all_tests($test_path);
 492     };
 493     select STDOUT;
 494
 495     # $? is unreliable in MacPerl, so we'll just fudge it.
 496     $failed->{estat} = $die_estat if $IsMacPerl and $failed;
 497
 498     SKIP: {
 499         skip "special tests for bailout", 1 unless $test eq 'bailout';
 500         like( $@, '/Further testing stopped: GERONI/i' );
 501     }
 502
 503     SKIP: {
 504         skip "don't apply to a bailout", 5 if $test eq 'bailout';
 505         is( $@, '' );
 506         is( Test::Harness::_all_ok($totals), $expect->{all_ok},
 507                                                   "$test - all ok" );
 508         ok( defined $expect->{total},             "$test - has total" );
 509         is_deeply( {map { $_=>$totals->{$_} } keys %{$expect->{total}}},
 510                    $expect->{total},
 511                                                   "$test - totals" );
 512         is_deeply( {map { $_=>$failed->{$test_path}{$_} }
 513                     keys %{$expect->{failed}}},
 514                    $expect->{failed},
 515                                                   "$test - failed" );
 516     }
 517
 518     SKIP: {
 519         skip "special tests for bignum", 1 unless $test eq 'bignum';
 520         is( $warning, <<WARN );
 521 Enormous test number seen [test 100001]
 522 Can't detailize, too big.
 523 Enormous test number seen [test 136211425]
 524 Can't detailize, too big.
 525 WARN
 526
 527     }
 528
 529     SKIP: {
 530         skip "bignum has known warnings", 1 if $test eq 'bignum';
 531         is( $warning, '' );
 532     }
 533 }