lib/Test/Harness/t/test-harness.t

   1 #!/usr/bin/perl -w
   2
   3 BEGIN {
   4     if( $ENV{PERL_CORE} ) {
   5         chdir 't';
   6         @INC = ('../lib', 'lib');
   7     }
   8     else {
   9         unshift @INC, 't/lib';
  10     }
  11 }
  12
  13 use strict;
  14 use File::Spec;
  15
  16 my $Curdir = File::Spec->curdir;
  17 my $SAMPLE_TESTS = $ENV{PERL_CORE}
  18                     ? File::Spec->catdir($Curdir, 'lib', 'sample-tests')
  19                     : File::Spec->catdir($Curdir, 't',   'sample-tests');
  20
  21
  22 # For shutting up Test::Harness.
  23 # Has to work on 5.004 which doesn't have Tie::StdHandle.
  24 package My::Dev::Null;
  25
  26 sub WRITE  {}
  27 sub PRINT  {}
  28 sub PRINTF {}
  29 sub TIEHANDLE {
  30     my $class = shift;
  31     my $fh    = do { local *HANDLE;  \*HANDLE };
  32     return bless $fh, $class;
  33 }
  34 sub READ {}
  35 sub READLINE {}
  36 sub GETC {}
  37
  38
  39 package main;
  40
  41 use Test::More;
  42
  43 my $IsMacOS   = $^O eq 'MacOS';
  44 my $IsVMS     = $^O eq 'VMS';
  45
  46 # VMS uses native, not POSIX, exit codes.
  47 my $die_estat = $IsVMS ? 44 : $IsMacOS ? 0 : 1;
  48
  49 my %samples = (
  50             simple            => {
  51                                   total => {
  52                                             bonus      => 0,
  53                                             max        => 5,
  54                                             'ok'       => 5,
  55                                             files      => 1,
  56                                             bad        => 0,
  57                                             good       => 1,
  58                                             tests      => 1,
  59                                             sub_skipped=> 0,
  60                                             'todo'     => 0,
  61                                             skipped    => 0,
  62                                            },
  63                                   failed => { },
  64                                   all_ok => 1,
  65                                  },
  66             simple_fail      => {
  67                                  total => {
  68                                            bonus       => 0,
  69                                            max         => 5,
  70                                            'ok'        => 3,
  71                                            files       => 1,
  72                                            bad         => 1,
  73                                            good        => 0,
  74                                            tests       => 1,
  75                                            sub_skipped => 0,
  76                                            'todo'      => 0,
  77                                            skipped     => 0,
  78                                           },
  79                                  failed => {
  80                                             canon      => '2 5',
  81                                            },
  82                                  all_ok => 0,
  83                                 },
  84             descriptive       => {
  85                                   total => {
  86                                             bonus      => 0,
  87                                             max        => 5,
  88                                             'ok'       => 5,
  89                                             files      => 1,
  90                                             bad        => 0,
  91                                             good       => 1,
  92                                             tests      => 1,
  93                                             sub_skipped=> 0,
  94                                             'todo'     => 0,
  95                                             skipped    => 0,
  96                                            },
  97                                   failed => { },
  98                                   all_ok => 1,
  99                                  },
 100             no_nums           => {
 101                                   total => {
 102                                             bonus      => 0,
 103                                             max        => 5,
 104                                             'ok'       => 4,
 105                                             files      => 1,
 106                                             bad        => 1,
 107                                             good       => 0,
 108                                             tests      => 1,
 109                                             sub_skipped=> 0,
 110                                             'todo'     => 0,
 111                                             skipped    => 0,
 112                                            },
 113                                   failed => {
 114                                              canon     => '3',
 115                                             },
 116                                   all_ok => 0,
 117                                  },
 118             'todo'            => {
 119                                   total => {
 120                                             bonus      => 1,
 121                                             max        => 5,
 122                                             'ok'       => 5,
 123                                             files      => 1,
 124                                             bad        => 0,
 125                                             good       => 1,
 126                                             tests      => 1,
 127                                             sub_skipped=> 0,
 128                                             'todo'     => 2,
 129                                             skipped    => 0,
 130                                            },
 131                                   failed => { },
 132                                   all_ok => 1,
 133                                  },
 134             todo_inline       => {
 135                                   total => {
 136                                             bonus       => 1,
 137                                             max         => 3,
 138                                             'ok'        => 3,
 139                                             files       => 1,
 140                                             bad         => 0,
 141                                             good        => 1,
 142                                             tests       => 1,
 143                                             sub_skipped => 0,
 144                                             'todo'      => 2,
 145                                             skipped     => 0,
 146                                            },
 147                                   failed => { },
 148                                   all_ok => 1,
 149                                  },
 150             'skip'            => {
 151                                   total => {
 152                                             bonus      => 0,
 153                                             max        => 5,
 154                                             'ok'       => 5,
 155                                             files      => 1,
 156                                             bad        => 0,
 157                                             good       => 1,
 158                                             tests      => 1,
 159                                             sub_skipped=> 1,
 160                                             'todo'     => 0,
 161                                             skipped    => 0,
 162                                            },
 163                                   failed => { },
 164                                   all_ok => 1,
 165                                  },
 166             'skip_nomsg'      => {
 167                                   total => {
 168                                             bonus      => 0,
 169                                             max        => 1,
 170                                             'ok'       => 1,
 171                                             files      => 1,
 172                                             bad        => 0,
 173                                             good       => 1,
 174                                             tests      => 1,
 175                                             sub_skipped=> 1,
 176                                             'todo'     => 0,
 177                                             skipped    => 0,
 178                                            },
 179                                   failed => { },
 180                                   all_ok => 1,
 181                                  },
 182             bailout           => 0,
 183             combined          => {
 184                                   total => {
 185                                             bonus      => 1,
 186                                             max        => 10,
 187                                             'ok'       => 8,
 188                                             files      => 1,
 189                                             bad        => 1,
 190                                             good       => 0,
 191                                             tests      => 1,
 192                                             sub_skipped=> 1,
 193                                             'todo'     => 2,
 194                                             skipped    => 0
 195                                            },
 196                                   failed => {
 197                                              canon     => '3 9',
 198                                             },
 199                                   all_ok => 0,
 200                                  },
 201             duplicates        => {
 202                                   total => {
 203                                             bonus      => 0,
 204                                             max        => 10,
 205                                             'ok'       => 11,
 206                                             files      => 1,
 207                                             bad        => 1,
 208                                             good       => 0,
 209                                             tests      => 1,
 210                                             sub_skipped=> 0,
 211                                             'todo'     => 0,
 212                                             skipped    => 0,
 213                                            },
 214                                   failed => {
 215                                              canon     => '??',
 216                                             },
 217                                   all_ok => 0,
 218                                  },
 219             head_end          => {
 220                                   total => {
 221                                             bonus      => 0,
 222                                             max        => 4,
 223                                             'ok'       => 4,
 224                                             files      => 1,
 225                                             bad        => 0,
 226                                             good       => 1,
 227                                             tests      => 1,
 228                                             sub_skipped=> 0,
 229                                             'todo'     => 0,
 230                                             skipped    => 0,
 231                                            },
 232                                   failed => { },
 233                                   all_ok => 1,
 234                                  },
 235             head_fail         => {
 236                                   total => {
 237                                             bonus      => 0,
 238                                             max        => 4,
 239                                             'ok'       => 3,
 240                                             files      => 1,
 241                                             bad        => 1,
 242                                             good       => 0,
 243                                             tests      => 1,
 244                                             sub_skipped=> 0,
 245                                             'todo'     => 0,
 246                                             skipped    => 0,
 247                                            },
 248                                   failed => {
 249                                              canon      => '2',
 250                                             },
 251                                   all_ok => 0,
 252                                  },
 253             skipall          => {
 254                                   total => {
 255                                             bonus      => 0,
 256                                             max        => 0,
 257                                             'ok'       => 0,
 258                                             files      => 1,
 259                                             bad        => 0,
 260                                             good       => 1,
 261                                             tests      => 1,
 262                                             sub_skipped=> 0,
 263                                             'todo'     => 0,
 264                                             skipped    => 1,
 265                                            },
 266                                   failed => { },
 267                                   all_ok => 1,
 268                                  },
 269             skipall_nomsg   => {
 270                                   total => {
 271                                             bonus      => 0,
 272                                             max        => 0,
 273                                             'ok'       => 0,
 274                                             files      => 1,
 275                                             bad        => 0,
 276                                             good       => 1,
 277                                             tests      => 1,
 278                                             sub_skipped=> 0,
 279                                             'todo'     => 0,
 280                                             skipped    => 1,
 281                                            },
 282                                   failed => { },
 283                                   all_ok => 1,
 284                                  },
 285             with_comments     => {
 286                                   total => {
 287                                             bonus      => 2,
 288                                             max        => 5,
 289                                             'ok'       => 5,
 290                                             files      => 1,
 291                                             bad        => 0,
 292                                             good       => 1,
 293                                             tests      => 1,
 294                                             sub_skipped=> 0,
 295                                             'todo'     => 4,
 296                                             skipped    => 0,
 297                                            },
 298                                   failed => { },
 299                                   all_ok => 1,
 300                                  },
 301             taint             => {
 302                                   total => {
 303                                             bonus      => 0,
 304                                             max        => 1,
 305                                             'ok'       => 1,
 306                                             files      => 1,
 307                                             bad        => 0,
 308                                             good       => 1,
 309                                             tests      => 1,
 310                                             sub_skipped=> 0,
 311                                             'todo'     => 0,
 312                                             skipped    => 0,
 313                                            },
 314                                   failed => { },
 315                                   all_ok => 1,
 316                                  },
 317
 318             'die'             => {
 319                                   total => {
 320                                             bonus      => 0,
 321                                             max        => 0,
 322                                             'ok'       => 0,
 323                                             files      => 1,
 324                                             bad        => 1,
 325                                             good       => 0,
 326                                             tests      => 1,
 327                                             sub_skipped=> 0,
 328                                             'todo'     => 0,
 329                                             skipped    => 0,
 330                                            },
 331                                   failed => {
 332                                              estat      => $die_estat,
 333                                              max        => '??',
 334                                              failed     => '??',
 335                                              canon      => '??',
 336                                             },
 337                                   all_ok => 0,
 338                                  },
 339
 340             die_head_end      => {
 341                                   total => {
 342                                             bonus      => 0,
 343                                             max        => 0,
 344                                             'ok'       => 4,
 345                                             files      => 1,
 346                                             bad        => 1,
 347                                             good       => 0,
 348                                             tests      => 1,
 349                                             sub_skipped=> 0,
 350                                             'todo'     => 0,
 351                                             skipped    => 0,
 352                                            },
 353                                   failed => {
 354                                              estat      => $die_estat,
 355                                              max        => '??',
 356                                              failed     => '??',
 357                                              canon      => '??',
 358                                             },
 359                                   all_ok => 0,
 360                                  },
 361
 362             die_last_minute   => {
 363                                   total => {
 364                                             bonus      => 0,
 365                                             max        => 4,
 366                                             'ok'       => 4,
 367                                             files      => 1,
 368                                             bad        => 1,
 369                                             good       => 0,
 370                                             tests      => 1,
 371                                             sub_skipped=> 0,
 372                                             'todo'     => 0,
 373                                             skipped    => 0,
 374                                            },
 375                                   failed => {
 376                                              estat      => $die_estat,
 377                                              max        => 4,
 378                                              failed     => 0,
 379                                              canon      => '??',
 380                                             },
 381                                   all_ok => 0,
 382                                  },
 383             bignum            => {
 384                                   total => {
 385                                             bonus      => 0,
 386                                             max        => 2,
 387                                             'ok'       => 4,
 388                                             files      => 1,
 389                                             bad        => 1,
 390                                             good       => 0,
 391                                             tests      => 1,
 392                                             sub_skipped=> 0,
 393                                             'todo'     => 0,
 394                                             skipped    => 0,
 395                                            },
 396                                   failed => {
 397                                              canon      => '??',
 398                                             },
 399                                   all_ok => 0,
 400                                  },
 401             'shbang_misparse' => {
 402                                   total => {
 403                                             bonus      => 0,
 404                                             max        => 2,
 405                                             'ok'       => 2,
 406                                             files      => 1,
 407                                             bad        => 0,
 408                                             good       => 1,
 409                                             tests      => 1,
 410                                             sub_skipped=> 0,
 411                                             'todo'     => 0,
 412                                             skipped    => 0,
 413                                            },
 414                                   failed => { },
 415                                   all_ok => 1,
 416                                  },
 417            );
 418
 419 plan tests => (keys(%samples) * 8) + 1;
 420
 421 use Test::Harness;
 422 use_ok('Test::Harness');
 423
 424
 425 tie *NULL, 'My::Dev::Null' or die $!;
 426
 427 while (my($test, $expect) = each %samples) {
 428     # _run_all_tests() runs the tests but skips the formatting.
 429     my($totals, $failed);
 430     my $warning = '';
 431     my $test_path = File::Spec->catfile($SAMPLE_TESTS, $test);
 432
 433     eval {
 434         select NULL;    # _run_all_tests() isn't as quiet as it should be.
 435         local $SIG{__WARN__} = sub { $warning .= join '', @_; };
 436         ($totals, $failed) =
 437           Test::Harness::_run_all_tests($test_path);
 438     };
 439     select STDOUT;
 440
 441     # $? is unreliable in MacPerl, so we'll simply fudge it.
 442     $failed->{estat} = $die_estat if $IsMacOS and $failed;
 443
 444     SKIP: {
 445         skip "special tests for bailout", 1 unless $test eq 'bailout';
 446         like( $@, '/Further testing stopped: GERONI/i' );
 447     }
 448
 449     SKIP: {
 450         skip "don't apply to a bailout", 5 if $test eq 'bailout';
 451         is( $@, '' );
 452         is( Test::Harness::_all_ok($totals), $expect->{all_ok},
 453                                                   "$test - all ok" );
 454         ok( defined $expect->{total},             "$test - has total" );
 455         is_deeply( {map { $_=>$totals->{$_} } keys %{$expect->{total}}},
 456                    $expect->{total},
 457                                                   "$test - totals" );
 458         is_deeply( {map { $_=>$failed->{$test_path}{$_} }
 459                     keys %{$expect->{failed}}},
 460                    $expect->{failed},
 461                                                   "$test - failed" );
 462     }
 463
 464     SKIP: {
 465         skip "special tests for bignum", 1 unless $test eq 'bignum';
 466         is( $warning, <<WARN );
 467 Enormous test number seen [test 100001]
 468 Can't detailize, too big.
 469 Enormous test number seen [test 136211425]
 470 Can't detailize, too big.
 471 WARN
 472
 473     }
 474
 475     SKIP: {
 476         skip "bignum has known warnings", 1 if $test eq 'bignum';
 477         is( $warning, '' );
 478     }
 479 }