3 # Check the functionality of the Porting/bench.pl executable;
4 # in particular, its argument handling and its ability to produce
5 # the expected output for particular arguments.
7 # See also t/porting/bench_selftest.pl
10 chdir '..' if -f 'test.pl';
12 require './t/test.pl';
20 # Only test on git checkouts - this is more of a perl core developer
21 # tool than an end-user tool.
22 # Only test on a platform likely to support forking, pipes, cachegrind
23 # etc. Add other platforms if you think they're safe.
25 skip_all "not devel" unless -d "./.git";
26 skip_all "not linux" unless $^O eq 'linux';
27 skip_all "no valgrind" unless -x '/bin/valgrind' || -x '/usr/bin/valgrind';
28 # Address sanitizer clashes horribly with cachegrind
29 skip_all "not with ASAN" if $Config{ccflags} =~ /sanitize=address/;
30 # If this takes more than 15 second then something is very wrong
31 skip_all "cachegrind broken" if system "( ulimit -c 0; ulimit -t 15; valgrind -q --tool=cachegrind --cachegrind-out-file=/dev/null $^X -e0 ) 2>/dev/null";
34 my $bench_pl = "Porting/bench.pl";
36 ok -e $bench_pl, "$bench_pl exists and is executable";
38 my $bench_cmd = "$^X -Ilib $bench_pl";
42 # Read in the expected output format templates and create qr//s from them.
52 die "invalid format line: $_" unless /^FORMAT:\s+(\w+)\s*$/;
54 die "duplicate format: '$cur'\n" if exists $formats{$cur};
60 for my $name (sort keys %formats) {
61 my $f = $formats{$name};
63 # expand "%%SUB_FORMAT%%
64 $f =~ s{^ \s* %% (\w+) %% [ \t]* \n}
66 my $f1 = $formats{$1};
67 die "No such sub-format '%%$1%%' in format '$name'\n"
74 # convert NNNN.NN placeholders into a regex
84 # convert run of space chars into ' +' or ' *'
86 $f =~ s/(\A|\n)(\\ )+/$1 */g;
89 # convert '---' placeholders into a regex
90 $f =~ s/(\\-){2,}/-+/g;
92 $format_qrs{$name} = qr/\A$f\z/;
97 # ---------------------------------------------------
103 "Unknown option: boz\nUse the -h option for usage information.\n",
104 "croak: basic unknown option"
108 "Error: --fields: unknown field 'Boz'\n",
109 "croak: unknown --field"
113 "Error: unrecognised action 'boz'\nmust be one of: grind, selftest\n",
114 "croak: unknown --action"
118 "Error: --sort argument should be of the form field:perl: 'boz'\n",
119 "croak: invalid --sort"
123 "Error: --sort: unknown field 'boz'\n",
124 "croak: unknown --sort field"
127 "-action=selftest perl",
128 "Error: no perl executables may be specified with selftest\n",
129 "croak: --action-selftest with executable"
133 "Error: --tests regex must be of the form /.../\n",
134 "croak: invalid --tests regex"
137 "--tests=call::sub::empty,foo::bar::baz::boz perl",
138 "Error: no such test found: 'foo::bar::baz::boz'\n"
139 . "Re-run with --verbose for a list of valid tests.\n",
140 "croak: unknown test in --tests"
143 "--verbose --tests=call::sub::empty,foo::bar::baz::boz --read=t/porting/bench/callsub.json",
144 "Error: no such test found: 'foo::bar::baz::boz'\n"
145 . "Valid test names are:\n"
146 . " call::sub::amp_empty\n"
147 . " call::sub::empty\n",
148 "croak: unknown test in --tests --verbose"
151 "--tests=/foo::bar::baz::boz/ perl",
152 "Error: no tests to run\n",
153 "croak: no --tests to run "
156 "--benchfile=no-such-file-boz perl",
157 qr/\AError: can't read 'no-such-file-boz':/,
158 "croak: non-existent --benchfile "
161 "--benchfile=t/porting/bench/synerr perl",
162 qr{\AError: can't parse 't/porting/bench/synerr':\nsyntax error},
163 "croak: --benchfile with syntax error"
166 "--benchfile=t/porting/bench/ret0 perl",
167 "Error: can't load 't/porting/bench/ret0': code didn't return a true value\n",
168 "croak: --benchfile which returns 0"
171 "--benchfile=t/porting/bench/oddentry perl",
172 qr{\AError: 't/porting/bench/oddentry' does not contain evenly paired test names and hashes\n},
173 "croak: --benchfile with odd number of entries"
176 "--benchfile=t/porting/bench/badname perl",
177 qr{\AError: 't/porting/bench/badname': invalid test name: '1='\n},
178 "croak: --benchfile with invalid test name"
181 "--benchfile=t/porting/bench/badhash perl",
182 qr{\AError: 't/porting/bench/badhash': invalid key 'blah' for test 'foo::bar'\n},
183 "croak: --benchfile with invalid test hash key"
186 "--norm=2 ./miniperl ./perl",
187 "Error: --norm value 2 outside range 0..1\n",
188 "croak: select-a-perl out of range"
191 "--norm=-0 ./miniperl ./perl",
192 "Error: --norm value -0 outside range -1..-2\n",
193 "croak: select-a-perl out of range"
196 "--norm=-3 ./miniperl ./perl",
197 "Error: --norm value -3 outside range -1..-2\n",
198 "croak: select-a-perl out of range"
201 "--sort=Ir:myperl ./miniperl ./perl",
202 "Error: --sort: unrecognised perl 'myperl'\n"
203 . "Valid perl names are:\n"
206 "croak: select-a-perl unrecognised"
209 "--compact=./perl ./perl=A ./perl=B",
210 "Error: --compact: ambiguous perl './perl'\n",
211 "croak: select-a-perl ambiguous"
215 "Error: unrecognised executable switch '--foo'\n",
216 "croak: ./perl --foo"
220 "Error: --args without a preceding executable name\n",
221 "croak: --args without perl"
225 "Error: --env without a preceding executable name\n",
226 "croak: --env without perl"
230 "Error: --args is missing value\n",
231 "croak: --args without value"
235 "Error: --env is missing value\n",
236 "croak: --env without value"
239 "./perl --env='FOO'",
240 "Error: --env is missing =value\n",
241 "croak: --env without =value"
245 "Error: duplicate label './perl': each executable must have a unique label\n",
246 "croak: duplicate label ./perl ./perl"
249 "./perl=A ./miniperl=A",
250 "Error: duplicate label 'A': each executable must have a unique label\n",
251 "croak: duplicate label =A =A"
254 "--read=t/porting/bench/callsub.json --read=t/porting/bench/callsub.json",
255 "Error: duplicate label './perl': seen in file 't/porting/bench/callsub.json'\n",
256 "croak: duplicate label --read=... --read=..."
259 "--read=t/porting/bench/callsub.json ./perl",
260 "Error: duplicate label './perl': seen both in --read file and on command line\n",
261 "croak: duplicate label --read=... ./perl"
265 qr{^\QError: unable to execute './nosuch-perl': },
266 "croak: no such perl"
269 "--grindargs=Boz --debug --tests=call::sub::empty ./perl=A ./perl=B",
270 qr{Error: .*?(unexpected code or cachegrind output|gave return status)}s,
271 "croak: cachegrind output format "
275 "Error: --bisect option must be of form 'field,integer,integer'\n",
280 "Error: --bisect option must be of form 'field,integer,integer'\n",
281 "croak: --bisect=Ir,1"
285 "Error: --bisect option must be of form 'field,integer,integer'\n",
286 "croak: --bisect=Ir,1,2,3"
290 "Error: --bisect option must be of form 'field,integer,integer'\n",
291 "croak: --bisect=Ir,1,x"
295 "Error: --bisect option must be of form 'field,integer,integer'\n",
296 "croak: --bisect=Ir,x,2"
300 "Error: unrecognised field 'boz' in --bisect option\n",
301 "croak: --bisect=boz,1,2"
305 "Error: --bisect min (2) must be <= max (1)\n",
306 "croak: --bisect=boz,2,1"
309 "--read=no-such-file-boz",
310 qr/\AError: can't open 'no-such-file-boz' for reading:/,
311 "croak: non-existent --read file "
314 "--read=t/porting/bench/badversion.json",
315 "Error: unsupported version 9999.9 in file 't/porting/bench/badversion.json' (too new)\n",
316 "croak: --read version"
319 "--read=t/porting/bench/callsub.json --benchfile=t/perf/benchmarks ./perl ",
320 "Error: --benchfile cannot be used when --read is present\n",
321 "croak: benchfile with read"
325 "Error: nothing to do: no perls to run, no data to read.\n",
330 "Error: need at least 2 perls for comparison.\n",
331 "croak: need 2 perls"
334 "--bisect=Ir,1,2 ./perl=A ./perl=B",
335 "Error: exactly one perl executable must be specified for bisect\n",
336 "croak: --bisect, need 1 perls"
339 "--bisect=Ir,1,2 --tests=/call/ ./perl=A",
340 "Error: only a single test may be specified with --bisect\n",
341 "croak: --bisect one test only"
343 # note that callsub.json was created using
344 # ./perl -Ilib Porting/bench.pl --tests='/call::sub::(amp_)?empty/' \
345 # --write=t/porting/bench/callsub.json ./perl
347 "--read=t/porting/bench/callsub.json --write=no/such/file/boz",
348 qr{\AError: can't open 'no/such/file/boz' for writing: },
349 "croak: --write open error"
351 # note that callsub2.json was created using
352 # ./perl -Ilib Porting/bench.pl \
353 # --tests='call::sub::empty,call::sub::args3' \
354 # --write=t/porting/bench/callsub2.json ./perl=perl2
356 "--read=t/porting/bench/callsub.json "
357 . " --read=t/porting/bench/callsub2.json",
358 "Can't merge multiple read files: they contain differing test sets.\n"
359 . "Re-run with --verbose to see the differences.\n",
360 "croak: --read callsub, callsub2"
363 "--read=t/porting/bench/callsub.json "
364 . " --read=t/porting/bench/callsub2.json"
366 "Can't merge multiple read files: they contain differing test sets.\n"
367 . "Previous tests:\n"
368 . " call::sub::amp_empty\n"
369 . " call::sub::empty\n"
370 . "tests from 't/porting/bench/callsub2.json':\n"
371 . " call::sub::args3\n"
372 . " call::sub::empty\n",
373 "croak: --read callsub, callsub2 --verbose"
376 # these ones aren't tested (and nor are any "Panic:" ones):
378 # Error: can't parse '$field' field from cachegrind output
379 # Error: while starting cachegrind subprocess for NNNN:
380 # File '$file' contains no results
381 # File '$file' contains differing test and results names
382 # File '$file' contains differing test and sort order names
383 # Can't merge multiple read files: differing loop counts:
386 my ($args, $expected, $desc) = @$test;
387 $out = qx($bench_cmd $args 2>&1);
388 if (ref($expected)) {
389 like $out, $expected, $desc;
392 is $out, $expected, $desc;
396 # ---------------------------------------------------
400 my $resultfile1 = tempfile(); # benchmark results for 1 perl
401 my $resultfile2 = tempfile(); # benchmark results for 2 perls
403 # Run a real cachegrind session and write results to file.
404 # the -j 2 is to minimally exercise its parallel facility.
406 note("running cachegrind for 1st perl; may be slow...");
407 $out = qx($bench_cmd -j 2 --write=$resultfile1 --tests=call::sub::empty $^X=p0 2>&1);
408 is $out, "", "--write should produce no output (1 perl)";
409 ok -s $resultfile1, "--write should create a non-empty results file (1 perl)";
411 # and again with 2 perls. This is also tests the 'mix read and new
412 # perls' functionality.
414 note("running cachegrind for 2nd perl; may be slow...");
415 $out = qx($bench_cmd -j 2 --read=$resultfile1 --write=$resultfile2 $^X=p1 2>&1);
416 is $out, "", "--write should produce no output (2 perls)"
417 or diag("got: $out");
418 ok -s $resultfile2, "--write should create a non-empty results file (2 perls)";
422 # read back the results in raw form
424 $out = qx($bench_cmd --read=$resultfile1 --raw 2>&1);
425 like $out, $format_qrs{raw1}, "basic cachegrind raw format; 1 perl";
427 # and read back the results in raw compact form
429 $out = qx($bench_cmd --read=$resultfile1 --raw --compact=0 2>&1);
430 like $out, $format_qrs{raw_compact}, "basic cachegrind raw compact format; 1 perl";
432 # and read back the results in raw average form
434 $out = qx($bench_cmd --read=$resultfile1 --raw --average 2>&1);
435 like $out, $format_qrs{raw_average1}, "basic cachegrind raw average format; 1 perl";
437 # and read back the results with raw selected fields
439 $out = qx($bench_cmd --read=$resultfile1 --raw --fields=Ir,Dr 2>&1);
440 like $out, $format_qrs{fields1}, "basic cachegrind --fields; 1 perl";
444 # read back the results in relative-percent form
446 $out = qx($bench_cmd --read=$resultfile2 2>&1);
447 like $out, $format_qrs{percent2}, "basic cachegrind percent format; 2 perls";
449 # read back the results in relative-percent form with norm
451 $out = qx($bench_cmd --read=$resultfile2 --norm=0 2>&1);
452 like $out, $format_qrs{percent2}, "basic cachegrind percent format, norm; 2 perls";
454 # ditto with negative norm
456 $out = qx($bench_cmd --read=$resultfile2 --norm=-2 2>&1);
457 like $out, $format_qrs{percent2}, "basic cachegrind percent format, norm -2; 2 perls";
459 # read back the results in relative-percent form with sort
461 $out = qx($bench_cmd --read=$resultfile2 --sort=Ir:0 2>&1);
462 like $out, $format_qrs{percent2}, "basic cachegrind percent format, sort; 2 perls";
464 # read back the results in relative-percent form with sort and norm
466 $out = qx($bench_cmd --read=$resultfile2 --sort=Ir:0 --norm=0 2>&1);
467 like $out, $format_qrs{percent2}, "basic cachegrind percent format, sort, norm; 2 perls";
469 # and read back the results in raw form
471 $out = qx($bench_cmd --read=$resultfile2 --raw 2>&1);
472 like $out, $format_qrs{raw2}, "basic cachegrind raw format; 2 perls";
474 # and read back the results in raw form with norm
476 $out = qx($bench_cmd --read=$resultfile2 --raw --norm=0 2>&1);
477 like $out, $format_qrs{raw2}, "basic cachegrind raw format, norm; 2 perls";
479 # and read back the results in raw form with sort
481 $out = qx($bench_cmd --read=$resultfile2 --raw --sort=Ir:0 2>&1);
482 like $out, $format_qrs{raw2}, "basic cachegrind raw format, sort, norm; 2 perls";
484 # and read back the results in raw form with sort and norm
486 $out = qx($bench_cmd --read=$resultfile2 --raw --sort=Ir:0 --norm=0 2>&1);
487 like $out, $format_qrs{raw2}, "basic cachegrind raw format, sort, norm; 2 perls";
489 # and read back the results in compact form
491 $out = qx($bench_cmd --read=$resultfile2 --compact=1 2>&1);
492 like $out, $format_qrs{compact}, "basic cachegrind compact format; 2 perls";
494 # and read back the results in average form
496 $out = qx($bench_cmd --read=$resultfile2 --average 2>&1);
497 like $out, $format_qrs{average}, "basic cachegrind average format; 2 perls";
499 # and read back the results with selected fields
501 $out = qx($bench_cmd --read=$resultfile2 --fields=Ir,Dr 2>&1);
502 like $out, $format_qrs{fields2}, "basic cachegrind --fields; 2 perls";
504 # and read back the results in compact form with selected fields
506 $out = qx($bench_cmd --read=$resultfile2 --compact=1 --fields=Ir,Dr 2>&1);
507 like $out, $format_qrs{compact_fields}, "basic cachegrind compact, fields; 2 perls";
509 # and read back the results with 1 selected fields (this is more compact)
511 $out = qx($bench_cmd --read=$resultfile2 --fields=Ir 2>&1);
512 like $out, $format_qrs{'1field'}, "basic cachegrind 1 field; 2 perls";
517 # the Ir range here is intended such that the bisect will always fail
518 $out = qx($bench_cmd --read=t/porting/bench/callsub.json --tests=call::sub::empty --bisect=Ir,100000,100001 2>&1);
520 is $?, 1 << 8, "--bisect: exit result: should not match";
521 like $out, qr/^Bisect: Ir had the value -?\d+\n/,
522 "--bisect: got expected output";
524 # multiple reads with differing test sets but common --tests subset
526 $out = qx($bench_cmd --read=t/porting/bench/callsub.json --read=t/porting/bench/callsub2.json --tests=call::sub::empty 2>&1);
527 $out =~ s{\Q./perl perl2}{ p0 p1};
528 $out =~ s{^\./perl}{p0}m;
529 like $out, $format_qrs{percent2}, "2 reads; overlapping test sets";
531 # A read defines what benchmarks to run
533 note("running cachegrind on 1 perl; may be slow...");
534 $out = qx($bench_cmd --read=t/porting/bench/callsub.json --tests=call::sub::empty $^X=p1 2>&1);
535 $out =~ s{^\./perl}{p0}m;
536 $out =~ s{\Q./perl}{ p0};
537 like $out, $format_qrs{percent2}, "1 read; 1 generate";
539 # Process environment and optional args.
540 # This is a minimal test that it runs - it doesn't test whether
541 # the environment and args are getting applied correctly, apart from the
542 # fact that the perls in question are being successfully executed.
544 # Also check the --autolabel feature
546 note("running cachegrind on 2 perls; may be slow...");
549 --read=t/porting/bench/callsub.json
550 --read=t/porting/bench/callsub2.json
551 --tests=call::sub::empty
554 $^X --args='-Ifoo/bar -Mstrict' --env='FOO=foo'
555 $^X --args='-Ifoo/bar' --env='BAR=bar' --env='BAZ=baz'
560 $out =~ s{^\./perl}{p0}m;
561 $out =~ s{\Q ./perl perl2 p-0 p-1}
563 like $out, $format_qrs{percent4}, "4 perls with autolabel and args and env";
569 # Templates for expected output formats.
571 # Lines starting with '#' are skipped.
573 # Lines of the form 'FORMAT: foo' start and name a new template
575 # All other lines are part of the template
577 # Entries of the form NNNN.NN are converted into a regex of the form
578 # ( \s* -? \d+\.\d\d | - )
579 # i.e. it expects number with a fixed number of digits after the point,
582 # Any runs of space chars (but not tab) are converted into ' +',
583 # or ' *' if at the start of a line
585 # Entries of the form --- are converted into [-]+
587 # Lines of the form %%FOO%% are substituted with format 'FOO'
591 # ===================================================================
597 COND conditional branches
598 IND indirect branches
599 _m branch predict miss
600 _m1 level 1 cache miss
601 _mm last cache (e.g. L3) miss
602 - indeterminate percentage (e.g. 1/0)
603 # ===================================================================
607 The numbers represent relative counts per loop iteration, compared to
609 Higher is better: for example, using half as many instructions gives 200%,
610 while using twice as many gives 50%.
613 function call with no args or body
633 # ===================================================================
637 The numbers represent relative counts per loop iteration, compared to
639 Higher is better: for example, using half as many instructions gives 200%,
640 while using twice as many gives 50%.
643 function call with no args or body
646 ------ ------ ------ ------
647 Ir 100.00 NNN.NN NNN.NN NNN.NN
648 Dr 100.00 NNN.NN NNN.NN NNN.NN
649 Dw 100.00 NNN.NN NNN.NN NNN.NN
650 COND 100.00 NNN.NN NNN.NN NNN.NN
651 IND 100.00 NNN.NN NNN.NN NNN.NN
653 COND_m 100.00 NNN.NN NNN.NN NNN.NN
654 IND_m 100.00 NNN.NN NNN.NN NNN.NN
656 Ir_m1 100.00 NNN.NN NNN.NN NNN.NN
657 Dr_m1 100.00 NNN.NN NNN.NN NNN.NN
658 Dw_m1 100.00 NNN.NN NNN.NN NNN.NN
660 Ir_mm 100.00 NNN.NN NNN.NN NNN.NN
661 Dr_mm 100.00 NNN.NN NNN.NN NNN.NN
662 Dw_mm 100.00 NNN.NN NNN.NN NNN.NN
663 # ===================================================================
667 The numbers represent relative counts per loop iteration, compared to
669 Higher is better: for example, using half as many instructions gives 200%,
670 while using twice as many gives 50%.
673 function call with no args or body
679 # ===================================================================
683 The numbers represent raw counts per loop iteration.
686 function call with no args or body
706 # ===================================================================
710 The numbers represent raw counts per loop iteration.
732 # ===================================================================
736 The numbers represent raw counts per loop iteration.
739 function call with no args or body
745 # ===================================================================
749 The numbers represent raw counts per loop iteration.
752 function call with no args or body
759 COND NNNNNN.N NNNNNN.N
760 IND NNNNNN.N NNNNNN.N
762 COND_m NNNNNN.N NNNNNN.N
763 IND_m NNNNNN.N NNNNNN.N
765 Ir_m1 NNNNNN.N NNNNNN.N
766 Dr_m1 NNNNNN.N NNNNNN.N
767 Dw_m1 NNNNNN.N NNNNNN.N
769 Ir_mm NNNNNN.N NNNNNN.N
770 Dr_mm NNNNNN.N NNNNNN.N
771 Dw_mm NNNNNN.N NNNNNN.N
772 # ===================================================================
776 The numbers represent relative counts per loop iteration, compared to
778 Higher is better: for example, using half as many instructions gives 200%,
779 while using twice as many gives 50%.
783 Ir Dr Dw COND IND COND_m IND_m Ir_m1 Dr_m1 Dw_m1 Ir_mm Dr_mm Dw_mm
784 ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------
785 NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN call::sub::empty function call with no args or body
786 # ===================================================================
787 FORMAT: compact_fields
790 The numbers represent relative counts per loop iteration, compared to
792 Higher is better: for example, using half as many instructions gives 200%,
793 while using twice as many gives 50%.
799 NNN.NN NNN.NN call::sub::empty function call with no args or body
800 # ===================================================================
804 The numbers represent relative counts per loop iteration, compared to
806 Higher is better: for example, using half as many instructions gives 200%,
807 while using twice as many gives 50%.
813 call::sub::empty NNN.NN NNN.NN
814 # ===================================================================
818 The numbers represent relative counts per loop iteration, compared to
820 Higher is better: for example, using half as many instructions gives 200%,
821 while using twice as many gives 50%.
843 # ===================================================================
847 The numbers represent raw counts per loop iteration.
851 Ir Dr Dw COND IND COND_m IND_m Ir_m1 Dr_m1 Dw_m1 Ir_mm Dr_mm Dw_mm
852 ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------
853 NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N call::sub::empty function call with no args or body
854 # ===================================================================