From: David Mitchell Date: Fri, 14 Nov 2014 12:18:12 +0000 (+0000) Subject: add Porting/bench.pl X-Git-Tag: v5.21.7~438 X-Git-Url: https://perl5.git.perl.org/perl5.git/commitdiff_plain/9e7973fa06e83f9e8592f277685d066e2ff6abef add Porting/bench.pl This tool runs code snippets found in t/perf/benchmarks (or similar) under cachegrind, in order to calculate how many instruction reads, data writes, branches, cache misses, etc. that one execution of the snippet uses. It will run them against two or more perl executables and show how much each test has gotten better or worse. It is modelled on the perlbench tool, but since it measures instruction reads etc., rather than timings, it is much more precise and reproducible. It is also considerably faster, and is capable or running tests in parallel. Rather than displaying a single relative percentage per test/perl combination, it displays values for 13 different measurements, such as instruction reads, conditional branch misses etc. This commit also changes the format of t/perf/benchmarks slightly; it becomes an AoH rather than a HoH (to allow checking for duplicate keys), and the test names themselves become a :: hierarchy. --- diff --git a/MANIFEST b/MANIFEST index 0988715..14228ba 100644 --- a/MANIFEST +++ b/MANIFEST @@ -4701,6 +4701,7 @@ pod/splitpod Splits perlfunc into multiple pod pages Policy_sh.SH Hold site-wide preferences between Configure runs. Porting/acknowledgements.pl Generate perldelta acknowledgements text Porting/add-package.pl Add/Update CPAN modules that are part of Core +Porting/bench.pl Run benchmarks against t/perf/benchmarks Porting/bisect-example.sh Example script to use with git bisect run Porting/bisect.pl A tool to make bisecting easy Porting/bisect-runner.pl Tool to be called by git bisect run @@ -5371,6 +5372,7 @@ t/perf/speed.t See if optimisations are keeping things fast t/perl.supp Perl valgrind suppressions t/porting/args_assert.t Check that all PERL_ARGS_ASSERT* macros are used t/porting/authors.t Check that all authors have been acknowledged +t/porting/bench.t Check that Porting/bench.pl is okay t/porting/bincompat.t Check that {non_,}bincompat_options are ordered t/porting/checkcase.t Check whether we are case-insensitive-fs-friendly t/porting/checkcfgvar.t Check that all config.sh-like files are good diff --git a/Porting/README.pod b/Porting/README.pod index 52a32bc..84020f6 100644 --- a/Porting/README.pod +++ b/Porting/README.pod @@ -16,6 +16,10 @@ text. Program to prepare dual-life distributions for insertion into the Perl 5 F and F directories. Now thought to be largely superseded. +=head2 F + +Do performance analysis on the code snippets in F. + =head2 F, F and F Use C to pinpoint changes. diff --git a/Porting/bench.pl b/Porting/bench.pl new file mode 100755 index 0000000..743ea67 --- /dev/null +++ b/Porting/bench.pl @@ -0,0 +1,1205 @@ +#!/usr/bin/perl +# +# A tool for analysing the performance of the code snippets found in +# t/perf/benchmarks or similar + + +=head1 NAME + +bench.pl - Compare the performance of perl code snippets across multiple +perls. + +=head1 SYNOPSIS + + # Basic: run the tests in t/perf/benchmarks against two or more perls + + bench.pl [options] perl1=[label1] perl2[=label2] ... + + # Run bench.pl's own built-in sanity tests + + bench.pl --action=selftest + +=head1 DESCRIPTION + +By default, F will run code snippets found in +F (or similar) under cachegrind, in order to calculate +how many instruction reads, data writes, branches, cache misses, etc. that +one execution of the snippet uses. It will run them against two or more +perl executables and show how much each test has gotten better or worse. + +It is modelled on the F tool, but since it measures instruction +reads etc., rather than timings, it is much more precise and reproducible. +It is also considerably faster, and is capable or running tests in +parallel (with C<-j>). Rather than displaying a single relative +percentage per test/perl combination, it displays values for 13 different +measurements, such as instruction reads, conditional branch misses etc. + +There are options to write the raw data to a file, and to read it back. +This means that you can view the same run data in different views with +different selection and sort options. + +The optional C<=label> after each perl executable is used in the display +output. + +=head1 OPTIONS + +=over 4 + +=item * + +--action=I + +What action to perform. The default is I, which runs the benchmarks +using I as the back end. The only other action at the moment is +I, which runs some basic sanity checks and produces TAP output. + +=item * + +--average + +Only display the overall average, rather than the results for each +individual test. + +=item * + +--benchfile=I + +The path of the file which contains the benchmarks (F +by default). + +=item * + +--bisect=I + +Run a single test against one perl and exit with a zero status if the +named field is in the specified range; exit 1 otherwise. It will complain +if more than one test or perl has been specified. It is intended to be +called as part of a bisect run, to determine when something changed. +For example, + + bench.pl -j 8 --tests=foo --bisect=Ir,100,105 --perlargs=-Ilib ./miniperl + +might be called from bisect to find when the number of instruction reads +for test I falls outside the range 100..105. + +=item * + +--debug + +Enable verbose debugging output. + +=item * + +--fields=I + +Display only the specified fields; for example, + + --fields=Ir,Ir_m,Ir_mm + +If only one field is selected, the output is in more compact form. + +=item * + +--grindargs=I + +Optional command-line arguments to pass to cachegrind invocations. + +=item * + +---help + +Display basic usage information. + +=item * + +-j I +--jobs=I + +Run I jobs in parallel (default 1). This determines how many cachegrind +process will running at a time, and should generally be set to the number +of CPUs available. + +=item * + +--norm=I + +Specify which perl column in the output to treat as the 100% norm. +It may be a column number (0..N-1) or a perl executable name or label. +It defaults to the leftmost column. + +=item * + +--perlargs=I + +Optional command-line arguments to pass to each perl that is run as part of +a cachegrind session. For example, C<--perlargs=-Ilib>. + +=item * + +--raw + +Display raw data counts rather than percentages in the outputs. This +allows you to see the exact number of intruction reads, branch misses etc. +for each test/perl combination. It also causes the C display +per field to be calculated based on the average of each tests's count +rather than average of each percentage. This means that tests with very +high counts will dominate. + +=item * + +--sort=I + +Order the tests in the output based on the value of I in the +column I. The I value is as per C<--norm>. For example + + bench.pl --sort=Dw:perl-5.20.0 perl-5.16.0 perl-5.18.0 perl-5.20.0 + +=item * + +-r I +--read=I + +Read in saved data from a previous C<--write> run from the specified file. + +Requires C to be available. + +=item * + +--tests=I + +Specify a subset of tests to run (or in the case of C<--read>, to display). +It may be either a comma-separated list of test names, or a regular +expression. For example + + --tests=expr::assign::scalar_lex,expr::assign::2list_lex + --tests=/^expr::/ + +=item * + +--verbose + +Display progress information. + +=item * + +-w I +--write=I + +Save the raw data to the specified file. It can be read back later with +C<--read>. + +Requires C to be available. + +=back + +=cut + + + +use 5.010000; +use warnings; +use strict; +use Getopt::Long qw(:config no_auto_abbrev); +use IPC::Open2 (); +use IO::Select; +use POSIX ":sys_wait_h"; + +# The version of the file format used to save data. We refuse to process +# the file if the integer component differs. + +my $FORMAT_VERSION = 1.0; + +# The fields we know about + +my %VALID_FIELDS = map { $_ => 1 } + qw(Ir Ir_m1 Ir_mm Dr Dr_m1 Dr_mm Dw Dw_m1 Dw_mm COND COND_m IND IND_m); + +sub usage { + die < 'grind', + average => 0, + benchfile => 't/perf/benchmarks', + bisect => undef, + debug => 0, + grindargs => '', + fields => undef, + jobs => 1, + norm => 0, + perlargs => '', + raw => 0, + read => undef, + sort => undef, + tests => undef, + verbose => 0, + write => undef, +); + + +# process command-line args and call top-level action + +{ + GetOptions( + 'action=s' => \$OPTS{action}, + 'average' => \$OPTS{average}, + 'benchfile=s' => \$OPTS{benchfile}, + 'bisect=s' => \$OPTS{bisect}, + 'debug' => \$OPTS{debug}, + 'grindargs=s' => \$OPTS{grindargs}, + 'help' => \$OPTS{help}, + 'fields=s' => \$OPTS{fields}, + 'jobs|j=i' => \$OPTS{jobs}, + 'norm=s' => \$OPTS{norm}, + 'perlargs=s' => \$OPTS{perlargs}, + 'raw' => \$OPTS{raw}, + 'read|r=s' => \$OPTS{read}, + 'sort=s' => \$OPTS{sort}, + 'tests=s' => \$OPTS{tests}, + 'verbose' => \$OPTS{verbose}, + 'write|w=s' => \$OPTS{write}, + ) or usage; + + usage if $OPTS{help}; + + + if (defined $OPTS{read} and defined $OPTS{write}) { + die "Error: can't specify both --read and --write options\n"; + } + + if (defined $OPTS{read} or defined $OPTS{write}) { + # fail early if it's not present + require JSON::PP; + } + + if (defined $OPTS{fields}) { + my @f = split /,/, $OPTS{fields}; + for (@f) { + die "Error: --fields: unknown field '$_'\n" + unless $VALID_FIELDS{$_}; + } + my %f = map { $_ => 1 } @f; + $OPTS{fields} = \%f; + } + + my %valid_actions = qw(grind 1 selftest 1); + unless ($valid_actions{$OPTS{action}}) { + die "Error: unrecognised action '$OPTS{action}'\n" + . "must be one of: " . join(', ', sort keys %valid_actions)."\n"; + } + + if (defined $OPTS{sort}) { + my @s = split /:/, $OPTS{sort}; + if (@s != 2) { + die "Error: --sort argument should be of the form field:perl: " + . "'$OPTS{sort}'\n"; + } + my ($field, $perl) = @s; + die "Error: --sort: unknown field '$field\n" + unless $VALID_FIELDS{$field}; + # the 'perl' value will be validated later, after we have processed + # the perls + $OPTS{'sort-field'} = $field; + $OPTS{'sort-perl'} = $perl; + } + + if ($OPTS{action} eq 'selftest') { + if (@ARGV) { + die "Error: no perl executables may be specified with --read\n" + } + } + elsif (defined $OPTS{bisect}) { + die "Error: exactly one perl executable must be specified for bisect\n" + unless @ARGV == 1; + die "Error: Can't specify both --bisect and --read\n" + if defined $OPTS{read}; + die "Error: Can't specify both --bisect and --write\n" + if defined $OPTS{write}; + } + elsif (defined $OPTS{read}) { + if (@ARGV) { + die "Error: no perl executables may be specified with --read\n" + } + } + elsif ($OPTS{raw}) { + unless (@ARGV) { + die "Error: at least one perl executable must be specified\n"; + } + } + else { + unless (@ARGV >= 2) { + die "Error: at least two perl executables must be specified\n"; + } + } + + if ($OPTS{action} eq 'grind') { + do_grind(\@ARGV); + } + elsif ($OPTS{action} eq 'selftest') { + do_selftest(); + } +} +exit 0; + + +# Given a hash ref keyed by test names, filter it by deleting unwanted +# tests, based on $OPTS{tests}. + +sub filter_tests { + my ($tests) = @_; + + my $opt = $OPTS{tests}; + return unless defined $opt; + + my @tests; + + if ($opt =~ m{^/}) { + $opt =~ s{^/(.+)/$}{$1} + or die "Error: --tests regex must be of the form /.../\n"; + for (keys %$tests) { + delete $tests->{$_} unless /$opt/; + } + } + else { + my %t; + for (split /,/, $opt) { + die "Error: no such test found: '$_'\n" unless exists $tests->{$_}; + $t{$_} = 1; + } + for (keys %$tests) { + delete $tests->{$_} unless exists $t{$_}; + } + } +} + + +# Read in the test file, and filter out any tests excluded by $OPTS{tests} + +sub read_tests_file { + my ($file) = @_; + + my $ta = do $file; + unless ($ta) { + die "Error: can't parse '$file': $@\n" if $@; + die "Error: can't read '$file': $!\n"; + } + + my $t = { @$ta }; + filter_tests($t); + return $t; +} + + +# Process the perl/column argument of options like --norm and --sort. +# Return the index of the matching perl. + +sub select_a_perl { + my ($perl, $perls, $who) = @_; + + if ($perl =~ /^[0-9]$/) { + die "Error: $who value $perl outside range 0.." . $#$perls . "\n" + unless $perl < @$perls; + return $perl; + } + else { + my @perl = grep $perls->[$_][0] eq $perl + || $perls->[$_][1] eq $perl, + 0..$#$perls; + die "Error: $who: unrecognised perl '$perl'\n" + unless @perl; + die "Error: $who: ambiguous perl '$perl'\n" + if @perl > 1; + return $perl[0]; + } +} + + +# Validate the list of perl=label on the command line. +# Also validate $OPTS{norm}, $OPTS{sort}; +# Return a list of [ exe, label ] pairs. + +sub process_perls { + my @results; + for my $p (@_) { + my ($perl, $label) = split /=/, $p, 2; + $label //= $perl; + my $r = qx($perl -e 'print qq(ok\n)' 2>&1); + die "Error: unable to execute '$perl': $r" if $r ne "ok\n"; + push @results, [ $perl, $label ]; + } + + $OPTS{norm} = select_a_perl($OPTS{norm}, \@results, "--norm"); + if (defined $OPTS{'sort-perl'}) { + $OPTS{'sort-perl'} = + select_a_perl($OPTS{'sort-perl'}, \@results, "--sort"); + } + + return @results; +} + + +# Return a string containing perl test code wrapped in a loop +# that runs $ARGV[0] times + +sub make_perl_prog { + my ($test, $desc, $setup, $code) = @_; + + return < $bisect_max; + } + + if (defined $OPTS{read}) { + open my $in, '<:encoding(UTF-8)', $OPTS{read} + or die " Error: can't open $OPTS{read} for reading: $!\n"; + my $data = do { local $/; <$in> }; + close $in; + + my $hash = JSON::PP::decode_json($data); + if (int($FORMAT_VERSION) < int($hash->{version})) { + die "Error: unsupported version $hash->{version} in file" + . "'$OPTS{read}' (too new)\n"; + } + ($loop_counts, $perls, $results, $tests) = + @$hash{qw(loop_counts perls results tests)}; + + filter_tests($results); + filter_tests($tests); + } + else { + # How many times to execute the loop for the two trials. The lower + # value is intended to do the loop enough times that branch + # prediction has taken hold; the higher loop allows us to see the + # branch misses after that + $loop_counts = [10, 20]; + + $tests = read_tests_file($OPTS{benchfile}); + die "Error: only a single test may be specified with --bisect\n" + if defined $OPTS{bisect} and keys %$tests != 1; + + $perls = [ process_perls(@$perl_args) ]; + $results = grind_run($tests, $perls, $loop_counts); + } + + if (defined $OPTS{write}) { + my $json = JSON::PP::encode_json({ + version => $FORMAT_VERSION, + loop_counts => $loop_counts, + perls => $perls, + results => $results, + tests => $tests, + }); + + open my $out, '>:encoding(UTF-8)', $OPTS{write} + or die " Error: can't open $OPTS{write} for writing: $!\n"; + print $out $json or die "Error: writing to file '$OPTS{write}': $!\n"; + close $out or die "Error: closing file '$OPTS{write}': $!\n"; + } + else { + my ($processed, $averages) = + grind_process($results, $perls, $loop_counts); + + if (defined $OPTS{bisect}) { + my @r = values %$results; + die "Panic: expected exactly one test result in bisect\n" + if @r != 1; + @r = values %{$r[0]}; + die "Panic: expected exactly one perl result in bisect\n" + if @r != 1; + my $c = $r[0]{$bisect_field}; + die "Panic: no result in bisect for field '$bisect_field'\n" + unless defined $c; + exit 0 if $bisect_min <= $c and $c <= $bisect_max; + exit 1; + } + else { + grind_print($processed, $averages, $perls, $tests); + } + } +} + + +# Run cachegrind for every test/perl combo. +# It may run several processes in parallel when -j is specified. +# Return a hash ref suitable for input to grind_process() + +sub grind_run { + my ($tests, $perls, $counts) = @_; + + # Build a list of all the jobs to run + + my @jobs; + + for my $test (sort keys %$tests) { + + # Create two test progs: one with an empty loop and one with code. + # Note that the empty loop is actually '{1;}' rather than '{}'; + # this causes the loop to have a single nextstate rather than a + # stub op, so more closely matches the active loop; e.g.: + # {1;} => nextstate; unstack + # {$x=1;} => nextstate; const; gvsv; sassign; unstack + my @prog = ( + make_perl_prog($test, @{$tests->{$test}}{qw(desc setup)}, '1'), + make_perl_prog($test, @{$tests->{$test}}{qw(desc setup code)}), + ); + + for my $p (@$perls) { + my ($perl, $label) = @$p; + + # Run both the empty loop and the active loop + # $counts->[0] and $counts->[1] times. + + for my $i (0,1) { + for my $j (0,1) { + my $cmd = "PERL_HASH_SEED=0 " + . "valgrind --tool=cachegrind --branch-sim=yes " + . "--cachegrind-out-file=/dev/null " + . "$OPTS{grindargs} " + . "$perl $OPTS{perlargs} - $counts->[$j] 2>&1"; + # for debugging and error messages + my $id = "$test/$perl " + . ($i ? "active" : "empty") . "/" + . ($j ? "long" : "short") . " loop"; + + push @jobs, { + test => $test, + perl => $perl, + plabel => $label, + cmd => $cmd, + prog => $prog[$i], + active => $i, + loopix => $j, + id => $id, + }; + } + } + } + } + + # Execute each cachegrind and store the results in %results. + + local $SIG{PIPE} = 'IGNORE'; + + my $max_jobs = $OPTS{jobs}; + my $running = 0; # count of executing jobs + my %pids; # map pids to jobs + my %fds; # map fds to jobs + my %results; + my $select = IO::Select->new(); + + while (@jobs or $running) { + + if ($OPTS{debug}) { + printf "Main loop: pending=%d running=%d\n", + scalar(@jobs), $running; + } + + # Start new jobs + + while (@jobs && $running < $max_jobs) { + my $job = shift @jobs; + my ($id, $cmd) =@$job{qw(id cmd)}; + + my ($in, $out, $pid); + warn "Starting $id\n" if $OPTS{verbose}; + eval { $pid = IPC::Open2::open2($out, $in, $cmd); 1; } + or die "Error: while starting cachegrind subprocess" + ." for $id:\n$@"; + $running++; + $pids{$pid} = $job; + $fds{"$out"} = $job; + $job->{out_fd} = $out; + $job->{output} = ''; + $job->{pid} = $pid; + + $out->blocking(0); + $select->add($out); + + if ($OPTS{debug}) { + print "Started pid $pid for $id\n"; + } + + # Note: + # In principle we should write to $in in the main select loop, + # since it may block. In reality, + # a) the code we write to the perl process's stdin is likely + # to be less than the OS's pipe buffer size; + # b) by the time the perl process has read in all its stdin, + # the only output it should have generated is a few lines + # of cachegrind output preamble. + # If these assumptions change, then perform the following print + # in the select loop instead. + + print $in $job->{prog}; + close $in; + } + + # Get output of running jobs + + if ($OPTS{debug}) { + printf "Select: waiting on (%s)\n", + join ', ', sort { $a <=> $b } map $fds{$_}{pid}, + $select->handles; + } + + my @ready = $select->can_read; + + if ($OPTS{debug}) { + printf "Select: pids (%s) ready\n", + join ', ', sort { $a <=> $b } map $fds{$_}{pid}, @ready; + } + + unless (@ready) { + die "Panic: select returned no file handles\n"; + } + + for my $fd (@ready) { + my $j = $fds{"$fd"}; + my $r = sysread $fd, $j->{output}, 8192, length($j->{output}); + unless (defined $r) { + die "Panic: Read from process running $j->{id} gave:\n$!"; + } + next if $r; + + # EOF + + if ($OPTS{debug}) { + print "Got eof for pid $fds{$fd}{pid} ($j->{id})\n"; + } + + $select->remove($j->{out_fd}); + close($j->{out_fd}) + or die "Panic: closing output fh on $j->{id} gave:\n$!\n"; + $running--; + delete $fds{"$j->{out_fd}"}; + my $output = $j->{output}; + + if ($OPTS{debug}) { + my $p = $j->{prog}; + $p =~ s/^/ : /mg; + my $o = $output; + $o =~ s/^/ : /mg; + + print "\n$j->{id}/\nCommand: $j->{cmd}\n" + . "Input:\n$p" + . "Output\n$o"; + } + + $results{$j->{test}}{$j->{perl}}[$j->{active}][$j->{loopix}] + = parse_cachegrind($output, $j->{id}, $j->{perl}); + } + + # Reap finished jobs + + while (1) { + my $kid = waitpid(-1, WNOHANG); + my $ret = $?; + last if $kid <= 0; + + unless (exists $pids{$kid}) { + die "Panic: reaped unexpected child $kid"; + } + my $j = $pids{$kid}; + if ($ret) { + die sprintf("Error: $j->{id} gave return status 0x%04x\n", $ret) + . "with the following output\n:$j->{output}\n"; + } + delete $pids{$kid}; + } + } + + return \%results; +} + + + + +# grind_process(): process the data that has been extracted from +# cachgegrind's output. +# +# $res is of the form ->{benchmark_name}{perl_name}[active][count]{field_name}, +# where active is 0 or 1 indicating an empty or active loop, +# count is 0 or 1 indicating a short or long loop. E.g. +# +# $res->{'expr::assign::scalar_lex'}{perl-5.21.1}[0][10]{Dw_mm} +# +# The $res data structure is modified in-place by this sub. +# +# $perls is [ [ perl-exe, perl-label], .... ]. +# +# $counts is [ N, M ] indicating the counts for the short and long loops. +# +# +# return \%output, \%averages, where +# +# $output{benchmark_name}{perl_name}{field_name} = N +# $averages{perl_name}{field_name} = M +# +# where N is the raw count ($OPTS{raw}), or count_perl0/count_perlI otherwise; +# M is the average raw count over all tests ($OPTS{raw}), or +# 1/(sum(count_perlI/count_perl0)/num_tests) otherwise. + +sub grind_process { + my ($res, $perls, $counts) = @_; + + # Process the four results for each test/perf combo: + # Convert + # $res->{benchmark_name}{perl_name}[active][count]{field_name} = n + # to + # $res->{benchmark_name}{perl_name}{field_name} = averaged_n + # + # $r[0][1] - $r[0][0] is the time to do ($counts->[1]-$counts->[0]) + # empty loops, eliminating startup time + # $r[1][1] - $r[1][0] is the time to do ($counts->[1]-$counts->[0]) + # active loops, eliminating startup time + # (the two startup times may be different because different code + # is being compiled); the difference of the two results above + # divided by the count difference is the time to execute the + # active code once, eliminating both startup and loop overhead. + + for my $tests (values %$res) { + for my $r (values %$tests) { + my $r2; + for (keys %{$r->[0][0]}) { + my $n = ( ($r->[1][1]{$_} - $r->[1][0]{$_}) + - ($r->[0][1]{$_} - $r->[0][0]{$_}) + ) / ($counts->[1] - $counts->[0]); + $r2->{$_} = $n; + } + $r = $r2; + } + } + + my %totals; + my %counts; + my %data; + + my $perl_norm = $perls->[$OPTS{norm}][0]; # the name of the reference perl + + for my $test_name (keys %$res) { + my $res1 = $res->{$test_name}; + my $res2_norm = $res1->{$perl_norm}; + for my $perl (keys %$res1) { + my $res2 = $res1->{$perl}; + for my $field (keys %$res2) { + my ($p, $q) = ($res2_norm->{$field}, $res2->{$field}); + + if ($OPTS{raw}) { + # Avoid annoying '-0.0' displays. Ideally this number + # should never be negative, but fluctuations in + # startup etc can theoretically make this happen + $q = 0 if ($q <= 0 && $q > -0.1); + $totals{$perl}{$field} += $q; + $counts{$perl}{$field}++; + $data{$test_name}{$perl}{$field} = $q; + next; + } + + # $p and $q are notionally integer counts, but + # due to variations in startup etc, it's possible for a + # count which is supposedly zero to be calculated as a + # small positive or negative value. + # In this case, set it to zero. Further below we + # special-case zeros to avoid division by zero errors etc. + + $p = 0.0 if $p < 0.01; + $q = 0.0 if $q < 0.01; + + if ($p == 0.0 && $q == 0.0) { + # Both perls gave a count of zero, so no change: + # treat as 100% + $totals{$perl}{$field} += 1; + $counts{$perl}{$field}++; + $data{$test_name}{$perl}{$field} = 1; + } + elsif ($p == 0.0 || $q == 0.0) { + # If either count is zero, there were too few events + # to give a meaningful ratio (and we will end up with + # division by zero if we try). Mark the result undef, + # indicating that it shouldn't be displayed; and skip + # adding to the average + $data{$test_name}{$perl}{$field} = undef; + } + else { + # For averages, we record q/p rather than p/q. + # Consider a test where perl_norm took 1000 cycles + # and perlN took 800 cycles. For the individual + # results we display p/q, or 1.25; i.e. a quarter + # quicker. For the averages, we instead sum all + # the 0.8's, which gives the total cycles required to + # execute all tests, with all tests given equal + # weight. Later we reciprocate the final result, + # i.e. 1/(sum(qi/pi)/n) + + $totals{$perl}{$field} += $q/$p; + $counts{$perl}{$field}++; + $data{$test_name}{$perl}{$field} = $p/$q; + } + } + } + } + + # Calculate averages based on %totals and %counts accumulated earlier. + + my %averages; + for my $perl (keys %totals) { + my $t = $totals{$perl}; + for my $field (keys %$t) { + $averages{$perl}{$field} = $OPTS{raw} + ? $t->{$field} / $counts{$perl}{$field} + # reciprocal - see comments above + : $counts{$perl}{$field} / $t->{$field}; + } + } + + return \%data, \%averages; +} + + +# grind_print(): display the tabulated results of all the cachegrinds. +# +# Arguments are of the form: +# $results->{benchmark_name}{perl_name}{field_name} = N +# $averages->{perl_name}{field_name} = M +# $perls = [ [ perl-exe, perl-label ], ... ] +# $tests->{test_name}{desc => ..., ...} + +sub grind_print { + my ($results, $averages, $perls, $tests) = @_; + + my @perl_names = map $_->[0], @$perls; + my %perl_labels; + $perl_labels{$_->[0]} = $_->[1] for @$perls; + + my $field_label_width = 6; + # Calculate the width to display for each column. + my $min_width = $OPTS{raw} ? 8 : 6; + my @widths = map { length($_) < $min_width ? $min_width : length($_) } + @perl_labels{@perl_names}; + + # Print header. + + print <[$perlix][0]; + @test_names = sort + { + $results->{$a}{$perl}{$field} + <=> $results->{$b}{$perl}{$field} + } + keys %$results; + } + else { + @test_names = sort(keys %$results); + } + } + + # No point in displaying average for only one test. + push @test_names, 'AVERAGE' unless @test_names == 1; + + # If only a single field is to be displayed, use a more compact + # format with only a single line of output per test. + + my $one_field = defined $OPTS{fields} && keys(%{$OPTS{fields}}) == 1; + + if ($one_field) { + + # The first column will now contain test names rather than + # field names; Calculate the max width. + + $field_label_width = 0; + for (@test_names) { + $field_label_width = length if length > $field_label_width; + } + + # Print the perl executables header. + + print "\n"; + for my $i (0,1) { + print " " x $field_label_width; + for (0..$#widths) { + printf " %*s", $widths[$_], + $i ? ('-' x$widths[$_]) : $perl_labels{$perl_names[$_]}; + } + print "\n"; + } + } + + # Dump the results for each test. + + for my $test_name (@test_names) { + my $doing_ave = ($test_name eq 'AVERAGE'); + my $res1 = $doing_ave ? $averages : $results->{$test_name}; + + unless ($one_field) { + print "\n$test_name"; + print "\n$tests->{$test_name}{desc}" unless $doing_ave; + print "\n\n"; + + # Print the perl executables header. + for my $i (0,1) { + print " " x $field_label_width; + for (0..$#widths) { + printf " %*s", $widths[$_], + $i ? ('-' x$widths[$_]) : $perl_labels{$perl_names[$_]}; + } + print "\n"; + } + } + + for my $field (qw(Ir Dr Dw COND IND + N + COND_m IND_m + N + Ir_m1 Dr_m1 Dw_m1 + N + Ir_mm Dr_mm Dw_mm + )) + { + next if $OPTS{fields} and ! exists $OPTS{fields}{$field}; + + if ($field eq 'N') { + print "\n"; + next; + } + + printf "%*s", $field_label_width, + $one_field ? $test_name : $field; + + for my $i (0..$#widths) { + my $res2 = $res1->{$perl_names[$i]}; + my $p = $res2->{$field}; + if (!defined $p) { + printf " %*s", $widths[$i], '-'; + } + elsif ($OPTS{raw}) { + printf " %*.1f", $widths[$i], $p; + } + else { + printf " %*.2f", $widths[$i], $p * 100; + } + } + print "\n"; + } + } +} + + +# do_selftest(): check that we can parse known cachegrind() +# output formats. If the output of cachegrind changes, add a *new* +# test here; keep the old tests to make sure we continue to parse +# old cachegrinds + +sub do_selftest { + + my @tests = ( + 'standard', + <<'EOF', +==32350== Cachegrind, a cache and branch-prediction profiler +==32350== Copyright (C) 2002-2013, and GNU GPL'd, by Nicholas Nethercote et al. +==32350== Using Valgrind-3.9.0 and LibVEX; rerun with -h for copyright info +==32350== Command: perl5211o /tmp/uiS2gjdqe5 1 +==32350== +--32350-- warning: L3 cache found, using its data for the LL simulation. +==32350== +==32350== I refs: 1,124,055 +==32350== I1 misses: 5,573 +==32350== LLi misses: 3,338 +==32350== I1 miss rate: 0.49% +==32350== LLi miss rate: 0.29% +==32350== +==32350== D refs: 404,275 (259,191 rd + 145,084 wr) +==32350== D1 misses: 9,608 ( 6,098 rd + 3,510 wr) +==32350== LLd misses: 5,794 ( 2,781 rd + 3,013 wr) +==32350== D1 miss rate: 2.3% ( 2.3% + 2.4% ) +==32350== LLd miss rate: 1.4% ( 1.0% + 2.0% ) +==32350== +==32350== LL refs: 15,181 ( 11,671 rd + 3,510 wr) +==32350== LL misses: 9,132 ( 6,119 rd + 3,013 wr) +==32350== LL miss rate: 0.5% ( 0.4% + 2.0% ) +==32350== +==32350== Branches: 202,372 (197,050 cond + 5,322 ind) +==32350== Mispredicts: 19,153 ( 17,742 cond + 1,411 ind) +==32350== Mispred rate: 9.4% ( 9.0% + 26.5% ) +EOF + { + COND => 197050, + COND_m => 17742, + Dr => 259191, + Dr_m1 => 6098, + Dr_mm => 2781, + Dw => 145084, + Dw_m1 => 3510, + Dw_mm => 3013, + IND => 5322, + IND_m => 1411, + Ir => 1124055, + Ir_m1 => 5573, + Ir_mm => 3338, + }, + ); + + for ('t', '.') { + last if require "$_/test.pl"; + } + plan(@tests / 3 * keys %VALID_FIELDS); + + while (@tests) { + my $desc = shift @tests; + my $output = shift @tests; + my $expected = shift @tests; + my $p = parse_cachegrind($output); + for (sort keys %VALID_FIELDS) { + is($p->{$_}, $expected->{$_}, "$desc, $_"); + } + } +} diff --git a/Porting/exec-bit.txt b/Porting/exec-bit.txt index a89d496..a0c91e0 100644 --- a/Porting/exec-bit.txt +++ b/Porting/exec-bit.txt @@ -29,6 +29,7 @@ runtests.SH t/TEST Porting/Maintainers.pl Porting/add-package.pl +Porting/bench.pl Porting/bisect.pl Porting/bisect-example.sh Porting/bisect-runner.pl diff --git a/pod/perlhack.pod b/pod/perlhack.pod index 8fcd618..b966542 100644 --- a/pod/perlhack.pod +++ b/pod/perlhack.pod @@ -961,6 +961,27 @@ available memory can be run safely. See also the documentation for the Test and Test::Harness modules, for more environment variables that affect testing. +=head2 Performance testing + +The file F contains snippets of perl code which are +intended to be benchmarked across a range of perls by the +F tool. If you fix or enhance a performance issue, you +may want to add a representative code sample to the file, then run +F against the previous and current perls to see what difference +it has made, and whether anything else has slowed down as a consequence. + +The file F is designed to test whether a particular +code snippet has been compiled into an optree containing specified +numbers of particular op types. This is good for testing whether +optimisations which alter ops, such as converting an C op into an +C op, are really doing that. + +The files F and F are designed to test +things that run thousands of times slower if a particular optimisation +is broken (for example, the utf8 length cache on long utf8 strings). +Add a test that will take a fraction of a second normally, and minutes +otherwise, causing the test file to time out on failure. + =head1 MORE READING FOR GUTS HACKERS To hack on the Perl guts, you'll need to read the following things: diff --git a/t/perf/benchmarks b/t/perf/benchmarks index 6424934..c137d0c 100644 --- a/t/perf/benchmarks +++ b/t/perf/benchmarks @@ -1,6 +1,6 @@ #!perl -# This file specifies a hash-of-hashes that define snippets of code that +# This file specifies an array-of-hashes that define snippets of code that # can be run by various measurement and profiling tools. # # The basic idea is that any time you add an optimisation that is intended @@ -15,16 +15,30 @@ # will be written that can run selected (or all) snippets in various # environments. These will not be run as part of a normal test suite run. # +# It is intended that the tests in this file will be lightweight; e.g. +# a hash access, an empty function call, or a single regex match etc. +# # This file is designed to be read in by 'do' (and in such a way that # multiple versions of this file from different releases can be read in # by a single process). # -# Each key of the top-level hash is a token that describes a particular -# test. Code will be compiled in the package named after the token, so it -# should match /^\w+$/a. It is intended that this can be used on the -# command line of tools to select particular tests, . +# The top-level array has name/hash pairs (we use an array rather than a +# hash so that duplicate keys can be spotted) Each name is a token that +# describes a particular test. Code will be compiled in the package named +# after the token, so it should match /^(\w|::)+$/a. It is intended that +# this can be used on the command line of tools to select particular +# tests. +# In addition, the package names are arranged into an informal hierarchy +# whose top members are (this is subject to change): +# +# call:: subroutine and method handling +# expr:: expressions: e.g. $x=1, $foo{bar}[0] +# loop:: structural code like for, while(), etc +# regex:: regular expressions +# string:: string handling # -# Each value is also a hash, with three fields: +# +# Each hash has three fields: # # desc is a description of the test # setup is a string containing setup code @@ -33,13 +47,27 @@ # So typically a benchmark tool might do something like # # eval "package $token; $setup; for (1..1000000) { $code }" +# +# Currently the only tool that uses this file is Porting/bench.pl; +# try C for more info -{ - arg_assignment => { - desc => 'assignment to local vars from @_', - setup => 'sub arg_assignment { my ($a, $b, $c) = @_ }', - code => 'arg_assignment(1,2,3)', +[ + 'call::sub::3_args' => { + desc => 'function call with 3 local lexical vars', + setup => 'sub f { my ($a, $b, $c) = @_ }', + code => 'f(1,2,3)', + }, + + 'expr::assign::scalar_lex' => { + desc => 'lexical $x = 1', + setup => 'my $x', + code => '$x = 1', + }, + 'expr::assign::2list_lex' => { + desc => 'lexical ($x, $y) = (1, 2)', + setup => 'my ($x, $y)', + code => '($x, $y) = (1, 2)', }, -}; +]; diff --git a/t/perf/benchmarks.t b/t/perf/benchmarks.t index 4e6c338..873f8db 100644 --- a/t/perf/benchmarks.t +++ b/t/perf/benchmarks.t @@ -14,27 +14,41 @@ use strict; my $file = 'perf/benchmarks'; -my $benchmarks = do $file; +my $benchmark_array = do $file; die $@ if $@; die "$! while trying to read '$file'" if $!; -die "'$file' did not return a hash ref\n" unless ref $benchmarks eq 'HASH'; +die "'$file' did not return an array ref\n" + unless ref $benchmark_array eq 'ARRAY'; + +die "Not an even number of key value pairs in '$file'\n" + if @$benchmark_array % 2; + +my %benchmarks; +while (@$benchmark_array) { + my $key = shift @$benchmark_array; + my $hash = shift @$benchmark_array; + die "Duplicate key '$key' in '$file'\n" if exists $benchmarks{$key}; + $benchmarks{$key} = $hash; +} -plan keys(%$benchmarks) * 3; +plan keys(%benchmarks) * 3; # check the hash of hashes is minimally consistent in format -for my $token (sort keys %$benchmarks) { - like($token, qr/^[a-zA-Z]\w*$/a, "legal token: $token"); - my $keys = join('-', sort keys %{$benchmarks->{$token}}); +for my $token (sort keys %benchmarks) { + like($token, qr/^[a-zA-Z](\w|::)+$/a, "legal token: $token"); + my $keys = join('-', sort keys %{$benchmarks{$token}}); is($keys, 'code-desc-setup', "legal keys: $token"); } # check that each bit of code compiles and runs -for my $token (sort keys %$benchmarks) { - my $b = $benchmarks->{$token}; +for my $token (sort keys %benchmarks) { + my $b = $benchmarks{$token}; my $code = "package $token; $b->{setup}; for (1..1) { $b->{code} } 1;"; + no warnings; + no strict; ok(eval $code, "running $token") or do { diag("code:"); diff --git a/t/perf/opcount.t b/t/perf/opcount.t index 8897604..659a80e 100644 --- a/t/perf/opcount.t +++ b/t/perf/opcount.t @@ -17,7 +17,7 @@ BEGIN { @INC = '../lib'; } -plan 3; +plan 28; use B (); @@ -72,3 +72,46 @@ test_opcount(0, "basic aelemfast", 'ex-aelem' => 1, } ); + +# Porting/bench.pl tries to create an empty and active loop, with the +# ops executed being exactly the same apart from the additional ops +# in the active loop. Check that this remains true. + +{ + test_opcount(0, "bench.pl empty loop", + sub { for my $x (1..$ARGV[0]) { 1; } }, + { + aelemfast => 1, + and => 1, + const => 1, + enteriter => 1, + iter => 1, + leaveloop => 1, + leavesub => 1, + lineseq => 2, + nextstate => 2, + null => 1, + pushmark => 1, + unstack => 1, + } + ); + + test_opcount(0, "bench.pl active loop", + sub { for my $x (1..$ARGV[0]) { $x; } }, + { + aelemfast => 1, + and => 1, + const => 1, + enteriter => 1, + iter => 1, + leaveloop => 1, + leavesub => 1, + lineseq => 2, + nextstate => 2, + null => 1, + padsv => 1, # this is the additional active op + pushmark => 1, + unstack => 1, + } + ); +} diff --git a/t/porting/bench.t b/t/porting/bench.t new file mode 100644 index 0000000..527995a --- /dev/null +++ b/t/porting/bench.t @@ -0,0 +1,15 @@ +#!./perl -w + +# run Porting/bench.pl's selftest + +BEGIN { + @INC = '..' if -f '../TestInit.pm'; +} +use TestInit qw(T A); # T is chdir to the top level, A makes paths absolute +use strict; + +require 't/test.pl'; +my $source = find_git_or_skip('all'); +chdir $source or die "Can't chdir to $source: $!"; + +system "$^X Porting/bench.pl --action=selftest";