[perl5.git] / t / porting / bench.t

#!/usr/bin/perl

# Check the functionality of the Porting/bench.pl executable;
# in particular, its argument handling and its ability to produce
# the expected output for particular arguments.
#
# See also t/porting/bench_selftest.pl

use warnings;
use strict;

BEGIN {
    chdir '..' if -f 'test.pl' && -f 'thread_it.pl';
    require './t/test.pl';
}

# Only test on git checkouts - this is more of a perl core developer
# tool than an end-user tool.
# Only test on a platform likely to support forking, pipes, cachegrind
# etc.  Add other platforms if you think they're safe.

skip_all "not devel"   unless -d ".git";
skip_all "not linux"   unless $^O eq 'linux';
skip_all "no valgrind" unless -x '/bin/valgrind' || -x '/usr/bin/valgrind';


my $bench_pl = "Porting/bench.pl";

ok -e $bench_pl, "$bench_pl exists and is executable";

my $bench_cmd = "$^X -Ilib $bench_pl";

my $out;

# Read in the expected output format templates and create qr//s from them.

my %formats;
my %format_qrs;

{
    my $cur;
    while (<DATA>) {
        next if /^#/;
        if (/^FORMAT:/) {
            die "invalid format line: $_" unless /^FORMAT:\s+(\w+)\s*$/;
            $cur = $1;
            die "duplicate format: '$cur'\n" if exists $formats{$cur};
            next;
        }
        $formats{$cur} .= $_;
    }

    for my $name (sort keys %formats) {
        my $f = $formats{$name};

        # expand "%%SUB_FORMAT%%
        $f =~ s{^ \s* %% (\w+) %% [ \t]* \n}
               {
                    my $f1 = $formats{$1};
                    die "No such sub-format '%%$1%%' in format '$name'\n"
                        unless defined $f1;
                    $f1;
               }gmxe;

        $f = quotemeta $f;

        # convert NNNN.NN placeholders into a regex
        $f =~ s{(N+)\\.(N+)}
               {
                    "("
                    . "\\s*-?\\d+\."
                    . "\\d" x length($2)
                    ."|-)"
               }ge;
        $format_qrs{$name} = qr/\A$f\z/;
    }
}


# ---------------------------------------------------
# check croaks

for my $test (
    [
        "--boz",
        "Unknown option: boz\nUse the -h option for usage information.\n",
        "croak: basic unknown option"
    ],
    [
        "--fields=Ir,Boz",
        "Error: --fields: unknown field 'Boz'\n",
        "croak: unknown --field"
    ],
    [
        "--action=boz",
        "Error: unrecognised action 'boz'\nmust be one of: grind, selftest\n",
        "croak: unknown --action"
    ],
    [
        "--sort=boz",
        "Error: --sort argument should be of the form field:perl: 'boz'\n",
        "croak: invalid --sort"
    ],
    [
        "--sort=boz:perl",
        "Error: --sort: unknown field 'boz'\n",
        "croak: unknown --sort field"
    ],
    [
        "-action=selftest perl",
        "Error: no perl executables may be specified with selftest\n",
        "croak: --action-selftest with executable"
    ],
    [
        "--tests=/boz perl",
        "Error: --tests regex must be of the form /.../\n",
        "croak: invalid --tests regex"
    ],
    [
        "--tests=call::sub::empty,foo::bar::baz::boz perl",
        "Error: no such test found: 'foo::bar::baz::boz'\n",
        "croak: unknown test in --tests"
    ],
    [
        "--tests=/foo::bar::baz::boz/ perl",
        "Error: no tests to run\n",
        "croak: no --tests to run "
    ],
    [
        "--benchfile=no-such-file-boz perl",
        qr/\AError: can't read 'no-such-file-boz':/,
        "croak: non-existent --benchfile "
    ],
    [
        "--benchfile=t/porting/bench/synerr perl",
        qr{\AError: can't parse 't/porting/bench/synerr':\nsyntax error},
        "croak: --benchfile with syntax error"
    ],
    [
        "--benchfile=t/porting/bench/ret0 perl",
        "Error: can't load 't/porting/bench/ret0': code didn't return a true value\n",
        "croak: --benchfile which returns 0"
    ],
    [
        "--norm=2 ./miniperl ./perl",
        "Error: --norm value 2 outside range 0..1\n",
        "croak: select-a-perl out of range"
    ],
    [
        "--sort=Ir:myperl ./miniperl ./perl",
        "Error: --sort: unrecognised perl 'myperl'\n",
        "croak: select-a-perl unrecognised"
    ],
    [
        "--compact=./perl ./perl=A ./perl=B",
        "Error: --compact: ambiguous perl './perl'\n",
        "croak: select-a-perl ambiguous"
    ],
    [
        " ./perl=A ./miniperl=A",
        "A cannot be used on 2 different perls under test\n",
        "croak: duplicate label"
    ],
    [
        "--grindargs=Boz --tests=call::sub::empty ./perl=A ./perl=B",
        qr{Error: while executing call::sub::empty/A empty/short loop:\nunexpected code or cachegrind output:\n},
        "croak: cachegrind output format "
    ],
    [
        "--bisect=Ir",,
        "Error: --bisect option must be of form 'field,integer,integer'\n",
        "croak: --bisect=Ir"
    ],
    [
        "--bisect=Ir,1",,
        "Error: --bisect option must be of form 'field,integer,integer'\n",
        "croak: --bisect=Ir,1"
    ],
    [
        "--bisect=Ir,1,2,3",
        "Error: --bisect option must be of form 'field,integer,integer'\n",
        "croak: --bisect=Ir,1,2,3"
    ],
    [
        "--bisect=Ir,1,x",
        "Error: --bisect option must be of form 'field,integer,integer'\n",
        "croak: --bisect=Ir,1,x"
    ],
    [
        "--bisect=Ir,x,2",
        "Error: --bisect option must be of form 'field,integer,integer'\n",
        "croak: --bisect=Ir,x,2"
    ],
    [
        "--bisect=boz,1,2",
        "Error: unrecognised field 'boz' in --bisect option\n",
        "croak: --bisect=boz,1,2"
    ],
    [
        "--bisect=Ir,2,1",
        "Error: --bisect min (2) must be <= max (1)\n",
        "croak: --bisect=boz,2,1"
    ],
    [
        "--read=no-such-file-boz",
        qr/\AError: can't open 'no-such-file-boz' for reading:/,
        "croak: non-existent --read file "
    ],
    [
        "--read=t/porting/bench/badversion.json",
        "Error: unsupported version 9999.9 in file 't/porting/bench/badversion.json' (too new)\n",
        "croak: --read version"
    ],
    [
        "",
        "Error: nothing to do: no perls to run, no data to read.\n",
        "croak: no input"
    ],
    [
        "./perl",
        "Error: need at least 2 perls for comparison.\n",
        "croak: need 2 perls"
    ],
    [
        "--bisect=Ir,1,2 ./perl=A ./perl=B",
        "Error: exactly one perl executable must be specified for bisect\n",
        "croak: --bisect, need 1 perls"
    ],
    [
        "--bisect=Ir,1,2 --tests=/call/ ./perl=A",
        "Error: only a single test may be specified with --bisect\n",
        "croak: --bisect one test only"
    ],
    # note that callsub.json was created using
    # ./perl -Ilib Porting/bench.pl --tests='/call::sub::(amp_)?empty/' \
    #                     --write=t/porting/bench/callsub.json ./perl

    [
        "--read=t/porting/bench/callsub.json --write=no/such/file/boz",
        qr{\AError: can't open 'no/such/file/boz' for writing: },
        "croak: --write open error"
    ],

    # these ones isn't tested:

    # Error: can't parse '$field' field from cachegrind output
    # Error: while starting cachegrind subprocess for NNNN:
)
{
    my ($args, $expected, $desc) = @$test;
    $out = qx($bench_cmd $args 2>&1);
    if (ref($expected)) {
        like $out, $expected, $desc;
    }
    else {
        is $out, $expected, $desc;
    }
}

# ---------------------------------------------------
# run benchmarks


my $resultfile1 = tempfile(); # benchmark results for 1 perl
my $resultfile2 = tempfile(); # benchmark results for 2 perls

# Run a real cachegrind session and write results to file.
# the -j 2 is to minimally exercise its parallel facility.

note("running cachegrind for 1st perl; may be slow...");
$out = qx($bench_cmd -j 2 --write=$resultfile1 --tests=call::sub::empty $^X=p0 2>&1);
is length($out), 0, "--write should produce no output (1 perl)";
ok -s $resultfile1, "--write should create a non-empty results file (1 perl)";

# and again with 2 perls. This is also tests the 'mix read and new new
# perls' functionality.

note("running cachegrind for 2nd perl; may be slow...");
$out = qx($bench_cmd -j 2 --read=$resultfile1 --write=$resultfile2 $^X=p1 2>&1);
is length($out), 0, "--write should produce no output (2 perls)"
    or diag("got: $out");
ok -s $resultfile2, "--write should create a non-empty results file (2 perls)";

# 1 perl:

# read back the results in raw form

$out = qx($bench_cmd --read=$resultfile1 --raw 2>&1);
like $out, $format_qrs{raw1}, "basic cachegrind raw format; 1 perl";

# and read back the results in raw compact form

$out = qx($bench_cmd --read=$resultfile1 --raw --compact=0 2>&1);
like $out, $format_qrs{raw_compact}, "basic cachegrind raw compact format; 1 perl";

# 2 perls:

# read back the results in relative-percent form

$out = qx($bench_cmd --read=$resultfile2 2>&1);
like $out, $format_qrs{percent2}, "basic cachegrind percent format; 2 perls";

# and read back the results in raw form

$out = qx($bench_cmd --read=$resultfile2 --raw 2>&1);
like $out, $format_qrs{raw2}, "basic cachegrind raw format; 2 perls";

# and read back the results in compact form

$out = qx($bench_cmd --read=$resultfile2 --compact=1 2>&1);
like $out, $format_qrs{compact}, "basic cachegrind compact format; 2 perls";


# bisect

note("running cachegrind bisect on 1 perl; may be slow...");

# the Ir range here is intended such that the bisect will always fail
$out = qx($bench_cmd --tests=call::sub::empty --bisect=Ir,100000,100001 $^X=p0 2>&1);

is $?, 1 << 8, "--bisect should not match";
is length($out), 0, "--bisect should produce no output"
    or diag("got: $out");


done_testing();


# Templates for expected output formats.
#
# Lines starting with '#' are skipped.
# Lines of the form 'FORMAT: foo' start and name a new template
# All other lines are part of the template
# Entries of the form NNNN.NN are converted into a regex of the form
#    ( \s* -? \d+\.\d\d | - )
# i.e. it expects number with a fixed number of digits after the point,
# or a '-'.
# Lines of the form %%FOO%% are substituted with format 'FOO'


__END__
# ===================================================================
FORMAT: STD_HEADER
Key:
    Ir   Instruction read
    Dr   Data read
    Dw   Data write
    COND conditional branches
    IND  indirect branches
    _m   branch predict miss
    _m1  level 1 cache miss
    _mm  last cache (e.g. L3) miss
    -    indeterminate percentage (e.g. 1/0)
# ===================================================================
FORMAT: percent2
%%STD_HEADER%%

The numbers represent relative counts per loop iteration, compared to
p0 at 100.0%.
Higher is better: for example, using half as many instructions gives 200%,
while using twice as many gives 50%.

call::sub::empty
function call with no args or body

           p0     p1
       ------ ------
    Ir 100.00 NNN.NN
    Dr 100.00 NNN.NN
    Dw 100.00 NNN.NN
  COND 100.00 NNN.NN
   IND 100.00 NNN.NN

COND_m 100.00 NNN.NN
 IND_m 100.00 NNN.NN

 Ir_m1 100.00 NNN.NN
 Dr_m1 100.00 NNN.NN
 Dw_m1 100.00 NNN.NN

 Ir_mm 100.00 NNN.NN
 Dr_mm 100.00 NNN.NN
 Dw_mm 100.00 NNN.NN
# ===================================================================
FORMAT: raw1
%%STD_HEADER%%

The numbers represent raw counts per loop iteration.

call::sub::empty
function call with no args or body

             p0
       --------
    Ir NNNNNN.N
    Dr NNNNNN.N
    Dw NNNNNN.N
  COND NNNNNN.N
   IND NNNNNN.N

COND_m NNNNNN.N
 IND_m NNNNNN.N

 Ir_m1 NNNNNN.N
 Dr_m1 NNNNNN.N
 Dw_m1 NNNNNN.N

 Ir_mm NNNNNN.N
 Dr_mm NNNNNN.N
 Dw_mm NNNNNN.N
# ===================================================================
FORMAT: raw2
%%STD_HEADER%%

The numbers represent raw counts per loop iteration.

call::sub::empty
function call with no args or body

             p0       p1
       -------- --------
    Ir NNNNNN.N NNNNNN.N
    Dr NNNNNN.N NNNNNN.N
    Dw NNNNNN.N NNNNNN.N
  COND NNNNNN.N NNNNNN.N
   IND NNNNNN.N NNNNNN.N

COND_m NNNNNN.N NNNNNN.N
 IND_m NNNNNN.N NNNNNN.N

 Ir_m1 NNNNNN.N NNNNNN.N
 Dr_m1 NNNNNN.N NNNNNN.N
 Dw_m1 NNNNNN.N NNNNNN.N

 Ir_mm NNNNNN.N NNNNNN.N
 Dr_mm NNNNNN.N NNNNNN.N
 Dw_mm NNNNNN.N NNNNNN.N
# ===================================================================
FORMAT: compact
%%STD_HEADER%%

The numbers represent relative counts per loop iteration, compared to
p0 at 100.0%.
Higher is better: for example, using half as many instructions gives 200%,
while using twice as many gives 50%.

Results for p1

     Ir     Dr     Dw   COND    IND COND_m  IND_m  Ir_m1  Dr_m1  Dw_m1  Ir_mm  Dr_mm  Dw_mm
 ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------
 NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN  call::sub::empty
# ===================================================================
FORMAT: raw_compact
%%STD_HEADER%%

The numbers represent raw counts per loop iteration.

Results for p0

      Ir      Dr      Dw    COND     IND  COND_m   IND_m   Ir_m1   Dr_m1   Dw_m1   Ir_mm   Dr_mm   Dw_mm
  ------  ------  ------  ------  ------  ------  ------  ------  ------  ------  ------  ------  ------
 NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N  call::sub::empty
# ===================================================================
Commit	Line	Data
6568b26d DM	1	#!/usr/bin/perl
	2
	3	# Check the functionality of the Porting/bench.pl executable;
	4	# in particular, its argument handling and its ability to produce
	5	# the expected output for particular arguments.
	6	#
	7	# See also t/porting/bench_selftest.pl
	8
	9	use warnings;
	10	use strict;
	11
	12	BEGIN {
	13	chdir '..' if -f 'test.pl' && -f 'thread_it.pl';
	14	require './t/test.pl';
	15	}
	16
	17	# Only test on git checkouts - this is more of a perl core developer
	18	# tool than an end-user tool.
	19	# Only test on a platform likely to support forking, pipes, cachegrind
	20	# etc. Add other platforms if you think they're safe.
	21
	22	skip_all "not devel" unless -d ".git";
	23	skip_all "not linux" unless $^O eq 'linux';
	24	skip_all "no valgrind" unless -x '/bin/valgrind' \|\| -x '/usr/bin/valgrind';
	25
	26
	27	my $bench_pl = "Porting/bench.pl";
	28
	29	ok -e $bench_pl, "$bench_pl exists and is executable";
	30
	31	my $bench_cmd = "$^X -Ilib $bench_pl";
	32
	33	my $out;
	34
	35	# Read in the expected output format templates and create qr//s from them.
	36
	37	my %formats;
	38	my %format_qrs;
	39
	40	{
	41	my $cur;
	42	while (<DATA>) {
	43	next if /^#/;
	44	if (/^FORMAT:/) {
	45	die "invalid format line: $_" unless /^FORMAT:\s+(\w+)\s*$/;
	46	$cur = $1;
	47	die "duplicate format: '$cur'\n" if exists $formats{$cur};
	48	next;
	49	}
	50	$formats{$cur} .= $_;
	51	}
	52
	53	for my $name (sort keys %formats) {
	54	my $f = $formats{$name};
	55
	56	# expand "%%SUB_FORMAT%%
	57	$f =~ s{^ \s* %% (\w+) %% [ \t]* \n}
	58	{
	59	my $f1 = $formats{$1};
	60	die "No such sub-format '%%$1%%' in format '$name'\n"
	61	unless defined $f1;
	62	$f1;
	63	}gmxe;
	64
65	$f = quotemeta $f;
66
67	# convert NNNN.NN placeholders into a regex
68	$f =~ s{(N+)\\.(N+)}
69	{
70	"("
71	. "\\s*-?\\d+\."
72	. "\\d" x length($2)
73	."\|-)"
74	}ge;
75	$format_qrs{$name} = qr/\A$f\z/;
76	}
77	}
78
79
1137c9fa DM	80	# ---------------------------------------------------
	81	# check croaks
	82
	83	for my $test (
	84	[
	85	"--boz",
	86	"Unknown option: boz\nUse the -h option for usage information.\n",
	87	"croak: basic unknown option"
	88	],
	89	[
	90	"--fields=Ir,Boz",
	91	"Error: --fields: unknown field 'Boz'\n",
	92	"croak: unknown --field"
	93	],
	94	[
	95	"--action=boz",
	96	"Error: unrecognised action 'boz'\nmust be one of: grind, selftest\n",
	97	"croak: unknown --action"
	98	],
	99	[
	100	"--sort=boz",
	101	"Error: --sort argument should be of the form field:perl: 'boz'\n",
	102	"croak: invalid --sort"
	103	],
	104	[
	105	"--sort=boz:perl",
	106	"Error: --sort: unknown field 'boz'\n",
	107	"croak: unknown --sort field"
	108	],
	109	[
	110	"-action=selftest perl",
	111	"Error: no perl executables may be specified with selftest\n",
	112	"croak: --action-selftest with executable"
	113	],
	114	[
	115	"--tests=/boz perl",
	116	"Error: --tests regex must be of the form /.../\n",
	117	"croak: invalid --tests regex"
	118	],
	119	[
	120	"--tests=call::sub::empty,foo::bar::baz::boz perl",
	121	"Error: no such test found: 'foo::bar::baz::boz'\n",
	122	"croak: unknown test in --tests"
	123	],
	124	[
	125	"--tests=/foo::bar::baz::boz/ perl",
	126	"Error: no tests to run\n",
	127	"croak: no --tests to run "
	128	],
	129	[
	130	"--benchfile=no-such-file-boz perl",
	131	qr/\AError: can't read 'no-such-file-boz':/,
	132	"croak: non-existent --benchfile "
	133	],
	134	[
	135	"--benchfile=t/porting/bench/synerr perl",
	136	qr{\AError: can't parse 't/porting/bench/synerr':\nsyntax error},
	137	"croak: --benchfile with syntax error"
	138	],
	139	[
	140	"--benchfile=t/porting/bench/ret0 perl",
	141	"Error: can't load 't/porting/bench/ret0': code didn't return a true value\n",
	142	"croak: --benchfile which returns 0"
	143	],
144	[
145	"--norm=2 ./miniperl ./perl",
146	"Error: --norm value 2 outside range 0..1\n",
147	"croak: select-a-perl out of range"
148	],
149	[
150	"--sort=Ir:myperl ./miniperl ./perl",
151	"Error: --sort: unrecognised perl 'myperl'\n",
152	"croak: select-a-perl unrecognised"
153	],
154	[
155	"--compact=./perl ./perl=A ./perl=B",
156	"Error: --compact: ambiguous perl './perl'\n",
157	"croak: select-a-perl ambiguous"
158	],
159	[
160	" ./perl=A ./miniperl=A",
161	"A cannot be used on 2 different perls under test\n",
162	"croak: duplicate label"
163	],
164	[
165	"--grindargs=Boz --tests=call::sub::empty ./perl=A ./perl=B",
166	qr{Error: while executing call::sub::empty/A empty/short loop:\nunexpected code or cachegrind output:\n},
167	"croak: cachegrind output format "
168	],
169	[
170	"--bisect=Ir",,
171	"Error: --bisect option must be of form 'field,integer,integer'\n",
172	"croak: --bisect=Ir"
173	],
174	[
175	"--bisect=Ir,1",,
176	"Error: --bisect option must be of form 'field,integer,integer'\n",
177	"croak: --bisect=Ir,1"
178	],
179	[
180	"--bisect=Ir,1,2,3",
181	"Error: --bisect option must be of form 'field,integer,integer'\n",
182	"croak: --bisect=Ir,1,2,3"
183	],
184	[
185	"--bisect=Ir,1,x",
186	"Error: --bisect option must be of form 'field,integer,integer'\n",
187	"croak: --bisect=Ir,1,x"
188	],
189	[
190	"--bisect=Ir,x,2",
191	"Error: --bisect option must be of form 'field,integer,integer'\n",
192	"croak: --bisect=Ir,x,2"
193	],
194	[
195	"--bisect=boz,1,2",
196	"Error: unrecognised field 'boz' in --bisect option\n",
197	"croak: --bisect=boz,1,2"
198	],
199	[
200	"--bisect=Ir,2,1",
201	"Error: --bisect min (2) must be <= max (1)\n",
202	"croak: --bisect=boz,2,1"
203	],
204	[
205	"--read=no-such-file-boz",
206	qr/\AError: can't open 'no-such-file-boz' for reading:/,
207	"croak: non-existent --read file "
208	],
209	[
210	"--read=t/porting/bench/badversion.json",
211	"Error: unsupported version 9999.9 in file 't/porting/bench/badversion.json' (too new)\n",
212	"croak: --read version"
213	],
214	[
215	"",
216	"Error: nothing to do: no perls to run, no data to read.\n",
217	"croak: no input"
218	],
219	[
220	"./perl",
221	"Error: need at least 2 perls for comparison.\n",
222	"croak: need 2 perls"
223	],
224	[
225	"--bisect=Ir,1,2 ./perl=A ./perl=B",
226	"Error: exactly one perl executable must be specified for bisect\n",
227	"croak: --bisect, need 1 perls"
228	],
229	[
230	"--bisect=Ir,1,2 --tests=/call/ ./perl=A",
231	"Error: only a single test may be specified with --bisect\n",
232	"croak: --bisect one test only"
233	],
234	# note that callsub.json was created using
235	# ./perl -Ilib Porting/bench.pl --tests='/call::sub::(amp_)?empty/' \
236	# --write=t/porting/bench/callsub.json ./perl
237
238	[
239	"--read=t/porting/bench/callsub.json --write=no/such/file/boz",
240	qr{\AError: can't open 'no/such/file/boz' for writing: },
241	"croak: --write open error"
242	],
243
244	# these ones isn't tested:
245
246	# Error: can't parse '$field' field from cachegrind output
247	# Error: while starting cachegrind subprocess for NNNN:
248	)
249	{
250	my ($args, $expected, $desc) = @$test;
251	$out = qx($bench_cmd $args 2>&1);
252	if (ref($expected)) {
253	like $out, $expected, $desc;
254	}
255	else {
256	is $out, $expected, $desc;
257	}
258	}
259
260	# ---------------------------------------------------
261	# run benchmarks
262
263
6568b26d DM	264	my $resultfile1 = tempfile(); # benchmark results for 1 perl
	265	my $resultfile2 = tempfile(); # benchmark results for 2 perls
	266
	267	# Run a real cachegrind session and write results to file.
	268	# the -j 2 is to minimally exercise its parallel facility.
	269
	270	note("running cachegrind for 1st perl; may be slow...");
	271	$out = qx($bench_cmd -j 2 --write=$resultfile1 --tests=call::sub::empty $^X=p0 2>&1);
	272	is length($out), 0, "--write should produce no output (1 perl)";
	273	ok -s $resultfile1, "--write should create a non-empty results file (1 perl)";
	274
	275	# and again with 2 perls. This is also tests the 'mix read and new new
	276	# perls' functionality.
	277
	278	note("running cachegrind for 2nd perl; may be slow...");
	279	$out = qx($bench_cmd -j 2 --read=$resultfile1 --write=$resultfile2 $^X=p1 2>&1);
d9b91f79 DM	280	is length($out), 0, "--write should produce no output (2 perls)"
d9b91f79 DM	281	or diag("got: $out");
6568b26d DM	282	ok -s $resultfile2, "--write should create a non-empty results file (2 perls)";
	283
	284	# 1 perl:
	285
	286	# read back the results in raw form
	287
	288	$out = qx($bench_cmd --read=$resultfile1 --raw 2>&1);
	289	like $out, $format_qrs{raw1}, "basic cachegrind raw format; 1 perl";
	290
	291	# and read back the results in raw compact form
	292
	293	$out = qx($bench_cmd --read=$resultfile1 --raw --compact=0 2>&1);
	294	like $out, $format_qrs{raw_compact}, "basic cachegrind raw compact format; 1 perl";
	295
	296	# 2 perls:
	297
	298	# read back the results in relative-percent form
	299
	300	$out = qx($bench_cmd --read=$resultfile2 2>&1);
	301	like $out, $format_qrs{percent2}, "basic cachegrind percent format; 2 perls";
	302
	303	# and read back the results in raw form
	304
	305	$out = qx($bench_cmd --read=$resultfile2 --raw 2>&1);
	306	like $out, $format_qrs{raw2}, "basic cachegrind raw format; 2 perls";
	307
	308	# and read back the results in compact form
	309
	310	$out = qx($bench_cmd --read=$resultfile2 --compact=1 2>&1);
	311	like $out, $format_qrs{compact}, "basic cachegrind compact format; 2 perls";
	312
	313
d9b91f79 DM	314	# bisect
	315
	316	note("running cachegrind bisect on 1 perl; may be slow...");
	317
	318	# the Ir range here is intended such that the bisect will always fail
	319	$out = qx($bench_cmd --tests=call::sub::empty --bisect=Ir,100000,100001 $^X=p0 2>&1);
	320
	321	is $?, 1 << 8, "--bisect should not match";
	322	is length($out), 0, "--bisect should produce no output"
	323	or diag("got: $out");
6568b26d	324
1137c9fa	325
6568b26d DM	326	done_testing();
	327
	328
	329	# Templates for expected output formats.
	330	#
	331	# Lines starting with '#' are skipped.
	332	# Lines of the form 'FORMAT: foo' start and name a new template
	333	# All other lines are part of the template
	334	# Entries of the form NNNN.NN are converted into a regex of the form
	335	# ( \s* -? \d+\.\d\d \| - )
	336	# i.e. it expects number with a fixed number of digits after the point,
	337	# or a '-'.
	338	# Lines of the form %%FOO%% are substituted with format 'FOO'
	339
	340
	341	__END__
	342	# ===================================================================
	343	FORMAT: STD_HEADER
	344	Key:
	345	Ir Instruction read
	346	Dr Data read
	347	Dw Data write
	348	COND conditional branches
	349	IND indirect branches
	350	_m branch predict miss
	351	_m1 level 1 cache miss
	352	_mm last cache (e.g. L3) miss
	353	- indeterminate percentage (e.g. 1/0)
	354	# ===================================================================
	355	FORMAT: percent2
	356	%%STD_HEADER%%
	357
	358	The numbers represent relative counts per loop iteration, compared to
	359	p0 at 100.0%.
	360	Higher is better: for example, using half as many instructions gives 200%,
	361	while using twice as many gives 50%.
	362
	363	call::sub::empty
	364	function call with no args or body
	365
	366	p0 p1
	367	------ ------
	368	Ir 100.00 NNN.NN
	369	Dr 100.00 NNN.NN
	370	Dw 100.00 NNN.NN
	371	COND 100.00 NNN.NN
	372	IND 100.00 NNN.NN
	373
	374	COND_m 100.00 NNN.NN
	375	IND_m 100.00 NNN.NN
	376
	377	Ir_m1 100.00 NNN.NN
	378	Dr_m1 100.00 NNN.NN
	379	Dw_m1 100.00 NNN.NN
	380
	381	Ir_mm 100.00 NNN.NN
	382	Dr_mm 100.00 NNN.NN
	383	Dw_mm 100.00 NNN.NN
	384	# ===================================================================
	385	FORMAT: raw1
	386	%%STD_HEADER%%
	387
	388	The numbers represent raw counts per loop iteration.
	389
390	call::sub::empty
391	function call with no args or body
392
393	p0
394	--------
395	Ir NNNNNN.N
396	Dr NNNNNN.N
397	Dw NNNNNN.N
398	COND NNNNNN.N
399	IND NNNNNN.N
400
401	COND_m NNNNNN.N
402	IND_m NNNNNN.N
403
404	Ir_m1 NNNNNN.N
405	Dr_m1 NNNNNN.N
406	Dw_m1 NNNNNN.N
407
408	Ir_mm NNNNNN.N
409	Dr_mm NNNNNN.N
410	Dw_mm NNNNNN.N
411	# ===================================================================
412	FORMAT: raw2
413	%%STD_HEADER%%
414
415	The numbers represent raw counts per loop iteration.
416
417	call::sub::empty
418	function call with no args or body
419
420	p0 p1
421	-------- --------
422	Ir NNNNNN.N NNNNNN.N
423	Dr NNNNNN.N NNNNNN.N
424	Dw NNNNNN.N NNNNNN.N
425	COND NNNNNN.N NNNNNN.N
426	IND NNNNNN.N NNNNNN.N
427
428	COND_m NNNNNN.N NNNNNN.N
429	IND_m NNNNNN.N NNNNNN.N
430
431	Ir_m1 NNNNNN.N NNNNNN.N
432	Dr_m1 NNNNNN.N NNNNNN.N
433	Dw_m1 NNNNNN.N NNNNNN.N
434
435	Ir_mm NNNNNN.N NNNNNN.N
436	Dr_mm NNNNNN.N NNNNNN.N
437	Dw_mm NNNNNN.N NNNNNN.N
438	# ===================================================================
439	FORMAT: compact
440	%%STD_HEADER%%
441
442	The numbers represent relative counts per loop iteration, compared to
443	p0 at 100.0%.
444	Higher is better: for example, using half as many instructions gives 200%,
445	while using twice as many gives 50%.
446
447	Results for p1
448
449	Ir Dr Dw COND IND COND_m IND_m Ir_m1 Dr_m1 Dw_m1 Ir_mm Dr_mm Dw_mm
450	------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------
451	NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN call::sub::empty
452	# ===================================================================
453	FORMAT: raw_compact
454	%%STD_HEADER%%
455
456	The numbers represent raw counts per loop iteration.
457
458	Results for p0
459
460	Ir Dr Dw COND IND COND_m IND_m Ir_m1 Dr_m1 Dw_m1 Ir_mm Dr_mm Dw_mm
461	------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------
462	NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N call::sub::empty
463	# ===================================================================