Commit | Line | Data |
---|---|---|
6568b26d DM |
1 | #!/usr/bin/perl |
2 | ||
3 | # Check the functionality of the Porting/bench.pl executable; | |
4 | # in particular, its argument handling and its ability to produce | |
5 | # the expected output for particular arguments. | |
6 | # | |
7 | # See also t/porting/bench_selftest.pl | |
8 | ||
6568b26d DM |
9 | BEGIN { |
10 | chdir '..' if -f 'test.pl' && -f 'thread_it.pl'; | |
11 | require './t/test.pl'; | |
54ad4e11 | 12 | push @INC, 'lib'; |
6568b26d DM |
13 | } |
14 | ||
54ad4e11 DM |
15 | use warnings; |
16 | use strict; | |
17 | use Config; | |
18 | ||
19 | ||
6568b26d DM |
20 | # Only test on git checkouts - this is more of a perl core developer |
21 | # tool than an end-user tool. | |
22 | # Only test on a platform likely to support forking, pipes, cachegrind | |
23 | # etc. Add other platforms if you think they're safe. | |
24 | ||
25 | skip_all "not devel" unless -d ".git"; | |
26 | skip_all "not linux" unless $^O eq 'linux'; | |
27 | skip_all "no valgrind" unless -x '/bin/valgrind' || -x '/usr/bin/valgrind'; | |
54ad4e11 DM |
28 | # Address sanitizer clashes horribly with cachegrind |
29 | skip_all "not with ASAN" if $Config{ccflags} =~ /sanitize=address/; | |
6568b26d DM |
30 | |
31 | ||
32 | my $bench_pl = "Porting/bench.pl"; | |
33 | ||
34 | ok -e $bench_pl, "$bench_pl exists and is executable"; | |
35 | ||
36 | my $bench_cmd = "$^X -Ilib $bench_pl"; | |
37 | ||
1e072f25 | 38 | my ($out, $cmd); |
6568b26d DM |
39 | |
40 | # Read in the expected output format templates and create qr//s from them. | |
41 | ||
42 | my %formats; | |
43 | my %format_qrs; | |
44 | ||
45 | { | |
46 | my $cur; | |
47 | while (<DATA>) { | |
48 | next if /^#/; | |
49 | if (/^FORMAT:/) { | |
50 | die "invalid format line: $_" unless /^FORMAT:\s+(\w+)\s*$/; | |
51 | $cur = $1; | |
52 | die "duplicate format: '$cur'\n" if exists $formats{$cur}; | |
53 | next; | |
54 | } | |
55 | $formats{$cur} .= $_; | |
56 | } | |
57 | ||
58 | for my $name (sort keys %formats) { | |
59 | my $f = $formats{$name}; | |
60 | ||
61 | # expand "%%SUB_FORMAT%% | |
62 | $f =~ s{^ \s* %% (\w+) %% [ \t]* \n} | |
63 | { | |
64 | my $f1 = $formats{$1}; | |
65 | die "No such sub-format '%%$1%%' in format '$name'\n" | |
66 | unless defined $f1; | |
67 | $f1; | |
68 | }gmxe; | |
69 | ||
70 | $f = quotemeta $f; | |
71 | ||
72 | # convert NNNN.NN placeholders into a regex | |
73 | $f =~ s{(N+)\\.(N+)} | |
74 | { | |
1e072f25 | 75 | my $l = length($2); |
6568b26d | 76 | "(" |
1e072f25 DM |
77 | . "\\s*-?\\d+\\." |
78 | . "\\d" x $l | |
79 | ."|\\s{" | |
80 | . ($l + 1) | |
81 | . ",}-)" | |
6568b26d DM |
82 | }ge; |
83 | $format_qrs{$name} = qr/\A$f\z/; | |
84 | } | |
85 | } | |
86 | ||
87 | ||
1137c9fa DM |
88 | # --------------------------------------------------- |
89 | # check croaks | |
90 | ||
91 | for my $test ( | |
92 | [ | |
93 | "--boz", | |
94 | "Unknown option: boz\nUse the -h option for usage information.\n", | |
95 | "croak: basic unknown option" | |
96 | ], | |
97 | [ | |
98 | "--fields=Ir,Boz", | |
99 | "Error: --fields: unknown field 'Boz'\n", | |
100 | "croak: unknown --field" | |
101 | ], | |
102 | [ | |
103 | "--action=boz", | |
104 | "Error: unrecognised action 'boz'\nmust be one of: grind, selftest\n", | |
105 | "croak: unknown --action" | |
106 | ], | |
107 | [ | |
108 | "--sort=boz", | |
109 | "Error: --sort argument should be of the form field:perl: 'boz'\n", | |
110 | "croak: invalid --sort" | |
111 | ], | |
112 | [ | |
113 | "--sort=boz:perl", | |
114 | "Error: --sort: unknown field 'boz'\n", | |
115 | "croak: unknown --sort field" | |
116 | ], | |
117 | [ | |
118 | "-action=selftest perl", | |
119 | "Error: no perl executables may be specified with selftest\n", | |
120 | "croak: --action-selftest with executable" | |
121 | ], | |
122 | [ | |
123 | "--tests=/boz perl", | |
124 | "Error: --tests regex must be of the form /.../\n", | |
125 | "croak: invalid --tests regex" | |
126 | ], | |
127 | [ | |
128 | "--tests=call::sub::empty,foo::bar::baz::boz perl", | |
e89a8e10 DM |
129 | "Error: no such test found: 'foo::bar::baz::boz'\n" |
130 | . "Re-run with --verbose for a list of valid tests.\n", | |
1137c9fa DM |
131 | "croak: unknown test in --tests" |
132 | ], | |
133 | [ | |
e89a8e10 DM |
134 | "--verbose --tests=call::sub::empty,foo::bar::baz::boz --read=t/porting/bench/callsub.json", |
135 | "Error: no such test found: 'foo::bar::baz::boz'\n" | |
136 | . "Valid test names are:\n" | |
137 | . " call::sub::amp_empty\n" | |
138 | . " call::sub::empty\n", | |
139 | "croak: unknown test in --tests --verbose" | |
140 | ], | |
141 | [ | |
1137c9fa DM |
142 | "--tests=/foo::bar::baz::boz/ perl", |
143 | "Error: no tests to run\n", | |
144 | "croak: no --tests to run " | |
145 | ], | |
146 | [ | |
147 | "--benchfile=no-such-file-boz perl", | |
148 | qr/\AError: can't read 'no-such-file-boz':/, | |
149 | "croak: non-existent --benchfile " | |
150 | ], | |
151 | [ | |
152 | "--benchfile=t/porting/bench/synerr perl", | |
153 | qr{\AError: can't parse 't/porting/bench/synerr':\nsyntax error}, | |
154 | "croak: --benchfile with syntax error" | |
155 | ], | |
156 | [ | |
157 | "--benchfile=t/porting/bench/ret0 perl", | |
158 | "Error: can't load 't/porting/bench/ret0': code didn't return a true value\n", | |
159 | "croak: --benchfile which returns 0" | |
160 | ], | |
161 | [ | |
162 | "--norm=2 ./miniperl ./perl", | |
163 | "Error: --norm value 2 outside range 0..1\n", | |
164 | "croak: select-a-perl out of range" | |
165 | ], | |
166 | [ | |
167 | "--sort=Ir:myperl ./miniperl ./perl", | |
168 | "Error: --sort: unrecognised perl 'myperl'\n", | |
169 | "croak: select-a-perl unrecognised" | |
170 | ], | |
171 | [ | |
172 | "--compact=./perl ./perl=A ./perl=B", | |
173 | "Error: --compact: ambiguous perl './perl'\n", | |
174 | "croak: select-a-perl ambiguous" | |
175 | ], | |
176 | [ | |
99b1e78b DM |
177 | "./perl --foo", |
178 | "Error: unrecognised executable switch '--foo'\n", | |
179 | "croak: ./perl --foo" | |
180 | ], | |
181 | [ | |
182 | "-- --args=foo", | |
183 | "Error: --args without a preceding executable name\n", | |
184 | "croak: --args without perl" | |
185 | ], | |
186 | [ | |
187 | "-- --env=foo=bar", | |
188 | "Error: --env without a preceding executable name\n", | |
189 | "croak: --env without perl" | |
190 | ], | |
191 | [ | |
192 | "./perl --args", | |
193 | "Error: --args is missing value\n", | |
194 | "croak: --args without value" | |
195 | ], | |
196 | [ | |
197 | "./perl --env", | |
198 | "Error: --env is missing value\n", | |
199 | "croak: --env without value" | |
200 | ], | |
201 | [ | |
202 | "./perl --env='FOO'", | |
203 | "Error: --env is missing =value\n", | |
204 | "croak: --env without =value" | |
205 | ], | |
206 | [ | |
207 | "./perl ./perl", | |
81cb9d79 DM |
208 | "Error: duplicate label './perl': each executable must have a unique label\n", |
209 | "croak: duplicate label ./perl ./perl" | |
210 | ], | |
211 | [ | |
99b1e78b | 212 | "./perl=A ./miniperl=A", |
81cb9d79 DM |
213 | "Error: duplicate label 'A': each executable must have a unique label\n", |
214 | "croak: duplicate label =A =A" | |
215 | ], | |
216 | [ | |
217 | "--read=t/porting/bench/callsub.json --read=t/porting/bench/callsub.json", | |
218 | "Error: duplicate label './perl': seen in file 't/porting/bench/callsub.json'\n", | |
219 | "croak: duplicate label --read=... --read=..." | |
220 | ], | |
221 | [ | |
222 | "--read=t/porting/bench/callsub.json ./perl", | |
223 | "Error: duplicate label './perl': seen both in --read file and on command line\n", | |
224 | "croak: duplicate label --read=... ./perl" | |
1137c9fa DM |
225 | ], |
226 | [ | |
99b1e78b DM |
227 | "./nosuch-perl", |
228 | qr{^\QError: unable to execute './nosuch-perl': }, | |
229 | "croak: no such perl" | |
230 | ], | |
231 | [ | |
132771f7 | 232 | "--grindargs=Boz --debug --tests=call::sub::empty ./perl=A ./perl=B", |
1137c9fa DM |
233 | qr{Error: while executing call::sub::empty/A empty/short loop:\nunexpected code or cachegrind output:\n}, |
234 | "croak: cachegrind output format " | |
235 | ], | |
236 | [ | |
237 | "--bisect=Ir",, | |
238 | "Error: --bisect option must be of form 'field,integer,integer'\n", | |
239 | "croak: --bisect=Ir" | |
240 | ], | |
241 | [ | |
242 | "--bisect=Ir,1",, | |
243 | "Error: --bisect option must be of form 'field,integer,integer'\n", | |
244 | "croak: --bisect=Ir,1" | |
245 | ], | |
246 | [ | |
247 | "--bisect=Ir,1,2,3", | |
248 | "Error: --bisect option must be of form 'field,integer,integer'\n", | |
249 | "croak: --bisect=Ir,1,2,3" | |
250 | ], | |
251 | [ | |
252 | "--bisect=Ir,1,x", | |
253 | "Error: --bisect option must be of form 'field,integer,integer'\n", | |
254 | "croak: --bisect=Ir,1,x" | |
255 | ], | |
256 | [ | |
257 | "--bisect=Ir,x,2", | |
258 | "Error: --bisect option must be of form 'field,integer,integer'\n", | |
259 | "croak: --bisect=Ir,x,2" | |
260 | ], | |
261 | [ | |
262 | "--bisect=boz,1,2", | |
263 | "Error: unrecognised field 'boz' in --bisect option\n", | |
264 | "croak: --bisect=boz,1,2" | |
265 | ], | |
266 | [ | |
267 | "--bisect=Ir,2,1", | |
268 | "Error: --bisect min (2) must be <= max (1)\n", | |
269 | "croak: --bisect=boz,2,1" | |
270 | ], | |
271 | [ | |
272 | "--read=no-such-file-boz", | |
273 | qr/\AError: can't open 'no-such-file-boz' for reading:/, | |
274 | "croak: non-existent --read file " | |
275 | ], | |
276 | [ | |
277 | "--read=t/porting/bench/badversion.json", | |
278 | "Error: unsupported version 9999.9 in file 't/porting/bench/badversion.json' (too new)\n", | |
279 | "croak: --read version" | |
280 | ], | |
4533e88f DM |
281 | [ |
282 | "--read=t/porting/bench/callsub.json --benchfile=t/perf/benchmarks ./perl ", | |
283 | "Error: --benchfile cannot be used when --read is present\n", | |
284 | "croak: benchfile with read" | |
285 | ], | |
1137c9fa DM |
286 | [ |
287 | "", | |
288 | "Error: nothing to do: no perls to run, no data to read.\n", | |
289 | "croak: no input" | |
290 | ], | |
291 | [ | |
292 | "./perl", | |
293 | "Error: need at least 2 perls for comparison.\n", | |
294 | "croak: need 2 perls" | |
295 | ], | |
296 | [ | |
297 | "--bisect=Ir,1,2 ./perl=A ./perl=B", | |
298 | "Error: exactly one perl executable must be specified for bisect\n", | |
299 | "croak: --bisect, need 1 perls" | |
300 | ], | |
301 | [ | |
302 | "--bisect=Ir,1,2 --tests=/call/ ./perl=A", | |
303 | "Error: only a single test may be specified with --bisect\n", | |
304 | "croak: --bisect one test only" | |
305 | ], | |
306 | # note that callsub.json was created using | |
307 | # ./perl -Ilib Porting/bench.pl --tests='/call::sub::(amp_)?empty/' \ | |
308 | # --write=t/porting/bench/callsub.json ./perl | |
1137c9fa DM |
309 | [ |
310 | "--read=t/porting/bench/callsub.json --write=no/such/file/boz", | |
311 | qr{\AError: can't open 'no/such/file/boz' for writing: }, | |
312 | "croak: --write open error" | |
313 | ], | |
68de41bc DM |
314 | # note that callsub2.json was created using |
315 | # ./perl -Ilib Porting/bench.pl \ | |
316 | # --tests='call::sub::empty,call::sub::args3' \ | |
81cb9d79 | 317 | # --write=t/porting/bench/callsub2.json ./perl=perl2 |
68de41bc DM |
318 | [ |
319 | "--read=t/porting/bench/callsub.json " | |
320 | . " --read=t/porting/bench/callsub2.json", | |
321 | "Can't merge multiple read files: they contain differing test sets.\n" | |
322 | . "Re-run with --verbose to see the differences.\n", | |
323 | "croak: --read callsub, callsub2" | |
324 | ], | |
325 | [ | |
326 | "--read=t/porting/bench/callsub.json " | |
327 | . " --read=t/porting/bench/callsub2.json" | |
328 | . " --verbose", | |
329 | "Can't merge multiple read files: they contain differing test sets.\n" | |
330 | . "Previous tests:\n" | |
331 | . " call::sub::amp_empty\n" | |
332 | . " call::sub::empty\n" | |
333 | . "tests from 't/porting/bench/callsub2.json':\n" | |
334 | . " call::sub::args3\n" | |
335 | . " call::sub::empty\n", | |
336 | "croak: --read callsub, callsub2 --verbose" | |
337 | ], | |
1137c9fa | 338 | |
68de41bc | 339 | # these ones aren't tested (and nor are any "Panic:" ones): |
1137c9fa DM |
340 | |
341 | # Error: can't parse '$field' field from cachegrind output | |
342 | # Error: while starting cachegrind subprocess for NNNN: | |
68de41bc DM |
343 | # File '$file' contains no results |
344 | # File '$file' contains differing test and results names | |
345 | # File '$file' contains differing test and sort order names | |
346 | # Can't merge multiple read files: differing loop counts: | |
1137c9fa DM |
347 | ) |
348 | { | |
349 | my ($args, $expected, $desc) = @$test; | |
350 | $out = qx($bench_cmd $args 2>&1); | |
351 | if (ref($expected)) { | |
352 | like $out, $expected, $desc; | |
353 | } | |
354 | else { | |
355 | is $out, $expected, $desc; | |
356 | } | |
357 | } | |
358 | ||
359 | # --------------------------------------------------- | |
360 | # run benchmarks | |
361 | ||
362 | ||
6568b26d DM |
363 | my $resultfile1 = tempfile(); # benchmark results for 1 perl |
364 | my $resultfile2 = tempfile(); # benchmark results for 2 perls | |
365 | ||
366 | # Run a real cachegrind session and write results to file. | |
367 | # the -j 2 is to minimally exercise its parallel facility. | |
368 | ||
369 | note("running cachegrind for 1st perl; may be slow..."); | |
370 | $out = qx($bench_cmd -j 2 --write=$resultfile1 --tests=call::sub::empty $^X=p0 2>&1); | |
54ad4e11 | 371 | is $out, "", "--write should produce no output (1 perl)"; |
6568b26d DM |
372 | ok -s $resultfile1, "--write should create a non-empty results file (1 perl)"; |
373 | ||
374 | # and again with 2 perls. This is also tests the 'mix read and new new | |
375 | # perls' functionality. | |
376 | ||
377 | note("running cachegrind for 2nd perl; may be slow..."); | |
378 | $out = qx($bench_cmd -j 2 --read=$resultfile1 --write=$resultfile2 $^X=p1 2>&1); | |
54ad4e11 | 379 | is $out, "", "--write should produce no output (2 perls)" |
d9b91f79 | 380 | or diag("got: $out"); |
6568b26d DM |
381 | ok -s $resultfile2, "--write should create a non-empty results file (2 perls)"; |
382 | ||
383 | # 1 perl: | |
384 | ||
385 | # read back the results in raw form | |
386 | ||
387 | $out = qx($bench_cmd --read=$resultfile1 --raw 2>&1); | |
388 | like $out, $format_qrs{raw1}, "basic cachegrind raw format; 1 perl"; | |
389 | ||
390 | # and read back the results in raw compact form | |
391 | ||
392 | $out = qx($bench_cmd --read=$resultfile1 --raw --compact=0 2>&1); | |
393 | like $out, $format_qrs{raw_compact}, "basic cachegrind raw compact format; 1 perl"; | |
394 | ||
16dbd072 DM |
395 | # and read back the results in raw average form |
396 | ||
397 | $out = qx($bench_cmd --read=$resultfile1 --raw --average 2>&1); | |
398 | like $out, $format_qrs{raw_average1}, "basic cachegrind raw average format; 1 perl"; | |
399 | ||
400 | # and read back the results with raw selected fields | |
401 | ||
402 | $out = qx($bench_cmd --read=$resultfile1 --raw --fields=Ir,Dr 2>&1); | |
403 | like $out, $format_qrs{fields1}, "basic cachegrind --fields; 1 perl"; | |
404 | ||
6568b26d DM |
405 | # 2 perls: |
406 | ||
407 | # read back the results in relative-percent form | |
408 | ||
409 | $out = qx($bench_cmd --read=$resultfile2 2>&1); | |
410 | like $out, $format_qrs{percent2}, "basic cachegrind percent format; 2 perls"; | |
411 | ||
16dbd072 DM |
412 | # read back the results in relative-percent form with norm |
413 | ||
414 | $out = qx($bench_cmd --read=$resultfile2 --norm=0 2>&1); | |
415 | like $out, $format_qrs{percent2}, "basic cachegrind percent format, norm; 2 perls"; | |
416 | ||
417 | # read back the results in relative-percent form with sort | |
418 | ||
419 | $out = qx($bench_cmd --read=$resultfile2 --sort=Ir:0 2>&1); | |
420 | like $out, $format_qrs{percent2}, "basic cachegrind percent format, sort; 2 perls"; | |
421 | ||
422 | # read back the results in relative-percent form with sort and norm | |
423 | ||
424 | $out = qx($bench_cmd --read=$resultfile2 --sort=Ir:0 --norm=0 2>&1); | |
425 | like $out, $format_qrs{percent2}, "basic cachegrind percent format, sort, norm; 2 perls"; | |
426 | ||
6568b26d DM |
427 | # and read back the results in raw form |
428 | ||
429 | $out = qx($bench_cmd --read=$resultfile2 --raw 2>&1); | |
430 | like $out, $format_qrs{raw2}, "basic cachegrind raw format; 2 perls"; | |
431 | ||
16dbd072 DM |
432 | # and read back the results in raw form with norm |
433 | ||
434 | $out = qx($bench_cmd --read=$resultfile2 --raw --norm=0 2>&1); | |
435 | like $out, $format_qrs{raw2}, "basic cachegrind raw format, norm; 2 perls"; | |
436 | ||
437 | # and read back the results in raw form with sort | |
438 | ||
439 | $out = qx($bench_cmd --read=$resultfile2 --raw --sort=Ir:0 2>&1); | |
440 | like $out, $format_qrs{raw2}, "basic cachegrind raw format, sort, norm; 2 perls"; | |
441 | ||
442 | # and read back the results in raw form with sort and norm | |
443 | ||
444 | $out = qx($bench_cmd --read=$resultfile2 --raw --sort=Ir:0 --norm=0 2>&1); | |
445 | like $out, $format_qrs{raw2}, "basic cachegrind raw format, sort, norm; 2 perls"; | |
446 | ||
6568b26d DM |
447 | # and read back the results in compact form |
448 | ||
449 | $out = qx($bench_cmd --read=$resultfile2 --compact=1 2>&1); | |
450 | like $out, $format_qrs{compact}, "basic cachegrind compact format; 2 perls"; | |
451 | ||
16dbd072 DM |
452 | # and read back the results in average form |
453 | ||
454 | $out = qx($bench_cmd --read=$resultfile2 --average 2>&1); | |
455 | like $out, $format_qrs{average}, "basic cachegrind average format; 2 perls"; | |
456 | ||
457 | # and read back the results with selected fields | |
458 | ||
459 | $out = qx($bench_cmd --read=$resultfile2 --fields=Ir,Dr 2>&1); | |
460 | like $out, $format_qrs{fields2}, "basic cachegrind --fields; 2 perls"; | |
b9248ec1 | 461 | |
16dbd072 DM |
462 | # and read back the results in compact form with selected fields |
463 | ||
464 | $out = qx($bench_cmd --read=$resultfile2 --compact=1 --fields=Ir,Dr 2>&1); | |
465 | like $out, $format_qrs{compact_fields}, "basic cachegrind compact, fields; 2 perls"; | |
466 | ||
b9248ec1 DM |
467 | # and read back the results with 1 selected fields (this is more compact) |
468 | ||
469 | $out = qx($bench_cmd --read=$resultfile2 --fields=Ir 2>&1); | |
470 | like $out, $format_qrs{'1field'}, "basic cachegrind 1 field; 2 perls"; | |
471 | ||
6568b26d | 472 | |
d9b91f79 DM |
473 | # bisect |
474 | ||
d9b91f79 | 475 | # the Ir range here is intended such that the bisect will always fail |
30792d9e | 476 | $out = qx($bench_cmd --read=t/porting/bench/callsub.json --tests=call::sub::empty --bisect=Ir,100000,100001 2>&1); |
d9b91f79 | 477 | |
a387d7f0 DM |
478 | is $?, 1 << 8, "--bisect: exit result: should not match"; |
479 | like $out, qr/^Bisect: Ir had the value -?\d+\n/, | |
480 | "--bisect: got expected output"; | |
6568b26d | 481 | |
68de41bc DM |
482 | # multiple reads with differing test sets but common --tests subset |
483 | ||
484 | $out = qx($bench_cmd --read=t/porting/bench/callsub.json --read=t/porting/bench/callsub2.json --tests=call::sub::empty 2>&1); | |
81cb9d79 | 485 | $out =~ s{\Q./perl perl2}{ p0 p1}; |
68de41bc | 486 | $out =~ s{^\./perl}{p0}m; |
68de41bc DM |
487 | like $out, $format_qrs{percent2}, "2 reads; overlapping test sets"; |
488 | ||
489 | # A read defines what benchmarks to run | |
490 | ||
491 | note("running cachegrind on 1 perl; may be slow..."); | |
492 | $out = qx($bench_cmd --read=t/porting/bench/callsub.json --tests=call::sub::empty $^X=p1 2>&1); | |
493 | $out =~ s{^\./perl}{p0}m; | |
494 | $out =~ s{\Q./perl}{ p0}; | |
495 | like $out, $format_qrs{percent2}, "1 read; 1 generate"; | |
496 | ||
99b1e78b DM |
497 | # Process environment and optional args. |
498 | # This is a minimal test that it runs - it doesn't test whether | |
499 | # the environment and args are getting applied correctly, apart from the | |
500 | # fact that the perls in question are being successfully executed. | |
1e072f25 DM |
501 | # |
502 | # Also check the --autolabel feature | |
99b1e78b DM |
503 | |
504 | note("running cachegrind on 2 perls; may be slow..."); | |
1e072f25 DM |
505 | $cmd = <<EOF; |
506 | $bench_cmd | |
507 | --read=t/porting/bench/callsub.json | |
508 | --read=t/porting/bench/callsub2.json | |
509 | --tests=call::sub::empty | |
510 | --autolabel | |
511 | --perlargs=-Ilib | |
512 | $^X --args='-Ifoo/bar -Mstrict' --env='FOO=foo' | |
513 | $^X --args='-Ifoo/bar' --env='BAR=bar' --env='BAZ=baz' | |
514 | 2>&1 | |
515 | EOF | |
516 | $cmd =~ s/\n\s+/ /g; | |
517 | $out = qx($cmd); | |
518 | $out =~ s{^\./perl}{p0}m; | |
519 | $out =~ s{\Q ./perl perl2 0 1} | |
520 | { p0 p1 p2 p3}; | |
521 | like $out, $format_qrs{percent4}, "4 perls with autolabel and args and env"; | |
68de41bc | 522 | |
1137c9fa | 523 | |
6568b26d DM |
524 | done_testing(); |
525 | ||
526 | ||
527 | # Templates for expected output formats. | |
528 | # | |
529 | # Lines starting with '#' are skipped. | |
530 | # Lines of the form 'FORMAT: foo' start and name a new template | |
531 | # All other lines are part of the template | |
532 | # Entries of the form NNNN.NN are converted into a regex of the form | |
533 | # ( \s* -? \d+\.\d\d | - ) | |
534 | # i.e. it expects number with a fixed number of digits after the point, | |
535 | # or a '-'. | |
536 | # Lines of the form %%FOO%% are substituted with format 'FOO' | |
537 | ||
538 | ||
539 | __END__ | |
540 | # =================================================================== | |
541 | FORMAT: STD_HEADER | |
542 | Key: | |
543 | Ir Instruction read | |
544 | Dr Data read | |
545 | Dw Data write | |
546 | COND conditional branches | |
547 | IND indirect branches | |
548 | _m branch predict miss | |
549 | _m1 level 1 cache miss | |
550 | _mm last cache (e.g. L3) miss | |
551 | - indeterminate percentage (e.g. 1/0) | |
552 | # =================================================================== | |
553 | FORMAT: percent2 | |
554 | %%STD_HEADER%% | |
555 | ||
556 | The numbers represent relative counts per loop iteration, compared to | |
557 | p0 at 100.0%. | |
558 | Higher is better: for example, using half as many instructions gives 200%, | |
559 | while using twice as many gives 50%. | |
560 | ||
561 | call::sub::empty | |
562 | function call with no args or body | |
563 | ||
564 | p0 p1 | |
565 | ------ ------ | |
566 | Ir 100.00 NNN.NN | |
567 | Dr 100.00 NNN.NN | |
568 | Dw 100.00 NNN.NN | |
569 | COND 100.00 NNN.NN | |
570 | IND 100.00 NNN.NN | |
571 | ||
572 | COND_m 100.00 NNN.NN | |
573 | IND_m 100.00 NNN.NN | |
574 | ||
575 | Ir_m1 100.00 NNN.NN | |
576 | Dr_m1 100.00 NNN.NN | |
577 | Dw_m1 100.00 NNN.NN | |
578 | ||
579 | Ir_mm 100.00 NNN.NN | |
580 | Dr_mm 100.00 NNN.NN | |
581 | Dw_mm 100.00 NNN.NN | |
582 | # =================================================================== | |
1e072f25 DM |
583 | FORMAT: percent4 |
584 | %%STD_HEADER%% | |
585 | ||
586 | The numbers represent relative counts per loop iteration, compared to | |
587 | p0 at 100.0%. | |
588 | Higher is better: for example, using half as many instructions gives 200%, | |
589 | while using twice as many gives 50%. | |
590 | ||
591 | call::sub::empty | |
592 | function call with no args or body | |
593 | ||
594 | p0 p1 p2 p3 | |
595 | ------ ------ ------ ------ | |
596 | Ir 100.00 NNN.NN NNN.NN NNN.NN | |
597 | Dr 100.00 NNN.NN NNN.NN NNN.NN | |
598 | Dw 100.00 NNN.NN NNN.NN NNN.NN | |
599 | COND 100.00 NNN.NN NNN.NN NNN.NN | |
600 | IND 100.00 NNN.NN NNN.NN NNN.NN | |
601 | ||
602 | COND_m 100.00 NNN.NN NNN.NN NNN.NN | |
603 | IND_m 100.00 NNN.NN NNN.NN NNN.NN | |
604 | ||
605 | Ir_m1 100.00 NNN.NN NNN.NN NNN.NN | |
606 | Dr_m1 100.00 NNN.NN NNN.NN NNN.NN | |
607 | Dw_m1 100.00 NNN.NN NNN.NN NNN.NN | |
608 | ||
609 | Ir_mm 100.00 NNN.NN NNN.NN NNN.NN | |
610 | Dr_mm 100.00 NNN.NN NNN.NN NNN.NN | |
611 | Dw_mm 100.00 NNN.NN NNN.NN NNN.NN | |
612 | # =================================================================== | |
16dbd072 DM |
613 | FORMAT: fields2 |
614 | %%STD_HEADER%% | |
615 | ||
616 | The numbers represent relative counts per loop iteration, compared to | |
617 | p0 at 100.0%. | |
618 | Higher is better: for example, using half as many instructions gives 200%, | |
619 | while using twice as many gives 50%. | |
620 | ||
621 | call::sub::empty | |
622 | function call with no args or body | |
623 | ||
624 | p0 p1 | |
625 | ------ ------ | |
626 | Ir 100.00 NNN.NN | |
627 | Dr 100.00 NNN.NN | |
628 | # =================================================================== | |
6568b26d DM |
629 | FORMAT: raw1 |
630 | %%STD_HEADER%% | |
631 | ||
632 | The numbers represent raw counts per loop iteration. | |
633 | ||
634 | call::sub::empty | |
635 | function call with no args or body | |
636 | ||
637 | p0 | |
638 | -------- | |
639 | Ir NNNNNN.N | |
640 | Dr NNNNNN.N | |
641 | Dw NNNNNN.N | |
642 | COND NNNNNN.N | |
643 | IND NNNNNN.N | |
644 | ||
645 | COND_m NNNNNN.N | |
646 | IND_m NNNNNN.N | |
647 | ||
648 | Ir_m1 NNNNNN.N | |
649 | Dr_m1 NNNNNN.N | |
650 | Dw_m1 NNNNNN.N | |
651 | ||
652 | Ir_mm NNNNNN.N | |
653 | Dr_mm NNNNNN.N | |
654 | Dw_mm NNNNNN.N | |
655 | # =================================================================== | |
16dbd072 DM |
656 | FORMAT: raw_average1 |
657 | %%STD_HEADER%% | |
658 | ||
659 | The numbers represent raw counts per loop iteration. | |
660 | ||
661 | AVERAGE | |
662 | ||
663 | p0 | |
664 | -------- | |
665 | Ir NNNNNN.N | |
666 | Dr NNNNNN.N | |
667 | Dw NNNNNN.N | |
668 | COND NNNNNN.N | |
669 | IND NNNNNN.N | |
670 | ||
671 | COND_m NNNNNN.N | |
672 | IND_m NNNNNN.N | |
673 | ||
674 | Ir_m1 NNNNNN.N | |
675 | Dr_m1 NNNNNN.N | |
676 | Dw_m1 NNNNNN.N | |
677 | ||
678 | Ir_mm NNNNNN.N | |
679 | Dr_mm NNNNNN.N | |
680 | Dw_mm NNNNNN.N | |
681 | # =================================================================== | |
682 | FORMAT: fields1 | |
683 | %%STD_HEADER%% | |
684 | ||
685 | The numbers represent raw counts per loop iteration. | |
686 | ||
687 | call::sub::empty | |
688 | function call with no args or body | |
689 | ||
690 | p0 | |
691 | -------- | |
692 | Ir NNNNNN.N | |
693 | Dr NNNNNN.N | |
694 | # =================================================================== | |
6568b26d DM |
695 | FORMAT: raw2 |
696 | %%STD_HEADER%% | |
697 | ||
698 | The numbers represent raw counts per loop iteration. | |
699 | ||
700 | call::sub::empty | |
701 | function call with no args or body | |
702 | ||
703 | p0 p1 | |
704 | -------- -------- | |
705 | Ir NNNNNN.N NNNNNN.N | |
706 | Dr NNNNNN.N NNNNNN.N | |
707 | Dw NNNNNN.N NNNNNN.N | |
708 | COND NNNNNN.N NNNNNN.N | |
709 | IND NNNNNN.N NNNNNN.N | |
710 | ||
711 | COND_m NNNNNN.N NNNNNN.N | |
712 | IND_m NNNNNN.N NNNNNN.N | |
713 | ||
714 | Ir_m1 NNNNNN.N NNNNNN.N | |
715 | Dr_m1 NNNNNN.N NNNNNN.N | |
716 | Dw_m1 NNNNNN.N NNNNNN.N | |
717 | ||
718 | Ir_mm NNNNNN.N NNNNNN.N | |
719 | Dr_mm NNNNNN.N NNNNNN.N | |
720 | Dw_mm NNNNNN.N NNNNNN.N | |
721 | # =================================================================== | |
722 | FORMAT: compact | |
723 | %%STD_HEADER%% | |
724 | ||
725 | The numbers represent relative counts per loop iteration, compared to | |
726 | p0 at 100.0%. | |
727 | Higher is better: for example, using half as many instructions gives 200%, | |
728 | while using twice as many gives 50%. | |
729 | ||
730 | Results for p1 | |
731 | ||
732 | Ir Dr Dw COND IND COND_m IND_m Ir_m1 Dr_m1 Dw_m1 Ir_mm Dr_mm Dw_mm | |
733 | ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ | |
734 | NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN NNN.NN call::sub::empty | |
735 | # =================================================================== | |
16dbd072 DM |
736 | FORMAT: compact_fields |
737 | %%STD_HEADER%% | |
738 | ||
739 | The numbers represent relative counts per loop iteration, compared to | |
740 | p0 at 100.0%. | |
741 | Higher is better: for example, using half as many instructions gives 200%, | |
742 | while using twice as many gives 50%. | |
743 | ||
744 | Results for p1 | |
745 | ||
746 | Ir Dr | |
747 | ------ ------ | |
748 | NNN.NN NNN.NN call::sub::empty | |
749 | # =================================================================== | |
b9248ec1 DM |
750 | FORMAT: 1field |
751 | %%STD_HEADER%% | |
752 | ||
753 | The numbers represent relative counts per loop iteration, compared to | |
754 | p0 at 100.0%. | |
755 | Higher is better: for example, using half as many instructions gives 200%, | |
756 | while using twice as many gives 50%. | |
757 | ||
758 | Results for field Ir. | |
759 | ||
760 | p0 p1 | |
761 | ------ ------ | |
762 | call::sub::empty NNN.NN NNN.NN | |
763 | # =================================================================== | |
16dbd072 DM |
764 | FORMAT: average |
765 | %%STD_HEADER%% | |
766 | ||
767 | The numbers represent relative counts per loop iteration, compared to | |
768 | p0 at 100.0%. | |
769 | Higher is better: for example, using half as many instructions gives 200%, | |
770 | while using twice as many gives 50%. | |
771 | ||
772 | AVERAGE | |
773 | ||
774 | p0 p1 | |
775 | ------ ------ | |
776 | Ir 100.00 NNN.NN | |
777 | Dr 100.00 NNN.NN | |
778 | Dw 100.00 NNN.NN | |
779 | COND 100.00 NNN.NN | |
780 | IND 100.00 NNN.NN | |
781 | ||
782 | COND_m 100.00 NNN.NN | |
783 | IND_m 100.00 NNN.NN | |
784 | ||
785 | Ir_m1 100.00 NNN.NN | |
786 | Dr_m1 100.00 NNN.NN | |
787 | Dw_m1 100.00 NNN.NN | |
788 | ||
789 | Ir_mm 100.00 NNN.NN | |
790 | Dr_mm 100.00 NNN.NN | |
791 | Dw_mm 100.00 NNN.NN | |
792 | # =================================================================== | |
6568b26d DM |
793 | FORMAT: raw_compact |
794 | %%STD_HEADER%% | |
795 | ||
796 | The numbers represent raw counts per loop iteration. | |
797 | ||
798 | Results for p0 | |
799 | ||
800 | Ir Dr Dw COND IND COND_m IND_m Ir_m1 Dr_m1 Dw_m1 Ir_mm Dr_mm Dw_mm | |
801 | ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ | |
802 | NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N NNNNN.N call::sub::empty | |
803 | # =================================================================== |