[perl5.git] / regen_perly.pl

#!/usr/bin/perl
#
# regen_perly.pl, DAPM 12-Feb-04
#
# Copyright (c) 2004, 2005 Larry Wall
#
# Given an input file perly.y, run bison on it and produce
# the following output files:
#
# perly.h	standard bison header file with minor doctoring of
#		#line directives plus adding a #ifdef PERL_CORE
#
# perly.tab	the parser table C definitions extracted from the bison output
#		plus an extra table generated by this script.
#
# perly.act	the action case statements extracted from the bison output
#
# Note that perly.c is *not* regenerated - this is now a static file which
# is not dependent on perly.y any more.
#
# If a filename of the form foo.y is given on the command line, then
# this is used instead as the basename for all the files mentioned
# above.
#
# Note that temporary files of the form perlytmp.h and perlytmp.c are
# created and then deleted during this process
#
# Note also that this script is intended to be run on a UNIX system;
# it may work elsewhere but no specific attempt has been made to make it
# portable.

sub usage { die "usage: $0 [ -b bison_executable ] [ file.y ]\n" }

use warnings;
use strict;

my $bison = 'bison';

if (@ARGV >= 2 and $ARGV[0] eq '-b') {
    shift;
    $bison = shift;
}

my $y_file = shift || 'perly.y';

usage unless @ARGV==0 && $y_file =~ /\.y$/;

(my $h_file    = $y_file) =~ s/\.y$/.h/;
(my $act_file  = $y_file) =~ s/\.y$/.act/;
(my $tab_file  = $y_file) =~ s/\.y$/.tab/;
(my $tmpc_file = $y_file) =~ s/\.y$/tmp.c/;
(my $tmph_file = $y_file) =~ s/\.y$/tmp.h/;

# the yytranslate[] table generated by bison is ASCII/EBCDIC sensitive

die "$0: must be run on an ASCII system\n" unless ord 'A' == 65;

# check for correct version number. The constraints are:
#  * must be >= 1.24 to avoid licensing issues.
#  * it must generate the yystos[] table. Version 1.28 doesn't generate
#    this; 1.35+ does
#  * Must produce output which is extractable by the regexes below
#  * Must produce the right values.
# These last two contstraints  may well be met by earlier versions, but
# I simply haven't tested them yet. If it works for you, then modify
# the test below to allow that version too. DAPM Feb 04.

my $version = `$bison -V`;
unless ($version) { die <<EOF; }
Could not find a version of bison in your path. Please install bison.
EOF

unless ($version =~ /\b(1\.875[a-z]?|2\.[0134])\b/) { die <<EOF; }

You have the wrong version of bison in your path; currently 1.875
2.0, 2.1, 2.3 or 2.4 is required.  Try installing
    http://ftp.gnu.org/gnu/bison/bison-2.4.1.tar.gz
or similar.  Your bison identifies itself as:

$version
EOF

# creates $tmpc_file and $tmph_file
my_system("$bison -d -o $tmpc_file $y_file");

open CTMPFILE, $tmpc_file or die "Can't open $tmpc_file: $!\n";
my $clines;
{ local $/; $clines = <CTMPFILE>; }
die "failed to read $tmpc_file: length mismatch\n"
    unless length $clines == -s $tmpc_file;
close CTMPFILE;

my ($actlines, $tablines) = extract($clines);

$tablines .= make_type_tab($y_file, $tablines);

chmod 0644, $act_file;
open ACTFILE, ">$act_file" or die "can't open $act_file: $!\n";
print ACTFILE $actlines;
close ACTFILE;
chmod 0444, $act_file;

chmod 0644, $tab_file;
open TABFILE, ">$tab_file" or die "can't open $tab_file: $!\n";
print TABFILE $tablines;
close TABFILE;
chmod 0444, $tab_file;

unlink $tmpc_file;

# Wrap PERL_CORE round the symbol definitions. Also,  the
# C<#line 30 "perly.y"> confuses the Win32 resource compiler and the
# C<#line 188 "perlytmp.h"> gets picked up by make depend, so remove them.

open TMPH_FILE, $tmph_file or die "Can't open $tmph_file: $!\n";
chmod 0644, $h_file;
open H_FILE, ">$h_file" or die "Can't open $h_file: $!\n";
my $endcore_done = 0;
# Token macros need to be generated manually on bison 2.4
my $gather_tokens = ($version =~ /\b2\.4\b/ ? undef : 0);
my $tokens;
while (<TMPH_FILE>) {
    print H_FILE "#ifdef PERL_CORE\n" if $. == 1;
    if (!$endcore_done and /YYSTYPE_IS_DECLARED/) {
	print H_FILE "#endif /* PERL_CORE */\n";
	$endcore_done = 1;
    }
    next if /^#line \d+ ".*"/;
    if (not defined $gather_tokens) {
	$gather_tokens = 1 if /^\s* enum \s* yytokentype \s* \{/x;
    }
    elsif ($gather_tokens) {
	if (/^\# \s* endif/x) { # The #endif just after the end of the token enum
	    $gather_tokens = 0;
	    $_ .= "\n/* Tokens.  */\n$tokens";
	}
	else {
	    my ($tok, $val) = /(\w+) \s* = \s* (\d+)/x;
	    $tokens .= "#define $tok $val\n" if $tok;
	}
    }
    print H_FILE $_;
}
close TMPH_FILE;
close H_FILE;
chmod 0444, $h_file;
unlink $tmph_file;

print "rebuilt:  $h_file $tab_file $act_file\n";

exit 0;


sub extract {
    my $clines = shift;
    my $tablines;
    my $actlines;

    $clines =~ m@
	(?:
	    ^/* YYFINAL[^\n]+\n		#optional comment
	)?
	\# \s* define \s* YYFINAL	# first #define
	.*?				# other defines + most tables
	yystos\[\]\s*=			# start of last table
	.*?
	}\s*;				# end of last table
    @xms
	or die "Can't extract tables from $tmpc_file\n";
    $tablines = $&;


    $clines =~ m@
	switch \s* \( \s* \w+ \s* \) \s* { \s*
	(
	    case \s* \d+ \s* :
	    \s*
	    (?: \s* /\* .*? \*/ \s* )*	# optional C-comments
	    \s*
	    \#line [^\n]+"\Q$y_file\E"
	    .*?
	)
	}
	\s*
	(?: \s* /\* .*? \*/ \s* )*	# optional C-comments
	\s*
	(
	    \#line[^\n]+\.c"
	|
	    \#line[^\n]+\.simple"
	|
	    YY_SYMBOL_PRINT
	)
    @xms
	or die "Can't extract actions from $tmpc_file\n";
    $actlines = $1;

    # Remove extraneous comments from bison 2.4
    $actlines =~ s!\s* /\* \s* Line \s* \d+ \s* of \s* yacc\.c \s* \*/!!gx;

    # C<#line 188 "perlytmp.c"> gets picked up by make depend, so remove them.
    $actlines =~ s/^#line \d+ "\Q$tmpc_file\E".*$//gm;

    # convert yyvsp[nnn] into ps[nnn].val

    $actlines =~ s/yyvsp\[(.*?)\]/ps[$1].val/g
	or die "Can't convert value stack name\n";

    return $actlines. "\n", $tablines. "\n";
}

# Generate a table, yy_type_tab[], that specifies for each token, what
# type of value it holds.
#
# Read the .y file and extract a list of all the token names and
# non-terminal names; then scan the string $tablines for the table yytname,
# which gives the token index of each token/non-terminal; then use this to
# create yy_type_tab.
#
# ie given (in perly.y),
#
#   %token <opval> A
#   %token <ival>  B
#   %type  <pval>  C
#   %type  <opval> D
#
# and (in $tablines),
#
#   yytname[] = { "A" "B", "C", "D", "E" };
#
# then return
#
#    typedef enum { toketype_ival, toketype_opval, toketype_pval } toketypes;
#
#    static const toketypes yy_type_tab[]
#          = { toketype_opval, toketype_ival, toketype_pval,
#                toketype_opval, toketype_ival }
#
# where "E" has the default type. The default type is determined
# by the __DEFAULT__ comment  next to the appropriate union member in
# perly.y

sub make_type_tab {
    my ($y_file, $tablines) = @_;
    my %tokens;
    my %types;
    my $default_token;
    open my $fh, '<', $y_file or die "Can't open $y_file: $!\n";
    while (<$fh>) {
	if (/(\$\d+)\s*=/) {
	    warn "$y_file:$.: dangerous assignment to $1: $_";
	}

	if (/__DEFAULT__/) {
	    m{(\w+) \s* ; \s* /\* \s* __DEFAULT__}x
		or die "$y_file: can't parse __DEFAULT__ line: $_";
	    die "$y_file: duplicate __DEFAULT__ line: $_"
		    if defined $default_token;
	    $default_token = $1;
	    next;
	}

	next unless /^%(token|type)/;
	s/^%(token|type)\s+<(\w+)>\s+//
	    or die "$y_file: unparseable token/type line: $_";
	$tokens{$_} = $2 for (split ' ', $_);
	$types{$2} = 1;
    }
    die "$y_file: no __DEFAULT__ token defined\n" unless $default_token;
    $types{$default_token} = 1;

    $tablines =~ /^\Qstatic const char *const yytname[] =\E\n
	    {\n
	    (.*?)
	    ^};
	    /xsm
	or die "Can't extract yytname[] from table string\n";
    my $fields = $1;
    $fields =~ s{"([^"]+)"}
		{ "toketype_" .
		    (defined $tokens{$1} ? $tokens{$1} : $default_token)
		}ge;
    $fields =~ s/, \s* 0 \s* $//x
	or die "make_type_tab: couldn't delete trailing ',0'\n";

    return 
	  "\ntypedef enum {\n\t"
	. join(", ", map "toketype_$_", sort keys %types)
	. "\n} toketypes;\n\n"
	. "/* type of each token/terminal */\n"
	. "static const toketypes yy_type_tab[] =\n{\n"
	. $fields
	. "\n};\n";
}


sub my_system {
    system(@_);
    if ($? == -1) {
	die "failed to execute command '@_': $!\n";
    }
    elsif ($? & 127) {
	die sprintf "command '@_' died with signal %d\n",
	    ($? & 127);
    }
    elsif ($? >> 8) {
	die sprintf "command '@_' exited with value %d\n", $? >> 8;
    }
}
Commit	Line	Data
0de566d7 DM	1	#!/usr/bin/perl
	2	#
	3	# regen_perly.pl, DAPM 12-Feb-04
	4	#
1d325971	5	# Copyright (c) 2004, 2005 Larry Wall
0de566d7 DM	6	#
	7	# Given an input file perly.y, run bison on it and produce
	8	# the following output files:
	9	#
	10	# perly.h standard bison header file with minor doctoring of
	11	# #line directives plus adding a #ifdef PERL_CORE
	12	#
	13	# perly.tab the parser table C definitions extracted from the bison output
0539ab63	14	# plus an extra table generated by this script.
0de566d7 DM	15	#
	16	# perly.act the action case statements extracted from the bison output
	17	#
	18	# Note that perly.c is not regenerated - this is now a static file which
	19	# is not dependent on perly.y any more.
	20	#
	21	# If a filename of the form foo.y is given on the command line, then
	22	# this is used instead as the basename for all the files mentioned
	23	# above.
	24	#
	25	# Note that temporary files of the form perlytmp.h and perlytmp.c are
	26	# created and then deleted during this process
	27	#
	28	# Note also that this script is intended to be run on a UNIX system;
	29	# it may work elsewhere but no specific attempt has been made to make it
	30	# portable.
	31
	32	sub usage { die "usage: $0 [ -b bison_executable ] [ file.y ]\n" }
	33
	34	use warnings;
	35	use strict;
	36
	37	my $bison = 'bison';
	38
	39	if (@ARGV >= 2 and $ARGV[0] eq '-b') {
	40	shift;
	41	$bison = shift;
	42	}
	43
	44	my $y_file = shift \|\| 'perly.y';
	45
	46	usage unless @ARGV==0 && $y_file =~ /\.y$/;
	47
	48	(my $h_file = $y_file) =~ s/\.y$/.h/;
	49	(my $act_file = $y_file) =~ s/\.y$/.act/;
	50	(my $tab_file = $y_file) =~ s/\.y$/.tab/;
	51	(my $tmpc_file = $y_file) =~ s/\.y$/tmp.c/;
	52	(my $tmph_file = $y_file) =~ s/\.y$/tmp.h/;
	53
	54	# the yytranslate[] table generated by bison is ASCII/EBCDIC sensitive
	55
	56	die "$0: must be run on an ASCII system\n" unless ord 'A' == 65;
	57
	58	# check for correct version number. The constraints are:
	59	# * must be >= 1.24 to avoid licensing issues.
	60	# * it must generate the yystos[] table. Version 1.28 doesn't generate
	61	# this; 1.35+ does
	62	# * Must produce output which is extractable by the regexes below
	63	# * Must produce the right values.
	64	# These last two contstraints may well be met by earlier versions, but
	65	# I simply haven't tested them yet. If it works for you, then modify
	66	# the test below to allow that version too. DAPM Feb 04.
	67
	68	my $version = `$bison -V`;
7b931d50 LB	69	unless ($version) { die <<EOF; }
	70	Could not find a version of bison in your path. Please install bison.
	71	EOF
	72
ce1534ab	73	unless ($version =~ /\b(1\.875[a-z]?\|2\.[0134])\b/) { die <<EOF; }
0de566d7	74
2080282f	75	You have the wrong version of bison in your path; currently 1.875
aae7e7b3 VP	76	2.0, 2.1, 2.3 or 2.4 is required. Try installing
aae7e7b3 VP	77	http://ftp.gnu.org/gnu/bison/bison-2.4.1.tar.gz
0de566d7 DM	78	or similar. Your bison identifies itself as:
	79
	80	$version
	81	EOF
	82
	83	# creates $tmpc_file and $tmph_file
	84	my_system("$bison -d -o $tmpc_file $y_file");
	85
	86	open CTMPFILE, $tmpc_file or die "Can't open $tmpc_file: $!\n";
	87	my $clines;
	88	{ local $/; $clines = <CTMPFILE>; }
	89	die "failed to read $tmpc_file: length mismatch\n"
	90	unless length $clines == -s $tmpc_file;
	91	close CTMPFILE;
	92
	93	my ($actlines, $tablines) = extract($clines);
	94
d5c6462e	95	$tablines .= make_type_tab($y_file, $tablines);
0539ab63	96
0de566d7 DM	97	chmod 0644, $act_file;
	98	open ACTFILE, ">$act_file" or die "can't open $act_file: $!\n";
	99	print ACTFILE $actlines;
	100	close ACTFILE;
	101	chmod 0444, $act_file;
	102
	103	chmod 0644, $tab_file;
	104	open TABFILE, ">$tab_file" or die "can't open $tab_file: $!\n";
	105	print TABFILE $tablines;
	106	close TABFILE;
	107	chmod 0444, $tab_file;
	108
	109	unlink $tmpc_file;
	110
	111	# Wrap PERL_CORE round the symbol definitions. Also, the
96f4e226 SH	112	# C<#line 30 "perly.y"> confuses the Win32 resource compiler and the
96f4e226 SH	113	# C<#line 188 "perlytmp.h"> gets picked up by make depend, so remove them.
0de566d7 DM	114
	115	open TMPH_FILE, $tmph_file or die "Can't open $tmph_file: $!\n";
	116	chmod 0644, $h_file;
	117	open H_FILE, ">$h_file" or die "Can't open $h_file: $!\n";
	118	my $endcore_done = 0;
ce1534ab VP	119	# Token macros need to be generated manually on bison 2.4
	120	my $gather_tokens = ($version =~ /\b2\.4\b/ ? undef : 0);
	121	my $tokens;
0de566d7 DM	122	while (<TMPH_FILE>) {
	123	print H_FILE "#ifdef PERL_CORE\n" if $. == 1;
	124	if (!$endcore_done and /YYSTYPE_IS_DECLARED/) {
	125	print H_FILE "#endif /* PERL_CORE */\n";
	126	$endcore_done = 1;
	127	}
96f4e226	128	next if /^#line \d+ ".*"/;
ce1534ab VP	129	if (not defined $gather_tokens) {
	130	$gather_tokens = 1 if /^\s* enum \s* yytokentype \s* \{/x;
	131	}
	132	elsif ($gather_tokens) {
	133	if (/^\# \s* endif/x) { # The #endif just after the end of the token enum
	134	$gather_tokens = 0;
	135	$_ .= "\n/* Tokens. */\n$tokens";
	136	}
	137	else {
	138	my ($tok, $val) = /(\w+) \s* = \s* (\d+)/x;
	139	$tokens .= "#define $tok $val\n" if $tok;
	140	}
	141	}
0de566d7 DM	142	print H_FILE $_;
	143	}
	144	close TMPH_FILE;
	145	close H_FILE;
	146	chmod 0444, $h_file;
	147	unlink $tmph_file;
	148
	149	print "rebuilt: $h_file $tab_file $act_file\n";
	150
	151	exit 0;
	152
	153
	154	sub extract {
	155	my $clines = shift;
	156	my $tablines;
	157	my $actlines;
	158
	159	$clines =~ m@
	160	(?:
	161	^/* YYFINAL[^\n]+\n #optional comment
	162	)?
	163	\# \s* define \s* YYFINAL # first #define
	164	.*? # other defines + most tables
	165	yystos\[\]\s*= # start of last table
	166	.*?
	167	}\s*; # end of last table
	168	@xms
	169	or die "Can't extract tables from $tmpc_file\n";
	170	$tablines = $&;
	171
	172
	173	$clines =~ m@
	174	switch \s* \( \s* \w+ \s* \) \s* { \s*
	175	(
ce1534ab VP	176	case \s* \d+ \s* :
	177	\s*
	178	(?: \s* /\* .? \/ \s* )* # optional C-comments
	179	\s*
2ade6388	180	\#line [^\n]+"\Q$y_file\E"
0de566d7 DM	181	.*?
	182	)
	183	}
	184	\s*
ce1534ab	185	(?: \s* /\* .? \/ \s* )* # optional C-comments
0de566d7 DM	186	\s*
	187	(
	188	\#line[^\n]+\.c"
	189	\|
	190	\#line[^\n]+\.simple"
3797f23d DM	191	\|
3797f23d DM	192	YY_SYMBOL_PRINT
0de566d7 DM	193	)
	194	@xms
	195	or die "Can't extract actions from $tmpc_file\n";
	196	$actlines = $1;
	197
ce1534ab VP	198	# Remove extraneous comments from bison 2.4
	199	$actlines =~ s!\s* /\* \s* Line \s* \d+ \s* of \s* yacc\.c \s* \*/!!gx;
	200
0d6f9730 DM	201	# C<#line 188 "perlytmp.c"> gets picked up by make depend, so remove them.
	202	$actlines =~ s/^#line \d+ "\Q$tmpc_file\E".*$//gm;
	203
1654d593 DM	204	# convert yyvsp[nnn] into ps[nnn].val
	205
	206	$actlines =~ s/yyvsp\[(.*?)\]/ps[$1].val/g
	207	or die "Can't convert value stack name\n";
	208
0de566d7 DM	209	return $actlines. "\n", $tablines. "\n";
	210	}
	211
d5c6462e DM	212	# Generate a table, yy_type_tab[], that specifies for each token, what
d5c6462e DM	213	# type of value it holds.
0539ab63	214	#
d5c6462e DM	215	# Read the .y file and extract a list of all the token names and
	216	# non-terminal names; then scan the string $tablines for the table yytname,
	217	# which gives the token index of each token/non-terminal; then use this to
	218	# create yy_type_tab.
0539ab63	219	#
d5c6462e DM	220	# ie given (in perly.y),
	221	#
	222	# %token <opval> A
	223	# %token <ival> B
	224	# %type <pval> C
	225	# %type <opval> D
	226	#
	227	# and (in $tablines),
	228	#
	229	# yytname[] = { "A" "B", "C", "D", "E" };
0539ab63 DM	230	#
0539ab63 DM	231	# then return
d5c6462e DM	232	#
	233	# typedef enum { toketype_ival, toketype_opval, toketype_pval } toketypes;
	234	#
	235	# static const toketypes yy_type_tab[]
	236	# = { toketype_opval, toketype_ival, toketype_pval,
	237	# toketype_opval, toketype_ival }
	238	#
	239	# where "E" has the default type. The default type is determined
	240	# by the __DEFAULT__ comment next to the appropriate union member in
	241	# perly.y
0539ab63	242
d5c6462e	243	sub make_type_tab {
0539ab63 DM	244	my ($y_file, $tablines) = @_;
0539ab63 DM	245	my %tokens;
d5c6462e DM	246	my %types;
d5c6462e DM	247	my $default_token;
0539ab63 DM	248	open my $fh, '<', $y_file or die "Can't open $y_file: $!\n";
0539ab63 DM	249	while (<$fh>) {
29522234 DM	250	if (/(\$\d+)\s*=/) {
	251	warn "$y_file:$.: dangerous assignment to $1: $_";
	252	}
	253
d5c6462e DM	254	if (/__DEFAULT__/) {
	255	m{(\w+) \s* ; \s* /\* \s* __DEFAULT__}x
	256	or die "$y_file: can't parse __DEFAULT__ line: $_";
	257	die "$y_file: duplicate __DEFAULT__ line: $_"
	258	if defined $default_token;
	259	$default_token = $1;
	260	next;
	261	}
	262
	263	next unless /^%(token\|type)/;
	264	s/^%(token\|type)\s+<(\w+)>\s+//
	265	or die "$y_file: unparseable token/type line: $_";
	266	$tokens{$_} = $2 for (split ' ', $_);
	267	$types{$2} = 1;
0539ab63	268	}
d5c6462e DM	269	die "$y_file: no __DEFAULT__ token defined\n" unless $default_token;
d5c6462e DM	270	$types{$default_token} = 1;
0539ab63 DM	271
	272	$tablines =~ /^\Qstatic const char *const yytname[] =\E\n
	273	{\n
	274	(.*?)
	275	^};
	276	/xsm
	277	or die "Can't extract yytname[] from table string\n";
	278	my $fields = $1;
d5c6462e DM	279	$fields =~ s{"([^"]+)"}
	280	{ "toketype_" .
	281	(defined $tokens{$1} ? $tokens{$1} : $default_token)
	282	}ge;
	283	$fields =~ s/, \s* 0 \s* $//x
	284	or die "make_type_tab: couldn't delete trailing ',0'\n";
	285
0539ab63	286	return
d5c6462e DM	287	"\ntypedef enum {\n\t"
	288	. join(", ", map "toketype_$_", sort keys %types)
	289	. "\n} toketypes;\n\n"
	290	. "/* type of each token/terminal */\n"
d5c6462e DM	291	. "static const toketypes yy_type_tab[] =\n{\n"
	292	. $fields
	293	. "\n};\n";
0539ab63 DM	294	}
	295
	296
0de566d7 DM	297	sub my_system {
	298	system(@_);
	299	if ($? == -1) {
d5c6462e	300	die "failed to execute command '@_': $!\n";
0de566d7 DM	301	}
	302	elsif ($? & 127) {
	303	die sprintf "command '@_' died with signal %d\n",
	304	($? & 127);
	305	}
	306	elsif ($? >> 8) {
	307	die sprintf "command '@_' exited with value %d\n", $? >> 8;
	308	}
	309	}