[perl5.git] / Porting / sync-with-cpan

#!/usr/bin/env perl

=head1 NAME

Porting/sync-with-cpan

=head1 SYNOPSIS

  perl Porting/sync-with-cpan <module>

where <module> is the name it appears in the C<%Modules> hash
of F<Porting/Maintainers.pl>

=head1 DESCRIPTION

Script to help out with syncing cpan distros.

Does the following:

=over 4

=item *

Fetches the package list from CPAN. Finds the current version of the given
package. [1]

=item *

Downloads the relevant tarball; unpacks the tarball. [1]

=item *

Clean out the old directory (C<git clean -dfx>)

=item *

Moves the old directory out of the way, moves the new directory in place.

=item *

Restores any F<.gitignore> file.

=item *

Removes files from C<@IGNORE> and C<EXCLUDED>

=item *

C<git add> any new files.

=item *

C<git rm> any files that are gone.

=item *

Remove the +x bit on files in F<t/>

=item *

Remove the +x bit on files that don't have it enabled in the current dir

=item *

Restore files mentioned in C<CUSTOMIZED>

=item *

Adds new files to F<MANIFEST>

=item *

Runs a C<make> (assumes a configure has been run)

=item *

Cleans up

=item *

Runs tests for the package

=item *

Runs the porting tests

=back

[1]  If the C<--tarball> option is given, then CPAN is not consulted.
C<--tarball> should be the path to the tarball; the version is extracted
from the filename -- but can be overwritten by the C<--version> option.

=head1 TODO

=over 4

=item *

Delete files from F<MANIFEST>

=item *

Update F<Porting/Maintainers.pl>

=item *

Optional, run a full test suite

=item *

Handle complicated C<FILES>

=back

This is an initial version; no attempt has been made yet to make this
portable. It shells out instead of trying to find a Perl solution.
In particular, it assumes wget, git, tar, chmod, perl, make, and rm
to be available.

=cut


package Maintainers;

use 5.010;

use strict;
use warnings;
use Getopt::Long;

$| = 1;

die "This does not look like a top level directory"
     unless -d "cpan" && -d "Porting";

our @IGNORABLE;
our %Modules;

use autodie;

require "Porting/Maintainers.pl";

my %IGNORABLE    = map {$_ => 1} @IGNORABLE;

my $package      = "02packages.details.txt";
my $package_url  = "http://www.cpan.org/modules/$package";
my $package_file = "/tmp/$package";


GetOptions ('tarball=s'  =>  \my $tarball,
            'version=s'  =>  \my $version,
             force       =>  \my $force,)
        or  die "Failed to parse arguments";

die "Usage: $0 module [args] [cpan package]" unless @ARGV == 1 || @ARGV == 2;

my ($module)  = shift;
my  $cpan_mod = @ARGV ? shift : $module;


my  $info         = $Modules {$module} or die "Cannot find module $module";
my  $distribution = $$info {DISTRIBUTION};

my @files         = glob $$info {FILES};
if (@files != 1 || !-d $files [0] || $$info {MAP}) {
    say "This looks like a setup $0 cannot handle (yet)";
    unless ($force) {
        say "Will not continue without a --force option";
        exit 1;
    }
    say "--force is in effect, so we'll soldier on. Wish me luck!";
}


chdir "cpan";

my  $pkg_dir      = $$info {FILES};
    $pkg_dir      =~ s!.*/!!;

my ($old_version) = $distribution =~ /-([0-9.]+)\.tar\.gz/;

my  $o_module     = $module;
if ($cpan_mod =~ /-/ && $cpan_mod !~ /::/) {
    $cpan_mod =~ s/-/::/g;
}

#
# Find the information from CPAN.
#
my $new_file;
my $new_version;
unless ($tarball) {
    #
    # Poor man's cache
    #
    unless (-f $package_file && -M $package_file < 1) {
        system wget => $package_url, '-qO', $package_file;
    }

    my  $new_line = `grep '^$cpan_mod ' $package_file`
                     or die "Cannot find $cpan_mod on CPAN\n";
    chomp $new_line;
    (undef, $new_version, my $new_path) = split ' ', $new_line;
    if (defined $version) {
        $new_path =~ s/-$new_version\./-$version\./;
        $new_version = $version;
    }
    $new_file = (split '/', $new_path) [-1];

    my $url = "http://search.cpan.org/CPAN/authors/id/$new_path";
    say "Fetching $url";
    #
    # Fetch the new distro
    #
    system wget => $url, '-qO', $new_file;
}
else {
    $new_file     = $tarball;
    $new_version  = $version // ($new_file =~ /-([0-9._]+)\.tar\.gz/) [0];
}

my  $old_dir      = "$pkg_dir-$old_version";

say "Cleaning out old directory";
system git => 'clean', '-dfxq', $pkg_dir;

say "Unpacking $new_file";

system tar => 'xfz', $new_file;
(my $new_dir = $new_file) =~ s/\.tar\.gz//;
# ensure 'make' will update all files
system('find', $new_dir, '-exec', 'touch', '{}', ';');

say "Renaming directories";
rename $pkg_dir => $old_dir;
rename $new_dir => $pkg_dir;


if (-f "$old_dir/.gitignore") {
    say "Restoring .gitignore";
    system git => 'checkout', "$pkg_dir/.gitignore";
}

my @new_files = `find $pkg_dir -type f`;
chomp @new_files;
@new_files = grep {$_ ne $pkg_dir} @new_files;
s!^[^/]+/!! for @new_files;
my %new_files = map {$_ => 1} @new_files;

my @old_files = `find $old_dir -type f`;
chomp @old_files;
@old_files = grep {$_ ne $old_dir} @old_files;
s!^[^/]+/!! for @old_files;
my %old_files = map {$_ => 1} @old_files;

#
# Find files that can be deleted.
#
my @EXCLUDED_QR;
my %EXCLUDED_QQ;
if ($$info {EXCLUDED}) {
    foreach my $entry (@{$$info {EXCLUDED}}) {
        if (ref $entry) {push @EXCLUDED_QR => $entry}
        else            {$EXCLUDED_QQ {$entry} = 1}
    }
}

my @delete;
my @commit;
my @gone;
FILE:
foreach my $file (@new_files) {
    next if -d "$pkg_dir/$file";   # Ignore directories.
    next if $old_files {$file};    # It's already there.
    if ($IGNORABLE {$file}) {
        push @delete => $file;
        next;
    }
    if ($EXCLUDED_QQ {$file}) {
        push @delete => $file;
        next;
    }
    foreach my $pattern (@EXCLUDED_QR) {
        if ($file =~ /$pattern/) {
            push @delete => $file;
            next FILE;
        }
    }
    push @commit => $file;
}
foreach my $file (@old_files) {
    next if -d "$old_dir/$file";
    next if $new_files {$file};
    push @gone => $file;
}

#
# Find all files with an exec bit
#
my @exec = `find $pkg_dir -type f -perm +111`;
chomp @exec;
my @de_exec;
foreach my $file (@exec) {
    # Remove leading dir
    $file =~ s!^[^/]+/!!;
    if ($file =~ m!^t/!) {
        push @de_exec => $file;
        next;
    }
    # Check to see if the file exists; if it doesn't and doesn't have
    # the exec bit, remove it.
    if ($old_files {$file}) {
        unless (-x "$old_dir/$file") {
            push @de_exec => $file;
        }
    }
}

#
# No need to change the +x bit on files that will be deleted.
#
if (@de_exec && @delete) {
    my %delete = map {+"$pkg_dir/$_" => 1} @delete;
    @de_exec = grep {!$delete {$_}} @de_exec;
}

say "unlink $pkg_dir/$_" for @delete;
say "git add $pkg_dir/$_" for @commit;
say "git rm -f $pkg_dir/$_" for @gone;
say "chmod a-x $pkg_dir/$_" for @de_exec;

print "Hit return to continue; ^C to abort "; <STDIN>;

unlink "$pkg_dir/$_"                      for @delete;
system git   => 'add', "$pkg_dir/$_"      for @commit;
system git   => 'rm', '-f', "$pkg_dir/$_" for @gone;
system chmod => 'a-x', "$pkg_dir/$_"      for @de_exec;

#
# Restore anything that is customized.
# We don't really care whether we've deleted the file - since we
# do a git restore, it's going to be resurrected if necessary.
#
if ($$info {CUSTOMIZED}) {
    say "Restoring customized files";
    foreach my $file (@{$$info {CUSTOMIZED}}) {
        system git => "checkout", "$pkg_dir/$file";
    }
}

chdir "..";
if (@commit) {
    say "Fixing MANIFEST";
    my $MANIFEST      = "MANIFEST";
    my $MANIFEST_SORT = "$MANIFEST.sorted";
    open my $fh, ">>", $MANIFEST;
    say $fh "cpan/$pkg_dir/$_" for @commit;
    close $fh;
    system perl => "Porting/manisort", '--output', $MANIFEST_SORT;
    rename $MANIFEST_SORT => $MANIFEST;
}


print "Running a make ... ";
system "make > make.log 2>&1" and die "Running make failed, see make.log";
print "done\n";

#
# Must clean up, or else t/porting/FindExt.t will fail.
# Note that we can always retrieve the orginal directory with a git checkout.
#
print "About to clean up; hit return or abort (^C) "; <STDIN>;

chdir "cpan";
system rm => '-r', $old_dir;
unlink $new_file unless $tarball;


#
# Run the tests. First the test belonging to the module, followed by the
# the tests in t/porting
#
chdir "../t";
say "Running module tests";
my @test_files = `find ../cpan/$pkg_dir -name '*.t' -type f`;
chomp @test_files;
my $output = `./perl TEST @test_files`;
unless ($output =~ /All tests successful/) {
    say $output;
    exit 1;
}

print "Running tests in t/porting ";
my @tests = `ls porting/*.t`;
chomp @tests;
my @failed;
foreach my $t (@tests) {
    my @not = `./perl -I../lib -I.. $t | grep ^not | grep -v "# TODO"`;
    print @not ? '!' : '.';
    push @failed => $t if @not;
}
print "\n";
say "Failed tests: @failed" if @failed;


print "Now you ought to run a make; make test ...\n";

say "Do not forget to update Porting/Maintainers.pl before committing";
say "$o_module is now version $new_version";


__END__
Commit	Line	Data
33e80a47	1	#!/usr/bin/env perl
418f4069	2
c5e3e317 JL	3	=head1 NAME
c5e3e317 JL	4
0c41dfff	5	Porting/sync-with-cpan
c5e3e317 JL	6
	7	=head1 SYNOPSIS
	8
	9	perl Porting/sync-with-cpan <module>
	10
	11	where <module> is the name it appears in the C<%Modules> hash
	12	of F<Porting/Maintainers.pl>
	13
	14	=head1 DESCRIPTION
	15
	16	Script to help out with syncing cpan distros.
	17
	18	Does the following:
	19
	20	=over 4
	21
	22	=item *
	23
	24	Fetches the package list from CPAN. Finds the current version of the given
	25	package. [1]
	26
	27	=item *
	28
	29	Downloads the relevant tarball; unpacks the tarball. [1]
	30
	31	=item *
	32
	33	Clean out the old directory (C<git clean -dfx>)
	34
	35	=item *
	36
	37	Moves the old directory out of the way, moves the new directory in place.
	38
	39	=item *
	40
	41	Restores any F<.gitignore> file.
	42
	43	=item *
	44
	45	Removes files from C<@IGNORE> and C<EXCLUDED>
	46
	47	=item *
	48
	49	C<git add> any new files.
	50
	51	=item *
	52
	53	C<git rm> any files that are gone.
	54
	55	=item *
	56
	57	Remove the +x bit on files in F<t/>
	58
	59	=item *
	60
	61	Remove the +x bit on files that don't have it enabled in the current dir
	62
	63	=item *
	64
	65	Restore files mentioned in C<CUSTOMIZED>
	66
	67	=item *
	68
	69	Adds new files to F<MANIFEST>
70
71	=item *
72
73	Runs a C<make> (assumes a configure has been run)
74
75	=item *
76
77	Cleans up
78
79	=item *
80
81	Runs tests for the package
82
83	=item *
84
85	Runs the porting tests
86
87	=back
88
89	[1] If the C<--tarball> option is given, then CPAN is not consulted.
90	C<--tarball> should be the path to the tarball; the version is extracted
91	from the filename -- but can be overwritten by the C<--version> option.
92
93	=head1 TODO
94
95	=over 4
96
97	=item *
98
99	Delete files from F<MANIFEST>
100
101	=item *
102
103	Update F<Porting/Maintainers.pl>
104
105	=item *
106
107	Optional, run a full test suite
108
109	=item *
110
111	Handle complicated C<FILES>
112
113	=back
114
115	This is an initial version; no attempt has been made yet to make this
116	portable. It shells out instead of trying to find a Perl solution.
117	In particular, it assumes wget, git, tar, chmod, perl, make, and rm
118	to be available.
119
120	=cut
121
418f4069	122
4d18e0a2 A	123	package Maintainers;
4d18e0a2 A	124
418f4069 A	125	use 5.010;
	126
	127	use strict;
	128	use warnings;
4d18e0a2	129	use Getopt::Long;
418f4069 A	130
	131	$\| = 1;
	132
07a826df	133	die "This does not look like a top level directory"
418f4069 A	134	unless -d "cpan" && -d "Porting";
418f4069 A	135
418f4069 A	136	our @IGNORABLE;
	137	our %Modules;
	138
	139	use autodie;
	140
	141	require "Porting/Maintainers.pl";
	142
418f4069 A	143	my %IGNORABLE = map {$_ => 1} @IGNORABLE;
	144
	145	my $package = "02packages.details.txt";
	146	my $package_url = "http://www.cpan.org/modules/$package";
	147	my $package_file = "/tmp/$package";
	148
418f4069	149
4d18e0a2	150	GetOptions ('tarball=s' => \my $tarball,
b5bf278a A	151	'version=s' => \my $version,
b5bf278a A	152	force => \my $force,)
4d18e0a2 A	153	or die "Failed to parse arguments";
	154
	155	die "Usage: $0 module [args] [cpan package]" unless @ARGV == 1 \|\| @ARGV == 2;
418f4069	156
5b73aae5 A	157	my ($module) = shift;
5b73aae5 A	158	my $cpan_mod = @ARGV ? shift : $module;
418f4069	159
4d18e0a2	160
418f4069 A	161	my $info = $Modules {$module} or die "Cannot find module $module";
418f4069 A	162	my $distribution = $$info {DISTRIBUTION};
b5bf278a A	163
	164	my @files = glob $$info {FILES};
	165	if (@files != 1 \|\| !-d $files [0] \|\| $$info {MAP}) {
	166	say "This looks like a setup $0 cannot handle (yet)";
	167	unless ($force) {
	168	say "Will not continue without a --force option";
	169	exit 1;
	170	}
	171	say "--force is in effect, so we'll soldier on. Wish me luck!";
	172	}
	173
	174
	175	chdir "cpan";
	176
418f4069 A	177	my $pkg_dir = $$info {FILES};
	178	$pkg_dir =~ s!.*/!!;
	179
	180	my ($old_version) = $distribution =~ /-([0-9.]+)\.tar\.gz/;
	181
	182	my $o_module = $module;
5b73aae5 A	183	if ($cpan_mod =~ /-/ && $cpan_mod !~ /::/) {
5b73aae5 A	184	$cpan_mod =~ s/-/::/g;
418f4069 A	185	}
	186
	187	#
	188	# Find the information from CPAN.
	189	#
4d18e0a2 A	190	my $new_file;
	191	my $new_version;
	192	unless ($tarball) {
	193	#
	194	# Poor man's cache
	195	#
	196	unless (-f $package_file && -M $package_file < 1) {
	197	system wget => $package_url, '-qO', $package_file;
	198	}
	199
	200	my $new_line = `grep '^$cpan_mod ' $package_file`
	201	or die "Cannot find $cpan_mod on CPAN\n";
	202	chomp $new_line;
	203	(undef, $new_version, my $new_path) = split ' ', $new_line;
3a4316cc JL	204	if (defined $version) {
	205	$new_path =~ s/-$new_version\./-$version\./;
	206	$new_version = $version;
	207	}
4d18e0a2 A	208	$new_file = (split '/', $new_path) [-1];
	209
	210	my $url = "http://search.cpan.org/CPAN/authors/id/$new_path";
	211	say "Fetching $url";
	212	#
	213	# Fetch the new distro
	214	#
	215	system wget => $url, '-qO', $new_file;
	216	}
	217	else {
	218	$new_file = $tarball;
	219	$new_version = $version // ($new_file =~ /-([0-9._]+)\.tar\.gz/) [0];
	220	}
418f4069 A	221
418f4069 A	222	my $old_dir = "$pkg_dir-$old_version";
418f4069 A	223
	224	say "Cleaning out old directory";
	225	system git => 'clean', '-dfxq', $pkg_dir;
	226
418f4069 A	227	say "Unpacking $new_file";
	228
	229	system tar => 'xfz', $new_file;
618ac2f6	230	(my $new_dir = $new_file) =~ s/\.tar\.gz//;
3f7808eb JL	231	# ensure 'make' will update all files
3f7808eb JL	232	system('find', $new_dir, '-exec', 'touch', '{}', ';');
418f4069 A	233
	234	say "Renaming directories";
	235	rename $pkg_dir => $old_dir;
	236	rename $new_dir => $pkg_dir;
	237
	238
	239	if (-f "$old_dir/.gitignore") {
	240	say "Restoring .gitignore";
	241	system git => 'checkout', "$pkg_dir/.gitignore";
	242	}
	243
ad9b4e6f	244	my @new_files = `find $pkg_dir -type f`;
418f4069 A	245	chomp @new_files;
	246	@new_files = grep {$_ ne $pkg_dir} @new_files;
	247	s!^[^/]+/!! for @new_files;
	248	my %new_files = map {$_ => 1} @new_files;
	249
ad9b4e6f	250	my @old_files = `find $old_dir -type f`;
418f4069 A	251	chomp @old_files;
	252	@old_files = grep {$_ ne $old_dir} @old_files;
	253	s!^[^/]+/!! for @old_files;
	254	my %old_files = map {$_ => 1} @old_files;
	255
	256	#
	257	# Find files that can be deleted.
	258	#
	259	my @EXCLUDED_QR;
	260	my %EXCLUDED_QQ;
	261	if ($$info {EXCLUDED}) {
	262	foreach my $entry (@{$$info {EXCLUDED}}) {
	263	if (ref $entry) {push @EXCLUDED_QR => $entry}
	264	else {$EXCLUDED_QQ {$entry} = 1}
	265	}
	266	}
	267
	268	my @delete;
	269	my @commit;
	270	my @gone;
418f4069 A	271	FILE:
	272	foreach my $file (@new_files) {
	273	next if -d "$pkg_dir/$file"; # Ignore directories.
	274	next if $old_files {$file}; # It's already there.
	275	if ($IGNORABLE {$file}) {
	276	push @delete => $file;
	277	next;
	278	}
	279	if ($EXCLUDED_QQ {$file}) {
	280	push @delete => $file;
	281	next;
	282	}
	283	foreach my $pattern (@EXCLUDED_QR) {
	284	if ($file =~ /$pattern/) {
	285	push @delete => $file;
	286	next FILE;
	287	}
	288	}
	289	push @commit => $file;
	290	}
	291	foreach my $file (@old_files) {
	292	next if -d "$old_dir/$file";
	293	next if $new_files {$file};
	294	push @gone => $file;
	295	}
ad9b4e6f A	296
	297	#
	298	# Find all files with an exec bit
	299	#
	300	my @exec = `find $pkg_dir -type f -perm +111`;
418f4069	301	chomp @exec;
ad9b4e6f A	302	my @de_exec;
	303	foreach my $file (@exec) {
	304	# Remove leading dir
	305	$file =~ s!^[^/]+/!!;
	306	if ($file =~ m!^t/!) {
	307	push @de_exec => $file;
	308	next;
	309	}
	310	# Check to see if the file exists; if it doesn't and doesn't have
	311	# the exec bit, remove it.
	312	if ($old_files {$file}) {
	313	unless (-x "$old_dir/$file") {
	314	push @de_exec => $file;
	315	}
	316	}
	317	}
418f4069 A	318
	319	#
	320	# No need to change the +x bit on files that will be deleted.
	321	#
ad9b4e6f	322	if (@de_exec && @delete) {
a9f5d1d4	323	my %delete = map {+"$pkg_dir/$_" => 1} @delete;
ad9b4e6f	324	@de_exec = grep {!$delete {$_}} @de_exec;
418f4069 A	325	}
	326
	327	say "unlink $pkg_dir/$_" for @delete;
	328	say "git add $pkg_dir/$_" for @commit;
	329	say "git rm -f $pkg_dir/$_" for @gone;
ad9b4e6f	330	say "chmod a-x $pkg_dir/$_" for @de_exec;
418f4069 A	331
	332	print "Hit return to continue; ^C to abort "; <STDIN>;
	333
	334	unlink "$pkg_dir/$_" for @delete;
	335	system git => 'add', "$pkg_dir/$_" for @commit;
	336	system git => 'rm', '-f', "$pkg_dir/$_" for @gone;
ad9b4e6f	337	system chmod => 'a-x', "$pkg_dir/$_" for @de_exec;
418f4069	338
9c259538 A	339	#
	340	# Restore anything that is customized.
	341	# We don't really care whether we've deleted the file - since we
	342	# do a git restore, it's going to be resurrected if necessary.
	343	#
	344	if ($$info {CUSTOMIZED}) {
	345	say "Restoring customized files";
	346	foreach my $file (@{$$info {CUSTOMIZED}}) {
	347	system git => "checkout", "$pkg_dir/$file";
	348	}
	349	}
	350
a8121781	351	chdir "..";
418f4069 A	352	if (@commit) {
418f4069 A	353	say "Fixing MANIFEST";
a8121781	354	my $MANIFEST = "MANIFEST";
418f4069 A	355	my $MANIFEST_SORT = "$MANIFEST.sorted";
418f4069 A	356	open my $fh, ">>", $MANIFEST;
a8121781	357	say $fh "cpan/$pkg_dir/$_" for @commit;
418f4069	358	close $fh;
a8121781	359	system perl => "Porting/manisort", '--output', $MANIFEST_SORT;
418f4069 A	360	rename $MANIFEST_SORT => $MANIFEST;
	361	}
	362
	363
418f4069 A	364	print "Running a make ... ";
	365	system "make > make.log 2>&1" and die "Running make failed, see make.log";
	366	print "done\n";
	367
	368	#
	369	# Must clean up, or else t/porting/FindExt.t will fail.
	370	# Note that we can always retrieve the orginal directory with a git checkout.
	371	#
	372	print "About to clean up; hit return or abort (^C) "; <STDIN>;
	373
	374	chdir "cpan";
	375	system rm => '-r', $old_dir;
e81fec2c	376	unlink $new_file unless $tarball;
418f4069 A	377
418f4069 A	378
ad9b4e6f A	379	#
	380	# Run the tests. First the test belonging to the module, followed by the
	381	# the tests in t/porting
	382	#
418f4069	383	chdir "../t";
ad9b4e6f A	384	say "Running module tests";
	385	my @test_files = `find ../cpan/$pkg_dir -name '*.t' -type f`;
	386	chomp @test_files;
	387	my $output = `./perl TEST @test_files`;
	388	unless ($output =~ /All tests successful/) {
	389	say $output;
	390	exit 1;
	391	}
	392
418f4069 A	393	print "Running tests in t/porting ";
	394	my @tests = `ls porting/*.t`;
	395	chomp @tests;
	396	my @failed;
	397	foreach my $t (@tests) {
	398	my @not = `./perl -I../lib -I.. $t \| grep ^not \| grep -v "# TODO"`;
	399	print @not ? '!' : '.';
	400	push @failed => $t if @not;
	401	}
	402	print "\n";
	403	say "Failed tests: @failed" if @failed;
	404
	405
	406	print "Now you ought to run a make; make test ...\n";
	407
	408	say "Do not forget to update Porting/Maintainers.pl before committing";
	409	say "$o_module is now version $new_version";
	410
	411
	412	__END__