This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Elide use of `ls`, `find` and `touch`
[perl5.git] / Porting / sync-with-cpan
CommitLineData
33e80a47 1#!/usr/bin/env perl
418f4069 2
c5e3e317
JL
3=head1 NAME
4
f703fc96 5Porting/sync-with-cpan - Synchronize with CPAN distributions
c5e3e317
JL
6
7=head1 SYNOPSIS
8
9 perl Porting/sync-with-cpan <module>
10
11where <module> is the name it appears in the C<%Modules> hash
12of F<Porting/Maintainers.pl>
13
14=head1 DESCRIPTION
15
16Script to help out with syncing cpan distros.
17
18Does the following:
19
20=over 4
21
22=item *
23
24Fetches the package list from CPAN. Finds the current version of the given
25package. [1]
26
27=item *
28
29Downloads the relevant tarball; unpacks the tarball. [1]
30
31=item *
32
33Clean out the old directory (C<git clean -dfx>)
34
35=item *
36
37Moves the old directory out of the way, moves the new directory in place.
38
39=item *
40
41Restores any F<.gitignore> file.
42
43=item *
44
45Removes files from C<@IGNORE> and C<EXCLUDED>
46
47=item *
48
49C<git add> any new files.
50
51=item *
52
53C<git rm> any files that are gone.
54
55=item *
56
57Remove the +x bit on files in F<t/>
58
59=item *
60
61Remove the +x bit on files that don't have it enabled in the current dir
62
63=item *
64
65Restore files mentioned in C<CUSTOMIZED>
66
67=item *
68
69Adds new files to F<MANIFEST>
70
71=item *
72
73Runs a C<make> (assumes a configure has been run)
74
75=item *
76
77Cleans up
78
79=item *
80
81Runs tests for the package
82
83=item *
84
85Runs the porting tests
86
87=back
88
89[1] If the C<--tarball> option is given, then CPAN is not consulted.
90C<--tarball> should be the path to the tarball; the version is extracted
91from the filename -- but can be overwritten by the C<--version> option.
92
93=head1 TODO
94
95=over 4
96
97=item *
98
99Delete files from F<MANIFEST>
100
101=item *
102
103Update F<Porting/Maintainers.pl>
104
105=item *
106
107Optional, run a full test suite
108
109=item *
110
111Handle complicated C<FILES>
112
113=back
114
115This is an initial version; no attempt has been made yet to make this
116portable. It shells out instead of trying to find a Perl solution.
192f56b0 117In particular, it assumes git, chmod, perl, and make
c5e3e317
JL
118to be available.
119
120=cut
121
418f4069 122
4d18e0a2
A
123package Maintainers;
124
418f4069
A
125use 5.010;
126
127use strict;
128use warnings;
4d18e0a2 129use Getopt::Long;
a1450e8b 130use Archive::Tar;
192f56b0 131use File::Path qw( remove_tree );
fc134225 132use File::Find;
418f4069
A
133
134$| = 1;
135
07a826df 136die "This does not look like a top level directory"
418f4069
A
137 unless -d "cpan" && -d "Porting";
138
418f4069
A
139our @IGNORABLE;
140our %Modules;
141
142use autodie;
143
144require "Porting/Maintainers.pl";
145
418f4069
A
146my %IGNORABLE = map {$_ => 1} @IGNORABLE;
147
fc134225
MM
148my $tmpdir= $ENV{ TEMP } // '/tmp';
149
418f4069
A
150my $package = "02packages.details.txt";
151my $package_url = "http://www.cpan.org/modules/$package";
fc134225 152my $package_file = "$tmpdir/$package"; # this is a cache
418f4069 153
b7e2b692
JL
154my @problematic = (
155 'podlators', # weird CUSTOMIZED section due to .PL files
156);
157
418f4069 158
4d18e0a2 159GetOptions ('tarball=s' => \my $tarball,
b5bf278a
A
160 'version=s' => \my $version,
161 force => \my $force,)
4d18e0a2
A
162 or die "Failed to parse arguments";
163
164die "Usage: $0 module [args] [cpan package]" unless @ARGV == 1 || @ARGV == 2;
418f4069 165
fc134225
MM
166sub find_type_f {
167 my @res;
168 find( { no_chdir => 1, wanted => sub {
169 my $file= $File::Find::name;
170 return unless -f $file;
171 push @res, $file
172 }}, @_ );
173 @res
174};
175
5b73aae5
A
176my ($module) = shift;
177my $cpan_mod = @ARGV ? shift : $module;
418f4069 178
4d18e0a2 179
418f4069
A
180my $info = $Modules {$module} or die "Cannot find module $module";
181my $distribution = $$info {DISTRIBUTION};
b5bf278a
A
182
183my @files = glob $$info {FILES};
b7e2b692 184if (!-d $files [0] || grep { $_ eq $module } @problematic) {
b5bf278a
A
185 say "This looks like a setup $0 cannot handle (yet)";
186 unless ($force) {
187 say "Will not continue without a --force option";
188 exit 1;
189 }
190 say "--force is in effect, so we'll soldier on. Wish me luck!";
191}
192
193
194chdir "cpan";
195
83d3dd1d 196my $pkg_dir = $files[0];
418f4069
A
197 $pkg_dir =~ s!.*/!!;
198
199my ($old_version) = $distribution =~ /-([0-9.]+)\.tar\.gz/;
200
201my $o_module = $module;
5b73aae5
A
202if ($cpan_mod =~ /-/ && $cpan_mod !~ /::/) {
203 $cpan_mod =~ s/-/::/g;
418f4069
A
204}
205
206#
207# Find the information from CPAN.
208#
4d18e0a2
A
209my $new_file;
210my $new_version;
211unless ($tarball) {
212 #
213 # Poor man's cache
214 #
215 unless (-f $package_file && -M $package_file < 1) {
132246f2
MM
216 eval {
217 require HTTP::Tiny;
218 my $http= HTTP::Tiny->new();
219 $http->mirror( $package_url => $package_file );
220 1
221 } or system wget => $package_url, '-qO', $package_file;
4d18e0a2
A
222 }
223
224 my $new_line = `grep '^$cpan_mod ' $package_file`
225 or die "Cannot find $cpan_mod on CPAN\n";
226 chomp $new_line;
227 (undef, $new_version, my $new_path) = split ' ', $new_line;
3a4316cc
JL
228 if (defined $version) {
229 $new_path =~ s/-$new_version\./-$version\./;
230 $new_version = $version;
231 }
4d18e0a2
A
232 $new_file = (split '/', $new_path) [-1];
233
234 my $url = "http://search.cpan.org/CPAN/authors/id/$new_path";
235 say "Fetching $url";
236 #
237 # Fetch the new distro
238 #
132246f2
MM
239 eval {
240 require HTTP::Tiny;
241 my $http= HTTP::Tiny->new();
242 $http->mirror( $url => $new_file );
243 1
244 } or system wget => $url, '-qO', $new_file;
4d18e0a2
A
245}
246else {
247 $new_file = $tarball;
248 $new_version = $version // ($new_file =~ /-([0-9._]+)\.tar\.gz/) [0];
249}
418f4069
A
250
251my $old_dir = "$pkg_dir-$old_version";
418f4069
A
252
253say "Cleaning out old directory";
254system git => 'clean', '-dfxq', $pkg_dir;
255
418f4069 256say "Unpacking $new_file";
5fb91d48 257Archive::Tar->extract_archive( $new_file );
418f4069 258
618ac2f6 259(my $new_dir = $new_file) =~ s/\.tar\.gz//;
3f7808eb 260# ensure 'make' will update all files
fc134225
MM
261my $t= time;
262for my $file (find_type_f($new_dir)) {
263 open(my $fh,">>$file") || die "Cannot write $file:$!";
264 close($fh);
265 utime($t,$t,$file);
266};
418f4069
A
267
268say "Renaming directories";
269rename $pkg_dir => $old_dir;
418f4069 270
83d3dd1d
JL
271say "Creating new package directory";
272mkdir $pkg_dir;
273
274say "Populating new package directory";
275my $map = $$info {MAP};
276my @EXCLUDED_QR;
277my %EXCLUDED_QQ;
278if ($$info {EXCLUDED}) {
279 foreach my $entry (@{$$info {EXCLUDED}}) {
280 if (ref $entry) {push @EXCLUDED_QR => $entry}
281 else {$EXCLUDED_QQ {$entry} = 1}
282 }
283}
284
fc134225 285FILE: for my $file ( find_type_f( $new_dir )) {
83d3dd1d
JL
286 my $old_file = $file;
287 $file =~ s{^$new_dir/}{};
288
289 next if $EXCLUDED_QQ{$file};
290 for my $qr (@EXCLUDED_QR) {
291 next FILE if $file =~ $qr;
292 }
293
294 if ( $map ) {
295 for my $key ( sort { length $b <=> length $a } keys %$map ) {
296 my $val = $map->{$key};
297 last if $file =~ s/^$key/$val/;
298 }
299 }
7bbb137d
JL
300 else {
301 $file = $files[0] . '/' . $file;
302 }
83d3dd1d
JL
303
304 if ( $file =~ m{^cpan/} ) {
305 $file =~ s{^cpan/}{};
306 }
307 else {
308 $file = '../' . $file;
309 }
310
311 my $prefix = '';
312 my @parts = split '/', $file;
313 pop @parts;
314 for my $part (@parts) {
315 $prefix .= '/' if $prefix;
316 $prefix .= $part;
317 mkdir $prefix unless -d $prefix;
318 }
319
320 rename $old_file => $file;
321}
192f56b0 322remove_tree( $new_dir );
418f4069
A
323
324if (-f "$old_dir/.gitignore") {
325 say "Restoring .gitignore";
326 system git => 'checkout', "$pkg_dir/.gitignore";
327}
328
fc134225 329my @new_files = find_type_f( $pkg_dir );
418f4069
A
330@new_files = grep {$_ ne $pkg_dir} @new_files;
331s!^[^/]+/!! for @new_files;
332my %new_files = map {$_ => 1} @new_files;
333
fc134225 334my @old_files = find_type_f( $old_dir );
418f4069
A
335@old_files = grep {$_ ne $old_dir} @old_files;
336s!^[^/]+/!! for @old_files;
337my %old_files = map {$_ => 1} @old_files;
338
418f4069
A
339my @delete;
340my @commit;
341my @gone;
418f4069
A
342FILE:
343foreach my $file (@new_files) {
344 next if -d "$pkg_dir/$file"; # Ignore directories.
345 next if $old_files {$file}; # It's already there.
346 if ($IGNORABLE {$file}) {
347 push @delete => $file;
348 next;
349 }
418f4069
A
350 push @commit => $file;
351}
352foreach my $file (@old_files) {
353 next if -d "$old_dir/$file";
354 next if $new_files {$file};
355 push @gone => $file;
356}
ad9b4e6f
A
357
358#
359# Find all files with an exec bit
360#
fc134225 361my @exec = find_type_f( $pkg_dir );
ad9b4e6f
A
362my @de_exec;
363foreach my $file (@exec) {
364 # Remove leading dir
365 $file =~ s!^[^/]+/!!;
366 if ($file =~ m!^t/!) {
367 push @de_exec => $file;
368 next;
369 }
370 # Check to see if the file exists; if it doesn't and doesn't have
371 # the exec bit, remove it.
372 if ($old_files {$file}) {
373 unless (-x "$old_dir/$file") {
374 push @de_exec => $file;
375 }
376 }
377}
418f4069
A
378
379#
380# No need to change the +x bit on files that will be deleted.
381#
ad9b4e6f 382if (@de_exec && @delete) {
a9f5d1d4 383 my %delete = map {+"$pkg_dir/$_" => 1} @delete;
ad9b4e6f 384 @de_exec = grep {!$delete {$_}} @de_exec;
418f4069
A
385}
386
387say "unlink $pkg_dir/$_" for @delete;
388say "git add $pkg_dir/$_" for @commit;
389say "git rm -f $pkg_dir/$_" for @gone;
ad9b4e6f 390say "chmod a-x $pkg_dir/$_" for @de_exec;
418f4069
A
391
392print "Hit return to continue; ^C to abort "; <STDIN>;
393
394unlink "$pkg_dir/$_" for @delete;
395system git => 'add', "$pkg_dir/$_" for @commit;
396system git => 'rm', '-f', "$pkg_dir/$_" for @gone;
ad9b4e6f 397system chmod => 'a-x', "$pkg_dir/$_" for @de_exec;
418f4069 398
9c259538
A
399#
400# Restore anything that is customized.
401# We don't really care whether we've deleted the file - since we
402# do a git restore, it's going to be resurrected if necessary.
403#
404if ($$info {CUSTOMIZED}) {
405 say "Restoring customized files";
406 foreach my $file (@{$$info {CUSTOMIZED}}) {
407 system git => "checkout", "$pkg_dir/$file";
408 }
409}
410
a8121781 411chdir "..";
418f4069
A
412if (@commit) {
413 say "Fixing MANIFEST";
a8121781 414 my $MANIFEST = "MANIFEST";
418f4069
A
415 my $MANIFEST_SORT = "$MANIFEST.sorted";
416 open my $fh, ">>", $MANIFEST;
a8121781 417 say $fh "cpan/$pkg_dir/$_" for @commit;
418f4069 418 close $fh;
a8121781 419 system perl => "Porting/manisort", '--output', $MANIFEST_SORT;
418f4069
A
420 rename $MANIFEST_SORT => $MANIFEST;
421}
422
423
418f4069 424print "Running a make ... ";
fc134225 425system "$Config{make} > make.log 2>&1" and die "Running make failed, see make.log";
418f4069
A
426print "done\n";
427
428#
429# Must clean up, or else t/porting/FindExt.t will fail.
730ad6b9 430# Note that we can always retrieve the original directory with a git checkout.
418f4069
A
431#
432print "About to clean up; hit return or abort (^C) "; <STDIN>;
433
192f56b0
MM
434remove_tree( "cpan/$old_dir" );
435unlink "cpan/$new_file" unless $tarball;
418f4069
A
436
437
ad9b4e6f
A
438#
439# Run the tests. First the test belonging to the module, followed by the
440# the tests in t/porting
441#
192f56b0 442chdir "t";
ad9b4e6f 443say "Running module tests";
fc134225
MM
444my @test_files = grep { /\.t$/ } find_type_f( $pkg_dir );
445my $exe_dir= $^O =~ /MSWin/ ? "..\\" : './';
446my $output = `${exe_dir}perl$Config{_exe} TEST @test_files`;
ad9b4e6f
A
447unless ($output =~ /All tests successful/) {
448 say $output;
449 exit 1;
450}
451
418f4069 452print "Running tests in t/porting ";
fc134225 453my @tests = glob 'porting/*.t';
418f4069
A
454chomp @tests;
455my @failed;
456foreach my $t (@tests) {
457 my @not = `./perl -I../lib -I.. $t | grep ^not | grep -v "# TODO"`;
458 print @not ? '!' : '.';
459 push @failed => $t if @not;
460}
461print "\n";
462say "Failed tests: @failed" if @failed;
463
464
9807c17b
JL
465say "Attempting to update Maintainers.pl";
466chdir '..';
467
468open my $Maintainers_pl, '<', 'Porting/Maintainers.pl';
469open my $new_Maintainers_pl, '>', 'Maintainers.pl';
470
471my $found;
472my $in_mod_section;
473while (<$Maintainers_pl>) {
474 if (!$found) {
475 if ($in_mod_section) {
476 if (/DISTRIBUTION/) {
33c6567b 477 if (s/\Q$old_version/$new_version/) {
9807c17b
JL
478 $found = 1;
479 }
480 }
481
482 if (/^ }/) {
483 $in_mod_section = 0;
484 }
485 }
486
487 if (/\Q$cpan_mod/) {
488 $in_mod_section = 1;
489 }
490 }
491
492 print $new_Maintainers_pl $_;
493}
494
495if ($found) {
496 unlink 'Porting/Maintainers.pl';
497 rename 'Maintainers.pl' => 'Porting/Maintainers.pl';
498 system chmod => 'a+x', 'Porting/Maintainers.pl';
499}
500else {
501 say "Could not update Porting/Maintainers.pl.";
502 say "Make sure you update this by hand before committing.";
503}
418f4069 504
418f4069 505say "$o_module is now version $new_version";
9807c17b 506say "Now you ought to run a make; make test ...";
418f4069
A
507
508
509__END__