Add =encoding utf8 to all core non-ASCII POD files + test script

It's now possible to run:

    perl Porting/checkpodencoding.pl

To check if the core contains any naughty POD that uses non-ASCII
without declaring an encoding.

With this patch all the POD in core (except POD tests we're ignoring)
has a correct =encoding directive. The script also flags problems in
lib/* and cpan/* which aren't being fixed as part of this commit.

diff --git a/Porting/checkpodencoding.pl b/Porting/checkpodencoding.pl
new file mode 100644 (file)
index 0000000..a2d12df
--- /dev/null
+++ b/Porting/checkpodencoding.pl
@@ -0,0 +1,69 @@
+#!/usr/bin/env perl
+use 5.010;
+use open qw< :encoding(utf8) :std >;
+use autodie;
+use strict;
+use File::Find;
+use Encode::Guess;
+
+# Check if POD files contain non-ASCII without specifying
+# =encoding. Run it as:
+
+## perl Porting/checkpodencoding.pl
+
+find(
+    {
+        wanted => \&finder,
+        no_chdir => 1,
+    },
+    '.'
+);
+
+sub finder {
+    my $file = $_;
+
+    return if -B $file;
+
+    open my $fh, '<', $file;
+
+    #say STDERR "Checking $file";
+
+    next if
+        # Test cases
+        $file ~~ m[Pod-Simple/t];
+
+    my ($in_pod, $has_encoding, @non_ascii);
+
+    FILE: while (my $line = <$fh>) {
+        chomp $line;
+        if ($line ~~ /^=[a-z]+/) {
+            $in_pod = 1;
+        }
+
+        if ($in_pod) {
+            if ($line ~~ /^=encoding (\S+)/) {
+                $has_encoding = 1;
+                last FILE;
+            } elsif ($line ~~ /[^[:ascii:]]/) {
+                my $encoding = guess_encoding($line);
+                push @non_ascii => {
+                    num => $.,
+                    line => $line,
+                    encoding => (ref $encoding ? "$encoding->{Name}?" : 'unknown!'),
+                };
+            }
+        }
+
+        if ($line ~~ /^=cut/) {
+            $in_pod = 0;
+        }
+    }
+
+    if (@non_ascii and not $has_encoding) {
+        say "$file:";
+        $DB::single = 1;
+        for (@non_ascii) {
+            say "    $_->{num} ($_->{encoding}): $_->{line}";
+        }
+    }
+}

diff --git a/Porting/release_managers_guide.pod b/Porting/release_managers_guide.pod
index a07268a..d552650 100644 (file)
--- a/Porting/release_managers_guide.pod
+++ b/Porting/release_managers_guide.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 NAME
 
 release_managers_guide - Releasing a new version of perl 5.x

diff --git a/Porting/release_schedule.pod b/Porting/release_schedule.pod
index 45a6fcc..c0b74f4 100644 (file)
--- a/Porting/release_schedule.pod
+++ b/Porting/release_schedule.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 Release schedule
 
 This document lists the release engineers for at least the next

diff --git a/pod/perl588delta.pod b/pod/perl588delta.pod
index a3d1df3..b2203bc 100644 (file)
--- a/pod/perl588delta.pod
+++ b/pod/perl588delta.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 NAME
 
 perl588delta - what is new for perl v5.8.8

diff --git a/pod/perl593delta.pod b/pod/perl593delta.pod
index 6c8587a..11f53ab 100644 (file)
--- a/pod/perl593delta.pod
+++ b/pod/perl593delta.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 NAME
 
 perl593delta - what is new for perl v5.9.3

diff --git a/pod/perlebcdic.pod b/pod/perlebcdic.pod
index f178912..7c1082d 100644 (file)
--- a/pod/perlebcdic.pod
+++ b/pod/perlebcdic.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 NAME
 
 perlebcdic - Considerations for running Perl on EBCDIC platforms

diff --git a/pod/perlhist.pod b/pod/perlhist.pod
index 2f74127..b3ed001 100644 (file)
--- a/pod/perlhist.pod
+++ b/pod/perlhist.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 NAME
 
 perlhist - the Perl history records

diff --git a/pod/perlpodspec.pod b/pod/perlpodspec.pod
index 0bf84e0..dbe0539 100644 (file)
--- a/pod/perlpodspec.pod
+++ b/pod/perlpodspec.pod
@@ -1,3 +1,4 @@
+=encoding utf8
 
 =head1 NAME
 

diff --git a/pod/perlrepository.pod b/pod/perlrepository.pod
index feccf1a..e7ef58a 100644 (file)
--- a/pod/perlrepository.pod
+++ b/pod/perlrepository.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =for comment
 Consistent formatting of this file is achieved with:
   perl ./Porting/podtidy pod/perlrepository.pod

diff --git a/pod/perlthrtut.pod b/pod/perlthrtut.pod
index 18a6f46..fb947b6 100644 (file)
--- a/pod/perlthrtut.pod
+++ b/pod/perlthrtut.pod
@@ -1,3 +1,5 @@
+=encoding utf8
+
 =head1 NAME
 
 perlthrtut - Tutorial on threads in Perl

diff --git a/symbian/TODO b/symbian/TODO
index 3fb1534..ac8ee3d 100644 (file)
--- a/symbian/TODO
+++ b/symbian/TODO
@@ -1,9 +1,11 @@
+=encoding utf8
+
 =head1 BASE PORT
 
 =head2 Console
 
 - The Console only does "ASCII" input: e.g. pressing the "2"
-  key five times, "aaaaa", does not produce "ä" ("a diaeresis"),
+  key five times, "aaaaa", does not produce "ä" ("a diaeresis"),
   but instead the "2" key rotates through "abc2abc2...".
   This is a pity because the Console is actually capable of full
   Unicode input and output (if you have the fonts, that is).  You