PATCH: [perl #112208]: Set utf8 flag on $! appropriately

This patch sets the utf8 flag on $! if the error string passes utf8
validity tests and has some bytes with the upper bit set.  (If none
have that bit set, is an ASCII string, and whether or not it is UTF-8 is
irrelevant.)  This is a heuristic that could fail, but as the reference
in the comments points out this is unlikely.

One can reasonably assume that a UTF-8 locale will return a UTF-8
result.  So another approach would be to look at that (but we wouldn't
want to turn the flag on for a purely ASCII string anyway, as that could
change the semantics from existing behavior by making the string follow
Unicode rules, whereas it didn't necessarily before.)  To do this, we
could keep track of the utf8ness of the LC_MESSAGES locale.  But until
the heuristic in this patch is shown to not be good enough, I don't see
the need to do this extra work.

diff --git a/lib/locale.t b/lib/locale.t
index 96f9045..79c2fd9 100644 (file)
--- a/lib/locale.t
+++ b/lib/locale.t
@@ -1014,6 +1014,7 @@ foreach $Locale (@Locale) {
     my $ok11;
     my $ok12;
     my $ok13;
+    my $ok14;
 
     my $c;
     my $d;
@@ -1069,6 +1070,7 @@ foreach $Locale (@Locale) {
             $ok11 = $f == $c;
             $ok12 = abs(($f + $g) - 3.57) < 0.01;
             $ok13 = $w == 0;
+            $ok14 = 1;  # Skip for non-utf8 locales
         }
     }
     else {
@@ -1112,6 +1114,26 @@ foreach $Locale (@Locale) {
             $ok11 = $f == $c;
             $ok12 = abs(($f + $g) - 3.57) < 0.01;
             $ok13 = $w == 0;
+
+            # Look for non-ASCII error messages, and verify that the first
+            # such is in UTF-8 (the others almost certainly will be like the
+            # first).
+            $ok14 = 1;
+            foreach my $err (keys %!) {
+                use Errno;
+                $! = eval "&Errno::$err";   # Convert to strerror() output
+                my $strerror = "$!";
+                if ("$strerror" =~ /\P{ASCII}/) {
+                    my $utf8_strerror = $strerror;
+                    utf8::upgrade($utf8_strerror);
+
+                    # If $! was already in UTF-8, the upgrade was a no-op;
+                    # otherwise they will be different byte strings.
+                    use bytes;
+                    $ok14 = $utf8_strerror eq $strerror;
+                    last;
+                }
+            }
         }
     }
 
@@ -1165,6 +1187,9 @@ foreach $Locale (@Locale) {
     tryneoalpha($Locale, ++$locales_test_number, $ok13);
     $test_names{$locales_test_number} = 'Verify that don\'t get warning under "==" even if radix is not a dot';
 
+    tryneoalpha($Locale, ++$locales_test_number, $ok14);
+    $test_names{$locales_test_number} = 'Verify that non-ASCII UTF-8 error messages are in UTF-8';
+
     debug "# $first_f_test..$locales_test_number: \$f = $f, \$g = $g, back to locale = $Locale\n";
 
     # Does taking lc separately differ from taking

diff --git a/mg.c b/mg.c
index 7ff78c1..518d108 100644 (file)
--- a/mg.c
+++ b/mg.c
@@ -1043,7 +1043,35 @@ Perl_magic_get(pTHX_ SV *sv, MAGIC *mg)
            sv_setpv(sv, os2error(Perl_rc));
        else
 #endif
-       sv_setpv(sv, errno ? Strerror(errno) : "");
+       if (! errno) {
+            sv_setpvs(sv, "");
+        }
+        else {
+
+            /* Strerror can return NULL on some platforms, which will result in
+             * 'sv' not being considered SvOK.  The SvNOK_on() below will cause
+             * just the number part to be valid */
+            sv_setpv(sv, Strerror(errno));
+
+            /* In some locales the error string may come back as UTF-8, in
+             * which case we should turn on that flag.  This didn't use to
+             * happen, and to avoid any possible backward compatibility issues,
+             * we don't turn on the flag unless we have to.  So the flag stays
+             * off for an entirely ASCII string.  We assume that if the string
+             * looks like UTF-8, it really is UTF-8:  "text in any other
+             * encoding that uses bytes with the high bit set is extremely
+             * unlikely to pass a UTF-8 validity test"
+             * (http://en.wikipedia.org/wiki/Charset_detection).  There is a
+             * potential that we will get it wrong however, especially on short
+             * error message text.  (If it turns out to be necessary, we could
+             * also keep track if the current LC_MESSAGES locale is UTF-8) */
+            if (SvOK(sv)    /* It could be that Strerror returned invalid */
+                && ! is_ascii_string((U8*) SvPVX_const(sv), SvCUR(sv))
+                && is_utf8_string((U8*) SvPVX_const(sv), SvCUR(sv)))
+            {
+                SvUTF8_on(sv);
+            }
+        }
        RESTORE_ERRNO;
        }