Fix bug where charnames xlator doesn't return utf8

A user-supplied charnames translator for \N{} processing doesn't have to
return in UTF-8.  Prior to this patch, the code in toke.c assumed it
did.

diff --git a/t/re/pat_advanced.t b/t/re/pat_advanced.t
index cb48060..5296aeb 100644 (file)
--- a/t/re/pat_advanced.t
+++ b/t/re/pat_advanced.t
@@ -1036,6 +1036,19 @@ sub run_tests {
         }
 
         undef $w;
+        my $Cedilla_Latin1 = "GAR"
+                           . latin1_to_native("\xC7")
+                           . "ON";
+        my $Cedilla_utf8 = $Cedilla_Latin1;
+        utf8::upgrade($Cedilla_utf8);
+        eval qq[is("\\N{$Cedilla_Latin1}", "$Cedilla_Latin1", "A cedilla in character name works")];
+        undef $w;
+            {
+            use feature 'unicode_eval';
+            eval qq[use utf8; is("\\N{$Cedilla_utf8}", "$Cedilla_utf8", "... same under 'use utf8': they work")];
+        }
+
+        undef $w;
         {
             BEGIN { no strict; *CnameTest:: = *{"_charnames\0A::" } }
             package CnameTest { sub translator { pop } }

diff --git a/toke.c b/toke.c
index 46b5e34..fa13514 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -3766,6 +3766,10 @@ S_scan_const(pTHX_ char *start)
                            const STRLEN off = d - SvPVX_const(sv);
                            d = off + SvGROW(sv, off + len + (STRLEN)(send - s) + 1);
                        }
+                        if (! SvUTF8(res)) {    /* Make sure is \N{} return is UTF-8 */
+                            sv_utf8_upgrade(res);
+                            str = SvPV_const(res, len);
+                        }
                        Copy(str, d, len, char);
                        d += len;
                    }