[perl #124113] Make check for multi-dimensional arrays be UTF8-aware

During parsing, toke.c checks if the user is attempting provide multiple
indexes to an array index:

    $a[ $foo, $bar ];

However, while checking for word characters in variable names is aware
of multi-byte characters if "use utf8" is enabled, the loop is only
advanced one byte at a time, not one character at a time.  As such,
multibyte variables in array indexes incorrectly yield warnings:

    Passing malformed UTF-8 to "XPosixWord" is deprecated
    Malformed UTF-8 character (unexpected continuation byte 0x9d, with
      no preceding start byte)

Switch the loop to advance character-by-character if UTF-8 semantics are
in use.

diff --git a/t/lib/warnings/toke b/t/lib/warnings/toke
index 5d31104..018f188 100644 (file)
--- a/t/lib/warnings/toke
+++ b/t/lib/warnings/toke
@@ -1521,3 +1521,13 @@ Use of literal control characters in variable names is deprecated at (eval 2) li
 -a;
 ;-a;
 EXPECT
+########
+# toke.c
+# [perl #124113] Compile-time warning with UTF8 variable in array index
+use warnings;
+use utf8;
+my $𝛃 = 0;
+my @array = (0);
+my $v = $array[ 0 + $𝛃 ];
+   $v = $array[ $𝛃 + 0 ];
+EXPECT

diff --git a/toke.c b/toke.c
index ddc2431..56c5186 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -6049,7 +6049,7 @@ Perl_yylex(pTHX)
                        char *t = s+1;
 
                        while (isSPACE(*t) || isWORDCHAR_lazy_if(t,UTF) || *t == '$')
-                           t++;
+                           t += UTF ? UTF8SKIP(t) : 1;
                        if (*t++ == ',') {
                            PL_bufptr = skipspace(PL_bufptr); /* XXX can realloc */
                            while (t < PL_bufend && *t != ']')