regen/regcharclass.pl: Work on early Unicodes

[perl5.git] / perly.c

diff --git a/perly.c b/perly.c
index 44d6ce6..abb4d4e 100644 (file)
--- a/perly.c
+++ b/perly.c
@@ -318,17 +318,8 @@ Perl_yyparse (pTHX_ int gramtype)
 
     /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol.  */
     if (parser->yychar == YYEMPTY) {
-       YYDPRINTF ((Perl_debug_log, "Reading a token: "));
+       YYDPRINTF ((Perl_debug_log, "Reading a token:\n"));
        parser->yychar = yylex();
-
-/* perly.tab is shipped based on an ASCII system; if it were to be regenerated
- * on a platform that doesn't use ASCII, this translation back would need to be
- * removed */
-#  ifdef EBCDIC
-       if (parser->yychar >= 0) {
-           parser->yychar = NATIVE_TO_UNI(parser->yychar);
-       }
-#  endif
     }
 
     if (parser->yychar <= YYEOF) {
@@ -336,7 +327,16 @@ Perl_yyparse (pTHX_ int gramtype)
        YYDPRINTF ((Perl_debug_log, "Now at end of input.\n"));
     }
     else {
-       yytoken = YYTRANSLATE (parser->yychar);
+        /* perly.tab is shipped based on an ASCII system, so need to index it
+         * with characters translated to ASCII.  Although it's not designed for
+         * this purpose, we can use NATIVE_TO_UNI here.  It returns its
+         * argument on ASCII platforms, and on EBCDIC translates native to
+         * ascii in the 0-255 range, leaving everything else unchanged.  This
+         * jibes with yylex() returning some bare characters in that range, but
+         * all tokens it returns are either 0, or above 255.  There could be a
+         * problem if NULs weren't 0, or were ever returned as raw chars by
+         * yylex() */
+        yytoken = YYTRANSLATE (NATIVE_TO_UNI(parser->yychar));
        YYDSYMPRINTF ("Next token is", yytoken, &parser->yylval);
     }
 
@@ -577,11 +577,5 @@ Perl_yyparse (pTHX_ int gramtype)
 }
 
 /*
- * Local variables:
- * c-indentation-style: bsd
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- *
  * ex: set ts=8 sts=4 sw=4 et:
  */