This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
merge perly.y and madlu.y
[perl5.git] / perly.y
diff --git a/perly.y b/perly.y
index 77caeb6..f5b4203 100644 (file)
--- a/perly.y
+++ b/perly.y
@@ -1,6 +1,6 @@
 /*    perly.y
  *
- *    Copyright (c) 1991-2002, 2003, 2004 Larry Wall
+ *    Copyright (c) 1991-2002, 2003, 2004, 2005, 2006 Larry Wall
  *
  *    You may distribute under the terms of either the GNU General Public
  *    License or the Artistic License, as specified in the README file.
 /*
  * 'I see,' laughed Strider.  'I look foul and feel fair.  Is that it?
  * All that is gold does not glitter, not all those who wander are lost.'
- */
-
-/* This file holds the grammar for the Perl language. If edited, you need
+ *
+ * This file holds the grammar for the Perl language. If edited, you need
  * to run regen_perly.pl, which re-creates the files perly.h, perly.tab
  * and perly.act which are derived from this.
  *
+ * Note that these derived files are included and compiled twice; once
+ * from perly.c, and once from madly.c. The second time, a number of MAD
+ * macros are defined, which compile in extra code that allows the parse
+ * tree to be accurately dumped. In particular:
+ *
+ * MAD            defined if compiling madly.c
+ * DO_MAD(A)      expands to A  under madly.c, to null otherwise
+ * IF_MAD(a,b)    expands to A under madly.c, to B otherwise
+ * TOKEN_GETMAD() expands to token_getmad() under madly.c, to null otherwise
+ * TOKEN_FREE()   similarly
+ * OP_GETMAD()    similarly
+ * IVAL(i)        expands to (i)->tk_lval.ival or (i)
+ * PVAL(p)        expands to (p)->tk_lval.pval or (p)
+ *
  * The main job of of this grammar is to call the various newFOO()
  * functions in op.c to build a syntax tree of OP structs.
  * It relies on the lexer in toke.c to do the tokenizing.
 
 %pure_parser
 
+/* FIXME for MAD - is the new mintro on while and until important?  */
+
 %start prog
 
 %union {
     I32        ival;
     char *pval;
-#ifdef PERL_MAD
-    TOKEN* tkval;
-#endif
     OP *opval;
     GV *gvval;
+#ifdef PERL_IN_MADLY_C
+    TOKEN* p_tkval;
+    TOKEN* tkval;
+#else
+    char *p_tkval;
+    I32        tkval;
+#endif
 }
 
-%token <ival> '{'
+%token <tkval> '{' '}' '[' ']' '-' '+' '$' '@' '%' '*' '&' ';'
 
 %token <opval> WORD METHOD FUNCMETH THING PMFUNC PRIVATEREF
 %token <opval> FUNC0SUB UNIOPSUB LSTOPSUB
-%token <pval> LABEL
-%token <ival> FORMAT SUB ANONSUB PACKAGE USE
-%token <ival> WHILE UNTIL IF UNLESS ELSE ELSIF CONTINUE FOR
-%token <ival> GIVEN WHEN DEFAULT
-%token <ival> LOOPEX DOTDOT
-%token <ival> FUNC0 FUNC1 FUNC UNIOP LSTOP
-%token <ival> RELOP EQOP MULOP ADDOP
-%token <ival> DOLSHARP DO HASHBRACK NOAMP
-%token <ival> LOCAL MY MYSUB REQUIRE
-%token COLONATTR
-
-%type <ival> prog decl format startsub startanonsub startformsub mintro
-%type <ival> progstart remember mremember '&' savescope mydefsv
+%token <p_tkval> LABEL
+%token <tkval> FORMAT SUB ANONSUB PACKAGE USE
+%token <tkval> WHILE UNTIL IF UNLESS ELSE ELSIF CONTINUE FOR
+%token <tkval> GIVEN WHEN DEFAULT
+%token <tkval> LOOPEX DOTDOT
+%token <tkval> FUNC0 FUNC1 FUNC UNIOP LSTOP
+%token <tkval> RELOP EQOP MULOP ADDOP
+%token <tkval> DOLSHARP DO HASHBRACK NOAMP
+%token <tkval> LOCAL MY MYSUB REQUIRE
+%token <tkval> COLONATTR
+
+%type <ival> prog progstart remember mremember savescope
+%type <ival>  startsub startanonsub startformsub
+/* FIXME for MAD - are these two ival? */
+%type <ival> mydefsv mintro
+
+%type <opval> decl format subrout mysubrout package use peg
+
 %type <opval> block mblock lineseq line loop cond else
 %type <opval> expr term subscripted scalar ary hsh arylen star amper sideff
 %type <opval> argexpr nexpr texpr iexpr mexpr mnexpr miexpr
 %type <opval> listexpr listexprcom indirob listop method
 %type <opval> formname subname proto subbody cont my_scalar
-%type <opval> subattrlist myattrlist mysubrout myattrterm myterm
+%type <opval> subattrlist myattrlist myattrterm myterm
 %type <opval> termbinop termunop anonymous termdo
 %type <opval> switch case
-%type <pval> label
+%type <p_tkval> label
 
-%nonassoc PREC_LOW
+%nonassoc <tkval> PREC_LOW
 %nonassoc LOOPEX
 
-%left <ival> OROP DOROP
-%left ANDOP
-%right NOTOP
+%left <tkval> OROP DOROP
+%left <tkval> ANDOP
+%right <tkval> NOTOP
 %nonassoc LSTOP LSTOPSUB
-%left ','
-%right <ival> ASSIGNOP
-%right '?' ':'
+%left <tkval> ','
+%right <tkval> ASSIGNOP
+%right <tkval> '?' ':'
 %nonassoc DOTDOT
-%left OROR DORDOR
-%left ANDAND
-%left <ival> BITOROP
-%left <ival> BITANDOP
+%left <tkval> OROR DORDOR
+%left <tkval> ANDAND
+%left <tkval> BITOROP
+%left <tkval> BITANDOP
 %nonassoc EQOP
 %nonassoc RELOP
 %nonassoc UNIOP UNIOPSUB
 %nonassoc REQUIRE
-%left <ival> SHIFTOP
+%left <tkval> SHIFTOP
 %left ADDOP
 %left MULOP
-%left <ival> MATCHOP
-%right '!' '~' UMINUS REFGEN
-%right <ival> POWOP
-%nonassoc PREINC PREDEC POSTINC POSTDEC
-%left ARROW
-%nonassoc <ival> ')'
-%left '('
+%left <tkval> MATCHOP
+%right <tkval> '!' '~' UMINUS REFGEN
+%right <tkval> POWOP
+%nonassoc <tkval> PREINC PREDEC POSTINC POSTDEC
+%left <tkval> ARROW
+%nonassoc <tkval> ')'
+%left <tkval> '('
 %left '[' '{'
 
-%token PEG
+%token <tkval> PEG
 
 %% /* RULES */
 
@@ -107,9 +131,12 @@ prog       :       progstart
 
 /* An ordinary block */
 block  :       '{' remember lineseq '}'
-                       { if (PL_copline > (line_t)$1)
-                             PL_copline = (line_t)$1;
-                         $$ = block_end($2, $3); }
+                       { if (PL_copline > (line_t)IVAL($1))
+                             PL_copline = (line_t)IVAL($1);
+                         $$ = block_end($2, $3);
+                         TOKEN_GETMAD($1,$$,'{');
+                         TOKEN_GETMAD($4,$$,'}');
+                       }
        ;
 
 remember:      /* NULL */      /* start a full lexical scope */
@@ -128,9 +155,12 @@ progstart:
 
 
 mblock :       '{' mremember lineseq '}'
-                       { if (PL_copline > (line_t)$1)
-                             PL_copline = (line_t)$1;
-                         $$ = block_end($2, $3); }
+                       { if (PL_copline > (line_t)IVAL($1))
+                             PL_copline = (line_t)IVAL($1);
+                         $$ = block_end($2, $3);
+                         TOKEN_GETMAD($1,$$,'{');
+                         TOKEN_GETMAD($4,$$,'}');
+                       }
        ;
 
 mremember:     /* NULL */      /* start a partial lexical scope */
@@ -144,7 +174,12 @@ savescope: /* NULL */      /* remember stack pos in case of error */
 lineseq        :       /* NULL */
                        { $$ = Nullop; }
        |       lineseq decl
-                       { $$ = $1; }
+                       {
+                       $$ = IF_MAD(
+                               append_list(OP_LINESEQ,
+                                   (LISTOP*)$1, (LISTOP*)$2),
+                               $1);
+                       }
        |       lineseq savescope line
                        {   LEAVE_SCOPE($2);
                            $$ = append_list(OP_LINESEQ,
@@ -155,24 +190,46 @@ lineseq   :       /* NULL */
 
 /* A "line" in the program */
 line   :       label cond
-                       { $$ = newSTATEOP(0, $1, $2); }
+                       { $$ = newSTATEOP(0, PVAL($1), $2);
+                         TOKEN_GETMAD($1,((LISTOP*)$$)->op_first,'L'); }
        |       loop    /* loops add their own labels */
        |       switch  /* ... and so do switches */
                        { $$ = $1; }
        |       label case
-                       { $$ = newSTATEOP(0, $1, $2); }
+                       { $$ = newSTATEOP(0, PVAL($1), $2); }
        |       label ';'
-                       { if ($1 != Nullch) {
-                             $$ = newSTATEOP(0, $1, newOP(OP_NULL, 0));
-                           }
-                           else {
-                             $$ = Nullop;
-                             PL_copline = NOLINE;
-                           }
-                           PL_expect = XSTATE; }
+                       {
+                         if (PVAL($1)) {
+                             $$ = newSTATEOP(0, PVAL($1), newOP(OP_NULL, 0));
+                             TOKEN_GETMAD($1,$$,'L');
+                             TOKEN_GETMAD($2,((LISTOP*)$$)->op_first,';');
+                         }
+                         else {
+                             $$ = IF_MAD(
+                                       newOP(OP_NULL, 0),
+                                       Nullop);
+                              PL_copline = NOLINE;
+                             TOKEN_FREE($1);
+                             TOKEN_GETMAD($2,$$,';');
+                         }
+                         PL_expect = XSTATE;
+                       }
        |       label sideff ';'
-                       { $$ = newSTATEOP(0, $1, $2);
-                         PL_expect = XSTATE; }
+                       {
+                         $$ = newSTATEOP(0, PVAL($1), $2);
+                         PL_expect = XSTATE;
+                         DO_MAD(
+                             /* sideff might already have a nexstate */
+                             OP* op = ((LISTOP*)$$)->op_first;
+                             if (op) {
+                                 while (op->op_sibling &&
+                                    op->op_sibling->op_type == OP_NEXTSTATE)
+                                       op = op->op_sibling;
+                                 token_getmad($1,op,'L');
+                                 token_getmad($3,op,';');
+                             }
+                         )
+                       }
        ;
 
 /* An expression which may have a side-effect */
@@ -181,38 +238,62 @@ sideff    :       error
        |       expr
                        { $$ = $1; }
        |       expr IF expr
-                       { $$ = newLOGOP(OP_AND, 0, $3, $1); }
+                       { $$ = newLOGOP(OP_AND, 0, $3, $1);
+                         TOKEN_GETMAD($2,$$,'i');
+                       }
        |       expr UNLESS expr
-                       { $$ = newLOGOP(OP_OR, 0, $3, $1); }
+                       { $$ = newLOGOP(OP_OR, 0, $3, $1);
+                         TOKEN_GETMAD($2,$$,'i');
+                       }
        |       expr WHILE expr
-                       { $$ = newLOOPOP(OPf_PARENS, 1, scalar($3), $1); }
+                       { $$ = newLOOPOP(OPf_PARENS, 1, scalar($3), $1);
+                         TOKEN_GETMAD($2,$$,'w');
+                       }
        |       expr UNTIL iexpr
-                       { $$ = newLOOPOP(OPf_PARENS, 1, $3, $1);}
+                       { $$ = newLOOPOP(OPf_PARENS, 1, $3, $1);
+                         TOKEN_GETMAD($2,$$,'w');
+                       }
        |       expr FOR expr
-                       { $$ = newFOROP(0, Nullch, (line_t)$2,
-                                       Nullop, $3, $1, Nullop); }
+                       { $$ = newFOROP(0, Nullch, (line_t)IVAL($2),
+                                       Nullop, $3, $1, Nullop);
+                         TOKEN_GETMAD($2,((LISTOP*)$$)->op_first->op_sibling,'w');
+                       }
        ;
 
 /* else and elsif blocks */
 else   :       /* NULL */
                        { $$ = Nullop; }
        |       ELSE mblock
-                       { ($2)->op_flags |= OPf_PARENS; $$ = scope($2); }
+                       { ($2)->op_flags |= OPf_PARENS; $$ = scope($2);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       ELSIF '(' mexpr ')' mblock else
-                       { PL_copline = (line_t)$1;
+                       { PL_copline = (line_t)IVAL($1);
                            $$ = newCONDOP(0, $3, scope($5), $6);
-                           PL_hints |= HINT_BLOCK_SCOPE; }
+                           PL_hints |= HINT_BLOCK_SCOPE;
+                         TOKEN_GETMAD($1,$$,'I');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        ;
 
 /* Real conditional expressions */
 cond   :       IF '(' remember mexpr ')' mblock else
-                       { PL_copline = (line_t)$1;
+                       { PL_copline = (line_t)IVAL($1);
                            $$ = block_end($3,
-                                  newCONDOP(0, $4, scope($6), $7)); }
+                                  newCONDOP(0, $4, scope($6), $7));
+                         TOKEN_GETMAD($1,$$,'I');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
        |       UNLESS '(' remember miexpr ')' mblock else
-                       { PL_copline = (line_t)$1;
+                       { PL_copline = (line_t)IVAL($1);
                            $$ = block_end($3,
-                                  newCONDOP(0, $4, scope($6), $7)); }
+                                  newCONDOP(0, $4, scope($6), $7));
+                         TOKEN_GETMAD($1,$$,'I');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
        ;
 
 /* Cases for a switch statement */
@@ -227,41 +308,94 @@ case      :       WHEN '(' remember mexpr ')' mblock
 cont   :       /* NULL */
                        { $$ = Nullop; }
        |       CONTINUE block
-                       { $$ = scope($2); }
+                       { $$ = scope($2);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        ;
 
 /* Loops: while, until, for, and a bare block */
 loop   :       label WHILE '(' remember texpr ')' mintro mblock cont
-                       { PL_copline = (line_t)$2;
+                       { OP *innerop;
+                         PL_copline = (line_t)$2;
                            $$ = block_end($4,
-                                  newSTATEOP(0, $1,
-                                    newWHILEOP(0, 1, (LOOP*)Nullop,
-                                               $2, $5, $8, $9, $7))); }
+                                  newSTATEOP(0, PVAL($1),
+                                    innerop = newWHILEOP(0, 1, (LOOP*)Nullop,
+                                               IVAL($2), $5, $8, $9, $7)));
+                         TOKEN_GETMAD($1,innerop,'L');
+                         TOKEN_GETMAD($2,innerop,'W');
+                         TOKEN_GETMAD($3,innerop,'(');
+                         TOKEN_GETMAD($6,innerop,')');
+                       }
+
        |       label UNTIL '(' remember iexpr ')' mintro mblock cont
-                       { PL_copline = (line_t)$2;
+                       { OP *innerop;
+                         PL_copline = (line_t)$2;
                            $$ = block_end($4,
-                                  newSTATEOP(0, $1,
-                                    newWHILEOP(0, 1, (LOOP*)Nullop,
-                                               $2, $5, $8, $9, $7))); }
+                                  newSTATEOP(0, PVAL($1),
+                                    innerop = newWHILEOP(0, 1, (LOOP*)Nullop,
+                                               IVAL($2), $5, $8, $9, $7)));
+                         TOKEN_GETMAD($1,innerop,'L');
+                         TOKEN_GETMAD($2,innerop,'W');
+                         TOKEN_GETMAD($3,innerop,'(');
+                         TOKEN_GETMAD($6,innerop,')');
+                       }
        |       label FOR MY remember my_scalar '(' mexpr ')' mblock cont
-                       { $$ = block_end($4,
-                                newFOROP(0, $1, (line_t)$2, $5, $7, $9, $10)); }
+                       { OP *innerop;
+                         $$ = block_end($4,
+                            innerop = newFOROP(0, PVAL($1), (line_t)IVAL($2),
+                                           $5, $7, $9, $10));
+                         TOKEN_GETMAD($1,((LISTOP*)innerop)->op_first,'L');
+                         TOKEN_GETMAD($2,((LISTOP*)innerop)->op_first->op_sibling,'W');
+                         TOKEN_GETMAD($3,((LISTOP*)innerop)->op_first->op_sibling,'d');
+                         TOKEN_GETMAD($6,((LISTOP*)innerop)->op_first->op_sibling,'(');
+                         TOKEN_GETMAD($8,((LISTOP*)innerop)->op_first->op_sibling,')');
+                       }
        |       label FOR scalar '(' remember mexpr ')' mblock cont
-                       { $$ = block_end($5,
-                                newFOROP(0, $1, (line_t)$2, mod($3, OP_ENTERLOOP),
-                                         $6, $8, $9)); }
+                       { OP *innerop;
+                         $$ = block_end($5,
+                            innerop = newFOROP(0, PVAL($1), (line_t)IVAL($2),
+                                   mod($3, OP_ENTERLOOP), $6, $8, $9));
+                         TOKEN_GETMAD($1,((LISTOP*)innerop)->op_first,'L');
+                         TOKEN_GETMAD($2,((LISTOP*)innerop)->op_first->op_sibling,'W');
+                         TOKEN_GETMAD($4,((LISTOP*)innerop)->op_first->op_sibling,'(');
+                         TOKEN_GETMAD($7,((LISTOP*)innerop)->op_first->op_sibling,')');
+                       }
        |       label FOR '(' remember mexpr ')' mblock cont
-                       { $$ = block_end($4,
-                                newFOROP(0, $1, (line_t)$2, Nullop, $5, $7, $8)); }
+                       { OP *innerop;
+                         $$ = block_end($4,
+                            innerop = newFOROP(0, PVAL($1), (line_t)IVAL($2),
+                                                   Nullop, $5, $7, $8));
+                         TOKEN_GETMAD($1,((LISTOP*)innerop)->op_first,'L');
+                         TOKEN_GETMAD($2,((LISTOP*)innerop)->op_first->op_sibling,'W');
+                         TOKEN_GETMAD($3,((LISTOP*)innerop)->op_first->op_sibling,'(');
+                         TOKEN_GETMAD($6,((LISTOP*)innerop)->op_first->op_sibling,')');
+                       }
        |       label FOR '(' remember mnexpr ';' texpr ';' mintro mnexpr ')'
                    mblock
                        /* basically fake up an initialize-while lineseq */
                        { OP *forop;
-                         PL_copline = (line_t)$2;
-                         forop = newSTATEOP(0, $1,
+                         PL_copline = (line_t)IVAL($2);
+                         forop = newSTATEOP(0, PVAL($1),
                                            newWHILEOP(0, 1, (LOOP*)Nullop,
-                                               $2, scalar($7),
+                                               IVAL($2), scalar($7),
                                                $12, $10, $9));
+#ifdef MAD
+                         if (!$5)
+                               $5 = newOP(OP_NULL, 0);
+                         forop = newUNOP(OP_NULL, 0, append_elem(OP_LINESEQ,
+                               newSTATEOP(0,
+                                          (($1)->tk_lval.pval
+                                          ?savepv(($1)->tk_lval.pval):Nullch),
+                                          $5),
+                               forop));
+
+                         token_getmad($2,forop,'3');
+                         token_getmad($3,forop,'(');
+                         token_getmad($6,forop,'1');
+                         token_getmad($8,forop,'2');
+                         token_getmad($11,forop,')');
+                         token_getmad($1,forop,'L');
+#else
                          if ($5) {
                                forop = append_elem(OP_LINESEQ,
                                         newSTATEOP(0, ($1?savepv($1):Nullch),
@@ -269,18 +403,21 @@ loop      :       label WHILE '(' remember texpr ')' mintro mblock cont
                                        forop);
                          }
 
+
+#endif
                          $$ = block_end($4, forop); }
        |       label block cont  /* a block is a loop that happens once */
-                       { $$ = newSTATEOP(0, $1,
+                       { $$ = newSTATEOP(0, PVAL($1),
                                 newWHILEOP(0, 1, (LOOP*)Nullop,
-                                           NOLINE, Nullop, $2, $3, 0)); }
+                                           NOLINE, Nullop, $2, $3, 0));
+                         TOKEN_GETMAD($1,((LISTOP*)$$)->op_first,'L'); }
        ;
 
 /* Switch blocks */
 switch :       label GIVEN '(' remember mydefsv mexpr ')' mblock
                        { PL_copline = (line_t) $2;
                            $$ = block_end($4,
-                               newSTATEOP(0, $1,
+                               newSTATEOP(0, PVAL($1),
                                    newGIVENOP($6, scope($8),
                                        (PADOFFSET) $5) )); }
        ;
@@ -299,7 +436,9 @@ nexpr       :       /* NULL */
 
 /* Boolean expression */
 texpr  :       /* NULL means true */
-                       { (void)scan_num("1", &yylval); $$ = yylval.opval; }
+                       { YYSTYPE tmplval;
+                         (void)scan_num("1", &tmplval);
+                         $$ = tmplval.opval; }
        |       expr
        ;
 
@@ -323,26 +462,53 @@ miexpr    :       iexpr
 
 /* Optional "MAIN:"-style loop labels */
 label  :       /* empty */
-                       { $$ = Nullch; }
+                       {
+#ifdef MAD
+                         YYSTYPE tmplval;
+                         tmplval.pval = Nullch;
+                         $$ = newTOKEN(OP_NULL, tmplval, 0);
+#else
+                         $$ = Nullch;
+#endif
+                       }
        |       LABEL
        ;
 
-/* Some kind of declaration - does not take part in the parse tree */
+/* Some kind of declaration - just hang on peg in the parse tree */
 decl   :       format
-                       { $$ = 0; }
+                       { $$ = $1; }
        |       subrout
-                       { $$ = 0; }
+                       { $$ = $1; }
        |       mysubrout
-                       { $$ = 0; }
+                       { $$ = $1; }
        |       package
-                       { $$ = 0; }
+                       { $$ = $1; }
        |       use
-                       { $$ = 0; }
+                       { $$ = $1; }
+
+    /* these two are only used by MAD */
+
+       |       peg
+                       { $$ = $1; }
+       ;
+
+peg    :       PEG
+                       { $$ = newOP(OP_NULL,0);
+                         TOKEN_GETMAD($1,$$,'p');
+                       }
        ;
 
 format :       FORMAT startformsub formname block
                        { SvREFCNT_inc(PL_compcv);
-                         newFORM($2, $3, $4); }
+#ifdef MAD
+                         $$ = newFORM($2, $3, $4);
+                         prepend_madprops($1->tk_mad, $$, 'F');
+                         $1->tk_mad = 0;
+                         token_free($1);
+#else
+                         newFORM($2, $3, $4);
+#endif
+                       }
        ;
 
 formname:      WORD            { $$ = $1; }
@@ -352,18 +518,40 @@ formname: WORD            { $$ = $1; }
 /* Unimplemented "my sub foo { }" */
 mysubrout:     MYSUB startsub subname proto subattrlist subbody
                        { SvREFCNT_inc(PL_compcv);
-                         newMYSUB($2, $3, $4, $5, $6); }
+#ifdef MAD
+                         $$ = newMYSUB($2, $3, $4, $5, $6);
+                         token_getmad($1,$$,'d');
+#else
+                         newMYSUB($2, $3, $4, $5, $6);
+#endif
+                       }
        ;
 
 /* Subroutine definition */
 subrout        :       SUB startsub subname proto subattrlist subbody
                        { SvREFCNT_inc(PL_compcv);
-                         newATTRSUB($2, $3, $4, $5, $6); }
+#ifdef MAD
+                         OP* o = newSVOP(OP_ANONCODE, 0,
+                           (SV*)newATTRSUB($2, $3, $4, $5, $6));
+                         $$ = newOP(OP_NULL,0);
+                         op_getmad(o,$$,'&');
+                         op_getmad($3,$$,'n');
+                         op_getmad($4,$$,'s');
+                         op_getmad($5,$$,'a');
+                         token_getmad($1,$$,'d');
+                         append_madprops($6->op_madprop, $$, 0);
+                         $6->op_madprop = 0;
+#else
+                         newATTRSUB($2, $3, $4, $5, $6);
+                         $$ = Nullop;
+#endif
+                       }
        ;
 
 startsub:      /* NULL */      /* start a regular subroutine scope */
                        { $$ = start_subparse(FALSE, 0);
                            SAVEFREESV(PL_compcv); }
+
        ;
 
 startanonsub:  /* NULL */      /* start an anonymous subroutine scope */
@@ -395,89 +583,169 @@ proto    :       /* NULL */
 subattrlist:   /* NULL */
                        { $$ = Nullop; }
        |       COLONATTR THING
-                       { $$ = $2; }
+                       { $$ = $2;
+                         TOKEN_GETMAD($1,$$,':');
+                       }
        |       COLONATTR
-                       { $$ = Nullop; }
+                       { $$ = IF_MAD(
+                                   newOP(OP_NULL, 0),
+                                   Nullop
+                               );
+                         TOKEN_GETMAD($1,$$,':');
+                       }
        ;
 
 /* List of attributes for a "my" variable declaration */
 myattrlist:    COLONATTR THING
-                       { $$ = $2; }
+                       { $$ = $2;
+                         TOKEN_GETMAD($1,$$,':');
+                       }
        |       COLONATTR
-                       { $$ = Nullop; }
+                       { $$ = IF_MAD(
+                                   newOP(OP_NULL, 0),
+                                   Nullop
+                               );
+                         TOKEN_GETMAD($1,$$,':');
+                       }
        ;
 
 /* Subroutine body - either null or a block */
 subbody        :       block   { $$ = $1; }
-       |       ';'     { $$ = Nullop; PL_expect = XSTATE; }
+       |       ';'     { $$ = IF_MAD(
+                                   newOP(OP_NULL,0),
+                                   Nullop
+                               );
+                         PL_expect = XSTATE;
+                         TOKEN_GETMAD($1,$$,';');
+                       }
        ;
 
 package :      PACKAGE WORD ';'
-                       { package($2); }
+                       {
+#ifdef MAD
+                         $$ = package($2);
+                         token_getmad($1,$$,'o');
+                         token_getmad($3,$$,';');
+#else
+                         package($2);
+#endif
+                       }
        ;
 
 use    :       USE startsub
                        { CvSPECIAL_on(PL_compcv); /* It's a BEGIN {} */ }
                    WORD WORD listexpr ';'
-                        { SvREFCNT_inc(PL_compcv);
-                          utilize($1, $2, $4, $5, $6); }
+                       { SvREFCNT_inc(PL_compcv);
+#ifdef MAD
+                         $$ = utilize(IVAL($1), $2, $4, $5, $6);
+                         token_getmad($1,$$,'o');
+                         token_getmad($7,$$,';');
+                         if (PL_rsfp_filters && AvFILLp(PL_rsfp_filters) >= 0)
+                             append_madprops(newMADPROP('!', MAD_PV, "", 0), $$, 0);
+#else
+                         utilize(IVAL($1), $2, $4, $5, $6);
+#endif
+                       }
        ;
 
 /* Ordinary expressions; logical combinations */
 expr   :       expr ANDOP expr
-                       { $$ = newLOGOP(OP_AND, 0, $1, $3); }
+                       { $$ = newLOGOP(OP_AND, 0, $1, $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       expr OROP expr
-                       { $$ = newLOGOP($2, 0, $1, $3); }
+                       { $$ = newLOGOP(IVAL($2), 0, $1, $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       expr DOROP expr
-                       { $$ = newLOGOP(OP_DOR, 0, $1, $3); }
+                       { $$ = newLOGOP(OP_DOR, 0, $1, $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       argexpr %prec PREC_LOW
        ;
 
 /* Expressions are a list of terms joined by commas */
 argexpr        :       argexpr ','
-                       { $$ = $1; }
+                       {
+#ifdef MAD
+                         OP* op = newNULLLIST();
+                         token_getmad($2,op,',');
+                         $$ = append_elem(OP_LIST, $1, op);
+#else
+                         $$ = $1;
+#endif
+                       }
        |       argexpr ',' term
-                       { $$ = append_elem(OP_LIST, $1, $3); }
+                       { 
+                         DO_MAD(
+                             $3 = newUNOP(OP_NULL, 0, $3);
+                             token_getmad($2,$3,',');
+                         )
+                         $$ = append_elem(OP_LIST, $1, $3);
+                       }
        |       term %prec PREC_LOW
        ;
 
 /* List operators */
 listop :       LSTOP indirob argexpr /* map {...} @args or print $fh @args */
-                       { $$ = convert($1, OPf_STACKED,
-                               prepend_elem(OP_LIST, newGVREF($1,$2), $3) ); }
+                       { $$ = convert(IVAL($1), OPf_STACKED,
+                               prepend_elem(OP_LIST, newGVREF(IVAL($1),$2), $3) );
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       FUNC '(' indirob expr ')'      /* print ($fh @args */
-                       { $$ = convert($1, OPf_STACKED,
-                               prepend_elem(OP_LIST, newGVREF($1,$3), $4) ); }
+                       { $$ = convert(IVAL($1), OPf_STACKED,
+                               prepend_elem(OP_LIST, newGVREF(IVAL($1),$3), $4) );
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
        |       term ARROW method '(' listexprcom ')' /* $foo->bar(list) */
                        { $$ = convert(OP_ENTERSUB, OPf_STACKED,
                                append_elem(OP_LIST,
                                    prepend_elem(OP_LIST, scalar($1), $5),
-                                   newUNOP(OP_METHOD, 0, $3))); }
+                                   newUNOP(OP_METHOD, 0, $3)));
+                         TOKEN_GETMAD($2,$$,'A');
+                         TOKEN_GETMAD($4,$$,'(');
+                         TOKEN_GETMAD($6,$$,')');
+                       }
        |       term ARROW method                     /* $foo->bar */
                        { $$ = convert(OP_ENTERSUB, OPf_STACKED,
                                append_elem(OP_LIST, scalar($1),
-                                   newUNOP(OP_METHOD, 0, $3))); }
+                                   newUNOP(OP_METHOD, 0, $3)));
+                         TOKEN_GETMAD($2,$$,'A');
+                       }
        |       METHOD indirob listexpr              /* new Class @args */
                        { $$ = convert(OP_ENTERSUB, OPf_STACKED,
                                append_elem(OP_LIST,
                                    prepend_elem(OP_LIST, $2, $3),
-                                   newUNOP(OP_METHOD, 0, $1))); }
+                                   newUNOP(OP_METHOD, 0, $1)));
+                       }
        |       FUNCMETH indirob '(' listexprcom ')' /* method $object (@args) */
                        { $$ = convert(OP_ENTERSUB, OPf_STACKED,
                                append_elem(OP_LIST,
                                    prepend_elem(OP_LIST, $2, $4),
-                                   newUNOP(OP_METHOD, 0, $1))); }
+                                   newUNOP(OP_METHOD, 0, $1)));
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
        |       LSTOP listexpr                       /* print @args */
-                       { $$ = convert($1, 0, $2); }
+                       { $$ = convert(IVAL($1), 0, $2);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       FUNC '(' listexprcom ')'             /* print (@args) */
-                       { $$ = convert($1, 0, $3); }
+                       { $$ = convert(IVAL($1), 0, $3);
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        |       LSTOPSUB startanonsub block /* sub f(&@);   f { foo } ... */
                        { SvREFCNT_inc(PL_compcv);
                          $3 = newANONATTRSUB($2, 0, Nullop, $3); }
                    listexpr            %prec LSTOP  /* ... @bar */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
                                 append_elem(OP_LIST,
-                                  prepend_elem(OP_LIST, $3, $5), $1)); }
+                                  prepend_elem(OP_LIST, $3, $5), $1));
+                       }
        ;
 
 /* Names of methods. May use $object->$methodname */
@@ -490,128 +758,259 @@ subscripted:    star '{' expr ';' '}'        /* *main::{something} */
                         /* In this and all the hash accessors, ';' is
                          * provided by the tokeniser */
                        { $$ = newBINOP(OP_GELEM, 0, $1, scalar($3));
-                           PL_expect = XOPERATOR; }
+                           PL_expect = XOPERATOR;
+                         TOKEN_GETMAD($2,$$,'{');
+                         TOKEN_GETMAD($4,$$,';');
+                         TOKEN_GETMAD($5,$$,'}');
+                       }
        |       scalar '[' expr ']'          /* $array[$element] */
-                       { $$ = newBINOP(OP_AELEM, 0, oopsAV($1), scalar($3)); }
+                       { $$ = newBINOP(OP_AELEM, 0, oopsAV($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'[');
+                         TOKEN_GETMAD($4,$$,']');
+                       }
        |       term ARROW '[' expr ']'      /* somearef->[$element] */
                        { $$ = newBINOP(OP_AELEM, 0,
                                        ref(newAVREF($1),OP_RV2AV),
-                                       scalar($4));}
+                                       scalar($4));
+                         TOKEN_GETMAD($2,$$,'a');
+                         TOKEN_GETMAD($3,$$,'[');
+                         TOKEN_GETMAD($5,$$,']');
+                       }
        |       subscripted '[' expr ']'    /* $foo->[$bar]->[$baz] */
                        { $$ = newBINOP(OP_AELEM, 0,
                                        ref(newAVREF($1),OP_RV2AV),
-                                       scalar($3));}
+                                       scalar($3));
+                         TOKEN_GETMAD($2,$$,'[');
+                         TOKEN_GETMAD($4,$$,']');
+                       }
        |       scalar '{' expr ';' '}'    /* $foo->{bar();} */
                        { $$ = newBINOP(OP_HELEM, 0, oopsHV($1), jmaybe($3));
-                           PL_expect = XOPERATOR; }
+                           PL_expect = XOPERATOR;
+                         TOKEN_GETMAD($2,$$,'{');
+                         TOKEN_GETMAD($4,$$,';');
+                         TOKEN_GETMAD($5,$$,'}');
+                       }
        |       term ARROW '{' expr ';' '}' /* somehref->{bar();} */
                        { $$ = newBINOP(OP_HELEM, 0,
                                        ref(newHVREF($1),OP_RV2HV),
                                        jmaybe($4));
-                           PL_expect = XOPERATOR; }
+                           PL_expect = XOPERATOR;
+                         TOKEN_GETMAD($2,$$,'a');
+                         TOKEN_GETMAD($3,$$,'{');
+                         TOKEN_GETMAD($5,$$,';');
+                         TOKEN_GETMAD($6,$$,'}');
+                       }
        |       subscripted '{' expr ';' '}' /* $foo->[bar]->{baz;} */
                        { $$ = newBINOP(OP_HELEM, 0,
                                        ref(newHVREF($1),OP_RV2HV),
                                        jmaybe($3));
-                           PL_expect = XOPERATOR; }
+                           PL_expect = XOPERATOR;
+                         TOKEN_GETMAD($2,$$,'{');
+                         TOKEN_GETMAD($4,$$,';');
+                         TOKEN_GETMAD($5,$$,'}');
+                       }
        |       term ARROW '(' ')'          /* $subref->() */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-                                  newCVREF(0, scalar($1))); }
+                                  newCVREF(0, scalar($1)));
+                         TOKEN_GETMAD($2,$$,'a');
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        |       term ARROW '(' expr ')'     /* $subref->(@args) */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
                                   append_elem(OP_LIST, $4,
-                                      newCVREF(0, scalar($1)))); }
+                                      newCVREF(0, scalar($1))));
+                         TOKEN_GETMAD($2,$$,'a');
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
 
        |       subscripted '(' expr ')'   /* $foo->{bar}->(@args) */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
                                   append_elem(OP_LIST, $3,
-                                              newCVREF(0, scalar($1)))); }
+                                              newCVREF(0, scalar($1))));
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        |       subscripted '(' ')'        /* $foo->{bar}->() */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-                                  newCVREF(0, scalar($1))); }
+                                  newCVREF(0, scalar($1)));
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                       }
        |       '(' expr ')' '[' expr ']'            /* list slice */
-                       { $$ = newSLICEOP(0, $5, $2); }
+                       { $$ = newSLICEOP(0, $5, $2);
+                         TOKEN_GETMAD($1,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                         TOKEN_GETMAD($4,$$,'[');
+                         TOKEN_GETMAD($6,$$,']');
+                       }
        |       '(' ')' '[' expr ']'                 /* empty list slice! */
-                       { $$ = newSLICEOP(0, $4, Nullop); }
+                       { $$ = newSLICEOP(0, $4, Nullop);
+                         TOKEN_GETMAD($1,$$,'(');
+                         TOKEN_GETMAD($2,$$,')');
+                         TOKEN_GETMAD($3,$$,'[');
+                         TOKEN_GETMAD($5,$$,']');
+                       }
     ;
 
 /* Binary operators between terms */
-termbinop      :       term ASSIGNOP term             /* $x = $y */
-                       { $$ = newASSIGNOP(OPf_STACKED, $1, $2, $3); }
+termbinop:     term ASSIGNOP term                     /* $x = $y */
+                       { $$ = newASSIGNOP(OPf_STACKED, $1, IVAL($2), $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term POWOP term                        /* $x ** $y */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term MULOP term                        /* $x * $y, $x x $y */
-                       {   if ($2 != OP_REPEAT)
+                       {   if (IVAL($2) != OP_REPEAT)
                                scalar($1);
-                           $$ = newBINOP($2, 0, $1, scalar($3)); }
+                           $$ = newBINOP(IVAL($2), 0, $1, scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term ADDOP term                        /* $x + $y */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term SHIFTOP term                      /* $x >> $y, $x << $y */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term RELOP term                        /* $x > $y, etc. */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term EQOP term                         /* $x == $y, $x eq $y */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term BITANDOP term                     /* $x & $y */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term BITOROP term                      /* $x | $y */
-                       { $$ = newBINOP($2, 0, scalar($1), scalar($3)); }
+                       { $$ = newBINOP(IVAL($2), 0, scalar($1), scalar($3));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term DOTDOT term                       /* $x..$y, $x...$y */
-                       { $$ = newRANGE($2, scalar($1), scalar($3));}
+                       {
+                         $$ = newRANGE(IVAL($2), scalar($1), scalar($3));
+                         DO_MAD(
+                             UNOP *op;
+                             op = (UNOP*)$$;
+                             op = (UNOP*)op->op_first; /* get to flop */
+                             op = (UNOP*)op->op_first; /* get to flip */
+                             op = (UNOP*)op->op_first; /* get to range */
+                             token_getmad($2,(OP*)op,'o');
+                           )
+                       }
        |       term ANDAND term                       /* $x && $y */
-                       { $$ = newLOGOP(OP_AND, 0, $1, $3); }
+                       { $$ = newLOGOP(OP_AND, 0, $1, $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term OROR term                         /* $x || $y */
-                       { $$ = newLOGOP(OP_OR, 0, $1, $3); }
+                       { $$ = newLOGOP(OP_OR, 0, $1, $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term DORDOR term                       /* $x // $y */
-                       { $$ = newLOGOP(OP_DOR, 0, $1, $3); }
+                       { $$ = newLOGOP(OP_DOR, 0, $1, $3);
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term MATCHOP term                      /* $x =~ /$y/ */
-                       { $$ = bind_match($2, $1, $3); }
+                       { $$ = bind_match(IVAL($2), $1, $3);
+                         TOKEN_GETMAD($2,
+                               ($$->op_type == OP_NOT
+                                   ? ((UNOP*)$$)->op_first : $$),
+                               '~');
+                       }
     ;
 
 /* Unary operators and terms */
 termunop : '-' term %prec UMINUS                       /* -$x */
-                       { $$ = newUNOP(OP_NEGATE, 0, scalar($2)); }
+                       { $$ = newUNOP(OP_NEGATE, 0, scalar($2));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       '+' term %prec UMINUS                  /* +$x */
-                       { $$ = $2; }
+                       { $$ = IF_MAD(
+                                   newUNOP(OP_NULL, 0, $2),
+                                   $2
+                               );
+                         TOKEN_GETMAD($1,$$,'+');
+                       }
        |       '!' term                               /* !$x */
-                       { $$ = newUNOP(OP_NOT, 0, scalar($2)); }
+                       { $$ = newUNOP(OP_NOT, 0, scalar($2));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       '~' term                               /* ~$x */
-                       { $$ = newUNOP(OP_COMPLEMENT, 0, scalar($2));}
+                       { $$ = newUNOP(OP_COMPLEMENT, 0, scalar($2));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       term POSTINC                           /* $x++ */
                        { $$ = newUNOP(OP_POSTINC, 0,
-                                       mod(scalar($1), OP_POSTINC)); }
+                                       mod(scalar($1), OP_POSTINC));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       term POSTDEC                           /* $x-- */
                        { $$ = newUNOP(OP_POSTDEC, 0,
-                                       mod(scalar($1), OP_POSTDEC)); }
+                                       mod(scalar($1), OP_POSTDEC));
+                         TOKEN_GETMAD($2,$$,'o');
+                       }
        |       PREINC term                            /* ++$x */
                        { $$ = newUNOP(OP_PREINC, 0,
-                                       mod(scalar($2), OP_PREINC)); }
+                                       mod(scalar($2), OP_PREINC));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       PREDEC term                            /* --$x */
                        { $$ = newUNOP(OP_PREDEC, 0,
-                                       mod(scalar($2), OP_PREDEC)); }
+                                       mod(scalar($2), OP_PREDEC));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
 
     ;
 
 /* Constructors for anonymous data */
 anonymous:     '[' expr ']'
-                       { $$ = newANONLIST($2); }
+                       { $$ = newANONLIST($2);
+                         TOKEN_GETMAD($1,$$,'[');
+                         TOKEN_GETMAD($3,$$,']');
+                       }
        |       '[' ']'
-                       { $$ = newANONLIST(Nullop); }
+                       { $$ = newANONLIST(Nullop);
+                         TOKEN_GETMAD($1,$$,'[');
+                         TOKEN_GETMAD($2,$$,']');
+                       }
        |       HASHBRACK expr ';' '}'  %prec '(' /* { foo => "Bar" } */
-                       { $$ = newANONHASH($2); }
+                       { $$ = newANONHASH($2);
+                         TOKEN_GETMAD($1,$$,'{');
+                         TOKEN_GETMAD($3,$$,';');
+                         TOKEN_GETMAD($4,$$,'}');
+                       }
        |       HASHBRACK ';' '}'       %prec '(' /* { } (';' by tokener) */
-                       { $$ = newANONHASH(Nullop); }
+                       { $$ = newANONHASH(Nullop);
+                         TOKEN_GETMAD($1,$$,'{');
+                         TOKEN_GETMAD($2,$$,';');
+                         TOKEN_GETMAD($3,$$,'}');
+                       }
        |       ANONSUB startanonsub proto subattrlist block    %prec '('
                        { SvREFCNT_inc(PL_compcv);
-                         $$ = newANONATTRSUB($2, $3, $4, $5); }
+                         $$ = newANONATTRSUB($2, $3, $4, $5);
+                         TOKEN_GETMAD($1,$$,'o');
+                         OP_GETMAD($3,$$,'s');
+                         OP_GETMAD($4,$$,'a');
+                       }
 
     ;
 
 /* Things called with "do" */
 termdo :       DO term %prec UNIOP                     /* do $filename */
-                       { $$ = dofile($2, $1); }
+                       { $$ = dofile($2, $1);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       DO block        %prec '('               /* do { code */
-                       { $$ = newUNOP(OP_NULL, OPf_SPECIAL, scope($2)); }
+                       { $$ = newUNOP(OP_NULL, OPf_SPECIAL, scope($2));
+                         TOKEN_GETMAD($1,$$,'D');
+                       }
        |       DO WORD '(' ')'                         /* do somesub() */
                        { $$ = newUNOP(OP_ENTERSUB,
                            OPf_SPECIAL|OPf_STACKED,
@@ -619,7 +1018,11 @@ termdo    :       DO term %prec UNIOP                     /* do $filename */
                                scalar(newCVREF(
                                    (OPpENTERSUB_AMPER<<8),
                                    scalar($2)
-                               )),Nullop)); dep();}
+                               )),Nullop)); dep();
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        |       DO WORD '(' expr ')'                    /* do somesub(@args) */
                        { $$ = newUNOP(OP_ENTERSUB,
                            OPf_SPECIAL|OPf_STACKED,
@@ -628,16 +1031,28 @@ termdo   :       DO term %prec UNIOP                     /* do $filename */
                                scalar(newCVREF(
                                    (OPpENTERSUB_AMPER<<8),
                                    scalar($2)
-                               )))); dep();}
+                               )))); dep();
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
        |       DO scalar '(' ')'                      /* do $subref () */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_SPECIAL|OPf_STACKED,
                            prepend_elem(OP_LIST,
-                               scalar(newCVREF(0,scalar($2))), Nullop)); dep();}
+                               scalar(newCVREF(0,scalar($2))), Nullop)); dep();
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        |       DO scalar '(' expr ')'                 /* do $subref (@args) */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_SPECIAL|OPf_STACKED,
                            prepend_elem(OP_LIST,
                                $4,
-                               scalar(newCVREF(0,scalar($2))))); dep();}
+                               scalar(newCVREF(0,scalar($2))))); dep();
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($3,$$,'(');
+                         TOKEN_GETMAD($5,$$,')');
+                       }
 
         ;
 
@@ -646,17 +1061,30 @@ term     :       termbinop
        |       anonymous
        |       termdo
        |       term '?' term ':' term
-                       { $$ = newCONDOP(0, $1, $3, $5); }
+                       { $$ = newCONDOP(0, $1, $3, $5);
+                         TOKEN_GETMAD($2,$$,'?');
+                         TOKEN_GETMAD($4,$$,':');
+                       }
        |       REFGEN term                          /* \$x, \@y, \%z */
-                       { $$ = newUNOP(OP_REFGEN, 0, mod($2,OP_REFGEN)); }
+                       { $$ = newUNOP(OP_REFGEN, 0, mod($2,OP_REFGEN));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       myattrterm      %prec UNIOP
                        { $$ = $1; }
        |       LOCAL term      %prec UNIOP
-                       { $$ = localize($2,$1); }
+                       { $$ = localize($2,IVAL($1));
+                         TOKEN_GETMAD($1,$$,'d');
+                       }
        |       '(' expr ')'
-                       { $$ = sawparens($2); }
+                       { $$ = sawparens(IF_MAD(newUNOP(OP_NULL,0,$2), $2));
+                         TOKEN_GETMAD($1,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                       }
        |       '(' ')'
-                       { $$ = sawparens(newNULLLIST()); }
+                       { $$ = sawparens(newNULLLIST());
+                         TOKEN_GETMAD($1,$$,'(');
+                         TOKEN_GETMAD($2,$$,')');
+                       }
        |       scalar  %prec '('
                        { $$ = $1; }
        |       star    %prec '('
@@ -674,42 +1102,76 @@ term     :       termbinop
                                newOP(OP_PUSHMARK, 0),
                                    newLISTOP(OP_ASLICE, 0,
                                        list($3),
-                                       ref($1, OP_ASLICE))); }
+                                       ref($1, OP_ASLICE)));
+                         TOKEN_GETMAD($2,$$,'[');
+                         TOKEN_GETMAD($4,$$,']');
+                       }
        |       ary '{' expr ';' '}'                 /* @hash{@keys} */
                        { $$ = prepend_elem(OP_HSLICE,
                                newOP(OP_PUSHMARK, 0),
                                    newLISTOP(OP_HSLICE, 0,
                                        list($3),
                                        ref(oopsHV($1), OP_HSLICE)));
-                           PL_expect = XOPERATOR; }
+                           PL_expect = XOPERATOR;
+                         TOKEN_GETMAD($2,$$,'{');
+                         TOKEN_GETMAD($4,$$,';');
+                         TOKEN_GETMAD($5,$$,'}');
+                       }
        |       THING   %prec '('
                        { $$ = $1; }
        |       amper                                /* &foo; */
                        { $$ = newUNOP(OP_ENTERSUB, 0, scalar($1)); }
        |       amper '(' ')'                        /* &foo() */
-                       { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED, scalar($1)); }
+                       { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED, scalar($1));
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                       }
        |       amper '(' expr ')'                   /* &foo(@args) */
-                       { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-                           append_elem(OP_LIST, $3, scalar($1))); }
+                       {
+                         $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
+                               append_elem(OP_LIST, $3, scalar($1)));
+                         DO_MAD(
+                             OP* op = $$;
+                             if (op->op_type == OP_CONST) { /* defeat const fold */
+                               op = (OP*)op->op_madprop->mad_val;
+                             }
+                             token_getmad($2,op,'(');
+                             token_getmad($4,op,')');
+                         )
+                       }
        |       NOAMP WORD listexpr                  /* foo(@args) */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
-                           append_elem(OP_LIST, $3, scalar($2))); }
+                           append_elem(OP_LIST, $3, scalar($2)));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       LOOPEX  /* loop exiting command (goto, last, dump, etc) */
-                       { $$ = newOP($1, OPf_SPECIAL);
-                           PL_hints |= HINT_BLOCK_SCOPE; }
+                       { $$ = newOP(IVAL($1), OPf_SPECIAL);
+                           PL_hints |= HINT_BLOCK_SCOPE;
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       LOOPEX term
-                       { $$ = newLOOPEX($1,$2); }
+                       { $$ = newLOOPEX(IVAL($1),$2);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       NOTOP argexpr                        /* not $foo */
-                       { $$ = newUNOP(OP_NOT, 0, scalar($2)); }
+                       { $$ = newUNOP(OP_NOT, 0, scalar($2));
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       UNIOP                                /* Unary op, $_ implied */
-                       { $$ = newOP($1, 0); }
-       |       UNIOP block                          /* eval { foo } */
-                       { $$ = newUNOP($1, 0, $2); }
+                       { $$ = newOP(IVAL($1), 0);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
+       |       UNIOP block                          /* eval { foo }* */
+                       { $$ = newUNOP(IVAL($1), 0, $2);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       UNIOP term                           /* Unary op */
-                       { $$ = newUNOP($1, 0, $2); }
-       |       REQUIRE                              /* require, $_ implied */
+                       { $$ = newUNOP(IVAL($1), 0, $2);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
+       |       REQUIRE                              /* require, $_ implied *//* FIMXE for MAD needed? */
                        { $$ = newOP(OP_REQUIRE, $1 ? OPf_SPECIAL : 0); }
-       |       REQUIRE term                         /* require Foo */
+       |       REQUIRE term                         /* require Foo *//* FIMXE for MAD needed? */
                        { $$ = newUNOP(OP_REQUIRE, $1 ? OPf_SPECIAL : 0, $2); }
        |       UNIOPSUB
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED, scalar($1)); }
@@ -717,35 +1179,68 @@ term     :       termbinop
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
                            append_elem(OP_LIST, $2, scalar($1))); }
        |       FUNC0                                /* Nullary operator */
-                       { $$ = newOP($1, 0); }
+                       { $$ = newOP(IVAL($1), 0);
+                         TOKEN_GETMAD($1,$$,'o');
+                       }
        |       FUNC0 '(' ')'
-                       { $$ = newOP($1, 0); }
+                       { $$ = newOP(IVAL($1), 0);
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                       }
        |       FUNC0SUB                             /* Sub treated as nullop */
                        { $$ = newUNOP(OP_ENTERSUB, OPf_STACKED,
                                scalar($1)); }
        |       FUNC1 '(' ')'                        /* not () */
-                       { $$ = $1 == OP_NOT ? newUNOP($1, 0, newSVOP(OP_CONST, 0, newSViv(0)))
-                                           : newOP($1, OPf_SPECIAL); }
+                       { $$ = (IVAL($1) == OP_NOT)
+                           ? newUNOP(IVAL($1), 0, newSVOP(OP_CONST, 0, newSViv(0)))
+                           : newOP(IVAL($1), OPf_SPECIAL);
+
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                       }
        |       FUNC1 '(' expr ')'                   /* not($foo) */
-                       { $$ = newUNOP($1, 0, $3); }
-       |       PMFUNC '(' argexpr ')'           /* m//, s///, tr/// */
-                       { $$ = pmruntime($1, $3, 1); }
+                       { $$ = newUNOP(IVAL($1), 0, $3);
+                         TOKEN_GETMAD($1,$$,'o');
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
+       |       PMFUNC '(' argexpr ')'          /* m//, s///, tr/// */
+                       { $$ = pmruntime($1, $3, 1);
+                         TOKEN_GETMAD($2,$$,'(');
+                         TOKEN_GETMAD($4,$$,')');
+                       }
        |       WORD
        |       listop
        ;
 
 /* "my" declarations, with optional attributes */
 myattrterm:    MY myterm myattrlist
-                       { $$ = my_attrs($2,$3); }
+                       { $$ = my_attrs($2,$3);
+                         DO_MAD(
+                             token_getmad($1,$$,'d');
+                             append_madprops($3->op_madprop, $$, 'a');
+                             $3->op_madprop = 0;
+                         )
+                       }
        |       MY myterm
-                       { $$ = localize($2,$1); }
+                       { $$ = localize($2,IVAL($1));
+                         TOKEN_GETMAD($1,$$,'d');
+                       }
        ;
 
 /* Things that can be "my"'d */
 myterm :       '(' expr ')'
-                       { $$ = sawparens($2); }
+                       { $$ = sawparens($2);
+                         TOKEN_GETMAD($1,$$,'(');
+                         TOKEN_GETMAD($3,$$,')');
+                       }
        |       '(' ')'
-                       { $$ = sawparens(newNULLLIST()); }
+                       { $$ = sawparens(newNULLLIST());
+                         TOKEN_GETMAD($1,$$,'(');
+                         TOKEN_GETMAD($2,$$,')');
+                       }
        |       scalar  %prec '('
                        { $$ = $1; }
        |       hsh     %prec '('
@@ -766,7 +1261,16 @@ listexprcom:      /* NULL */
        |       expr
                        { $$ = $1; }
        |       expr ','
-                       { $$ = $1; }
+                       {
+#ifdef MAD
+                         OP* op = newNULLLIST();
+                         token_getmad($2,op,',');
+                         $$ = append_elem(OP_LIST, $1, op);
+#else
+                         $$ = $1;
+#endif
+
+                       }
        ;
 
 /* A little bit of trickery to make "for my $foo (@bar)" actually be
@@ -776,34 +1280,46 @@ my_scalar:       scalar
        ;
 
 amper  :       '&' indirob
-                       { $$ = newCVREF($1,$2); }
+                       { $$ = newCVREF(IVAL($1),$2);
+                         TOKEN_GETMAD($1,$$,'&');
+                       }
        ;
 
 scalar :       '$' indirob
-                       { $$ = newSVREF($2); }
+                       { $$ = newSVREF($2);
+                         TOKEN_GETMAD($1,$$,'$');
+                       }
        ;
 
 ary    :       '@' indirob
-                       { $$ = newAVREF($2); }
+                       { $$ = newAVREF($2);
+                         TOKEN_GETMAD($1,$$,'@');
+                       }
        ;
 
 hsh    :       '%' indirob
-                       { $$ = newHVREF($2); }
+                       { $$ = newHVREF($2);
+                         TOKEN_GETMAD($1,$$,'%');
+                       }
        ;
 
 arylen :       DOLSHARP indirob
-                       { $$ = newAVREF($2); }
+                       { $$ = newAVREF($2);
+                         TOKEN_GETMAD($1,$$,'l');
+                       }
        ;
 
 star   :       '*' indirob
-                       { $$ = newGVREF(0,$2); }
+                       { $$ = newGVREF(0,$2);
+                         TOKEN_GETMAD($1,$$,'*');
+                       }
        ;
 
 /* Indirect objects */
 indirob        :       WORD
                        { $$ = scalar($1); }
        |       scalar %prec PREC_LOW
-                       { $$ = scalar($1);  }
+                       { $$ = scalar($1); }
        |       block
                        { $$ = scope($1); }