Re: push parser

[Joel E. Denny]

Below is another patch I haven't yet committed.

After this one, only one more patch should be necessary to finish 
committing the large poorly documented patch I posted a few days ago.  
That next patch would need to implement a yyparse wrapping yypush_parse.  
Once we agree on how that should go, we'll be able to run the test suite 
in push mode so we can be sure our push code is working right.

Index: ChangeLog
===================================================================
RCS file: /sources/bison/bison/ChangeLog,v
retrieving revision 1.1636
diff -p -u -r1.1636 ChangeLog
--- ChangeLog   20 Dec 2006 07:48:57 -0000      1.1636
+++ ChangeLog   20 Dec 2006 08:35:22 -0000
@@ -1,5 +1,31 @@
 2006-12-20  Joel E. Denny  <address@hidden>
 
+       Enable push parsers to operate in impure mode.  Thus, %push-parser no
+       longer implies %pure-parser.  The point of this change is to move
+       towards being able to test the push parser code by running the entire
+       test suite as if %push-parser had been declared.
+       * data/push.c (yypush_parse): For impure mode, remove the
+       yypushed_char, yypushed_val, and yypushed_loc arguments.
+       Instead, declare these as local variables initialized to the global
+       yychar, yylval, and yylloc.
+       For the first yypush_parse invocation only, restore the initial values
+       of these global variables when it's time to read a token since they
+       have been overwritten.
+       * src/parse-gram.y (prologue_declaration): Don't set pure_parser for
+       %push-parser.
+       * tests/calc.at (Simple LALR(1) Calculator): Always set %pure-parser
+       along with %push-parser since this test case was designed for pure push
+       parsers.
+        * tests/local.at (AT_PURE_OR_PUSH_IF): Remove unused.
+       (AT_YACC_OR_PUSH_IF): New.
+       (AT_YYERROR_SEES_LOC_IF): Fix enough that the test suite passes, but
+       add a note that it's still wrong for some cases (as it has been for a
+       while).
+       (AT_PURE_LEX_IF): Use AT_PURE_IF instead of AT_PURE_OR_PUSH_IF since
+       %push-parser no longer implies %pure-parser.
+
+2006-12-20  Joel E. Denny  <address@hidden>
+
        Remove some unnecessary differences between the pull parser code and
        the push parser code.  This patch enables yynerrs in push mode.
        * data/push.c: Reformat M4 a little.
Index: data/push.c
===================================================================
RCS file: /sources/bison/bison/data/push.c,v
retrieving revision 1.23
diff -p -u -r1.23 push.c
--- data/push.c 20 Dec 2006 07:48:57 -0000      1.23
+++ data/push.c 20 Dec 2006 08:35:22 -0000
@@ -226,10 +226,10 @@ enum { YYPUSH_MORE = 4 };
 b4_c_function_decl([[yypstate_delete]], [[void]],
                    [[[yypstate *yyps]], [[yyps]]])
 b4_c_function_decl([[yypush_parse]], [[int]],
-  [[[yypstate *yyps]], [[yyps]]],
+  [[[yypstate *yyps]], [[yyps]]]b4_pure_if([,
   [[[int yypushed_char]], [[yypushed_char]]],
   [[[YYSTYPE const *yypushed_val]], [[yypushed_val]]]b4_locations_if([,
-  [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])m4_ifset([b4_parse_param], [,
+  [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])])m4_ifset([b4_parse_param], [,
   b4_parse_param]))[
 #endif
 ]])
@@ -1106,10 +1106,11 @@ b4_push_if(
 `-------------------------*/
 
 ]b4_push_if([
-b4_c_function_def([[yypush_parse]], [[int]], [[[yypstate *yyps]], [[yyps]]], 
+b4_c_function_def([[yypush_parse]], [[int]],
+  [[[yypstate *yyps]], [[yyps]]]b4_pure_if([,
   [[[int yypushed_char]], [[yypushed_char]]],
   [[[YYSTYPE const *yypushed_val]], [[yypushed_val]]]b4_locations_if([,
-  [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])m4_ifset([b4_parse_param], [,
+  [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])])m4_ifset([b4_parse_param], [,
   b4_parse_param]))], [
 #ifdef YYPARSE_PARAM
 b4_c_function_def([yyparse], [int], [[void *YYPARSE_PARAM], [YYPARSE_PARAM]])
@@ -1118,7 +1119,11 @@ b4_c_function_def([yyparse], [int], b4_p
 #endif])[
 {
   ]b4_pure_if([b4_declare_scanner_communication_variables])
-  b4_push_if([], [b4_declare_parser_state_variables])[
+  b4_push_if([b4_pure_if([], [[  int yypushed_char = yychar;
+  YYSTYPE yypushed_val = yylval;
+  ]b4_locations_if([[YYLTYPE yypushed_loc = yylloc;
+]])])],
+  [b4_declare_parser_state_variables])[
 
 #if YYERROR_VERBOSE
   /* Buffer for error messages, and its allocated size.  */
@@ -1286,14 +1291,22 @@ yybackup:
           goto yypushreturn;
         }
       yyps->yynew = 0;
-
+]b4_pure_if([], [[
+      /* Restoring the pushed token is only necessary for the first
+         yypush_parse invocation since subsequent invocations don't overwrite
+         it before jumping to yyread_pushed_token.  */
+      yychar = yypushed_char;
+      yylval = yypushed_val;
+      ]b4_locations_if([[yylloc = yypushed_loc;
+]])])[
 yyread_pushed_token:
 ]])[      YYDPRINTF ((stderr, "Reading a token: "));
-]b4_push_if([[      yychar = yypushed_char;
+]b4_push_if([b4_pure_if([[      yychar = yypushed_char;
       if (yypushed_val)
         yylval = *yypushed_val;
 ]b4_locations_if([[      if (yypushed_loc)
-        yylloc = *yypushed_loc;]])],
+        yylloc = *yypushed_loc;
+]])])],
 [[      yychar = YYLEX;
 ]])[    }
 
@@ -1643,10 +1656,10 @@ enum { YYPUSH_MORE = 4 };
 b4_c_function_decl([b4_prefix[pstate_delete]], [[void]],
                    [[b4_prefix[pstate *yyps]], [[yyps]]])
 b4_c_function_decl([b4_prefix[push_parse]], [[int]],
-  [[b4_prefix[pstate *yyps]], [[yyps]]],
+  [[b4_prefix[pstate *yyps]], [[yyps]]]b4_pure_if([,
   [[[int yypushed_char]], [[yypushed_char]]],
   [[[YYSTYPE const *yypushed_val]], [[yypushed_val]]]b4_locations_if([,
-  [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])m4_ifset([b4_parse_param], [,
+  [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])])m4_ifset([b4_parse_param], [,
   b4_parse_param]))[
 #endif
 ]])
Index: src/parse-gram.y
===================================================================
RCS file: /sources/bison/bison/src/parse-gram.y,v
retrieving revision 1.102
diff -p -u -r1.102 parse-gram.y
--- src/parse-gram.y    19 Dec 2006 00:34:36 -0000      1.102
+++ src/parse-gram.y    20 Dec 2006 08:35:24 -0000
@@ -257,7 +257,7 @@ prologue_declaration:
 | "%output" "=" STRING          { spec_outfile = $3; }  /* deprecated */
 | "%parse-param" "{...}"       { add_param ("parse_param", $2, @2); }
 | "%pure-parser"                { pure_parser = true; }
-| "%push-parser"                { push_parser = true; pure_parser = true; }
+| "%push-parser"                { push_parser = true; }
 | "%require" STRING             { version_check (&@2, $2); }
 | "%skeleton" STRING            { skeleton_arg ($2, 1, &@1); }
 | "%token-table"                { token_table_flag = true; }
Index: tests/calc.at
===================================================================
RCS file: /sources/bison/bison/tests/calc.at,v
retrieving revision 1.99
diff -p -u -r1.99 calc.at
--- tests/calc.at       19 Dec 2006 00:34:37 -0000      1.99
+++ tests/calc.at       20 Dec 2006 08:35:24 -0000
@@ -572,7 +572,7 @@ AT_CHECK_CALC_LALR([%yacc])
 AT_CHECK_CALC_LALR([%error-verbose])
 
 AT_CHECK_CALC_LALR([%pure-parser %locations])
-AT_CHECK_CALC_LALR([%push-parser %locations %skeleton "push.c"])
+AT_CHECK_CALC_LALR([%push-parser %pure-parser %locations %skeleton "push.c"])
 AT_CHECK_CALC_LALR([%error-verbose %locations])
 
 AT_CHECK_CALC_LALR([%error-verbose %locations %defines %name-prefix "calc" 
%verbose %yacc])
@@ -581,7 +581,7 @@ AT_CHECK_CALC_LALR([%debug])
 AT_CHECK_CALC_LALR([%error-verbose %debug %locations %defines %name-prefix 
"calc" %verbose %yacc])
 
 AT_CHECK_CALC_LALR([%pure-parser %error-verbose %debug %locations %defines 
%name-prefix "calc" %verbose %yacc])
-AT_CHECK_CALC_LALR([%push-parser %error-verbose %debug %locations %defines 
%name-prefix "calc" %verbose %yacc %skeleton "push.c"])
+AT_CHECK_CALC_LALR([%push-parser %pure-parser %error-verbose %debug %locations 
%defines %name-prefix "calc" %verbose %yacc %skeleton "push.c"])
 
 AT_CHECK_CALC_LALR([%pure-parser %error-verbose %debug %locations %defines 
%name-prefix "calc" %verbose %yacc %parse-param {semantic_value *result} 
%parse-param {int *count}])
 
Index: tests/local.at
===================================================================
RCS file: /sources/bison/bison/tests/local.at,v
retrieving revision 1.20
diff -p -u -r1.20 local.at
--- tests/local.at      19 Dec 2006 00:34:37 -0000      1.20
+++ tests/local.at      20 Dec 2006 08:35:24 -0000
@@ -60,8 +60,8 @@ m4_pushdef([AT_PURE_IF],
 [m4_bmatch([$3], [%pure-parser], [$1], [$2])])
 m4_pushdef([AT_PUSH_IF],
 [m4_bmatch([$3], [%push-parser], [$1], [$2])])
-m4_pushdef([AT_PURE_OR_PUSH_IF],
-[m4_bmatch([$3], [%pure-parser\|%push-parser], [$1], [$2])])
+m4_pushdef([AT_YACC_OR_PUSH_IF],
+[AT_YACC_IF([$1], [AT_PUSH_IF([$1], [$2])])])
 m4_pushdef([AT_PURE_AND_LOC_IF],
 [m4_bmatch([$3], [%locations.*%pure-parser\|%pure-parser.*%locations],
           [$1], [$2])])
@@ -76,17 +76,17 @@ m4_pushdef([AT_YYERROR_ARG_LOC_IF],
 [AT_GLR_OR_PARAM_IF([AT_PURE_AND_LOC_IF([$1], [$2])],
                    [$2])])
 # yyerror always sees the locations (when activated), except if
-# push or (yacc & pure & !param).
+# (yacc & pure & !param).  FIXME: This is wrong.  See the manual.
 m4_pushdef([AT_YYERROR_SEES_LOC_IF],
-[AT_PUSH_IF([$2],[AT_LOCATION_IF([AT_YACC_IF([AT_PURE_IF([AT_PARAM_IF([$1], 
[$2])],
-                                       [$1])],
-                           [$1])],
-               [$2])])])
+[AT_LOCATION_IF([AT_YACC_OR_PUSH_IF([AT_PURE_IF([AT_PARAM_IF([$1], [$2])],
+                                               [$1])],
+                                   [$1])],
+               [$2])])
 
 # The interface is pure: either because %pure-parser, or because we
 # are using the C++ parsers.
 m4_pushdef([AT_PURE_LEX_IF],
-[AT_PURE_OR_PUSH_IF([$1],
+[AT_PURE_IF([$1],
            [AT_SKEL_CC_IF([$1], [$2])])])
 
 AT_PURE_LEX_IF(