[PATCH 5/6] yacc.c: push: initialize the pstate variables in pstate_new

[Akim Demaille]

Currently pstate_new does not set up its variables, this task is left
to yypush_parse.  This was probably to share more code with usual pull
parsers, where these (local) variables are indeed initialized by
yyparse.

But as a consequence yyexpected_tokens crashes at the very beginning
of the parse, since, for instance, the stacks are not even set up.
See https://lists.gnu.org/r/bison-patches/2020-03/msg00001.html.

The fix could have very simple, but the documentation actually makes
it very clear that we can reuse a pstate for several parses:

    After yypush_parse returns a status other than YYPUSH_MORE, the
    parser instance yyps may be reused for a new parse.

so we need to restore the parser to its pristine state so that (i) it
is ready to run the next parse, (ii) it properly supports
yyexpected_tokens for the next run.

* data/skeletons/yacc.c (b4_initialize_parser_state_variables): New,
extracted from the top of yyparse/yypush_parse.
(yypstate_clear): New.
(yypstate_new): Use it when push parsers are enabled.
Define after the yyps macros so that we can use the same code as the
regular pull parsers.
(yyparse): Use it when push parsers are _not_ enabled.

* examples/c/bistromathic/bistromathic.test: Check the completion on
the beginning of the line.
---
 TODO                                      |   3 +
 data/skeletons/yacc.c                     | 105 +++++++++++++---------
 examples/c/bistromathic/bistromathic.test |  10 +++
 examples/c/bistromathic/local.mk          |   2 +-
 4 files changed, 76 insertions(+), 44 deletions(-)

diff --git a/TODO b/TODO
index 65f9148d..46cc29ad 100644
--- a/TODO
+++ b/TODO
@@ -95,6 +95,9 @@ See also the item "$undefined" below.
 Consider deprecating impure push parsers.  They add a lot of complexity, for
 a bad feature.
 
+It's not clear to me that yyerror_range really needs to be in pstate,
+instead of just being a local variable.
+
 * Bison 3.7
 ** Unit rules / Injection rules (Akim Demaille)
 Maybe we could expand unit rules (or "injections", see
diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c
index 9de6f7b0..38c73977 100644
--- a/data/skeletons/yacc.c
+++ b/data/skeletons/yacc.c
@@ -195,6 +195,9 @@ m4_define([b4_declare_parser_state_variables],
        Refer to the stacks through separate pointers, to allow yyoverflow
        to reallocate them elsewhere.  */
 
+    /* Their size.  */
+    YYPTRDIFF_T yystacksize;
+
     /* The state stack.  */
     yy_state_t yyssa[YYINITDEPTH];
     yy_state_t *yyss;
@@ -211,15 +214,34 @@ m4_define([b4_declare_parser_state_variables],
     YYLTYPE *yylsp;
 
     /* The locations where the error started and ended.  */
-    YYLTYPE yyerror_range[3];]])[
-
-    YYPTRDIFF_T yystacksize;]b4_lac_if([[
+    YYLTYPE yyerror_range[3];]])[]b4_lac_if([[
 
     yy_state_t 
yyesa@{]b4_percent_define_get([[parse.lac.es-capacity-initial]])[@};
     yy_state_t *yyes;
     YYPTRDIFF_T yyes_capacity;]])])
 
 
+# b4_initialize_parser_state_variables
+# ------------------------------------
+# Initialize these variables.
+m4_define([b4_initialize_parser_state_variables],
+[[  yynerrs = 0;
+  yystate = 0;
+  yyerrstatus = 0;
+
+  yystacksize = YYINITDEPTH;
+  yyssp = yyss = yyssa;
+  yyvsp = yyvs = yyvsa;]b4_locations_if([[
+  yylsp = yyls = yylsa;]])[]b4_lac_if([[
+
+  yyes = yyesa;
+  yyes_capacity = ]b4_percent_define_get([[parse.lac.es-capacity-initial]])[;
+  if (YYMAXDEPTH < yyes_capacity)
+    yyes_capacity = YYMAXDEPTH;]])[
+]])
+
+
+
 # _b4_declare_yyparse_push
 # ------------------------
 # Declaration of yyparse (and dependencies) when using the push parser
@@ -328,6 +350,7 @@ m4_if(b4_api_prefix, [yy], [],
 #define yypush_parse    ]b4_prefix[push_parse]b4_pull_if([[
 #define yypull_parse    ]b4_prefix[pull_parse]])[
 #define yypstate_new    ]b4_prefix[pstate_new
+#define yypstate_clear  ]b4_prefix[pstate_clear
 #define yypstate_delete ]b4_prefix[pstate_delete
 #define yypstate        ]b4_prefix[pstate]])[
 #define yylex           ]b4_prefix[lex
@@ -1382,8 +1405,7 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
 ]b4_push_if([[
 struct yypstate
   {]b4_declare_parser_state_variables[
-    /* Used to determine if this is the first time this instance has
-       been used.  */
+    /* Whether this instance has not started parsing yet.  */
     int yynew;
   };]b4_pure_if([], [[
 
@@ -1425,6 +1447,31 @@ yypull_parse (yypstate *yyps]b4_user_formals[)
   return yystatus;
 }]])[
 
+]b4_pure_if([[#define ]b4_prefix[nerrs yyps->]b4_prefix[nerrs]])[
+#define yystate yyps->yystate
+#define yyerrstatus yyps->yyerrstatus
+#define yyssa yyps->yyssa
+#define yyss yyps->yyss
+#define yyssp yyps->yyssp
+#define yyvsa yyps->yyvsa
+#define yyvs yyps->yyvs
+#define yyvsp yyps->yyvsp]b4_locations_if([[
+#define yylsa yyps->yylsa
+#define yyls yyps->yyls
+#define yylsp yyps->yylsp
+#define yyerror_range yyps->yyerror_range]])[
+#define yystacksize yyps->yystacksize]b4_lac_if([[
+#define yyesa yyps->yyesa
+#define yyes yyps->yyes
+#define yyes_capacity yyps->yyes_capacity]])[
+
+/* Initialize the parser data structure.  */
+void
+yypstate_clear (yypstate *yyps)
+{
+]b4_initialize_parser_state_variables[
+}
+
 /* Initialize the parser data structure.  */
 yypstate *
 yypstate_new (void)
@@ -1437,6 +1484,7 @@ yypstate_new (void)
     return YY_NULLPTR;
   yyps->yynew = 1;]b4_pure_if([], [[
   yypstate_allocated = 1;]])[
+  yypstate_clear (yyps);
   return yyps;
 }
 
@@ -1448,33 +1496,15 @@ yypstate_delete (yypstate *yyps)
 #ifndef yyoverflow
       /* If the stack was reallocated but the parse did not complete, then the
          stack still needs to be freed.  */
-      if (!yyps->yynew && yyps->yyss != yyps->yyssa)
-        YYSTACK_FREE (yyps->yyss);
+      if (!yyps->yynew && yyss != yyssa)
+        YYSTACK_FREE (yyss);
 #endif]b4_lac_if([[
-      if (!yyps->yynew && yyps->yyes != yyps->yyesa)
-        YYSTACK_FREE (yyps->yyes);]])[
+      if (!yyps->yynew && yyes != yyesa)
+        YYSTACK_FREE (yyes);]])[
       free (yyps);]b4_pure_if([], [[
       yypstate_allocated = 0;]])[
     }
 }
-]b4_pure_if([[
-#define ]b4_prefix[nerrs yyps->]b4_prefix[nerrs]])[
-#define yystate yyps->yystate
-#define yyerrstatus yyps->yyerrstatus
-#define yyssa yyps->yyssa
-#define yyss yyps->yyss
-#define yyssp yyps->yyssp
-#define yyvsa yyps->yyvsa
-#define yyvs yyps->yyvs
-#define yyvsp yyps->yyvsp]b4_locations_if([[
-#define yylsa yyps->yylsa
-#define yyls yyps->yyls
-#define yylsp yyps->yylsp
-#define yyerror_range yyps->yyerror_range]])[
-#define yystacksize yyps->yystacksize]b4_lac_if([[
-#define yyesa yyps->yyesa
-#define yyes yyps->yyes
-#define yyes_capacity yyps->yyes_capacity]])[
 ]])[
 
 ]b4_push_if([[
@@ -1522,29 +1552,17 @@ yyparse (]m4_ifset([b4_parse_param], 
[b4_formals(b4_parse_param)], [void])[)]])[
 
   /* The number of symbols on the RHS of the reduced rule.
      Keep to zero when no symbol should be popped.  */
-  int yylen = 0;]b4_push_if([[
-
+  int yylen = 0;
+]b4_push_if([[
   if (!yyps->yynew)
     {
       yyn = yypact[yystate];
       goto yyread_pushed_token;
-    }]])[
-
-  yyssp = yyss = yyssa;
-  yyvsp = yyvs = yyvsa;]b4_locations_if([[
-  yylsp = yyls = yylsa;]])[
-  yystacksize = YYINITDEPTH;]b4_lac_if([[
-
-  yyes = yyesa;
-  yyes_capacity = ]b4_percent_define_get([[parse.lac.es-capacity-initial]])[;
-  if (YYMAXDEPTH < yyes_capacity)
-    yyes_capacity = YYMAXDEPTH;]])[
+    }]], [
+b4_initialize_parser_state_variables])[
 
   YYDPRINTF ((stderr, "Starting parse\n"));
 
-  yystate = 0;
-  yyerrstatus = 0;
-  yynerrs = 0;
   yychar = YYEMPTY; /* Cause a token to be read.  */
 ]m4_ifdef([b4_initial_action], [
 b4_dollar_pushdef([m4_define([b4_dollar_dollar_used])yylval], [], [],
@@ -2033,6 +2051,7 @@ yyreturn:
 #endif]b4_lac_if([[
   if (yyes != yyesa)
     YYSTACK_FREE (yyes);]])b4_push_if([[
+  yypstate_clear (yyps);
   yyps->yynew = 1;
 
 
diff --git a/examples/c/bistromathic/bistromathic.test 
b/examples/c/bistromathic/bistromathic.test
index a877bea3..4f64b1a1 100755
--- a/examples/c/bistromathic/bistromathic.test
+++ b/examples/c/bistromathic/bistromathic.test
@@ -102,3 +102,13 @@ sed -e 's/\\t/     /g' >input <<EOF
 EOF
 run 0 '> (atan ( ''
 > err: 1.9: syntax error: expected - or ( or double precision number or 
 > function or variable before end of file'
+
+# Check the completion at the very beginning.
+sed -e 's/\\t/ /g' >input <<EOF
+e\t\t
+EOF
+run 0 '> e
+end of file  exit         exp          ''
+> e
+0
+> err: '
diff --git a/examples/c/bistromathic/local.mk b/examples/c/bistromathic/local.mk
index 920265f6..cab026e2 100644
--- a/examples/c/bistromathic/local.mk
+++ b/examples/c/bistromathic/local.mk
@@ -23,7 +23,7 @@ check_PROGRAMS += %D%/bistromathic
 TESTS += %D%/bistromathic.test
 EXTRA_DIST += %D%/bistromathic.test
 nodist_%C%_bistromathic_SOURCES = %D%/parse.y %D%/parse.h
-%D%/calc.c: $(dependencies)
+%D%/parse.c: $(dependencies)
 
 # Don't use gnulib's system headers.
 %C%_bistromathic_CPPFLAGS = -I$(top_srcdir)/%D% -I$(top_builddir)/%D%
-- 
2.25.1