Fix %error-verbose for conflicts resolved by %nonassoc.

[Joel E. Denny]

Verbose syntax error messages used to include error action tokens in the 
expected token list.  There was a FIXME in conflicts.at about this.  To 
fix it, I pushed the following patches to master and similar ones to 
branch-2.5.  I feel this area is a bit too subtle to include this change 
in branch-2.4.2.

I'm not proud of propagating more preprocessor macros to yacc.c.  Maybe 
this is a good place for some more functions like the ones Akim has 
discussed recently.  Also, we could probably find better names than NINF.  
For example, error_action or default_action.

Anyway, for now, I mainly wanted to fix a bug.

>From 87412882128fc3ae807f47db23884552f5841e74 Mon Sep 17 00:00:00 2001
From: Joel E. Denny <address@hidden>
Date: Tue, 25 Aug 2009 01:12:37 -0400
Subject: [PATCH] Some code and documentation improvements.

* data/c.m4 (b4_table_value_equals): New macro to capture
some repeated code.
* data/glr.c (yyis_pact_ninf): Use it here.
(yyis_table_ninf): Likewise.
(yyreportSyntaxError): Improve internal comments.
* data/yacc.c (yyis_pact_ninf): New macro copied from glr.c.
Use it everywhere possible.
(yyis_table_ninf): Likewise.
(yysyntax_error): Improve internal comments.
* data/lalr1.cc (yysyntax_error_): Likewise.
* data/lalr1.java (yysyntax_error): Likewise.
* src/tables.h: Improve comments about yypact, yytable, etc.
---
 ChangeLog       |   16 ++++++++++++++++
 data/c.m4       |   10 ++++++++++
 data/glr.c      |   11 ++++-------
 data/lalr1.cc   |    3 ++-
 data/lalr1.java |    3 ++-
 data/yacc.c     |   15 +++++++++++----
 src/tables.h    |   13 +++++++++++--
 7 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index b831d11..dc11d51 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+2009-08-25  Joel E. Denny  <address@hidden>
+
+       Some code and documentation improvements.
+       * data/c.m4 (b4_table_value_equals): New macro to capture
+       some repeated code.
+       * data/glr.c (yyis_pact_ninf): Use it here.
+       (yyis_table_ninf): Likewise.
+       (yyreportSyntaxError): Improve internal comments.
+       * data/yacc.c (yyis_pact_ninf): New macro copied from glr.c.
+       Use it everywhere possible.
+       (yyis_table_ninf): Likewise.
+       (yysyntax_error): Improve internal comments.
+       * data/lalr1.cc (yysyntax_error_): Likewise.
+       * data/lalr1.java (yysyntax_error): Likewise.
+       * src/tables.h: Improve comments about yypact, yytable, etc.
+
 2009-08-21  Joel E. Denny  <address@hidden>
 
        Use locale when quoting.
diff --git a/data/c.m4 b/data/c.m4
index 33a4186..3ba48db 100644
--- a/data/c.m4
+++ b/data/c.m4
@@ -157,6 +157,16 @@ m4_define([b4_int_type_for],
 [b4_int_type($1_min, $1_max)])
 
 
+# b4_table_value_equals(TABLE, VALUE, LITERAL)
+# --------------------------------------------
+# Without inducing a comparison warning from the compiler, check if the
+# literal value LITERAL equals VALUE from table TABLE, which must have
+# TABLE_min and TABLE_max defined.
+m4_define([b4_table_value_equals],
+[m4_if(m4_eval($3 < m4_indir([b4_]$1[_min])
+               || m4_indir([b4_]$1[_max]) < $3), [1],
+       [[YYID (0)]],
+       [[((]$2[) == (]$3[))]])])
 
 ## ---------##
 ## Values.  ##
diff --git a/data/glr.c b/data/glr.c
index a4b921f..b0ad100 100644
--- a/data/glr.c
+++ b/data/glr.c
@@ -954,9 +954,7 @@ yylhsNonterm (yyRuleNum yyrule)
 }
 
 #define yyis_pact_ninf(yystate) \
-  ]m4_if(m4_eval(b4_pact_ninf < b4_pact_min), [1],
-        [0],
-        [((yystate) == YYPACT_NINF)])[
+  ]b4_table_value_equals([[pact]], [[yystate]], [b4_pact_ninf])[
 
 /** True iff LR state STATE has only a default reduction (regardless
  *  of token).  */
@@ -974,9 +972,7 @@ yydefaultAction (yyStateNum yystate)
 }
 
 #define yyis_table_ninf(yytable_value) \
-  ]m4_if(m4_eval(b4_table_ninf < b4_table_min), [1],
-        [YYID (0)],
-        [((yytable_value) == YYTABLE_NINF)])[
+  ]b4_table_value_equals([[table]], [[yytable_value]], [b4_table_ninf])[
 
 /** Set *YYACTION to the action to take in YYSTATE on seeing YYTOKEN.
  *  Result R means
@@ -2048,7 +2044,8 @@ yyreportSyntaxError (yyGLRStack* 
yystackp]b4_user_formals[)
          char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
 
          /* Start YYX at -YYN if negative to avoid negative indexes in
-            YYCHECK.  */
+            YYCHECK.  In other words, skip the first -YYN actions for this
+            state because they are default actions.  */
          int yyxbegin = yyn < 0 ? -yyn : 0;
 
          /* Stay within bounds of both yycheck and yytname.  */
diff --git a/data/lalr1.cc b/data/lalr1.cc
index 2a9316d..5ce0b47 100644
--- a/data/lalr1.cc
+++ b/data/lalr1.cc
@@ -954,7 +954,8 @@ b4_error_verbose_if([state_type yystate, int yytoken],
     if (yypact_ninf_ < yyn && yyn <= yylast_)
       {
        /* Start YYX at -YYN if negative to avoid negative indexes in
-          YYCHECK.  */
+          YYCHECK.  In other words, skip the first -YYN actions for this
+          state because they are default actions.  */
        int yyxbegin = yyn < 0 ? -yyn : 0;
 
        /* Stay within bounds of both yycheck and yytname.  */
diff --git a/data/lalr1.java b/data/lalr1.java
index e68650d..7e274d6 100644
--- a/data/lalr1.java
+++ b/data/lalr1.java
@@ -735,7 +735,8 @@ m4_popdef([b4_at_dollar])])dnl
             StringBuffer res;
 
             /* Start YYX at -YYN if negative to avoid negative indexes in
-               YYCHECK.  */
+               YYCHECK.  In other words, skip the first -YYN actions for this
+               state because they are default actions.  */
             int yyxbegin = yyn < 0 ? -yyn : 0;
 
             /* Stay within bounds of both yycheck and yytname.  */
diff --git a/data/yacc.c b/data/yacc.c
index fa14cc5..9689223 100644
--- a/data/yacc.c
+++ b/data/yacc.c
@@ -525,8 +525,14 @@ static const ]b4_int_type_for([b4_toknum])[ yytoknum[] =
 
 #define YYPACT_NINF ]b4_pact_ninf[
 
+#define yyis_pact_ninf(yystate) \
+  ]b4_table_value_equals([[pact]], [[yystate]], [b4_pact_ninf])[
+
 #define YYTABLE_NINF ]b4_table_ninf[
 
+#define yyis_table_ninf(yytable_value) \
+  ]b4_table_value_equals([[table]], [[yytable_value]], [b4_table_ninf])[
+
 ]b4_parser_tables_define[
 
 #define yyerrok                (yyerrstatus = 0)
@@ -848,7 +854,8 @@ yysyntax_error (char *yyresult, int yystate, int yytoken)
       char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
 
       /* Start YYX at -YYN if negative to avoid negative indexes in
-        YYCHECK.  */
+        YYCHECK.  In other words, skip the first -YYN actions for this
+        state because they are default actions.  */
       int yyxbegin = yyn < 0 ? -yyn : 0;
 
       /* Stay within bounds of both yycheck and yytname.  */
@@ -1271,7 +1278,7 @@ yybackup:
 
   /* First try to decide what to do without reference to lookahead token.  */
   yyn = yypact[yystate];
-  if (yyn == YYPACT_NINF)
+  if (yyis_pact_ninf (yyn))
     goto yydefault;
 
   /* Not known => get a lookahead token if don't already have one.  */
@@ -1321,7 +1328,7 @@ yyread_pushed_token:]])[
   yyn = yytable[yyn];
   if (yyn <= 0)
     {
-      if (yyn == 0 || yyn == YYTABLE_NINF)
+      if (yyn == 0 || yyis_table_ninf (yyn))
        goto yyerrlab;
       yyn = -yyn;
       goto yyreduce;
@@ -1505,7 +1512,7 @@ yyerrlab1:
   for (;;)
     {
       yyn = yypact[yystate];
-      if (yyn != YYPACT_NINF)
+      if (!yyis_pact_ninf (yyn))
        {
          yyn += YYTERROR;
          if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR)
diff --git a/src/tables.h b/src/tables.h
index a44cdb8..b21fa7b 100644
--- a/src/tables.h
+++ b/src/tables.h
@@ -54,8 +54,15 @@
    something else to do.
 
    YYPACT[S] = index in YYTABLE of the portion describing state S.
-   The lookahead token's type is used to index that portion to find
-   out what to do.
+   The lookahead token's number, I, is used to index that portion of
+   YYTABLE to find out what action to perform.
+
+   If YYPACT[S] == YYPACT_NINF, if YYPACT[S] + I is outside the bounds
+   of YYTABLE (from 0 to YYLAST), or if YYCHECK indicates that I is
+   outside the bounds of the portion for S, then the default action
+   (from YYDEFACT and YYDEFGOTO) should be used instead of YYTABLE.
+   Otherwise, the value YYTABLE[YYPACT[S] + I] should be used even if
+   YYPACT[S] < 0.
 
    If the value in YYTABLE is positive, we shift the token and go to
    that state.
@@ -64,6 +71,8 @@
 
    If the value is zero, the default action from YYDEFACT[S] is used.
 
+   If the value is YYTABLE_NINF, it's a syntax error.
+
    YYPGOTO[I] = the index in YYTABLE of the portion describing what to
    do after reducing a rule that derives variable I + NTOKENS.  This
    portion is indexed by the parser state number, S, as of before the
-- 
1.5.4.3


>From 53f036ce027289d3f5e70c88735b88aa6725381d Mon Sep 17 00:00:00 2001
From: Joel E. Denny <address@hidden>
Date: Tue, 25 Aug 2009 01:13:02 -0400
Subject: [PATCH] Fix %error-verbose for conflicts resolved by %nonassoc.

* NEWS (2.5): Document.
* data/glr.c (yyreportSyntaxError): Fix this by checking
yyis_table_ninf.
* data/yacc.c (yysyntax_error): Likewise.
* data/lalr1.cc (yysyntax_error_): Fix this by checking
yytable_ninf_.
* data/lalr1.java (yysyntax_error): Likewise.
* tests/conflicts.at (%nonassoc and eof): Update expected output
and remove FIXME.
---
 ChangeLog          |   13 +
 NEWS               |    9 +
 data/glr.c         |    3 +-
 data/lalr1.cc      |    3 +-
 data/lalr1.java    |    6 +-
 data/yacc.c        |    3 +-
 src/parse-gram.c   |  652 ++++++++++++++++++++++++++--------------------------
 src/parse-gram.h   |    8 +-
 tests/conflicts.at |    9 +-
 9 files changed, 369 insertions(+), 337 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index dc11d51..6624a0c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,18 @@
 2009-08-25  Joel E. Denny  <address@hidden>
 
+       Fix %error-verbose for conflicts resolved by %nonassoc.
+       * NEWS (2.5): Document.
+       * data/glr.c (yyreportSyntaxError): Fix this by checking
+       yyis_table_ninf.
+       * data/yacc.c (yysyntax_error): Likewise.
+       * data/lalr1.cc (yysyntax_error_): Fix this by checking
+       yytable_ninf_.
+       * data/lalr1.java (yysyntax_error): Likewise.
+       * tests/conflicts.at (%nonassoc and eof): Update expected output
+       and remove FIXME.
+
+2009-08-25  Joel E. Denny  <address@hidden>
+
        Some code and documentation improvements.
        * data/c.m4 (b4_table_value_equals): New macro to capture
        some repeated code.
diff --git a/NEWS b/NEWS
index 54a10d8..7a5009c 100644
--- a/NEWS
+++ b/NEWS
@@ -146,6 +146,15 @@ Bison News
   Bison now warns when a character literal is not of length one.  In
   some future release, Bison will report an error instead.
 
+** Verbose error messages fixed for nonassociative tokens.
+
+  When %error-verbose is specified, syntax error messages produced by
+  the generated parser include the unexpected token as well as a list of
+  expected tokens.  Previously, this list erroneously included tokens
+  that would actually induce a syntax error because conflicts for them
+  were resolved with %nonassoc.  Such tokens are now properly omitted
+  from the list.
+
 * Changes in version 2.4.2 (????-??-??):
 
 ** Detection of GNU M4 1.4.6 or newer during configure is improved.
diff --git a/data/glr.c b/data/glr.c
index b0ad100..85a5aff 100644
--- a/data/glr.c
+++ b/data/glr.c
@@ -2060,7 +2060,8 @@ yyreportSyntaxError (yyGLRStack* 
yystackp]b4_user_formals[)
          yyarg[yycount++] = yytokenName (yytoken);
 
          for (yyx = yyxbegin; yyx < yyxend; ++yyx)
-           if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR)
+           if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR
+               && !yyis_table_ninf (yytable[yyx + yyn]))
              {
                if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
                  {
diff --git a/data/lalr1.cc b/data/lalr1.cc
index 5ce0b47..1e4c856 100644
--- a/data/lalr1.cc
+++ b/data/lalr1.cc
@@ -971,7 +971,8 @@ b4_error_verbose_if([state_type yystate, int yytoken],
         char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
         yyarg[yycount++] = yytname_[yytoken];
        for (int yyx = yyxbegin; yyx < yyxend; ++yyx)
-         if (yycheck_[yyx + yyn] == yyx && yyx != yyterror_)
+         if (yycheck_[yyx + yyn] == yyx && yyx != yyterror_
+             && yytable_[yyx + yyn] != yytable_ninf_)
           {
             if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
             {
diff --git a/data/lalr1.java b/data/lalr1.java
index 7e274d6..646b777 100644
--- a/data/lalr1.java
+++ b/data/lalr1.java
@@ -744,7 +744,8 @@ m4_popdef([b4_at_dollar])])dnl
             int yyxend = yychecklim < yyntokens_ ? yychecklim : yyntokens_;
             int count = 0;
             for (int x = yyxbegin; x < yyxend; ++x)
-              if (yycheck_[x + yyn] == x && x != yyterror_)
+              if (yycheck_[x + yyn] == x && x != yyterror_
+                  && yycheck_[x + yyn] != yytable_ninf_)
                 ++count;
 
             // FIXME: This method of building the message is not compatible
@@ -755,7 +756,8 @@ m4_popdef([b4_at_dollar])])dnl
               {
                 count = 0;
                 for (int x = yyxbegin; x < yyxend; ++x)
-                  if (yycheck_[x + yyn] == x && x != yyterror_)
+                  if (yycheck_[x + yyn] == x && x != yyterror_
+                      && yycheck_[x + yyn] != yytable_ninf_)
                     {
                       res.append (count++ == 0 ? ", expecting " : " or ");
                       res.append (yytnamerr_ (yytname_[x]));
diff --git a/data/yacc.c b/data/yacc.c
index 9689223..acc1118 100644
--- a/data/yacc.c
+++ b/data/yacc.c
@@ -869,7 +869,8 @@ yysyntax_error (char *yyresult, int yystate, int yytoken)
       yyarg[yycount++] = yytname[yytoken];
 
       for (yyx = yyxbegin; yyx < yyxend; ++yyx)
-       if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR)
+       if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR
+           && !yyis_table_ninf (yytable[yyx + yyn]))
          {
            if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
              {
diff --git a/tests/conflicts.at b/tests/conflicts.at
index 98b7d41..f2f7861 100644
--- a/tests/conflicts.at
+++ b/tests/conflicts.at
@@ -99,20 +99,17 @@ AT_BISON_CHECK([-o input.c input.y])
 AT_COMPILE([input])
 
 AT_PARSER_CHECK([./input '0<0'])
-# FIXME: This is an actual bug, but a new one, in the sense that
-# no one has ever spotted it!  The messages are *wrong*: there should
-# be nothing there, it should be expected eof.
 AT_PARSER_CHECK([./input '0<0<0'], [1], [],
-         [syntax error, unexpected '<', expecting '<' or '>'
+         [syntax error, unexpected '<'
 ])
 
 AT_PARSER_CHECK([./input '0>0'])
 AT_PARSER_CHECK([./input '0>0>0'], [1], [],
-         [syntax error, unexpected '>', expecting '<' or '>'
+         [syntax error, unexpected '>'
 ])
 
 AT_PARSER_CHECK([./input '0<0>0'], [1], [],
-         [syntax error, unexpected '>', expecting '<' or '>'
+         [syntax error, unexpected '>'
 ])
 
 AT_CLEANUP
-- 
1.5.4.3