yacc: comply with recent POSIX updates: declare yyerror and yylex

[Akim Demaille]

Hi Paul,

I'd be happy to have your opinion on this, particularly the wording.

Cheers!

commit fda40895114e5fdeb8c5fc3091c259a3952babac
Author: Akim Demaille <akim.demaille@gmail.com>
Date:   Thu Aug 5 08:39:24 2021 +0200

    yacc: comply with recent POSIX updates: declare yyerror and yylex
    
    In POSIX Yacc mode, declare yyerror and yylex unless already #defined,
    or if YYERROR_IS_DECLARED/YYLEX_IS_DECLARED are defined (for
    consistency with Bison's YYSTYPE_IS_DECLARED/YYLTYPE_IS_DECLARED).
    See <https://austingroupbugs.net/view.php?id=1388#c5220>.
    
    * data/skeletons/c.m4 (b4_function_declare): Resurect.
    (b4_lex_formals): Since we will possibly expose this prototype
    in the header, take the prefix into account.
    * data/skeletons/yacc.c (b4_declare_yyerror_and_yylex): New.
    (b4_shared_declarations): Use it.
    
    * tests/local.at (AT_YACC_IF): New.
    When in Yacc mode, set the `yacc` Autotest keyword.
    (AT_YYERROR_DECLARE(c)): Don't declare in Yacc mode,
    to avoid clashes (since this signature is static).
    (AT_YYERROR_DEFINE(c)): Don't define as static in Yacc mode.
    * tests/regression.at (Early token definitions with --yacc): Specify
    that we are in Yacc mode.

diff --git a/NEWS b/NEWS
index 8cecaf68..a908bdb0 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,14 @@ GNU Bison NEWS
   now generates a *.gv file by default, instead of *.dot.  A transition
   started in Bison 3.4.
 
+  To comply with the latest POSIX standard, in Yacc compatibility mode
+  (options `-y`/`--yacc`) Bison now generates prototypes for yyerror and
+  yylex.  In some situations, this is breaking compatibility: if the user
+  has already declared these functions but with some differences (e.g., to
+  declare them as static, or to use specific attributes), the generated
+  parser will fail to compile.  To disable these prototypes, #define yyerror
+  (to `yyerror`), and likewise for yylex.
+
 ** Deprecated features
 
   Support for the YYPRINT macro is removed. It worked only with yacc.c and
diff --git a/TODO b/TODO
index 2eab7097..2e33075c 100644
--- a/TODO
+++ b/TODO
@@ -1,8 +1,4 @@
 * Soon
-** POSIX updates
-See the recent changes about function prototypes in POSIX Yacc.  Implement
-them.
-
 ** Missing tests
 commit c22902e360e0fbbe9fd5657dcf107e03166da309
 Author: Akim Demaille <akim.demaille@gmail.com>
diff --git a/data/skeletons/c.m4 b/data/skeletons/c.m4
index a0e35ae0..2a992f51 100644
--- a/data/skeletons/c.m4
+++ b/data/skeletons/c.m4
@@ -112,8 +112,8 @@ b4_percent_define_default([[api.symbol.prefix]], 
[[YYSYMBOL_]])
 # All the yylex formal arguments.
 # b4_lex_param arrives quoted twice, but we want to keep only one level.
 m4_define([b4_lex_formals],
-[b4_pure_if([[[[YYSTYPE *yylvalp]], [[&yylval]]][]dnl
-b4_locations_if([, [[YYLTYPE *yyllocp], [&yylloc]]])])dnl
+[b4_pure_if([[[b4_api_PREFIX[STYPE *yylvalp]], [[&yylval]]][]dnl
+b4_locations_if([, [b4_api_PREFIX[LTYPE *yyllocp], [&yylloc]]])])dnl
 m4_ifdef([b4_lex_param], [, ]b4_lex_param)])
 
 
@@ -662,6 +662,14 @@ m4_define([b4_formal],
 [$1])
 
 
+# b4_function_declare(NAME, RETURN-VALUE, [DECL1, NAME1], ...)
+# ------------------------------------------------------------
+# Declare the function NAME.
+m4_define([b4_function_declare],
+[$2 $1 (b4_formals(m4_shift2($@)));[]dnl
+])
+
+
 
 ## --------------------- ##
 ## Calling C functions.  ##
diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c
index 98671322..3fa8b14f 100644
--- a/data/skeletons/yacc.c
+++ b/data/skeletons/yacc.c
@@ -102,6 +102,16 @@ m4_define([b4_yyerror_arg_loc_if],
                           [1], [m4_ifset([b4_parse_param], [$1])],
                           [2], [$1])])])
 
+# b4_yyerror_formals
+# ------------------
+m4_define([b4_yyerror_formals],
+[b4_pure_if([b4_locations_if([, [[const ]b4_api_PREFIX[LTYPE *yyllocp], 
[&yylloc]]])[]dnl
+m4_ifdef([b4_parse_param], [, b4_parse_param])[]dnl
+,])dnl
+[[const char *msg], [msg]]])
+
+
+
 # b4_yyerror_args
 # ---------------
 # Arguments passed to yyerror: user args plus yylloc.
@@ -352,17 +362,32 @@ m4_define([b4_declare_yyparse],
 ])
 
 
+# b4_declare_yyerror_and_yylex
+# ----------------------------
+# Comply with POSIX Yacc.
+# <https://austingroupbugs.net/view.php?id=1388#c5220>
+m4_define([b4_declare_yyerror_and_yylex],
+[b4_yacc_if([[#if !defined ]b4_prefix[error && !defined 
]b4_api_PREFIX[ERROR_IS_DECLARED
+]b4_function_declare([b4_prefix[error]], void, b4_yyerror_formals)[
+#endif
+#if !defined ]b4_prefix[lex && !defined ]b4_api_PREFIX[LEX_IS_DECLARED
+]b4_function_declare([b4_prefix[lex]], int, b4_lex_formals)[
+#endif
+]])dnl
+])
+
 
 # b4_shared_declarations
 # ----------------------
-# Declaration that might either go into the header (if --header)
-# or open coded in the parser body.
+# Declarations that might either go into the header (if --header)
+# or into the implementation file.
 m4_define([b4_shared_declarations],
 [b4_cpp_guard_open([b4_spec_mapped_header_file])[
 ]b4_declare_yydebug[
 ]b4_percent_code_get([[requires]])[
 ]b4_token_enums_defines[
 ]b4_declare_yylstype[
+]b4_declare_yyerror_and_yylex[
 ]b4_declare_yyparse[
 ]b4_percent_code_get([[provides]])[
 ]b4_cpp_guard_close([b4_spec_mapped_header_file])[]dnl
diff --git a/doc/bison.texi b/doc/bison.texi
index beee6f92..71fb7245 100644
--- a/doc/bison.texi
+++ b/doc/bison.texi
@@ -6117,11 +6117,13 @@ @node Decl Summary
 @end deffn
 
 @deffn {Directive} %yacc
-Pretend the option @option{--yacc} was given, i.e., imitate Yacc, including
-its naming conventions.  Only makes sense with the @file{yacc.c}
+Pretend the option @option{--yacc} was given
+(@pxref{option-yacc,,@option{--yacc}}), i.e., imitate Yacc, including its
+naming conventions.  Only makes sense with the @file{yacc.c}
 skeleton. @xref{Tuning the Parser}, for more.
 
-Of course @code{%yacc} is a Bison extension@dots{}
+Of course, being a Bison extension, @code{%yacc} is somewhat
+self-contradictory@dots{}
 @end deffn
 
 
@@ -11826,8 +11828,9 @@ @node Tuning the Parser
 @item -p @var{prefix}
 @itemx --name-prefix=@var{prefix}
 Pretend that @code{%name-prefix "@var{prefix}"} was specified (@pxref{Decl
-Summary}).  Obsoleted by @option{-Dapi.prefix=@var{prefix}}.  @xref{Multiple
-Parsers}.
+Summary}).  The option @option{-p} is specified by POSIX.  When POSIX
+compatibility is not a requirement, @option{-Dapi.prefix=@var{prefix}} is a
+better option (@pxref{Multiple Parsers}).
 
 @item -l
 @itemx --no-lines
@@ -11859,26 +11862,46 @@ @node Tuning the Parser
 Pretend that @code{%token-table} was specified.  @xref{Decl Summary}.
 
 @item -y
-@itemx --yacc
-Act more like the traditional @command{yacc} command.  This can cause
-different diagnostics to be generated (it implies @option{-Wyacc}), and may
-change behavior in other minor ways.  Most importantly, imitate Yacc's
-output file name conventions, so that the parser implementation file is
-called @file{y.tab.c}, and the other outputs are called @file{y.output} and
-@file{y.tab.h}.  Also, generate @code{#define} statements in addition to an
-@code{enum} to associate token codes with token kind names.  Thus, the
-following shell script can substitute for Yacc, and the Bison distribution
-contains such a script for compatibility with POSIX:
-
+@itemx @anchor{option-yacc} --yacc
+Act more like the traditional @command{yacc} command:
+@itemize
+@item
+Generate different diagnostics (it implies @option{-Wyacc}).
+@item
+Generate @code{#define} statements in addition to an @code{enum} to
+associate token codes with token kind names.
+@item
+Generate prototypes for @code{yyerror} and @code{yylex} (since Bison 3.8):
 @example
-#! /bin/sh
-bison -y "$@@"
+int yylex (void);
+void yyerror (const char *);
 @end example
+As a Bison extension, additional arguments required by @code{%pure-parser},
+@code{%locations}, @code{%lex-param} and @code{%parse-param} are taken into
+account.  You may disable @code{yyerror}'s prototype with @samp{#define
+yyerror yyerror} (as specified by POSIX), or with @samp{#define
+YYERROR_IS_DECLARED} (a Bison extension).  Likewise for @code{yylex}.
+@item
+Imitate Yacc's output file name conventions, so that the parser
+implementation file is called @file{y.tab.c}, and the other outputs are
+called @file{y.output} and @file{y.tab.h}.  Do not use @option{--yacc} just
+to change the output file names since it also triggers all the
+aforementioned behavior changes; rather use @samp{-o y.tab.c}.
+@end itemize
 
 The @option{-y}/@option{--yacc} option is intended for use with traditional
 Yacc grammars.  This option only makes sense for the default C skeleton,
 @file{yacc.c}.  If your grammar uses Bison extensions Bison cannot be
 Yacc-compatible, even if this option is specified.
+
+Thus, the following shell script can substitute for Yacc, and the Bison
+distribution contains such a @command{yacc} script for compatibility with
+POSIX:
+
+@example
+#! /bin/sh
+bison -y "$@@"
+@end example
 @end table
 
 @node Output Files
diff --git a/tests/README.md b/tests/README.md
index aa409a54..970fb3e9 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -27,6 +27,7 @@ synonyms.
 - report: for automaton dumps
 - %union
 - variant
+- yacc: POSIX yacc (%yacc, -y, --yacc)
 
 # Calculator
 The grammar features several special directives:
diff --git a/tests/local.at b/tests/local.at
index 5dde77c2..2231461b 100644
--- a/tests/local.at
+++ b/tests/local.at
@@ -266,6 +266,8 @@ m4_pushdef([AT_MULTISTART_IF],
 [m4_bmatch([$3], [%start [_a-zA-Z]+ [_a-zA-Z]+], [$1], [$2])])
 m4_pushdef([AT_PARAM_IF],
 [m4_bmatch([$3], [%parse-param], [$1], [$2])])
+m4_pushdef([AT_YACC_IF],
+[m4_bmatch([$3], [%yacc], [$1], [$2])])
 
 # Comma-terminated list of formals parse-parameters.
 # E.g., %parse-param { int x } %parse-param {int y} -> "int x, int y, ".
@@ -418,6 +420,7 @@ m4_pushdef([AT_YYLTYPE],
 AT_GLR_IF([AT_KEYWORDS([glr])])
 AT_MULTISTART_IF([AT_KEYWORDS([multistart])])
 AT_PUSH_IF([AT_KEYWORDS([push])])
+AT_YACC_IF([AT_KEYWORDS([yacc])])
 ])# _AT_BISON_OPTION_PUSHDEFS
 
 
@@ -457,6 +460,7 @@ m4_define([AT_BISON_OPTION_POPDEFS],
 m4_popdef([AT_PUSH_IF])
 m4_popdef([AT_PURE_IF])
 m4_popdef([AT_PARSER_CLASS])
+m4_popdef([AT_YACC_IF])
 m4_popdef([AT_PARAM_IF])
 m4_popdef([AT_MULTISTART_IF])
 m4_popdef([AT_LEXPARAM_IF])
@@ -674,7 +678,7 @@ m4_define([AT_YYERROR_DECLARE_EXTERN(c)],
 m4_define([AT_YYERROR_DECLARE(c)],
 [[#include <stdio.h>
 ]AT_LOCATION_PRINT_DECLARE[
-static ]AT_YYERROR_DECLARE_EXTERN])
+]AT_YACC_IF([], [[static ]AT_YYERROR_DECLARE_EXTERN])])
 
 
 # "%define parse.error custom" uses a different format, easy to check.
@@ -720,7 +724,7 @@ m4_define([AT_YYERROR_DEFINE(c)],
 ]])[
 
 /* A C error reporting function.  */
-static
+]AT_YACC_IF([], [static])[
 ]AT_YYERROR_PROTOTYPE[
 {]m4_bpatsubst(m4_defn([AT_PARSE_PARAMS]),
               [[^,]+[^A-Za-z_0-9]\([A-Za-z_][A-Za-z_0-9]*\),* *], [
diff --git a/tests/regression.at b/tests/regression.at
index 52ef5e15..464ec247 100644
--- a/tests/regression.at
+++ b/tests/regression.at
@@ -87,7 +87,7 @@ AT_SETUP([Early token definitions with --yacc])
 # Found in GCJ: they expect the tokens to be defined before the user
 # prologue, so that they can use the token definitions in it.
 
-AT_BISON_OPTION_PUSHDEFS
+AT_BISON_OPTION_PUSHDEFS([%yacc])
 
 # Not AT_DATA_GRAMMAR, which uses %code, which is not supported by Yacc.
 AT_DATA([input.y],
@@ -112,7 +112,7 @@ AT_SETUP([Early token definitions with --yacc])
 ]])
 AT_BISON_OPTION_POPDEFS
 
-AT_BISON_CHECK([-y -o input.c input.y])
+AT_BISON_CHECK([--yacc -o input.c input.y])
 AT_COMPILE([input.o])
 
 AT_CLEANUP