c++: fix token constructors for types with commas

[Akim Demaille]

I'm afraid I should release 3.2.3 because of this issue.
I was working on Frank's proposal about make_symbol when I
fell onto this.

That's stupid, I should have seen it coming.

commit dcaceccd4dc5f905641cae58b467789c44fbf5c5
Author: Akim Demaille <address@hidden>
Date:   Tue Dec 18 13:22:03 2018 +0100

    c++: fix token constructors for types with commas
    
    Bitten by macros, again.
    See 680b715518795c8648360fcf05f3772c04d2eed2.
    
    * data/variant.hh (_b4_symbol_constructor_declare)
    (_b4_symbol_constructor_define): Do not use user types, which can
    include commas as in `std::pair<int, int>`, to macros.
    
    * tests/local.at: Adjust the lex related macros to support the
    case of token constructors.
    * tests/types.at: Also check token constructors on types with commas.

diff --git a/data/variant.hh b/data/variant.hh
index 47ff9cbd..1b71eba2 100644
--- a/data/variant.hh
+++ b/data/variant.hh
@@ -340,13 +340,21 @@ m4_define([b4_symbol_value_template],
 # Declare make_SYMBOL for SYMBOL-NUM.  Use at class-level.
 m4_define([_b4_symbol_constructor_declare],
 [b4_symbol_if([$1], [is_token], [b4_symbol_if([$1], [has_id],
-[    static
+[# if 201103L <= YY_CPLUSPLUS
+    static
     symbol_type
     make_[]_b4_symbol([$1], [id]) (dnl
 b4_join(b4_symbol_if([$1], [has_type],
-                     [YY_COPY (b4_symbol([$1], [type])) v]),
-        b4_locations_if([YY_COPY (location_type) l])));
-
+                     [b4_symbol([$1], [type]) v]),
+        b4_locations_if([location_type l])));
+#else
+    static
+    symbol_type
+    make_[]_b4_symbol([$1], [id]) (dnl
+b4_join(b4_symbol_if([$1], [has_type],
+                     [const b4_symbol([$1], [type])& v]),
+        b4_locations_if([const location_type& l])));
+#endif
 ])])])
 
 
@@ -365,18 +373,31 @@ b4_symbol_foreach([_b4_symbol_constructor_declare])])
 # Define make_SYMBOL for SYMBOL-NUM.
 m4_define([_b4_symbol_constructor_define],
 [b4_symbol_if([$1], [is_token], [b4_symbol_if([$1], [has_id],
-[  inline
+[# if 201103L <= YY_CPLUSPLUS
+  inline
   b4_parser_class_name::symbol_type
   b4_parser_class_name::make_[]_b4_symbol([$1], [id]) (dnl
 b4_join(b4_symbol_if([$1], [has_type],
-                     [YY_COPY (b4_symbol([$1], [type])) v]),
-        b4_locations_if([YY_COPY (location_type) l])))
+                     [b4_symbol([$1], [type]) v]),
+        b4_locations_if([location_type l])))
   {
     return symbol_type (b4_join([token::b4_symbol([$1], [id])],
-                                b4_symbol_if([$1], [has_type], [YY_MOVE (v)]),
-                                b4_locations_if([YY_MOVE (l)])));
+                                b4_symbol_if([$1], [has_type], [std::move 
(v)]),
+                                b4_locations_if([std::move (l)])));
   }
-
+#else
+  inline
+  b4_parser_class_name::symbol_type
+  b4_parser_class_name::make_[]_b4_symbol([$1], [id]) (dnl
+b4_join(b4_symbol_if([$1], [has_type],
+                     [const b4_symbol([$1], [type])& v]),
+        b4_locations_if([const location_type& l])))
+  {
+    return symbol_type (b4_join([token::b4_symbol([$1], [id])],
+                                b4_symbol_if([$1], [has_type], [v]),
+                                b4_locations_if([l])));
+  }
+#endif
 ])])])
 
 
diff --git a/tests/local.at b/tests/local.at
index b31b7719..739ae2ed 100644
--- a/tests/local.at
+++ b/tests/local.at
@@ -248,16 +248,25 @@ m4_pushdef([AT_YYLTYPE],
 [AT_CXX_IF([AT_NAMESPACE[::parser::location_type]],
                [AT_API_PREFIX[LTYPE]])])
 
-
-AT_PURE_LEX_IF(
+AT_TOKEN_CTOR_IF(
+[m4_pushdef([AT_LOC], [[(]AT_NAME_PREFIX[lloc)]])
+ m4_pushdef([AT_VAL], [[(]AT_NAME_PREFIX[lval)]])
+ m4_pushdef([AT_YYLEX_FORMALS],     [])
+ m4_pushdef([AT_YYLEX_RETURN],      [yy::parser::symbol_type])
+ m4_pushdef([AT_YYLEX_ARGS],        [])
+ m4_pushdef([AT_USE_LEX_ARGS],      [])
+ m4_pushdef([AT_YYLEX_PRE_FORMALS], [])
+ m4_pushdef([AT_YYLEX_PRE_ARGS],    [])],
+[AT_PURE_LEX_IF(
 [m4_pushdef([AT_LOC], [(*llocp)])
  m4_pushdef([AT_VAL], [(*lvalp)])
  m4_pushdef([AT_YYLEX_FORMALS],
             [AT_YYSTYPE *lvalp[]AT_LOCATION_IF([, AT_YYLTYPE *llocp])])
+ m4_pushdef([AT_YYLEX_RETURN], [int])
  m4_pushdef([AT_YYLEX_ARGS],
             [lvalp[]AT_LOCATION_IF([, llocp])])
  m4_pushdef([AT_USE_LEX_ARGS],
-            [(void) lvalp;AT_LOCATION_IF([(void) llocp])])
+            [(void) lvalp;AT_LOCATION_IF([(void) llocp;])])
  m4_pushdef([AT_YYLEX_PRE_FORMALS],
             [AT_YYLEX_FORMALS, ])
  m4_pushdef([AT_YYLEX_PRE_ARGS],
@@ -266,11 +275,12 @@ AT_PURE_LEX_IF(
 [m4_pushdef([AT_LOC], [[(]AT_NAME_PREFIX[lloc)]])
  m4_pushdef([AT_VAL], [[(]AT_NAME_PREFIX[lval)]])
  m4_pushdef([AT_YYLEX_FORMALS],     [void])
+ m4_pushdef([AT_YYLEX_RETURN],      [int])
  m4_pushdef([AT_YYLEX_ARGS],        [])
- m4_pushdef([AT_USE_LEX_ARGS],    [])
+ m4_pushdef([AT_USE_LEX_ARGS],      [])
  m4_pushdef([AT_YYLEX_PRE_FORMALS], [])
  m4_pushdef([AT_YYLEX_PRE_ARGS],    [])
-])
+])])
 
 # Handle the different types of location components.
 
@@ -403,7 +413,7 @@ $2])
 # ACTION may compute yylval for instance, using "res" as token type,
 # and "toknum" as the number of calls to yylex (starting at 0).
 m4_define([AT_YYLEX_PROTOTYPE],
-[int AT_NAME_PREFIX[]lex (]AT_YYLEX_FORMALS[)[]dnl
+[AT_YYLEX_RETURN AT_NAME_PREFIX[]lex (]AT_YYLEX_FORMALS[)[]dnl
 ])
 
 m4_define([AT_YYLEX_DECLARE_EXTERN],
@@ -424,15 +434,15 @@ static
              [[static int const input[] = ]$1])[;
   static size_t toknum = 0;
   int res;
-  ]AT_USE_LEX_ARGS[;
+  ]AT_USE_LEX_ARGS[
   assert (toknum < sizeof input / sizeof input[0]);
   res = input[toknum++];
-  ]$2[;]AT_LOCATION_IF([[
+  ]$2[;]AT_TOKEN_CTOR_IF([], [[
+  ]AT_LOCATION_IF([[
   ]AT_LOC_FIRST_LINE[ = ]AT_LOC_LAST_LINE[ = 1;
   ]AT_LOC_FIRST_COLUMN[ = ]AT_LOC_LAST_COLUMN[ = ]AT_CXX_IF([(unsigned )], 
[(int)])[toknum;]])[
-  return res;
-}]dnl
-])
+  return res;]])[
+}]])
 
 # AT_YYERROR_FORMALS
 # AT_YYERROR_PROTOTYPE
diff --git a/tests/types.at b/tests/types.at
index 1bbb958d..71110083 100644
--- a/tests/types.at
+++ b/tests/types.at
@@ -288,7 +288,6 @@ m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], 
[glr.cc]],
                AT_VAL.build (std::make_pair<std::string, std::string> ("two", 
"deux"));],
             [10:11, two:deux])
 
-    # Move-only types, and variadic emplace.
     AT_TEST([%skeleton "]b4_skel["
              %code requires { #include <memory> }
              %define api.value.type variant],
@@ -304,7 +303,29 @@ m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], 
[glr.cc]],
              else if (res == '2')
                ]AT_VAL[.emplace <std::pair<int, int>> (21, 22);]],
             [10, 21, 22],
-            [AT_REQUIRE_CXX_STD(14, [echo "$at_std not supported"; 
continue])])])
+            [AT_REQUIRE_CXX_STD(14, [echo "$at_std not supported"; continue])])
+
+    AT_TEST([%skeleton "]b4_skel["
+             %code requires { #include <memory> }
+             %define api.value.type variant
+             %define api.token.constructor],
+            [[%token <std::unique_ptr<int>> ONE;
+             %token <std::pair<int, int>> TWO;
+             %token EOI 0;]],
+            [ONE TWO { std::cout << *$1 << ", "
+                                 << $2.first << ", "
+                                 << $2.second << '\n'; }],
+            ["12"],
+            [[if (res == '1')
+               return yy::parser::make_ONE (std::make_unique<int> (10));
+             else if (res == '2')
+               return yy::parser::make_TWO (std::make_pair<int, int> (21, 22));
+             else
+               return yy::parser::make_EOI ()]],
+            [10, 21, 22],
+            [AT_REQUIRE_CXX_STD(14, [echo "$at_std not supported"; continue])])
+
+    ])
   ])
 ])