Implement __VA_OPT__

This implements __VA_OPT__, a new preprocessor feature added in C++2A.
The paper can be found here:

http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0306r4.html

gcc/ChangeLog

        * doc/cpp.texi (Variadic Macros): Document __VA_OPT__.

gcc/testsuite/ChangeLog

        * c-c++-common/cpp/va-opt-pedantic.c: New file.
        * c-c++-common/cpp/va-opt.c: New file.
        * c-c++-common/cpp/va-opt-error.c: New file.

libcpp/ChangeLog

        * pch.c (cpp_read_state): Set n__VA_OPT__.
        * macro.c (vaopt_state): New class.
        (_cpp_arguments_ok): Check va_opt flag.
        (replace_args, create_iso_definition): Use vaopt_state.
        * lex.c (lex_identifier_intern): Possibly issue errors for
        __VA_OPT__.
        (lex_identifier): Likewise.
        (maybe_va_opt_error): New function.
        * internal.h (struct lexer_state) <va_args_ok>: Update comment.
        (struct spec_nodes) <n__VA_OPT__>: New field.
        * init.c (struct lang_flags) <va_opt>: New field.
        (lang_defaults): Add entries for C++2A.  Update all entries for
        va_opt.
        (cpp_set_lang): Initialize va_opt.
        * include/cpplib.h (struct cpp_options) <va_opt>: New field.
        * identifiers.c (_cpp_init_hashtable): Initialize n__VA_OPT__.

From-SVN: r254707

8 files changed, 248 insertions, 24 deletions

diff --git a/libcpp/ChangeLog b/libcpp/ChangeLog
index 35bcef9..70c834c 100644
--- a/libcpp/ChangeLog
+++ b/libcpp/ChangeLog
@@ -1,3 +1,22 @@
+2017-11-13  Tom Tromey  <tom@tromey.com>
+
+	* pch.c (cpp_read_state): Set n__VA_OPT__.
+	* macro.c (vaopt_state): New class.
+	(_cpp_arguments_ok): Check va_opt flag.
+	(replace_args, create_iso_definition): Use vaopt_state.
+	* lex.c (lex_identifier_intern): Possibly issue errors for
+	__VA_OPT__.
+	(lex_identifier): Likewise.
+	(maybe_va_opt_error): New function.
+	* internal.h (struct lexer_state) <va_args_ok>: Update comment.
+	(struct spec_nodes) <n__VA_OPT__>: New field.
+	* init.c (struct lang_flags) <va_opt>: New field.
+	(lang_defaults): Add entries for C++2A.  Update all entries for
+	va_opt.
+	(cpp_set_lang): Initialize va_opt.
+	* include/cpplib.h (struct cpp_options) <va_opt>: New field.
+	* identifiers.c (_cpp_init_hashtable): Initialize n__VA_OPT__.
+
 2017-11-13  David Malcolm  <dmalcolm@redhat.com>
 
 	* include/line-map.h (linenum_type): Move this typedef and the
diff --git a/libcpp/identifiers.c b/libcpp/identifiers.c
index 220f9b9..e456fd3 100644
--- a/libcpp/identifiers.c
+++ b/libcpp/identifiers.c
@@ -70,6 +70,8 @@ _cpp_init_hashtable (cpp_reader *pfile, cpp_hash_table *table)
   s->n_false		= cpp_lookup (pfile, DSC("false"));
   s->n__VA_ARGS__       = cpp_lookup (pfile, DSC("__VA_ARGS__"));
   s->n__VA_ARGS__->flags |= NODE_DIAGNOSTIC;
+  s->n__VA_OPT__        = cpp_lookup (pfile, DSC("__VA_OPT__"));
+  s->n__VA_OPT__->flags |= NODE_DIAGNOSTIC;
   s->n__has_include__   = cpp_lookup (pfile, DSC("__has_include__"));
   s->n__has_include_next__ = cpp_lookup (pfile, DSC("__has_include_next__"));
 }
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 5a14858..101b33a 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -478,6 +478,9 @@ struct cpp_options
   /* Nonzero for C++ 2014 Standard digit separators.  */
   unsigned char digit_separators;
 
+  /* Nonzero for C++2a __VA_OPT__ feature.  */
+  unsigned char va_opt;
+
   /* Holds the name of the target (execution) character set.  */
   const char *narrow_charset;
 
diff --git a/libcpp/init.c b/libcpp/init.c
index ecc81e31..8423656 100644
--- a/libcpp/init.c
+++ b/libcpp/init.c
@@ -91,30 +91,31 @@ struct lang_flags
   char digit_separators;
   char trigraphs;
   char utf8_char_literals;
+  char va_opt;
 };
 
 static const struct lang_flags lang_defaults[] =
-{ /*              c99 c++ xnum xid c11 std digr ulit rlit udlit bincst digsep trig u8chlit */
-  /* GNUC89   */  { 0,  0,  1,  0,  0,  0,  1,   0,   0,   0,    0,     0,     0,   0 },
-  /* GNUC99   */  { 1,  0,  1,  1,  0,  0,  1,   1,   1,   0,    0,     0,     0,   0 },
-  /* GNUC11   */  { 1,  0,  1,  1,  1,  0,  1,   1,   1,   0,    0,     0,     0,   0 },
-  /* GNUC17   */  { 1,  0,  1,  1,  1,  0,  1,   1,   1,   0,    0,     0,     0,   0 },
-  /* STDC89   */  { 0,  0,  0,  0,  0,  1,  0,   0,   0,   0,    0,     0,     1,   0 },
-  /* STDC94   */  { 0,  0,  0,  0,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0 },
-  /* STDC99   */  { 1,  0,  1,  1,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0 },
-  /* STDC11   */  { 1,  0,  1,  1,  1,  1,  1,   1,   0,   0,    0,     0,     1,   0 },
-  /* STDC17   */  { 1,  0,  1,  1,  1,  1,  1,   1,   0,   0,    0,     0,     1,   0 },
-  /* GNUCXX   */  { 0,  1,  1,  1,  0,  0,  1,   0,   0,   0,    0,     0,     0,   0 },
-  /* CXX98    */  { 0,  1,  0,  1,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0 },
-  /* GNUCXX11 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    0,     0,     0,   0 },
-  /* CXX11    */  { 1,  1,  0,  1,  1,  1,  1,   1,   1,   1,    0,     0,     1,   0 },
-  /* GNUCXX14 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    1,     1,     0,   0 },
-  /* CXX14    */  { 1,  1,  0,  1,  1,  1,  1,   1,   1,   1,    1,     1,     1,   0 },
-  /* GNUCXX17 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    1,     1,     0,   1 },
-  /* CXX17    */  { 1,  1,  1,  1,  1,  1,  1,   1,   1,   1,    1,     1,     0,   1 },
-  /* GNUCXX2A */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    1,     1,     0,   1 },
-  /* CXX2A    */  { 1,  1,  1,  1,  1,  1,  1,   1,   1,   1,    1,     1,     0,   1 },
-  /* ASM      */  { 0,  0,  1,  0,  0,  0,  0,   0,   0,   0,    0,     0,     0,   0 }
+{ /*              c99 c++ xnum xid c11 std digr ulit rlit udlit bincst digsep trig u8chlit vaopt */
+  /* GNUC89   */  { 0,  0,  1,  0,  0,  0,  1,   0,   0,   0,    0,     0,     0,   0,      1 },
+  /* GNUC99   */  { 1,  0,  1,  1,  0,  0,  1,   1,   1,   0,    0,     0,     0,   0,      1 },
+  /* GNUC11   */  { 1,  0,  1,  1,  1,  0,  1,   1,   1,   0,    0,     0,     0,   0,      1 },
+  /* GNUC17   */  { 1,  0,  1,  1,  1,  0,  1,   1,   1,   0,    0,     0,     0,   0,      1 },
+  /* STDC89   */  { 0,  0,  0,  0,  0,  1,  0,   0,   0,   0,    0,     0,     1,   0,      0 },
+  /* STDC94   */  { 0,  0,  0,  0,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0,      0 },
+  /* STDC99   */  { 1,  0,  1,  1,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0,      0 },
+  /* STDC11   */  { 1,  0,  1,  1,  1,  1,  1,   1,   0,   0,    0,     0,     1,   0,      0 },
+  /* STDC17   */  { 1,  0,  1,  1,  1,  1,  1,   1,   0,   0,    0,     0,     1,   0,      0 },
+  /* GNUCXX   */  { 0,  1,  1,  1,  0,  0,  1,   0,   0,   0,    0,     0,     0,   0,      1 },
+  /* CXX98    */  { 0,  1,  0,  1,  0,  1,  1,   0,   0,   0,    0,     0,     1,   0,      0 },
+  /* GNUCXX11 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    0,     0,     0,   0,      1 },
+  /* CXX11    */  { 1,  1,  0,  1,  1,  1,  1,   1,   1,   1,    0,     0,     1,   0,      0 },
+  /* GNUCXX14 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    1,     1,     0,   0,      1 },
+  /* CXX14    */  { 1,  1,  0,  1,  1,  1,  1,   1,   1,   1,    1,     1,     1,   0,      0 },
+  /* GNUCXX17 */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    1,     1,     0,   1,      1 },
+  /* CXX17    */  { 1,  1,  1,  1,  1,  1,  1,   1,   1,   1,    1,     1,     0,   1,      0 },
+  /* GNUCXX2A */  { 1,  1,  1,  1,  1,  0,  1,   1,   1,   1,    1,     1,     0,   1,      1 },
+  /* CXX2A    */  { 1,  1,  1,  1,  1,  1,  1,   1,   1,   1,    1,     1,     0,   1,      1 },
+  /* ASM      */  { 0,  0,  1,  0,  0,  0,  0,   0,   0,   0,    0,     0,     0,   0,      0 }
 };
 
 /* Sets internal flags correctly for a given language.  */
@@ -139,6 +140,7 @@ cpp_set_lang (cpp_reader *pfile, enum c_lang lang)
   CPP_OPTION (pfile, digit_separators)		 = l->digit_separators;
   CPP_OPTION (pfile, trigraphs)			 = l->trigraphs;
   CPP_OPTION (pfile, utf8_char_literals)	 = l->utf8_char_literals;
+  CPP_OPTION (pfile, va_opt)			 = l->va_opt;
 }
 
 /* Initialize library global state.  */
diff --git a/libcpp/internal.h b/libcpp/internal.h
index f24e85c..0a33aba 100644
--- a/libcpp/internal.h
+++ b/libcpp/internal.h
@@ -246,7 +246,7 @@ struct lexer_state
      all directives apart from #define.  */
   unsigned char save_comments;
 
-  /* Nonzero if lexing __VA_ARGS__ is valid.  */
+  /* Nonzero if lexing __VA_ARGS__ and __VA_OPT__ are valid.  */
   unsigned char va_args_ok;
 
   /* Nonzero if lexing poisoned identifiers is valid.  */
@@ -282,6 +282,7 @@ struct spec_nodes
   cpp_hashnode *n_true;			/* C++ keyword true */
   cpp_hashnode *n_false;		/* C++ keyword false */
   cpp_hashnode *n__VA_ARGS__;		/* C99 vararg macros */
+  cpp_hashnode *n__VA_OPT__;		/* C++ vararg macros */
   cpp_hashnode *n__has_include__;	/* __has_include__ operator */
   cpp_hashnode *n__has_include_next__;	/* __has_include_next__ operator */
 };
diff --git a/libcpp/lex.c b/libcpp/lex.c
index 8af09e5..a8dc3ba 100644
--- a/libcpp/lex.c
+++ b/libcpp/lex.c
@@ -1352,6 +1352,28 @@ forms_identifier_p (cpp_reader *pfile, int first,
   return false;
 }
 
+/* Helper function to issue error about improper __VA_OPT__ use.  */
+static void
+maybe_va_opt_error (cpp_reader *pfile)
+{
+  if (CPP_PEDANTIC (pfile) && !CPP_OPTION (pfile, va_opt))
+    {
+      /* __VA_OPT__ should not be accepted at all, but allow it in
+	 system headers.  */
+      if (!cpp_in_system_header (pfile))
+	cpp_error (pfile, CPP_DL_PEDWARN,
+		   "__VA_OPT__ is not available until C++2a");
+    }
+  else if (!pfile->state.va_args_ok)
+    {
+      /* __VA_OPT__ should only appear in the replacement list of a
+	 variadic macro.  */
+      cpp_error (pfile, CPP_DL_PEDWARN,
+		 "__VA_OPT__ can only appear in the expansion"
+		 " of a C++2a variadic macro");
+    }
+}
+
 /* Helper function to get the cpp_hashnode of the identifier BASE.  */
 static cpp_hashnode *
 lex_identifier_intern (cpp_reader *pfile, const uchar *base)
@@ -1396,6 +1418,9 @@ lex_identifier_intern (cpp_reader *pfile, const uchar *base)
 		       " of a C99 variadic macro");
 	}
 
+      if (result == pfile->spec_nodes.n__VA_OPT__)
+	maybe_va_opt_error (pfile);
+
       /* For -Wc++-compat, warn about use of C++ named operators.  */
       if (result->flags & NODE_WARN_OPERATOR)
 	cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
@@ -1485,6 +1510,11 @@ lex_identifier (cpp_reader *pfile, const uchar *base, bool starts_ucn,
 		       " of a C99 variadic macro");
 	}
 
+      /* __VA_OPT__ should only appear in the replacement list of a
+	 variadic macro.  */
+      if (result == pfile->spec_nodes.n__VA_OPT__)
+	maybe_va_opt_error (pfile);
+
       /* For -Wc++-compat, warn about use of C++ named operators.  */
       if (result->flags & NODE_WARN_OPERATOR)
 	cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
diff --git a/libcpp/macro.c b/libcpp/macro.c
index fab1cb0..bf473ea 100644
--- a/libcpp/macro.c
+++ b/libcpp/macro.c
@@ -89,6 +89,155 @@ struct macro_arg_saved_data {
   union _cpp_hashnode_value value;
 };
 
+static const char *vaopt_paste_error =
+  N_("'##' cannot appear at either end of __VA_OPT__");
+
+/* A class for tracking __VA_OPT__ state while iterating over a
+   sequence of tokens.  This is used during both macro definition and
+   expansion.  */
+class vaopt_state {
+
+ public:
+
+  /* Initialize the state tracker.  ANY_ARGS is true if variable
+     arguments were provided to the macro invocation.  */
+  vaopt_state (cpp_reader *pfile, bool is_variadic, bool any_args)
+    : m_pfile (pfile),
+    m_allowed (any_args),
+    m_variadic (is_variadic),
+    m_state (0),
+    m_last_was_paste (false),
+    m_paste_location (0),
+    m_location (0)
+  {
+  }
+
+  enum update_type
+  {
+    ERROR,
+    DROP,
+    INCLUDE
+  };
+
+  /* Given a token, update the state of this tracker and return a
+     boolean indicating whether the token should be be included in the
+     expansion.  */
+  update_type update (const cpp_token *token)
+  {
+    /* If the macro isn't variadic, just don't bother.  */
+    if (!m_variadic)
+      return INCLUDE;
+
+    if (token->type == CPP_NAME
+	&& token->val.node.node == m_pfile->spec_nodes.n__VA_OPT__)
+      {
+	if (m_state > 0)
+	  {
+	    cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
+			  "__VA_OPT__ may not appear in a __VA_OPT__");
+	    return ERROR;
+	  }
+	++m_state;
+	m_location = token->src_loc;
+	return DROP;
+      }
+    else if (m_state == 1)
+      {
+	if (token->type != CPP_OPEN_PAREN)
+	  {
+	    cpp_error_at (m_pfile, CPP_DL_ERROR, m_location,
+			  "__VA_OPT__ must be followed by an "
+			  "open parenthesis");
+	    return ERROR;
+	  }
+	++m_state;
+	return DROP;
+      }
+    else if (m_state >= 2)
+      {
+	if (m_state == 2 && token->type == CPP_PASTE)
+	  {
+	    cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
+			  vaopt_paste_error);
+	    return ERROR;
+	  }
+	/* Advance states before further considering this token, in
+	   case we see a close paren immediately after the open
+	   paren.  */
+	if (m_state == 2)
+	  ++m_state;
+
+	bool was_paste = m_last_was_paste;
+	m_last_was_paste = false;
+	if (token->type == CPP_PASTE)
+	  {
+	    m_last_was_paste = true;
+	    m_paste_location = token->src_loc;
+	  }
+	else if (token->type == CPP_OPEN_PAREN)
+	  ++m_state;
+	else if (token->type == CPP_CLOSE_PAREN)
+	  {
+	    --m_state;
+	    if (m_state == 2)
+	      {
+		/* Saw the final paren.  */
+		m_state = 0;
+
+		if (was_paste)
+		  {
+		    cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
+				  vaopt_paste_error);
+		    return ERROR;
+		  }
+
+		return DROP;
+	      }
+	  }
+	return m_allowed ? INCLUDE : DROP;
+      }
+
+    /* Nothing to do with __VA_OPT__.  */
+    return INCLUDE;
+  }
+
+  /* Ensure that any __VA_OPT__ was completed.  If ok, return true.
+     Otherwise, issue an error and return false.  */
+  bool completed ()
+  {
+    if (m_variadic && m_state != 0)
+      cpp_error_at (m_pfile, CPP_DL_ERROR, m_location,
+		    "unterminated __VA_OPT__");
+    return m_state == 0;
+  }
+
+ private:
+
+  /* The cpp_reader.  */
+  cpp_reader *m_pfile;
+
+  /* True if there were varargs.  */
+  bool m_allowed;
+  /* True if the macro is variadic.  */
+  bool m_variadic;
+
+  /* The state variable:
+     0 means not parsing
+     1 means __VA_OPT__ seen, looking for "("
+     2 means "(" seen (so the next token can't be "##")
+     >= 3 means looking for ")", the number encodes the paren depth.  */
+  int m_state;
+
+  /* If true, the previous token was ##.  This is used to detect when
+     a paste occurs at the end of the sequence.  */
+  bool m_last_was_paste;
+  /* The location of the paste token.  */
+  source_location m_paste_location;
+
+  /* Location of the __VA_OPT__ token.  */
+  source_location m_location;
+};
+
 /* Macro expansion.  */
 
 static int enter_macro_context (cpp_reader *, cpp_hashnode *,
@@ -776,7 +925,8 @@ _cpp_arguments_ok (cpp_reader *pfile, cpp_macro *macro, const cpp_hashnode *node
 
   if (argc < macro->paramc)
     {
-      /* As an extension, variadic arguments are allowed to not appear in
+      /* In C++2a (here the va_opt flag is used), and also as a GNU
+	 extension, variadic arguments are allowed to not appear in
 	 the invocation at all.
 	 e.g. #define debug(format, args...) something
 	 debug("string");
@@ -786,7 +936,8 @@ _cpp_arguments_ok (cpp_reader *pfile, cpp_macro *macro, const cpp_hashnode *node
 
       if (argc + 1 == macro->paramc && macro->variadic)
 	{
-	  if (CPP_PEDANTIC (pfile) && ! macro->syshdr)
+	  if (CPP_PEDANTIC (pfile) && ! macro->syshdr
+	      && ! CPP_OPTION (pfile, va_opt))
 	    {
 	      if (CPP_OPTION (pfile, cplusplus))
 		cpp_error (pfile, CPP_DL_PEDWARN,
@@ -1678,6 +1829,8 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 				 num_macro_tokens);
     }
   i = 0;
+  vaopt_state vaopt_tracker (pfile, macro->variadic,
+			     args[macro->paramc - 1].count > 0);
   for (src = macro->exp.tokens; src < limit; src++)
     {
       unsigned int arg_tokens_count;
@@ -1685,6 +1838,10 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
       const cpp_token **paste_flag = NULL;
       const cpp_token **tmp_token_ptr;
 
+      /* __VA_OPT__ handling.  */
+      if (vaopt_tracker.update (src) != vaopt_state::INCLUDE)
+	continue;
+
       if (src->type != CPP_MACRO_ARG)
 	{
 	  /* Allocate a virtual location for token SRC, and add that
@@ -3076,6 +3233,9 @@ create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
       *token = *ctoken;
     }
 
+  /* The argument doesn't matter here.  */
+  vaopt_state vaopt_tracker (pfile, macro->variadic, true);
+
   for (;;)
     {
       /* Check the stringifying # constraint 6.10.3.2.1 of
@@ -3144,10 +3304,16 @@ create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
 	    }
 	}
 
+      if (vaopt_tracker.update (token) == vaopt_state::ERROR)
+	return false;
+
       following_paste_op = (token->type == CPP_PASTE);
       token = lex_expansion_token (pfile, macro);
     }
 
+  if (!vaopt_tracker.completed ())
+    return false;
+
   macro->exp.tokens = (cpp_token *) BUFF_FRONT (pfile->a_buff);
   macro->traditional = 0;
 
diff --git a/libcpp/pch.c b/libcpp/pch.c
index cad4b87..b685a38 100644
--- a/libcpp/pch.c
+++ b/libcpp/pch.c
@@ -835,6 +835,7 @@ cpp_read_state (cpp_reader *r, const char *name, FILE *f,
     s->n_true		= cpp_lookup (r, DSC("true"));
     s->n_false		= cpp_lookup (r, DSC("false"));
     s->n__VA_ARGS__     = cpp_lookup (r, DSC("__VA_ARGS__"));
+    s->n__VA_OPT__      = cpp_lookup (r, DSC("__VA_OPT__"));
     s->n__has_include__ = cpp_lookup (r, DSC("__has_include__"));
     s->n__has_include_next__ = cpp_lookup (r, DSC("__has_include_next__"));
   }