libcpp: __VA_OPT__ p1042r1 placemarker changes [PR101488]

So, besides missing #__VA_OPT__ patch for which I've posted patch last week,
P1042R1 introduced some placemarker changes for __VA_OPT__, most notably
the addition of before "removal of placemarker tokens," rescanning ...
and the
 #define H4(X, ...) __VA_OPT__(a X ## X) ## b
H4(, 1)  // replaced by a b
example mentioned there where we replace it currently with ab

The following patch are the minimum changes (except for the
__builtin_expect) that achieve the same preprocessing between current
clang++ and patched gcc on all the testcases I've tried (i.e. gcc __VA_OPT__
testsuite in c-c++-common/cpp/va-opt* including the new test and the clang
clang/test/Preprocessor/macro_va_opt* testcases).

At one point I was trying to implement the __VA_OPT__(args) case as if
for non-empty __VA_ARGS__ it expanded as if __VA_OPT__( and ) were missing,
but from the tests it seems that is not how it should work, in particular
if after (or before) we have some macro argument and it is not followed
(or preceded) by ##, then it should be macro expanded even when __VA_OPT__
is after ## or ) is followed by ##.  And it seems that not removing any
padding tokens isn't possible either, because the expansion of the arguments
typically has a padding token at the start and end and those at least
according to the testsuite need to go.  It is unclear if it would be enough
to remove just one or if all padding tokens should be removed.
Anyway, e.g. the previous removal of all padding tokens at the end of
__VA_OPT__ is undesirable, as it e.g. eats also the padding tokens needed
for the H4 example from the paper.

2021-09-01  Jakub Jelinek  <jakub@redhat.com>

	PR preprocessor/101488
	* macro.c (replace_args): Fix up handling of CPP_PADDING tokens at the
	start or end of __VA_OPT__ arguments when preceeded or followed by ##.

	* c-c++-common/cpp/va-opt-3.c: Adjust expected output.
	* c-c++-common/cpp/va-opt-7.c: New test.

1 files changed, 18 insertions, 12 deletions

diff --git a/libcpp/macro.c b/libcpp/macro.c
index c317a43..b3ba352 100644
--- a/libcpp/macro.c
+++ b/libcpp/macro.c
@@ -2026,6 +2026,7 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
   i = 0;
   vaopt_state vaopt_tracker (pfile, macro->variadic, &args[macro->paramc - 1]);
   const cpp_token **vaopt_start = NULL;
+  unsigned vaopt_padding_tokens = 0;
   for (src = macro->exp.tokens; src < limit; src++)
     {
       unsigned int arg_tokens_count;
@@ -2035,7 +2036,7 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 
       /* __VA_OPT__ handling.  */
       vaopt_state::update_type vostate = vaopt_tracker.update (src);
-      if (vostate != vaopt_state::INCLUDE)
+      if (__builtin_expect (vostate != vaopt_state::INCLUDE, false))
 	{
 	  if (vostate == vaopt_state::BEGIN)
 	    {
@@ -2060,7 +2061,9 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 
 	      /* Remove any tail padding from inside the __VA_OPT__.  */
 	      paste_flag = tokens_buff_last_token_ptr (buff);
-	      while (paste_flag && paste_flag != start
+	      while (vaopt_padding_tokens--
+		     && paste_flag
+		     && paste_flag != start
 		     && (*paste_flag)->type == CPP_PADDING)
 		{
 		  tokens_buff_remove_last_token (buff);
@@ -2104,6 +2107,7 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 	  continue;
 	}
 
+      vaopt_padding_tokens = 0;
       if (src->type != CPP_MACRO_ARG)
 	{
 	  /* Allocate a virtual location for token SRC, and add that
@@ -2181,11 +2185,8 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 		  else
 		    paste_flag = tmp_token_ptr;
 		}
-	      /* Remove the paste flag if the RHS is a placemarker, unless the
-		 previous emitted token is at the beginning of __VA_OPT__;
-		 placemarkers within __VA_OPT__ are ignored in that case.  */
-	      else if (arg_tokens_count == 0
-		       && tmp_token_ptr != vaopt_start)
+	      /* Remove the paste flag if the RHS is a placemarker.  */
+	      else if (arg_tokens_count == 0)
 		paste_flag = tmp_token_ptr;
 	    }
 	}
@@ -2215,7 +2216,8 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 
       /* Padding on the left of an argument (unless RHS of ##).  */
       if ((!pfile->state.in_directive || pfile->state.directive_wants_padding)
-	  && src != macro->exp.tokens && !(src[-1].flags & PASTE_LEFT)
+	  && src != macro->exp.tokens
+	  && !(src[-1].flags & PASTE_LEFT)
 	  && !last_token_is (buff, vaopt_start))
 	{
 	  const cpp_token *t = padding_token (pfile, src);
@@ -2260,8 +2262,12 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 		token_index += j;
 
 	      index = expanded_token_index (pfile, macro, src, token_index);
-	      tokens_buff_add_token (buff, virt_locs,
-				     macro_arg_token_iter_get_token (&from),
+	      const cpp_token *tok = macro_arg_token_iter_get_token (&from);
+	      if (tok->type == CPP_PADDING)
+		vaopt_padding_tokens++;
+	      else
+		vaopt_padding_tokens = 0;
+	      tokens_buff_add_token (buff, virt_locs, tok,
 				     macro_arg_token_iter_get_location (&from),
 				     src->src_loc, map, index);
 	      macro_arg_token_iter_forward (&from);
@@ -2301,13 +2307,13 @@ replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
 		     NODE_NAME (node), src->val.macro_arg.arg_no);
 
       /* Avoid paste on RHS (even case count == 0).  */
-      if (!pfile->state.in_directive && !(src->flags & PASTE_LEFT)
-	  && !last_token_is (buff, vaopt_start))
+      if (!pfile->state.in_directive && !(src->flags & PASTE_LEFT))
 	{
 	  const cpp_token *t = &pfile->avoid_paste;
 	  tokens_buff_add_token (buff, virt_locs,
 				 t, t->src_loc, t->src_loc,
 				 NULL, 0);
+	  vaopt_padding_tokens++;
 	}
 
       /* Add a new paste flag, or remove an unwanted one.  */