Preserve original spellings of extended identifiers.

This patch makes cpplib track the original spellings of extended
identifiers, as well as the canonical UTF-8 version, in order to
follow standard semantics properly without needing a convoluted and
undocumented canonicalization in translation phase 1 (see bug 9449
comments 39-46 regarding such a canonicalization).

The spelling is tracked in cpp_identifier and cpp_macro_arg without
making cpp_token any larger.  The original spelling is used for checks
of duplicate macro definitions, stringizing (see the C++ tests added;
this case is only an issue for C++ not C because C makes it
implementation-defined whether a \ is inserted before the \ of a UCN
in a string or character constant when stringizing, while C++ does
not), pasting (relevant when the result is then stringized for C++)
and when macro definitions are output as text (e.g. for -d options).

Once a macro has been defined, only the original spelling of the
argument names needs keeping in the argument list.  While it is being
defined, however, both spellings are needed: the original one for
subsequent saving for checks of duplicate macro definitions, and the
canonical one which is the node marked specially to generate macro
argument tokens rather than normal identifier tokens.  The buffer that
is used to save the original values of the identifier tokens is
changed so that it stores both those original values and a pointer to
the canonical hash nodes, so that those canonical nodes can be found
when their values need restoring after the macro definition has been
parsed.

I believe this covers the known standards issues in extended
identifiers support (the remaining unimplemented C99 areas in GCC all
being floating-point-related), except for C++ translation of extended
characters to UCNs in phase 1 (which I have no plans to work on).
There are however probably issues left with handling of extended
identifiers in other places, as listed in
<https://gcc.gnu.org/ml/gcc-patches/2014-11/msg00337.html> (those
issues are generally the sort of thing that could be addressed as bugs
outside development stage 1).  (The bulk of the potential issues Zack
was concerned about in 2003-5, that resulted in extended identifiers
being disabled in the absence of -fextended-identifiers, were
effectively eliminated by the audit and fixes I did in 2009, however;
that todo list reflects what was left over after that audit.)

Bootstrapped with no regressions on x86_64-unknown-linux-gnu.

libcpp:
	* include/cpp-id-data.h (struct cpp_macro): Update comment
	regarding parameters.
	* include/cpplib.h (struct cpp_macro_arg, struct cpp_identifier):
	Add spelling fields.
	(struct cpp_token): Update comment on macro_arg.
	* internal.h (_cpp_save_parameter): Add extra argument.
	(_cpp_spell_ident_ucns): New declaration.
	* lex.c (lex_identifier): Add SPELLING argument.  Set *SPELLING to
	original spelling of identifier.
	(_cpp_lex_direct): Update calls to lex_identifier.
	(_cpp_spell_ident_ucns): New function, factored out of
	cpp_spell_token.
	(cpp_spell_token): Adjust FORSTRING argument semantics to return
	original spelling of identifiers.  Use _cpp_spell_ident_ucns in
	!FORSTRING case.
	(_cpp_equiv_tokens): Check spellings of identifiers and macro
	arguments are identical.
	* macro.c (macro_arg_saved_data): New structure.
	(paste_tokens): Use original spellings of identifiers from
	cpp_spell_token.
	(_cpp_save_parameter): Add argument SPELLING.  Save both canonical
	node and its value.
	(parse_params): Update calls to _cpp_save_parameter.
	(lex_expansion_token): Save spelling of macro argument tokens.
	(_cpp_create_definition): Extract canonical node from saved data.
	(cpp_macro_definition): Use UCNs in spelling of macro name.  Use
	original spellings of macro argument tokens and identifiers.
	* traditional.c (scan_parameters): Update call to
	_cpp_save_parameter.

gcc:
	* doc/invoke.texi (-std=c99, -std=c11): Don't refer to corner
	cases of extended identifiers.

gcc/testsuite:
	* g++.dg/cpp/ucnid-2.C, g++.dg/cpp/ucnid-3.C,
	gcc.dg/cpp/ucnid-11.c, gcc.dg/cpp/ucnid-12.c,
	gcc.dg/cpp/ucnid-13.c, gcc.dg/cpp/ucnid-14.c,
	gcc.dg/cpp/ucnid-15.c: New tests.

From-SVN: r217202

1 files changed, 39 insertions, 20 deletions

diff --git a/libcpp/macro.c b/libcpp/macro.c
index 8fa9770..678bf2b 100644
--- a/libcpp/macro.c
+++ b/libcpp/macro.c
@@ -80,6 +80,15 @@ struct macro_arg_token_iter
 #endif
 };
 
+/* Saved data about an identifier being used as a macro argument
+   name.  */
+struct macro_arg_saved_data {
+  /* The canonical (UTF-8) spelling of this identifier.  */
+  cpp_hashnode *canonical_node;
+  /* The previous value of this identifier.  */
+  union _cpp_hashnode_value value;
+};
+
 /* Macro expansion.  */
 
 static int enter_macro_context (cpp_reader *, cpp_hashnode *,
@@ -590,7 +599,7 @@ paste_tokens (cpp_reader *pfile, source_location location,
 
   len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 1;
   buf = (unsigned char *) alloca (len);
-  end = lhsend = cpp_spell_token (pfile, *plhs, buf, false);
+  end = lhsend = cpp_spell_token (pfile, *plhs, buf, true);
 
   /* Avoid comment headers, since they are still processed in stage 3.
      It is simpler to insert a space here, rather than modifying the
@@ -600,7 +609,7 @@ paste_tokens (cpp_reader *pfile, source_location location,
     *end++ = ' ';
   /* In one obscure case we might see padding here.  */
   if (rhs->type != CPP_PADDING)
-    end = cpp_spell_token (pfile, rhs, end, false);
+    end = cpp_spell_token (pfile, rhs, end, true);
   *end = '\n';
 
   cpp_push_buffer (pfile, buf, end - buf, /* from_stage3 */ true);
@@ -2748,10 +2757,12 @@ _cpp_free_definition (cpp_hashnode *h)
   h->flags &= ~(NODE_BUILTIN | NODE_DISABLED | NODE_USED);
 }
 
-/* Save parameter NODE to the parameter list of macro MACRO.  Returns
-   zero on success, nonzero if the parameter is a duplicate.  */
+/* Save parameter NODE (spelling SPELLING) to the parameter list of
+   macro MACRO.  Returns zero on success, nonzero if the parameter is
+   a duplicate.  */
 bool
-_cpp_save_parameter (cpp_reader *pfile, cpp_macro *macro, cpp_hashnode *node)
+_cpp_save_parameter (cpp_reader *pfile, cpp_macro *macro, cpp_hashnode *node,
+		     cpp_hashnode *spelling)
 {
   unsigned int len;
   /* Constraint 6.10.3.6 - duplicate parameter names.  */
@@ -2766,17 +2777,20 @@ _cpp_save_parameter (cpp_reader *pfile, cpp_macro *macro, cpp_hashnode *node)
       < (macro->paramc + 1) * sizeof (cpp_hashnode *))
     _cpp_extend_buff (pfile, &pfile->a_buff, sizeof (cpp_hashnode *));
 
-  ((cpp_hashnode **) BUFF_FRONT (pfile->a_buff))[macro->paramc++] = node;
+  ((cpp_hashnode **) BUFF_FRONT (pfile->a_buff))[macro->paramc++] = spelling;
   node->flags |= NODE_MACRO_ARG;
-  len = macro->paramc * sizeof (union _cpp_hashnode_value);
+  len = macro->paramc * sizeof (struct macro_arg_saved_data);
   if (len > pfile->macro_buffer_len)
     {
       pfile->macro_buffer = XRESIZEVEC (unsigned char, pfile->macro_buffer,
                                         len);
       pfile->macro_buffer_len = len;
     }
-  ((union _cpp_hashnode_value *) pfile->macro_buffer)[macro->paramc - 1]
-    = node->value;
+  struct macro_arg_saved_data save;
+  save.value = node->value;
+  save.canonical_node = node;
+  ((struct macro_arg_saved_data *) pfile->macro_buffer)[macro->paramc - 1]
+    = save;
   
   node->value.arg_index  = macro->paramc;
   return false;
@@ -2816,7 +2830,8 @@ parse_params (cpp_reader *pfile, cpp_macro *macro)
 	    }
 	  prev_ident = 1;
 
-	  if (_cpp_save_parameter (pfile, macro, token->val.node.node))
+	  if (_cpp_save_parameter (pfile, macro, token->val.node.node,
+				   token->val.node.spelling))
 	    return false;
 	  continue;
 
@@ -2839,6 +2854,7 @@ parse_params (cpp_reader *pfile, cpp_macro *macro)
 	  if (!prev_ident)
 	    {
 	      _cpp_save_parameter (pfile, macro,
+				   pfile->spec_nodes.n__VA_ARGS__,
 				   pfile->spec_nodes.n__VA_ARGS__);
 	      pfile->state.va_args_ok = 1;
 	      if (! CPP_OPTION (pfile, c99)
@@ -2909,8 +2925,10 @@ lex_expansion_token (cpp_reader *pfile, cpp_macro *macro)
   if (token->type == CPP_NAME
       && (token->val.node.node->flags & NODE_MACRO_ARG) != 0)
     {
+      cpp_hashnode *spelling = token->val.node.spelling;
       token->type = CPP_MACRO_ARG;
       token->val.macro_arg.arg_no = token->val.node.node->value.arg_index;
+      token->val.macro_arg.spelling = spelling;
     }
   else if (CPP_WTRADITIONAL (pfile) && macro->paramc > 0
 	   && (token->type == CPP_STRING || token->type == CPP_CHAR))
@@ -3162,9 +3180,11 @@ _cpp_create_definition (cpp_reader *pfile, cpp_hashnode *node)
   /* Clear the fast argument lookup indices.  */
   for (i = macro->paramc; i-- > 0; )
     {
-      struct cpp_hashnode *node = macro->params[i];
+      struct macro_arg_saved_data *save =
+	&((struct macro_arg_saved_data *) pfile->macro_buffer)[i];
+      struct cpp_hashnode *node = save->canonical_node;
       node->flags &= ~ NODE_MACRO_ARG;
-      node->value = ((union _cpp_hashnode_value *) pfile->macro_buffer)[i];
+      node->value = save->value;
     }
 
   if (!ok)
@@ -3285,7 +3305,7 @@ cpp_macro_definition (cpp_reader *pfile, cpp_hashnode *node)
 
   macro = node->value.macro;
   /* Calculate length.  */
-  len = NODE_LEN (node) + 2;			/* ' ' and NUL.  */
+  len = NODE_LEN (node) * 10 + 2;		/* ' ' and NUL.  */
   if (macro->fun_like)
     {
       len += 4;		/* "()" plus possible final ".." of named
@@ -3305,7 +3325,7 @@ cpp_macro_definition (cpp_reader *pfile, cpp_hashnode *node)
 	  cpp_token *token = &macro->exp.tokens[i];
 
 	  if (token->type == CPP_MACRO_ARG)
-	    len += NODE_LEN (macro->params[token->val.macro_arg.arg_no - 1]);
+	    len += NODE_LEN (token->val.macro_arg.spelling);
 	  else
 	    len += cpp_token_len (token);
 
@@ -3327,8 +3347,7 @@ cpp_macro_definition (cpp_reader *pfile, cpp_hashnode *node)
 
   /* Fill in the buffer.  Start with the macro name.  */
   buffer = pfile->macro_buffer;
-  memcpy (buffer, NODE_NAME (node), NODE_LEN (node));
-  buffer += NODE_LEN (node);
+  buffer = _cpp_spell_ident_ucns (buffer, node);
 
   /* Parameter names.  */
   if (macro->fun_like)
@@ -3377,12 +3396,12 @@ cpp_macro_definition (cpp_reader *pfile, cpp_hashnode *node)
 	  if (token->type == CPP_MACRO_ARG)
 	    {
 	      memcpy (buffer,
-		      NODE_NAME (macro->params[token->val.macro_arg.arg_no - 1]),
-		      NODE_LEN (macro->params[token->val.macro_arg.arg_no - 1]));
-	      buffer += NODE_LEN (macro->params[token->val.macro_arg.arg_no - 1]);
+		      NODE_NAME (token->val.macro_arg.spelling),
+		      NODE_LEN (token->val.macro_arg.spelling));
+	      buffer += NODE_LEN (token->val.macro_arg.spelling);
 	    }
 	  else
-	    buffer = cpp_spell_token (pfile, token, buffer, false);
+	    buffer = cpp_spell_token (pfile, token, buffer, true);
 
 	  if (token->flags & PASTE_LEFT)
 	    {