PR c++/94147 - mangling of lambdas assigned to globals

This patch implements Jason's suggestion of pushing a lambda scope
when parsing a global variable initializer.  That bit worked fine, but
happened to cause g++.dg/opt/dump1.C to not give any
used-but-not-defined warnings.

The reason was no_linkage_check, which considers any lambda that has
an extra-scope to have linkage.  Which is technically correct.  Except
that we think that all types that have linkage have external linkage.

Our representation of linkage and visibility is somewhat inaccurate,
particularly when it comes to types.  We have TREE_PUBLIC,
DECL_EXTERNAL, DECL_VISIBILITY, DECL_COMDAT, DECL_NOT_REALLY_EXTERN.
It could really do with a through cleanup, but that won't be a simple
task.

The best I could come up with was seeing if the extra scope was a
VAR_DECL, and if that was TREE_PUBLIC and the var was inline (its
COMDATness is sadly not set at that point) or a template
instantiation, then the lambda had linkage.  Otherwise it's as-if it
has no-linkage from the POV of compiler internals.

This is an ABI change (so we should document it), but it's changing
mangling from an unpredictable (in practice) counter, to something the
ABI defines.  So I'm not concerned about mangling-changed warnings, or
preserving the broken mangling under some ABI selection flag.  Code
that did this worked by accident within a single TU.  It'll continue
to work by design there, and across TUs.

	* parser.c (cp_parser_init_declarator): Namespace-scope variables
	provide a lambda scope.
	* tree.c (no_linkage_check): Lambdas with a variable for extra
	scope have a linkage from the variable.

5 files changed, 92 insertions, 9 deletions

diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog
index 938504b..bb5f77f 100644
--- a/gcc/cp/ChangeLog
+++ b/gcc/cp/ChangeLog
@@ -1,3 +1,11 @@
+2020-03-18  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/94147 - mangling of lambdas assigned to globals
+	* parser.c (cp_parser_init_declarator): Namespace-scope variables
+	provide a lambda scope.
+	* tree.c (no_linkage_check): Lambdas with a variable for extra
+	scope have a linkage from the variable.
+
 2020-03-18  Jakub Jelinek  <jakub@redhat.com>
 
 	* constraint.cc (resolve_function_concept_check, subsumes_constraints,
diff --git a/gcc/cp/parser.c b/gcc/cp/parser.c
index 26e0236..198ab97 100644
--- a/gcc/cp/parser.c
+++ b/gcc/cp/parser.c
@@ -20761,16 +20761,24 @@ cp_parser_init_declarator (cp_parser* parser,
       else
 	{
 	  /* We want to record the extra mangling scope for in-class
-	     initializers of class members and initializers of static data
-	     member templates.  The former involves deferring
-	     parsing of the initializer until end of class as with default
-	     arguments.  So right here we only handle the latter.  */
-	  if (!member_p && processing_template_decl && decl != error_mark_node)
+	     initializers of class members and initializers of static
+	     data member templates and namespace-scope initializers.
+	     The former involves deferring parsing of the initializer
+	     until end of class as with default arguments.  So right
+	     here we only handle the latter two.  */
+	  bool has_lambda_scope = false;
+
+	  if (decl != error_mark_node
+	      && !member_p
+	      && (processing_template_decl || DECL_NAMESPACE_SCOPE_P (decl)))
+	    has_lambda_scope = true;
+
+	  if (has_lambda_scope)
 	    start_lambda_scope (decl);
 	  initializer = cp_parser_initializer (parser,
 					       &is_direct_init,
 					       &is_non_constant_init);
-	  if (!member_p && processing_template_decl && decl != error_mark_node)
+	  if (has_lambda_scope)
 	    finish_lambda_scope ();
 	  if (initializer == error_mark_node)
 	    cp_parser_skip_to_end_of_statement (parser);
diff --git a/gcc/cp/tree.c b/gcc/cp/tree.c
index a412345..da2e7fd 100644
--- a/gcc/cp/tree.c
+++ b/gcc/cp/tree.c
@@ -2794,9 +2794,23 @@ no_linkage_check (tree t, bool relaxed_p)
      fix it up later if not.  We need to check this even in templates so
      that we properly handle a lambda-expression in the signature.  */
   if (LAMBDA_TYPE_P (t)
-      && CLASSTYPE_LAMBDA_EXPR (t) != error_mark_node
-      && LAMBDA_TYPE_EXTRA_SCOPE (t) == NULL_TREE)
-    return t;
+      && CLASSTYPE_LAMBDA_EXPR (t) != error_mark_node)
+    {
+      tree extra = LAMBDA_TYPE_EXTRA_SCOPE (t);
+      if (!extra)
+	return t;
+
+      /* If the mangling scope is internal-linkage or not repeatable
+	 elsewhere, the lambda effectively has no linkage.  (Sadly
+	 we're not very careful with the linkages of types.)  */
+      if (TREE_CODE (extra) == VAR_DECL
+	  && !(TREE_PUBLIC (extra)
+	       && (processing_template_decl
+		   || (DECL_LANG_SPECIFIC (extra) && DECL_USE_TEMPLATE (extra))
+		   /* DECL_COMDAT is set too late for us to check.  */
+		   || DECL_VAR_DECLARED_INLINE_P (extra))))
+	return t;
+    }
 
   /* Otherwise there's no point in checking linkage on template functions; we
      can't know their complete types.  */
diff --git a/gcc/testsuite/g++.dg/abi/lambda-vis.C b/gcc/testsuite/g++.dg/abi/lambda-vis.C
new file mode 100644
index 0000000..c3eb157
--- /dev/null
+++ b/gcc/testsuite/g++.dg/abi/lambda-vis.C
@@ -0,0 +1,23 @@
+// { dg-do compile { target c++17 } }
+// { dg-options "-fno-inline" }
+
+template<typename T> int sfoo (T); // { dg-warning "used but never defined" }
+template<typename T> int gfoo (T); // { dg-warning "used but never defined" }
+template<typename T> int ifoo (T); // OK
+template<typename T> struct Wrapper {};
+template<typename T> Wrapper<T> capture (T &&) {return Wrapper<T> ();}
+
+static int svar = sfoo (capture ([]{}));
+
+int gvar = gfoo (capture ([]{}));
+
+inline int ivar = ifoo (capture ([]{}));
+
+// { dg-final { scan-assembler {_Z7captureINL4svarMUlvE_EE7WrapperIT_EOS2_:} } }
+// { dg-final { scan-assembler {_Z7captureIN4gvarMUlvE_EE7WrapperIT_EOS2_:} } }
+// { dg-final { scan-assembler {_Z7captureIN4ivarMUlvE_EE7WrapperIT_EOS2_:} } }
+
+// Calls to the foos are emitted.
+// { dg-final { scan-assembler {call[ \t]*_Z4sfooI7WrapperINL4svarMUlvE_EEEiT_} { target { i?86-*-* x86_64-*-* } } } }
+// { dg-final { scan-assembler {call[ \t]*_Z4gfooI7WrapperIN4gvarMUlvE_EEEiT_} { target { i?86-*-* x86_64-*-* } } } }
+// { dg-final { scan-assembler {call[ \t]*_Z4ifooI7WrapperIN4ivarMUlvE_EEEiT_} { target { i?86-*-* x86_64-*-* } } } }
diff --git a/gcc/testsuite/g++.dg/abi/mangle74.C b/gcc/testsuite/g++.dg/abi/mangle74.C
new file mode 100644
index 0000000..4e1c632
--- /dev/null
+++ b/gcc/testsuite/g++.dg/abi/mangle74.C
@@ -0,0 +1,30 @@
+// { dg-do compile { target c++17 } }
+// { dg-options "-fno-inline -O0" }
+
+inline auto var = [] () {return 2;};
+
+int bob ()
+{
+return var ();
+}
+
+struct Foo
+{
+  static inline auto bar = [] () {return 4;};
+};
+
+int bill ()
+{
+  return Foo::bar ();
+}
+
+// this one should have internal linkage (from svar)
+static auto svar = [] () {return 8;};
+int thorn ()
+{
+  return svar ();
+}
+
+// { dg-final { scan-assembler "_ZNK3varMUlvE_clEv:" } }
+// { dg-final { scan-assembler "_ZNK3Foo3barMUlvE_clEv:" { xfail *-*-* } } }
+// { dg-final { scan-assembler-not "_ZNK3FooUlvE_clEv:" { xfail *-*-* } } }