c++: lambdas with internal linkage are different to no-linkage [PR94426]

My fix for 94147 was confusing no-linkage with internal linkage, at
the language level.  That's wrong. (the std is confusing here, because
it describes linkage of names (which is wrong), and lambdas have no
names)

Lambdas with extra-scope, have linkage.  However, at the
implementation-level that linkage is at least as restricted as the
linkage of the extra-scope decl.

Further, when instantiating a variable initialized by a lambda, we
must determine the visibility of the variable itself, before
instantiating its initializer.  If the template arguments are internal
(or no-linkage), the variable will have internal linkage, regardless
of the linkage of the template it is instantiated from.  We need to
know that before instantiating the lambda, so we can restrict its
linkage correctly.

	* decl2.c (determine_visibility): A lambda's visibility is
	affected by its extra scope.
	* pt.c (instantiate_decl): Determine var's visibility before
	instantiating its initializer.
	* tree.c (no_linkage_check): Revert code looking at visibility of
	lambda's extra scope.
`	gcc/cp/
	* g++.dg/cpp0x/lambda/pr94426-[12].C: New.
	* g++.dg/abi/lambda-vis.C: Drop a warning.
	* g++.dg/cpp0x/lambda/lambda-mangle.C: Lambda visibility on
	variable changes.
	* g++.dg/opt/dump1.C: Drop warnings of no import.

10 files changed, 93 insertions, 21 deletions

diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog
index 27c79be..9de64c0 100644
--- a/gcc/cp/ChangeLog
+++ b/gcc/cp/ChangeLog
@@ -1,3 +1,13 @@
+2020-04-13  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/94426  lambdas with internal linkage are different to no-linkage
+	* decl2.c (determine_visibility): A lambda's visibility is
+	affected by its extra scope.
+	* pt.c (instantiate_decl): Determine var's visibility before
+	instantiating its initializer.
+	* tree.c (no_linkage_check): Revert code looking at visibility of
+	lambda's extra scope.
+`
 2020-04-10  Iain Sandoe  <iain@sandoe.co.uk>
 
 	PR c++/94528
diff --git a/gcc/cp/decl2.c b/gcc/cp/decl2.c
index 6cf72b4..293df99 100644
--- a/gcc/cp/decl2.c
+++ b/gcc/cp/decl2.c
@@ -2527,6 +2527,21 @@ determine_visibility (tree decl)
   else if (DECL_LANG_SPECIFIC (decl) && DECL_USE_TEMPLATE (decl))
     template_decl = decl;
 
+  if (TREE_CODE (decl) == TYPE_DECL
+      && LAMBDA_TYPE_P (TREE_TYPE (decl))
+      && CLASSTYPE_LAMBDA_EXPR (TREE_TYPE (decl)) != error_mark_node)
+    if (tree extra = LAMBDA_TYPE_EXTRA_SCOPE (TREE_TYPE (decl)))
+      {
+	/* The lambda's visibility is limited by that of its extra
+	   scope.  */
+	int vis = 0;
+	if (TYPE_P (extra))
+	  vis = type_visibility (extra);
+	else
+	  vis = expr_visibility (extra);
+	constrain_visibility (decl, vis, false);
+      }
+
   /* If DECL is a member of a class, visibility specifiers on the
      class can influence the visibility of the DECL.  */
   tree class_type = NULL_TREE;
diff --git a/gcc/cp/pt.c b/gcc/cp/pt.c
index 050a57b..0a8ec31 100644
--- a/gcc/cp/pt.c
+++ b/gcc/cp/pt.c
@@ -25541,6 +25541,14 @@ instantiate_decl (tree d, bool defer_ok, bool expl_inst_class_mem_p)
       c_inhibit_evaluation_warnings = 0;
     }
 
+  if (VAR_P (d))
+    {
+      /* The variable might be a lambda's extra scope, and that
+	 lambda's visibility depends on D's.  */
+      maybe_commonize_var (d);
+      determine_visibility (d);
+    }
+
   /* Mark D as instantiated so that recursive calls to
      instantiate_decl do not try to instantiate it again.  */
   DECL_TEMPLATE_INSTANTIATED (d) = 1;
diff --git a/gcc/cp/tree.c b/gcc/cp/tree.c
index d1192b7..1d311b0 100644
--- a/gcc/cp/tree.c
+++ b/gcc/cp/tree.c
@@ -2780,9 +2780,10 @@ verify_stmt_tree (tree t)
   cp_walk_tree (&t, verify_stmt_tree_r, &statements, NULL);
 }
 
-/* Check if the type T depends on a type with no linkage and if so, return
-   it.  If RELAXED_P then do not consider a class type declared within
-   a vague-linkage function to have no linkage.  */
+/* Check if the type T depends on a type with no linkage and if so,
+   return it.  If RELAXED_P then do not consider a class type declared
+   within a vague-linkage function to have no linkage.  Remember:
+   no-linkage is not the same as internal-linkage*/
 
 tree
 no_linkage_check (tree t, bool relaxed_p)
@@ -2801,17 +2802,6 @@ no_linkage_check (tree t, bool relaxed_p)
       tree extra = LAMBDA_TYPE_EXTRA_SCOPE (t);
       if (!extra)
 	return t;
-
-      /* If the mangling scope is internal-linkage or not repeatable
-	 elsewhere, the lambda effectively has no linkage.  (Sadly
-	 we're not very careful with the linkages of types.)  */
-      if (TREE_CODE (extra) == VAR_DECL
-	  && !(TREE_PUBLIC (extra)
-	       && (processing_template_decl
-		   || (DECL_LANG_SPECIFIC (extra) && DECL_USE_TEMPLATE (extra))
-		   /* DECL_COMDAT is set too late for us to check.  */
-		   || DECL_VAR_DECLARED_INLINE_P (extra))))
-	return t;
     }
 
   /* Otherwise there's no point in checking linkage on template functions; we
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index b1f232e..9f64453 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,12 @@
+2020-04-13  Nathan Sidwell  <nathan@acm.org>
+
+	PR c++/94426
+	* g++.dg/cpp0x/lambda/pr94426-[12].C: New.
+	* g++.dg/abi/lambda-vis.C: Drop a warning.
+	* g++.dg/cpp0x/lambda/lambda-mangle.C: Lambda visibility on
+	variable changes.
+	* g++.dg/opt/dump1.C: Drop warnings of no import.
+
 2020-04-13  Jozef Lawrynowicz  <jozef.l@mittosystems.com>
 
 	* gcc.target/msp430/operand-modifiers.c: New test.
diff --git a/gcc/testsuite/g++.dg/abi/lambda-vis.C b/gcc/testsuite/g++.dg/abi/lambda-vis.C
index 89683b2..c1033f5 100644
--- a/gcc/testsuite/g++.dg/abi/lambda-vis.C
+++ b/gcc/testsuite/g++.dg/abi/lambda-vis.C
@@ -2,7 +2,7 @@
 // { dg-options "-fno-inline" }
 
 template<typename T> int sfoo (T); // { dg-warning "used but never defined" }
-template<typename T> int gfoo (T); // { dg-warning "used but never defined" }
+template<typename T> int gfoo (T); // OK, but not completable
 template<typename T> int ifoo (T); // OK
 template<typename T> struct Wrapper {};
 template<typename T> Wrapper<T> capture (T &&) {return Wrapper<T> ();}
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle.C
index 7894ef3..ef4bad8 100644
--- a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle.C
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle.C
@@ -54,9 +54,12 @@ void bar()
   []{}();
 }
 
-// lambdas used in non-template, non-class body initializers are internal.
+// lambdas used in namespace-scope initializers have the linkage of
+// the decl
 // { dg-final { scan-assembler-not "weak\[^\n\r\]*_ZNKUlv" } }
-// { dg-final { scan-assembler-not "weak\[^\n\r\]*variable" } }
+// { dg-final { scan-assembler "weak\[^\n\r\]*variableMUlvE_clEv" { target c++14_down } } }
+// in c++17 and up, this operator() become constexpr, no not emitted
+// { dg-final { scan-assembler-not "weak\[^\n\r\]*variableMUlvE_clEv" { target c++17 } } }
 int variable = []{return 1;}();
 
 // And a template instantiated with such a lambda is also internal.
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/pr94426-1.C b/gcc/testsuite/g++.dg/cpp0x/lambda/pr94426-1.C
new file mode 100644
index 0000000..ae7cbf0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/pr94426-1.C
@@ -0,0 +1,17 @@
+// { dg-do compile { target c++14 } }
+// PR 94426 ICE mangling lambda
+// { dg-options {-flto -O2} }
+
+template <bool> using Void = void;
+
+template <typename U> bool Init (U) {return true;}
+template <typename> bool VAR = Init ([] {});
+
+template <typename T>
+Void<false && VAR<T>> Foo (T)
+{}
+
+void q ()
+{
+  Foo ([] {});
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/pr94426-2.C b/gcc/testsuite/g++.dg/cpp0x/lambda/pr94426-2.C
new file mode 100644
index 0000000..3db864c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/pr94426-2.C
@@ -0,0 +1,20 @@
+// { dg-do compile { target c++14 } }
+// PR 94426 ICE mangling lambda
+
+template <bool> using Void = void;
+
+template <typename U> bool Init (U) {return true;}
+template <typename> bool VAR = Init ([] {});
+
+template <typename T>
+Void<false && VAR<T>> Foo (T)
+{}
+
+void q ()
+{
+  Foo ([] {});
+}
+
+// The instantiation of VAR becomes local
+// { dg-final { scan-assembler {.local	_Z3VARIZ1qvEUlvE_E} { target { i?86-*-* x86_64-*-* } } } }
+// { dg-final { scan-assembler {.comm	_Z3VARIZ1qvEUlvE_E,1,1} { target { i?86-*-* x86_64-*-* } } } }
diff --git a/gcc/testsuite/g++.dg/opt/dump1.C b/gcc/testsuite/g++.dg/opt/dump1.C
index f813044..38ed055 100644
--- a/gcc/testsuite/g++.dg/opt/dump1.C
+++ b/gcc/testsuite/g++.dg/opt/dump1.C
@@ -312,7 +312,7 @@ namespace std __attribute__ ((__visibility__ ("default")))
     typename __add_ref<
                       typename tuple_element<__i, tuple<_Elements...>>::type
                     >::type
-    get(tuple<_Elements...>& __t) noexcept; // { dg-warning "used but never defined" }
+    get(tuple<_Elements...>& __t) noexcept;
   template<std::size_t... _Indexes>
     struct _Index_tuple
     {};
@@ -387,7 +387,7 @@ namespace std __attribute__ ((__visibility__ ("default")))
     };
   template<typename _Callable, typename... _Args>
     typename _Bind_simple_helper<_Callable, _Args...>::__type
-    __bind_simple(_Callable&& __callable, _Args&&... __args)  // { dg-warning "used but never defined" }
+    __bind_simple(_Callable&& __callable, _Args&&... __args) 
   ;
   union _Any_data
   ;
@@ -404,7 +404,7 @@ namespace std __attribute__ ((__visibility__ ("default")))
       {
       protected:
  static _Functor*
- _M_get_pointer(const _Any_data& __source)  // { dg-warning "used but never defined" }
+ _M_get_pointer(const _Any_data& __source)
  ;
       };
   };
@@ -511,7 +511,7 @@ namespace std __attribute__ ((__visibility__ ("default")))
         _S_construct(_Alloc&, _Tp* __p, _Args&&... __args)
  { ::new((void*)__p) _Tp(std::forward<_Args>(__args)...); }
       static pointer
-      allocate(_Alloc& __a, size_type __n)  // { dg-warning "used but never defined" }
+      allocate(_Alloc& __a, size_type __n) 
       ;
       template<typename _Tp, typename... _Args>
  static auto construct(_Alloc& __a, _Tp* __p, _Args&&... __args)