Fix invalid devirtualization when combining final keyword and anonymous types

this patch fixes a wrong code issue where we incorrectly devirtualize to
__builtin_unreachable.  The problem occurs in combination of anonymous
namespaces and final keyword used on methods.  We do two optimizations here
 1) when reacing final method we cut the search for possible new targets
 2) if the type is anonymous we detect whether it is ever instatiated by
    looking if its vtable is referred to.
Now this goes wrong when thre is an anonymous type with final method that
is not instantiated while its derived type is.  So if 1 triggers we need
to make 2 to look for vtables of all derived types as done by this patch.

Bootstrpaped/regtested x86_64-linux

Honza

gcc/ChangeLog:

2022-08-10  Jan Hubicka  <hubicka@ucw.cz>

	PR middle-end/106057
	* ipa-devirt.cc (type_or_derived_type_possibly_instantiated_p): New
	function.
	(possible_polymorphic_call_targets): Use it.

gcc/testsuite/ChangeLog:

2022-08-10  Jan Hubicka  <hubicka@ucw.cz>

	PR middle-end/106057
	* g++.dg/tree-ssa/pr101839.C: New test.

2 files changed, 82 insertions, 8 deletions

diff --git a/gcc/ipa-devirt.cc b/gcc/ipa-devirt.cc
index 412ca14..265d07b 100644
--- a/gcc/ipa-devirt.cc
+++ b/gcc/ipa-devirt.cc
@@ -285,6 +285,19 @@ type_possibly_instantiated_p (tree t)
   return vnode && vnode->definition;
 }
 
+/* Return true if T or type derived from T may have instance.  */
+
+static bool
+type_or_derived_type_possibly_instantiated_p (odr_type t)
+{
+  if (type_possibly_instantiated_p (t->type))
+    return true;
+  for (auto derived : t->derived_types)
+    if (type_or_derived_type_possibly_instantiated_p (derived))
+      return true;
+  return false;
+}
+
 /* Hash used to unify ODR types based on their mangled name and for anonymous
    namespace types.  */
 
@@ -3172,6 +3185,7 @@ possible_polymorphic_call_targets (tree otr_type,
     {
       odr_type speculative_outer_type;
       bool speculation_complete = true;
+      bool check_derived_types = false;
 
       /* First insert target from type itself and check if it may have
 	 derived types.  */
@@ -3190,8 +3204,12 @@ possible_polymorphic_call_targets (tree otr_type,
 	 to walk derivations.  */
       if (target && DECL_FINAL_P (target))
 	context.speculative_maybe_derived_type = false;
-      if (type_possibly_instantiated_p (speculative_outer_type->type))
-	maybe_record_node (nodes, target, &inserted, can_refer, &speculation_complete);
+      if (check_derived_types
+	  ? type_or_derived_type_possibly_instantiated_p
+		 (speculative_outer_type)
+	  : type_possibly_instantiated_p (speculative_outer_type->type))
+	maybe_record_node (nodes, target, &inserted, can_refer,
+			   &speculation_complete);
       if (binfo)
 	matched_vtables.add (BINFO_VTABLE (binfo));
 
@@ -3212,6 +3230,7 @@ possible_polymorphic_call_targets (tree otr_type,
 
   if (!speculative || !nodes.length ())
     {
+      bool check_derived_types = false;
       /* First see virtual method of type itself.  */
       binfo = get_binfo_at_offset (TYPE_BINFO (outer_type->type),
 				   context.offset, otr_type);
@@ -3229,16 +3248,18 @@ possible_polymorphic_call_targets (tree otr_type,
       if (target && DECL_CXX_DESTRUCTOR_P (target))
 	context.maybe_in_construction = false;
 
-      if (target)
+      /* In the case we get complete method, we don't need 
+	 to walk derivations.  */
+      if (target && DECL_FINAL_P (target))
 	{
-	  /* In the case we get complete method, we don't need 
-	     to walk derivations.  */
-	  if (DECL_FINAL_P (target))
-	    context.maybe_derived_type = false;
+	  check_derived_types = true;
+	  context.maybe_derived_type = false;
 	}
 
       /* If OUTER_TYPE is abstract, we know we are not seeing its instance.  */
-      if (type_possibly_instantiated_p (outer_type->type))
+      if (check_derived_types
+	  ? type_or_derived_type_possibly_instantiated_p (outer_type)
+	  : type_possibly_instantiated_p (outer_type->type))
 	maybe_record_node (nodes, target, &inserted, can_refer, &complete);
       else
 	skipped = true;
diff --git a/gcc/testsuite/g++.dg/tree-ssa/pr101839.C b/gcc/testsuite/g++.dg/tree-ssa/pr101839.C
new file mode 100644
index 0000000..bb7b61c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/tree-ssa/pr101839.C
@@ -0,0 +1,53 @@
+// { dg-do run }                                                            
+// { dg-options "-O2 -fdump-tree-optimized" }                                   
+// { dg-require-effective-target c++11 }
+
+#include <string.h>
+#include <iostream>
+#include <map>
+namespace {
+  struct Buf {
+    char * buf; int a{0}; int b{0};
+    Buf(char * b) : buf(b) { }
+    void add(int v) {
+      ::memcpy(buf, &v, sizeof(v));
+      a += sizeof(v);
+      b += sizeof(v);
+    }
+  };
+  struct A {
+    virtual void fill(Buf &buf) {
+      buf.add(type());
+      buf.add(type());
+    }
+    virtual ~A() {}
+    virtual int type() = 0;
+  };
+  struct BA : A {
+    void fill(Buf &buf) {
+      A::fill(buf);
+      buf.add(type());
+      buf.add(type());
+    }
+    int type() final {
+      return 1;
+    }
+  };
+  struct CBA final : BA {
+  };
+  struct CA final : A {
+    ::std::map<int, int> m;
+    int type() final {
+      return 2;
+    }
+  };
+}
+int main(int argc, char ** ) {
+  char d[1024];
+  CBA cba;
+  Buf buf(d);
+  cba.fill(buf);
+  CA ca;
+  return 0;
+}
+// { dg-final { scan-tree-dump-not "__builtin_unreachable" "optimized" } }