c++: comptypes ICE with BOUND_TEMPLATE_TEMPLATE_PARMs [PR107539]

Here we end up giving the two BOUND_TEMPLATE_TEMPLATE_PARMs
C<decltype(f::t)> and C<decltype(g::t)> the same TYPE_CANONICAL because
the hash table that interns TYPE_CANONICAL for template type parameters
doesn't set the comparing_specializations flag which controls how
PARM_DECLs from different contexts compare equal.

Later, from spec_hasher::equal for the corresponding two specializations
A<C<decltype(f::t)>> and A<C<decltype(g::t)>>, we compare the two bound
ttps with comparing_specializations set hence they now (structurally)
compare different despite having the same TYPE_CANONICAL, and so we get
the error:

  internal compiler error: same canonical type node for different types
    'C<decltype (t)>' and 'C<decltype (t)>'

This suggests that we should be setting comparing_specializations from
ctp_hasher::equal to match spec_hasher::equal.  But doing so introduces
a separate ICE in cpp2a/concepts-placeholder3.C:

  internal compiler error: canonical types differ for identical types
    'auto [requires ::same_as<<placeholder>, decltype(f::x)>]' and
    'auto [requires ::same_as<<placeholder>, decltype(g::x)>]'

because norm_hasher::equal doesn't set comparing_specializations either.

I'm not sure when exactly we need to set comparing_specializations given
what it controls (TYPENAME_TYPE equality/hashing and PARM_DECL equality)
but it seems to be the conservative choice to set the flag wherever we
have a global hash table that relies on type equality.  To that end this
patch sets comparing_specializations in ctp_hasher and norm_hasher, as
well as in atom_hasher and sat_hasher for good measure.  This turns out
to be a compile time win of about 2% in some concepts tests, probably
because of the improved TYPENAME_TYPE hashing enabled by the flag.

	PR c++/107539

gcc/cp/ChangeLog:

	* constraint.cc (norm_hasher::hash, norm_hasher::equal): Set
	comparing_specializations.
	(sat_hasher::hash, sat_hasher::equal): Likewise.
	* cp-tree.h (atom_hasher::hash, atom_hasher::equal): Likewise.
	* pt.cc (ctp_hasher::hash, ctp_hasher::equal): Likewise.

gcc/testsuite/ChangeLog:

	* g++.dg/template/canon-type-19.C: New test.

4 files changed, 47 insertions, 6 deletions

diff --git a/gcc/cp/constraint.cc b/gcc/cp/constraint.cc
index ab0f66b..37eae03 100644
--- a/gcc/cp/constraint.cc
+++ b/gcc/cp/constraint.cc
@@ -715,14 +715,20 @@ struct norm_hasher : ggc_ptr_hash<norm_entry>
 {
   static hashval_t hash (norm_entry *e)
   {
-    hashval_t hash = iterative_hash_template_arg (e->tmpl, 0);
-    return iterative_hash_template_arg (e->args, hash);
+    ++comparing_specializations;
+    hashval_t val = iterative_hash_template_arg (e->tmpl, 0);
+    val = iterative_hash_template_arg (e->args, val);
+    --comparing_specializations;
+    return val;
   }
 
   static bool equal (norm_entry *e1, norm_entry *e2)
   {
-    return e1->tmpl == e2->tmpl
+    ++comparing_specializations;
+    bool eq = e1->tmpl == e2->tmpl
       && template_args_equal (e1->args, e2->args);
+    --comparing_specializations;
+    return eq;
   }
 };
 
@@ -2530,6 +2536,9 @@ struct sat_hasher : ggc_ptr_hash<sat_entry>
 {
   static hashval_t hash (sat_entry *e)
   {
+    auto cso = make_temp_override (comparing_specializations);
+    ++comparing_specializations;
+
     if (ATOMIC_CONSTR_MAP_INSTANTIATED_P (e->atom))
       {
 	/* Atoms with instantiated mappings are built during satisfaction.
@@ -2564,6 +2573,9 @@ struct sat_hasher : ggc_ptr_hash<sat_entry>
 
   static bool equal (sat_entry *e1, sat_entry *e2)
   {
+    auto cso = make_temp_override (comparing_specializations);
+    ++comparing_specializations;
+
     if (ATOMIC_CONSTR_MAP_INSTANTIATED_P (e1->atom)
 	!= ATOMIC_CONSTR_MAP_INSTANTIATED_P (e2->atom))
       return false;
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index 548b533..addd26e 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -8418,12 +8418,18 @@ struct atom_hasher : default_hash_traits<tree>
 {
   static hashval_t hash (tree t)
   {
-    return hash_atomic_constraint (t);
+    ++comparing_specializations;
+    hashval_t val = hash_atomic_constraint (t);
+    --comparing_specializations;
+    return val;
   }
 
   static bool equal (tree t1, tree t2)
   {
-    return atomic_constraints_identical_p (t1, t2);
+    ++comparing_specializations;
+    bool eq = atomic_constraints_identical_p (t1, t2);
+    --comparing_specializations;
+    return eq;
   }
 };
 
diff --git a/gcc/cp/pt.cc b/gcc/cp/pt.cc
index 08de273..3169161 100644
--- a/gcc/cp/pt.cc
+++ b/gcc/cp/pt.cc
@@ -4492,18 +4492,23 @@ struct ctp_hasher : ggc_ptr_hash<tree_node>
 {
   static hashval_t hash (tree t)
   {
+    ++comparing_specializations;
     tree_code code = TREE_CODE (t);
     hashval_t val = iterative_hash_object (code, 0);
     val = iterative_hash_object (TEMPLATE_TYPE_LEVEL (t), val);
     val = iterative_hash_object (TEMPLATE_TYPE_IDX (t), val);
     if (TREE_CODE (t) == BOUND_TEMPLATE_TEMPLATE_PARM)
       val = iterative_hash_template_arg (TYPE_TI_ARGS (t), val);
+    --comparing_specializations;
     return val;
   }
 
   static bool equal (tree t, tree u)
   {
-    return comptypes (t, u, COMPARE_STRUCTURAL);
+    ++comparing_specializations;
+    bool eq = comptypes (t, u, COMPARE_STRUCTURAL);
+    --comparing_specializations;
+    return eq;
   }
 };
 
diff --git a/gcc/testsuite/g++.dg/template/canon-type-19.C b/gcc/testsuite/g++.dg/template/canon-type-19.C
new file mode 100644
index 0000000..27ef3f1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/canon-type-19.C
@@ -0,0 +1,18 @@
+// PR c++/107539
+// { dg-do compile { target c++11 } }
+
+template<class T>
+struct A { };
+
+template<template<class> class C>
+struct B {
+  template<class T>
+  void f(T t) {
+    A<C<decltype(t)>> a1;
+  }
+
+  template<class T>
+  void g(T t) {
+    A<C<decltype(t)>> a2;
+  }
+};