[nvptx] Handle DCmode in define_expand "omp_simt_xchg_{bfly,idx}"

For a test-case doing an openmp target simd reduction on a complex double:
...
  DOUBLE COMPLEX :: counter_N0
  ...
  !$OMP TARGET SIMD reduction(+: counter_N0)
...
we run into:
...
during RTL pass: expand
b.f90: In function ‘MAIN__._omp_fn.0’:
b.f90:23:32: internal compiler error: in expand_insn, at optabs.cc:8029
   23 |     counter_N0 = counter_N0 + 1.
      |                                ^
0x10f1cd3 expand_insn(insn_code, unsigned int, expand_operand*)
        gcc/optabs.cc:8029
0xeac435 expand_GOMP_SIMT_XCHG_BFLY
        gcc/internal-fn.cc:375
...

Fix this by handling DCmode and CDImode in define_expand
"omp_simt_xchg_{bfly,idx}".

Tested on x86_64 with nvptx accelerator.

gcc/ChangeLog:

2022-02-28  Tom de Vries  <tdevries@suse.de>

	PR target/102429
	* config/nvptx/nvptx.cc (nvptx_gen_shuffle): Handle DCmode and CDImode.
	* config/nvptx/nvptx.md
	(define_predicate "nvptx_register_or_complex_di_df_register_operand"):
	New predicate.
	(define_expand "omp_simt_xchg_bfly", define_expand "omp_simt_xchg_idx"):
	Use nvptx_register_or_complex_di_df_register_operand.

2 files changed, 33 insertions, 4 deletions

diff --git a/gcc/config/nvptx/nvptx.cc b/gcc/config/nvptx/nvptx.cc
index f3179ef..6ca99a6 100644
--- a/gcc/config/nvptx/nvptx.cc
+++ b/gcc/config/nvptx/nvptx.cc
@@ -1941,6 +1941,23 @@ nvptx_gen_shuffle (rtx dst, rtx src, rtx idx, nvptx_shuffle_kind kind)
 
   switch (GET_MODE (dst))
     {
+      case E_DCmode:
+      case E_CDImode:
+	{
+	  gcc_assert (GET_CODE (dst) == CONCAT);
+	  gcc_assert (GET_CODE (src) == CONCAT);
+	  rtx dst_real = XEXP (dst, 0);
+	  rtx dst_imag = XEXP (dst, 1);
+	  rtx src_real = XEXP (src, 0);
+	  rtx src_imag = XEXP (src, 1);
+
+	  start_sequence ();
+	  emit_insn (nvptx_gen_shuffle (dst_real, src_real, idx, kind));
+	  emit_insn (nvptx_gen_shuffle (dst_imag, src_imag, idx, kind));
+	  res = get_insns ();
+	  end_sequence ();
+	}
+	break;
     case E_SImode:
       res = gen_nvptx_shufflesi (dst, src, idx, GEN_INT (kind));
       break;
diff --git a/gcc/config/nvptx/nvptx.md b/gcc/config/nvptx/nvptx.md
index 4989b56..a453c1d 100644
--- a/gcc/config/nvptx/nvptx.md
+++ b/gcc/config/nvptx/nvptx.md
@@ -94,6 +94,18 @@
   return register_operand (op, mode);
 })
 
+(define_predicate "nvptx_register_or_complex_di_df_register_operand"
+  (ior (match_code "reg")
+       (match_code "concat"))
+{
+  if (GET_CODE (op) == CONCAT)
+    return ((GET_MODE (op) == DCmode || GET_MODE (op) == CDImode)
+	    && nvptx_register_operand (XEXP (op, 0), mode)
+	    && nvptx_register_operand (XEXP (op, 1), mode));
+
+  return nvptx_register_operand (op, mode);
+})
+
 (define_predicate "nvptx_nonimmediate_operand"
   (match_code "mem,reg")
 {
@@ -1902,8 +1914,8 @@
 ;; Implement IFN_GOMP_SIMT_XCHG_BFLY: perform a "butterfly" exchange
 ;; across lanes
 (define_expand "omp_simt_xchg_bfly"
-  [(match_operand 0 "nvptx_register_operand" "=R")
-   (match_operand 1 "nvptx_register_operand" "R")
+  [(match_operand 0 "nvptx_register_or_complex_di_df_register_operand" "=R")
+   (match_operand 1 "nvptx_register_or_complex_di_df_register_operand" "R")
    (match_operand:SI 2 "nvptx_nonmemory_operand" "Ri")]
   ""
 {
@@ -1915,8 +1927,8 @@
 ;; Implement IFN_GOMP_SIMT_XCHG_IDX: broadcast value in operand 1
 ;; from lane given by index in operand 2 to operand 0 in all lanes
 (define_expand "omp_simt_xchg_idx"
-  [(match_operand 0 "nvptx_register_operand" "=R")
-   (match_operand 1 "nvptx_register_operand" "R")
+  [(match_operand 0 "nvptx_register_or_complex_di_df_register_operand" "=R")
+   (match_operand 1 "nvptx_register_or_complex_di_df_register_operand" "R")
    (match_operand:SI 2 "nvptx_nonmemory_operand" "Ri")]
   ""
 {