libcpp: Implement C++23 P2290R3 - Delimited escape sequences [PR106645]

The following patch implements the C++23 P2290R3 paper.

2022-08-20  Jakub Jelinek  <jakub@redhat.com>

	PR c++/106645
libcpp/
	* include/cpplib.h (struct cpp_options): Implement
	P2290R3 - Delimited escape sequences.  Add delimite_escape_seqs
	member.
	* init.cc (struct lang_flags): Likewise.
	(lang_defaults): Add delim column.
	(cpp_set_lang): Copy over delimite_escape_seqs.
	* charset.cc (extend_char_range): New function.
	(_cpp_valid_ucn): Use it.  Handle delimited escape sequences.
	(convert_hex): Likewise.
	(convert_oct): Likewise.
	(convert_ucn): Use extend_char_range.
	(convert_escape): Call convert_oct even for \o.
	(_cpp_interpret_identifier): Handle delimited escape sequences.
	* lex.cc (get_bidi_ucn_1): Likewise.  Add end argument, fill it in.
	(get_bidi_ucn): Adjust get_bidi_ucn_1 caller.  Use end argument to
	compute num_bytes.
gcc/testsuite/
	* c-c++-common/cpp/delimited-escape-seq-1.c: New test.
	* c-c++-common/cpp/delimited-escape-seq-2.c: New test.
	* c-c++-common/cpp/delimited-escape-seq-3.c: New test.
	* c-c++-common/Wbidi-chars-24.c: New test.
	* gcc.dg/cpp/delimited-escape-seq-1.c: New test.
	* gcc.dg/cpp/delimited-escape-seq-2.c: New test.
	* g++.dg/cpp/delimited-escape-seq-1.C: New test.
	* g++.dg/cpp/delimited-escape-seq-2.C: New test.

8 files changed, 207 insertions, 0 deletions

diff --git a/gcc/testsuite/c-c++-common/Wbidi-chars-24.c b/gcc/testsuite/c-c++-common/Wbidi-chars-24.c
new file mode 100644
index 0000000..608a5d6
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wbidi-chars-24.c
@@ -0,0 +1,28 @@
+/* PR preprocessor/103026 */
+/* { dg-do compile } */
+/* { dg-options "-Wbidi-chars=ucn,unpaired" } */
+/* Test nesting of bidi chars in various contexts.  */
+
+void
+g1 ()
+{
+  const char *s1 = "a b c LRE\u{202a} 1 2 3 PDI\u{00000000000000000000000002069} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+  const char *s2 = "a b c RLE\u{00202b} 1 2 3 PDI\u{2069} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+  const char *s3 = "a b c LRO\u{000000202d} 1 2 3 PDI\u{02069} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+  const char *s4 = "a b c RLO\u{202e} 1 2 3 PDI\u{00000002069} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+  const char *s5 = "a b c LRI\u{002066} 1 2 3 PDF\u{202C} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+  const char *s6 = "a b c RLI\u{02067} 1 2 3 PDF\u{202c} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+  const char *s7 = "a b c FSI\u{0002068} 1 2 3 PDF\u{0202c} x y z";
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+}
+
+int A\u{202a}B\u{2069}C;
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
+int a\u{00000202b}B\u{000000002069}c;
+/* { dg-warning "unpaired" "" { target *-*-* } .-1 } */
diff --git a/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-1.c b/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-1.c
new file mode 100644
index 0000000..04d822d
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-1.c
@@ -0,0 +1,92 @@
+/* P2290R3 - Delimited escape sequences */
+/* { dg-do run } */
+/* { dg-require-effective-target wchar } */
+/* { dg-options "-std=gnu99 -Wno-c++-compat" { target c } } */
+/* { dg-options "-std=c++23" { target c++ } } */
+
+#ifndef __cplusplus
+#include <wchar.h>
+typedef __CHAR16_TYPE__ char16_t;
+typedef __CHAR32_TYPE__ char32_t;
+#endif
+
+const char32_t *a = U"\u{1234}\u{10fffd}\u{000000000000000000000000000000000000000000000000000000000001234}\u{10FFFD}";
+const char32_t *b = U"\x{1234}\x{10fffd}\x{000000000000000000000000000000000000000000000000000000000001234}";
+const char32_t *c = U"\o{1234}\o{4177775}\o{000000000000000000000000000000000000000000000000000000000000000000000000004177775}";
+const char16_t *d = u"\u{1234}\u{bFFd}\u{00000000000000000000000000000001234}";
+const char16_t *e = u"\x{1234}\x{BffD}\x{000001234}";
+const char16_t *f = u"\o{1234}\o{137775}\o{000000000000000137775}";
+const wchar_t *g = L"\u{1234}\u{bFFd}\u{00000000000000000000000000000001234}";
+const wchar_t *h = L"\x{1234}\x{bFFd}\x{000001234}";
+const wchar_t *i = L"\o{1234}\o{137775}\o{000000000000000137775}";
+#ifdef __cplusplus
+const char *j = "\u{34}\u{000000000000000003D}";
+#endif
+const char *k = "\x{34}\x{000000000000000003D}";
+const char *l = "\o{34}\o{000000000000000176}";
+
+#if U'\u{1234}' != U'\u1234' || U'\u{10fffd}' != U'\U0010FFFD' \
+    || U'\x{00000001234}' != U'\x1234' || U'\x{010fffd}' != U'\x10FFFD' \
+    || U'\o{1234}' != U'\x29c' || U'\o{004177775}' != U'\x10FFFD' \
+    || u'\u{1234}' != u'\u1234' || u'\u{0bffd}' != u'\uBFFD' \
+    || u'\x{00000001234}' != u'\x1234' || u'\x{0Bffd}' != u'\x0bFFD' \
+    || u'\o{1234}' != u'\x29c' || u'\o{00137775}' != u'\xBFFD' \
+    || L'\u{1234}' != L'\u1234' || L'\u{0bffd}' != L'\uBFFD' \
+    || L'\x{00000001234}' != L'\x1234' || L'\x{0bffd}' != L'\x0bFFD' \
+    || L'\o{1234}' != L'\x29c' || L'\o{00137775}' != L'\xBFFD' \
+    || '\x{34}' != '\x034' || '\x{0003d}' != '\x003D' \
+    || '\o{34}' != '\x1C' || '\o{176}' != '\x007E'
+#error Bad
+#endif
+#ifdef __cplusplus
+#if '\u{0000000034}' != '\u0034' || '\u{3d}' != '\u003D'
+#error Bad
+#endif
+#endif
+
+int
+main ()
+{
+  if (a[0] != U'\u1234' || a[0] != U'\u{1234}'
+      || a[1] != U'\U0010FFFD' || a[1] != U'\u{000010fFfD}'
+      || a[2] != a[0]
+      || a[3] != a[1]
+      || b[0] != U'\x1234' || b[0] != U'\x{001234}'
+      || b[1] != U'\x10FFFD' || b[1] != U'\x{0010fFfD}'
+      || b[2] != b[0]
+      || c[0] != U'\x29c' || c[0] != U'\o{001234}'
+      || c[1] != U'\x10FFFD' || c[1] != U'\o{4177775}'
+      || c[2] != c[1])
+    __builtin_abort ();
+  if (d[0] != u'\u1234' || d[0] != u'\u{1234}'
+      || d[1] != u'\U0000BFFD' || d[1] != u'\u{00000bFfD}'
+      || d[2] != d[0]
+      || e[0] != u'\x1234' || e[0] != u'\x{001234}'
+      || e[1] != u'\xBFFD' || e[1] != u'\x{00bFfD}'
+      || e[2] != e[0]
+      || f[0] != u'\x29c' || f[0] != u'\o{001234}'
+      || f[1] != u'\xbFFD' || f[1] != u'\o{137775}'
+      || f[2] != f[1])
+    __builtin_abort ();
+  if (g[0] != L'\u1234' || g[0] != L'\u{1234}'
+      || g[1] != L'\U0000BFFD' || g[1] != L'\u{00000bFfD}'
+      || g[2] != g[0]
+      || h[0] != L'\x1234' || h[0] != L'\x{001234}'
+      || h[1] != L'\xBFFD' || h[1] != L'\x{00bFfD}'
+      || h[2] != h[0]
+      || i[0] != L'\x29c' || i[0] != L'\o{001234}'
+      || i[1] != L'\xbFFD' || i[1] != L'\o{137775}'
+      || i[2] != i[1])
+    __builtin_abort ();
+#ifdef __cplusplus
+  if (j[0] != '\u0034' || j[0] != '\u{034}'
+      || j[1] != '\U0000003D' || j[1] != '\u{000003d}')
+    __builtin_abort ();
+#endif
+  if (k[0] != '\x034' || k[0] != '\x{0034}'
+      || k[1] != '\x3D' || k[1] != '\x{3d}'
+      || l[0] != '\x1c' || l[0] != '\o{0034}'
+      || l[1] != '\x07e' || l[1] != '\o{176}' || l[1] != '\176')
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-2.c b/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-2.c
new file mode 100644
index 0000000..b23b83f
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-2.c
@@ -0,0 +1,18 @@
+/* P2290R3 - Delimited escape sequences */
+/* { dg-do compile } */
+/* { dg-options "-std=gnu99 -Wno-c++-compat" { target c } } */
+/* { dg-options "-std=c++23" { target c++ } } */
+
+int jalape\u{f1}o = 42;
+
+int
+caf\u{000e9} (void)
+{
+  return jalape\u00F1o;
+}
+
+int
+test (void)
+{
+  return caf\u00e9 ();
+}
diff --git a/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-3.c b/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-3.c
new file mode 100644
index 0000000..ef9e5b5
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/delimited-escape-seq-3.c
@@ -0,0 +1,33 @@
+/* P2290R3 - Delimited escape sequences */
+/* { dg-do compile } */
+/* { dg-require-effective-target wchar } */
+/* { dg-options "-std=gnu99 -Wno-c++-compat" { target c } } */
+/* { dg-options "-std=c++23" { target c++ } } */
+
+#ifndef __cplusplus
+typedef __CHAR32_TYPE__ char32_t;
+#endif
+
+const char32_t *a = U"\u{}";				/* { dg-error "empty delimited escape sequence" } */
+							/* { dg-error "is not a valid universal character" "" { target c } .-1 } */
+const char32_t *b = U"\u{12" "34}";			/* { dg-error "'\\\\u\\{' not terminated with '\\}' after" } */
+const char32_t *c = U"\u{0000ffffffff}";		/* { dg-error "is not a valid universal character" } */
+const char32_t *d = U"\u{010000edcb}";			/* { dg-error "is not a valid universal character" } */
+const char32_t *e = U"\u{02000000000000000000edcb}";	/* { dg-error "is not a valid universal character" } */
+const char32_t *f = U"\u{123ghij}";			/* { dg-error "'\\\\u\\{' not terminated with '\\}' after" } */
+const char32_t *g = U"\u{123.}";			/* { dg-error "'\\\\u\\{' not terminated with '\\}' after" } */
+const char32_t *h = U"\u{.}";				/* { dg-error "'\\\\u\\{' not terminated with '\\}' after" } */
+const char32_t *i = U"\x{}";				/* { dg-error "empty delimited escape sequence" } */
+const char32_t *j = U"\x{12" "34}";			/* { dg-error "'\\\\x\\{' not terminated with '\\}' after" } */
+const char32_t *k = U"\x{0000ffffffff}";
+const char32_t *l = U"\x{010000edcb}";			/* { dg-warning "hex escape sequence out of range" } */
+const char32_t *m = U"\x{02000000000000000000edcb}";	/* { dg-warning "hex escape sequence out of range" } */
+const char32_t *n = U"\x{123ghij}";			/* { dg-error "'\\\\x\\{' not terminated with '\\}' after" } */
+const char32_t *o = U"\x{123.}";			/* { dg-error "'\\\\x\\{' not terminated with '\\}' after" } */
+const char32_t *p = U"\o{}";				/* { dg-error "empty delimited escape sequence" } */
+const char32_t *q = U"\o{12" "34}";			/* { dg-error "'\\\\o\\{' not terminated with '\\}' after" } */
+const char32_t *r = U"\o{0000037777777777}";
+const char32_t *s = U"\o{040000166713}";		/* { dg-warning "octal escape sequence out of range" } */
+const char32_t *t = U"\o{02000000000000000000000166713}";/* { dg-warning "octal escape sequence out of range" } */
+const char32_t *u = U"\o{1238}";			/* { dg-error "'\\\\o\\{' not terminated with '\\}' after" } */
+const char32_t *v = U"\o{.}";				/* { dg-error "'\\\\o\\{' not terminated with '\\}' after" } */
diff --git a/gcc/testsuite/g++.dg/cpp/delimited-escape-seq-1.C b/gcc/testsuite/g++.dg/cpp/delimited-escape-seq-1.C
new file mode 100644
index 0000000..a9c8a0c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp/delimited-escape-seq-1.C
@@ -0,0 +1,8 @@
+// P2290R3 - Delimited escape sequences
+// { dg-do compile { target c++11 } }
+// { dg-require-effective-target wchar }
+// { dg-options "-pedantic" }
+
+const char32_t *a = U"\u{1234}";	// { dg-warning "delimited escape sequences are only valid in" "" { target c++20_down } }
+const char32_t *b = U"\x{1234}";	// { dg-warning "delimited escape sequences are only valid in" "" { target c++20_down } }
+const char32_t *c = U"\o{1234}";	// { dg-warning "delimited escape sequences are only valid in" "" { target c++20_down } }
diff --git a/gcc/testsuite/g++.dg/cpp/delimited-escape-seq-2.C b/gcc/testsuite/g++.dg/cpp/delimited-escape-seq-2.C
new file mode 100644
index 0000000..51af741
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp/delimited-escape-seq-2.C
@@ -0,0 +1,8 @@
+// P2290R3 - Delimited escape sequences
+// { dg-do compile { target c++11 } }
+// { dg-require-effective-target wchar }
+// { dg-options "-pedantic-errors" }
+
+const char32_t *a = U"\u{1234}";	// { dg-error "delimited escape sequences are only valid in" "" { target c++20_down } }
+const char32_t *b = U"\x{1234}";	// { dg-error "delimited escape sequences are only valid in" "" { target c++20_down } }
+const char32_t *c = U"\o{1234}";	// { dg-error "delimited escape sequences are only valid in" "" { target c++20_down } }
diff --git a/gcc/testsuite/gcc.dg/cpp/delimited-escape-seq-1.c b/gcc/testsuite/gcc.dg/cpp/delimited-escape-seq-1.c
new file mode 100644
index 0000000..e08775f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/cpp/delimited-escape-seq-1.c
@@ -0,0 +1,10 @@
+/* P2290R3 - Delimited escape sequences */
+/* { dg-do compile } */
+/* { dg-require-effective-target wchar } */
+/* { dg-options "-std=gnu99 -Wno-c++-compat -pedantic" } */
+
+typedef __CHAR32_TYPE__ char32_t;
+
+const char32_t *a = U"\u{1234}";	/* { dg-warning "delimited escape sequences are only valid in" } */
+const char32_t *b = U"\x{1234}";	/* { dg-warning "delimited escape sequences are only valid in" } */
+const char32_t *c = U"\o{1234}";	/* { dg-warning "delimited escape sequences are only valid in" } */
diff --git a/gcc/testsuite/gcc.dg/cpp/delimited-escape-seq-2.c b/gcc/testsuite/gcc.dg/cpp/delimited-escape-seq-2.c
new file mode 100644
index 0000000..71c9865
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/cpp/delimited-escape-seq-2.c
@@ -0,0 +1,10 @@
+/* P2290R3 - Delimited escape sequences */
+/* { dg-do compile } */
+/* { dg-require-effective-target wchar } */
+/* { dg-options "-std=gnu99 -Wno-c++-compat -pedantic-errors" } */
+
+typedef __CHAR32_TYPE__ char32_t;
+
+const char32_t *a = U"\u{1234}";	/* { dg-error "delimited escape sequences are only valid in" } */
+const char32_t *b = U"\x{1234}";	/* { dg-error "delimited escape sequences are only valid in" } */
+const char32_t *c = U"\o{1234}";	/* { dg-error "delimited escape sequences are only valid in" } */