Add generic C.UTF-8 locale (Bug 17318)

We add a new C.UTF-8 locale. This locale is not builtin to glibc, but
is provided as a distinct locale. The locale provides full support for
UTF-8 and this includes full code point sorting via STRCMP-based
collation (strcmp or wcscmp).

The collation uses a new keyword 'codepoint_collation' which drops all
collation rules and generates an empty zero rules collation to enable
STRCMP usage in collation. This ensures that we get full code point
sorting for C.UTF-8 with a minimal 1406 bytes of overhead (LC_COLLATE
structure information and ASCII collating tables).

The new locale is added to SUPPORTED. Minimal test data for specific
code points (minus those not supported by collate-test) is provided in
C.UTF-8.in, and this verifies code point sorting is working reasonably
across the range. The locale was tested manually with the full set of
code points without failure.

The locale is harmonized with locales already shipping in various
downstream distributions. A new tst-iconv9 test is added which verifies
the C.UTF-8 locale is generally usable.

Testing for fnmatch, regexec, and recomp is provided by extending
bug-regex1, bugregex19, bug-regex4, bug-regex6, transbug, tst-fnmatch,
tst-regcomp-truncated, and tst-regex to use C.UTF-8.

Tested on x86_64 or i686 without regression.

Reviewed-by: Florian Weimer <fweimer@redhat.com>

2 files changed, 106 insertions, 3 deletions

diff --git a/iconv/Makefile b/iconv/Makefile
index 07d77c9..9993f2d 100644
--- a/iconv/Makefile
+++ b/iconv/Makefile
@@ -43,8 +43,19 @@ CFLAGS-charmap.c += -DCHARMAP_PATH='"$(i18ndir)/charmaps"' \
 CFLAGS-linereader.c += -DNO_TRANSLITERATION
 CFLAGS-simple-hash.c += -I../locale
 
-tests	= tst-iconv1 tst-iconv2 tst-iconv3 tst-iconv4 tst-iconv5 tst-iconv6 \
-	  tst-iconv7 tst-iconv8 tst-iconv-mt tst-iconv-opt
+tests = \
+	tst-iconv1 \
+	tst-iconv2 \
+	tst-iconv3 \
+	tst-iconv4 \
+	tst-iconv5 \
+	tst-iconv6 \
+	tst-iconv7 \
+	tst-iconv8 \
+	tst-iconv9 \
+	tst-iconv-mt \
+	tst-iconv-opt \
+	# tests
 
 others		= iconv_prog iconvconfig
 install-others-programs	= $(inst_bindir)/iconv
@@ -83,10 +94,15 @@ endif
 include ../Rules
 
 ifeq ($(run-built-tests),yes)
-LOCALES := en_US.UTF-8
+# We have to generate locales (list sorted alphabetically)
+LOCALES := \
+	C.UTF-8 \
+	en_US.UTF-8 \
+	# LOCALES
 include ../gen-locales.mk
 
 $(objpfx)tst-iconv-opt.out: $(gen-locales)
+$(objpfx)tst-iconv9.out: $(gen-locales)
 endif
 
 $(inst_bindir)/iconv: $(objpfx)iconv_prog $(+force)
diff --git a/iconv/tst-iconv9.c b/iconv/tst-iconv9.c
new file mode 100644
index 0000000..c46b183
--- /dev/null
+++ b/iconv/tst-iconv9.c
@@ -0,0 +1,87 @@
+/* Verify that using C.UTF-8 works.
+
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <iconv.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+#include <support/support.h>
+#include <support/check.h>
+
+/* This test does two things:
+   (1) Verify that we have likely included translit_combining in C.UTF-8.
+   (2) Verify default_missing is '?' as expected.  */
+
+/* ISO-8859-1 encoding of "für".  */
+char iso88591_in[] = { 0x66, 0xfc, 0x72, 0x0 };
+/* ASCII transliteration is "fur" with C.UTF-8 translit_combining.  */
+char ascii_exp[] = { 0x66, 0x75, 0x72, 0x0 };
+
+/* First 3-byte UTF-8 code point.  */
+char utf8_in[] = { 0xe0, 0xa0, 0x80, 0x0 };
+/* There is no ASCII transliteration for SAMARITAN LETTER ALAF
+   so we get default_missing used which is '?'.  */
+char default_missing_exp[] = { 0x3f, 0x0 };
+
+static int
+do_test (void)
+{
+  char ascii_out[5];
+  iconv_t cd;
+  char *inbuf;
+  char *outbuf;
+  size_t inbytes;
+  size_t outbytes;
+  size_t n;
+
+  /* The C.UTF-8 locale should include translit_combining, which provides
+     the transliteration for "LATIN SMALL LETTER U WITH DIAERESIS" which
+     is not provided by locale/C-translit.h.in.  */
+  xsetlocale (LC_ALL, "C.UTF-8");
+
+  /* From ISO-8859-1 to ASCII.  */
+  cd = iconv_open ("ASCII//TRANSLIT,IGNORE", "ISO-8859-1");
+  TEST_VERIFY (cd != (iconv_t) -1);
+  inbuf = iso88591_in;
+  inbytes = 3;
+  outbuf = ascii_out;
+  outbytes = 3;
+  n = iconv (cd, &inbuf, &inbytes, &outbuf, &outbytes);
+  TEST_VERIFY (n != -1);
+  *outbuf = '\0';
+  TEST_COMPARE_BLOB (ascii_out, 3, ascii_exp, 3);
+  TEST_VERIFY (iconv_close (cd) == 0);
+
+  /* From UTF-8 to ASCII.  */
+  cd = iconv_open ("ASCII//TRANSLIT,IGNORE", "UTF-8");
+  TEST_VERIFY (cd != (iconv_t) -1);
+  inbuf = utf8_in;
+  inbytes = 3;
+  outbuf = ascii_out;
+  outbytes = 3;
+  n = iconv (cd, &inbuf, &inbytes, &outbuf, &outbytes);
+  TEST_VERIFY (n != -1);
+  *outbuf = '\0';
+  TEST_COMPARE_BLOB (ascii_out, 1, default_missing_exp, 1);
+  TEST_VERIFY (iconv_close (cd) == 0);
+
+  return 0;
+}
+
+#include <support/test-driver.c>