diagnostics: ensure that .sarif files are UTF-8 encoded [PR109098]

PR analyzer/109098 notes that the SARIF spec mandates that .sarif
files are UTF-8 encoded, but -fdiagnostics-format=sarif-file naively
assumes that the source files are UTF-8 encoded when quoting source
artefacts in the .sarif output, which can lead to us writing out
.sarif files with non-UTF-8 bytes in them (which break my reporting
scripts).

The root cause is that sarif_builder::maybe_make_artifact_content_object
was using maybe_read_file to load the file content as bytes, and
assuming they were UTF-8 encoded.

This patch reworks both overloads of this function (one used for the
whole file, the other for snippets of quoted lines) so that they go
through input.cc's file cache, which attempts to decode the input files
according to the input charset, and then encode as UTF-8.  They also
check that the result actually is UTF-8, for cases where the input
charset is missing, or incorrectly specified, and omit the quoted
source for such awkward cases.

Doing so fixes all of the cases I've encountered.

The patch adds a new:
  { dg-final { verify-sarif-file } }
directive to all SARIF test cases in the test suite, which verifies
that the output is UTF-8 encoded, and is valid JSON.  In particular
it verifies that when we complain about encoding problems, the .sarif
report we emit is itself correctly encoded.

gcc/ChangeLog:
	PR analyzer/109098
	* diagnostic-format-sarif.cc (read_until_eof): Delete.
	(maybe_read_file): Delete.
	(sarif_builder::maybe_make_artifact_content_object): Use
	get_source_file_content rather than maybe_read_file.
	Reject it if it's not valid UTF-8.
	* input.cc (file_cache_slot::get_full_file_content): New.
	(get_source_file_content): New.
	(selftest::check_cpp_valid_utf8_p): New.
	(selftest::test_cpp_valid_utf8_p): New.
	(selftest::input_cc_tests): Call selftest::test_cpp_valid_utf8_p.
	* input.h (get_source_file_content): New prototype.

gcc/testsuite/ChangeLog:
	PR analyzer/109098
	* c-c++-common/diagnostic-format-sarif-file-1.c: Add
	verify-sarif-file directive.
	* c-c++-common/diagnostic-format-sarif-file-2.c: Likewise.
	* c-c++-common/diagnostic-format-sarif-file-3.c: Likewise.
	* c-c++-common/diagnostic-format-sarif-file-4.c: Likewise.
	* c-c++-common/diagnostic-format-sarif-file-Wbidi-chars.c: New
	test case, adapted from Wbidi-chars-1.c.
	* c-c++-common/diagnostic-format-sarif-file-bad-utf8-pr109098-1.c:
	New test case.
	* c-c++-common/diagnostic-format-sarif-file-bad-utf8-pr109098-2.c:
	New test case.
	* c-c++-common/diagnostic-format-sarif-file-bad-utf8-pr109098-3.c:
	New test case, adapted from cpp/Winvalid-utf8-1.c.
	* c-c++-common/diagnostic-format-sarif-file-valid-CP850.c: New
	test case, adapted from gcc.dg/diagnostic-input-charset-1.c.
	* gcc.dg/plugin/crash-test-ice-sarif.c: Add verify-sarif-file
	directive.
	* gcc.dg/plugin/crash-test-write-though-null-sarif.c: Likewise.
	* gcc.dg/plugin/diagnostic-test-paths-5.c: Likewise.
	* lib/scansarif.exp (verify-sarif-file): New procedure.
	* lib/verify-sarif-file.py: New support script.

libcpp/ChangeLog:
	PR analyzer/109098
	* charset.cc (cpp_valid_utf8_p): New function.
	* include/cpplib.h (cpp_valid_utf8_p): New prototype.

Signed-off-by: David Malcolm <dmalcolm@redhat.com>

1 files changed, 125 insertions, 0 deletions

diff --git a/gcc/input.cc b/gcc/input.cc
index 9702dbd..eaf301e 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -67,6 +67,7 @@ public:
   {
     return m_missing_trailing_newline;
   }
+  char_span get_full_file_content ();
 
   void inc_use_count () { m_use_count++; }
 
@@ -459,6 +460,20 @@ file_cache::add_file (const char *file_path)
   return r;
 }
 
+/* Get a borrowed char_span to the full content of this file
+   as decoded according to the input charset, encoded as UTF-8.  */
+
+char_span
+file_cache_slot::get_full_file_content ()
+{
+  char *line;
+  ssize_t line_len;
+  while (get_next_line (&line, &line_len))
+    {
+    }
+  return char_span (m_data, m_nb_read);
+}
+
 /* Populate this slot for use on FILE_PATH and FP, dropping any
    existing cached content within it.  */
 
@@ -1047,6 +1062,18 @@ get_source_text_between (location_t start, location_t end)
   return xstrdup (buf);
 }
 
+/* Get a borrowed char_span to the full content of FILE_PATH
+   as decoded according to the input charset, encoded as UTF-8.  */
+
+char_span
+get_source_file_content (const char *file_path)
+{
+  diagnostic_file_cache_init ();
+
+  file_cache_slot *c = global_dc->m_file_cache->lookup_or_add_file (file_path);
+  return c->get_full_file_content ();
+}
+
 /* Determine if FILE_PATH missing a trailing newline on its final line.
    Only valid to call once all of the file has been loaded, by
    requesting a line number beyond the end of the file.  */
@@ -4045,7 +4072,104 @@ void test_cpp_utf8 ()
 	  ASSERT_EQ (byte_col2, byte_col);
       }
   }
+}
+
+static bool
+check_cpp_valid_utf8_p (const char *str)
+{
+  return cpp_valid_utf8_p (str, strlen (str));
+}
+
+/* Check that cpp_valid_utf8_p works as expected.  */
+
+static void
+test_cpp_valid_utf8_p ()
+{
+  ASSERT_TRUE (check_cpp_valid_utf8_p ("hello world"));
+
+  /* 2-byte char (pi).  */
+  ASSERT_TRUE (check_cpp_valid_utf8_p("\xcf\x80"));
+
+  /* 3-byte chars (the Japanese word "mojibake").  */
+  ASSERT_TRUE (check_cpp_valid_utf8_p
+	       (
+		/* U+6587 CJK UNIFIED IDEOGRAPH-6587
+		   UTF-8: 0xE6 0x96 0x87
+		   C octal escaped UTF-8: \346\226\207.  */
+		"\346\226\207"
+		/* U+5B57 CJK UNIFIED IDEOGRAPH-5B57
+		   UTF-8: 0xE5 0xAD 0x97
+		   C octal escaped UTF-8: \345\255\227.  */
+		"\345\255\227"
+		/* U+5316 CJK UNIFIED IDEOGRAPH-5316
+		   UTF-8: 0xE5 0x8C 0x96
+		   C octal escaped UTF-8: \345\214\226.  */
+		"\345\214\226"
+		/* U+3051 HIRAGANA LETTER KE
+		   UTF-8: 0xE3 0x81 0x91
+		   C octal escaped UTF-8: \343\201\221.  */
+		"\343\201\221"));
+
+  /* 4-byte char: an emoji.  */
+  ASSERT_TRUE (check_cpp_valid_utf8_p ("\xf0\x9f\x98\x82"));
+
+  /* Control codes, including the NUL byte.  */
+  ASSERT_TRUE (cpp_valid_utf8_p ("\r\n\v\0\1", 5));
+
+  ASSERT_FALSE (check_cpp_valid_utf8_p ("\xf0!\x9f!\x98!\x82!"));
+
+  /* Unexpected continuation bytes.  */
+  for (unsigned char continuation_byte = 0x80;
+       continuation_byte <= 0xbf;
+       continuation_byte++)
+    ASSERT_FALSE (cpp_valid_utf8_p ((const char *)&continuation_byte, 1));
+
+  /* "Lonely start characters" for 2-byte sequences.  */
+  {
+    unsigned char buf[2];
+    buf[1] = ' ';
+    for (buf[0] = 0xc0;
+	 buf[0] <= 0xdf;
+	 buf[0]++)
+      ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
+  }
+
+  /* "Lonely start characters" for 3-byte sequences.  */
+  {
+    unsigned char buf[2];
+    buf[1] = ' ';
+    for (buf[0] = 0xe0;
+	 buf[0] <= 0xef;
+	 buf[0]++)
+      ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
+  }
+
+  /* "Lonely start characters" for 4-byte sequences.  */
+  {
+    unsigned char buf[2];
+    buf[1] = ' ';
+    for (buf[0] = 0xf0;
+	 buf[0] <= 0xf4;
+	 buf[0]++)
+      ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
+  }
+
+  /* Invalid start characters (formerly valid for 5-byte and 6-byte
+     sequences).  */
+  {
+    unsigned char buf[2];
+    buf[1] = ' ';
+    for (buf[0] = 0xf5;
+	 buf[0] <= 0xfd;
+	 buf[0]++)
+      ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
+  }
 
+  /* Impossible bytes.  */
+  ASSERT_FALSE (check_cpp_valid_utf8_p ("\xc0"));
+  ASSERT_FALSE (check_cpp_valid_utf8_p ("\xc1"));
+  ASSERT_FALSE (check_cpp_valid_utf8_p ("\xfe"));
+  ASSERT_FALSE (check_cpp_valid_utf8_p ("\xff"));
 }
 
 /* Run all of the selftests within this file.  */
@@ -4091,6 +4215,7 @@ input_cc_tests ()
   test_line_offset_overflow ();
 
   test_cpp_utf8 ();
+  test_cpp_valid_utf8_p ();
 }
 
 } // namespace selftest