testsuite: generalize support for Python tests for SARIF output

In r15-2354-g4d1f71d49e396c I added the ability to use Python to write
tests of SARIF output via a new "run-sarif-pytest" based
on "run-gcov-pytest", with a sarif.py support script in
testsuite/gcc.dg/sarif-output.

This followup patch:
(a) removes the limitation of such tests needing to be in
testsuite/gcc.dg/sarif-output by moving sarif.py to testsuite/lib
and adding logic to add that directory to PYTHONPATH when invoking
pytest.

(b) uses this to replace fragile regexp-based tests in
gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c with
Python logic that verifies the structure within the generated JSON,
and to add test coverage for SARIF output relating to GCC plugins.

gcc/ChangeLog:
	* diagnostic-format-sarif.cc: Add comments noting that we don't
	yet capture any diagnostic_metadata::rules associated with a
	diagnostic.

gcc/testsuite/ChangeLog:
	* gcc.dg/plugin/diagnostic-test-metadata-sarif.c: New test,
	based on diagnostic-test-metadata.c.
	* gcc.dg/plugin/diagnostic-test-metadata-sarif.py: New script.
	* gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c:
	Replace scan-sarif-file directives with run-sarif-pytest, to
	run...
	* gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py:
	...this new test.
	* gcc.dg/plugin/plugin.exp (plugin_test_list): Add
	diagnostic-test-metadata-sarif.c.
	* gcc.dg/sarif-output/sarif.py: Move to...
	* lib/sarif.py: ...here.
	* lib/scansarif.exp (run-sarif-pytest): Prepend "lib" to
	PYTHONPATH before running python scripts.

Signed-off-by: David Malcolm <dmalcolm@redhat.com>

8 files changed, 208 insertions, 16 deletions

diff --git a/gcc/diagnostic-format-sarif.cc b/gcc/diagnostic-format-sarif.cc
index 963a185..1d99c90 100644
--- a/gcc/diagnostic-format-sarif.cc
+++ b/gcc/diagnostic-format-sarif.cc
@@ -580,7 +580,9 @@ public:
      (SARIF v2.1.0 section 3.27.13).
    - doesn't capture -Werror cleanly
    - doesn't capture inlining information (can SARIF handle this?)
-   - doesn't capture macro expansion information (can SARIF handle this?).  */
+   - doesn't capture macro expansion information (can SARIF handle this?).
+   - doesn't capture any diagnostic_metadata::rules associated with
+     a diagnostic.  */
 
 class sarif_builder
 {
@@ -1522,6 +1524,8 @@ sarif_builder::make_result_object (diagnostic_context &context,
 	}
 
       diagnostic.metadata->maybe_add_sarif_properties (*result_obj);
+
+      /* We don't yet support diagnostic_metadata::rule.  */
     }
 
   /* "level" property (SARIF v2.1.0 section 3.27.10).  */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c
new file mode 100644
index 0000000..246a842
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-format=sarif-file" } */
+
+extern char *gets (char *s);
+
+void test_cwe (void)
+{
+  char buf[1024];
+  gets (buf);
+}
+
+/* Verify that some JSON was written to a file with the expected name.  */
+/* { dg-final { verify-sarif-file } } */
+
+/* Use a Python script to verify various properties about the generated
+   .sarif file:
+   { dg-final { run-sarif-pytest diagnostic-test-metadata-sarif.c "diagnostic-test-metadata-sarif.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py
new file mode 100644
index 0000000..959e6f2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-metadata-sarif.py
@@ -0,0 +1,55 @@
+# We expect a warning with this textual form:
+#
+# . PATH/diagnostic-test-metadata-sarif.c: In function 'test_cwe':
+# . PATH/diagnostic-test-metadata-sarif.c:8:3: warning: never use 'gets' [CWE-242] [STR34-C]
+
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_basics(sarif):
+    schema = sarif['$schema']
+    assert schema == "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
+
+    version = sarif['version']
+    assert version == "2.1.0"
+
+def test_plugin_metadata(sarif):
+    runs = sarif['runs']
+    assert len(runs) == 1
+
+    run = runs[0]
+    tool = run['tool']
+
+    # We expect an extension, for the plugin.
+    extensions = tool['extensions']
+    assert len(extensions) == 1
+
+    extension = extensions[0]
+    assert extension['name'] == 'diagnostic_plugin_test_metadata'
+    assert 'diagnostic_plugin_test_metadata' in extension['fullName']
+
+    # TODO: ideally there would be a rule for [STR34-C] within
+    # the extension for the plugin with
+    #   'id': 'STR34-C',
+    #   'helpUri': 'https://example.com/'
+
+def test_result(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+    results = run['results']
+
+    assert len(results) == 1
+    
+    result = results[0]
+    assert result['level'] == 'warning'
+    assert result['message']['text'] == "never use 'gets'"
+
+    assert len(result['taxa']) == 1
+    taxon = result['taxa'][0]
+    assert taxon['id'] == '242'
+    assert taxon['toolComponent']['name'] == 'cwe'
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
index 727d1bb..f0f31d0 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.c
@@ -19,17 +19,6 @@ void bar ()
 /* Verify that some JSON was written to a file with the expected name.  */
 /* { dg-final { verify-sarif-file } } */
 
-/* We expect various properties.
-   The indentation here reflects the expected hierarchy, though these tests
-   don't check for that, merely the string fragments we expect.
-
-   { dg-final { scan-sarif-file {"version": "2.1.0"} } }
-     { dg-final { scan-sarif-file {"text": "deadlock due to inconsistent lock acquisition order"} } }
-     { dg-final { scan-sarif-file {"id": "Thread 1"} } }
-       { dg-final { scan-sarif-file {"executionOrder": 1} } }
-       { dg-final { scan-sarif-file {"executionOrder": 2} } }
-       { dg-final { scan-sarif-file {"executionOrder": 5} } }
-     { dg-final { scan-sarif-file {"id": "Thread 2"} } }
-       { dg-final { scan-sarif-file {"executionOrder": 3} } }
-       { dg-final { scan-sarif-file {"executionOrder": 4} } }
-       { dg-final { scan-sarif-file {"executionOrder": 6} } }  */
+/* Use a Python script to verify various properties about the generated
+   .sarif file:
+   { dg-final { run-sarif-pytest diagnostic-test-paths-multithreaded-sarif.c "diagnostic-test-paths-multithreaded-sarif.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py
new file mode 100644
index 0000000..cff78aa
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-paths-multithreaded-sarif.py
@@ -0,0 +1,109 @@
+# We expect a warning with this textual form:
+#
+# . warning: deadlock due to inconsistent lock acquisition order
+# .    17 |   acquire_lock_a (); /* { dg-warning "deadlock due to inconsistent lock acquisition order" } */
+# .       |   ^~~~~~~~~~~~~~~~~
+# . Thread: 'Thread 1'
+# .   'foo': event 1
+# .     |
+# .     |    9 | {
+# .     |      | ^
+# .     |      | |
+# .     |      | (1) entering 'foo'
+# .     |
+# .     +--> 'foo': event 2
+# .            |
+# .            |   10 |   acquire_lock_a ();
+# .            |      |   ^~~~~~~~~~~~~~~~~
+# .            |      |   |
+# .            |      |   (2) lock a is now held by thread 1
+# .            |
+# . 
+# . Thread: 'Thread 2'
+# .   'bar': event 3
+# .     |
+# .     |   15 | {
+# .     |      | ^
+# .     |      | |
+# .     |      | (3) entering 'bar'
+# .     |
+# .     +--> 'bar': event 4
+# .            |
+# .            |   16 |   acquire_lock_b ();
+# .            |      |   ^~~~~~~~~~~~~~~~~
+# .            |      |   |
+# .            |      |   (4) lock b is now held by thread 2
+# .            |
+
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_basics(sarif):
+    schema = sarif['$schema']
+    assert schema == "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
+
+    version = sarif['version']
+    assert version == "2.1.0"
+
+def test_execution_successful(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+
+    invocations = run['invocations']
+    assert len(invocations) == 1
+    invocation = invocations[0]
+
+    # We expect a mere 'warning' to allow executionSuccessful be true
+    assert invocation['executionSuccessful'] == True
+
+def test_result(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+    results = run['results']
+
+    assert len(results) == 1
+    
+    result = results[0]
+    assert result['level'] == 'warning'
+    assert result['message']['text'] == "deadlock due to inconsistent lock acquisition order"
+
+    code_flows = result['codeFlows']
+    assert len(code_flows) == 1
+
+    code_flow = code_flows[0]
+
+    thread_flows = code_flow['threadFlows']
+    assert len(thread_flows) == 2
+
+    tf0 = thread_flows[0]
+    assert tf0['id'] == 'Thread 1'
+    
+    tf1 = thread_flows[1]
+    assert tf1['id'] == 'Thread 2'
+
+    assert len(tf0['locations']) == 3
+    assert tf0['locations'][0]['executionOrder'] == 1
+    assert tf0['locations'][0]['location']['message']['text'] \
+        == "entering 'foo'"
+    assert tf0['locations'][1]['executionOrder'] == 2
+    assert tf0['locations'][1]['location']['message']['text'] \
+        == "lock a is now held by thread 1"
+    assert tf0['locations'][2]['executionOrder'] == 5
+    assert tf0['locations'][2]['location']['message']['text'] \
+        == "deadlocked due to waiting for lock b in thread 1..."
+
+    assert len(tf1['locations']) == 3
+    assert tf1['locations'][0]['executionOrder'] == 3
+    assert tf1['locations'][0]['location']['message']['text'] \
+        == "entering 'bar'"
+    assert tf1['locations'][1]['executionOrder'] == 4
+    assert tf1['locations'][1]['location']['message']['text'] \
+        == "lock b is now held by thread 2"
+    assert tf1['locations'][2]['executionOrder'] == 6
+    assert tf1['locations'][2]['location']['message']['text'] \
+        == "...whilst waiting for lock a in thread 2"
diff --git a/gcc/testsuite/gcc.dg/plugin/plugin.exp b/gcc/testsuite/gcc.dg/plugin/plugin.exp
index 933f9a5..2c2d919 100644
--- a/gcc/testsuite/gcc.dg/plugin/plugin.exp
+++ b/gcc/testsuite/gcc.dg/plugin/plugin.exp
@@ -101,7 +101,9 @@ set plugin_test_list [list \
 	  diagnostic-test-inlining-2.c \
 	  diagnostic-test-inlining-3.c \
 	  diagnostic-test-inlining-4.c } \
-    { diagnostic_plugin_test_metadata.c diagnostic-test-metadata.c } \
+    { diagnostic_plugin_test_metadata.c \
+	  diagnostic-test-metadata.c \
+	  diagnostic-test-metadata-sarif.c } \
     { diagnostic_plugin_test_paths.c \
 	  diagnostic-test-paths-1.c \
 	  diagnostic-test-paths-2.c \
diff --git a/gcc/testsuite/gcc.dg/sarif-output/sarif.py b/gcc/testsuite/lib/sarif.py
index 7daf35b..7daf35b 100644
--- a/gcc/testsuite/gcc.dg/sarif-output/sarif.py
+++ b/gcc/testsuite/lib/sarif.py
diff --git a/gcc/testsuite/lib/scansarif.exp b/gcc/testsuite/lib/scansarif.exp
index e08f80c..ea6a883 100644
--- a/gcc/testsuite/lib/scansarif.exp
+++ b/gcc/testsuite/lib/scansarif.exp
@@ -135,8 +135,24 @@ proc run-sarif-pytest { args } {
     }
 
     setenv SARIF_PATH $testcase
+    set libdir "${srcdir}/lib"
+
+    # Set/prepend libdir to PYTHONPATH
+    if [info exists ::env(PYTHONPATH)] {
+	set old_PYTHONPATH $::env(PYTHONPATH)
+	setenv PYTHONPATH "${libdir}:${old_PYTHONPATH}"
+    } else {
+	setenv PYTHONPATH "${libdir}"
+    }
+    
+    verbose "PYTHONPATH=[getenv PYTHONPATH]" 2
+
     spawn -noecho python3 -m pytest --color=no -rap -s --tb=no $srcdir/$subdir/$pytest_script
 
+    if [info exists old_PYTHONPATH] {
+	setenv PYTHONPATH ${old_PYTHONPATH}
+    }
+
     set prefix "\[^\r\n\]*"
     expect {
       -re "FAILED($prefix)\[^\r\n\]+\r\n" {