Add expected stdout for failing-meson and warning-meson tests

Initially produced using:

  for d in "test cases/failing/"* ; do rm -r _build ; ./meson.py setup "$d" _build | grep ERROR >"$d"/expected_stdout.txt; done

then converted to json with jq using:

  jq --raw-input --slurp 'split("\n") | {stdout: map({line: select(. != "")})}' expected_stdout.txt >test.json

or merged with existing json using:

  jq --slurp '.[0] + .[1]' test.json expected.json >test.json.new

v2:
Add some comments to explain the match when it isn't totally obvious

v3:
Add or adjust existing re: in expected output to handle '/' or '\' path
separators appearing in message, not location.

v4:
Put expected stdout in test.json, rather than a separate expected_stdout.txt file
Park comments in an unused 'comments' key, as JSON doesn't have a syntax for comments

95 files changed, 707 insertions, 10 deletions

diff --git a/test cases/failing/1 project not first/test.json b/test cases/failing/1 project not first/test.json
new file mode 100644
index 0000000..70f3c41
--- /dev/null
+++ b/test cases/failing/1 project not first/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "ERROR: First statement must be a call to project"
+    }
+  ]
+}
diff --git a/test cases/failing/10 out of bounds/test.json b/test cases/failing/10 out of bounds/test.json
new file mode 100644
index 0000000..e27d990
--- /dev/null
+++ b/test cases/failing/10 out of bounds/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/10 out of bounds/meson.build:4:0: ERROR: Index 0 out of bounds of array of size 0."
+    }
+  ]
+}
diff --git a/test cases/failing/11 object arithmetic/test.json b/test cases/failing/11 object arithmetic/test.json
new file mode 100644
index 0000000..5339fac
--- /dev/null
+++ b/test cases/failing/11 object arithmetic/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/11 object arithmetic/meson\\.build:3:0: ERROR: Invalid use of addition: .*"
+    }
+  ]
+}
diff --git a/test cases/failing/12 string arithmetic/test.json b/test cases/failing/12 string arithmetic/test.json
new file mode 100644
index 0000000..476f9bb
--- /dev/null
+++ b/test cases/failing/12 string arithmetic/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/12 string arithmetic/meson\\.build:3:0: ERROR: Invalid use of addition: .*"
+    }
+  ]
+}
diff --git a/test cases/failing/13 array arithmetic/test.json b/test cases/failing/13 array arithmetic/test.json
new file mode 100644
index 0000000..55056ce
--- /dev/null
+++ b/test cases/failing/13 array arithmetic/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/13 array arithmetic/meson.build:3:0: ERROR: Multiplication works only with integers."
+    }
+  ]
+}
diff --git a/test cases/failing/14 invalid option name/test.json b/test cases/failing/14 invalid option name/test.json
new file mode 100644
index 0000000..71e685d
--- /dev/null
+++ b/test cases/failing/14 invalid option name/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/14 invalid option name/meson_options.txt:1:0: ERROR: Option names can only contain letters, numbers or dashes."
+    }
+  ]
+}
diff --git a/test cases/failing/15 kwarg before arg/test.json b/test cases/failing/15 kwarg before arg/test.json
new file mode 100644
index 0000000..c7f72c3
--- /dev/null
+++ b/test cases/failing/15 kwarg before arg/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/15 kwarg before arg/meson.build:3:0: ERROR: All keyword arguments must be after positional arguments."
+    }
+  ]
+}
diff --git a/test cases/failing/16 extract from subproject/test.json b/test cases/failing/16 extract from subproject/test.json
new file mode 100644
index 0000000..78d45a5
--- /dev/null
+++ b/test cases/failing/16 extract from subproject/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/16 extract from subproject/meson.build:6:0: ERROR: Tried to extract objects from a subproject target."
+    }
+  ]
+}
diff --git a/test cases/failing/17 same target/test.json b/test cases/failing/17 same target/test.json
new file mode 100644
index 0000000..0005ba4
--- /dev/null
+++ b/test cases/failing/17 same target/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/17 same target/meson.build:4:0: ERROR: Tried to create target \"foo\", but a target of that name already exists."
+    }
+  ]
+}
diff --git a/test cases/failing/18 wrong plusassign/test.json b/test cases/failing/18 wrong plusassign/test.json
new file mode 100644
index 0000000..c698f85
--- /dev/null
+++ b/test cases/failing/18 wrong plusassign/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/18 wrong plusassign/meson.build:3:0: ERROR: Plusassignment target must be an id."
+    }
+  ]
+}
diff --git a/test cases/failing/19 target clash/test.json b/test cases/failing/19 target clash/test.json
new file mode 100644
index 0000000..d22b894
--- /dev/null
+++ b/test cases/failing/19 target clash/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "ERROR: Multiple producers for Ninja target \"clash\". Please rename your targets."
+    }
+  ]
+}
diff --git a/test cases/failing/2 missing file/test.json b/test cases/failing/2 missing file/test.json
new file mode 100644
index 0000000..b95b8b0
--- /dev/null
+++ b/test cases/failing/2 missing file/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/2 missing file/meson.build:3:0: ERROR: File missing.c does not exist."
+    }
+  ]
+}
diff --git a/test cases/failing/20 version/test.json b/test cases/failing/20 version/test.json
new file mode 100644
index 0000000..f330624
--- /dev/null
+++ b/test cases/failing/20 version/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/20 version/meson\\.build:1:0: ERROR: Meson version is .* but project requires >100\\.0\\.0"
+    }
+  ]
+}
diff --git a/test cases/failing/21 subver/test.json b/test cases/failing/21 subver/test.json
new file mode 100644
index 0000000..f8cfd3a
--- /dev/null
+++ b/test cases/failing/21 subver/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/21 subver/meson.build:3:0: ERROR: Subproject foo version is 1.0.0 but >1.0.0 required."
+    }
+  ]
+}
diff --git a/test cases/failing/22 assert/test.json b/test cases/failing/22 assert/test.json
new file mode 100644
index 0000000..edae999
--- /dev/null
+++ b/test cases/failing/22 assert/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/22 assert/meson.build:3:0: ERROR: Assert failed: I am fail."
+    }
+  ]
+}
diff --git a/test cases/failing/23 rel testdir/test.json b/test cases/failing/23 rel testdir/test.json
new file mode 100644
index 0000000..ba983ab
--- /dev/null
+++ b/test cases/failing/23 rel testdir/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/23 rel testdir/meson.build:4:0: ERROR: Workdir keyword argument must be an absolute path."
+    }
+  ]
+}
diff --git a/test cases/failing/24 int conversion/test.json b/test cases/failing/24 int conversion/test.json
new file mode 100644
index 0000000..e749928
--- /dev/null
+++ b/test cases/failing/24 int conversion/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/24 int conversion/meson.build:3:13: ERROR: String 'notanumber' cannot be converted to int"
+    }
+  ]
+}
diff --git a/test cases/failing/25 badlang/test.json b/test cases/failing/25 badlang/test.json
new file mode 100644
index 0000000..0b23fd7
--- /dev/null
+++ b/test cases/failing/25 badlang/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/25 badlang/meson.build:3:0: ERROR: Tried to use unknown language \"nonexisting\"."
+    }
+  ]
+}
diff --git a/test cases/failing/26 output subdir/test.json b/test cases/failing/26 output subdir/test.json
new file mode 100644
index 0000000..796468d
--- /dev/null
+++ b/test cases/failing/26 output subdir/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/26 output subdir/meson.build:3:0: ERROR: Output file name must not contain a subdirectory."
+    }
+  ]
+}
diff --git a/test cases/failing/27 noprog use/test.json b/test cases/failing/27 noprog use/test.json
new file mode 100644
index 0000000..b84562e
--- /dev/null
+++ b/test cases/failing/27 noprog use/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/27 noprog use/meson.build:5:0: ERROR: Tried to use not-found external program in \"command\""
+    }
+  ]
+}
diff --git a/test cases/failing/28 no crossprop/test.json b/test cases/failing/28 no crossprop/test.json
new file mode 100644
index 0000000..a186a68
--- /dev/null
+++ b/test cases/failing/28 no crossprop/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/28 no crossprop/meson.build:3:0: ERROR: Unknown cross property: nonexisting."
+    }
+  ]
+}
diff --git a/test cases/failing/29 nested ternary/test.json b/test cases/failing/29 nested ternary/test.json
new file mode 100644
index 0000000..ba05013
--- /dev/null
+++ b/test cases/failing/29 nested ternary/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/29 nested ternary/meson.build:3:12: ERROR: Nested ternary operators are not allowed."
+    }
+  ]
+}
diff --git a/test cases/failing/3 missing subdir/test.json b/test cases/failing/3 missing subdir/test.json
new file mode 100644
index 0000000..562de25
--- /dev/null
+++ b/test cases/failing/3 missing subdir/test.json
@@ -0,0 +1,9 @@
+{
+  "stdout": [
+    {
+        "comment": "'missing/meson.build' gets transformed with os.path.sep separators",
+        "match": "re",
+        "line": "test cases/failing/3 missing subdir/meson\\.build:3:0: ERROR: Non\\-existent build file 'missing[\\\\/]meson\\.build'"
+    }
+  ]
+}
diff --git a/test cases/failing/30 invalid man extension/test.json b/test cases/failing/30 invalid man extension/test.json
new file mode 100644
index 0000000..3f77a04
--- /dev/null
+++ b/test cases/failing/30 invalid man extension/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/30 invalid man extension/meson.build:2:0: ERROR: Man file must have a file extension of a number between 1 and 8"
+    }
+  ]
+}
diff --git a/test cases/failing/31 no man extension/test.json b/test cases/failing/31 no man extension/test.json
new file mode 100644
index 0000000..6e1f542
--- /dev/null
+++ b/test cases/failing/31 no man extension/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/31 no man extension/meson.build:2:0: ERROR: Man file must have a file extension of a number between 1 and 8"
+    }
+  ]
+}
diff --git a/test cases/failing/32 exe static shared/test.json b/test cases/failing/32 exe static shared/test.json
new file mode 100644
index 0000000..51d3804
--- /dev/null
+++ b/test cases/failing/32 exe static shared/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/32 exe static shared/meson.build:9:0: ERROR: Can't link non-PIC static library 'stat' into shared library 'shr2'. Use the 'pic' option to static_library to build with PIC."
+    }
+  ]
+}
diff --git a/test cases/failing/33 non-root subproject/test.json b/test cases/failing/33 non-root subproject/test.json
new file mode 100644
index 0000000..a14cece
--- /dev/null
+++ b/test cases/failing/33 non-root subproject/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/33 non-root subproject/some/meson.build:1:0: ERROR: Subproject directory not found and someproj.wrap file not found"
+    }
+  ]
+}
diff --git a/test cases/failing/34 dependency not-required then required/test.json b/test cases/failing/34 dependency not-required then required/test.json
new file mode 100644
index 0000000..bed1a45
--- /dev/null
+++ b/test cases/failing/34 dependency not-required then required/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/34 dependency not\\-required then required/meson\\.build:4:0: ERROR: Dependency \"foo\\-bar\\-xyz\\-12\\.3\" not found, tried .*"
+    }
+  ]
+}
diff --git a/test cases/failing/35 project argument after target/test.json b/test cases/failing/35 project argument after target/test.json
new file mode 100644
index 0000000..f5efd9b
--- /dev/null
+++ b/test cases/failing/35 project argument after target/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/35 project argument after target/meson.build:7:0: ERROR: Tried to use 'add_project_arguments' after a build target has been declared."
+    }
+  ]
+}
diff --git a/test cases/failing/37 has function external dependency/test.json b/test cases/failing/37 has function external dependency/test.json
new file mode 100644
index 0000000..81d6f91
--- /dev/null
+++ b/test cases/failing/37 has function external dependency/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/37 has function external dependency/meson.build:8:3: ERROR: Dependencies must be external dependencies"
+    }
+  ]
+}
diff --git a/test cases/failing/38 libdir must be inside prefix/test.json b/test cases/failing/38 libdir must be inside prefix/test.json
index 1cd893c..d9256d1 100644
--- a/test cases/failing/38 libdir must be inside prefix/test.json
+++ b/test cases/failing/38 libdir must be inside prefix/test.json
@@ -1,3 +1,10 @@
 {
-  "do_not_set_opts": ["libdir"]
+  "do_not_set_opts": [
+    "libdir"
+  ],
+  "stdout": [
+    {
+      "line": "test cases/failing/38 libdir must be inside prefix/meson.build:1:0: ERROR: The value of the 'libdir' option is '/opt/lib' which must be a subdir of the prefix '/usr'."
+    }
+  ]
 }
diff --git a/test cases/failing/39 prefix absolute/test.json b/test cases/failing/39 prefix absolute/test.json
index 4e0f6cd..2770243 100644
--- a/test cases/failing/39 prefix absolute/test.json
+++ b/test cases/failing/39 prefix absolute/test.json
@@ -1,3 +1,11 @@
 {
-  "do_not_set_opts": ["prefix"]
+  "do_not_set_opts": [
+    "prefix"
+  ],
+  "stdout": [
+    {
+      "comment": "literal 'some/path/notabs' appears in output, irrespective of os.path.sep, as that's the prefix",
+      "line": "test cases/failing/39 prefix absolute/meson.build:1:0: ERROR: prefix value 'some/path/notabs' must be an absolute path"
+    }
+  ]
 }
diff --git a/test cases/failing/4 missing meson.build/test.json b/test cases/failing/4 missing meson.build/test.json
new file mode 100644
index 0000000..3857090
--- /dev/null
+++ b/test cases/failing/4 missing meson.build/test.json
@@ -0,0 +1,9 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "comment": "'subdir/meson.build' gets transformed with os.path.sep separators",
+      "line": "test cases/failing/4 missing meson\\.build/meson\\.build:3:0: ERROR: Non\\-existent build file 'subdir[\\\\/]meson\\.build'"
+    }
+  ]
+}
diff --git a/test cases/failing/40 kwarg assign/test.json b/test cases/failing/40 kwarg assign/test.json
new file mode 100644
index 0000000..671eb3f
--- /dev/null
+++ b/test cases/failing/40 kwarg assign/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/40 kwarg assign/meson.build:3:0: ERROR: Tried to assign values inside an argument list."
+    }
+  ]
+}
diff --git a/test cases/failing/41 custom target plainname many inputs/test.json b/test cases/failing/41 custom target plainname many inputs/test.json
new file mode 100644
index 0000000..8c15cda
--- /dev/null
+++ b/test cases/failing/41 custom target plainname many inputs/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/41 custom target plainname many inputs/meson.build:5:0: ERROR: Output cannot contain @PLAINNAME@ or @BASENAME@ when there is more than one input (we can't know which to use)"
+    }
+  ]
+}
diff --git a/test cases/failing/42 custom target outputs not matching install_dirs/test.json b/test cases/failing/42 custom target outputs not matching install_dirs/test.json
index e59cb9f..f9e2ba7 100644
--- a/test cases/failing/42 custom target outputs not matching install_dirs/test.json
+++ b/test cases/failing/42 custom target outputs not matching install_dirs/test.json
@@ -1,10 +1,33 @@
 {
   "installed": [
-    {"type": "file", "file": "usr/include/diff.h"},
-    {"type": "file", "file": "usr/include/first.h"},
-    {"type": "file", "file": "usr/bin/diff.sh"},
-    {"type": "file", "file": "usr/bin/second.sh"},
-    {"type": "file", "file": "opt/same.h"},
-    {"type": "file", "file": "opt/same.sh"}
+    {
+      "type": "file",
+      "file": "usr/include/diff.h"
+    },
+    {
+      "type": "file",
+      "file": "usr/include/first.h"
+    },
+    {
+      "type": "file",
+      "file": "usr/bin/diff.sh"
+    },
+    {
+      "type": "file",
+      "file": "usr/bin/second.sh"
+    },
+    {
+      "type": "file",
+      "file": "opt/same.h"
+    },
+    {
+      "type": "file",
+      "file": "opt/same.sh"
+    }
+  ],
+  "stdout": [
+    {
+      "line": "ERROR: Target 'too-few-install-dirs' has 3 outputs: ['toofew.h', 'toofew.c', 'toofew.sh'], but only 2 \"install_dir\"s were found."
+    }
   ]
 }
diff --git a/test cases/failing/43 project name colon/test.json b/test cases/failing/43 project name colon/test.json
new file mode 100644
index 0000000..7a55574
--- /dev/null
+++ b/test cases/failing/43 project name colon/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/43 project name colon/meson.build:1:0: ERROR: Project name 'name with :' must not contain ':'"
+    }
+  ]
+}
diff --git a/test cases/failing/44 abs subdir/test.json b/test cases/failing/44 abs subdir/test.json
new file mode 100644
index 0000000..0aa56f6
--- /dev/null
+++ b/test cases/failing/44 abs subdir/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/44 abs subdir/meson.build:5:0: ERROR: Subdir argument must be a relative path."
+    }
+  ]
+}
diff --git a/test cases/failing/45 abspath to srcdir/test.json b/test cases/failing/45 abspath to srcdir/test.json
new file mode 100644
index 0000000..b6a87fe
--- /dev/null
+++ b/test cases/failing/45 abspath to srcdir/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/45 abspath to srcdir/meson.build:3:0: ERROR: Tried to form an absolute path to a source dir. You should not do that but use relative paths instead."
+    }
+  ]
+}
diff --git a/test cases/failing/46 pkgconfig variables reserved/test.json b/test cases/failing/46 pkgconfig variables reserved/test.json
new file mode 100644
index 0000000..b92ee17
--- /dev/null
+++ b/test cases/failing/46 pkgconfig variables reserved/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/46 pkgconfig variables reserved/meson.build:8:5: ERROR: Variable \"prefix\" is reserved"
+    }
+  ]
+}
diff --git a/test cases/failing/47 pkgconfig variables zero length/test.json b/test cases/failing/47 pkgconfig variables zero length/test.json
new file mode 100644
index 0000000..097fee1
--- /dev/null
+++ b/test cases/failing/47 pkgconfig variables zero length/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/47 pkgconfig variables zero length/meson.build:8:5: ERROR: Invalid variable \"=value\". Variables must be in 'name=value' format"
+    }
+  ]
+}
diff --git a/test cases/failing/48 pkgconfig variables zero length value/test.json b/test cases/failing/48 pkgconfig variables zero length value/test.json
new file mode 100644
index 0000000..50a35ce
--- /dev/null
+++ b/test cases/failing/48 pkgconfig variables zero length value/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/48 pkgconfig variables zero length value/meson.build:8:5: ERROR: Invalid variable \"key=\". Variables must be in 'name=value' format"
+    }
+  ]
+}
diff --git a/test cases/failing/49 pkgconfig variables not key value/test.json b/test cases/failing/49 pkgconfig variables not key value/test.json
new file mode 100644
index 0000000..cf07e62
--- /dev/null
+++ b/test cases/failing/49 pkgconfig variables not key value/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/49 pkgconfig variables not key value/meson.build:8:5: ERROR: Invalid variable \"this_should_be_key_value\". Variables must be in 'name=value' format"
+    }
+  ]
+}
diff --git a/test cases/failing/5 misplaced option/test.json b/test cases/failing/5 misplaced option/test.json
new file mode 100644
index 0000000..12afdf0
--- /dev/null
+++ b/test cases/failing/5 misplaced option/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/5 misplaced option/meson.build:3:0: ERROR: Tried to call option() in build description file. All options must be in the option file."
+    }
+  ]
+}
diff --git a/test cases/failing/50 executable comparison/test.json b/test cases/failing/50 executable comparison/test.json
new file mode 100644
index 0000000..585b382
--- /dev/null
+++ b/test cases/failing/50 executable comparison/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/50 executable comparison/meson.build:6:0: ERROR: exe1 can only be compared for equality."
+    }
+  ]
+}
diff --git a/test cases/failing/51 inconsistent comparison/test.json b/test cases/failing/51 inconsistent comparison/test.json
new file mode 100644
index 0000000..5867f0a
--- /dev/null
+++ b/test cases/failing/51 inconsistent comparison/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/51 inconsistent comparison/meson.build:5:0: ERROR: Values of different types (list, str) cannot be compared using <."
+    }
+  ]
+}
diff --git a/test cases/failing/52 slashname/test.json b/test cases/failing/52 slashname/test.json
new file mode 100644
index 0000000..180400a
--- /dev/null
+++ b/test cases/failing/52 slashname/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/52 slashname/meson.build:11:0: ERROR: Problem encountered: Re-enable me once slash in name is finally prohibited."
+    }
+  ]
+}
diff --git a/test cases/failing/53 reserved meson prefix/test.json b/test cases/failing/53 reserved meson prefix/test.json
new file mode 100644
index 0000000..502d96a
--- /dev/null
+++ b/test cases/failing/53 reserved meson prefix/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/53 reserved meson prefix/meson.build:3:0: ERROR: The \"meson-\" prefix is reserved and cannot be used for top-level subdir()."
+    }
+  ]
+}
diff --git a/test cases/failing/54 wrong shared crate type/test.json b/test cases/failing/54 wrong shared crate type/test.json
new file mode 100644
index 0000000..5cced6f
--- /dev/null
+++ b/test cases/failing/54 wrong shared crate type/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/54 wrong shared crate type/meson.build:7:0: ERROR: Crate type \"staticlib\" invalid for dynamic libraries; must be \"dylib\" or \"cdylib\""
+    }
+  ]
+}
diff --git a/test cases/failing/55 wrong static crate type/test.json b/test cases/failing/55 wrong static crate type/test.json
new file mode 100644
index 0000000..7073f7b
--- /dev/null
+++ b/test cases/failing/55 wrong static crate type/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/55 wrong static crate type/meson.build:7:0: ERROR: Crate type \"cdylib\" invalid for static libraries; must be \"rlib\" or \"staticlib\""
+    }
+  ]
+}
diff --git a/test cases/failing/56 or on new line/test.json b/test cases/failing/56 or on new line/test.json
new file mode 100644
index 0000000..c55cee6
--- /dev/null
+++ b/test cases/failing/56 or on new line/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/56 or on new line/meson.build:4:8: ERROR: Invalid or clause."
+    }
+  ]
+}
diff --git a/test cases/failing/57 kwarg in module/test.json b/test cases/failing/57 kwarg in module/test.json
new file mode 100644
index 0000000..cafb3ab
--- /dev/null
+++ b/test cases/failing/57 kwarg in module/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/57 kwarg in module/meson.build:3:0: ERROR: Function does not take keyword arguments."
+    }
+  ]
+}
diff --git a/test cases/failing/58 link with executable/test.json b/test cases/failing/58 link with executable/test.json
new file mode 100644
index 0000000..d3975c1
--- /dev/null
+++ b/test cases/failing/58 link with executable/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/58 link with executable/meson.build:4:0: ERROR: Link target 'prog' is not linkable."
+    }
+  ]
+}
diff --git a/test cases/failing/59 assign custom target index/test.json b/test cases/failing/59 assign custom target index/test.json
new file mode 100644
index 0000000..07ecb91
--- /dev/null
+++ b/test cases/failing/59 assign custom target index/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/59 assign custom target index/meson.build:24:0: ERROR: Assignment target must be an id."
+    }
+  ]
+}
diff --git a/test cases/failing/6 missing incdir/test.json b/test cases/failing/6 missing incdir/test.json
new file mode 100644
index 0000000..172d8a9
--- /dev/null
+++ b/test cases/failing/6 missing incdir/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/6 missing incdir/meson.build:3:0: ERROR: Include dir nosuchdir does not exist."
+    }
+  ]
+}
diff --git a/test cases/failing/60 getoption prefix/test.json b/test cases/failing/60 getoption prefix/test.json
new file mode 100644
index 0000000..03bf419
--- /dev/null
+++ b/test cases/failing/60 getoption prefix/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/60 getoption prefix/meson.build:5:0: ERROR: Having a colon in option name is forbidden, projects are not allowed to directly access options of other subprojects."
+    }
+  ]
+}
diff --git a/test cases/failing/61 bad option argument/test.json b/test cases/failing/61 bad option argument/test.json
new file mode 100644
index 0000000..4002005
--- /dev/null
+++ b/test cases/failing/61 bad option argument/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/61 bad option argument/meson_options.txt:1:0: ERROR: Invalid kwargs for option \"name\": \"vaule\""
+    }
+  ]
+}
diff --git a/test cases/failing/62 subproj filegrab/test.json b/test cases/failing/62 subproj filegrab/test.json
new file mode 100644
index 0000000..dd0d7bb
--- /dev/null
+++ b/test cases/failing/62 subproj filegrab/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/62 subproj filegrab/subprojects/a/meson.build:3:0: ERROR: Sandbox violation: Tried to grab file prog.c from a different subproject."
+    }
+  ]
+}
diff --git a/test cases/failing/63 grab subproj/test.json b/test cases/failing/63 grab subproj/test.json
new file mode 100644
index 0000000..8147905
--- /dev/null
+++ b/test cases/failing/63 grab subproj/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/63 grab subproj/meson.build:7:0: ERROR: Sandbox violation: Tried to grab file sub.c from a different subproject."
+    }
+  ]
+}
diff --git a/test cases/failing/64 grab sibling/test.json b/test cases/failing/64 grab sibling/test.json
new file mode 100644
index 0000000..1604d47
--- /dev/null
+++ b/test cases/failing/64 grab sibling/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/64 grab sibling/subprojects/a/meson.build:3:0: ERROR: Sandbox violation: Tried to grab file sneaky.c from a different subproject."
+    }
+  ]
+}
diff --git a/test cases/failing/65 string as link target/test.json b/test cases/failing/65 string as link target/test.json
new file mode 100644
index 0000000..e212482
--- /dev/null
+++ b/test cases/failing/65 string as link target/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/65 string as link target/meson.build:2:0: ERROR: '' is not a target."
+    }
+  ]
+}
diff --git a/test cases/failing/66 dependency not-found and required/test.json b/test cases/failing/66 dependency not-found and required/test.json
new file mode 100644
index 0000000..5b13316
--- /dev/null
+++ b/test cases/failing/66 dependency not-found and required/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/66 dependency not-found and required/meson.build:2:0: ERROR: Dependency is both required and not-found"
+    }
+  ]
+}
diff --git a/test cases/failing/68 wrong boost module/test.json b/test cases/failing/68 wrong boost module/test.json
new file mode 100644
index 0000000..9ef1b0f
--- /dev/null
+++ b/test cases/failing/68 wrong boost module/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/68 wrong boost module/meson.build:9:0: ERROR: Dependency \"boost\" not found"
+    }
+  ]
+}
diff --git a/test cases/failing/69 install_data rename bad size/test.json b/test cases/failing/69 install_data rename bad size/test.json
new file mode 100644
index 0000000..1329fec
--- /dev/null
+++ b/test cases/failing/69 install_data rename bad size/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/69 install_data rename bad size/meson.build:3:0: ERROR: Size of rename argument is different from number of sources"
+    }
+  ]
+}
diff --git a/test cases/failing/7 go to subproject/test.json b/test cases/failing/7 go to subproject/test.json
new file mode 100644
index 0000000..c254757
--- /dev/null
+++ b/test cases/failing/7 go to subproject/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/7 go to subproject/meson.build:3:0: ERROR: Must not go into subprojects dir with subdir(), use subproject() instead."
+    }
+  ]
+}
diff --git a/test cases/failing/70 skip only subdir/test.json b/test cases/failing/70 skip only subdir/test.json
new file mode 100644
index 0000000..3b40b66
--- /dev/null
+++ b/test cases/failing/70 skip only subdir/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/70 skip only subdir/meson.build:8:0: ERROR: File main.cpp does not exist."
+    }
+  ]
+}
diff --git a/test cases/failing/71 dual override/test.json b/test cases/failing/71 dual override/test.json
new file mode 100644
index 0000000..66409e6
--- /dev/null
+++ b/test cases/failing/71 dual override/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/71 dual override/meson.build:5:6: ERROR: Tried to override executable \"override\" which has already been overridden."
+    }
+  ]
+}
diff --git a/test cases/failing/72 override used/test.json b/test cases/failing/72 override used/test.json
new file mode 100644
index 0000000..29a58f1
--- /dev/null
+++ b/test cases/failing/72 override used/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/72 override used/meson.build:5:6: ERROR: Tried to override finding of executable \"something.py\" which has already been found."
+    }
+  ]
+}
diff --git a/test cases/failing/73 run_command unclean exit/test.json b/test cases/failing/73 run_command unclean exit/test.json
new file mode 100644
index 0000000..beda187
--- /dev/null
+++ b/test cases/failing/73 run_command unclean exit/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/73 run_command unclean exit/meson\\.build:4:0: ERROR: Command \".*[\\\\/]test cases[\\\\/]failing[\\\\/]73 run_command unclean exit[\\\\/]\\.[\\\\/]returncode\\.py 1\" failed with status 1\\."
+    }
+  ]
+}
diff --git a/test cases/failing/74 int literal leading zero/test.json b/test cases/failing/74 int literal leading zero/test.json
new file mode 100644
index 0000000..78a735e
--- /dev/null
+++ b/test cases/failing/74 int literal leading zero/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "comment": "this error message is not very informative",
+      "line": "test cases/failing/74 int literal leading zero/meson.build:5:13: ERROR: Expecting eof got number."
+    }
+  ]
+}
diff --git a/test cases/failing/75 configuration immutable/test.json b/test cases/failing/75 configuration immutable/test.json
new file mode 100644
index 0000000..3365aae
--- /dev/null
+++ b/test cases/failing/75 configuration immutable/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/75 configuration immutable/meson.build:12:16: ERROR: Can not set values on configuration object that has been used."
+    }
+  ]
+}
diff --git a/test cases/failing/76 link with shared module on osx/test.json b/test cases/failing/76 link with shared module on osx/test.json
new file mode 100644
index 0000000..4e2856f
--- /dev/null
+++ b/test cases/failing/76 link with shared module on osx/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/76 link with shared module on osx/meson.build:8:0: ERROR: target links against shared modules."
+    }
+  ]
+}
diff --git a/test cases/failing/77 non ascii in ascii encoded configure file/test.json b/test cases/failing/77 non ascii in ascii encoded configure file/test.json
new file mode 100644
index 0000000..e35b95b
--- /dev/null
+++ b/test cases/failing/77 non ascii in ascii encoded configure file/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/77 non ascii in ascii encoded configure file/meson\\.build:5:0: ERROR: Could not write output file .*[\\\\/]config9\\.h: 'ascii' codec can't encode character '\\\\u0434' in position 17: ordinal not in range\\(128\\)"
+    }
+  ]
+}
diff --git a/test cases/failing/78 subproj dependency not-found and required/test.json b/test cases/failing/78 subproj dependency not-found and required/test.json
new file mode 100644
index 0000000..534b4f4
--- /dev/null
+++ b/test cases/failing/78 subproj dependency not-found and required/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/78 subproj dependency not-found and required/meson.build:2:0: ERROR: Subproject directory not found and missing.wrap file not found"
+    }
+  ]
+}
diff --git a/test cases/failing/79 unfound run/test.json b/test cases/failing/79 unfound run/test.json
new file mode 100644
index 0000000..6baafc0
--- /dev/null
+++ b/test cases/failing/79 unfound run/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/79 unfound run/meson.build:4:0: ERROR: Tried to use non-existing executable 'nonexisting_prog'"
+    }
+  ]
+}
diff --git a/test cases/failing/8 recursive/test.json b/test cases/failing/8 recursive/test.json
new file mode 100644
index 0000000..b4c964c
--- /dev/null
+++ b/test cases/failing/8 recursive/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/8 recursive/subprojects/b/meson.build:3:0: ERROR: Recursive include of subprojects: a => b => a."
+    }
+  ]
+}
diff --git a/test cases/failing/80 framework dependency with version/test.json b/test cases/failing/80 framework dependency with version/test.json
new file mode 100644
index 0000000..5cbc129
--- /dev/null
+++ b/test cases/failing/80 framework dependency with version/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/80 framework dependency with version/meson.build:8:0: ERROR: Unknown version of dependency 'appleframeworks', but need ['>0']."
+    }
+  ]
+}
diff --git a/test cases/failing/81 override exe config/test.json b/test cases/failing/81 override exe config/test.json
new file mode 100644
index 0000000..f19785b
--- /dev/null
+++ b/test cases/failing/81 override exe config/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/81 override exe config/meson.build:6:0: ERROR: Program 'bar' was overridden with the compiled executable 'foo' and therefore cannot be used during configuration"
+    }
+  ]
+}
diff --git a/test cases/failing/82 gl dependency with version/test.json b/test cases/failing/82 gl dependency with version/test.json
new file mode 100644
index 0000000..2c63a2c
--- /dev/null
+++ b/test cases/failing/82 gl dependency with version/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/82 gl dependency with version/meson.build:9:0: ERROR: Unknown version of dependency 'gl', but need ['>0']."
+    }
+  ]
+}
diff --git a/test cases/failing/83 threads dependency with version/test.json b/test cases/failing/83 threads dependency with version/test.json
new file mode 100644
index 0000000..b131be4
--- /dev/null
+++ b/test cases/failing/83 threads dependency with version/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/83 threads dependency with version/meson.build:3:0: ERROR: Unknown version of dependency 'threads', but need ['>0']."
+    }
+  ]
+}
diff --git a/test cases/failing/85 dub libray/test.json b/test cases/failing/85 dub libray/test.json
new file mode 100644
index 0000000..a8b3e28
--- /dev/null
+++ b/test cases/failing/85 dub libray/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/85 dub libray/meson.build:11:0: ERROR: Dependency \"dubtestproject\" not found"
+    }
+  ]
+}
diff --git a/test cases/failing/86 dub executable/test.json b/test cases/failing/86 dub executable/test.json
new file mode 100644
index 0000000..f9944af
--- /dev/null
+++ b/test cases/failing/86 dub executable/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/86 dub executable/meson.build:11:0: ERROR: Dependency \"dubtestproject:test1\" not found"
+    }
+  ]
+}
diff --git a/test cases/failing/87 dub compiler/test.json b/test cases/failing/87 dub compiler/test.json
index acb7da8..f28312f 100644
--- a/test cases/failing/87 dub compiler/test.json
+++ b/test cases/failing/87 dub compiler/test.json
@@ -2,8 +2,18 @@
   "matrix": {
     "options": {
       "warning_level": [
-        { "val": "1", "skip_on_env": [ "SINGLE_DUB_COMPILER" ] }
+        {
+          "val": "1",
+          "skip_on_env": [
+            "SINGLE_DUB_COMPILER"
+          ]
+        }
       ]
     }
-  }
+  },
+  "stdout": [
+    {
+      "line": "test cases/failing/87 dub compiler/meson.build:17:0: ERROR: Dependency \"dubtestproject:test2\" not found"
+    }
+  ]
 }
diff --git a/test cases/failing/88 subproj not-found dep/test.json b/test cases/failing/88 subproj not-found dep/test.json
new file mode 100644
index 0000000..a1c4231
--- /dev/null
+++ b/test cases/failing/88 subproj not-found dep/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/88 subproj not-found dep/meson.build:2:0: ERROR: Could not find dependency notfound_dep in subproject somesubproj"
+    }
+  ]
+}
diff --git a/test cases/failing/89 invalid configure file/test.json b/test cases/failing/89 invalid configure file/test.json
new file mode 100644
index 0000000..921ce61
--- /dev/null
+++ b/test cases/failing/89 invalid configure file/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/89 invalid configure file/meson.build:3:0: ERROR: \"install_dir\" must be specified when \"install\" in a configure_file is true"
+    }
+  ]
+}
diff --git a/test cases/failing/9 missing extra file/test.json b/test cases/failing/9 missing extra file/test.json
new file mode 100644
index 0000000..188b6a6
--- /dev/null
+++ b/test cases/failing/9 missing extra file/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/9 missing extra file/meson.build:3:0: ERROR: File missing.txt does not exist."
+    }
+  ]
+}
diff --git a/test cases/failing/90 kwarg dupe/test.json b/test cases/failing/90 kwarg dupe/test.json
new file mode 100644
index 0000000..a8df75d
--- /dev/null
+++ b/test cases/failing/90 kwarg dupe/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/90 kwarg dupe/meson.build:5:0: ERROR: Entry \"install\" defined both as a keyword argument and in a \"kwarg\" entry."
+    }
+  ]
+}
diff --git a/test cases/failing/91 missing pch file/test.json b/test cases/failing/91 missing pch file/test.json
new file mode 100644
index 0000000..166f627
--- /dev/null
+++ b/test cases/failing/91 missing pch file/test.json
@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "comment": "literal 'pch/prog.h' from meson.build appears in output, irrespective of os.path.sep",
+      "line": "test cases/failing/91 missing pch file/meson.build:2:0: ERROR: File pch/prog.h does not exist."
+    }
+  ]
+}
diff --git a/test cases/failing/92 pch source different folder/test.json b/test cases/failing/92 pch source different folder/test.json
new file mode 100644
index 0000000..d94db50
--- /dev/null
+++ b/test cases/failing/92 pch source different folder/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/92 pch source different folder/meson.build:4:0: ERROR: PCH files must be stored in the same folder."
+    }
+  ]
+}
diff --git a/test cases/failing/93 vala without c/test.json b/test cases/failing/93 vala without c/test.json
new file mode 100644
index 0000000..6185b7e
--- /dev/null
+++ b/test cases/failing/93 vala without c/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/93 vala without c/meson.build:2:0: ERROR: Compiling Vala requires C. Add C to your project languages and rerun Meson."
+    }
+  ]
+}
diff --git a/test cases/failing/94 unknown config tool/test.json b/test cases/failing/94 unknown config tool/test.json
new file mode 100644
index 0000000..a001152
--- /dev/null
+++ b/test cases/failing/94 unknown config tool/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/94 unknown config tool/meson.build:2:0: ERROR: Dependency \"no-such-config-tool\" not found"
+    }
+  ]
+}
diff --git a/test cases/failing/95 custom target install data/test.json b/test cases/failing/95 custom target install data/test.json
new file mode 100644
index 0000000..64ef530
--- /dev/null
+++ b/test cases/failing/95 custom target install data/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/95 custom target install data/meson.build:11:0: ERROR: Argument must be string or file."
+    }
+  ]
+}
diff --git a/test cases/failing/96 add dict non string key/test.json b/test cases/failing/96 add dict non string key/test.json
new file mode 100644
index 0000000..5fd4033
--- /dev/null
+++ b/test cases/failing/96 add dict non string key/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/96 add dict non string key/meson.build:9:0: ERROR: Key must be a string"
+    }
+  ]
+}
diff --git a/test cases/failing/97 add dict duplicate keys/test.json b/test cases/failing/97 add dict duplicate keys/test.json
new file mode 100644
index 0000000..9d01551
--- /dev/null
+++ b/test cases/failing/97 add dict duplicate keys/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/97 add dict duplicate keys/meson.build:9:0: ERROR: Duplicate dictionary key: myKey"
+    }
+  ]
+}
diff --git a/test cases/failing/99 no native prop/test.json b/test cases/failing/99 no native prop/test.json
new file mode 100644
index 0000000..8c320d9
--- /dev/null
+++ b/test cases/failing/99 no native prop/test.json
@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/99 no native prop/meson.build:3:0: ERROR: Unknown native property: nonexisting."
+    }
+  ]
+}