aboutsummaryrefslogtreecommitdiff
path: root/tests/Makefile.include
blob: 9422ddaece563003c3b7834ca5b477bff7fbd25f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
# -*- Mode: makefile -*-

.PHONY: check-help
check-help:
	@echo "Regression testing targets:"
	@echo " $(MAKE) check                  Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
	@echo " $(MAKE) bench                  Run speed tests"
	@echo
	@echo "Individual test suites:"
	@echo " $(MAKE) check-qtest-TARGET     Run qtest tests for given target"
	@echo " $(MAKE) check-qtest            Run qtest tests"
	@echo " $(MAKE) check-unit             Run qobject tests"
	@echo " $(MAKE) check-qapi-schema      Run QAPI schema tests"
	@echo " $(MAKE) check-block            Run block tests"
ifneq ($(filter $(all-check-targets), check-softfloat),)
	@echo " $(MAKE) check-tcg              Run TCG tests"
	@echo " $(MAKE) check-softfloat        Run FPU emulation tests"
endif
	@echo " $(MAKE) check-avocado          Run avocado (integration) tests for currently configured targets"
	@echo
	@echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report"
	@echo " $(MAKE) check-venv             Creates a Python venv for tests"
	@echo " $(MAKE) check-clean            Clean the tests and related data"
	@echo
	@echo "The following are useful for CI builds"
	@echo " $(MAKE) check-build            Build most test binaries"
	@echo " $(MAKE) get-vm-images          Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)"
	@echo
	@echo
	@echo "The variable SPEED can be set to control the gtester speed setting."
	@echo "Default options are -k and (for $(MAKE) V=1) --verbose; they can be"
	@echo "changed with variable GTESTER_OPTIONS."

ifneq ($(wildcard config-host.mak),)
export SRC_PATH

SPEED = quick

# Per guest TCG tests
BUILD_TCG_TARGET_RULES=$(patsubst %,build-tcg-tests-%, $(TCG_TESTS_TARGETS))
CLEAN_TCG_TARGET_RULES=$(patsubst %,clean-tcg-tests-%, $(TCG_TESTS_TARGETS))
DISTCLEAN_TCG_TARGET_RULES=$(patsubst %,distclean-tcg-tests-%, $(TCG_TESTS_TARGETS))
RUN_TCG_TARGET_RULES=$(patsubst %,run-tcg-tests-%, $(TCG_TESTS_TARGETS))

$(foreach TARGET,$(TCG_TESTS_TARGETS), \
        $(eval $(BUILD_DIR)/tests/tcg/config-$(TARGET).mak: config-host.mak))

.PHONY: $(TCG_TESTS_TARGETS:%=build-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=build-tcg-tests-%): build-tcg-tests-%: $(BUILD_DIR)/tests/tcg/config-%.mak
	$(call quiet-command, \
            $(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS), \
        "BUILD","$* guest-tests")

.PHONY: $(TCG_TESTS_TARGETS:%=run-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=run-tcg-tests-%): run-tcg-tests-%: build-tcg-tests-%
	$(call quiet-command, \
           $(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) SPEED=$(SPEED) run, \
        "RUN", "$* guest-tests")

.PHONY: $(TCG_TESTS_TARGETS:%=clean-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=clean-tcg-tests-%): clean-tcg-tests-%:
	$(call quiet-command, \
           $(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) clean, \
        "CLEAN", "$* guest-tests")

.PHONY: $(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%): distclean-tcg-tests-%:
	$(call quiet-command, \
           $(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) distclean, \
        "CLEAN", "$* guest-tests")

.PHONY: build-tcg
build-tcg: $(BUILD_TCG_TARGET_RULES)

.PHONY: check-tcg
.ninja-goals.check-tcg = all $(if $(CONFIG_PLUGIN),test-plugins)
check-tcg: $(RUN_TCG_TARGET_RULES)

.PHONY: clean-tcg
clean-tcg: $(CLEAN_TCG_TARGET_RULES)

.PHONY: distclean-tcg
distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES)

# Python venv for running tests

.PHONY: check-venv check-avocado check-acceptance check-acceptance-deprecated-warning

# Build up our target list from the filtered list of ninja targets
TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets)))

TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv
TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt
TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results
TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3
ifndef AVOCADO_TESTS
	AVOCADO_TESTS=tests/avocado
endif
# Controls the output generated by Avocado when running tests.
# Any number of command separated loggers are accepted.  For more
# information please refer to "avocado --help".
AVOCADO_SHOW=app
ifndef AVOCADO_TAGS
	AVOCADO_CMDLINE_TAGS=$(patsubst %-softmmu,-t arch:%, \
						 $(filter %-softmmu,$(TARGETS)))
else
	AVOCADO_CMDLINE_TAGS=$(addprefix -t , $(AVOCADO_TAGS))
endif

quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
    $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \
    "VENVPIP","$1")

$(TESTS_VENV_DIR): $(TESTS_VENV_REQ)
	$(call quiet-command, $(PYTHON) -m venv $@, VENV, $@)
	$(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
	$(call quiet-venv-pip,install -r $(TESTS_VENV_REQ))
	$(call quiet-command, touch $@)

$(TESTS_RESULTS_DIR):
	$(call quiet-command, mkdir -p $@, \
            MKDIR, $@)

check-venv: $(TESTS_VENV_DIR)

FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS)))
FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS))
FEDORA_31_ARCHES := x86_64 aarch64 ppc64le s390x
FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES))

# download one specific Fedora 31 image
get-vm-image-fedora-31-%: check-venv
	$(call quiet-command, \
             $(TESTS_PYTHON) -m avocado vmimage get \
             --distro=fedora --distro-version=31 --arch=$*, \
	"AVOCADO", "Downloading avocado tests VM image for $*")

# download all vm images, according to defined targets
get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOWNLOAD))

check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images
	$(call quiet-command, \
            $(TESTS_PYTHON) -m avocado \
            --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
            $(if $(AVOCADO_TAGS),, --filter-by-tags-include-empty \
			--filter-by-tags-include-empty-key) \
            $(AVOCADO_CMDLINE_TAGS) \
            $(if $(GITLAB_CI),,--failfast) $(AVOCADO_TESTS), \
            "AVOCADO", "tests/avocado")

check-acceptance-deprecated-warning:
	@echo
	@echo "Note '$(MAKE) check-acceptance' is deprecated, use '$(MAKE) check-avocado' instead."
	@echo

check-acceptance: check-acceptance-deprecated-warning | check-avocado

# Consolidated targets

.PHONY: check check-clean get-vm-images
check:

check-build: run-ninja

check-clean:
	rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR)

clean: check-clean clean-tcg
distclean: distclean-tcg

endif