diff options
| -rw-r--r-- | data/test.schema.json | 2 | ||||
| -rw-r--r-- | docs/markdown/Builtin-options.md | 17 | ||||
| -rw-r--r-- | docs/markdown/Python-module.md | 5 | ||||
| -rw-r--r-- | docs/markdown/snippets/python_extension_module_limited_api.md | 5 | ||||
| -rw-r--r-- | mesonbuild/coredata.py | 2 | ||||
| -rw-r--r-- | mesonbuild/dependencies/python.py | 12 | ||||
| -rw-r--r-- | mesonbuild/modules/python.py | 83 | ||||
| -rwxr-xr-x | mesonbuild/scripts/python_info.py | 15 | ||||
| -rwxr-xr-x | run_project_tests.py | 14 | ||||
| -rw-r--r-- | test cases/python/10 extmodule limited api disabled/meson.build | 10 | ||||
| -rw-r--r-- | test cases/python/10 extmodule limited api disabled/module.c | 17 | ||||
| -rw-r--r-- | test cases/python/9 extmodule limited api/limited.c | 19 | ||||
| -rw-r--r-- | test cases/python/9 extmodule limited api/meson.build | 16 | ||||
| -rw-r--r-- | test cases/python/9 extmodule limited api/not_limited.c | 59 | ||||
| -rw-r--r-- | test cases/python/9 extmodule limited api/test.json | 8 |
15 files changed, 264 insertions, 20 deletions
diff --git a/data/test.schema.json b/data/test.schema.json index a809388..98ae44e 100644 --- a/data/test.schema.json +++ b/data/test.schema.json @@ -26,9 +26,11 @@ "exe", "shared_lib", "python_lib", + "python_limited_lib", "pdb", "implib", "py_implib", + "py_limited_implib", "implibempty", "expr" ] diff --git a/docs/markdown/Builtin-options.md b/docs/markdown/Builtin-options.md index fed893e..ca4fd14 100644 --- a/docs/markdown/Builtin-options.md +++ b/docs/markdown/Builtin-options.md @@ -370,12 +370,13 @@ install prefix. For example: if the install prefix is `/usr` and the ### Python module -| Option | Default value | Possible values | Description | -| ------ | ------------- | ----------------- | ----------- | -| bytecompile | 0 | integer from -1 to 2 | What bytecode optimization level to use (Since 1.2.0) | -| install_env | prefix | {auto,prefix,system,venv} | Which python environment to install to (Since 0.62.0) | -| platlibdir | | Directory path | Directory for site-specific, platform-specific files (Since 0.60.0) | -| purelibdir | | Directory path | Directory for site-specific, non-platform-specific files (Since 0.60.0) | +| Option | Default value | Possible values | Description | +| ------ | ------------- | ----------------- | ----------- | +| bytecompile | 0 | integer from -1 to 2 | What bytecode optimization level to use (Since 1.2.0) | +| install_env | prefix | {auto,prefix,system,venv} | Which python environment to install to (Since 0.62.0) | +| platlibdir | | Directory path | Directory for site-specific, platform-specific files (Since 0.60.0) | +| purelibdir | | Directory path | Directory for site-specific, non-platform-specific files (Since 0.60.0) | +| allow_limited_api | true | true, false | Disables project-wide use of the Python Limited API (Since 1.3.0) | *Since 0.60.0* The `python.platlibdir` and `python.purelibdir` options are used by the python module methods `python.install_sources()` and @@ -405,3 +406,7 @@ python bytecode. Bytecode has 3 optimization levels: To this, Meson adds level `-1`, which is to not attempt to compile bytecode at all. + +*Since 1.3.0* The `python.allow_limited_api` option affects whether the +`limited_api` keyword argument of the `extension_module` method is respected. +If set to `false`, the effect of the `limited_api` argument is disabled. diff --git a/docs/markdown/Python-module.md b/docs/markdown/Python-module.md index f67262a..05ae57d 100644 --- a/docs/markdown/Python-module.md +++ b/docs/markdown/Python-module.md @@ -101,6 +101,11 @@ the addition of the following: `/usr/lib/site-packages`. When subdir is passed to this method, it will be appended to that location. This keyword argument is mutually exclusive with `install_dir` +- `limited_api`: *since 1.3.0* A string containing the Python version + of the [Py_LIMITED_API](https://docs.python.org/3/c-api/stable.html) that + the extension targets. For example, '3.7' to target Python 3.7's version of + the limited API. This behavior can be disabled by setting the value of + `python.allow_limited_api`. See [Python module options](Builtin-options.md#python-module). Additionally, the following diverge from [[shared_module]]'s default behavior: diff --git a/docs/markdown/snippets/python_extension_module_limited_api.md b/docs/markdown/snippets/python_extension_module_limited_api.md new file mode 100644 index 0000000..f5da969 --- /dev/null +++ b/docs/markdown/snippets/python_extension_module_limited_api.md @@ -0,0 +1,5 @@ +## Support targeting Python's limited C API + +The Python module's `extension_module` function has gained the ability +to build extensions which target Python's limited C API via a new keyword +argument: `limited_api`. diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index e930dff..7bbc09e 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -1300,6 +1300,8 @@ BUILTIN_CORE_OPTIONS: 'MutableKeyedOptionDictType' = OrderedDict([ BuiltinOption(UserStringOption, 'Directory for site-specific, platform-specific files.', '')), (OptionKey('purelibdir', module='python'), BuiltinOption(UserStringOption, 'Directory for site-specific, non-platform-specific files.', '')), + (OptionKey('allow_limited_api', module='python'), + BuiltinOption(UserBooleanOption, 'Whether to allow use of the Python Limited API', True)), ]) BUILTIN_OPTIONS = OrderedDict(chain(BUILTIN_DIR_OPTIONS.items(), BUILTIN_CORE_OPTIONS.items())) diff --git a/mesonbuild/dependencies/python.py b/mesonbuild/dependencies/python.py index 1607728..efb904e 100644 --- a/mesonbuild/dependencies/python.py +++ b/mesonbuild/dependencies/python.py @@ -44,6 +44,7 @@ if T.TYPE_CHECKING: paths: T.Dict[str, str] platform: str suffix: str + limited_api_suffix: str variables: T.Dict[str, str] version: str @@ -94,6 +95,7 @@ class BasicPythonExternalProgram(ExternalProgram): 'paths': {}, 'platform': 'sentinel', 'suffix': 'sentinel', + 'limited_api_suffix': 'sentinel', 'variables': {}, 'version': '0.0', } @@ -197,7 +199,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase): if self.link_libpython: # link args if mesonlib.is_windows(): - self.find_libpy_windows(environment) + self.find_libpy_windows(environment, limited_api=False) else: self.find_libpy(environment) else: @@ -259,7 +261,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase): mlog.log(f'Unknown Windows Python platform {self.platform!r}') return None - def get_windows_link_args(self) -> T.Optional[T.List[str]]: + def get_windows_link_args(self, limited_api: bool) -> T.Optional[T.List[str]]: if self.platform.startswith('win'): vernum = self.variables.get('py_version_nodot') verdot = self.variables.get('py_version_short') @@ -277,6 +279,8 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase): else: libpath = Path(f'python{vernum}.dll') else: + if limited_api: + vernum = vernum[0] libpath = Path('libs') / f'python{vernum}.lib' # For a debug build, pyconfig.h may force linking with # pythonX_d.lib (see meson#10776). This cannot be avoided @@ -317,7 +321,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase): return None return [str(lib)] - def find_libpy_windows(self, env: 'Environment') -> None: + def find_libpy_windows(self, env: 'Environment', limited_api: bool = False) -> None: ''' Find python3 libraries on Windows and also verify that the arch matches what we are building for. @@ -332,7 +336,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase): self.is_found = False return # This can fail if the library is not found - largs = self.get_windows_link_args() + largs = self.get_windows_link_args(limited_api) if largs is None: self.is_found = False return diff --git a/mesonbuild/modules/python.py b/mesonbuild/modules/python.py index d0067db..c8af224 100644 --- a/mesonbuild/modules/python.py +++ b/mesonbuild/modules/python.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import annotations -import copy, json, os, shutil +import copy, json, os, shutil, re import typing as T from . import ExtensionModule, ModuleInfo @@ -32,7 +32,7 @@ from ..interpreterbase import ( InvalidArguments, typed_pos_args, typed_kwargs, KwargInfo, FeatureNew, FeatureNewKwargs, disablerIfNotFound ) -from ..mesonlib import MachineChoice +from ..mesonlib import MachineChoice, OptionKey from ..programs import ExternalProgram, NonExistingExternalProgram if T.TYPE_CHECKING: @@ -65,7 +65,7 @@ if T.TYPE_CHECKING: MaybePythonProg = T.Union[NonExistingExternalProgram, 'PythonExternalProgram'] -mod_kwargs = {'subdir'} +mod_kwargs = {'subdir', 'limited_api'} mod_kwargs.update(known_shmod_kwargs) mod_kwargs -= {'name_prefix', 'name_suffix'} @@ -114,6 +114,7 @@ class PythonExternalProgram(BasicPythonExternalProgram): _PURE_KW = KwargInfo('pure', (bool, NoneType)) _SUBDIR_KW = KwargInfo('subdir', str, default='') +_LIMITED_API_KW = KwargInfo('limited_api', str, default='', since='1.3.0') _DEFAULTABLE_SUBDIR_KW = KwargInfo('subdir', (str, NoneType)) class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): @@ -124,6 +125,7 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): assert isinstance(prefix, str), 'for mypy' self.variables = info['variables'] self.suffix = info['suffix'] + self.limited_api_suffix = info['limited_api_suffix'] self.paths = info['paths'] self.pure = python.pure self.platlib_install_path = os.path.join(prefix, python.platlib) @@ -148,7 +150,7 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): @permittedKwargs(mod_kwargs) @typed_pos_args('python.extension_module', str, varargs=(str, mesonlib.File, CustomTarget, CustomTargetIndex, GeneratedList, StructuredSources, ExtractedObjects, BuildTarget)) - @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, allow_unknown=True) + @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, _LIMITED_API_KW, allow_unknown=True) def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], kwargs: ExtensionModuleKw) -> 'SharedModule': if 'install_dir' in kwargs: if kwargs['subdir'] is not None: @@ -161,9 +163,11 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): kwargs['install_dir'] = self._get_install_dir_impl(False, subdir) + target_suffix = self.suffix + new_deps = mesonlib.extract_as_list(kwargs, 'dependencies') - has_pydep = any(isinstance(dep, _PythonDependencyBase) for dep in new_deps) - if not has_pydep: + pydep = next((dep for dep in new_deps if isinstance(dep, _PythonDependencyBase)), None) + if pydep is None: pydep = self._dependency_method_impl({}) if not pydep.found(): raise mesonlib.MesonException('Python dependency not found') @@ -171,15 +175,62 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): FeatureNew.single_use('python_installation.extension_module with implicit dependency on python', '0.63.0', self.subproject, 'use python_installation.dependency()', self.current_node) + + limited_api_version = kwargs.pop('limited_api') + allow_limited_api = self.interpreter.environment.coredata.get_option(OptionKey('allow_limited_api', module='python')) + if limited_api_version != '' and allow_limited_api: + + target_suffix = self.limited_api_suffix + + limited_api_version_hex = self._convert_api_version_to_py_version_hex(limited_api_version, pydep.version) + limited_api_definition = f'-DPy_LIMITED_API={limited_api_version_hex}' + + new_c_args = mesonlib.extract_as_list(kwargs, 'c_args') + new_c_args.append(limited_api_definition) + kwargs['c_args'] = new_c_args + + new_cpp_args = mesonlib.extract_as_list(kwargs, 'cpp_args') + new_cpp_args.append(limited_api_definition) + kwargs['cpp_args'] = new_cpp_args + + # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib + # into the linker path when not running in debug mode via a series #pragma comment(lib, "") + # directives. We manually override these here as this interferes with the intended + # use of the 'limited_api' kwarg + for_machine = self.interpreter.machine_from_native_kwarg(kwargs) + compilers = self.interpreter.environment.coredata.compilers[for_machine] + if any(compiler.get_id() == 'msvc' for compiler in compilers.values()): + pydep_copy = copy.copy(pydep) + pydep_copy.find_libpy_windows(self.env, limited_api=True) + if not pydep_copy.found(): + raise mesonlib.MesonException('Python dependency supporting limited API not found') + + new_deps.remove(pydep) + new_deps.append(pydep_copy) + + pyver = pydep.version.replace('.', '') + python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib' + python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib' + + new_link_args = mesonlib.extract_as_list(kwargs, 'link_args') + + is_debug = self.interpreter.environment.coredata.options[OptionKey('debug')].value + if is_debug: + new_link_args.append(python_windows_debug_link_exception) + else: + new_link_args.append(python_windows_release_link_exception) + + kwargs['link_args'] = new_link_args + kwargs['dependencies'] = new_deps # msys2's python3 has "-cpython-36m.dll", we have to be clever # FIXME: explain what the specific cleverness is here - split, suffix = self.suffix.rsplit('.', 1) + split, target_suffix = target_suffix.rsplit('.', 1) args = (args[0] + split, args[1]) kwargs['name_prefix'] = '' - kwargs['name_suffix'] = suffix + kwargs['name_suffix'] = target_suffix if 'gnu_symbol_visibility' not in kwargs and \ (self.is_pypy or mesonlib.version_compare(self.version, '>=3.9')): @@ -187,6 +238,22 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): return self.interpreter.build_target(self.current_node, args, kwargs, SharedModule) + def _convert_api_version_to_py_version_hex(self, api_version: str, detected_version: str) -> str: + python_api_version_format = re.compile(r'[0-9]\.[0-9]{1,2}') + decimal_match = python_api_version_format.fullmatch(api_version) + if not decimal_match: + raise InvalidArguments(f'Python API version invalid: "{api_version}".') + if mesonlib.version_compare(api_version, '<3.2'): + raise InvalidArguments(f'Python Limited API version invalid: {api_version} (must be greater than 3.2)') + if mesonlib.version_compare(api_version, '>' + detected_version): + raise InvalidArguments(f'Python Limited API version too high: {api_version} (detected {detected_version})') + + version_components = api_version.split('.') + major = int(version_components[0]) + minor = int(version_components[1]) + + return '0x{:02x}{:02x}0000'.format(major, minor) + def _dependency_method_impl(self, kwargs: TYPE_kwargs) -> Dependency: for_machine = self.interpreter.machine_from_native_kwarg(kwargs) identifier = get_dep_identifier(self._full_path(), kwargs) diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py index 9c3a079..0f7787c 100755 --- a/mesonbuild/scripts/python_info.py +++ b/mesonbuild/scripts/python_info.py @@ -65,6 +65,20 @@ elif sys.version_info < (3, 8, 7): else: suffix = variables.get('EXT_SUFFIX') +limited_api_suffix = None +if sys.version_info >= (3, 2): + try: + from importlib.machinery import EXTENSION_SUFFIXES + limited_api_suffix = EXTENSION_SUFFIXES[1] + except Exception: + pass + +# pypy supports modules targetting the limited api but +# does not use a special suffix to distinguish them: +# https://doc.pypy.org/en/latest/cpython_differences.html#permitted-abi-tags-in-extensions +if '__pypy__' in sys.builtin_module_names: + limited_api_suffix = suffix + print(json.dumps({ 'variables': variables, 'paths': paths, @@ -76,4 +90,5 @@ print(json.dumps({ 'is_venv': sys.prefix != variables['base_prefix'], 'link_libpython': links_against_libpython(), 'suffix': suffix, + 'limited_api_suffix': limited_api_suffix, })) diff --git a/run_project_tests.py b/run_project_tests.py index facf1e9..27020ca 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -148,7 +148,7 @@ class InstalledFile: canonical_compiler = 'msvc' python_suffix = python.info['suffix'] - + python_limited_suffix = python.info['limited_api_suffix'] has_pdb = False if self.language in {'c', 'cpp'}: has_pdb = canonical_compiler == 'msvc' @@ -167,7 +167,7 @@ class InstalledFile: return None # Handle the different types - if self.typ in {'py_implib', 'python_lib', 'python_file'}: + if self.typ in {'py_implib', 'py_limited_implib', 'python_lib', 'python_limited_lib', 'python_file'}: val = p.as_posix() val = val.replace('@PYTHON_PLATLIB@', python.platlib) val = val.replace('@PYTHON_PURELIB@', python.purelib) @@ -176,6 +176,8 @@ class InstalledFile: return p if self.typ == 'python_lib': return p.with_suffix(python_suffix) + if self.typ == 'python_limited_lib': + return p.with_suffix(python_limited_suffix) if self.typ == 'py_implib': p = p.with_suffix(python_suffix) if env.machines.host.is_windows() and canonical_compiler == 'msvc': @@ -184,6 +186,14 @@ class InstalledFile: return p.with_suffix('.dll.a') else: return None + if self.typ == 'py_limited_implib': + p = p.with_suffix(python_limited_suffix) + if env.machines.host.is_windows() and canonical_compiler == 'msvc': + return p.with_suffix('.lib') + elif env.machines.host.is_windows() or env.machines.host.is_cygwin(): + return p.with_suffix('.dll.a') + else: + return None elif self.typ in {'file', 'dir'}: return p elif self.typ == 'shared_lib': diff --git a/test cases/python/10 extmodule limited api disabled/meson.build b/test cases/python/10 extmodule limited api disabled/meson.build new file mode 100644 index 0000000..42cd618 --- /dev/null +++ b/test cases/python/10 extmodule limited api disabled/meson.build @@ -0,0 +1,10 @@ +project('Python limited api disabled', 'c', + default_options : ['buildtype=release', 'werror=true', 'python.allow_limited_api=false']) + +py_mod = import('python') +py = py_mod.find_installation() + +module = py.extension_module('my_module', + 'module.c', + limited_api: '3.7', +) diff --git a/test cases/python/10 extmodule limited api disabled/module.c b/test cases/python/10 extmodule limited api disabled/module.c new file mode 100644 index 0000000..a5d3a87 --- /dev/null +++ b/test cases/python/10 extmodule limited api disabled/module.c @@ -0,0 +1,17 @@ +#include <Python.h> + +#if defined(Py_LIMITED_API) +#error "Py_LIMITED_API's definition by Meson should have been disabled." +#endif + +static struct PyModuleDef my_module = { + PyModuleDef_HEAD_INIT, + "my_module", + NULL, + -1, + NULL +}; + +PyMODINIT_FUNC PyInit_my_module(void) { + return PyModule_Create(&my_module); +} diff --git a/test cases/python/9 extmodule limited api/limited.c b/test cases/python/9 extmodule limited api/limited.c new file mode 100644 index 0000000..0d1c718 --- /dev/null +++ b/test cases/python/9 extmodule limited api/limited.c @@ -0,0 +1,19 @@ +#include <Python.h> + +#ifndef Py_LIMITED_API +#error Py_LIMITED_API must be defined. +#elif Py_LIMITED_API != 0x03070000 +#error Wrong value for Py_LIMITED_API +#endif + +static struct PyModuleDef limited_module = { + PyModuleDef_HEAD_INIT, + "limited_api_test", + NULL, + -1, + NULL +}; + +PyMODINIT_FUNC PyInit_limited(void) { + return PyModule_Create(&limited_module); +} diff --git a/test cases/python/9 extmodule limited api/meson.build b/test cases/python/9 extmodule limited api/meson.build new file mode 100644 index 0000000..68afc96 --- /dev/null +++ b/test cases/python/9 extmodule limited api/meson.build @@ -0,0 +1,16 @@ +project('Python limited api', 'c', + default_options : ['buildtype=release', 'werror=true']) + +py_mod = import('python') +py = py_mod.find_installation() + +ext_mod_limited = py.extension_module('limited', + 'limited.c', + limited_api: '3.7', + install: true, +) + +ext_mod = py.extension_module('not_limited', + 'not_limited.c', + install: true, +) diff --git a/test cases/python/9 extmodule limited api/not_limited.c b/test cases/python/9 extmodule limited api/not_limited.c new file mode 100644 index 0000000..105dbb8 --- /dev/null +++ b/test cases/python/9 extmodule limited api/not_limited.c @@ -0,0 +1,59 @@ +#include <Python.h> +#include <stdio.h> + +#ifdef Py_LIMITED_API +#error Py_LIMITED_API must not be defined. +#endif + +/* This function explicitly calls functions whose declaration is elided when + * Py_LIMITED_API is defined. This is to test that the linker is actually + * linking to the right version of the library on Windows. */ +static PyObject *meth_not_limited(PyObject *self, PyObject *args) +{ + PyObject *list; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "o", & list)) + return NULL; + + if (!PyList_Check(list)) { + PyErr_Format(PyExc_TypeError, "expected 'list'"); + return NULL; + } + + /* PyList_GET_SIZE and PyList_GET_ITEM are only available if Py_LIMITED_API + * is not defined. It seems likely that they will remain excluded from the + * limited API as their checked counterparts (PyList_GetSize and + * PyList_GetItem) are made available in that mode instead. */ + size = PyList_GET_SIZE(list); + for(Py_ssize_t i = 0; i < size; ++i) { + PyObject *element = PyList_GET_ITEM(list, i); + if (element == NULL) { + return NULL; + } + + if(PyObject_Print(element, stdout, Py_PRINT_RAW) == -1) { + return NULL; + } + } + + Py_RETURN_NONE; +} + +static struct PyMethodDef not_limited_methods[] = { + { "not_limited", meth_not_limited, METH_VARARGS, + "Calls functions whose declaration is elided by Py_LIMITED_API" }, + { NULL, NULL, 0, NULL } +}; + +static struct PyModuleDef not_limited_module = { + PyModuleDef_HEAD_INIT, + "not_limited_api_test", + NULL, + -1, + not_limited_methods +}; + +PyMODINIT_FUNC PyInit_not_limited(void) { + return PyModule_Create(¬_limited_module); +} diff --git a/test cases/python/9 extmodule limited api/test.json b/test cases/python/9 extmodule limited api/test.json new file mode 100644 index 0000000..06a1706 --- /dev/null +++ b/test cases/python/9 extmodule limited api/test.json @@ -0,0 +1,8 @@ +{ + "installed": [ + {"type": "python_limited_lib", "file": "usr/@PYTHON_PLATLIB@/limited"}, + {"type": "py_limited_implib", "file": "usr/@PYTHON_PLATLIB@/limited"}, + {"type": "python_lib", "file": "usr/@PYTHON_PLATLIB@/not_limited"}, + {"type": "py_implib", "file": "usr/@PYTHON_PLATLIB@/not_limited"} + ] +} |