From 893c2465502b1e0ccd2c88d08d163bf0b7d39ad7 Mon Sep 17 00:00:00 2001
From: Daniel Mensinger <daniel@mensinger-ka.de>
Date: Fri, 7 Feb 2020 22:25:42 +0100
Subject: boost: Rewrite boost_names.py generator
---
tools/boost_names.py | 371 +++++++++++++++++++++++++++++++--------------------
1 file changed, 228 insertions(+), 143 deletions(-)
(limited to 'tools')
diff --git a/tools/boost_names.py b/tools/boost_names.py
index af461d8..d26d34b 100755
--- a/tools/boost_names.py
+++ b/tools/boost_names.py
@@ -24,164 +24,249 @@ boost/$ path/to/meson/tools/boost_names.py >> path/to/meson/dependencies/misc.py
"""
import sys
-import os
-import collections
-import pprint
import json
import re
+import textwrap
+import functools
+import typing as T
+from pathlib import Path
+
+lib_dir = Path('libs')
+jamroot = Path('Jamroot')
+
+not_modules = ['config', 'disjoint_sets', 'headers']
+
+export_modules = False
+
+
+@functools.total_ordering
+class BoostLibrary():
+ def __init__(self, name: str, shared: T.List[str], static: T.List[str], single: T.List[str], multi: T.List[str]):
+ self.name = name
+ self.shared = shared
+ self.static = static
+ self.single = single
+ self.multi = multi
+
+ def __lt__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']:
+ if isinstance(other, BoostLibrary):
+ return self.name < other.name
+ return NotImplemented
+
+ def __eq__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']:
+ if isinstance(other, BoostLibrary):
+ return self.name == other.name
+ elif isinstance(other, str):
+ return self.name == other
+ return NotImplemented
+
+ def __hash__(self) -> int:
+ return hash(self.name)
+
+@functools.total_ordering
+class BoostModule():
+ def __init__(self, name: str, key: str, desc: str, libs: T.List[BoostLibrary]):
+ self.name = name
+ self.key = key
+ self.desc = desc
+ self.libs = libs
+
+ def __lt__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']:
+ if isinstance(other, BoostModule):
+ return self.key < other.key
+ return NotImplemented
-Module = collections.namedtuple('Module', ['dirname', 'name', 'libnames'])
-Module.__repr__ = lambda self: str((self.dirname, self.name, self.libnames)) # type: ignore
-
-LIBS = 'libs'
-
-manual_map = {
- 'callable_traits': 'Call Traits',
- 'crc': 'CRC',
- 'dll': 'DLL',
- 'gil': 'GIL',
- 'graph_parallel': 'GraphParallel',
- 'icl': 'ICL',
- 'io': 'IO State Savers',
- 'msm': 'Meta State Machine',
- 'mpi': 'MPI',
- 'mpl': 'MPL',
- 'multi_array': 'Multi-Array',
- 'multi_index': 'Multi-Index',
- 'numeric': 'Numeric Conversion',
- 'ptr_container': 'Pointer Container',
- 'poly_collection': 'PolyCollection',
- 'qvm': 'QVM',
- 'throw_exception': 'ThrowException',
- 'tti': 'TTI',
- 'vmd': 'VMD',
-}
-
-extra = [
- Module('utility', 'Compressed Pair', []),
- Module('core', 'Enable If', []),
- Module('functional', 'Functional/Factory', []),
- Module('functional', 'Functional/Forward', []),
- Module('functional', 'Functional/Hash', []),
- Module('functional', 'Functional/Overloaded Function', []),
- Module('utility', 'Identity Type', []),
- Module('utility', 'In Place Factory, Typed In Place Factory', []),
- Module('numeric', 'Interval', []),
- Module('math', 'Math Common Factor', []),
- Module('math', 'Math Octonion', []),
- Module('math', 'Math Quaternion', []),
- Module('math', 'Math/Special Functions', []),
- Module('math', 'Math/Statistical Distributions', []),
- Module('bind', 'Member Function', []),
- Module('algorithm', 'Min-Max', []),
- Module('numeric', 'Odeint', []),
- Module('utility', 'Operators', []),
- Module('core', 'Ref', []),
- Module('utility', 'Result Of', []),
- Module('algorithm', 'String Algo', []),
- Module('core', 'Swap', []),
- Module('', 'Tribool', []),
- Module('numeric', 'uBLAS', []),
- Module('utility', 'Value Initialized', []),
-]
-
-# Cannot find the following modules in the documentation of boost
-not_modules = ['beast', 'logic', 'mp11', 'winapi']
-
-def eprint(message):
- print(message, file=sys.stderr)
-
-def get_library_names(jamfile):
- libs = []
- with open(jamfile) as jamfh:
- jam = jamfh.read()
- res = re.finditer(r'^lib[\s]+([A-Za-z0-9_]+)([^;]*);', jam, re.MULTILINE | re.DOTALL)
- for matches in res:
- if ':' in matches.group(2):
- libs.append(matches.group(1))
- res = re.finditer(r'^boost-lib[\s]+([A-Za-z0-9_]+)([^;]*);', jam, re.MULTILINE | re.DOTALL)
- for matches in res:
- if ':' in matches.group(2):
- libs.append('boost_{}'.format(matches.group(1)))
- return libs
-def exists(modules, module):
- return len([x for x in modules if x.dirname == module.dirname]) != 0
+def get_boost_version() -> T.Optional[str]:
+ raw = jamroot.read_text()
+ m = re.search(r'BOOST_VERSION\s*:\s*([0-9\.]+)\s*;', raw)
+ if m:
+ return m.group(1)
+ return None
-def get_modules(init=extra):
- modules = init
- for directory in os.listdir(LIBS):
- if not os.path.isdir(os.path.join(LIBS, directory)):
+
+def get_libraries(jamfile: Path) -> T.List[BoostLibrary]:
+ # Extract libraries from the boost Jamfiles. This includes:
+ # - library name
+ # - compiler flags
+
+ libs: T.List[BoostLibrary] = []
+ raw = jamfile.read_text()
+ raw = re.sub(r'#.*\n', '\n', raw) # Remove comments
+ raw = re.sub(r'\s+', ' ', raw) # Force single space
+ raw = re.sub(r'}', ';', raw) # Cheat code blocks by converting } to ;
+
+ cmds = raw.split(';') # Commands always terminate with a ; (I hope)
+ cmds = [x.strip() for x in cmds] # Some cleanup
+
+ # "Parse" the relevant sections
+ for i in cmds:
+ parts = i.split(' ')
+ parts = [x for x in parts if x not in ['', ':']]
+ if not parts:
continue
- if directory in not_modules:
+
+ # Parese libraries
+ if parts[0] in ['lib', 'boost-lib']:
+ assert len(parts) >= 2
+
+ # Get and check the library name
+ lname = parts[1]
+ if parts[0] == 'boost-lib':
+ lname = f'boost_{lname}'
+ if not lname.startswith('boost_'):
+ continue
+
+ # Get shared / static defines
+ shared: T.List[str] = []
+ static: T.List[str] = []
+ single: T.List[str] = []
+ multi: T.List[str] = []
+ for j in parts:
+ m1 = re.match(r'<link>shared:<define>(.*)', j)
+ m2 = re.match(r'<link>static:<define>(.*)', j)
+ m3 = re.match(r'<threading>single:<define>(.*)', j)
+ m4 = re.match(r'<threading>multi:<define>(.*)', j)
+
+ if m1:
+ shared += [m1.group(1)]
+ if m2:
+ static += [m2.group(1)]
+ if m3:
+ single += [m3.group(1)]
+ if m4:
+ multi += [m4.group(1)]
+
+ shared = [f'-D{x}' for x in shared]
+ static = [f'-D{x}' for x in static]
+ libs += [BoostLibrary(lname, shared, static, single, multi)]
+
+ return libs
+
+
+def process_lib_dir(ldir: Path) -> T.List[BoostModule]:
+ meta_file = ldir / 'meta' / 'libraries.json'
+ bjam_file = ldir / 'build' / 'Jamfile.v2'
+ if not meta_file.exists():
+ print(f'WARNING: Meta file {meta_file} does not exist')
+ return []
+
+ # Extract libs
+ libs: T.List[BoostLibrary] = []
+ if bjam_file.exists():
+ libs = get_libraries(bjam_file)
+
+ # Extract metadata
+ data = json.loads(meta_file.read_text())
+ if not isinstance(data, list):
+ data = [data]
+
+ modules: T.List[BoostModule] = []
+ for i in data:
+ modules += [BoostModule(i['name'], i['key'], i['description'], libs)]
+
+ return modules
+
+
+def get_modules() -> T.List[BoostModule]:
+ modules: T.List[BoostModule] = []
+ for i in lib_dir.iterdir():
+ if not i.is_dir() or i.name in not_modules:
continue
- jamfile = os.path.join(LIBS, directory, 'build', 'Jamfile.v2')
- if os.path.isfile(jamfile):
- libs = get_library_names(jamfile)
- else:
- libs = []
- if directory in manual_map.keys():
- modname = manual_map[directory]
+
+ # numeric has sub libs
+ subdirs = i / 'sublibs'
+ metadir = i / 'meta'
+ if subdirs.exists() and not metadir.exists():
+ for j in i.iterdir():
+ if not j.is_dir():
+ continue
+ modules += process_lib_dir(j)
else:
- modname = directory.replace('_', ' ').title()
- modules.append(Module(directory, modname, libs))
+ modules += process_lib_dir(i)
+
return modules
-def get_modules_2():
- modules = []
- # The python module uses an older build system format and is not easily parseable.
- # We add the python module libraries manually.
- modules.append(Module('python', 'Python', ['boost_python', 'boost_python3', 'boost_numpy', 'boost_numpy3']))
- for (root, _, files) in os.walk(LIBS):
- for f in files:
- if f == "libraries.json":
- projectdir = os.path.dirname(root)
-
- jamfile = os.path.join(projectdir, 'build', 'Jamfile.v2')
- if os.path.isfile(jamfile):
- libs = get_library_names(jamfile)
- else:
- libs = []
-
- # Get metadata for module
- jsonfile = os.path.join(root, f)
- with open(jsonfile) as jsonfh:
- boost_modules = json.loads(jsonfh.read())
- if(isinstance(boost_modules, dict)):
- boost_modules = [boost_modules]
- for boost_module in boost_modules:
- modules.append(Module(boost_module['key'], boost_module['name'], libs))
-
- # Some subprojects do not have meta directory with json file. Find those
- jsonless_modules = [x for x in get_modules([]) if not exists(modules, x)]
- for module in jsonless_modules:
- eprint("WARNING: {} does not have meta/libraries.json. Will guess pretty name '{}'".format(module.dirname, module.name))
- modules.extend(jsonless_modules)
- return modules
+def main() -> int:
+ if not lib_dir.is_dir() or not jamroot.exists():
+ print("ERROR: script must be run in boost source directory")
+ return 1
+
+ vers = get_boost_version()
+ modules = get_modules()
+ modules = sorted(modules)
+ libraries = [x for y in modules for x in y.libs]
+ libraries = sorted(set(libraries))
+
+ print(textwrap.dedent(f'''\
+ #### ---- BEGIN GENERATED ---- ####
+ # #
+ # Generated with tools/boost_names.py:
+ # - boost version: {vers}
+ # - modules found: {len(modules)}
+ # - libraries found: {len(libraries)}
+ #
+
+ class BoostLibrary():
+ def __init__(self, name: str, shared: T.List[str], static: T.List[str], single: T.List[str], multi: T.List[str]):
+ self.name = name
+ self.shared = shared
+ self.static = static
+ self.single = single
+ self.multi = multi
+
+ class BoostModule():
+ def __init__(self, name: str, key: str, desc: str, libs: T.List[str]):
+ self.name = name
+ self.key = key
+ self.desc = desc
+ self.libs = libs
+
+
+ # dict of all know libraries with additional compile options
+ boost_libraries = {{\
+ '''))
+
+ for i in libraries:
+ print(textwrap.indent(textwrap.dedent(f"""\
+ '{i.name}': BoostLibrary(
+ name='{i.name}',
+ shared={i.shared},
+ static={i.static},
+ single={i.single},
+ multi={i.multi},
+ ),\
+ """), ' '))
+
+ if export_modules:
+ print(textwrap.dedent(f'''\
+ }}
+
-def main(args):
- if not os.path.isdir(LIBS):
- eprint("ERROR: script must be run in boost source directory")
+ # dict of all modules with metadata
+ boost_modules = {{\
+ '''))
- # It will pick jsonless algorithm if 1 is given as argument
- impl = 0
- if len(args) > 1:
- if args[1] == '1':
- impl = 1
+ for mod in modules:
+ desc_excaped = re.sub(r"'", "\\'", mod.desc)
+ print(textwrap.indent(textwrap.dedent(f"""\
+ '{mod.key}': BoostModule(
+ name='{mod.name}',
+ key='{mod.key}',
+ desc='{desc_excaped}',
+ libs={[x.name for x in mod.libs]},
+ ),\
+ """), ' '))
- if impl == 1:
- modules = get_modules()
- else:
- modules = get_modules_2()
+ print(textwrap.dedent(f'''\
+ }}
- sorted_modules = sorted(modules, key=lambda module: module.name.lower())
- sorted_modules = [x[2] for x in sorted_modules if x[2]]
- sorted_modules = sum(sorted_modules, [])
- sorted_modules = [x for x in sorted_modules if x.startswith('boost')]
+ # #
+ #### ---- END GENERATED ---- ####\
+ '''))
- pp = pprint.PrettyPrinter()
- pp.pprint(sorted_modules)
+ return 0
if __name__ == '__main__':
- main(sys.argv)
+ sys.exit(main())
--
cgit v1.1