boost: Rewrite boost_names.py generator

1 files changed, 228 insertions, 143 deletions

diff --git a/tools/boost_names.py b/tools/boost_names.py
index af461d8..d26d34b 100755
--- a/tools/boost_names.py
+++ b/tools/boost_names.py
@@ -24,164 +24,249 @@ boost/$ path/to/meson/tools/boost_names.py >> path/to/meson/dependencies/misc.py
 """
 
 import sys
-import os
-import collections
-import pprint
 import json
 import re
+import textwrap
+import functools
+import typing as T
+from pathlib import Path
+
+lib_dir = Path('libs')
+jamroot = Path('Jamroot')
+
+not_modules = ['config', 'disjoint_sets', 'headers']
+
+export_modules = False
+
+
+@functools.total_ordering
+class BoostLibrary():
+    def __init__(self, name: str, shared: T.List[str], static: T.List[str], single: T.List[str], multi: T.List[str]):
+        self.name = name
+        self.shared = shared
+        self.static = static
+        self.single = single
+        self.multi = multi
+
+    def __lt__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']:
+        if isinstance(other, BoostLibrary):
+            return self.name < other.name
+        return NotImplemented
+
+    def __eq__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']:
+        if isinstance(other, BoostLibrary):
+            return self.name == other.name
+        elif isinstance(other, str):
+            return self.name == other
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+@functools.total_ordering
+class BoostModule():
+    def __init__(self, name: str, key: str, desc: str, libs: T.List[BoostLibrary]):
+        self.name = name
+        self.key = key
+        self.desc = desc
+        self.libs = libs
+
+    def __lt__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']:
+        if isinstance(other, BoostModule):
+            return self.key < other.key
+        return NotImplemented
 
-Module = collections.namedtuple('Module', ['dirname', 'name', 'libnames'])
-Module.__repr__ = lambda self: str((self.dirname, self.name, self.libnames))  # type: ignore
-
-LIBS = 'libs'
-
-manual_map = {
-    'callable_traits': 'Call Traits',
-    'crc': 'CRC',
-    'dll': 'DLL',
-    'gil': 'GIL',
-    'graph_parallel': 'GraphParallel',
-    'icl': 'ICL',
-    'io': 'IO State Savers',
-    'msm': 'Meta State Machine',
-    'mpi': 'MPI',
-    'mpl': 'MPL',
-    'multi_array': 'Multi-Array',
-    'multi_index': 'Multi-Index',
-    'numeric': 'Numeric Conversion',
-    'ptr_container': 'Pointer Container',
-    'poly_collection': 'PolyCollection',
-    'qvm': 'QVM',
-    'throw_exception': 'ThrowException',
-    'tti': 'TTI',
-    'vmd': 'VMD',
-}
-
-extra = [
-    Module('utility', 'Compressed Pair', []),
-    Module('core', 'Enable If', []),
-    Module('functional', 'Functional/Factory', []),
-    Module('functional', 'Functional/Forward', []),
-    Module('functional', 'Functional/Hash', []),
-    Module('functional', 'Functional/Overloaded Function', []),
-    Module('utility', 'Identity Type', []),
-    Module('utility', 'In Place Factory, Typed In Place Factory', []),
-    Module('numeric', 'Interval', []),
-    Module('math', 'Math Common Factor', []),
-    Module('math', 'Math Octonion', []),
-    Module('math', 'Math Quaternion', []),
-    Module('math', 'Math/Special Functions', []),
-    Module('math', 'Math/Statistical Distributions', []),
-    Module('bind', 'Member Function', []),
-    Module('algorithm', 'Min-Max', []),
-    Module('numeric', 'Odeint', []),
-    Module('utility', 'Operators', []),
-    Module('core', 'Ref', []),
-    Module('utility', 'Result Of', []),
-    Module('algorithm', 'String Algo', []),
-    Module('core', 'Swap', []),
-    Module('', 'Tribool', []),
-    Module('numeric', 'uBLAS', []),
-    Module('utility', 'Value Initialized', []),
-]
-
-# Cannot find the following modules in the documentation of boost
-not_modules = ['beast', 'logic', 'mp11', 'winapi']
-
-def eprint(message):
-    print(message, file=sys.stderr)
-
-def get_library_names(jamfile):
-    libs = []
-    with open(jamfile) as jamfh:
-        jam = jamfh.read()
-        res = re.finditer(r'^lib[\s]+([A-Za-z0-9_]+)([^;]*);', jam, re.MULTILINE | re.DOTALL)
-        for matches in res:
-            if ':' in matches.group(2):
-                libs.append(matches.group(1))
-        res = re.finditer(r'^boost-lib[\s]+([A-Za-z0-9_]+)([^;]*);', jam, re.MULTILINE | re.DOTALL)
-        for matches in res:
-            if ':' in matches.group(2):
-                libs.append('boost_{}'.format(matches.group(1)))
-    return libs
 
-def exists(modules, module):
-    return len([x for x in modules if x.dirname == module.dirname]) != 0
+def get_boost_version() -> T.Optional[str]:
+    raw = jamroot.read_text()
+    m = re.search(r'BOOST_VERSION\s*:\s*([0-9\.]+)\s*;', raw)
+    if m:
+        return m.group(1)
+    return None
 
-def get_modules(init=extra):
-    modules = init
-    for directory in os.listdir(LIBS):
-        if not os.path.isdir(os.path.join(LIBS, directory)):
+
+def get_libraries(jamfile: Path) -> T.List[BoostLibrary]:
+    # Extract libraries from the boost Jamfiles. This includes:
+    #  - library name
+    #  - compiler flags
+
+    libs: T.List[BoostLibrary] = []
+    raw = jamfile.read_text()
+    raw = re.sub(r'#.*\n', '\n', raw)  # Remove comments
+    raw = re.sub(r'\s+', ' ', raw)     # Force single space
+    raw = re.sub(r'}', ';', raw)       # Cheat code blocks by converting } to ;
+
+    cmds = raw.split(';')              # Commands always terminate with a ; (I hope)
+    cmds = [x.strip() for x in cmds]   # Some cleanup
+
+    # "Parse" the relevant sections
+    for i in cmds:
+        parts = i.split(' ')
+        parts = [x for x in parts if x not in ['', ':']]
+        if not parts:
             continue
-        if directory in not_modules:
+
+        # Parese libraries
+        if parts[0] in ['lib', 'boost-lib']:
+            assert len(parts) >= 2
+
+            # Get and check the library name
+            lname = parts[1]
+            if parts[0] == 'boost-lib':
+                lname = f'boost_{lname}'
+            if not lname.startswith('boost_'):
+                continue
+
+            # Get shared / static defines
+            shared: T.List[str] = []
+            static: T.List[str] = []
+            single: T.List[str] = []
+            multi: T.List[str] = []
+            for j in parts:
+                m1 = re.match(r'<link>shared:<define>(.*)', j)
+                m2 = re.match(r'<link>static:<define>(.*)', j)
+                m3 = re.match(r'<threading>single:<define>(.*)', j)
+                m4 = re.match(r'<threading>multi:<define>(.*)', j)
+
+                if m1:
+                    shared += [m1.group(1)]
+                if m2:
+                    static += [m2.group(1)]
+                if m3:
+                    single += [m3.group(1)]
+                if m4:
+                    multi += [m4.group(1)]
+
+            shared = [f'-D{x}' for x in shared]
+            static = [f'-D{x}' for x in static]
+            libs += [BoostLibrary(lname, shared, static, single, multi)]
+
+    return libs
+
+
+def process_lib_dir(ldir: Path) -> T.List[BoostModule]:
+    meta_file = ldir / 'meta' / 'libraries.json'
+    bjam_file = ldir / 'build' / 'Jamfile.v2'
+    if not meta_file.exists():
+        print(f'WARNING: Meta file {meta_file} does not exist')
+        return []
+
+    # Extract libs
+    libs: T.List[BoostLibrary] = []
+    if bjam_file.exists():
+        libs = get_libraries(bjam_file)
+
+    # Extract metadata
+    data = json.loads(meta_file.read_text())
+    if not isinstance(data, list):
+        data = [data]
+
+    modules: T.List[BoostModule] = []
+    for i in data:
+        modules += [BoostModule(i['name'], i['key'], i['description'], libs)]
+
+    return modules
+
+
+def get_modules() -> T.List[BoostModule]:
+    modules: T.List[BoostModule] = []
+    for i in lib_dir.iterdir():
+        if not i.is_dir() or i.name in not_modules:
             continue
-        jamfile = os.path.join(LIBS, directory, 'build', 'Jamfile.v2')
-        if os.path.isfile(jamfile):
-            libs = get_library_names(jamfile)
-        else:
-            libs = []
-        if directory in manual_map.keys():
-            modname = manual_map[directory]
+
+        # numeric has sub libs
+        subdirs = i / 'sublibs'
+        metadir = i / 'meta'
+        if subdirs.exists() and not metadir.exists():
+            for j in i.iterdir():
+                if not j.is_dir():
+                    continue
+                modules += process_lib_dir(j)
         else:
-            modname = directory.replace('_', ' ').title()
-        modules.append(Module(directory, modname, libs))
+            modules += process_lib_dir(i)
+
     return modules
 
-def get_modules_2():
-    modules = []
-    # The python module uses an older build system format and is not easily parseable.
-    # We add the python module libraries manually.
-    modules.append(Module('python', 'Python', ['boost_python', 'boost_python3', 'boost_numpy', 'boost_numpy3']))
-    for (root, _, files) in os.walk(LIBS):
-        for f in files:
-            if f == "libraries.json":
-                projectdir = os.path.dirname(root)
-
-                jamfile = os.path.join(projectdir, 'build', 'Jamfile.v2')
-                if os.path.isfile(jamfile):
-                    libs = get_library_names(jamfile)
-                else:
-                    libs = []
-
-                # Get metadata for module
-                jsonfile = os.path.join(root, f)
-                with open(jsonfile) as jsonfh:
-                    boost_modules = json.loads(jsonfh.read())
-                    if(isinstance(boost_modules, dict)):
-                        boost_modules = [boost_modules]
-                    for boost_module in boost_modules:
-                        modules.append(Module(boost_module['key'], boost_module['name'], libs))
-
-    # Some subprojects do not have meta directory with json file. Find those
-    jsonless_modules = [x for x in get_modules([]) if not exists(modules, x)]
-    for module in jsonless_modules:
-        eprint("WARNING: {} does not have meta/libraries.json. Will guess pretty name '{}'".format(module.dirname, module.name))
-    modules.extend(jsonless_modules)
 
-    return modules
+def main() -> int:
+    if not lib_dir.is_dir() or not jamroot.exists():
+        print("ERROR: script must be run in boost source directory")
+        return 1
+
+    vers = get_boost_version()
+    modules = get_modules()
+    modules = sorted(modules)
+    libraries = [x for y in modules for x in y.libs]
+    libraries = sorted(set(libraries))
+
+    print(textwrap.dedent(f'''\
+        ####      ---- BEGIN GENERATED ----      ####
+        #                                           #
+        # Generated with tools/boost_names.py:
+        #  - boost version:   {vers}
+        #  - modules found:   {len(modules)}
+        #  - libraries found: {len(libraries)}
+        #
+
+        class BoostLibrary():
+            def __init__(self, name: str, shared: T.List[str], static: T.List[str], single: T.List[str], multi: T.List[str]):
+                self.name = name
+                self.shared = shared
+                self.static = static
+                self.single = single
+                self.multi = multi
+
+        class BoostModule():
+            def __init__(self, name: str, key: str, desc: str, libs: T.List[str]):
+                self.name = name
+                self.key = key
+                self.desc = desc
+                self.libs = libs
+
+
+        # dict of all know libraries with additional compile options
+        boost_libraries = {{\
+    '''))
+
+    for i in libraries:
+        print(textwrap.indent(textwrap.dedent(f"""\
+            '{i.name}': BoostLibrary(
+                name='{i.name}',
+                shared={i.shared},
+                static={i.static},
+                single={i.single},
+                multi={i.multi},
+            ),\
+        """), '    '))
+
+    if export_modules:
+        print(textwrap.dedent(f'''\
+            }}
+
 
-def main(args):
-    if not os.path.isdir(LIBS):
-        eprint("ERROR: script must be run in boost source directory")
+            # dict of all modules with metadata
+            boost_modules = {{\
+        '''))
 
-    # It will pick jsonless algorithm if 1 is given as argument
-    impl = 0
-    if len(args) > 1:
-        if args[1] == '1':
-            impl = 1
+        for mod in modules:
+            desc_excaped = re.sub(r"'", "\\'", mod.desc)
+            print(textwrap.indent(textwrap.dedent(f"""\
+                '{mod.key}': BoostModule(
+                    name='{mod.name}',
+                    key='{mod.key}',
+                    desc='{desc_excaped}',
+                    libs={[x.name for x in mod.libs]},
+                ),\
+            """), '    '))
 
-    if impl == 1:
-        modules = get_modules()
-    else:
-        modules = get_modules_2()
+    print(textwrap.dedent(f'''\
+        }}
 
-    sorted_modules = sorted(modules, key=lambda module: module.name.lower())
-    sorted_modules = [x[2] for x in sorted_modules if x[2]]
-    sorted_modules = sum(sorted_modules, [])
-    sorted_modules = [x for x in sorted_modules if x.startswith('boost')]
+        #                                           #
+        ####       ---- END GENERATED ----       ####\
+    '''))
 
-    pp = pprint.PrettyPrinter()
-    pp.pprint(sorted_modules)
+    return 0
 
 if __name__ == '__main__':
-    main(sys.argv)
+    sys.exit(main())