diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/boost_names.py | 56 | ||||
| -rwxr-xr-x | tools/build_website.py | 51 | ||||
| -rw-r--r-- | tools/copy_files.py | 55 | ||||
| -rwxr-xr-x | tools/dircondenser.py | 4 | ||||
| -rwxr-xr-x | tools/gen_data.py | 139 | ||||
| -rwxr-xr-x | tools/regenerate_docs.py | 150 |
6 files changed, 441 insertions, 14 deletions
diff --git a/tools/boost_names.py b/tools/boost_names.py index d26d34b..b66c6cc 100755 --- a/tools/boost_names.py +++ b/tools/boost_names.py @@ -43,10 +43,10 @@ export_modules = False class BoostLibrary(): def __init__(self, name: str, shared: T.List[str], static: T.List[str], single: T.List[str], multi: T.List[str]): self.name = name - self.shared = shared - self.static = static - self.single = single - self.multi = multi + self.shared = sorted(set(shared)) + self.static = sorted(set(static)) + self.single = sorted(set(single)) + self.multi = sorted(set(multi)) def __lt__(self, other: T.Any) -> T.Union[bool, 'NotImplemented']: if isinstance(other, BoostLibrary): @@ -99,15 +99,35 @@ def get_libraries(jamfile: Path) -> T.List[BoostLibrary]: cmds = raw.split(';') # Commands always terminate with a ; (I hope) cmds = [x.strip() for x in cmds] # Some cleanup + project_usage_requirements: T.List[str] = [] + # "Parse" the relevant sections for i in cmds: parts = i.split(' ') - parts = [x for x in parts if x not in ['', ':']] + parts = [x for x in parts if x not in ['']] if not parts: continue - # Parese libraries - if parts[0] in ['lib', 'boost-lib']: + # Parse project + if parts[0] in ['project']: + attributes: T.Dict[str, T.List[str]] = {} + curr: T.Optional[str] = None + + for j in parts: + if j == ':': + curr = None + elif curr is None: + curr = j + else: + if curr not in attributes: + attributes[curr] = [] + attributes[curr] += [j] + + if 'usage-requirements' in attributes: + project_usage_requirements = attributes['usage-requirements'] + + # Parse libraries + elif parts[0] in ['lib', 'boost-lib']: assert len(parts) >= 2 # Get and check the library name @@ -117,28 +137,36 @@ def get_libraries(jamfile: Path) -> T.List[BoostLibrary]: if not lname.startswith('boost_'): continue + # Count `:` to only select the 'usage-requirements' + # See https://boostorg.github.io/build/manual/master/index.html#bbv2.main-target-rule-syntax + colon_counter = 0 + usage_requirements: T.List[str] = [] + for j in parts: + if j == ':': + colon_counter += 1 + elif colon_counter >= 4: + usage_requirements += [j] + # Get shared / static defines shared: T.List[str] = [] static: T.List[str] = [] single: T.List[str] = [] multi: T.List[str] = [] - for j in parts: + for j in usage_requirements + project_usage_requirements: m1 = re.match(r'<link>shared:<define>(.*)', j) m2 = re.match(r'<link>static:<define>(.*)', j) m3 = re.match(r'<threading>single:<define>(.*)', j) m4 = re.match(r'<threading>multi:<define>(.*)', j) if m1: - shared += [m1.group(1)] + shared += [f'-D{m1.group(1)}'] if m2: - static += [m2.group(1)] + static += [f'-D{m2.group(1)}'] if m3: - single += [m3.group(1)] + single +=[f'-D{m3.group(1)}'] if m4: - multi += [m4.group(1)] + multi += [f'-D{m4.group(1)}'] - shared = [f'-D{x}' for x in shared] - static = [f'-D{x}' for x in static] libs += [BoostLibrary(lname, shared, static, single, multi)] return libs diff --git a/tools/build_website.py b/tools/build_website.py new file mode 100755 index 0000000..5486b69 --- /dev/null +++ b/tools/build_website.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import os, sys, subprocess, shutil + +assert(os.getcwd() == '/home/jpakkane') + +from glob import glob + +def purge(fname): + if not os.path.exists(fname): + return + if os.path.isdir(fname): + shutil.rmtree(fname) + os.unlink(fname) + +def update(): + webdir = 'mesonweb' + repodir = 'mesonwebbuild' + docdir = os.path.join(repodir, 'docs') + builddir = os.path.join(docdir, 'builddir') + htmldir = os.path.join(builddir, 'Meson documentation-doc/html') +# subprocess.check_call(['git', 'pull'], cwd=webdir) + subprocess.check_call(['git', 'fetch', '-a'], cwd=repodir) + subprocess.check_call(['git', 'reset', '--hard', 'origin/master'], + cwd=repodir) + if os.path.isdir(htmldir): + shutil.rmtree(htmldir) + if os.path.isdir(builddir): + shutil.rmtree(builddir) + env = os.environ.copy() + env['PATH'] = env['PATH'] + ':/home/jpakkane/.local/bin' + subprocess.check_call(['../meson.py', '.', 'builddir'], cwd=docdir, env=env) + subprocess.check_call(['ninja'], cwd=builddir) + old_files = glob(os.path.join(webdir, '*')) + for f in old_files: + base = f[len(webdir)+1:] + if base == 'CNAME' or base == 'favicon.png': + continue + subprocess.check_call(['git', 'rm', '-rf', base], cwd=webdir) + assert(os.path.isdir(webdir)) + new_entries = glob(os.path.join(htmldir, '*')) + for e in new_entries: + shutil.move(e, webdir) + subprocess.check_call('git add *', shell=True, cwd=webdir) + subprocess.check_call(['git', 'commit', '-a', '-m', 'Bleep. Bloop. I am a bot.'], + cwd=webdir) + subprocess.check_call(['git', 'push'], cwd=webdir) + shutil.rmtree(builddir) + +if __name__ == '__main__': + update() diff --git a/tools/copy_files.py b/tools/copy_files.py new file mode 100644 index 0000000..39eaa0a --- /dev/null +++ b/tools/copy_files.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + + +# Copyright 2018 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +Copy files +''' + +import argparse +import shutil +import typing as T +from pathlib import Path + +PathLike = T.Union[Path,str] + +def copy_files(files: T.List[str], input_dir: PathLike, output_dir: PathLike) -> None: + if not input_dir: + raise ValueError(f'Input directory value is not set') + if not output_dir: + raise ValueError(f'Output directory value is not set') + + input_dir = Path(input_dir).resolve() + output_dir = Path(output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + for f in files: + if (input_dir/f).is_dir(): + shutil.copytree(input_dir/f, output_dir/f) + else: + shutil.copy2(input_dir/f, output_dir/f) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Copy files') + parser.add_argument('files', metavar='FILE', nargs='*') + parser.add_argument('-C', dest='input_dir', required=True) + parser.add_argument('--output-dir', required=True) + + args = parser.parse_args() + + copy_files(files=args.files, + input_dir=args.input_dir, + output_dir=args.output_dir) diff --git a/tools/dircondenser.py b/tools/dircondenser.py index 023c14e..0e28bec 100755 --- a/tools/dircondenser.py +++ b/tools/dircondenser.py @@ -74,6 +74,10 @@ def condense(dirname: str): #print('git mv "%s" "%s"' % (old_name, new_name)) subprocess.check_call(['git', 'mv', old_name, new_name]) replacements.append((old_name, new_name)) + # update any appearances of old_name in expected stdout in test.json + json = os.path.join(new_name, 'test.json') + if os.path.isfile(json): + replace_source(json, [(old_name, new_name)]) os.chdir(curdir) replace_source('run_unittests.py', replacements) replace_source('run_project_tests.py', replacements) diff --git a/tools/gen_data.py b/tools/gen_data.py new file mode 100755 index 0000000..2cc05a4 --- /dev/null +++ b/tools/gen_data.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Daniel Mensinger + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import hashlib +import textwrap +import re +from pathlib import Path +from datetime import datetime +import typing as T + +class DataFile: + file_counter = 0 + + def __init__(self, path: Path, root: Path): + self.path = path + self.id = self.path.relative_to(root) + self.data_str = f'file_{DataFile.file_counter}_data_' + re.sub('[^a-zA-Z0-9]', '_', self.path.name) + DataFile.file_counter += 1 + + b = self.path.read_bytes() + self.data = b.decode() + self.sha256sum = hashlib.sha256(b).hexdigest() + + def __repr__(self) -> str: + return f'<{type(self).__name__}: [{self.sha256sum}] {self.id}>' + +def main() -> int: + root_dir = Path(__file__).resolve().parents[1] + mesonbuild_dir = root_dir / 'mesonbuild' + out_file = mesonbuild_dir / 'mesondata.py' + + data_dirs = mesonbuild_dir.glob('**/data') + + data_files: T.List[DataFile] = [] + + for d in data_dirs: + for p in d.iterdir(): + data_files += [DataFile(p, mesonbuild_dir)] + + print(f'Found {len(data_files)} data files') + + # Generate the data script + data = '' + + data += textwrap.dedent(f'''\ + # Copyright {datetime.today().year} The Meson development team + + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + + # http://www.apache.org/licenses/LICENSE-2.0 + + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + + #### + #### WARNING: This is an automatically generated file! Do not edit! + #### Generated by {Path(__file__).resolve().relative_to(root_dir)} + #### + + + from pathlib import Path + import typing as T + + if T.TYPE_CHECKING: + from .environment import Environment + + ###################### + # BEGIN Data section # + ###################### + + ''') + + for i in data_files: + data += f"{i.data_str} = '''\\\n{i.data}'''\n\n" + + data += textwrap.dedent(f''' + #################### + # END Data section # + #################### + + class DataFile: + def __init__(self, path: Path, sha256sum: str, data: str) -> None: + self.path = path + self.sha256sum = sha256sum + self.data = data + + def write_once(self, path: Path) -> None: + if not path.exists(): + path.write_text(self.data) + + def write_to_private(self, env: 'Environment') -> Path: + out_file = Path(env.scratch_dir) / 'data' / self.path.name + out_file.parent.mkdir(exist_ok=True) + self.write_once(out_file) + return out_file + + + mesondata = {{ + ''') + + for i in data_files: + data += textwrap.indent(textwrap.dedent(f"""\ + '{i.id}': DataFile( + Path('{i.id}'), + '{i.sha256sum}', + {i.data_str}, + ), + """), ' ') + + data += textwrap.dedent('''\ + } + ''') + + print(f'Updating {out_file}') + out_file.write_text(data) + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/regenerate_docs.py b/tools/regenerate_docs.py new file mode 100755 index 0000000..d443570 --- /dev/null +++ b/tools/regenerate_docs.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + + +# Copyright 2018 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +Regenerate markdown docs by using `meson.py` from the root dir +''' + +import argparse +import jinja2 +import os +import re +import subprocess +import sys +import textwrap +import typing as T +from pathlib import Path + +PathLike = T.Union[Path,str] + +def _get_meson_output(root_dir: Path, args: T.List): + env = os.environ.copy() + env['COLUMNS'] = '80' + return subprocess.run([str(sys.executable), str(root_dir/'meson.py')] + args, check=True, capture_output=True, text=True, env=env).stdout.strip() + +def get_commands_data(root_dir: Path): + usage_start_pattern = re.compile(r'^usage: ', re.MULTILINE) + positional_start_pattern = re.compile(r'^positional arguments:[\t ]*[\r\n]+', re.MULTILINE) + options_start_pattern = re.compile(r'^optional arguments:[\t ]*[\r\n]+', re.MULTILINE) + commands_start_pattern = re.compile(r'^[A-Za-z ]*[Cc]ommands:[\t ]*[\r\n]+', re.MULTILINE) + + def get_next_start(iterators, end): + return next((i.start() for i in iterators if i), end) + + def normalize_text(text): + # clean up formatting + out = text + out = re.sub(r'\r\n', r'\r', out, flags=re.MULTILINE) # replace newlines with a linux EOL + out = re.sub(r'^ +$', '', out, flags=re.MULTILINE) # remove trailing whitespace + out = re.sub(r'(?:^\n+|\n+$)', '', out) # remove trailing empty lines + return out + + def parse_cmd(cmd): + cmd_len = len(cmd) + usage = usage_start_pattern.search(cmd) + positionals = positional_start_pattern.search(cmd) + options = options_start_pattern.search(cmd) + commands = commands_start_pattern.search(cmd) + + arguments_start = get_next_start([positionals, options, commands], None) + assert arguments_start + + # replace `usage:` with `$` and dedent + dedent_size = (usage.end() - usage.start()) - len('$ ') + usage_text = textwrap.dedent(f'{dedent_size * " "}$ {normalize_text(cmd[usage.end():arguments_start])}') + + return { + 'usage': usage_text, + 'arguments': normalize_text(cmd[arguments_start:cmd_len]), + } + + def clean_dir_arguments(text): + # Remove platform specific defaults + args = [ + 'prefix', + 'bindir', + 'datadir', + 'includedir', + 'infodir', + 'libdir', + 'libexecdir', + 'localedir', + 'localstatedir', + 'mandir', + 'sbindir', + 'sharedstatedir', + 'sysconfdir' + ] + out = text + for a in args: + out = re.sub(r'(--' + a + r' .+?)\s+\(default:.+?\)(\.)?', r'\1\2', out, flags=re.MULTILINE|re.DOTALL) + return out + + output = _get_meson_output(root_dir, ['--help']) + commands = set(c.strip() for c in re.findall(r'usage:(?:.+)?{((?:[a-z]+,*)+?)}', output, re.MULTILINE|re.DOTALL)[0].split(',')) + commands.remove('help') + + cmd_data = dict() + + for cmd in commands: + cmd_output = _get_meson_output(root_dir, [cmd, '--help']) + cmd_data[cmd] = parse_cmd(cmd_output) + if cmd in ['setup', 'configure']: + cmd_data[cmd]['arguments'] = clean_dir_arguments(cmd_data[cmd]['arguments']) + + return cmd_data + +def regenerate_commands(root_dir: Path, output_dir: Path) -> None: + with open(root_dir/'docs'/'markdown_dynamic'/'Commands.md') as f: + template = f.read() + + cmd_data = get_commands_data(root_dir) + + t = jinja2.Template(template, undefined=jinja2.StrictUndefined, keep_trailing_newline=True) + content = t.render(cmd_help=cmd_data) + + output_file = output_dir/'Commands.md' + with open(output_file, 'w') as f: + f.write(content) + + print(f'`{output_file}` was regenerated') + +def regenerate_docs(output_dir: PathLike, + dummy_output_file: T.Optional[PathLike]) -> None: + if not output_dir: + raise ValueError(f'Output directory value is not set') + + output_dir = Path(output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + root_dir = Path(__file__).resolve().parent.parent + + regenerate_commands(root_dir, output_dir) + + if dummy_output_file: + with open(output_dir/dummy_output_file, 'w') as f: + f.write('dummy file for custom_target output') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate meson docs') + parser.add_argument('--output-dir', required=True) + parser.add_argument('--dummy-output-file', type=str) + + args = parser.parse_args() + + regenerate_docs(output_dir=args.output_dir, + dummy_output_file=args.dummy_output_file) |