From c38f679eebffd601d977330f24def69f4b620c93 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Mon, 22 Mar 2021 10:53:57 +0100 Subject: mklog: add new argument --directory. The argument is handy when one needs to generate ChangeLog entries for a different project (e.g. binutils). contrib/ChangeLog: * mklog.py: Add --directory argument. --- contrib/mklog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 6509886..00a001e 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -71,7 +71,7 @@ gcc_root = os.path.dirname(os.path.dirname(script_folder)) def find_changelog(path): folder = os.path.split(path)[0] while True: - if os.path.exists(os.path.join(gcc_root, folder, 'ChangeLog')): + if os.path.exists(os.path.join(args.directory, folder, 'ChangeLog')): return folder folder = os.path.dirname(folder) if folder == '': @@ -277,6 +277,9 @@ if __name__ == '__main__': help='Do not generate function names in ChangeLogs') parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', help='Download title of mentioned PRs') + parser.add_argument('-d', '--directory', default=gcc_root, + help='Root directory where to search for ChangeLog ' + 'files') parser.add_argument('-c', '--changelog', help='Append the ChangeLog to a git commit message ' 'file') -- cgit v1.1 From e54da1b6b39baadded02a28382bb6efb70e687c1 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Tue, 23 Mar 2021 08:49:25 +0100 Subject: mklog: fix test_mklog.py tests. contrib/ChangeLog: * mklog.py: Fix broken tests. --- contrib/mklog.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 00a001e..1604f05 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -65,13 +65,13 @@ PATCH must be generated using diff(1)'s -up or -cp options """ script_folder = os.path.realpath(__file__) -gcc_root = os.path.dirname(os.path.dirname(script_folder)) +root = os.path.dirname(os.path.dirname(script_folder)) def find_changelog(path): folder = os.path.split(path)[0] while True: - if os.path.exists(os.path.join(args.directory, folder, 'ChangeLog')): + if os.path.exists(os.path.join(root, folder, 'ChangeLog')): return folder folder = os.path.dirname(folder) if folder == '': @@ -277,7 +277,7 @@ if __name__ == '__main__': help='Do not generate function names in ChangeLogs') parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', help='Download title of mentioned PRs') - parser.add_argument('-d', '--directory', default=gcc_root, + parser.add_argument('-d', '--directory', help='Root directory where to search for ChangeLog ' 'files') parser.add_argument('-c', '--changelog', @@ -288,6 +288,8 @@ if __name__ == '__main__': args = parser.parse_args() if args.input == '-': args.input = None + if args.directory: + root = args.directory data = open(args.input) if args.input else sys.stdin if args.update_copyright: -- cgit v1.1 From fef084dc83d38cad32604bb1cee4caf42ffeec12 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Thu, 13 May 2021 15:12:36 +0200 Subject: mklog: Put detected PR entries before ChangeLogs contrib/ChangeLog: * mklog.py: Put PR entries before all ChangeLog entries (will be added to all ChangeLog locations by Daily bump script). * test_mklog.py: Test the new behavior. --- contrib/mklog.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 1604f05..5c93c70 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -169,13 +169,19 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False): if fill_pr_titles: out += get_pr_titles(prs) + # print list of PR entries before ChangeLog entries + if prs: + if not out: + out += '\n' + for pr in prs: + out += '\t%s\n' % pr + out += '\n' + # sort ChangeLog so that 'testsuite' is at the end for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x): files = changelogs[changelog] out += '%s:\n' % os.path.join(changelog, 'ChangeLog') out += '\n' - for pr in prs: - out += '\t%s\n' % pr # new and deleted files should be at the end for file in sorted(files, key=sort_changelog_files): assert file.path.startswith(changelog) -- cgit v1.1 From 58e3b17f4c42d050a1768b025712e6d18bcb76ae Mon Sep 17 00:00:00 2001 From: Jason Merrill Date: Wed, 16 Jun 2021 16:46:38 -0400 Subject: mklog: add subject line skeleton In the recent gcc-commit-mklog thread on gcc@ it occurred to me that the command could also fill in part of the subject line. If the first PR is foo/1234, and the commit does not yet have a subject line, this will add foo: [PR1234] contrib/ChangeLog: * mklog.py: Add an initial component: [PRnnnnn] line when we have a PR. --- contrib/mklog.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 5c93c70..1f59055 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -39,6 +39,7 @@ import requests from unidiff import PatchSet pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PPR [a-z+-]+\/[0-9]+)') +prnum_regex = re.compile(r'PR (?P[a-z+-]+)/(?P[0-9]+)') dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PDR [0-9]+)') dg_regex = re.compile(r'{\s+dg-(error|warning)') identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') @@ -67,6 +68,8 @@ PATCH must be generated using diff(1)'s -up or -cp options script_folder = os.path.realpath(__file__) root = os.path.dirname(os.path.dirname(script_folder)) +firstpr = '' + def find_changelog(path): folder = os.path.split(path)[0] @@ -134,6 +137,7 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False): prs = [] out = '' diff = PatchSet(data) + global firstpr for file in diff: # skip files that can't be parsed @@ -166,6 +170,9 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False): # Found dg-warning/dg-error line break + if prs: + firstpr = prs[0] + if fill_pr_titles: out += get_pr_titles(prs) @@ -308,8 +315,14 @@ if __name__ == '__main__': start = list(takewhile(lambda l: not l.startswith('#'), lines)) end = lines[len(start):] with open(args.changelog, 'w') as f: + if not start or not start[0]: + # initial commit subject line 'component: [PRnnnnn]' + m = prnum_regex.match(firstpr) + if m: + title = f'{m.group("comp")}: [PR{m.group("num")}]' + start.insert(0, title) if start: - # appent empty line + # append empty line if start[-1] != '': start.append('') else: -- cgit v1.1 From edf0c3ffb59d75c11e05bc722432dc554e275c72 Mon Sep 17 00:00:00 2001 From: Tobias Burnus Date: Mon, 21 Jun 2021 15:17:22 +0200 Subject: contrib/mklog.py: Improve PR handling Co-authored-by: Martin Sebor contrib/ChangeLog: * mklog.py (bugzilla_url): Fetch also component. (pr_filename_regex): New. (get_pr_titles): Update PR string with correct format and component. (generate_changelog): Take additional PRs; extract PR from the filename. (__main__): Add -b/--pr-numbers argument. * test_mklog.py (EXPECTED4): Update to expect a PR for the new file. --- contrib/mklog.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 1f59055..0b434f6 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -42,6 +42,7 @@ pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PPR [a-z+-]+\/[0-9]+)') prnum_regex = re.compile(r'PR (?P[a-z+-]+)/(?P[0-9]+)') dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PDR [0-9]+)') dg_regex = re.compile(r'{\s+dg-(error|warning)') +pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P\d{4,})') identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)') comment_regex = re.compile(r'^\/\*') struct_regex = re.compile(r'^(class|struct|union|enum)\s+' @@ -52,7 +53,7 @@ fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]') template_and_param_regex = re.compile(r'<[^<>]*>') md_def_regex = re.compile(r'\(define.*\s+"(.*)"') bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \ - 'include_fields=summary' + 'include_fields=summary,component' function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'} @@ -118,20 +119,23 @@ def sort_changelog_files(changed_file): def get_pr_titles(prs): - output = '' - for pr in prs: + output = [] + for idx, pr in enumerate(prs): pr_id = pr.split('/')[-1] r = requests.get(bugzilla_url % pr_id) bugs = r.json()['bugs'] if len(bugs) == 1: - output += '%s - %s\n' % (pr, bugs[0]['summary']) - print(output) + prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id) + out = '%s - %s\n' % (prs[idx], bugs[0]['summary']) + if out not in output: + output.append(out) if output: - output += '\n' - return output + output.append('') + return '\n'.join(output) -def generate_changelog(data, no_functions=False, fill_pr_titles=False): +def generate_changelog(data, no_functions=False, fill_pr_titles=False, + additional_prs=None): changelogs = {} changelog_list = [] prs = [] @@ -139,6 +143,8 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False): diff = PatchSet(data) global firstpr + if additional_prs: + prs = [pr for pr in additional_prs if pr not in prs] for file in diff: # skip files that can't be parsed if file.path == '/dev/null': @@ -154,21 +160,32 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False): # Only search first ten lines as later lines may # contains commented code which a note that it # has not been tested due to a certain PR or DR. + this_file_prs = [] for line in list(file)[0][0:10]: m = pr_regex.search(line.value) if m: pr = m.group('pr') if pr not in prs: prs.append(pr) + this_file_prs.append(pr.split('/')[-1]) else: m = dr_regex.search(line.value) if m: dr = m.group('dr') if dr not in prs: prs.append(dr) + this_file_prs.append(dr.split('/')[-1]) elif dg_regex.search(line.value): # Found dg-warning/dg-error line break + # PR number in the file name + fname = os.path.basename(file.path) + m = pr_filename_regex.search(fname) + if m: + pr = m.group('pr') + pr2 = 'PR ' + pr + if pr not in this_file_prs and pr2 not in prs: + prs.append(pr2) if prs: firstpr = prs[0] @@ -286,6 +303,9 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description=help_message) parser.add_argument('input', nargs='?', help='Patch file (or missing, read standard input)') + parser.add_argument('-b', '--pr-numbers', action='store', + type=lambda arg: arg.split(','), nargs="?", + help='Add the specified PRs (comma separated)') parser.add_argument('-s', '--no-functions', action='store_true', help='Do not generate function names in ChangeLogs') parser.add_argument('-p', '--fill-up-bug-titles', action='store_true', @@ -309,7 +329,7 @@ if __name__ == '__main__': update_copyright(data) else: output = generate_changelog(data, args.no_functions, - args.fill_up_bug_titles) + args.fill_up_bug_titles, args.pr_numbers) if args.changelog: lines = open(args.changelog).read().split('\n') start = list(takewhile(lambda l: not l.startswith('#'), lines)) -- cgit v1.1 From 48b312b4ba4bb738e22b92980760b38ffa6d68a5 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Tue, 22 Jun 2021 09:50:38 +0200 Subject: contrib: fix a flake8 issue contrib/ChangeLog: * mklog.py: Fix flake8 issue. --- contrib/mklog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 0b434f6..674c1dc 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -304,7 +304,7 @@ if __name__ == '__main__': parser.add_argument('input', nargs='?', help='Patch file (or missing, read standard input)') parser.add_argument('-b', '--pr-numbers', action='store', - type=lambda arg: arg.split(','), nargs="?", + type=lambda arg: arg.split(','), nargs='?', help='Add the specified PRs (comma separated)') parser.add_argument('-s', '--no-functions', action='store_true', help='Do not generate function names in ChangeLogs') -- cgit v1.1 From b838641bb0d4de5b25128b54012155ab46f452d0 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Mon, 28 Jun 2021 13:08:10 +0200 Subject: mklog: Handle correctly long lines. contrib/ChangeLog: * mklog.py: Handle correctly long lines. * test_mklog.py: Test it. --- contrib/mklog.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index 674c1dc..ba70af0 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -38,6 +38,9 @@ import requests from unidiff import PatchSet +LINE_LIMIT = 100 +TAB_WIDTH = 8 + pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PPR [a-z+-]+\/[0-9]+)') prnum_regex = re.compile(r'PR (?P[a-z+-]+)/(?P[0-9]+)') dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PDR [0-9]+)') @@ -134,6 +137,16 @@ def get_pr_titles(prs): return '\n'.join(output) +def append_changelog_line(out, relative_path, text): + line = f'\t* {relative_path}:' + if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT: + out += f'{line} {text}\n' + else: + out += f'{line}\n' + out += f'\t{text}\n' + return out + + def generate_changelog(data, no_functions=False, fill_pr_titles=False, additional_prs=None): changelogs = {} @@ -213,12 +226,12 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False, relative_path = file.path[len(changelog):].lstrip('/') functions = [] if file.is_added_file: - msg = 'New test' if in_tests else 'New file' - out += '\t* %s: %s.\n' % (relative_path, msg) + msg = 'New test.' if in_tests else 'New file.' + out = append_changelog_line(out, relative_path, msg) elif file.is_removed_file: - out += '\t* %s: Removed.\n' % (relative_path) + out = append_changelog_line(out, relative_path, 'Removed.') elif hasattr(file, 'is_rename') and file.is_rename: - out += '\t* %s: Moved to...\n' % (relative_path) + out = append_changelog_line(out, relative_path, 'Moved to...') new_path = file.target_file[2:] # A file can be theoretically moved to a location that # belongs to a different ChangeLog. Let user fix it. @@ -227,6 +240,7 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False, out += '\t* %s: ...here.\n' % (new_path) elif os.path.basename(file.path) in generated_files: out += '\t* %s: Regenerate.\n' % (relative_path) + append_changelog_line(out, relative_path, 'Regenerate.') else: if not no_functions: for hunk in file: -- cgit v1.1 From 84f906df4f0e99b94565f1ca72cf57f63420ad8a Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Mon, 12 Jul 2021 12:07:19 +0200 Subject: mklog: support '-b c/101343' format. contrib/ChangeLog: * mklog.py: Support additional PRs without PR prefix. --- contrib/mklog.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index ba70af0..d2aea85 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -157,7 +157,11 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False, global firstpr if additional_prs: - prs = [pr for pr in additional_prs if pr not in prs] + for apr in additional_prs: + if not apr.startswith('PR ') and '/' in apr: + apr = 'PR ' + apr + if apr not in prs: + prs.append(apr) for file in diff: # skip files that can't be parsed if file.path == '/dev/null': -- cgit v1.1 From 0684c8d3effab2c9c1b29938f5e56c77106af564 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Tue, 17 Aug 2021 14:57:40 +0200 Subject: commit-mklog: Add --co argument. The argument can be used for addition of Co-Authored-By lines with --trailer='Co-Authored-By=Mona Lisa Octocat '. contrib/ChangeLog: * gcc-git-customization.sh: Wrap $@ in quotes. * git-commit-mklog.py: Add new argument --co. * mklog.py: Skip the Co-Authored-By lines. --- contrib/mklog.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index d2aea85..d362be5 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -40,6 +40,7 @@ from unidiff import PatchSet LINE_LIMIT = 100 TAB_WIDTH = 8 +CO_AUTHORED_BY_PREFIX = 'co-authored-by: ' pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?PPR [a-z+-]+\/[0-9]+)') prnum_regex = re.compile(r'PR (?P[a-z+-]+)/(?P[0-9]+)') @@ -317,6 +318,12 @@ def update_copyright(data): f.write(content) +def skip_line_in_changelog(line): + if line.lower().startswith(CO_AUTHORED_BY_PREFIX) or line.startswith('#'): + return False + return True + + if __name__ == '__main__': parser = argparse.ArgumentParser(description=help_message) parser.add_argument('input', nargs='?', @@ -350,7 +357,7 @@ if __name__ == '__main__': args.fill_up_bug_titles, args.pr_numbers) if args.changelog: lines = open(args.changelog).read().split('\n') - start = list(takewhile(lambda l: not l.startswith('#'), lines)) + start = list(takewhile(skip_line_in_changelog, lines)) end = lines[len(start):] with open(args.changelog, 'w') as f: if not start or not start[0]: -- cgit v1.1 From e370a2482d41fd382055695b9a0a638ce75e1038 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Thu, 26 Aug 2021 10:20:20 +0200 Subject: mklog: fix renaming with version 0.7.0. contrib/ChangeLog: * mklog.py: Use file.{source,target}_file for proper rename handling. --- contrib/mklog.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'contrib/mklog.py') diff --git a/contrib/mklog.py b/contrib/mklog.py index d362be5..cd5ef0b 100755 --- a/contrib/mklog.py +++ b/contrib/mklog.py @@ -148,6 +148,13 @@ def append_changelog_line(out, relative_path, text): return out +def get_rel_path_if_prefixed(path, folder): + if path.startswith(folder): + return path[len(folder):].lstrip('/') + else: + return path + + def generate_changelog(data, no_functions=False, fill_pr_titles=False, additional_prs=None): changelogs = {} @@ -228,7 +235,7 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False, for file in sorted(files, key=sort_changelog_files): assert file.path.startswith(changelog) in_tests = 'testsuite' in changelog or 'testsuite' in file.path - relative_path = file.path[len(changelog):].lstrip('/') + relative_path = get_rel_path_if_prefixed(file.path, changelog) functions = [] if file.is_added_file: msg = 'New test.' if in_tests else 'New file.' @@ -236,13 +243,17 @@ def generate_changelog(data, no_functions=False, fill_pr_titles=False, elif file.is_removed_file: out = append_changelog_line(out, relative_path, 'Removed.') elif hasattr(file, 'is_rename') and file.is_rename: - out = append_changelog_line(out, relative_path, 'Moved to...') - new_path = file.target_file[2:] # A file can be theoretically moved to a location that # belongs to a different ChangeLog. Let user fix it. - if new_path.startswith(changelog): - new_path = new_path[len(changelog):].lstrip('/') - out += '\t* %s: ...here.\n' % (new_path) + # + # Since unidiff 0.7.0, path.file == path.target_file[2:], + # it used to be path.source_file[2:] + relative_path = get_rel_path_if_prefixed(file.source_file[2:], + changelog) + out = append_changelog_line(out, relative_path, 'Moved to...') + new_path = get_rel_path_if_prefixed(file.target_file[2:], + changelog) + out += f'\t* {new_path}: ...here.\n' elif os.path.basename(file.path) in generated_files: out += '\t* %s: Regenerate.\n' % (relative_path) append_changelog_line(out, relative_path, 'Regenerate.') -- cgit v1.1