#!/usr/bin/env python3
# Copyright (C) 2023-2024 Free Software Foundation, Inc.
#
# Script to regenerate FOO.opt.urls files for each FOO.opt in the
# source tree.
#
# This file is part of GCC.
#
# GCC is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 3, or (at your option) any later
# version.
#
# GCC is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with GCC; see the file COPYING3. If not see
# <http://www.gnu.org/licenses/>. */
DESCRIPTION = """
Parses the generated HTML (from "make html") to locate anchors
for options, then parses the .opt files within the source tree,
and generates a .opt.urls in the source tree for each .opt file,
giving URLs for each option, where it can.
Usage (from build/gcc subdirectory):
../../src/gcc/regenerate-opt-urls.py HTML/gcc-14.0.0/ ../../src
To run unit tests:
../../src/gcc/regenerate-opt-urls.py HTML/gcc-14.0.0/ ../../src --unit-test
"""
import argparse
import json
import os
from pathlib import Path
from pprint import pprint
import sys
import re
import unittest
def canonicalize_option_name(option_name):
if option_name.endswith('='):
option_name = option_name[0:-1]
return option_name
def canonicalize_url_suffix(url_suffix):
"""
Various options have anchors for both the positive and
negative form. For example -Wcpp has both:
'gcc/Warning-Options.html#index-Wno-cpp'
'gcc/Warning-Options.html#index-Wcpp'
Return a canonicalized version of the url_suffix that
strips out any "no-" prefixes, for use in deduplication.
Note that the resulting url suffix might not correspond to
an actual anchor in the HTML.
"""
url_suffix = re.sub('index-Wno-', 'index-W', url_suffix)
url_suffix = re.sub('index-fno-', 'index-f', url_suffix)
url_suffix = re.sub('_003d$', '', url_suffix)
url_suffix = re.sub('-([0-9]+)$', '', url_suffix)
return url_suffix
class Index:
def __init__(self):
# Map from language (or None) to map from option name to set of URL suffixes
self.entries = {}
def add_entry(self, matched_text, url_suffix, language, verbose=False):
if 'Attributes.html' in url_suffix:
return
matched_text = canonicalize_option_name(matched_text)
if language not in self.entries:
self.entries[language] = {}
per_lang_entries = self.entries[language]
if matched_text in per_lang_entries:
# Partition by canonicalized url_suffixes; add the
# first url_suffix in each such partition.
c_new = canonicalize_url_suffix(url_suffix)
for entry in per_lang_entries[matched_text]:
c_entry = canonicalize_url_suffix(entry)
if c_new == c_entry:
return
per_lang_entries[matched_text].add(url_suffix)
else:
per_lang_entries[matched_text] = set([url_suffix])
def get_languages(self):
return self.entries.keys()
def get_url_suffixes(self, text, language=None):
text = canonicalize_option_name(text)
per_lang_entries = self.entries.get(language)
if per_lang_entries:
return per_lang_entries.get(text)
def parse_option_index(self, input_filename, language, verbose=False):
with open(input_filename) as f:
dirname = input_filename.parent.name
for line in f:
self.parse_html_line_option_index(dirname, line, language, verbose)
def parse_html_line_option_index(self, dirname, line, language, verbose=False):
if verbose:
print(repr(line))
# Update for this in the GCC website's bin/preprocess process_html_file:
# | sed -e 's/_002d/-/g' -e 's/_002a/*/g' \
line = line.replace('_002d', '-')
line = line.replace('_002a', '*')
# e.g. <a href="Optimize-Options.html#index-fmodulo_002dsched"><code>fmodulo-sched</code></a>
m = re.search(r'<a href="([\S]+)"><code>([\S]+)</code></a>', line)
if not m:
return
if verbose:
print(m.groups())
url_suffix, index_text = m.groups()
option = '-' + index_text
# Strip off "no-" prefixes from options
if option[:5] == '-Wno-':
option = '-W' + option[5:]
if option[:5] == '-fno-':
option = '-f' + option[5:]
url_suffix = dirname + '/' + url_suffix
self.add_entry(option, url_suffix, language, verbose)
class TestParsingIndex(unittest.TestCase):
def test_parse_line(self):
index = Index()
index.parse_html_line_option_index('gcc',
'<a href="Optimize-Options.html#index-fmodulo_002dsched"><code>fmodulo-sched</code></a>',
None)
self.assertEqual(index.get_url_suffixes('-fmodulo-sched'),
{'gcc/Optimize-Options.html#index-fmodulo-sched'})
def test_negated_flag(self):
index = Index()
index.parse_html_line_option_index('gcc',
'<tr><td></td><td valign="top"><a href="Static-Analyzer-Options.html#index-fno_002danalyzer"><code>fno-analyzer</code></a>:</td><td> </td><td valign="top"><a href="Static-Analyzer-Options.html">Static Analyzer Options</a></td></tr>\n',
None)
self.assertEqual(index.get_url_suffixes('-fno-analyzer'), None)
self.assertEqual(index.get_url_suffixes('-fanalyzer'),
{'gcc/Static-Analyzer-Options.html#index-fno-analyzer'})
def test_negated_warning(self):
index = Index()
index.parse_html_line_option_index('gcc',
'<tr><td></td><td valign="top"><a href="Warning-Options.html#index-Wno_002dalloca"><code>Wno-alloca</code></a>:</td><td> </td><td valign="top"><a href="Warning-Options.html">Warning Options</a></td></tr>\n',
None)
self.assertEqual(index.get_url_suffixes('-Wno-alloca'),
None)
self.assertEqual(index.get_url_suffixes('-Walloca'),
{'gcc/Warning-Options.html#index-Wno-alloca'})
def test_parse_option_index(self):
index = Index()
index.parse_option_index(INPUT_HTML_PATH / 'gcc/Option-Index.html',
language=None)
self.assertEqual(index.get_url_suffixes('-fmodulo-sched'),
{'gcc/Optimize-Options.html#index-fmodulo-sched'})
self.assertEqual(index.get_url_suffixes('-O'),
{'gcc/Optimize-Options.html#index-O'})
self.assertEqual(index.get_url_suffixes('-O0'),
{'gcc/Optimize-Options.html#index-O0'})
self.assertEqual(index.get_url_suffixes('-Wframe-larger-than='),
{'gcc/Warning-Options.html#index-Wframe-larger-than_003d'})
# Check an option with duplicates: '-march'
# The url_suffixes will be of the form
# 'gcc/HPPA-Options.html#index-march-5',
# 'gcc/LoongArch-Options.html#index-march-7',
# etc, where the trailing number is, unfortunately, likely to
# change from release to release.
# Replace them with 'NN' for the purpose of this test:
em_arch_url_suffixes = [re.sub('(-[0-9]+)', '-NN', s)
for s in index.get_url_suffixes('-march')]
self.assertIn('gcc/ARM-Options.html#index-march-NN', em_arch_url_suffixes)
self.assertIn('gcc/x86-Options.html#index-march-NN', em_arch_url_suffixes)
self.assertEqual(index.get_url_suffixes('-Wcpp'),
{'gcc/Warning-Options.html#index-Wcpp'})
self.assertNotEqual(index.get_url_suffixes('-march'), None)
self.assertNotEqual(index.get_url_suffixes('-march='), None)
class OptFile:
def __init__(self, opt_path, rel_path):
"""
Parse a .opt file. Similar to opt-gather.awk.
"""
self.rel_path = rel_path
assert rel_path.startswith('gcc')
self.records = []
with open(opt_path) as f:
flag = 0
for line in f:
if re.match(r'[ \t]*(;|$)', line):
flag = 0
else:
if flag == 0:
self.records.append([line])
flag = 1
else:
self.records[-1].append(line)
# Mapping from target-specific page to subdirectory containing .opt files
# documented on that page.
TARGET_SPECIFIC_PAGES = {
'gcc/AArch64-Options.html' : 'gcc/config/aarch64/',
'gcc/AMD-GCN-Options.html' : 'gcc/config/gcn/',
'gcc/ARC-Options.html' : 'gcc/config/arc/',
'gcc/ARC-Options.html' : 'gcc/config/arc/',
'gcc/ARM-Options.html' : 'gcc/config/arm/',
'gcc/AVR-Options.html' : 'gcc/config/avr/',
'gcc/Adapteva-Epiphany-Options.html' : 'gcc/config/epiphany/',
'gcc/Blackfin-Options.html' : 'gcc/config/bfin/',
'gcc/C-SKY-Options.html' : 'gcc/config/csky/',
'gcc/C6X-Options.html' : 'gcc/config/c6x/',
'gcc/CRIS-Options.html' : 'gcc/config/cris/',
'gcc/DEC-Alpha-Options.html' : 'gcc/config/alpha/',
'gcc/FR30-Options.html' : 'gcc/config/fr30/',
'gcc/FRV-Options.html' : 'gcc/config/frv/',
'gcc/FT32-Options.html' : 'gcc/config/ft32/',
'gcc/H8_002f300-Options.html' : 'gcc/config/h8300/',
'gcc/HPPA-Options.html' : 'gcc/config/pa/',
'gcc/IA-64-Options.html' : 'gcc/config/ia64/',
'gcc/LoongArch-Options.html' : 'gcc/config/loongarch/',
'gcc/M32C-Options.html' : 'gcc/config/m32c/',
'gcc/M32R_002fD-Options.html' : 'gcc/config/m32r/',
'gcc/M680x0-Options.html' : 'gcc/config/m68k/',
'gcc/MCore-Options.html' : 'gcc/config/mcore/',
'gcc/MIPS-Options.html' : 'gcc/config/mips/',
'gcc/MMIX-Options.html' : 'gcc/config/mmix/',
'gcc/MN10300-Options.html' : 'gcc/config/mn10300/',
'gcc/MSP430-Options.html' : 'gcc/config/msp430/',
'gcc/MicroBlaze-Options.html' : 'gcc/config/microblaze/',
'gcc/Moxie-Options.html' : 'gcc/config/moxie/',
'gcc/NDS32-Options.html' : 'gcc/config/nds32/',
'gcc/Nios-II-Options.html' : 'gcc/config/nios2/',
'gcc/Nvidia-PTX-Options.html' : 'gcc/config/nvptx/',
'gcc/OpenRISC-Options.html' : 'gcc/config/or1k/',
'gcc/PDP-11-Options.html' : 'gcc/config/pdp11',
'gcc/PRU-Options.html' : 'gcc/config/pru/',
'gcc/RISC-V-Options.html' : 'gcc/config/riscv/',
'gcc/RL78-Options.html' : 'gcc/config/rl78/',
'gcc/RS_002f6000-and-PowerPC-Options.html' : 'gcc/config/rs6000/',
'gcc/RX-Options.html' : 'gcc/config/rx/',
'gcc/SH-Options.html' : 'gcc/config/sh/',
'gcc/SPARC-Options.html' : 'gcc/config/sparc/',
'gcc/S_002f390-and-zSeries-Options.html' : 'gcc/config/s390',
'gcc/V850-Options.html' : 'gcc/config/vax/',
'gcc/VAX-Options.html' : 'gcc/config/v850/',
'gcc/Visium-Options.html' : 'gcc/config/visium/',
'gcc/Xstormy16-Options.html' : 'gcc/config/stormy16/',
'gcc/Xtensa-Options.html' : 'gcc/config/xtensa/',
'gcc/eBPF-Options.html' : 'gcc/config/bpf/',
'gcc/x86-Options.html' : 'gcc/config/i386/',
}
def target_specific(url_suffix):
for page_prefix, subdir in TARGET_SPECIFIC_PAGES.items():
if url_suffix.startswith(page_prefix):
return subdir
def filter_urlsuffixes_for_optfile(optfile, url_suffixes):
"""
Filter out target-specific options for the wrong target.
"""
result = set()
for url_suffix in url_suffixes:
subdir = target_specific(url_suffix)
if subdir:
if not optfile.rel_path.startswith(subdir):
# Skip this
continue
result.add(url_suffix)
return result
class TestFiltering(unittest.TestCase):
def test_target_specific(self):
self.assertEqual(target_specific('gcc/Preprocessor-Options.html#index-A'),
None)
self.assertEqual(target_specific('gcc/MMIX-Options.html#index-mknuthdiv'),
'gcc/config/mmix/')
def test_filter(self):
s = {'gcc/MIPS-Options.html#index-munaligned-access-1',
'gcc/ARM-Options.html#index-munaligned-access'}
arm_optfile = OptFile('/dev/null', 'gcc/config/arm/arm.opt')
mips_optfile = OptFile('/dev/null', 'gcc/config/mips/mips.opt')
self.assertEqual(
filter_urlsuffixes_for_optfile(arm_optfile, s),
{'gcc/ARM-Options.html#index-munaligned-access'})
self.assertEqual(
filter_urlsuffixes_for_optfile(mips_optfile, s),
{'gcc/MIPS-Options.html#index-munaligned-access-1'})
def write_url_file(index, optfile, dstfile):
dstfile.write('; Autogenerated by regenerate-opt-urls.py from %s'
' and generated HTML\n\n'
% optfile.rel_path)
for record in optfile.records:
opt = '-' + record[0].strip()
url_suffixes_per_lang = {}
count = 0
for lang in index.get_languages():
this_lang_suffixes = index.get_url_suffixes(opt, language=lang)
url_suffixes_per_lang[lang] = this_lang_suffixes
if this_lang_suffixes:
count += len(this_lang_suffixes)
if not count:
continue
directives = []
for lang in index.get_languages():
if lang:
directive = 'LangUrlSuffix_%s for %r' % (lang, opt[1:])
else:
directive = 'UrlSuffix for %r' % opt[1:]
url_suffixes = url_suffixes_per_lang[lang]
if url_suffixes:
url_suffixes = filter_urlsuffixes_for_optfile(optfile, url_suffixes)
if url_suffixes:
if len(url_suffixes) == 1:
if lang:
directives.append('LangUrlSuffix_%s(%s)' % (lang, list(url_suffixes)[0]))
else:
directives.append('UrlSuffix(%s)' % list(url_suffixes)[0])
else:
dstfile.write('; skipping %s due to multiple URLs:\n'
% directive)
for u in sorted(url_suffixes):
dstfile.write('; duplicate: %r\n' % u)
else:
dstfile.write('; skipping %s due to finding no URLs\n'
% directive)
if directives:
dstfile.write('%s\n' % opt[1:])
dstfile.write(' '.join(directives) + '\n')
dstfile.write('\n')
# A list of (REL_PATH, LANG) pairs, where
# - REL_PATH is the relative path to a generated Option-Index.html file
# for a specific frontend, and
# - LANG is the name of the language (as specified in the "Language" record
# within the lang.opt file), or None for the language-independent
# documentation.
PER_LANGUAGE_OPTION_INDEXES = [
('gcc/Option-Index.html', None),
('gdc/Option-Index.html', 'D'),
('gfortran/Option-Index.html', 'Fortran')
]
def main(args):
index = Index()
for option_index_rel_path, lang in PER_LANGUAGE_OPTION_INDEXES:
index.parse_option_index(args.base_html_dir / option_index_rel_path,
language=lang)
for root, dirs, files in os.walk(args.src_gcc_dir):
for f in files:
if f.endswith('.opt'):
opt_path = os.path.join(root, f)
rel_path = os.path.relpath(opt_path, args.src_gcc_dir)
optfile = OptFile(opt_path, rel_path)
dstname = f + '.urls'
urlfile = os.path.join(root, dstname)
with open(urlfile, 'w') as dstfile:
write_url_file(index, optfile, dstfile)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=DESCRIPTION,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('base_html_dir', type=Path)
parser.add_argument('src_gcc_dir', type=Path)
parser.add_argument('--unit-test', action='store_true')
args = parser.parse_args()
if args.unit_test:
INPUT_HTML_PATH = args.base_html_dir
unittest.main(argv=[sys.argv[0], '-v'])
else:
main(args)