scripts: Extract glibcpp.py from check-obsolete-constructs.py

The C tokenizer is useful separately.

Reviewed-by: Siddhesh Poyarekar <siddhesh@sourceware.org>

2 files changed, 217 insertions, 184 deletions

diff --git a/scripts/check-obsolete-constructs.py b/scripts/check-obsolete-constructs.py
index 826568c..102f51b 100755
--- a/scripts/check-obsolete-constructs.py
+++ b/scripts/check-obsolete-constructs.py
@@ -24,193 +24,14 @@
 """
 
 import argparse
-import collections
+import os
 import re
 import sys
 
-# Simplified lexical analyzer for C preprocessing tokens.
-# Does not implement trigraphs.
-# Does not implement backslash-newline in the middle of any lexical
-#   item other than a string literal.
-# Does not implement universal-character-names in identifiers.
-# Treats prefixed strings (e.g. L"...") as two tokens (L and "...")
-# Accepts non-ASCII characters only within comments and strings.
-
-# Caution: The order of the outermost alternation matters.
-# STRING must be before BAD_STRING, CHARCONST before BAD_CHARCONST,
-# BLOCK_COMMENT before BAD_BLOCK_COM before PUNCTUATOR, and OTHER must
-# be last.
-# Caution: There should be no capturing groups other than the named
-# captures in the outermost alternation.
-
-# For reference, these are all of the C punctuators as of C11:
-#   [ ] ( ) { } , ; ? ~
-#   ! != * *= / /= ^ ^= = ==
-#   # ##
-#   % %= %> %: %:%:
-#   & &= &&
-#   | |= ||
-#   + += ++
-#   - -= -- ->
-#   . ...
-#   : :>
-#   < <% <: << <<= <=
-#   > >= >> >>=
-
-# The BAD_* tokens are not part of the official definition of pp-tokens;
-# they match unclosed strings, character constants, and block comments,
-# so that the regex engine doesn't have to backtrack all the way to the
-# beginning of a broken construct and then emit dozens of junk tokens.
-
-PP_TOKEN_RE_ = re.compile(r"""
-    (?P<STRING>        \"(?:[^\"\\\r\n]|\\(?:[\r\n -~]|\r\n))*\")
-   |(?P<BAD_STRING>    \"(?:[^\"\\\r\n]|\\[ -~])*)
-   |(?P<CHARCONST>     \'(?:[^\'\\\r\n]|\\(?:[\r\n -~]|\r\n))*\')
-   |(?P<BAD_CHARCONST> \'(?:[^\'\\\r\n]|\\[ -~])*)
-   |(?P<BLOCK_COMMENT> /\*(?:\*(?!/)|[^*])*\*/)
-   |(?P<BAD_BLOCK_COM> /\*(?:\*(?!/)|[^*])*\*?)
-   |(?P<LINE_COMMENT>  //[^\r\n]*)
-   |(?P<IDENT>         [_a-zA-Z][_a-zA-Z0-9]*)
-   |(?P<PP_NUMBER>     \.?[0-9](?:[0-9a-df-oq-zA-DF-OQ-Z_.]|[eEpP][+-]?)*)
-   |(?P<PUNCTUATOR>
-       [,;?~(){}\[\]]
-     | [!*/^=]=?
-     | \#\#?
-     | %(?:[=>]|:(?:%:)?)?
-     | &[=&]?
-     |\|[=|]?
-     |\+[=+]?
-     | -[=->]?
-     |\.(?:\.\.)?
-     | :>?
-     | <(?:[%:]|<(?:=|<=?)?)?
-     | >(?:=|>=?)?)
-   |(?P<ESCNL>         \\(?:\r|\n|\r\n))
-   |(?P<WHITESPACE>    [ \t\n\r\v\f]+)
-   |(?P<OTHER>         .)
-""", re.DOTALL | re.VERBOSE)
-
-HEADER_NAME_RE_ = re.compile(r"""
-    < [^>\r\n]+ >
-  | " [^"\r\n]+ "
-""", re.DOTALL | re.VERBOSE)
-
-ENDLINE_RE_ = re.compile(r"""\r|\n|\r\n""")
-
-# based on the sample code in the Python re documentation
-Token_ = collections.namedtuple("Token", (
-    "kind", "text", "line", "column", "context"))
-Token_.__doc__ = """
-   One C preprocessing token, comment, or chunk of whitespace.
-   'kind' identifies the token type, which will be one of:
-       STRING, CHARCONST, BLOCK_COMMENT, LINE_COMMENT, IDENT,
-       PP_NUMBER, PUNCTUATOR, ESCNL, WHITESPACE, HEADER_NAME,
-       or OTHER.  The BAD_* alternatives in PP_TOKEN_RE_ are
-       handled within tokenize_c, below.
-
-   'text' is the sequence of source characters making up the token;
-       no decoding whatsoever is performed.
-
-   'line' and 'column' give the position of the first character of the
-      token within the source file.  They are both 1-based.
-
-   'context' indicates whether or not this token occurred within a
-      preprocessing directive; it will be None for running text,
-      '<null>' for the leading '#' of a directive line (because '#'
-      all by itself on a line is a "null directive"), or the name of
-      the directive for tokens within a directive line, starting with
-      the IDENT for the name itself.
-"""
-
-def tokenize_c(file_contents, reporter):
-    """Yield a series of Token objects, one for each preprocessing
-       token, comment, or chunk of whitespace within FILE_CONTENTS.
-       The REPORTER object is expected to have one method,
-       reporter.error(token, message), which will be called to
-       indicate a lexical error at the position of TOKEN.
-       If MESSAGE contains the four-character sequence '{!r}', that
-       is expected to be replaced by repr(token.text).
-    """
+# Make available glibc Python modules.
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
 
-    Token = Token_
-    PP_TOKEN_RE = PP_TOKEN_RE_
-    ENDLINE_RE = ENDLINE_RE_
-    HEADER_NAME_RE = HEADER_NAME_RE_
-
-    line_num = 1
-    line_start = 0
-    pos = 0
-    limit = len(file_contents)
-    directive = None
-    at_bol = True
-    while pos < limit:
-        if directive == "include":
-            mo = HEADER_NAME_RE.match(file_contents, pos)
-            if mo:
-                kind = "HEADER_NAME"
-                directive = "after_include"
-            else:
-                mo = PP_TOKEN_RE.match(file_contents, pos)
-                kind = mo.lastgroup
-                if kind != "WHITESPACE":
-                    directive = "after_include"
-        else:
-            mo = PP_TOKEN_RE.match(file_contents, pos)
-            kind = mo.lastgroup
-
-        text = mo.group()
-        line = line_num
-        column = mo.start() - line_start
-        adj_line_start = 0
-        # only these kinds can contain a newline
-        if kind in ("WHITESPACE", "BLOCK_COMMENT", "LINE_COMMENT",
-                    "STRING", "CHARCONST", "BAD_BLOCK_COM", "ESCNL"):
-            for tmo in ENDLINE_RE.finditer(text):
-                line_num += 1
-                adj_line_start = tmo.end()
-            if adj_line_start:
-                line_start = mo.start() + adj_line_start
-
-        # Track whether or not we are scanning a preprocessing directive.
-        if kind == "LINE_COMMENT" or (kind == "WHITESPACE" and adj_line_start):
-            at_bol = True
-            directive = None
-        else:
-            if kind == "PUNCTUATOR" and text == "#" and at_bol:
-                directive = "<null>"
-            elif kind == "IDENT" and directive == "<null>":
-                directive = text
-            at_bol = False
-
-        # Report ill-formed tokens and rewrite them as their well-formed
-        # equivalents, so downstream processing doesn't have to know about them.
-        # (Rewriting instead of discarding provides better error recovery.)
-        if kind == "BAD_BLOCK_COM":
-            reporter.error(Token("BAD_BLOCK_COM", "", line, column+1, ""),
-                           "unclosed block comment")
-            text += "*/"
-            kind = "BLOCK_COMMENT"
-        elif kind == "BAD_STRING":
-            reporter.error(Token("BAD_STRING", "", line, column+1, ""),
-                           "unclosed string")
-            text += "\""
-            kind = "STRING"
-        elif kind == "BAD_CHARCONST":
-            reporter.error(Token("BAD_CHARCONST", "", line, column+1, ""),
-                           "unclosed char constant")
-            text += "'"
-            kind = "CHARCONST"
-
-        tok = Token(kind, text, line, column+1,
-                    "include" if directive == "after_include" else directive)
-        # Do not complain about OTHER tokens inside macro definitions.
-        # $ and @ appear in macros defined by headers intended to be
-        # included from assembly language, e.g. sysdeps/mips/sys/asm.h.
-        if kind == "OTHER" and directive != "define":
-            self.error(tok, "stray {!r} in program")
-
-        yield tok
-        pos = mo.end()
+import glibcpp
 
 #
 # Base and generic classes for individual checks.
@@ -446,7 +267,7 @@ class HeaderChecker:
 
         typedef_checker = ObsoleteTypedefChecker(self, self.fname)
 
-        for tok in tokenize_c(contents, self):
+        for tok in glibcpp.tokenize_c(contents, self):
             typedef_checker.examine(tok)
 
 def main():
diff --git a/scripts/glibcpp.py b/scripts/glibcpp.py
new file mode 100644
index 0000000..b44c6a4
--- /dev/null
+++ b/scripts/glibcpp.py
@@ -0,0 +1,212 @@
+#! /usr/bin/python3
+# Approximation to C preprocessing.
+# Copyright (C) 2019-2022 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+#
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <https://www.gnu.org/licenses/>.
+
+"""
+Simplified lexical analyzer for C preprocessing tokens.
+
+Does not implement trigraphs.
+
+Does not implement backslash-newline in the middle of any lexical
+item other than a string literal.
+
+Does not implement universal-character-names in identifiers.
+
+Treats prefixed strings (e.g. L"...") as two tokens (L and "...").
+
+Accepts non-ASCII characters only within comments and strings.
+"""
+
+import collections
+import re
+
+# Caution: The order of the outermost alternation matters.
+# STRING must be before BAD_STRING, CHARCONST before BAD_CHARCONST,
+# BLOCK_COMMENT before BAD_BLOCK_COM before PUNCTUATOR, and OTHER must
+# be last.
+# Caution: There should be no capturing groups other than the named
+# captures in the outermost alternation.
+
+# For reference, these are all of the C punctuators as of C11:
+#   [ ] ( ) { } , ; ? ~
+#   ! != * *= / /= ^ ^= = ==
+#   # ##
+#   % %= %> %: %:%:
+#   & &= &&
+#   | |= ||
+#   + += ++
+#   - -= -- ->
+#   . ...
+#   : :>
+#   < <% <: << <<= <=
+#   > >= >> >>=
+
+# The BAD_* tokens are not part of the official definition of pp-tokens;
+# they match unclosed strings, character constants, and block comments,
+# so that the regex engine doesn't have to backtrack all the way to the
+# beginning of a broken construct and then emit dozens of junk tokens.
+
+PP_TOKEN_RE_ = re.compile(r"""
+    (?P<STRING>        \"(?:[^\"\\\r\n]|\\(?:[\r\n -~]|\r\n))*\")
+   |(?P<BAD_STRING>    \"(?:[^\"\\\r\n]|\\[ -~])*)
+   |(?P<CHARCONST>     \'(?:[^\'\\\r\n]|\\(?:[\r\n -~]|\r\n))*\')
+   |(?P<BAD_CHARCONST> \'(?:[^\'\\\r\n]|\\[ -~])*)
+   |(?P<BLOCK_COMMENT> /\*(?:\*(?!/)|[^*])*\*/)
+   |(?P<BAD_BLOCK_COM> /\*(?:\*(?!/)|[^*])*\*?)
+   |(?P<LINE_COMMENT>  //[^\r\n]*)
+   |(?P<IDENT>         [_a-zA-Z][_a-zA-Z0-9]*)
+   |(?P<PP_NUMBER>     \.?[0-9](?:[0-9a-df-oq-zA-DF-OQ-Z_.]|[eEpP][+-]?)*)
+   |(?P<PUNCTUATOR>
+       [,;?~(){}\[\]]
+     | [!*/^=]=?
+     | \#\#?
+     | %(?:[=>]|:(?:%:)?)?
+     | &[=&]?
+     |\|[=|]?
+     |\+[=+]?
+     | -[=->]?
+     |\.(?:\.\.)?
+     | :>?
+     | <(?:[%:]|<(?:=|<=?)?)?
+     | >(?:=|>=?)?)
+   |(?P<ESCNL>         \\(?:\r|\n|\r\n))
+   |(?P<WHITESPACE>    [ \t\n\r\v\f]+)
+   |(?P<OTHER>         .)
+""", re.DOTALL | re.VERBOSE)
+
+HEADER_NAME_RE_ = re.compile(r"""
+    < [^>\r\n]+ >
+  | " [^"\r\n]+ "
+""", re.DOTALL | re.VERBOSE)
+
+ENDLINE_RE_ = re.compile(r"""\r|\n|\r\n""")
+
+# based on the sample code in the Python re documentation
+Token_ = collections.namedtuple("Token", (
+    "kind", "text", "line", "column", "context"))
+Token_.__doc__ = """
+   One C preprocessing token, comment, or chunk of whitespace.
+   'kind' identifies the token type, which will be one of:
+       STRING, CHARCONST, BLOCK_COMMENT, LINE_COMMENT, IDENT,
+       PP_NUMBER, PUNCTUATOR, ESCNL, WHITESPACE, HEADER_NAME,
+       or OTHER.  The BAD_* alternatives in PP_TOKEN_RE_ are
+       handled within tokenize_c, below.
+
+   'text' is the sequence of source characters making up the token;
+       no decoding whatsoever is performed.
+
+   'line' and 'column' give the position of the first character of the
+      token within the source file.  They are both 1-based.
+
+   'context' indicates whether or not this token occurred within a
+      preprocessing directive; it will be None for running text,
+      '<null>' for the leading '#' of a directive line (because '#'
+      all by itself on a line is a "null directive"), or the name of
+      the directive for tokens within a directive line, starting with
+      the IDENT for the name itself.
+"""
+
+def tokenize_c(file_contents, reporter):
+    """Yield a series of Token objects, one for each preprocessing
+       token, comment, or chunk of whitespace within FILE_CONTENTS.
+       The REPORTER object is expected to have one method,
+       reporter.error(token, message), which will be called to
+       indicate a lexical error at the position of TOKEN.
+       If MESSAGE contains the four-character sequence '{!r}', that
+       is expected to be replaced by repr(token.text).
+    """
+
+    Token = Token_
+    PP_TOKEN_RE = PP_TOKEN_RE_
+    ENDLINE_RE = ENDLINE_RE_
+    HEADER_NAME_RE = HEADER_NAME_RE_
+
+    line_num = 1
+    line_start = 0
+    pos = 0
+    limit = len(file_contents)
+    directive = None
+    at_bol = True
+    while pos < limit:
+        if directive == "include":
+            mo = HEADER_NAME_RE.match(file_contents, pos)
+            if mo:
+                kind = "HEADER_NAME"
+                directive = "after_include"
+            else:
+                mo = PP_TOKEN_RE.match(file_contents, pos)
+                kind = mo.lastgroup
+                if kind != "WHITESPACE":
+                    directive = "after_include"
+        else:
+            mo = PP_TOKEN_RE.match(file_contents, pos)
+            kind = mo.lastgroup
+
+        text = mo.group()
+        line = line_num
+        column = mo.start() - line_start
+        adj_line_start = 0
+        # only these kinds can contain a newline
+        if kind in ("WHITESPACE", "BLOCK_COMMENT", "LINE_COMMENT",
+                    "STRING", "CHARCONST", "BAD_BLOCK_COM", "ESCNL"):
+            for tmo in ENDLINE_RE.finditer(text):
+                line_num += 1
+                adj_line_start = tmo.end()
+            if adj_line_start:
+                line_start = mo.start() + adj_line_start
+
+        # Track whether or not we are scanning a preprocessing directive.
+        if kind == "LINE_COMMENT" or (kind == "WHITESPACE" and adj_line_start):
+            at_bol = True
+            directive = None
+        else:
+            if kind == "PUNCTUATOR" and text == "#" and at_bol:
+                directive = "<null>"
+            elif kind == "IDENT" and directive == "<null>":
+                directive = text
+            at_bol = False
+
+        # Report ill-formed tokens and rewrite them as their well-formed
+        # equivalents, so downstream processing doesn't have to know about them.
+        # (Rewriting instead of discarding provides better error recovery.)
+        if kind == "BAD_BLOCK_COM":
+            reporter.error(Token("BAD_BLOCK_COM", "", line, column+1, ""),
+                           "unclosed block comment")
+            text += "*/"
+            kind = "BLOCK_COMMENT"
+        elif kind == "BAD_STRING":
+            reporter.error(Token("BAD_STRING", "", line, column+1, ""),
+                           "unclosed string")
+            text += "\""
+            kind = "STRING"
+        elif kind == "BAD_CHARCONST":
+            reporter.error(Token("BAD_CHARCONST", "", line, column+1, ""),
+                           "unclosed char constant")
+            text += "'"
+            kind = "CHARCONST"
+
+        tok = Token(kind, text, line, column+1,
+                    "include" if directive == "after_include" else directive)
+        # Do not complain about OTHER tokens inside macro definitions.
+        # $ and @ appear in macros defined by headers intended to be
+        # included from assembly language, e.g. sysdeps/mips/sys/asm.h.
+        if kind == "OTHER" and directive != "define":
+            self.error(tok, "stray {!r} in program")
+
+        yield tok
+        pos = mo.end()