[C++ PATCH] simplify deferred parsing lexer

https://gcc.gnu.org/ml/gcc-patches/2019-10/msg01962.html

We use an eof_token global variable as a sentinel on a deferred parse
(such as in-class function definitions, or default args). This
complicates retrieving the next token in certain places.

As such deferred parses always nest properly and completely before
resuming the outer lexer, we can simply morph the token after the
deferred buffer into a CPP_EOF token and restore it afterwards. I
finally got around to implementing it with this patch.

One complication is that we have to change the discriminator for when
the token's value is a tree. We can't look at the token's type because
it might have been overwritten. I add a bool flag to the token
(there's several spare bits), and use that. This does simplify the
discriminator because we just check a single bit, rather than a set of
token types.

	* parser.h (struct cp_token): Drop {ENUM,BOOL}_BITFIELD C-ism.
	Add tree_check_p flag, use as nested union discriminator.
	(struct cp_lexer): Add saved_type & saved_keyword fields.
	* parser.c (eof_token): Delete.
	(cp_lexer_new_main): Always init last_token to last token of
	buffer.
	(cp_lexer_new_from_tokens): Overlay EOF token at end of range.
	(cp_lexer_destroy): Restore token under the EOF.
	(cp_lexer_previous_token_position): No check for eof_token here.
	(cp_lexer_get_preprocessor_token): Clear tree_check_p.
	(cp_lexer_peek_nth_token): Check CPP_EOF not eof_token.
	(cp_lexer_consume_token): Assert not CPP_EOF, no check for
	eof_token.
	(cp_lexer_purge_token): Likewise.
	(cp_lexer_purge_tokens_after): No check for EOF token.
	(cp_parser_nested_name_specifier, cp_parser_decltype)
	(cp_parser_template_id): Set tree_check_p.

From-SVN: r277514

1 files changed, 15 insertions, 11 deletions

diff --git a/gcc/cp/parser.h b/gcc/cp/parser.h
index 2004982..7fee244 100644
--- a/gcc/cp/parser.h
+++ b/gcc/cp/parser.h
@@ -41,34 +41,34 @@ struct GTY(()) tree_check {
 
 struct GTY (()) cp_token {
   /* The kind of token.  */
-  ENUM_BITFIELD (cpp_ttype) type : 8;
+  enum cpp_ttype type : 8;
   /* If this token is a keyword, this value indicates which keyword.
      Otherwise, this value is RID_MAX.  */
-  ENUM_BITFIELD (rid) keyword : 8;
+  enum rid keyword : 8;
   /* Token flags.  */
   unsigned char flags;
   /* True if this token is from a context where it is implicitly extern "C" */
-  BOOL_BITFIELD implicit_extern_c : 1;
+  bool implicit_extern_c : 1;
   /* True if an error has already been reported for this token, such as a
      CPP_NAME token that is not a keyword (i.e., for which KEYWORD is
      RID_MAX) iff this name was looked up and found to be ambiguous.  */
-  BOOL_BITFIELD error_reported : 1;
+  bool error_reported : 1;
   /* True for a token that has been purged.  If a token is purged,
      it is no longer a valid token and it should be considered
      deleted.  */
-  BOOL_BITFIELD purged_p : 1;
-  /* 5 unused bits.  */
+  bool purged_p : 1;
+  bool tree_check_p : 1;
+  /* 4 unused bits.  */
+
   /* The location at which this token was found.  */
   location_t location;
   /* The value associated with this token, if any.  */
   union cp_token_value {
     /* Used for compound tokens such as CPP_NESTED_NAME_SPECIFIER.  */
-    struct tree_check* GTY((tag ("1"))) tree_check_value;
+    struct tree_check* GTY((tag ("true"))) tree_check_value;
     /* Use for all other tokens.  */
-    tree GTY((tag ("0"))) value;
-  } GTY((desc ("(%1.type == CPP_TEMPLATE_ID)"
-	       "|| (%1.type == CPP_NESTED_NAME_SPECIFIER)"
-	       "|| (%1.type == CPP_DECLTYPE)"))) u;
+    tree GTY((tag ("false"))) value;
+  } GTY((desc ("%1.tree_check_p"))) u;
 };
 
 
@@ -99,6 +99,10 @@ struct GTY (()) cp_lexer {
      tokens.  */
   vec<cp_token_position> GTY ((skip)) saved_tokens;
 
+  /* Saved pieces of end token we replaced with the eof token.  */
+  enum cpp_ttype saved_type : 8;
+  enum rid saved_keyword : 8;
+
   /* The next lexer in a linked list of lexers.  */
   struct cp_lexer *next;