New macro expander.

2000-10-28  Neil Booth  <neilb@earthling.net>

	New macro expander.

	* cpplib.c (struct answer): New.
	(struct if_stack): Use cpp_lexer_pos rather than line and col.
	Rename cmacro mi_cmacro.
	(struct directive, KANDR, STDC89, EXTENSION, COND, IF_COND, INCL,
	IN_I): New directive and flags.
	(skip_rest_of_line, check_eol, run_directive, glue_header_name,
	parse_answer, parse_assertion, find_answer): New functions.
	(parse_ifdef, detect_if_not_defined, validate_else): Remove.
	(lex_macro_node): New function to replace parse_ifdef and
	get_define_node.

	(_cpp_handle_directive): New function, combines _cpp_check_directive
	and _cpp_check_linemarker.

	(do_define, do_undef, parse_include, do_include, do_import,
	do_include_next, read_line_number, do_line, do_ident, do_pragma,
	do_pragma_once, do_pragma_poison, do_pragma_dependency):
	Update for new token getting interface.

	(do_ifdef, do_ifndef, do_if, do_else, do_endif, push_conditional)
	: Update for new multiple-include optimisation technique.
	(do_elif): Don't forget to invalidate controlling macros.

	(unwind_if_stack, cpp_defined, cpp_push_buffer, cpp_pop_buffer): Update.
	(parse_assertion, parse_answer, find_answer, _cpp_test_assertion):
	Functions to handle assertions with the new token interface.
	(do_assert, do_unassert): Use them.

	(cpp_define, _cpp_define_builtin, cpp_undef, cpp_assert, cpp_unassert):
	Use run_directive.

	(_cpp_init_stacks): Register directive names.  Don't register special
	nodes.

	* cpperror.c (print_containing_files, _cpp_begin_message): Update to
	new position recording regime.
	(cpp_ice, cpp_fatal, cpp_error, cpp_error_with_line, cpp_warning,
	cpp_warning_with_line, cpp_pedwarn, cpp_pedwarn_with_line,
	cpp_pedwarn_with_file_and_line): Update for _cpp_begin_message changes.
	(cpp_type2name): Move to cpplex.c.

	* cppexp.c (parse_charconst): spec_nodes is no longer a pointer.
	(parse_defined): Update to handle new multiple include optimisation
	method.  Remove poisoned identifier warning.
	(parse_assertion, TYPE_NAME): Delete.
	(lex): Update for multiple include optimisation, removal of
	CPP_DEFINED, to use _cpp_test_assertion for assertions and
	cpp_token_as_text.
	(_cpp_parse_expr): Update for MI optimisation, and to use op_as_text.
	(op_as_text): New function, to wrap cpp_token_as_text.

	* cppfiles.c (stack_include_file, _cpp_pop_file_buffer):
	Update for MI optimisation.
	(_cpp_execute_include): Take a token rather than 3 arguments.  Fix
	segfault on diagnostic.
	(_cpp_compare_file_date): Take a token rather than 3 args.
	(cpp_read_file): Work correctly for zero-length files.

	* cpphash.c (_cpp_init_macros, _cpp_cleanup_macros): Rename
	_cpp_init_hashtable and _cpp_cleanup_hashtable.
	(cpp_lookup): Place identifiers at front of identifier pool
	for _cpp_lookup_with_hash.
	(_cpp_lookup_with_hash): Require identifiers to be at the front of
	the identifier pool.  Commit the memory if not already in the
	hash table.

	* cppinit.c (cpp_reader_init): Move cpp_init_completed test to top.
	Initialise various members of cpp_reader, memory pools, and the
	special nodes.
	(cpp_printer_init): Delete.
	(cpp_cleanup): Update.
	(struct builtin, builtin_array, initialize_builtins): Update for new
	hashnode definition and builtin handling.
	(cpp_start_read, cpp_finish): Don't take or initialise a
	printer.  Update.

	* cpplib.h (cpp_printer, cpp_toklist, CPP_DEFINED, BOL,
	PASTED, VAR_ARGS, BEG_OF_FILE, IN_DIRECTIVE, KNOWN_DIRECTIVE,
	T_VOID, T_SPECLINE, T_DATE, T_FILE, T_BASE_FILE, T_INCLUDE_LEVEL,
	T_TIME, T_STDC, T_OPERATOR, T_POISON, T_MACRO, T_ASSERTION): Delete.
	(struct cpp_pool, struct cpp_macro, struct cpp_lexer_pos,
	struct cpp_lookahead, CPP_DHASH, enum mi_state, enum mi_ind,
	NO_EXPAND, VARARGS_FIRST, struct cpp_token_with_pos,
	struct toklist, struct cpp_context, struct specnodes,
	TOKEN_LOOKAHEAD, TOKEN_BUFFSIZE, NODE_OPERATOR, NODE_POISONED,
	NODE_BUILTIN, NODE_DIAGNOSTIC, NT_VOID, NT_MACRO, NT_ASSERTION,
	enum builtin_type, cpp_can_paste): New.
	(struct cpp_token): Delete line and col members.
	(struct cpp_buffer): New member output_lineno.
	(struct lexer_state): Delete indented, in_lex_line, seen_dot.
	Add va_args_ok, poisoned_ok, prevent_expansion, parsing_args.
	(struct cpp_reader): New members lexer_pos, macro_pos, directive_pos,
	ident_pool, temp_string_pool, macro_pool, argument_pool, string_pool,
	base_context, context, directive, mi_state, mi_if_not_defined,
	mi_lexed, mi_cmacro, mi_ind_cmacro, la_read, la_write, la_unused,
	mlstring_pos, macro_buffer, macro_buffer_len.
	Delete members mls_line, mls_column, token_list, potential_control_macro,
	temp_tokens, temp_cap, temp_alloced, temp_used, first_directive_token,
	context_cap, cur_context, no_expand_level, paste_level, contexts, args,
	save_parameter_spellings, need_newline, .
	Change type of date, time and spec_nodes members.
	Change prototypes for include and ident callbacks.
	(struct cpp_hashnode): Change type of name.  Remove union members
	expansion and code.  Add members macro, operator and builtin.

	(cpp_token_len, cpp_token_as_text, cpp_spell_token, cpp_start_read,
	cpp_finish, cpp_avoid_paste, cpp_get_token, cpp_get_line,
	cpp_get_output_line, cpp_macro_definition, cpp_start_lookahead,
	cpp_stop_lookahead): New prototypes.
	(cpp_printer_init, cpp_dump_definition): Delete prototypes.

	(U_CHAR, U, ustrcmp, ustrncmp, ustrlen, uxstrdup, ustrchr, ufputs):
	Move from cpphash.h.

	* cpphash.h (U_CHAR, U, ustrcmp, ustrncmp, ustrlen, uxstrdup, ustrchr,
	ufputs): Move to cpplib.h.
	(enum spell_type, struct token_spelling, _cpp_token_spellings, TOKEN_SPELL,
	TOKEN_NAME, struct answer, FREE_ANSWER, KANDR, STDC89, EXTENSION,
	COND, EXPAND, INCL, COMMENTS, IN_I, struct directive, directive_handler,
	struct spec_nodes, _cpp_digraph_spellings, _cpp_free_temp_tokens,
	_cpp_init_input_buffer, _cpp_grow_token_buffer, _cpp_init_toklist,
	_cpp_clear_toklist, _cpp_expand_token_space, _cpp_expand_name_space,
	_cpp_equiv_tokens, _cpp_equiv_toklists, _cpp_process_directive,
	_cpp_run_directive, _cpp_get_line, _cpp_get_raw_token, _cpp_glue_header_name,
	_cpp_can_paste, _cpp_check_directive, _cpp_check_linemarker,
	_cpp_parse_assertion, _cpp_find_answer): Delete.
	(VALID_SIGN, ALIGN, POOL_FRONT, POOL_LIMIT, POOL_BASE, POOL_SIZE,
	POOL_USED, POOL_COMMIT, struct cpp_chunk, _cpp_lex_token, _cpp_init_pool,
	_cpp_free_pool, _cpp_pool_reserve, _cpp_pool_alloc, _cpp_next_chunk,
	_cpp_lock_pool, _cpp_unlock_pool, _cpp_test_assertion,
	_cpp_handle_directive, DSC): New.
	(struct include_file): New member defined.

	(DO_NOT_REREAD, _cpp_begin_message, _cpp_execute_include,
	_cpp_compare_file_date): Update.
	(_cpp_pop_context, _cpp_get_token, _cpp_free_lookaheads, _cpp_push_token): New.
	(_cpp_init_macros, _cpp_cleanup_macros): Rename to _cpp_init_hashtable,
	_cpp_cleanup_hashtable.

	* Makefile.in: Remove cppoutput.c.

	* cppoutput.c: Delete

	* fixheader.c (read_scan_file): Update for new cpp_get_token
	prototype.
	(recognized_function): New argument LINE.

	* scan-decls.c (skip_to_closing_brace, scan_decls): Update for
	new cpp_get_token prototype.

	* scan.h (recognized_function): Update prototype.

	* po/POTFILES.in: Remove cppoutput.c.

From-SVN: r37098

1 files changed, 519 insertions, 1881 deletions

diff --git a/gcc/cpplex.c b/gcc/cpplex.c
index 426e82d..fa737a9 100644
--- a/gcc/cpplex.c
+++ b/gcc/cpplex.c
@@ -20,79 +20,54 @@ You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
 
-/*
-
-Cleanups to do:-
-
-o Distinguish integers, floats, and 'other' pp-numbers.
-o Store ints and char constants as binary values.
-o New command-line assertion syntax.
-o Comment all functions, and describe macro expansion algorithm.
-o Move as much out of header files as possible.
-o Remove single quote pairs `', and some '', from diagnostics.
-o Correct pastability test for CPP_NAME and CPP_NUMBER.
-
-*/
+/* This lexer works with a single pass of the file.  Recently I
+   re-wrote it to minimize the places where we step backwards in the
+   input stream, to make future changes to support multi-byte
+   character sets fairly straight-forward.
+
+   There is now only one routine where we do step backwards:
+   skip_escaped_newlines.  This routine could probably also be changed
+   so that it doesn't need to step back.  One possibility is to use a
+   trick similar to that used in lex_period and lex_percent.  Two
+   extra characters might be needed, but skip_escaped_newlines itself
+   would probably be the only place that needs to be aware of that,
+   and changes to the remaining routines would probably only be needed
+   if they process a backslash.  */
 
 #include "config.h"
 #include "system.h"
-#include "intl.h"
 #include "cpplib.h"
 #include "cpphash.h"
 #include "symcat.h"
 
-const unsigned char *_cpp_digraph_spellings [] = {U"%:", U"%:%:", U"<:",
-						  U":>", U"<%", U"%>"};
-static const cpp_token placemarker_token = {0, 0, CPP_PLACEMARKER,
-					    0 UNION_INIT_ZERO};
-static const cpp_token eof_token = {0, 0, CPP_EOF, 0 UNION_INIT_ZERO};
-
-/* Flags for cpp_context.  */
-#define CONTEXT_PASTEL	(1 << 0) /* An argument context on LHS of ##.  */
-#define CONTEXT_PASTER	(1 << 1) /* An argument context on RHS of ##.  */
-#define CONTEXT_RAW	(1 << 2) /* If argument tokens already expanded.  */
-#define CONTEXT_ARG	(1 << 3) /* If an argument context.  */
-#define CONTEXT_VARARGS	(1 << 4) /* If a varargs argument context.  */
-
-typedef struct cpp_context cpp_context;
-struct cpp_context
+/* Tokens with SPELL_STRING store their spelling in the token list,
+   and it's length in the token->val.name.len.  */
+enum spell_type
 {
-  union
-  {
-    const cpp_toklist *list;	/* Used for macro contexts only.  */
-    const cpp_token **arg;	/* Used for arg contexts only.  */
-  } u;
-
-  /* Pushed token to be returned by next call to get_raw_token.  */
-  const cpp_token *pushed_token;
-
-  struct macro_args *args;	/* The arguments for a function-like
-				   macro.  NULL otherwise.  */
-  unsigned short posn;		/* Current posn, index into u.  */
-  unsigned short count;		/* No. of tokens in u.  */
-  unsigned short level;
-  unsigned char flags;
+  SPELL_OPERATOR = 0,
+  SPELL_CHAR,
+  SPELL_IDENT,
+  SPELL_STRING,
+  SPELL_NONE
 };
 
-typedef struct macro_args macro_args;
-struct macro_args
+struct token_spelling
 {
-  unsigned int *ends;
-  const cpp_token **tokens;
-  unsigned int capacity;
-  unsigned int used;
-  unsigned short level;
+  enum spell_type category;
+  const unsigned char *name;
 };
 
-static const cpp_token *get_raw_token PARAMS ((cpp_reader *));
-static const cpp_token *parse_arg PARAMS ((cpp_reader *, int, unsigned int,
-					   macro_args *, unsigned int *));
-static int parse_args PARAMS ((cpp_reader *, cpp_hashnode *, macro_args *));
-static void save_token PARAMS ((macro_args *, const cpp_token *));
-static int pop_context PARAMS ((cpp_reader *));
-static int push_macro_context PARAMS ((cpp_reader *, const cpp_token *));
-static void push_arg_context PARAMS ((cpp_reader *, const cpp_token *));
-static void free_macro_args PARAMS ((macro_args *));
+const unsigned char *digraph_spellings [] = {U"%:", U"%:%:", U"<:",
+					     U":>", U"<%", U"%>"};
+
+#define OP(e, s) { SPELL_OPERATOR, U s           },
+#define TK(e, s) { s,              U STRINGX (e) },
+const struct token_spelling token_spellings [N_TTYPES] = {TTYPE_TABLE };
+#undef OP
+#undef TK
+
+#define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
+#define TOKEN_NAME(token) (token_spellings[(token)->type].name)
 
 static cppchar_t handle_newline PARAMS ((cpp_buffer *, cppchar_t));
 static cppchar_t skip_escaped_newlines PARAMS ((cpp_buffer *, cppchar_t));
@@ -103,278 +78,18 @@ static int skip_line_comment PARAMS ((cpp_reader *));
 static void adjust_column PARAMS ((cpp_reader *));
 static void skip_whitespace PARAMS ((cpp_reader *, cppchar_t));
 static cpp_hashnode *parse_identifier PARAMS ((cpp_reader *, cppchar_t));
-static void parse_number PARAMS ((cpp_reader *, cpp_string *, cppchar_t));
+static void parse_number PARAMS ((cpp_reader *, cpp_string *, cppchar_t, int));
+static int unescaped_terminator_p PARAMS ((cpp_reader *, const U_CHAR *));
 static void parse_string PARAMS ((cpp_reader *, cpp_token *, cppchar_t));
-static void unterminated PARAMS ((cpp_reader *, unsigned int, int));
+static void unterminated PARAMS ((cpp_reader *, int));
 static int trigraph_ok PARAMS ((cpp_reader *, cppchar_t));
 static void save_comment PARAMS ((cpp_reader *, cpp_token *, const U_CHAR *));
 static void lex_percent PARAMS ((cpp_buffer *, cpp_token *));
 static void lex_dot PARAMS ((cpp_reader *, cpp_token *));
-static void lex_line PARAMS ((cpp_reader *, cpp_toklist *));
-static void lex_token PARAMS ((cpp_reader *, cpp_token *));
-static int lex_next PARAMS ((cpp_reader *, int));
-
-static int is_macro_disabled PARAMS ((cpp_reader *, const cpp_toklist *,
-				      const cpp_token *));
-
-static cpp_token *stringify_arg PARAMS ((cpp_reader *, const cpp_token *));
-static void expand_context_stack PARAMS ((cpp_reader *));
-static unsigned char * spell_token PARAMS ((cpp_reader *, const cpp_token *,
-					    unsigned char *));
-typedef unsigned int (* speller) PARAMS ((unsigned char *, cpp_toklist *,
-					  cpp_token *));
-static cpp_token *make_string_token PARAMS ((cpp_token *, const U_CHAR *,
-					    unsigned int));
-static cpp_token *alloc_number_token PARAMS ((cpp_reader *, int number));
-static const cpp_token *special_symbol PARAMS ((cpp_reader *, cpp_hashnode *,
-						const cpp_token *));
-static cpp_token *duplicate_token PARAMS ((cpp_reader *, const cpp_token *));
-static const cpp_token *maybe_paste_with_next PARAMS ((cpp_reader *,
-						       const cpp_token *));
-static unsigned int prevent_macro_expansion	PARAMS ((cpp_reader *));
-static void restore_macro_expansion	PARAMS ((cpp_reader *, unsigned int));
-static cpp_token *get_temp_token	PARAMS ((cpp_reader *));
-static void release_temp_tokens		PARAMS ((cpp_reader *));
-static U_CHAR * quote_string PARAMS ((U_CHAR *, const U_CHAR *, unsigned int));
-
-#define VALID_SIGN(c, prevc) \
-  (((c) == '+' || (c) == '-') && \
-   ((prevc) == 'e' || (prevc) == 'E' \
-    || (((prevc) == 'p' || (prevc) == 'P') && !CPP_OPTION (pfile, c89))))
-
-/* An upper bound on the number of bytes needed to spell a token,
-   including preceding whitespace.  */
-static inline size_t TOKEN_LEN PARAMS ((const cpp_token *));
-static inline size_t
-TOKEN_LEN (token)
-     const cpp_token *token;
-{
-  size_t len;
-
-  switch (TOKEN_SPELL (token))
-    {
-    default:		len = 0;			break;
-    case SPELL_STRING:	len = token->val.str.len;	break;
-    case SPELL_IDENT:	len = token->val.node->length;	break;
-    }
-  return len + 5;
-}
-
-#define IS_ARG_CONTEXT(c) ((c)->flags & CONTEXT_ARG)
-#define CURRENT_CONTEXT(pfile) ((pfile)->contexts + (pfile)->cur_context)
-
-#define ASSIGN_FLAGS_AND_POS(d, s) \
-  do {(d)->flags = (s)->flags & (PREV_WHITE | BOL | PASTE_LEFT); \
-      if ((d)->flags & BOL) {(d)->col = (s)->col; (d)->line = (s)->line;} \
-  } while (0)
-
-/* f is flags, just consisting of PREV_WHITE | BOL.  */
-#define MODIFY_FLAGS_AND_POS(d, s, f) \
-  do {(d)->flags &= ~(PREV_WHITE | BOL); (d)->flags |= (f); \
-      if ((f) & BOL) {(d)->col = (s)->col; (d)->line = (s)->line;} \
-  } while (0)
-
-#define OP(e, s) { SPELL_OPERATOR, U s           },
-#define TK(e, s) { s,              U STRINGX (e) },
-
-const struct token_spelling
-_cpp_token_spellings [N_TTYPES] = {TTYPE_TABLE };
-
-#undef OP
-#undef TK
-
-/* Helper routine used by parse_include, which can't see spell_token.
-   Reinterpret the current line as an h-char-sequence (< ... >); we are
-   looking at the first token after the <.  */
-const cpp_token *
-_cpp_glue_header_name (pfile)
-     cpp_reader *pfile;
-{
-  const cpp_token *t;
-  cpp_token *hdr;
-  U_CHAR *buf, *p;
-  size_t len, avail;
-
-  avail = 40;
-  len = 0;
-  buf = xmalloc (avail);
-
-  for (;;)
-    {
-      t = _cpp_get_token (pfile);
-      if (t->type == CPP_GREATER || t->type == CPP_EOF)
-	break;
-
-      if (len + TOKEN_LEN (t) > avail)
-	{
-	  avail = len + TOKEN_LEN (t) + 40;
-	  buf = xrealloc (buf, avail);
-	}
-
-      if (t->flags & PREV_WHITE)
-	buf[len++] = ' ';
-
-      p = spell_token (pfile, t, buf + len);
-      len = (size_t) (p - buf);  /* p known >= buf */
-    }
-
-  if (t->type == CPP_EOF)
-    cpp_error (pfile, "missing terminating > character");
-
-  buf = xrealloc (buf, len);
-
-  hdr = get_temp_token (pfile);
-  hdr->type = CPP_HEADER_NAME;
-  hdr->flags = 0;
-  hdr->val.str.text = buf;
-  hdr->val.str.len = len;
-  return hdr;
-}
-
-/* Token-buffer helper functions.  */
-
-/* Expand a token list's string space. It is *vital* that
-   list->tokens_used is correct, to get pointer fix-up right.  */
-void
-_cpp_expand_name_space (list, len)
-     cpp_toklist *list;
-     unsigned int len;
-{
-  const U_CHAR *old_namebuf;
-
-  old_namebuf = list->namebuf;
-  list->name_cap += len;
-  list->namebuf = (unsigned char *) xrealloc (list->namebuf, list->name_cap);
-
-  /* Fix up token text pointers.  */
-  if (list->namebuf != old_namebuf)
-    {
-      unsigned int i;
-
-      for (i = 0; i < list->tokens_used; i++)
-	if (TOKEN_SPELL (&list->tokens[i]) == SPELL_STRING)
-	  list->tokens[i].val.str.text += (list->namebuf - old_namebuf);
-    }
-}
-
-/* If there is not enough room for LEN more characters, expand the
-   list by just enough to have room for LEN characters.  */
-void
-_cpp_reserve_name_space (list, len)
-     cpp_toklist *list;
-     unsigned int len;
-{
-  unsigned int room = list->name_cap - list->name_used;
-
-  if (room < len)
-    _cpp_expand_name_space (list, len - room);
-}
-
-/* Expand the number of tokens in a list.  */
-void
-_cpp_expand_token_space (list, count)
-     cpp_toklist *list;
-     unsigned int count;
-{
-  list->tokens_cap += count;
-  list->tokens = (cpp_token *)
-    xrealloc (list->tokens, list->tokens_cap * sizeof (cpp_token));
-}
-
-/* Initialize a token list.  If EMPTY is false, some token and name
-   space is provided.  */
-void
-_cpp_init_toklist (list, empty)
-     cpp_toklist *list;
-     int empty;
-{
-  if (empty)
-    {
-      list->tokens_cap = 0;
-      list->tokens = 0;
-      list->name_cap = 0;
-      list->namebuf = 0;
-    }
-  else
-    {
-      /* Initialize token space.  */
-      list->tokens_cap = 256;	/* 4K's worth.  */
-      list->tokens = (cpp_token *)
-	xmalloc ((list->tokens_cap + 1) * sizeof (cpp_token));
-
-      /* Initialize name space.  */
-      list->name_cap = 1024;
-      list->namebuf = (unsigned char *) xmalloc (list->name_cap);
-    }
-
-  _cpp_clear_toklist (list);
-}
-
-/* Clear a token list.  */
-void
-_cpp_clear_toklist (list)
-     cpp_toklist *list;
-{
-  list->tokens_used = 0;
-  list->name_used = 0;
-  list->directive = 0;
-  list->paramc = 0;
-  list->params_len = 0;
-  list->flags = 0;
-}
-
-/* Free a token list.  Does not free the list itself, which may be
-   embedded in a larger structure.  */
-void
-_cpp_free_toklist (list)
-     const cpp_toklist *list;
-{
-  free (list->tokens);
-  free (list->namebuf);
-}
-
-/* Compare two tokens.  */
-int
-_cpp_equiv_tokens (a, b)
-     const cpp_token *a, *b;
-{
-  if (a->type == b->type && a->flags == b->flags)
-    switch (TOKEN_SPELL (a))
-      {
-      default:			/* Keep compiler happy.  */
-      case SPELL_OPERATOR:
-	return 1;
-      case SPELL_CHAR:
-      case SPELL_NONE:
-	return a->val.aux == b->val.aux; /* arg_no or character.  */
-      case SPELL_IDENT:
-	return a->val.node == b->val.node;
-      case SPELL_STRING:
-	return (a->val.str.len == b->val.str.len
-		&& !memcmp (a->val.str.text, b->val.str.text,
-			    a->val.str.len));
-      }
-
-  return 0;
-}
-
-/* Compare two token lists.  */
-int
-_cpp_equiv_toklists (a, b)
-     const cpp_toklist *a, *b;
-{
-  unsigned int i;
-
-  if (a->tokens_used != b->tokens_used
-      || a->flags != b->flags
-      || a->paramc != b->paramc)
-    return 0;
+static int name_p PARAMS ((cpp_reader *, const cpp_string *));
 
-  for (i = 0; i < a->tokens_used; i++)
-    if (! _cpp_equiv_tokens (&a->tokens[i], &b->tokens[i]))
-      return 0;
-  return 1;
-}
+static cpp_chunk *new_chunk PARAMS ((unsigned int));
+static int chunk_suitable PARAMS ((cpp_pool *, cpp_chunk *, unsigned int));
 
 /* Utility routine:
 
@@ -389,7 +104,7 @@ cpp_ideq (token, string)
   if (token->type != CPP_NAME)
     return 0;
 
-  return !ustrcmp (token->val.node->name, (const U_CHAR *)string);
+  return !ustrcmp (token->val.node->name, (const U_CHAR *) string);
 }
 
 /* Call when meeting a newline.  Returns the character after the newline
@@ -578,7 +293,7 @@ skip_block_comment (pfile)
 
     next_char:
       /* FIXME: For speed, create a new character class of characters
-	 of no interest inside block comments.  */
+	 of interest inside block comments.  */
       if (c == '?' || c == '\\')
 	c = skip_escaped_newlines (buffer, c);
 
@@ -692,7 +407,7 @@ skip_whitespace (pfile, c)
 	      warned = 1;
 	    }
 	}
-      else if (IN_DIRECTIVE (pfile) && CPP_PEDANTIC (pfile))
+      else if (pfile->state.in_directive && CPP_PEDANTIC (pfile))
 	cpp_pedwarn_with_line (pfile, CPP_BUF_LINE (buffer),
 			       CPP_BUF_COL (buffer),
 			       "%s in preprocessing directive",
@@ -710,6 +425,22 @@ skip_whitespace (pfile, c)
   buffer->read_ahead = c;
 }
 
+/* See if the characters of a number token are valid in a name (no
+   '.', '+' or '-').  */
+static int
+name_p (pfile, string)
+     cpp_reader *pfile;
+     const cpp_string *string;
+{
+  unsigned int i;
+
+  for (i = 0; i < string->len; i++)
+    if (!is_idchar (string->text[i]))
+      return 0;
+
+  return 1;  
+}
+
 /* Parse an identifier, skipping embedded backslash-newlines.
    Calculate the hash value of the token while parsing, for improved
    performance.  The hashing algorithm *must* match cpp_lookup().  */
@@ -719,18 +450,23 @@ parse_identifier (pfile, c)
      cpp_reader *pfile;
      cppchar_t c;
 {
+  cpp_hashnode *result;
   cpp_buffer *buffer = pfile->buffer;
+  unsigned char *dest, *limit;
   unsigned int r = 0, saw_dollar = 0;
-  unsigned int orig_used = pfile->token_list.name_used;
+
+  dest = POOL_FRONT (&pfile->ident_pool);
+  limit = POOL_LIMIT (&pfile->ident_pool);
 
   do
     {
       do
 	{
-	  if (pfile->token_list.name_used == pfile->token_list.name_cap)
-	    _cpp_expand_name_space (&pfile->token_list,
-				    pfile->token_list.name_used + 256);
-	  pfile->token_list.namebuf[pfile->token_list.name_used++] = c;
+	  /* Need room for terminating null.  */
+	  if (dest + 1 >= limit)
+	    limit = _cpp_next_chunk (&pfile->ident_pool, 0, &dest);
+
+	  *dest++ = c;
 	  r = HASHSTEP (r, c);
 
 	  if (c == '$')
@@ -751,86 +487,137 @@ parse_identifier (pfile, c)
     }
   while (is_idchar (c));
 
+  /* Remember the next character.  */
+  buffer->read_ahead = c;
+
   /* $ is not a identifier character in the standard, but is commonly
      accepted as an extension.  Don't warn about it in skipped
      conditional blocks.  */
   if (saw_dollar && CPP_PEDANTIC (pfile) && ! pfile->skipping)
     cpp_pedwarn (pfile, "'$' character(s) in identifier");
 
-  /* Remember the next character.  */
-  buffer->read_ahead = c;
-  return _cpp_lookup_with_hash (pfile, &pfile->token_list.namebuf[orig_used],
-				pfile->token_list.name_used - orig_used, r);
+  /* Identifiers are null-terminated.  */
+  *dest = '\0';
+
+  /* This routine commits the memory if necessary.  */
+  result = _cpp_lookup_with_hash (pfile,
+				  dest - POOL_FRONT (&pfile->ident_pool), r);
+
+  /* Some identifiers require diagnostics when lexed.  */
+  if (result->flags & NODE_DIAGNOSTIC && !pfile->skipping)
+    {
+      /* It is allowed to poison the same identifier twice.  */
+      if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
+	cpp_error (pfile, "attempt to use poisoned \"%s\"", result->name);
+
+      /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
+	 replacement list of a variable-arguments macro.  */
+      if (result == pfile->spec_nodes.n__VA_ARGS__
+	  && !pfile->state.va_args_ok)
+	cpp_pedwarn (pfile, "__VA_ARGS__ can only appear in the expansion of a C99 variable-argument macro");
+    }
+
+  return result;
 }
 
 /* Parse a number, skipping embedded backslash-newlines.  */
 static void
-parse_number (pfile, number, c)
+parse_number (pfile, number, c, leading_period)
      cpp_reader *pfile;
      cpp_string *number;
      cppchar_t c;
+     int leading_period;
 {
-  cppchar_t prevc;
   cpp_buffer *buffer = pfile->buffer;
-  unsigned int orig_used = pfile->token_list.name_used;
+  cpp_pool *pool = pfile->string_pool;
+  unsigned char *dest, *limit;
 
-  /* Reserve space for a leading period.  */
-  if (pfile->state.seen_dot)
-    pfile->token_list.name_used++;
+  dest = POOL_FRONT (pool);
+  limit = POOL_LIMIT (pool);
 
+  /* Place a leading period.  */
+  if (leading_period)
+    {
+      if (dest >= limit)
+	limit = _cpp_next_chunk (pool, 0, &dest);
+      *dest++ = '.';
+    }
+  
   do
     {
       do
 	{
-	  if (pfile->token_list.name_used >= pfile->token_list.name_cap)
-	    _cpp_expand_name_space (&pfile->token_list,
-				    pfile->token_list.name_used + 256);
-	  pfile->token_list.namebuf[pfile->token_list.name_used++] = c;
+	  /* Need room for terminating null.  */
+	  if (dest + 1 >= limit)
+	    limit = _cpp_next_chunk (pool, 0, &dest);
+	  *dest++ = c;
 
-	  prevc = c;
 	  c = EOF;
 	  if (buffer->cur == buffer->rlimit)
 	    break;
 
 	  c = *buffer->cur++;
 	}
-      while (is_numchar (c) || c == '.' || VALID_SIGN (c, prevc));
+      while (is_numchar (c) || c == '.' || VALID_SIGN (c, dest[-1]));
 
       /* Potential escaped newline?  */
       if (c != '?' && c != '\\')
 	break;
       c = skip_escaped_newlines (buffer, c);
     }
-  while (is_numchar (c) || c == '.' || VALID_SIGN (c, prevc));
-
-  /* Put any leading period in place, now we have the room.  */
-  if (pfile->state.seen_dot)
-    pfile->token_list.namebuf[orig_used] = '.';
+  while (is_numchar (c) || c == '.' || VALID_SIGN (c, dest[-1]));
 
   /* Remember the next character.  */
   buffer->read_ahead = c;
 
-  number->text = &pfile->token_list.namebuf[orig_used];
-  number->len = pfile->token_list.name_used - orig_used;
+  /* Null-terminate the number.  */
+  *dest = '\0';
+
+  number->text = POOL_FRONT (pool);
+  number->len = dest - number->text;
+  POOL_COMMIT (pool, number->len + 1);
 }
 
 /* Subroutine of parse_string.  Emits error for unterminated strings.  */
 static void
-unterminated (pfile, line, term)
+unterminated (pfile, term)
      cpp_reader *pfile;
-     unsigned int line;
      int term;
 {
   cpp_error (pfile, "missing terminating %c character", term);
 
-  if (term == '\"' && pfile->mls_line && pfile->mls_line != line)
+  if (term == '\"' && pfile->mlstring_pos.line
+      && pfile->mlstring_pos.line != pfile->lexer_pos.line)
     {
-      cpp_error_with_line (pfile, pfile->mls_line, pfile->mls_column,
+      cpp_error_with_line (pfile, pfile->mlstring_pos.line,
+			   pfile->mlstring_pos.col,
 			   "possible start of unterminated string literal");
-      pfile->mls_line = 0;
+      pfile->mlstring_pos.line = 0;
     }
 }
 
+/* Subroutine of parse_string.  */
+static int
+unescaped_terminator_p (pfile, dest)
+     cpp_reader *pfile;
+     const unsigned char *dest;
+{
+  const unsigned char *start, *temp;
+
+  /* In #include-style directives, terminators are not escapeable.  */
+  if (pfile->state.angled_headers)
+    return 1;
+
+  start = POOL_FRONT (pfile->string_pool);
+
+  /* An odd number of consecutive backslashes represents an escaped
+     terminator.  */
+  for (temp = dest; temp > start && temp[-1] == '\\'; temp--)
+    ;
+
+  return ((dest - temp) & 1) == 0;
+}
+
 /* Parses a string, character constant, or angle-bracketed header file
    name.  Handles embedded trigraphs and escaped newlines.
 
@@ -843,16 +630,20 @@ parse_string (pfile, token, terminator)
      cppchar_t terminator;
 {
   cpp_buffer *buffer = pfile->buffer;
-  unsigned int orig_used = pfile->token_list.name_used;
+  cpp_pool *pool = pfile->string_pool;
+  unsigned char *dest, *limit;
   cppchar_t c;
   unsigned int nulls = 0;
 
+  dest = POOL_FRONT (pool);
+  limit = POOL_LIMIT (pool);
+
   for (;;)
     {
       if (buffer->cur == buffer->rlimit)
 	{
 	  c = EOF;
-	  unterminated (pfile, token->line, terminator);
+	  unterminated (pfile, terminator);
 	  break;
 	}
       c = *buffer->cur++;
@@ -862,20 +653,10 @@ parse_string (pfile, token, terminator)
       if (c == '?' || c == '\\')
 	c = skip_escaped_newlines (buffer, c);
 
-      if (c == terminator)
+      if (c == terminator && unescaped_terminator_p (pfile, dest))
 	{
-	  unsigned int u = pfile->token_list.name_used;
-
-	  /* An odd number of consecutive backslashes represents an
-	     escaped terminator.  */
-	  while (u > orig_used && pfile->token_list.namebuf[u - 1] == '\\')
-	    u--;
-
-	  if ((pfile->token_list.name_used - u) % 2 == 0)
-	    {
-	      c = EOF;
-	      break;
-	    }
+	  c = EOF;
+	  break;
 	}
       else if (is_vspace (c))
 	{
@@ -888,18 +669,16 @@ parse_string (pfile, token, terminator)
 	  /* Character constants and header names may not extend over
 	     multiple lines.  In Standard C, neither may strings.
 	     Unfortunately, we accept multiline strings as an
-	     extension.  (Deprecatedly even in directives - otherwise,
-	     glibc's longlong.h breaks.)  */
+	     extension.  */
 	  if (terminator != '"')
 	    {
-	      unterminated (pfile, token->line, terminator);
+	      unterminated (pfile, terminator);
 	      break;
 	    }
 
-	  if (pfile->mls_line == 0)
+	  if (pfile->mlstring_pos.line == 0)
 	    {
-	      pfile->mls_line = token->line;
-	      pfile->mls_column = token->col;
+	      pfile->mlstring_pos = pfile->lexer_pos;
 	      if (CPP_PEDANTIC (pfile))
 		cpp_pedwarn (pfile, "multi-line string constant");
 	    }
@@ -913,11 +692,11 @@ parse_string (pfile, token, terminator)
 	    cpp_warning (pfile, "null character(s) preserved in literal");
 	}
 
-      if (pfile->token_list.name_used == pfile->token_list.name_cap)
-	_cpp_expand_name_space (&pfile->token_list,
-				pfile->token_list.name_used + 256);
+      /* No terminating null for strings - they could contain nulls.  */
+      if (dest >= limit)
+	limit = _cpp_next_chunk (pool, 0, &dest);
+      *dest++ = c;
 
-      pfile->token_list.namebuf[pfile->token_list.name_used++] = c;
       /* If we had a new line, the next character is in read_ahead.  */
       if (c != '\n')
 	continue;
@@ -926,14 +705,15 @@ parse_string (pfile, token, terminator)
 	goto have_char;
     }
 
+  /* Remember the next character.  */
   buffer->read_ahead = c;
 
-  token->val.str.text = &pfile->token_list.namebuf[orig_used];
-  token->val.str.len = pfile->token_list.name_used - orig_used;
+  token->val.str.text = POOL_FRONT (pool);
+  token->val.str.len = dest - token->val.str.text;
+  POOL_COMMIT (pool, token->val.str.len);
 }
 
-/* For output routine simplicity, the stored comment includes the
-   comment start and any terminator.  */
+/* The stored comment includes the comment start and any terminator.  */
 static void
 save_comment (pfile, token, from)
      cpp_reader *pfile;
@@ -942,12 +722,9 @@ save_comment (pfile, token, from)
 {
   unsigned char *buffer;
   unsigned int len;
-  cpp_toklist *list = &pfile->token_list;
   
   len = pfile->buffer->cur - from + 1; /* + 1 for the initial '/'.  */
-  _cpp_reserve_name_space (list, len);
-  buffer = list->namebuf + list->name_used;
-  list->name_used += len;
+  buffer = _cpp_pool_alloc (pfile->string_pool, len);
   
   token->type = CPP_COMMENT;
   token->val.str.len = len;
@@ -1029,9 +806,7 @@ lex_dot (pfile, result)
   if (c >= '0' && c <= '9')
     {
       result->type = CPP_NUMBER;
-      buffer->pfile->state.seen_dot = 1;
-      parse_number (pfile, &result->val.str, c);
-      buffer->pfile->state.seen_dot = 0;
+      parse_number (pfile, &result->val.str, c, 1);
     }
   else
     {
@@ -1053,26 +828,29 @@ lex_dot (pfile, result)
     }
 }
 
-static void
-lex_token (pfile, result)
+void
+_cpp_lex_token (pfile, result)
      cpp_reader *pfile;
      cpp_token *result;
 {
   cppchar_t c;
   cpp_buffer *buffer = pfile->buffer;
   const unsigned char *comment_start;
+  unsigned char was_skip_newlines = pfile->state.skip_newlines;
+  unsigned char newline_in_args = 0;
 
+  pfile->state.skip_newlines = 0;
   result->flags = 0;
  next_char:
-  result->line = CPP_BUF_LINE (buffer);
+  pfile->lexer_pos.line = buffer->lineno;
  next_char2:
-  result->col = CPP_BUF_COLUMN (buffer, buffer->cur);
+  pfile->lexer_pos.col = CPP_BUF_COLUMN (buffer, buffer->cur);
 
   c = buffer->read_ahead;
   if (c == EOF && buffer->cur < buffer->rlimit)
     {
       c = *buffer->cur++;
-      result->col++;
+      pfile->lexer_pos.col++;
     }
 
  do_switch:
@@ -1080,12 +858,11 @@ lex_token (pfile, result)
   switch (c)
     {
     case EOF:
-      /* Non-empty files should end in a newline.  Testing
-         skip_newlines ensures we only emit the warning once.  */
-      if (buffer->cur != buffer->line_base && buffer->cur != buffer->buf
-	  && pfile->state.skip_newlines)
-	cpp_pedwarn_with_line (pfile, buffer->lineno, CPP_BUF_COL (buffer),
-			       "no newline at end of file");
+      /* Non-empty files should end in a newline.  Ignore for command
+	 line - we get e.g. -A options with no trailing \n.  */
+      if (pfile->lexer_pos.col != 0 && pfile->done_initializing)
+	cpp_pedwarn (pfile, "no newline at end of file");
+      pfile->state.skip_newlines = 1;
       result->type = CPP_EOF;
       break;
 
@@ -1095,15 +872,35 @@ lex_token (pfile, result)
       goto next_char2;
 
     case '\n': case '\r':
-      result->type = CPP_EOF;
-      handle_newline (buffer, c);
-      /* Handling here will change significantly when moving to
-	 token-at-a-time.  */
-      if (pfile->state.skip_newlines)
+      /* Don't let directives spill over to the next line.  */
+      if (pfile->state.in_directive)
+	buffer->read_ahead = c;
+      else
 	{
-	  result->flags &= ~PREV_WHITE; /* Clear any whitespace flag.   */
-	  goto next_char;
+	  handle_newline (buffer, c);
+
+	  pfile->lexer_pos.output_line = buffer->lineno;
+
+	  /* Skip newlines in macro arguments (except in directives).  */
+	  if (pfile->state.parsing_args)
+	    {
+	      /* Set the whitespace flag.   */
+	      newline_in_args = 1;
+	      result->flags |= PREV_WHITE;
+	      goto next_char;
+	    }
+
+	  if (was_skip_newlines)
+	    {
+	      /* Clear any whitespace flag.   */
+	      result->flags &= ~PREV_WHITE;
+	      goto next_char;
+	    }
 	}
+
+      /* Next we're at BOL, so skip new lines.  */
+      pfile->state.skip_newlines = 1;
+      result->type = CPP_EOF;
       break;
 
     case '?':
@@ -1133,7 +930,7 @@ lex_token (pfile, result)
     case '0': case '1': case '2': case '3': case '4':
     case '5': case '6': case '7': case '8': case '9':
       result->type = CPP_NUMBER;
-      parse_number (pfile, &result->val.str, c);
+      parse_number (pfile, &result->val.str, c, 0);
       break;
 
     case '$':
@@ -1156,7 +953,7 @@ lex_token (pfile, result)
       result->val.node = parse_identifier (pfile, c);
 
       /* 'L' may introduce wide characters or strings.  */
-      if (result->val.node == pfile->spec_nodes->n_L)
+      if (result->val.node == pfile->spec_nodes.n_L)
 	{
 	  c = buffer->read_ahead; /* For make_string.  */
 	  if (c == '\'' || c == '"')
@@ -1166,10 +963,10 @@ lex_token (pfile, result)
 	    }
 	}
       /* Convert named operators to their proper types.  */
-      else if (result->val.node->type == T_OPERATOR)
+      else if (result->val.node->flags & NODE_OPERATOR)
 	{
 	  result->flags |= NAMED_OP;
-	  result->type = result->val.node->value.code;
+	  result->type = result->val.node->value.operator;
 	}
       break;
 
@@ -1193,7 +990,8 @@ lex_token (pfile, result)
       if (c == '*')
 	{
 	  if (skip_block_comment (pfile))
-	    cpp_error_with_line (pfile, result->line, result->col,
+	    cpp_error_with_line (pfile, pfile->lexer_pos.line,
+				 pfile->lexer_pos.col,
 				 "unterminated comment");
 	}
       else
@@ -1218,7 +1016,8 @@ lex_token (pfile, result)
 
 	  /* Skip_line_comment updates buffer->read_ahead.  */
 	  if (skip_line_comment (pfile))
-	    cpp_warning_with_line (pfile, result->line, result->col,
+	    cpp_warning_with_line (pfile, pfile->lexer_pos.line,
+				   pfile->lexer_pos.col,
 				   "multi-line comment");
 	}
 
@@ -1290,6 +1089,8 @@ lex_token (pfile, result)
 
     case '%':
       lex_percent (buffer, result);
+      if (result->type == CPP_HASH)
+	goto do_hash;
       break;
 
     case '.':
@@ -1349,9 +1150,21 @@ lex_token (pfile, result)
       break;
 	  
     case '#':
-      result->type = CPP_HASH;
       if (get_effective_char (buffer) == '#')
 	ACCEPT_CHAR (CPP_PASTE);
+      else
+	{
+	  result->type = CPP_HASH;
+	do_hash:
+	  /* CPP_DHASH is the hash introducing a directive.  */
+	  if (was_skip_newlines || newline_in_args)
+	    {
+	      result->type = CPP_DHASH;
+	      /* Get whitespace right - newline_in_args sets it.  */
+	      if (pfile->lexer_pos.col == 1)
+		result->flags &= ~PREV_WHITE;
+	    }
+	}
       break;
 
     case '|':
@@ -1423,117 +1236,30 @@ lex_token (pfile, result)
     }
 }
 
-/*
- *  The tokenizer's main loop.  Returns a token list, representing a
- *  logical line in the input file.  On EOF after some tokens have
- *  been processed, we return immediately.  Then in next call, or if
- *  EOF occurred at the beginning of a logical line, a single CPP_EOF
- *  token is placed in the list.
- */
-
-static void
-lex_line (pfile, list)
-     cpp_reader *pfile;
-     cpp_toklist *list;
+/* An upper bound on the number of bytes needed to spell a token,
+   including preceding whitespace.  */
+unsigned int
+cpp_token_len (token)
+     const cpp_token *token;
 {
-  unsigned int first_token;
-  cpp_token *cur_token, *first;
-  cpp_buffer *buffer = pfile->buffer;
-
-  pfile->state.in_lex_line = 1;
-  if (pfile->buffer->cur == pfile->buffer->buf)
-    list->flags |= BEG_OF_FILE;
-
- retry:
-  pfile->state.in_directive = 0;
-  pfile->state.angled_headers = 0;
-  pfile->state.skip_newlines = 1;
-  pfile->state.save_comments = ! CPP_OPTION (pfile, discard_comments);
-  first_token = list->tokens_used;
-  list->file = buffer->nominal_fname;
-
-  do
-    {
-      if (list->tokens_used >= list->tokens_cap)
-	_cpp_expand_token_space (list, 256);
-
-      cur_token = list->tokens + list->tokens_used;
-      lex_token (pfile, cur_token);
-
-      if (pfile->state.skip_newlines)
-	{
-	  pfile->state.skip_newlines = 0;
-	  list->line = buffer->lineno;
-	  if (cur_token->type == CPP_HASH)
-	    {
-	      pfile->state.in_directive = 1;
-	      pfile->state.save_comments = 0;
-	      pfile->state.indented = cur_token->flags & PREV_WHITE;
-	    }
-	  /* 6.10.3.10: Within the sequence of preprocessing tokens
-	     making up the invocation of a function-like macro, new
-	     line is considered a normal white-space character.  */
-	  else if (first_token != 0)
-	    cur_token->flags |= PREV_WHITE;
-	}
-      else if (IN_DIRECTIVE (pfile) && list->tokens_used == first_token + 1)
-	{
-	  if (cur_token->type == CPP_NUMBER)
-	    list->directive = _cpp_check_linemarker (pfile, cur_token);
-	  else
-	    list->directive = _cpp_check_directive (pfile, cur_token);
-	}
+  unsigned int len;
 
-      /* _cpp_get_line assumes list->tokens_used refers to the current
-	 token being lexed.  So do this after _cpp_check_directive to
-	 get the warnings therein correct.  */
-      list->tokens_used++;
-    }
-  while (cur_token->type != CPP_EOF);
-
-  /* All tokens are allocated, so the memory location is fixed.  */
-  first = &list->tokens[first_token];
-  first->flags |= BOL;
-  pfile->first_directive_token = first;
-
-  /* Don't complain about the null directive, nor directives in
-     assembly source: we don't know where the comments are, and # may
-     introduce assembler pseudo-ops.  Don't complain about invalid
-     directives in skipped conditional groups (6.10 p4).  */
-  if (IN_DIRECTIVE (pfile) && !KNOWN_DIRECTIVE (list) && !pfile->skipping
-      && !CPP_OPTION (pfile, lang_asm))
+  switch (TOKEN_SPELL (token))
     {
-      if (cur_token > first + 1)
-	{
-	  if (first[1].type == CPP_NAME)
-	    cpp_error_with_line (pfile, first->line, first->col,
-				 "invalid preprocessing directive #%s",
-				 first[1].val.node->name);
-	  else
-	    cpp_error_with_line (pfile, first->line, first->col,
-				 "invalid preprocessing directive");
-	}
-
-      /* Discard this line to prevent further errors from cc1.  */
-      _cpp_clear_toklist (list);
-      goto retry;
+    default:		len = 0;			break;
+    case SPELL_STRING:	len = token->val.str.len;	break;
+    case SPELL_IDENT:	len = token->val.node->length;	break;
     }
-
-  /* Drop the EOF unless really at EOF or in a directive.  */
-  if (cur_token != first && !KNOWN_DIRECTIVE (list)
-      && pfile->done_initializing)
-    list->tokens_used--;
-
-  pfile->state.in_lex_line = 0;
+  /* 1 for whitespace, 4 for comment delimeters.  */
+  return len + 5;
 }
 
 /* Write the spelling of a token TOKEN to BUFFER.  The buffer must
    already contain the enough space to hold the token's spelling.
    Returns a pointer to the character after the last character
    written.  */
-
-static unsigned char *
-spell_token (pfile, token, buffer)
+unsigned char *
+cpp_spell_token (pfile, token, buffer)
      cpp_reader *pfile;		/* Would be nice to be rid of this...  */
      const cpp_token *token;
      unsigned char *buffer;
@@ -1546,7 +1272,7 @@ spell_token (pfile, token, buffer)
 	unsigned char c;
 
 	if (token->flags & DIGRAPH)
-	  spelling = _cpp_digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
+	  spelling = digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
 	else if (token->flags & NAMED_OP)
 	  goto spell_ident;
 	else
@@ -1596,581 +1322,145 @@ spell_token (pfile, token, buffer)
   return buffer;
 }
 
-/* Macro expansion algorithm.
-
-Macro expansion is implemented by a single-pass algorithm; there are
-no rescan passes involved.  cpp_get_token expands just enough to be
-able to return a token to the caller, a consequence is that when it
-returns the preprocessor can be in a state of mid-expansion.  The
-algorithm does not work by fully expanding a macro invocation into
-some kind of token list, and then returning them one by one.
-
-Our expansion state is recorded in a context stack.  We start out with
-a single context on the stack, let's call it base context.  This
-consists of the token list returned by lex_line that forms the next
-logical line in the source file.
-
-The current level in the context stack is stored in the cur_context
-member of the cpp_reader structure.  The context it references keeps,
-amongst other things, a count of how many tokens form that context and
-our position within those tokens.
-
-Fundamentally, calling cpp_get_token will return the next token from
-the current context.  If we're at the end of the current context, that
-context is popped from the stack first, unless it is the base context,
-in which case the next logical line is lexed from the source file.
-
-However, before returning the token, if it is a CPP_NAME token
-_cpp_get_token checks to see if it is a macro and if it is enabled.
-Each time it encounters a macro name, it calls push_macro_context.
-This function checks that the macro should be expanded (with
-is_macro_enabled), and if so pushes a new macro context on the stack
-which becomes the current context.  It then loops back to read the
-first token of the macro context.
-
-A macro context basically consists of the token list representing the
-macro's replacement list, which was saved in the hash table by
-save_macro_expansion when its #define statement was parsed.  If the
-macro is function-like, it also contains the tokens that form the
-arguments to the macro.  I say more about macro arguments below, but
-for now just saying that each argument is a set of pointers to tokens
-is enough.
-
-When taking tokens from a macro context, we may get a CPP_MACRO_ARG
-token.  This represents an argument passed to the macro, with the
-argument number stored in the token's AUX field.  The argument should
-be substituted, this is achieved by pushing an "argument context".  An
-argument context is just refers to the tokens forming the argument,
-which are obtained directly from the macro context.  The STRINGIFY
-flag on a CPP_MACRO_ARG token indicates that the argument should be
-stringified.
-
-Here's a few simple rules the context stack obeys:-
-
-  1) The lex_line token list is always context zero.
-
-  2) Context 1, if it exists, must be a macro context.
-
-  3) An argument context can only appear above a macro context.
-
-  4) A macro context can appear above the base context, another macro
-  context, or an argument context.
-
-  5) These imply that the minimal level of an argument context is 2.
-
-The only tricky thing left is ensuring that macros are enabled and
-disabled correctly.  The algorithm controls macro expansion by the
-level of the context a token is taken from in the context stack.  If a
-token is taken from a level equal to no_expand_level (a member of
-struct cpp_reader), no expansion is performed.
-
-When popping a context off the stack, if no_expand_level equals the
-level of the popped context, it is reduced by one to match the new
-context level, so that expansion is still disabled.  It does not
-increase if a context is pushed, though.  It starts out life as
-UINT_MAX, which has the effect that initially macro expansion is
-enabled.  I explain how this mechanism works below.
-
-The standard requires:-
-
-  1) Arguments to be fully expanded before substitution.
-
-  2) Stringified arguments to not be expanded, nor the tokens
-  immediately surrounding a ## operator.
-
-  3) Continual rescanning until there are no more macros left to
-  replace.
-
-  4) Once a macro has been expanded in stage 1) or 3), it cannot be
-  expanded again during later rescans.  This prevents infinite
-  recursion.
-
-The first thing to observe is that stage 3) is mostly redundant.
-Since a macro is disabled once it has been expanded, how can a rescan
-find an unexpanded macro name?  There are only two cases where this is
-possible:-
-
-  a) If the macro name results from a token paste operation.
-
-  b) If the macro in question is a function-like macro that hasn't
-  already been expanded because previously there was not the required
-  '(' token immediately following it.  This is only possible when an
-  argument is substituted, and after substitution the last token of
-  the argument can bind with a parenthesis appearing in the tokens
-  following the substitution.  Note that if the '(' appears within the
-  argument, the ')' must too, as expanding macro arguments cannot
-  "suck in" tokens outside the argument.
-
-So we tackle this as follows.  When parsing the macro invocation for
-arguments, we record the tokens forming each argument as a list of
-pointers to those tokens.  We do not expand any tokens that are "raw",
-i.e. directly from the macro invocation, but other tokens that come
-from (nested) argument substitution are fully expanded.
-
-This is achieved by setting the no_expand_level to that of the macro
-invocation.  A CPP_MACRO_ARG token never appears in the list of tokens
-forming an argument, because parse_args (indirectly) calls
-get_raw_token which automatically pushes argument contexts and traces
-into them.  Since these contexts are at a higher level than the
-no_expand_level, they get fully macro expanded.
-
-"Raw" and non-raw tokens are separated in arguments by null pointers,
-with the policy that the initial state of an argument is raw.  If the
-first token is not raw, it should be preceded by a null pointer.  When
-tracing through the tokens of an argument context, each time
-get_raw_token encounters a null pointer, it toggles the flag
-CONTEXT_RAW.
-
-This flag, when set, indicates to is_macro_disabled that we are
-reading raw tokens which should be macro-expanded.  Similarly, if
-clear, is_macro_disabled suppresses re-expansion.
-
-It's probably time for an example.
-
-#define hash #
-#define str(x) #x
-#define xstr(y) str(y hash)
-str(hash)			// "hash"
-xstr(hash)			// "# hash"
-
-In the invocation of str, parse_args turns off macro expansion and so
-parses the argument as <hash>.  This is the only token (pointer)
-passed as the argument to str.  Since <hash> is raw there is no need
-for an initial null pointer.  stringify_arg is called from
-get_raw_token when tracing through the expansion of str, since the
-argument has the STRINGIFY flag set.  stringify_arg turns off
-macro_expansion by setting the no_expand_level to that of the argument
-context.  Thus it gets the token <hash> and stringifies it to "hash"
-correctly.
-
-Similary xstr is passed <hash>.  However, when parse_args is parsing
-the invocation of str() in xstr's expansion, get_raw_token encounters
-a CPP_MACRO_ARG token for y.  Transparently to parse_args, it pushes
-an argument context, and enters the tokens of the argument,
-i.e. <hash>.  This is at a higher context level than parse_args
-disabled, and so is_macro_disabled permits expansion of it and a macro
-context is pushed on top of the argument context.  This contains the
-<#> token, and the end result is that <hash> is macro expanded.
-However, after popping off the argument context, the <hash> of xstr's
-expansion does not get macro expanded because we're back at the
-no_expand_level.  The end result is that the argument passed to str is
-<NULL> <#> <NULL> <hash>.  Note the nulls - policy is we start off
-raw, <#> is not raw, but then <hash> is.
-
-*/
-
-
-/* Free the storage allocated for macro arguments.  */
-static void
-free_macro_args (args)
-     macro_args *args;
-{
-  if (args->tokens)
-    free ((PTR) args->tokens);
-  free (args->ends);
-  free (args);
-}
-
-/* Determines if a macro has been already used (and is therefore
-   disabled).  */
-static int
-is_macro_disabled (pfile, expansion, token)
+/* Returns a token as a null-terminated string.  The string is
+   temporary, and automatically freed later.  Useful for diagnostics.  */
+unsigned char *
+cpp_token_as_text (pfile, token)
      cpp_reader *pfile;
-     const cpp_toklist *expansion;
      const cpp_token *token;
 {
-  cpp_context *context = CURRENT_CONTEXT (pfile);
-
-  /* Arguments on either side of ## are inserted in place without
-     macro expansion (6.10.3.3.2).  Conceptually, any macro expansion
-     occurs during a later rescan pass.  The effect is that we expand
-     iff we would as part of the macro's expansion list, so we should
-     drop to the macro's context.  */
-  if (IS_ARG_CONTEXT (context))
-    {
-      if (token->flags & PASTED)
-	context--;
-      else if (!(context->flags & CONTEXT_RAW))
-	return 1;
-      else if (context->flags & (CONTEXT_PASTEL | CONTEXT_PASTER))
-	context--;
-    }
-
-  /* Have we already used this macro?  */
-  while (context->level > 0)
-    {
-      if (!IS_ARG_CONTEXT (context) && context->u.list == expansion)
-	return 1;
-      /* Raw argument tokens are judged based on the token list they
-         came from.  */
-      if (context->flags & CONTEXT_RAW)
-	context = pfile->contexts + context->level;
-      else
-	context--;
-    }
-
-  /* Function-like macros may be disabled if the '(' is not in the
-     current context.  We check this without disrupting the context
-     stack.  */
-  if (expansion->paramc >= 0)
-    {
-      const cpp_token *next;
-      unsigned int prev_nme;
+  unsigned int len = cpp_token_len (token);
+  unsigned char *start = _cpp_pool_alloc (&pfile->temp_string_pool, len), *end;
 
-      context = CURRENT_CONTEXT (pfile);
-      /* Drop down any contexts we're at the end of: the '(' may
-         appear in lower macro expansions, or in the rest of the file.  */
-      while (context->posn == context->count && context > pfile->contexts)
-	{
-	  context--;
-	  /* If we matched, we are disabled, as we appear in the
-	     expansion of each macro we meet.  */
-	  if (!IS_ARG_CONTEXT (context) && context->u.list == expansion)
-	    return 1;
-	}
+  end = cpp_spell_token (pfile, token, start);
+  end[0] = '\0';
 
-      prev_nme = pfile->no_expand_level;
-      pfile->no_expand_level = context - pfile->contexts;
-      next = _cpp_get_token (pfile);
-      restore_macro_expansion (pfile, prev_nme);
-
-      if (next->type != CPP_OPEN_PAREN)
-	{
-	  _cpp_push_token (pfile, next);
-	  if (CPP_WTRADITIONAL (pfile))
-	    cpp_warning (pfile,
-	 "function macro %s must be used with arguments in traditional C",
-			 token->val.node->name);
-	  return 1;
-	}
-    }
-
-  return 0;
+  return start;
 }
 
-/* Add a token to the set of tokens forming the arguments to the macro
-   being parsed in parse_args.  */
-static void
-save_token (args, token)
-     macro_args *args;
-     const cpp_token *token;
+/* Used by C front ends.  Should really move to using cpp_token_as_text.  */
+const char *
+cpp_type2name (type)
+     enum cpp_ttype type;
 {
-  if (args->used == args->capacity)
-    {
-      args->capacity += args->capacity + 100;
-      args->tokens = (const cpp_token **)
-	xrealloc ((PTR) args->tokens,
-		  args->capacity * sizeof (const cpp_token *));
-    }
-  args->tokens[args->used++] = token;
-}
-
-/* Take and save raw tokens until we finish one argument.  Empty
-   arguments are saved as a single CPP_PLACEMARKER token.  */
-static const cpp_token *
-parse_arg (pfile, var_args, paren_context, args, pcount)
-     cpp_reader *pfile;
-     int var_args;
-     unsigned int paren_context;
-     macro_args *args;
-     unsigned int *pcount;
-{
-  const cpp_token *token;
-  unsigned int paren = 0, count = 0;
-  int raw, was_raw = 1;
-  
-  for (count = 0;; count++)
-    {
-      token = _cpp_get_token (pfile);
-
-      switch (token->type)
-	{
-	default:
-	  break;
-
-	case CPP_OPEN_PAREN:
-	  paren++;
-	  break;
-
-	case CPP_CLOSE_PAREN:
-	  if (paren-- != 0)
-	    break;
-	  goto out;
-
-	case CPP_COMMA:
-	  /* Commas are not terminators within parantheses or var_args.  */
-	  if (paren || var_args)
-	    break;
-	  goto out;
-
-	case CPP_EOF:		/* Error reported by caller.  */
-	  goto out;
-	}
-
-      raw = pfile->cur_context <= paren_context;
-      if (raw != was_raw)
-	{
-	  was_raw = raw;
-	  save_token (args, 0);
-	  count++;
-	}
-      save_token (args, token);
-    }
-
- out:
-  if (count == 0)
-    {
-      /* Duplicate the placemarker.  Then we can set its flags and
-	 position and safely be using more than one.  */
-      save_token (args, duplicate_token (pfile, &placemarker_token));
-      count++;
-    }
-
-  *pcount = count;
-  return token;
+  return (const char *) token_spellings[type].name;
 }
 
-/* This macro returns true if the argument starting at offset O of arglist
-   A is empty - that is, it's either a single PLACEMARKER token, or a null
-   pointer followed by a PLACEMARKER.  */
-
-#define empty_argument(A, O) \
- ((A)->tokens[O] ? (A)->tokens[O]->type == CPP_PLACEMARKER \
-		 : (A)->tokens[(O)+1]->type == CPP_PLACEMARKER)
-   
-/* Parse the arguments making up a macro invocation.  Nested arguments
-   are automatically macro expanded, but immediate macros are not
-   expanded; this enables e.g. operator # to work correctly.  Returns
-   non-zero on error.  */
-static int
-parse_args (pfile, hp, args)
-     cpp_reader *pfile;
-     cpp_hashnode *hp;
-     macro_args *args;
+/* Writes the spelling of token to FP.  Separate from cpp_spell_token
+   for efficiency - to avoid double-buffering.  Also, outputs a space
+   if PREV_WHITE is flagged.  */
+void
+cpp_output_token (token, fp)
+     const cpp_token *token;
+     FILE *fp;
 {
-  const cpp_token *token;
-  const cpp_toklist *macro;
-  unsigned int total = 0;
-  unsigned int paren_context = pfile->cur_context;
-  int argc = 0;
-
-  macro = hp->value.expansion;
-  do
-    {
-      unsigned int count;
+  if (token->flags & PREV_WHITE)
+    putc (' ', fp);
 
-      token = parse_arg (pfile, (argc + 1 == macro->paramc
-				 && (macro->flags & VAR_ARGS)),
-			 paren_context, args, &count);
-      if (argc < macro->paramc)
-	{
-	  total += count;
-	  args->ends[argc] = total;
-	}
-      argc++;
-    }
-  while (token->type != CPP_CLOSE_PAREN && token->type != CPP_EOF);
-
-  if (token->type == CPP_EOF)
-    {
-      cpp_error(pfile, "unterminated argument list for macro \"%s\"", hp->name);
-      return 1;
-    }
-  else if (argc < macro->paramc)
-    {
-      /* A rest argument is allowed to not appear in the invocation at all.
-	 e.g. #define debug(format, args...) ...
-	 debug("string");
-	 This is exactly the same as if the rest argument had received no
-	 tokens - debug("string",);  This extension is deprecated.  */
-
-      if (argc + 1 == macro->paramc && (macro->flags & VAR_ARGS))
-	{
-	  /* Duplicate the placemarker.  Then we can set its flags and
-             position and safely be using more than one.  */
-	  save_token (args, duplicate_token (pfile, &placemarker_token));
-	  args->ends[argc] = total + 1;
-
-	  if (CPP_OPTION (pfile, c99) && CPP_PEDANTIC (pfile))
-	    cpp_pedwarn (pfile, "ISO C99 requires rest arguments to be used");
-
-	  return 0;
-	}
-      else
-	{
-	  cpp_error (pfile, "%u arguments is not enough for macro \"%s\"",
-		     argc, hp->name);
-	  return 1;
-	}
-    }
-  /* An empty argument to an empty function-like macro is fine.  */
-  else if (argc > macro->paramc
-	   && !(macro->paramc == 0 && argc == 1 && empty_argument (args, 0)))
-    {
-      cpp_error (pfile, "%u arguments is too many for macro \"%s\"",
-		 argc, hp->name);
-      return 1;
-    }
-
-  return 0;
-}
-
-/* Adds backslashes before all backslashes and double quotes appearing
-   in strings.  Non-printable characters are converted to octal.  */
-static U_CHAR *
-quote_string (dest, src, len)
-     U_CHAR *dest;
-     const U_CHAR *src;
-     unsigned int len;
-{
-  while (len--)
+  switch (TOKEN_SPELL (token))
     {
-      U_CHAR c = *src++;
-
-      if (c == '\\' || c == '"')
-	{
-	  *dest++ = '\\';
-	  *dest++ = c;
-	}
-      else
-	{
-	  if (ISPRINT (c))
-	    *dest++ = c;
-	  else
-	    {
-	      sprintf ((char *) dest, "\\%03o", c);
-	      dest += 4;
-	    }
-	}
-    }
+    case SPELL_OPERATOR:
+      {
+	const unsigned char *spelling;
 
-  return dest;
-}
+	if (token->flags & DIGRAPH)
+	  spelling = digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
+	else if (token->flags & NAMED_OP)
+	  goto spell_ident;
+	else
+	  spelling = TOKEN_NAME (token);
 
-/* Allocates a buffer to hold a token's TEXT, and converts TOKEN to a
-   CPP_STRING token containing TEXT in quoted form.  */
-static cpp_token *
-make_string_token (token, text, len)
-     cpp_token *token;
-     const U_CHAR *text;
-     unsigned int len;
-{
-  U_CHAR *buf;
- 
-  buf = (U_CHAR *) xmalloc (len * 4);
-  token->type = CPP_STRING;
-  token->flags = 0;
-  token->val.str.text = buf;
-  token->val.str.len = quote_string (buf, text, len) - buf;
-  return token;
-}
+	ufputs (spelling, fp);
+      }
+      break;
 
-/* Allocates and converts a temporary token to a CPP_NUMBER token,
-   evaluating to NUMBER.  */
-static cpp_token *
-alloc_number_token (pfile, number)
-     cpp_reader *pfile;
-     int number;
-{
-  cpp_token *result;
-  char *buf;
+    spell_ident:
+    case SPELL_IDENT:
+      ufputs (token->val.node->name, fp);
+    break;
 
-  result = get_temp_token (pfile);
-  buf = xmalloc (20);
-  sprintf (buf, "%d", number);
+    case SPELL_STRING:
+      {
+	int left, right, tag;
+	switch (token->type)
+	  {
+	  case CPP_STRING:	left = '"';  right = '"';  tag = '\0'; break;
+	  case CPP_WSTRING:	left = '"';  right = '"';  tag = 'L';  break;
+	  case CPP_OSTRING:	left = '"';  right = '"';  tag = '@';  break;
+	  case CPP_CHAR:	left = '\''; right = '\''; tag = '\0'; break;
+    	  case CPP_WCHAR:	left = '\''; right = '\''; tag = 'L';  break;
+	  case CPP_HEADER_NAME:	left = '<';  right = '>';  tag = '\0'; break;
+	  default:		left = '\0'; right = '\0'; tag = '\0'; break;
+	  }
+	if (tag) putc (tag, fp);
+	if (left) putc (left, fp);
+	fwrite (token->val.str.text, 1, token->val.str.len, fp);
+	if (right) putc (right, fp);
+      }
+      break;
 
-  result->type = CPP_NUMBER;
-  result->flags = 0;
-  result->val.str.text = (U_CHAR *) buf;
-  result->val.str.len = strlen (buf);
-  return result;
-}
+    case SPELL_CHAR:
+      putc (token->val.aux, fp);
+      break;
 
-/* Returns a temporary token from the temporary token store of PFILE.  */
-static cpp_token *
-get_temp_token (pfile)
-     cpp_reader *pfile;
-{
-  if (pfile->temp_used == pfile->temp_alloced)
-    {
-      if (pfile->temp_used == pfile->temp_cap)
-	{
-	  pfile->temp_cap += pfile->temp_cap + 20;
-	  pfile->temp_tokens = (cpp_token **) xrealloc
-	    (pfile->temp_tokens, pfile->temp_cap * sizeof (cpp_token *));
-	}
-      pfile->temp_tokens[pfile->temp_alloced++] = (cpp_token *) xmalloc
-	(sizeof (cpp_token));
+    case SPELL_NONE:
+      /* An error, most probably.  */
+      break;
     }
-
-  return pfile->temp_tokens[pfile->temp_used++];
 }
 
-/* Release (not free) for re-use the temporary tokens of PFILE.  */
-static void
-release_temp_tokens (pfile)
-     cpp_reader *pfile;
+/* Compare two tokens.  */
+int
+_cpp_equiv_tokens (a, b)
+     const cpp_token *a, *b;
 {
-  while (pfile->temp_used)
-    {
-      cpp_token *token = pfile->temp_tokens[--pfile->temp_used];
+  if (a->type == b->type && a->flags == b->flags)
+    switch (TOKEN_SPELL (a))
+      {
+      default:			/* Keep compiler happy.  */
+      case SPELL_OPERATOR:
+	return 1;
+      case SPELL_CHAR:
+	return a->val.aux == b->val.aux; /* Character.  */
+      case SPELL_NONE:
+	return (a->type != CPP_MACRO_ARG || a->val.aux == b->val.aux);
+      case SPELL_IDENT:
+	return a->val.node == b->val.node;
+      case SPELL_STRING:
+	return (a->val.str.len == b->val.str.len
+		&& !memcmp (a->val.str.text, b->val.str.text,
+			    a->val.str.len));
+      }
 
-      if (TOKEN_SPELL (token) == SPELL_STRING)
-	{
-	  free ((char *) token->val.str.text);
-	  token->val.str.text = 0;
-	}
-    }
+  return 0;
 }
 
-/* Free all of PFILE's dynamically-allocated temporary tokens.  */
-void
-_cpp_free_temp_tokens (pfile)
-     cpp_reader *pfile;
+#if 0
+/* Compare two token lists.  */
+int
+_cpp_equiv_toklists (a, b)
+     const struct toklist *a, *b;
 {
-  if (pfile->temp_tokens)
-    {
-      /* It is possible, though unlikely (looking for '(' of a funlike
-	 macro into EOF), that we haven't released the tokens yet.  */
-      release_temp_tokens (pfile);
-      while (pfile->temp_alloced)
-	free (pfile->temp_tokens[--pfile->temp_alloced]);
-      free (pfile->temp_tokens);
-    }
+  unsigned int i, count;
 
-  if (pfile->date)
-    {
-      free ((char *) pfile->date->val.str.text);
-      free (pfile->date);
-      free ((char *) pfile->time->val.str.text);
-      free (pfile->time);
-    }
-}
+  count = a->limit - a->first;
+  if (count != (b->limit - b->first))
+    return 0;
 
-/* Copy TOKEN into a temporary token from PFILE's store.  */
-static cpp_token *
-duplicate_token (pfile, token)
-     cpp_reader *pfile;
-     const cpp_token *token;
-{
-  cpp_token *result = get_temp_token (pfile);
+  for (i = 0; i < count; i++)
+    if (! _cpp_equiv_tokens (&a->first[i], &b->first[i]))
+      return 0;
 
-  *result = *token;
-  if (TOKEN_SPELL (token) == SPELL_STRING)
-    {
-      U_CHAR *buff = (U_CHAR *) xmalloc (token->val.str.len);
-      memcpy (buff, token->val.str.text, token->val.str.len);
-      result->val.str.text = buff;
-    }
-  return result;
+  return 1;
 }
+#endif
 
 /* Determine whether two tokens can be pasted together, and if so,
    what the resulting token is.  Returns CPP_EOF if the tokens cannot
    be pasted, or the appropriate type for the merged token if they
    can.  */
 enum cpp_ttype
-_cpp_can_paste (pfile, token1, token2, digraph)
+cpp_can_paste (pfile, token1, token2, digraph)
      cpp_reader * pfile;
      const cpp_token *token1, *token2;
      int* digraph;
@@ -2247,11 +1537,11 @@ _cpp_can_paste (pfile, token1, token2, digraph)
     case CPP_NAME:
       if (b == CPP_NAME)	return CPP_NAME;
       if (b == CPP_NUMBER
-	  && is_numstart(token2->val.str.text[0]))	 return CPP_NAME;
+	  && name_p (pfile, &token2->val.str)) return CPP_NAME;
       if (b == CPP_CHAR
-	  && token1->val.node == pfile->spec_nodes->n_L) return CPP_WCHAR;
+	  && token1->val.node == pfile->spec_nodes.n_L) return CPP_WCHAR;
       if (b == CPP_STRING
-	  && token1->val.node == pfile->spec_nodes->n_L) return CPP_WSTRING;
+	  && token1->val.node == pfile->spec_nodes.n_L) return CPP_WSTRING;
       break;
 
     case CPP_NUMBER:
@@ -2278,882 +1568,230 @@ _cpp_can_paste (pfile, token1, token2, digraph)
   return CPP_EOF;
 }
 
-/* Check if TOKEN is to be ##-pasted with the token after it.  */
-static const cpp_token *
-maybe_paste_with_next (pfile, token)
-     cpp_reader *pfile;
-     const cpp_token *token;
-{
-  cpp_token *pasted;
-  const cpp_token *second;
-  cpp_context *context = CURRENT_CONTEXT (pfile);
-
-  /* Is this token on the LHS of ## ? */
-
-  while ((token->flags & PASTE_LEFT)
-	 || ((context->flags & CONTEXT_PASTEL)
-	     && context->posn == context->count))
-    {
-      /* Suppress macro expansion for next token, but don't conflict
-	 with the other method of suppression.  If it is an argument,
-	 macro expansion within the argument will still occur.  */
-      pfile->paste_level = pfile->cur_context;
-      second = _cpp_get_token (pfile);
-      pfile->paste_level = 0;
-      context = CURRENT_CONTEXT (pfile);
-
-      /* Ignore placemarker argument tokens (cannot be from an empty
-	 macro since macros are not expanded).  */
-      if (token->type == CPP_PLACEMARKER)
-	pasted = duplicate_token (pfile, second);
-      else if (second->type == CPP_PLACEMARKER)
-	{
-	  /* GCC has special extended semantics for , ## b where b is
-	     a varargs parameter: the comma disappears if b was given
-	     no actual arguments (not merely if b is an empty
-	     argument).  */
-	  if (token->type == CPP_COMMA && (context->flags & CONTEXT_VARARGS))
-	    pasted = duplicate_token (pfile, second);
-	  else
-	    pasted = duplicate_token (pfile, token);
-	}
-      else
-	{
-	  int digraph = 0;
-	  enum cpp_ttype type = _cpp_can_paste (pfile, token, second, &digraph);
-
-	  if (type == CPP_EOF)
-	    {
-	      if (CPP_OPTION (pfile, warn_paste))
-		{
-		  /* Do not complain about , ## <whatever> if
-		     <whatever> came from a variable argument, because
-		     the author probably intended the ## to trigger
-		     the special extended semantics (see above).  */
-		  if (token->type == CPP_COMMA
-		      && (context->flags & CONTEXT_VARARGS))
-		    /* no warning */;
-		  else
-		    cpp_warning (pfile,
-			"pasting would not give a valid preprocessing token");
-		}
-	      _cpp_push_token (pfile, second);
-	      /* A short term hack to safely clear the PASTE_LEFT flag.  */
-	      pasted = duplicate_token (pfile, token);
-	      pasted->flags &= ~PASTE_LEFT;
-	      return pasted;
-	    }
-
-	  if (type == CPP_NAME || type == CPP_NUMBER)
-	    {
-	      /* Join spellings.  */
-	      U_CHAR *buf, *end;
-
-	      pasted = get_temp_token (pfile);
-	      buf = (U_CHAR *) alloca (TOKEN_LEN (token) + TOKEN_LEN (second));
-	      end = spell_token (pfile, token, buf);
-	      end = spell_token (pfile, second, end);
-	      *end = '\0';
+/* Returns nonzero if a space should be inserted to avoid an
+   accidental token paste for output.  For simplicity, it is
+   conservative, and occasionally advises a space where one is not
+   needed, e.g. "." and ".2".  */
 
-	      if (type == CPP_NAME)
-		pasted->val.node = cpp_lookup (pfile, buf, end - buf);
-	      else
-		{
-		  pasted->val.str.text = uxstrdup (buf);
-		  pasted->val.str.len = end - buf;
-		}
-	    }
-	  else if (type == CPP_WCHAR || type == CPP_WSTRING
-		   || type == CPP_OSTRING)
-	    pasted = duplicate_token (pfile, second);
-	  else
-	    {
-	      pasted = get_temp_token (pfile);
-	      pasted->val.integer = 0;
-	    }
-
-	  pasted->type = type;
-	  pasted->flags = digraph ? DIGRAPH : 0;
-
-	  if (type == CPP_NAME && pasted->val.node->type == T_OPERATOR)
-	    {
-	      pasted->type = pasted->val.node->value.code;
-	      pasted->flags |= NAMED_OP;
-	    }
-	}
-
-      /* The pasted token gets the whitespace flags and position of the
-	 first token, the PASTE_LEFT flag of the second token, plus the
-	 PASTED flag to indicate it is the result of a paste.  However, we
-	 want to preserve the DIGRAPH flag.  */
-      pasted->flags &= ~(PREV_WHITE | BOL | PASTE_LEFT);
-      pasted->flags |= ((token->flags & (PREV_WHITE | BOL))
-			| (second->flags & PASTE_LEFT) | PASTED);
-      pasted->col = token->col;
-      pasted->line = token->line;
-
-      /* See if there is another token to be pasted onto the one we just
-	 constructed.  */
-      token = pasted;
-      /* and loop */
-    }
-  return token;
-}
-
-/* Convert a token sequence to a single string token according to the
-   rules of the ISO C #-operator.  */
-#define INIT_SIZE 200
-static cpp_token *
-stringify_arg (pfile, token)
+int
+cpp_avoid_paste (pfile, token1, token2)
      cpp_reader *pfile;
-     const cpp_token *token;
+     const cpp_token *token1, *token2;
 {
-  cpp_token *result;
-  unsigned char *main_buf;
-  unsigned int prev_value, backslash_count = 0;
-  unsigned int buf_used = 0, whitespace = 0, buf_cap = INIT_SIZE;
-
-  push_arg_context (pfile, token);
-  prev_value  = prevent_macro_expansion (pfile);
-  main_buf = (unsigned char *) xmalloc (buf_cap);
-
-  result = get_temp_token (pfile);
-  ASSIGN_FLAGS_AND_POS (result, token);
-
-  for (; (token = _cpp_get_token (pfile))->type != CPP_EOF; )
-    {
-      int escape;
-      unsigned char *buf;
-      unsigned int len = TOKEN_LEN (token);
-
-      if (token->type == CPP_PLACEMARKER)
-	continue;
-
-      escape = (token->type == CPP_STRING || token->type == CPP_WSTRING
-		|| token->type == CPP_CHAR || token->type == CPP_WCHAR);
-      if (escape)
-	len *= 4 + 1;
-
-      if (buf_used + len > buf_cap)
-	{
-	  buf_cap = buf_used + len + INIT_SIZE;
-	  main_buf = xrealloc (main_buf, buf_cap);
-	}
-
-      if (whitespace && (token->flags & PREV_WHITE))
-	main_buf[buf_used++] = ' ';
-
-      if (escape)
-	buf = (unsigned char *) xmalloc (len);
-      else
-	buf = main_buf + buf_used;
-      
-      len = spell_token (pfile, token, buf) - buf;
-      if (escape)
-	{
-	  buf_used = quote_string (&main_buf[buf_used], buf, len) - main_buf;
-	  free (buf);
-	}
-      else
-	buf_used += len;
-
-      whitespace = 1;
-      if (token->type == CPP_BACKSLASH)
-	backslash_count++;
-      else
-	backslash_count = 0;
-    }
+  enum cpp_ttype a = token1->type, b = token2->type;
+  cppchar_t c;
 
-  /* Ignore the final \ of invalid string literals.  */
-  if (backslash_count & 1)
-    {
-      cpp_warning (pfile, "invalid string literal, ignoring final '\\'");
-      buf_used--;
-    }
+  if (token1->flags & NAMED_OP)
+    a = CPP_NAME;
+  if (token2->flags & NAMED_OP)
+    b = CPP_NAME;
 
-  result->type = CPP_STRING;
-  result->val.str.text = main_buf;
-  result->val.str.len = buf_used;
-  restore_macro_expansion (pfile, prev_value);
-  return result;
-}
+  c = EOF;
+  if (token2->flags & DIGRAPH)
+    c = digraph_spellings[b - CPP_FIRST_DIGRAPH][0];
+  else if (token_spellings[b].category == SPELL_OPERATOR)
+    c = token_spellings[b].name[0];
 
-/* Allocate more room on the context stack of PFILE.  */
-static void
-expand_context_stack (pfile)
-     cpp_reader *pfile;
-{
-  pfile->context_cap += pfile->context_cap + 20;
-  pfile->contexts = (cpp_context *)
-    xrealloc (pfile->contexts, pfile->context_cap * sizeof (cpp_context));
-}
+  /* Quickly get everything that can paste with an '='.  */
+  if (a <= CPP_LAST_EQ && c == '=')
+    return 1;
 
-/* Push the context of macro NODE onto the context stack.  TOKEN is
-   the CPP_NAME token invoking the macro.  */
-static int
-push_macro_context (pfile, token)
-     cpp_reader *pfile;
-     const cpp_token *token;
-{
-  unsigned char orig_flags;
-  macro_args *args;
-  cpp_context *context;
-  cpp_hashnode *node = token->val.node;
-
-  /* Token's flags may change when parsing args containing a nested
-     invocation of this macro.  */
-  orig_flags = token->flags & (PREV_WHITE | BOL);
-  args = 0;
-  if (node->value.expansion->paramc >= 0)
+  switch (a)
     {
-      unsigned int error, prev_nme;
-
-      /* Allocate room for the argument contexts, and parse them.  */
-      args  = (macro_args *) xmalloc (sizeof (macro_args));
-      args->ends = (unsigned int *)
-	xmalloc (node->value.expansion->paramc * sizeof (unsigned int));
-      args->tokens = 0;
-      args->capacity = 0;
-      args->used = 0;
-
-      prev_nme = prevent_macro_expansion (pfile);
-      pfile->args = args;
-      error = parse_args (pfile, node, args);
-      pfile->args = 0;
-      restore_macro_expansion (pfile, prev_nme);
-      if (error)
-	{
-	  free_macro_args (args);
-	  return 1;
-	}
-      /* Set the level after the call to parse_args.  */
-      args->level = pfile->cur_context;
+    case CPP_GREATER:	return c == '>' || c == '?';
+    case CPP_LESS:	return c == '<' || c == '?' || c == '%' || c == ':';
+    case CPP_PLUS:	return c == '+';
+    case CPP_MINUS:	return c == '-' || c == '>';
+    case CPP_DIV:	return c == '/' || c == '*'; /* Comments.  */
+    case CPP_MOD:	return c == ':' || c == '>';
+    case CPP_AND:	return c == '&';
+    case CPP_OR:	return c == '|';
+    case CPP_COLON:	return c == ':' || c == '>';
+    case CPP_DEREF:	return c == '*';
+    case CPP_DOT:	return c == '.' || c == '%';
+    case CPP_HASH:	return c == '#' || c == '%'; /* Digraph form.  */
+    case CPP_NAME:	return ((b == CPP_NUMBER
+				 && name_p (pfile, &token2->val.str))
+				|| b == CPP_NAME
+				|| b == CPP_CHAR || b == CPP_STRING); /* L */
+    case CPP_NUMBER:	return (b == CPP_NUMBER || b == CPP_NAME
+				|| c == '.' || c == '+' || c == '-');
+    case CPP_OTHER:	return (CPP_OPTION (pfile, objc)
+				&& token1->val.aux == '@'
+				&& (b == CPP_NAME || b == CPP_STRING));
+    default:		break;
     }
 
-  /* Now push its context.  */
-  pfile->cur_context++;
-  if (pfile->cur_context == pfile->context_cap)
-    expand_context_stack (pfile);
-
-  context = CURRENT_CONTEXT (pfile);
-  context->u.list = node->value.expansion;
-  context->args = args;
-  context->posn = 0;
-  context->count = context->u.list->tokens_used;
-  context->level = pfile->cur_context;
-  context->flags = 0;
-  context->pushed_token = 0;
-
-  /* Set the flags of the first token.  We know there must
-     be one, empty macros are a single placemarker token.  */
-  MODIFY_FLAGS_AND_POS (&context->u.list->tokens[0], token, orig_flags);
-
   return 0;
 }
 
-/* Push an argument to the current macro onto the context stack.
-   TOKEN is the MACRO_ARG token representing the argument expansion.  */
-static void
-push_arg_context (pfile, token)
-     cpp_reader *pfile;
-     const cpp_token *token;
-{
-  cpp_context *context;
-  macro_args *args;
-
-  pfile->cur_context++;
-  if (pfile->cur_context == pfile->context_cap)
-      expand_context_stack (pfile);
-
-  context = CURRENT_CONTEXT (pfile);
-  args = context[-1].args;
-
-  context->count = token->val.aux ? args->ends[token->val.aux - 1]: 0;
-  context->u.arg = args->tokens + context->count;
-  context->count = args->ends[token->val.aux] - context->count;
-  context->args = 0;
-  context->posn = 0;
-  context->level = args->level;
-  context->flags = CONTEXT_ARG | CONTEXT_RAW;
-  if ((context[-1].u.list->flags & VAR_ARGS)
-      && token->val.aux + 1 == (unsigned) context[-1].u.list->paramc)
-    context->flags |= CONTEXT_VARARGS;
-  context->pushed_token = 0;
-
-  /* Set the flags of the first token.  There is one.  */
-  {
-    const cpp_token *first = context->u.arg[0];
-    if (!first)
-      first = context->u.arg[1];
-
-    MODIFY_FLAGS_AND_POS ((cpp_token *) first, token,
-			  token->flags & (PREV_WHITE | BOL));
-  }
-
-  if (token->flags & PASTE_LEFT)
-    context->flags |= CONTEXT_PASTEL;
-  if (pfile->paste_level)
-    context->flags |= CONTEXT_PASTER;
-}
-
-/* "Unget" a token.  It is effectively inserted in the token queue and
-   will be returned by the next call to get_raw_token.  */
+/* Output all the remaining tokens on the current line, and a newline
+   character, to FP.  Leading whitespace is removed.  */
 void
-_cpp_push_token (pfile, token)
+cpp_output_line (pfile, fp)
      cpp_reader *pfile;
-     const cpp_token *token;
+     FILE *fp;
 {
-  cpp_context *context = CURRENT_CONTEXT (pfile);
+  cpp_token token;
 
-  if (context->posn > 0)
+  _cpp_get_token (pfile, &token);
+  token.flags &= ~PREV_WHITE;
+  while (token.type != CPP_EOF)
     {
-      const cpp_token *prev;
-      if (IS_ARG_CONTEXT (context))
-	prev = context->u.arg[context->posn - 1];
-      else
-	prev = &context->u.list->tokens[context->posn - 1];
-
-      if (prev == token)
-	{
-	  context->posn--;
-	  return;
-	}
+      cpp_output_token (&token, fp);
+      _cpp_get_token (pfile, &token);
     }
 
-  if (context->pushed_token)
-    cpp_ice (pfile, "two tokens pushed in a row");
-  if (token->type != CPP_EOF)
-    context->pushed_token = token;
-  /* Don't push back a directive's CPP_EOF, step back instead.  */
-  else if (pfile->cur_context == 0)
-    pfile->contexts[0].posn--;
+  putc ('\n', fp);
 }
 
-/* Handle a preprocessing directive.  TOKEN is the CPP_HASH token
-   introducing the directive.  */
-void
-_cpp_process_directive (pfile, token)
-     cpp_reader *pfile;
-     const cpp_token *token;
-{
-  const struct directive *d = pfile->token_list.directive;
-  int prev_nme = 0;
-
-  /* Skip over the directive name.  */
-  if (token[1].type == CPP_NAME)
-    _cpp_get_raw_token (pfile);
-  else if (token[1].type != CPP_NUMBER)
-    cpp_ice (pfile, "directive begins with %s?!", TOKEN_NAME (token));
-
-  if (! (d->flags & EXPAND))
-    prev_nme = prevent_macro_expansion (pfile);
-  (void) (*d->handler) (pfile);
-  if (! (d->flags & EXPAND))
-    restore_macro_expansion (pfile, prev_nme);
-  _cpp_skip_rest_of_line (pfile);
-}
+/* Memory pools.  */
 
-/* The external interface to return the next token.  All macro
-   expansion and directive processing is handled internally, the
-   caller only ever sees the output after preprocessing.  */
-const cpp_token *
-cpp_get_token (pfile)
-     cpp_reader *pfile;
+struct dummy
 {
-  const cpp_token *token;
-  /* Loop till we hit a non-directive, non-placemarker token.  */
-  for (;;)
-    {
-      token = _cpp_get_token (pfile);
-
-      if (token->type == CPP_PLACEMARKER)
-	continue;
+  char c;
+  union
+  {
+    double d;
+    int *p;
+  } u;
+};
 
-      if (token->type == CPP_HASH && token->flags & BOL
-	  && pfile->token_list.directive)
-	{
-	  _cpp_process_directive (pfile, token);
-	  continue;
-	}
+#define DEFAULT_ALIGNMENT (offsetof (struct dummy, u))
 
-      return token;
-    }
+static int
+chunk_suitable (pool, chunk, size)
+     cpp_pool *pool;
+     cpp_chunk *chunk;
+     unsigned int size;
+{
+  /* Being at least twice SIZE means we can use memcpy in
+     _cpp_next_chunk rather than memmove.  Besides, it's a good idea
+     anyway.  */
+  return (chunk && pool->locked != chunk
+	  && (unsigned int) (chunk->limit - chunk->base) >= size * 2);
 }
 
-/* The internal interface to return the next token.  There are two
-   differences between the internal and external interfaces: the
-   internal interface may return a PLACEMARKER token, and it does not
-   process directives.  */
-const cpp_token *
-_cpp_get_token (pfile)
-     cpp_reader *pfile;
+/* Returns the end of the new pool.  PTR points to a char in the old
+   pool, and is updated to point to the same char in the new pool.  */
+unsigned char *
+_cpp_next_chunk (pool, len, ptr)
+     cpp_pool *pool;
+     unsigned int len;
+     unsigned char **ptr;
 {
-  const cpp_token *token, *old_token;
-  cpp_hashnode *node;
+  cpp_chunk *chunk = pool->cur->next;
 
-  /* Loop until we hit a non-macro token.  */
-  for (;;)
+  /* LEN is the minimum size we want in the new pool.  */
+  len += POOL_ROOM (pool);
+  if (! chunk_suitable (pool, chunk, len))
     {
-      token = get_raw_token (pfile);
-
-      /* Short circuit EOF. */
-      if (token->type == CPP_EOF)
-	return token;
-
-      /* If we are skipping... */
-      if (pfile->skipping)
-	{
-	  /* we still have to process directives,  */
-	  if (pfile->token_list.directive)
-	    return token;
-
-	  /* but everything else is ignored.  */
-	  _cpp_skip_rest_of_line (pfile);
-	  continue;
-	}
+      chunk = new_chunk (POOL_SIZE (pool) * 2 + len);
 
-      /* If there's a potential control macro and we get here, then that
-	 #ifndef didn't cover the entire file and its argument shouldn't
-	 be taken as a control macro.  */
-      pfile->potential_control_macro = 0;
-
-      /* If we are rescanning preprocessed input, no macro expansion or
-	 token pasting may occur.  */
-      if (CPP_OPTION (pfile, preprocessed))
-	return token;
-
-      old_token = token;
-
-      /* See if there's a token to paste with this one.  */
-      if (!pfile->paste_level)
-	token = maybe_paste_with_next (pfile, token);
-
-      /* If it isn't a macro, return it now.  */
-      if (token->type != CPP_NAME || token->val.node->type == T_VOID)
-	return token;
-
-      /* Is macro expansion disabled in general, or are we in the
-	 middle of a token paste, or was this token just pasted?
-	 (Note we don't check token->flags & PASTED, because that
-	 counts tokens that were pasted at some point in the past,
-	 we're only interested in tokens that were pasted by this call
-	 to maybe_paste_with_next.)  */
-      if (pfile->no_expand_level == pfile->cur_context
-	  || pfile->paste_level
-	  || (token != old_token
-	      && pfile->no_expand_level + 1 == pfile->cur_context))
-	return token;
-
-      node = token->val.node;
-      if (node->type != T_MACRO)
-	return special_symbol (pfile, node, token);
-
-      if (is_macro_disabled (pfile, node->value.expansion, token))
-	return token;
-
-      if (push_macro_context (pfile, token))
-	return token;
-      /* else loop */
+      chunk->next = pool->cur->next;
+      pool->cur->next = chunk;
     }
-}
-
-/* Returns the next raw token, i.e. without performing macro
-   expansion.  Argument contexts are automatically entered.  */
-static const cpp_token *
-get_raw_token (pfile)
-     cpp_reader *pfile;
-{
-  const cpp_token *result;
-  cpp_context *context;
 
-  for (;;)
-    {
-      context = CURRENT_CONTEXT (pfile);
-      if (context->pushed_token)
-	{
-	  result = context->pushed_token;
-	  context->pushed_token = 0;
-	  return result;	/* Cannot be a CPP_MACRO_ARG */
-	}
-      else if (context->posn == context->count)
-	{
-	  if (pop_context (pfile))
-	    return &eof_token;
-	  continue;
-	}
-      else if (IS_ARG_CONTEXT (context))
-	{
-	  result = context->u.arg[context->posn++];
-	  if (result == 0)
-	    {
-	      context->flags ^= CONTEXT_RAW;
-	      result = context->u.arg[context->posn++];
-	    }
-	  return result;	/* Cannot be a CPP_MACRO_ARG */
-	}
-
-      result = &context->u.list->tokens[context->posn++];
-
-      if (result->type != CPP_MACRO_ARG)
-	return result;
+  /* Update the pointer before changing chunk's front.  */
+  if (ptr)
+    *ptr += chunk->base - POOL_FRONT (pool);
 
-      if (result->flags & STRINGIFY_ARG)
-	return stringify_arg (pfile, result);
-
-      push_arg_context (pfile, result);
-    }
-}
+  memcpy (chunk->base, POOL_FRONT (pool), POOL_ROOM (pool));
+  chunk->front = chunk->base;
 
-/* Internal interface to get the token without macro expanding.  */
-const cpp_token *
-_cpp_get_raw_token (pfile)
-     cpp_reader *pfile;
-{
-  int prev_nme = prevent_macro_expansion (pfile);
-  const cpp_token *result = _cpp_get_token (pfile);
-  restore_macro_expansion (pfile, prev_nme);
-  return result;
+  pool->cur = chunk;
+  return POOL_LIMIT (pool);
 }
 
-/* A thin wrapper to lex_line.  CLEAR is non-zero if the current token
-   list should be overwritten, or zero if we need to append
-   (typically, if we are within the arguments to a macro, or looking
-   for the '(' to start a function-like macro invocation).  */
-static int
-lex_next (pfile, clear)
-     cpp_reader *pfile;
-     int clear;
+static cpp_chunk *
+new_chunk (size)
+     unsigned int size;
 {
-  cpp_toklist *list = &pfile->token_list;
-  const cpp_token *old_list = list->tokens;
-  unsigned int old_used = list->tokens_used;
-
-  if (clear)
-    {
-      /* Release all temporary tokens.  */
-      _cpp_clear_toklist (list);
-      pfile->contexts[0].posn = 0;
-      if (pfile->temp_used)
-	release_temp_tokens (pfile);
-    }
-  lex_line (pfile, list);
-  pfile->contexts[0].count = list->tokens_used;
-
-  if (!clear && pfile->args)
-    {
-      /* Fix up argument token pointers.  */
-      if (old_list != list->tokens)
-	{
-	  unsigned int i;
-
-	  for (i = 0; i < pfile->args->used; i++)
-	    {
-	      const cpp_token *token = pfile->args->tokens[i];
-	      if (token >= old_list && token < old_list + old_used)
-		pfile->args->tokens[i] = (const cpp_token *)
-	        ((char *) token + ((char *) list->tokens - (char *) old_list));
-	    }
-	}
+  unsigned char *base;
+  cpp_chunk *result;
 
-      /* 6.10.3 paragraph 11: If there are sequences of preprocessing
-	 tokens within the list of arguments that would otherwise act as
-	 preprocessing directives, the behavior is undefined.
+  size = ALIGN (size, DEFAULT_ALIGNMENT);
+  base = (unsigned char *) xmalloc (size + sizeof (cpp_chunk));
+  /* Put the chunk descriptor at the end.  Then chunk overruns will
+     cause obvious chaos.  */
+  result = (cpp_chunk *) (base + size);
+  result->base = base;
+  result->front = base;
+  result->limit = base + size;
+  result->next = 0;
 
-	 This implementation will report a hard error and treat the
-	 'sequence of preprocessing tokens' as part of the macro argument,
-	 not a directive.  
-
-         Note if pfile->args == 0, we're OK since we're only inside a
-         macro argument after a '('.  */
-      if (list->directive)
-	{
-	  cpp_error_with_line (pfile, list->tokens[old_used].line,
-			       list->tokens[old_used].col,
-			       "#%s may not be used inside a macro argument",
-			       list->directive->name);
-	  return 1;
-	}
-    }
-
-  return 0;
-}
-
-/* Pops a context off the context stack.  If we're at the bottom, lexes
-   the next logical line.  Returns EOF if we're at the end of the
-   argument list to the # operator, or we should not "overflow"
-   into the rest of the file (e.g. 6.10.3.1.1).  */
-static int
-pop_context (pfile)
-     cpp_reader *pfile;
-{
-  cpp_context *context;
-
-  if (pfile->cur_context == 0)
-    {
-      /* If we are currently processing a directive, do not advance.  6.10
-	 paragraph 2: A new-line character ends the directive even if it
-	 occurs within what would otherwise be an invocation of a
-	 function-like macro.  */
-      if (pfile->token_list.directive)
-	return 1;
-
-      return lex_next (pfile, pfile->no_expand_level == UINT_MAX);
-    }
-
-  /* Argument contexts, when parsing args or handling # operator
-     return CPP_EOF at the end.  */
-  context = CURRENT_CONTEXT (pfile);
-  if (IS_ARG_CONTEXT (context) && pfile->cur_context == pfile->no_expand_level)
-    return 1;
-
-  /* Free resources when leaving macro contexts.  */
-  if (context->args)
-    free_macro_args (context->args);
-
-  if (pfile->cur_context == pfile->no_expand_level)
-    pfile->no_expand_level--;
-  pfile->cur_context--;
-
-  return 0;
+  return result;
 }
 
-/* Turn off macro expansion at the current context level.  */
-static unsigned int
-prevent_macro_expansion (pfile)
-     cpp_reader *pfile;
-{
-  unsigned int prev_value = pfile->no_expand_level;
-  pfile->no_expand_level = pfile->cur_context;
-  return prev_value;
+void
+_cpp_init_pool (pool, size, align, temp)
+     cpp_pool *pool;
+     unsigned int size, align, temp;
+{
+  if (align == 0)
+    align = DEFAULT_ALIGNMENT;
+  if (align & (align - 1))
+    abort ();
+  pool->align = align;
+  pool->cur = new_chunk (size);
+  pool->locked = 0;
+  pool->locks = 0;
+  if (temp)
+    pool->cur->next = pool->cur;
 }
 
-/* Restore macro expansion to its previous state.  */
-static void
-restore_macro_expansion (pfile, prev_value)
-     cpp_reader *pfile;
-     unsigned int prev_value;
+void
+_cpp_lock_pool (pool)
+     cpp_pool *pool;
 {
-  pfile->no_expand_level = prev_value;
+  if (pool->locks++ == 0)
+    pool->locked = pool->cur;
 }
 
-/* Used by cpperror.c to obtain the correct line and column to report
-   in a diagnostic.  */
-unsigned int
-_cpp_get_line (pfile, pcol)
-     cpp_reader *pfile;
-     unsigned int *pcol;
+void
+_cpp_unlock_pool (pool)
+     cpp_pool *pool;
 {
-  unsigned int index;
-  const cpp_token *cur_token;
-
-  if (pfile->state.in_lex_line)
-    index = pfile->token_list.tokens_used;
-  else
-    {
-      index = pfile->contexts[0].posn;
-
-      if (index == 0)
-	{
-	  if (pcol)
-	    *pcol = 0;
-	  return 0;
-	}
-      index--;
-    }
-
-  cur_token = &pfile->token_list.tokens[index];
-  if (pcol)
-    *pcol = cur_token->col;
-  return cur_token->line;
+  if (--pool->locks == 0)
+    pool->locked = 0;
 }
 
-#define DSC(str) (const U_CHAR *)str, sizeof str - 1
-static const char * const monthnames[] =
-{
-  "Jan", "Feb", "Mar", "Apr", "May", "Jun",
-  "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
-};
-
-/* Handle builtin macros like __FILE__.  */
-static const cpp_token *
-special_symbol (pfile, node, token)
-     cpp_reader *pfile;
-     cpp_hashnode *node;
-     const cpp_token *token;
+void
+_cpp_free_pool (pool)
+     cpp_pool *pool;
 {
-  cpp_token *result;
-  cpp_buffer *ip;
+  cpp_chunk *chunk = pool->cur, *next;
 
-  switch (node->type)
+  do
     {
-    case T_FILE:
-    case T_BASE_FILE:
-      {
-	const char *file;
-
-	ip = CPP_BUFFER (pfile);
-	if (ip == 0)
-	  file = "";
-	else
-	  {
-	    if (node->type == T_BASE_FILE)
-	      while (CPP_PREV_BUFFER (ip) != NULL)
-		ip = CPP_PREV_BUFFER (ip);
-
-	    file = ip->nominal_fname;
-	  }
-	result = make_string_token (get_temp_token (pfile), (U_CHAR *) file,
-				    strlen (file));
-      }
-      break;
-	
-    case T_INCLUDE_LEVEL:
-      /* pfile->include_depth counts the primary source as level 1,
-	 but historically __INCLUDE_DEPTH__ has called the primary
-	 source level 0.  */
-      result = alloc_number_token (pfile, pfile->include_depth - 1);
-      break;
-
-    case T_SPECLINE:
-      /* If __LINE__ is embedded in a macro, it must expand to the
-	 line of the macro's invocation, not its definition.
-	 Otherwise things like assert() will not work properly.  */
-      result = alloc_number_token (pfile, _cpp_get_line (pfile, NULL));
-      break;
-
-    case T_STDC:
-      {
-	int stdc = 1;
-
-#ifdef STDC_0_IN_SYSTEM_HEADERS
-	if (CPP_IN_SYSTEM_HEADER (pfile)
-	    && pfile->spec_nodes->n__STRICT_ANSI__->type == T_VOID)
-	  stdc = 0;
-#endif
-	result = alloc_number_token (pfile, stdc);
-      }
-      break;
-
-    case T_DATE:
-    case T_TIME:
-      if (pfile->date == 0)
-	{
-	  /* Allocate __DATE__ and __TIME__ from permanent storage,
-	     and save them in pfile so we don't have to do this again.
-	     We don't generate these strings at init time because
-	     time() and localtime() are very slow on some systems.  */
-	  time_t tt = time (NULL);
-	  struct tm *tb = localtime (&tt);
-
-	  pfile->date = make_string_token
-	    ((cpp_token *) xmalloc (sizeof (cpp_token)), DSC("Oct 11 1347"));
-	  pfile->time = make_string_token
-	    ((cpp_token *) xmalloc (sizeof (cpp_token)), DSC("12:34:56"));
-
-	  sprintf ((char *) pfile->date->val.str.text, "%s %2d %4d",
-		   monthnames[tb->tm_mon], tb->tm_mday, tb->tm_year + 1900);
-	  sprintf ((char *) pfile->time->val.str.text, "%02d:%02d:%02d",
-		   tb->tm_hour, tb->tm_min, tb->tm_sec);
-	}
-      result = node->type == T_DATE ? pfile->date: pfile->time;
-      break;
-
-    case T_POISON:
-      cpp_error (pfile, "attempt to use poisoned \"%s\"", node->name);
-      return token;
-
-    default:
-      cpp_ice (pfile, "invalid special hash type");
-      return token;
+      next = chunk->next;
+      free (chunk->base);
+      chunk = next;
     }
-
-  ASSIGN_FLAGS_AND_POS (result, token);
-  return result;
+  while (chunk && chunk != pool->cur);
 }
-#undef DSC
-
-/* Allocate pfile->input_buffer, and initialize _cpp_trigraph_map[]
-   if it hasn't happened already.  */
 
-void
-_cpp_init_input_buffer (pfile)
-     cpp_reader *pfile;
+/* Reserve LEN bytes from a memory pool.  */
+unsigned char *
+_cpp_pool_reserve (pool, len)
+     cpp_pool *pool;
+     unsigned int len;
 {
-  cpp_context *base;
-
-  _cpp_init_toklist (&pfile->token_list, 0);
-  pfile->no_expand_level = UINT_MAX;
-  pfile->context_cap = 20;
-  pfile->cur_context = 0;
-
-  pfile->contexts = (cpp_context *)
-    xmalloc (pfile->context_cap * sizeof (cpp_context));
-
-  /* Clear the base context.  */
-  base = &pfile->contexts[0];
-  base->u.list = &pfile->token_list;
-  base->posn = 0;
-  base->count = 0;
-  base->args = 0;
-  base->level = 0;
-  base->flags = 0;
-  base->pushed_token = 0;
-}
+  len = ALIGN (len, pool->align);
+  if (len > (unsigned int) POOL_ROOM (pool))
+    _cpp_next_chunk (pool, len, 0);
 
-/* Moves to the end of the directive line, popping contexts as
-   necessary.  */
-void
-_cpp_skip_rest_of_line (pfile)
-     cpp_reader *pfile;
-{
-  /* Discard all stacked contexts.  */
-  int i;
-  for (i = pfile->cur_context; i > 0; i--)
-    if (pfile->contexts[i].args)
-      free_macro_args (pfile->contexts[i].args);
-
-  if (pfile->no_expand_level <= pfile->cur_context)
-    pfile->no_expand_level = 0;
-  pfile->cur_context = 0;
-
-  /* Clear the base context, and clear the directive pointer so that
-     get_raw_token will advance to the next line.  */
-  pfile->contexts[0].count = 0;
-  pfile->contexts[0].posn = 0;
-  pfile->token_list.directive = 0;
+  return POOL_FRONT (pool);
 }
 
-/* Directive handler wrapper used by the command line option
-   processor.  */
-void
-_cpp_run_directive (pfile, dir, buf, count, name)
-     cpp_reader *pfile;
-     const struct directive *dir;
-     const char *buf;
-     size_t count;
-     const char *name;
+/* Allocate LEN bytes from a memory pool.  */
+unsigned char *
+_cpp_pool_alloc (pool, len)
+     cpp_pool *pool;
+     unsigned int len;
 {
-  if (cpp_push_buffer (pfile, (const U_CHAR *)buf, count) != NULL)
-    {
-      unsigned int prev_lvl = 0;
-
-      if (name)
-	CPP_BUFFER (pfile)->nominal_fname = name;
-      else
-	CPP_BUFFER (pfile)->nominal_fname = _("<command line>");
-      CPP_BUFFER (pfile)->lineno = (unsigned int)-1;
+  unsigned char *result = _cpp_pool_reserve (pool, len);
 
-      /* Scan the line now, else prevent_macro_expansion won't work.  */
-      lex_next (pfile, 1);
-      if (! (dir->flags & EXPAND))
-	prev_lvl = prevent_macro_expansion (pfile);
-
-      (void) (*dir->handler) (pfile);
-
-      if (! (dir->flags & EXPAND))
-	restore_macro_expansion (pfile, prev_lvl);
-      
-      _cpp_skip_rest_of_line (pfile);
-      cpp_pop_buffer (pfile);
-    }
+  POOL_COMMIT (pool, len);
+  return result;
 }