diff options
author | Neil Booth <neil@daikokuya.co.uk> | 2003-04-19 00:22:51 +0000 |
---|---|---|
committer | Neil Booth <neil@gcc.gnu.org> | 2003-04-19 00:22:51 +0000 |
commit | 26aea0737b2d0c4ee328d547555ae25ab5de9a0a (patch) | |
tree | 6e2502497aefe03b070107d7b41bbe6be454b727 /gcc/cpplex.c | |
parent | 606c99912372a1fd5354765992eaef014ab78ce2 (diff) | |
download | gcc-26aea0737b2d0c4ee328d547555ae25ab5de9a0a.zip gcc-26aea0737b2d0c4ee328d547555ae25ab5de9a0a.tar.gz gcc-26aea0737b2d0c4ee328d547555ae25ab5de9a0a.tar.bz2 |
cppfiles.c (ENABLE_VALGRIND_CHECKING, [...]): Remove.
* cppfiles.c (ENABLE_VALGRIND_CHECKING, VALGRIND_DISCARD,
MMAP_THRESHOLD, TEST_THRESHOLD, SHOULD_MMAP): Remove.
(struct include_file): Remove fefcnt, mapped members.
(open_file, stack_include_file, _cpp_pop_file_buffer): Disable caching.
(read_include_file): Don't use mmap, terminate buffers in '\r'.
(purge_cache): Don't use munmap.
* cpphash.h (CPP_BUF_COLUMN): Update.
(lexer_state): Remove lexing_comment.
(struct _cpp_line_note): New.
(struct cpp_buffer): New members cur_note, notes_used, notes_cap,
next_line and need_line. Remove col_adjust and saved_flags.
(_cpp_process_line_notes, _cpp_clean_line, _cpp_get_fresh_line,
_cpp_skip_block_comment, scan_out_logical_line): New.
(_cpp_init_mbchar): Remove.
* cppinit.c (init_library): Remove call to _cpp_init_mbchar.
(cpp_read_main_file): Set line to 1 earlier.
(post_options): -traditional-cpp doesn't want trigraphs.
* cpplex.c (MULTIBYTE_CHARS): Remove code predicated on this.
(add_line_note, _cpp_clean_line, _cpp_process_line_notes,
_cpp_get_fresh_line): New.
(handle_newline, skip_escaped_newlines, trigraph_p,
continue_after_nul, _cpp_init_mbchar): Remove.
(get_effective_char): Update.
(_cpp_skip_block_comment): Rename from skip_block_comment, simplify.
(skip_line_comment): Simplify.
(skip_whitespace, parse_identifier, parse_slow, parse_number,
parse_string): Update.
(cpp_lex_direct): Use clean lines and process line notes. Update.
(cpp_interpret_charconst): No MULTIBYTE_CHARS.
* cpplib.c (prepare_directive_trad): Call scan_out_logical_line
directly.
(_cpp_handle_directive): Don't set saved_flags.
(run_directive, destringize_and_run, cpp_define, cpp_define_builtin,
cpp_undef, handle_assertion, cpp_push_buffer): Update.
(_cpp_pop_buffer): Free notes.
* cppmacro.c (builtin_macro, paste_tokens): \n terminate buffer.
* cpppch.c (cpp_read_state): \n terminate buffer.
* cpptrad.c (skip_escaped_newlines, handle_newline): Remove.
(copy_comment): Use _cpp_skip_block_comment.
(skip_whitespace, lex_identifier, _cpp_read_logical_line_trad):
Simplify.
(_cpp_overlay_buffer, _cpp_remove_overlay, push_replacement_text,
save_replacement_text): Update.
(scan_out_logical_line): Update to use clean lines and process
line notes.
* fix-header.c (read_scan_file): Update.
testsuite:
* gcc.dg/cpp/_Pragma4.c: Remove stray space.
* gcc.dg/cpp/trad/escaped-eof.c: Correct line number.
From-SVN: r65808
Diffstat (limited to 'gcc/cpplex.c')
-rw-r--r-- | gcc/cpplex.c | 592 |
1 files changed, 222 insertions, 370 deletions
diff --git a/gcc/cpplex.c b/gcc/cpplex.c index 93e04b8..930e887 100644 --- a/gcc/cpplex.c +++ b/gcc/cpplex.c @@ -4,7 +4,6 @@ Based on CCCP program by Paul Rubin, June 1986 Adapted to ANSI C, Richard Stallman, Jan 1987 Broken out to separate file, Zack Weinberg, Mar 2000 - Single-pass line tokenization by Neil Booth, April 2000 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -27,11 +26,6 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include "cpplib.h" #include "cpphash.h" -#ifdef MULTIBYTE_CHARS -#include "mbchar.h" -#include <locale.h> -#endif - /* Tokens with SPELL_STRING store their spelling in the token list, and it's length in the token->val.name.len. */ enum spell_type @@ -63,24 +57,19 @@ static const struct token_spelling token_spellings[N_TTYPES] = { TTYPE_TABLE }; #define TOKEN_NAME(token) (token_spellings[(token)->type].name) #define BACKUP() do {buffer->cur = buffer->backup_to;} while (0) -static void handle_newline PARAMS ((cpp_reader *)); -static cppchar_t skip_escaped_newlines PARAMS ((cpp_reader *)); +static void add_line_note PARAMS ((cpp_buffer *, const uchar *, unsigned int)); static cppchar_t get_effective_char PARAMS ((cpp_reader *)); -static int skip_block_comment PARAMS ((cpp_reader *)); static int skip_line_comment PARAMS ((cpp_reader *)); -static void adjust_column PARAMS ((cpp_reader *)); -static int skip_whitespace PARAMS ((cpp_reader *, cppchar_t)); +static void skip_whitespace PARAMS ((cpp_reader *, cppchar_t)); static cpp_hashnode *parse_identifier PARAMS ((cpp_reader *)); static uchar *parse_slow PARAMS ((cpp_reader *, const uchar *, int, unsigned int *)); static void parse_number PARAMS ((cpp_reader *, cpp_string *, int)); static int unescaped_terminator_p PARAMS ((cpp_reader *, const uchar *)); static void parse_string PARAMS ((cpp_reader *, cpp_token *, cppchar_t)); -static bool trigraph_p PARAMS ((cpp_reader *)); static void save_comment PARAMS ((cpp_reader *, cpp_token *, const uchar *, cppchar_t)); -static bool continue_after_nul PARAMS ((cpp_reader *)); static int name_p PARAMS ((cpp_reader *, const cpp_string *)); static int maybe_read_ucs PARAMS ((cpp_reader *, const unsigned char **, const unsigned char *, cppchar_t *)); @@ -89,15 +78,6 @@ static tokenrun *next_tokenrun PARAMS ((tokenrun *)); static unsigned int hex_digit_value PARAMS ((unsigned int)); static _cpp_buff *new_buff PARAMS ((size_t)); -/* Change to the native locale for multibyte conversions. */ -void -_cpp_init_mbchar () -{ -#ifdef MULTIBYTE_CHARS - setlocale (LC_CTYPE, ""); - GET_ENVIRONMENT (literal_codeset, "LANG"); -#endif -} /* Utility routine: @@ -114,132 +94,158 @@ cpp_ideq (token, string) return !ustrcmp (NODE_NAME (token->val.node), (const uchar *) string); } -/* Call when meeting a newline, assumed to be in buffer->cur[-1]. - Returns with buffer->cur pointing to the character immediately - following the newline (combination). */ +/* Record a note TYPE at byte POS into the current cleaned logical + line. */ static void -handle_newline (pfile) - cpp_reader *pfile; +add_line_note (buffer, pos, type) + cpp_buffer *buffer; + const uchar *pos; + unsigned int type; { - cpp_buffer *buffer = pfile->buffer; - - /* Handle CR-LF and LF-CR. Most other implementations (e.g. java) - only accept CR-LF; maybe we should fall back to that behavior? */ - if (buffer->cur[-1] + buffer->cur[0] == '\r' + '\n') - buffer->cur++; + if (buffer->notes_used == buffer->notes_cap) + { + buffer->notes_cap = buffer->notes_cap * 2 + 200; + buffer->notes = (_cpp_line_note *) + xrealloc (buffer->notes, buffer->notes_cap * sizeof (_cpp_line_note)); + } - buffer->line_base = buffer->cur; - buffer->col_adjust = 0; - pfile->line++; + buffer->notes[buffer->notes_used].pos = pos; + buffer->notes[buffer->notes_used].type = type; + buffer->notes_used++; } -/* Subroutine of skip_escaped_newlines; called when a 3-character - sequence beginning with "??" is encountered. buffer->cur points to - the second '?'. - - Warn if necessary, and returns true if the sequence forms a - trigraph and the trigraph should be honored. */ -static bool -trigraph_p (pfile) +/* Returns with a logical line that contains no escaped newlines or + trigraphs. This is a time-critical inner loop. */ +void +_cpp_clean_line (pfile) cpp_reader *pfile; { - cpp_buffer *buffer = pfile->buffer; - cppchar_t from_char = buffer->cur[1]; - bool accept; - - if (!_cpp_trigraph_map[from_char]) - return false; + cpp_buffer *buffer; + const uchar *s; + uchar c, *d, *p; - accept = CPP_OPTION (pfile, trigraphs); + buffer = pfile->buffer; + buffer->cur_note = buffer->notes_used = 0; + buffer->cur = buffer->line_base = buffer->next_line; + buffer->need_line = false; + s = buffer->next_line - 1; - /* Don't warn about trigraphs in comments. */ - if (CPP_OPTION (pfile, warn_trigraphs) && !pfile->state.lexing_comment) + if (!buffer->from_stage3) { - if (accept) - cpp_error_with_line (pfile, DL_WARNING, - pfile->line, CPP_BUF_COL (buffer) - 1, - "trigraph ??%c converted to %c", - (int) from_char, - (int) _cpp_trigraph_map[from_char]); - else if (buffer->cur != buffer->last_Wtrigraphs) + d = (uchar *) s; + + for (;;) { - buffer->last_Wtrigraphs = buffer->cur; - cpp_error_with_line (pfile, DL_WARNING, - pfile->line, CPP_BUF_COL (buffer) - 1, - "trigraph ??%c ignored", (int) from_char); + c = *++s; + *++d = c; + + if (c == '\n' || c == '\r') + { + /* Handle DOS line endings. */ + if (c == '\r' && s != buffer->rlimit && s[1] == '\n') + s++; + if (s == buffer->rlimit) + break; + + /* Escaped? */ + p = d; + while (p != buffer->next_line && is_nvspace (p[-1])) + p--; + if (p == buffer->next_line || p[-1] != '\\') + break; + + add_line_note (buffer, p - 1, + p != d ? NOTE_ESC_SPACE_NL: NOTE_ESC_NL); + d = p - 2; + buffer->next_line = p - 1; + } + else if (c == '?' && s[1] == '?' && _cpp_trigraph_map[s[2]]) + { + /* Add a note regardless, for the benefit of -Wtrigraphs. */ + add_line_note (buffer, d, NOTE_TRIGRAPH); + if (CPP_OPTION (pfile, trigraphs)) + { + *d = _cpp_trigraph_map[s[2]]; + s += 2; + } + } } } + else + { + do + s++; + while (*s != '\n' && *s != '\r'); + d = (uchar *) s; + + /* Handle DOS line endings. */ + if (*s == '\r' && s != buffer->rlimit && s[1] == '\n') + s++; + } - return accept; + *d = '\n'; + add_line_note (buffer, d + 1, NOTE_NEWLINE); + buffer->next_line = s + 1; } -/* Skips any escaped newlines introduced by '?' or a '\\', assumed to - lie in buffer->cur[-1]. Returns the next byte, which will be in - buffer->cur[-1]. This routine performs preprocessing stages 1 and - 2 of the ISO C standard. */ -static cppchar_t -skip_escaped_newlines (pfile) +/* Process the notes created by add_line_note as far as the current + location. */ +void +_cpp_process_line_notes (pfile, in_comment) cpp_reader *pfile; + int in_comment; { cpp_buffer *buffer = pfile->buffer; - cppchar_t next = buffer->cur[-1]; - /* Only do this if we apply stages 1 and 2. */ - if (!buffer->from_stage3) + for (;;) { - const unsigned char *saved_cur; - cppchar_t next1; + _cpp_line_note *note = &buffer->notes[buffer->cur_note]; + unsigned int col; - do - { - if (next == '?') - { - if (buffer->cur[0] != '?' || !trigraph_p (pfile)) - break; - - /* Translate the trigraph. */ - next = _cpp_trigraph_map[buffer->cur[1]]; - buffer->cur += 2; - if (next != '\\') - break; - } + if (note->pos > buffer->cur) + break; - if (buffer->cur == buffer->rlimit) - break; + buffer->cur_note++; + col = CPP_BUF_COLUMN (buffer, note->pos + 1); - /* We have a backslash, and room for at least one more - character. Skip horizontal whitespace. */ - saved_cur = buffer->cur; - do - next1 = *buffer->cur++; - while (is_nvspace (next1) && buffer->cur < buffer->rlimit); + switch (note->type) + { + case NOTE_NEWLINE: + /* This note is a kind of sentinel we should never reach. */ + abort (); - if (!is_vspace (next1)) + case NOTE_TRIGRAPH: + if (!in_comment && CPP_OPTION (pfile, warn_trigraphs)) { - buffer->cur = saved_cur; - break; + if (CPP_OPTION (pfile, trigraphs)) + cpp_error_with_line (pfile, DL_WARNING, pfile->line, col, + "trigraph converted to %c", + (int) note->pos[0]); + else + cpp_error_with_line (pfile, DL_WARNING, pfile->line, col, + "trigraph ??%c ignored", + (int) note->pos[2]); } + break; - if (saved_cur != buffer->cur - 1 - && !pfile->state.lexing_comment) - cpp_error (pfile, DL_WARNING, - "backslash and newline separated by space"); - - handle_newline (pfile); - buffer->backup_to = buffer->cur; - if (buffer->cur == buffer->rlimit) + case NOTE_ESC_SPACE_NL: + if (!in_comment) + cpp_error_with_line (pfile, DL_WARNING, pfile->line, col, + "backslash and newline separated by space"); + /* Fall through... */ + case NOTE_ESC_NL: + if (buffer->next_line > buffer->rlimit) { - cpp_error (pfile, DL_PEDWARN, - "backslash-newline at end of file"); - next = EOF; + cpp_error_with_line (pfile, DL_PEDWARN, pfile->line, col, + "backslash-newline at end of file"); + /* Prevent "no newline at end of file" warning. */ + buffer->next_line = buffer->rlimit; } - else - next = *buffer->cur++; + + buffer->line_base = note->pos; + pfile->line++; } - while (next == '\\' || next == '?'); } - - return next; } /* Obtain the next character, after trigraph conversion and skipping @@ -251,42 +257,34 @@ static cppchar_t get_effective_char (pfile) cpp_reader *pfile; { - cppchar_t next; cpp_buffer *buffer = pfile->buffer; buffer->backup_to = buffer->cur; - next = *buffer->cur++; - if (__builtin_expect (next == '?' || next == '\\', 0)) - next = skip_escaped_newlines (pfile); - - return next; + return *buffer->cur++; } /* Skip a C-style block comment. We find the end of the comment by seeing if an asterisk is before every '/' we encounter. Returns nonzero if comment terminated by EOF, zero otherwise. */ -static int -skip_block_comment (pfile) +bool +_cpp_skip_block_comment (pfile) cpp_reader *pfile; { cpp_buffer *buffer = pfile->buffer; - cppchar_t c = EOF, prevc = EOF; + cppchar_t c; - pfile->state.lexing_comment = 1; - while (buffer->cur != buffer->rlimit) - { - prevc = c, c = *buffer->cur++; + if (*buffer->cur == '/') + buffer->cur++; - /* FIXME: For speed, create a new character class of characters - of interest inside block comments. */ - if (c == '?' || c == '\\') - c = skip_escaped_newlines (pfile); + for (;;) + { + c = *buffer->cur++; /* People like decorating comments with '*', so check for '/' instead for efficiency. */ if (c == '/') { - if (prevc == '*') + if (buffer->cur[-2] == '*') break; /* Warn about potential nested comments, but not if the '/' @@ -298,14 +296,18 @@ skip_block_comment (pfile) pfile->line, CPP_BUF_COL (buffer), "\"/*\" within comment"); } - else if (is_vspace (c)) - handle_newline (pfile); - else if (c == '\t') - adjust_column (pfile); + else if (c == '\n') + { + buffer->cur--; + _cpp_process_line_notes (pfile, true); + if (buffer->next_line >= buffer->rlimit) + return true; + _cpp_clean_line (pfile); + pfile->line++; + } } - pfile->state.lexing_comment = 0; - return c != '/' || prevc != '*'; + return false; } /* Skip a C++ line comment, leaving buffer->cur pointing to the @@ -317,72 +319,16 @@ skip_line_comment (pfile) { cpp_buffer *buffer = pfile->buffer; unsigned int orig_line = pfile->line; - cppchar_t c; -#ifdef MULTIBYTE_CHARS - wchar_t wc; - int char_len; -#endif - - pfile->state.lexing_comment = 1; -#ifdef MULTIBYTE_CHARS - /* Reset multibyte conversion state. */ - (void) local_mbtowc (NULL, NULL, 0); -#endif - do - { - if (buffer->cur == buffer->rlimit) - goto at_eof; - -#ifdef MULTIBYTE_CHARS - char_len = local_mbtowc (&wc, (const char *) buffer->cur, - buffer->rlimit - buffer->cur); - if (char_len == -1) - { - cpp_error (pfile, DL_WARNING, - "ignoring invalid multibyte character"); - char_len = 1; - c = *buffer->cur++; - } - else - { - buffer->cur += char_len; - c = wc; - } -#else - c = *buffer->cur++; -#endif - if (c == '?' || c == '\\') - c = skip_escaped_newlines (pfile); - } - while (!is_vspace (c)); - /* Step back over the newline, except at EOF. */ - buffer->cur--; - at_eof: + while (*buffer->cur != '\n') + buffer->cur++; - pfile->state.lexing_comment = 0; + _cpp_process_line_notes (pfile, true); return orig_line != pfile->line; } -/* pfile->buffer->cur is one beyond the \t character. Update - col_adjust so we track the column correctly. */ +/* Skips whitespace, saving the next non-whitespace character. */ static void -adjust_column (pfile) - cpp_reader *pfile; -{ - cpp_buffer *buffer = pfile->buffer; - unsigned int col = CPP_BUF_COL (buffer) - 1; /* Zero-based column. */ - - /* Round it up to multiple of the tabstop, but subtract 1 since the - tab itself occupies a character position. */ - buffer->col_adjust += (CPP_OPTION (pfile, tabstop) - - col % CPP_OPTION (pfile, tabstop)) - 1; -} - -/* Skips whitespace, saving the next non-whitespace character. - Adjusts pfile->col_adjust to account for tabs. Without this, - tokens might be assigned an incorrect column. */ -static int skip_whitespace (pfile, c) cpp_reader *pfile; cppchar_t c; @@ -393,15 +339,11 @@ skip_whitespace (pfile, c) do { /* Horizontal space always OK. */ - if (c == ' ') + if (c == ' ' || c == '\t') ; - else if (c == '\t') - adjust_column (pfile); /* Just \f \v or \0 left. */ else if (c == '\0') { - if (buffer->cur - 1 == buffer->rlimit) - return 0; if (!warned) { cpp_error (pfile, DL_WARNING, "null character(s) ignored"); @@ -420,7 +362,6 @@ skip_whitespace (pfile, c) while (is_nvspace (c)); buffer->cur--; - return 1; } /* See if the characters of a number token are valid in a name (no @@ -461,7 +402,7 @@ parse_identifier (pfile) cur++; /* Check for slow-path cases. */ - if (*cur == '?' || *cur == '\\' || *cur == '$') + if (*cur == '$') { unsigned int len; @@ -532,8 +473,6 @@ parse_slow (pfile, cur, number_p, plen) { /* Potential escaped newline? */ buffer->backup_to = buffer->cur - 1; - if (c == '?' || c == '\\') - c = skip_escaped_newlines (pfile); if (!is_idchar (c)) { @@ -590,7 +529,7 @@ parse_number (pfile, number, leading_period) cur++; /* Check for slow-path cases. */ - if (*cur == '?' || *cur == '\\' || *cur == '$') + if (*cur == '$') number->text = parse_slow (pfile, cur, 1 + leading_period, &number->len); else { @@ -648,18 +587,10 @@ parse_string (pfile, token, terminator) unsigned char *dest, *limit; cppchar_t c; bool warned_nulls = false; -#ifdef MULTIBYTE_CHARS - wchar_t wc; - int char_len; -#endif dest = BUFF_FRONT (pfile->u_buff); limit = BUFF_LIMIT (pfile->u_buff); -#ifdef MULTIBYTE_CHARS - /* Reset multibyte conversion state. */ - (void) local_mbtowc (NULL, NULL, 0); -#endif for (;;) { /* We need room for another char, possibly the terminating NUL. */ @@ -671,41 +602,19 @@ parse_string (pfile, token, terminator) limit = BUFF_LIMIT (pfile->u_buff); } -#ifdef MULTIBYTE_CHARS - char_len = local_mbtowc (&wc, (const char *) buffer->cur, - buffer->rlimit - buffer->cur); - if (char_len == -1) - { - cpp_error (pfile, DL_WARNING, - "ignoring invalid multibyte character"); - char_len = 1; - c = *buffer->cur++; - } - else - { - buffer->cur += char_len; - c = wc; - } -#else c = *buffer->cur++; -#endif - - /* Handle trigraphs, escaped newlines etc. */ - if (c == '?' || c == '\\') - c = skip_escaped_newlines (pfile); if (c == terminator) { if (unescaped_terminator_p (pfile, dest)) break; } - else if (is_vspace (c)) + else if (c == '\n') { /* No string literal may extend over multiple lines. In assembly language, suppress the error except for <> includes. This is a kludge around not knowing where comments are. */ - unterminated: if (CPP_OPTION (pfile, lang) != CLK_ASM || terminator == '>') cpp_error (pfile, DL_ERROR, "missing terminating %c character", (int) terminator); @@ -714,8 +623,6 @@ parse_string (pfile, token, terminator) } else if (c == '\0') { - if (buffer->cur - 1 == buffer->rlimit) - goto unterminated; if (!warned_nulls) { warned_nulls = true; @@ -723,14 +630,6 @@ parse_string (pfile, token, terminator) "null character(s) preserved in literal"); } } -#ifdef MULTIBYTE_CHARS - if (char_len > 1) - { - for ( ; char_len > 0; --char_len) - *dest++ = (*buffer->cur - char_len); - } - else -#endif *dest++ = c; } @@ -890,55 +789,55 @@ _cpp_lex_token (pfile) return result; } -/* A NUL terminates the current buffer. For ISO preprocessing this is - EOF, but for traditional preprocessing it indicates we need a line - refill. Returns TRUE to continue preprocessing a new buffer, FALSE - to return a CPP_EOF to the caller. */ -static bool -continue_after_nul (pfile) +/* Returns true if a fresh line has been loaded. */ +bool +_cpp_get_fresh_line (pfile) cpp_reader *pfile; { - cpp_buffer *buffer = pfile->buffer; - bool more = false; + /* We can't get a new line until we leave the current directive. */ + if (pfile->state.in_directive) + return false; - buffer->saved_flags = BOL; - if (CPP_OPTION (pfile, traditional)) + for (;;) { - if (pfile->state.in_directive) - return false; + cpp_buffer *buffer = pfile->buffer; - _cpp_remove_overlay (pfile); - more = _cpp_read_logical_line_trad (pfile); - _cpp_overlay_buffer (pfile, pfile->out.base, - pfile->out.cur - pfile->out.base); - pfile->line = pfile->out.first_line; - } - else - { - /* Stop parsing arguments with a CPP_EOF. When we finally come - back here, do the work of popping the buffer. */ - if (!pfile->state.parsing_args) + if (!buffer->need_line) + return true; + + if (buffer->next_line < buffer->rlimit) { - if (buffer->cur != buffer->line_base) - { - /* Non-empty files should end in a newline. Don't warn - for command line and _Pragma buffers. */ - if (!buffer->from_stage3) - cpp_error (pfile, DL_PEDWARN, "no newline at end of file"); - handle_newline (pfile); - } + _cpp_clean_line (pfile); + return true; + } - /* Similarly, finish an in-progress directive with CPP_EOF - before popping the buffer. */ - if (!pfile->state.in_directive && buffer->prev) - { - more = !buffer->return_at_eof; - _cpp_pop_buffer (pfile); - } + /* First, get out of parsing arguments state. */ + if (pfile->state.parsing_args) + return false; + + /* End of buffer. Non-empty files should end in a newline. */ + if (buffer->buf != buffer->rlimit + && buffer->next_line > buffer->rlimit + && !buffer->from_stage3) + { + /* Only warn once. */ + buffer->next_line = buffer->rlimit; + cpp_error_with_line (pfile, DL_PEDWARN, pfile->line - 1, + CPP_BUF_COLUMN (buffer, buffer->cur), + "no newline at end of file"); + } + + if (buffer->return_at_eof) + { + buffer->return_at_eof = false; + return false; } - } - return more; + if (!buffer->prev) + return false; + + _cpp_pop_buffer (pfile); + } } #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \ @@ -973,74 +872,49 @@ _cpp_lex_direct (pfile) cpp_token *result = pfile->cur_token++; fresh_line: + result->flags = 0; + if (pfile->buffer->need_line) + { + if (!_cpp_get_fresh_line (pfile)) + { + result->type = CPP_EOF; + return result; + } + if (!pfile->keep_tokens) + { + pfile->cur_run = &pfile->base_run; + result = pfile->base_run.base; + pfile->cur_token = result + 1; + } + result->flags = BOL; + if (pfile->state.parsing_args == 2) + result->flags |= PREV_WHITE; + } buffer = pfile->buffer; - result->flags = buffer->saved_flags; - buffer->saved_flags = 0; update_tokens_line: result->line = pfile->line; skipped_white: + if (buffer->cur >= buffer->notes[buffer->cur_note].pos + && !pfile->overlaid_buffer) + { + _cpp_process_line_notes (pfile, false); + result->line = pfile->line; + } c = *buffer->cur++; result->col = CPP_BUF_COLUMN (buffer, buffer->cur); - trigraph: switch (c) { case ' ': case '\t': case '\f': case '\v': case '\0': result->flags |= PREV_WHITE; - if (skip_whitespace (pfile, c)) - goto skipped_white; - - /* End of buffer. */ - buffer->cur--; - if (continue_after_nul (pfile)) - goto fresh_line; - result->type = CPP_EOF; - break; + skip_whitespace (pfile, c); + goto skipped_white; - case '\n': case '\r': - handle_newline (pfile); - buffer->saved_flags = BOL; - if (! pfile->state.in_directive) - { - if (pfile->state.parsing_args == 2) - buffer->saved_flags |= PREV_WHITE; - if (!pfile->keep_tokens) - { - pfile->cur_run = &pfile->base_run; - result = pfile->base_run.base; - pfile->cur_token = result + 1; - } - goto fresh_line; - } - result->type = CPP_EOF; - break; - - case '?': - case '\\': - /* These could start an escaped newline, or '?' a trigraph. Let - skip_escaped_newlines do all the work. */ - { - unsigned int line = pfile->line; - - c = skip_escaped_newlines (pfile); - if (line != pfile->line) - { - buffer->cur--; - /* We had at least one escaped newline of some sort. - Update the token's line and column. */ - goto update_tokens_line; - } - } - - /* We are either the original '?' or '\\', or a trigraph. */ - if (c == '?') - result->type = CPP_QUERY; - else if (c == '\\') - goto random_char; - else - goto trigraph; - break; + case '\n': + pfile->line++; + buffer->need_line = true; + goto fresh_line; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -1100,7 +974,7 @@ _cpp_lex_direct (pfile) if (c == '*') { - if (skip_block_comment (pfile)) + if (_cpp_skip_block_comment (pfile)) cpp_error (pfile, DL_ERROR, "unterminated comment"); } else if (c == '/' && (CPP_OPTION (pfile, cplusplus_comments) @@ -1331,6 +1205,7 @@ _cpp_lex_direct (pfile) case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break; case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); break; + case '?': result->type = CPP_QUERY; break; case '~': result->type = CPP_COMPL; break; case ',': result->type = CPP_COMMA; break; case '(': result->type = CPP_OPEN_PAREN; break; @@ -1349,7 +1224,6 @@ _cpp_lex_direct (pfile) goto start_ident; /* Fall through... */ - random_char: default: result->type = CPP_OTHER; result->val.c = c; @@ -1927,10 +1801,6 @@ cpp_interpret_charconst (pfile, token, pchars_seen, unsignedp) cppchar_t c, mask, result = 0; bool unsigned_p; -#ifdef MULTIBYTE_CHARS - (void) local_mbtowc (NULL, NULL, 0); -#endif - /* Width in bits. */ if (token->type == CPP_CHAR) { @@ -1952,25 +1822,7 @@ cpp_interpret_charconst (pfile, token, pchars_seen, unsignedp) while (str < limit) { -#ifdef MULTIBYTE_CHARS - wchar_t wc; - int char_len; - - char_len = local_mbtowc (&wc, (const char *)str, limit - str); - if (char_len == -1) - { - cpp_error (pfile, DL_WARNING, - "ignoring invalid multibyte character"); - c = *str++; - } - else - { - str += char_len; - c = wc; - } -#else c = *str++; -#endif if (c == '\\') c = cpp_parse_escape (pfile, &str, limit, token->type == CPP_WCHAR); |