diff options
Diffstat (limited to 'libcpp/charset.cc')
| -rw-r--r-- | libcpp/charset.cc | 2510 |
1 files changed, 2510 insertions, 0 deletions
diff --git a/libcpp/charset.cc b/libcpp/charset.cc new file mode 100644 index 0000000..b306f5d --- /dev/null +++ b/libcpp/charset.cc @@ -0,0 +1,2510 @@ +/* CPP Library - charsets + Copyright (C) 1998-2022 Free Software Foundation, Inc. + + Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges. + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; either version 3, or (at your option) any +later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; see the file COPYING3. If not see +<http://www.gnu.org/licenses/>. */ + +#include "config.h" +#include "system.h" +#include "cpplib.h" +#include "internal.h" + +/* Character set handling for C-family languages. + + Terminological note: In what follows, "charset" or "character set" + will be taken to mean both an abstract set of characters and an + encoding for that set. + + The C99 standard discusses two character sets: source and execution. + The source character set is used for internal processing in translation + phases 1 through 4; the execution character set is used thereafter. + Both are required by 5.2.1.2p1 to be multibyte encodings, not wide + character encodings (see 3.7.2, 3.7.3 for the standardese meanings + of these terms). Furthermore, the "basic character set" (listed in + 5.2.1p3) is to be encoded in each with values one byte wide, and is + to appear in the initial shift state. + + It is not explicitly mentioned, but there is also a "wide execution + character set" used to encode wide character constants and wide + string literals; this is supposed to be the result of applying the + standard library function mbstowcs() to an equivalent narrow string + (6.4.5p5). However, the behavior of hexadecimal and octal + \-escapes is at odds with this; they are supposed to be translated + directly to wchar_t values (6.4.4.4p5,6). + + The source character set is not necessarily the character set used + to encode physical source files on disk; translation phase 1 converts + from whatever that encoding is to the source character set. + + The presence of universal character names in C99 (6.4.3 et seq.) + forces the source character set to be isomorphic to ISO 10646, + that is, Unicode. There is no such constraint on the execution + character set; note also that the conversion from source to + execution character set does not occur for identifiers (5.1.1.2p1#5). + + For convenience of implementation, the source character set's + encoding of the basic character set should be identical to the + execution character set OF THE HOST SYSTEM's encoding of the basic + character set, and it should not be a state-dependent encoding. + + cpplib uses UTF-8 or UTF-EBCDIC for the source character set, + depending on whether the host is based on ASCII or EBCDIC (see + respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode + Technical Report #16). With limited exceptions, it relies on the + system library's iconv() primitive to do charset conversion + (specified in SUSv2). */ + +#if !HAVE_ICONV +/* Make certain that the uses of iconv(), iconv_open(), iconv_close() + below, which are guarded only by if statements with compile-time + constant conditions, do not cause link errors. */ +#define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1) +#define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1) +#define iconv_close(x) (void)0 +#define ICONV_CONST +#endif + +#if HOST_CHARSET == HOST_CHARSET_ASCII +#define SOURCE_CHARSET "UTF-8" +#define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e +#elif HOST_CHARSET == HOST_CHARSET_EBCDIC +#define SOURCE_CHARSET "UTF-EBCDIC" +#define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF +#else +#error "Unrecognized basic host character set" +#endif + +#ifndef EILSEQ +#define EILSEQ EINVAL +#endif + +/* This structure is used for a resizable string buffer throughout. */ +/* Don't call it strbuf, as that conflicts with unistd.h on systems + such as DYNIX/ptx where unistd.h includes stropts.h. */ +struct _cpp_strbuf +{ + uchar *text; + size_t asize; + size_t len; +}; + +/* This is enough to hold any string that fits on a single 80-column + line, even if iconv quadruples its size (e.g. conversion from + ASCII to UTF-32) rounded up to a power of two. */ +#define OUTBUF_BLOCK_SIZE 256 + +/* Conversions between UTF-8 and UTF-16/32 are implemented by custom + logic. This is because a depressing number of systems lack iconv, + or have have iconv libraries that do not do these conversions, so + we need a fallback implementation for them. To ensure the fallback + doesn't break due to neglect, it is used on all systems. + + UTF-32 encoding is nice and simple: a four-byte binary number, + constrained to the range 00000000-7FFFFFFF to avoid questions of + signedness. We do have to cope with big- and little-endian + variants. + + UTF-16 encoding uses two-byte binary numbers, again in big- and + little-endian variants, for all values in the 00000000-0000FFFF + range. Values in the 00010000-0010FFFF range are encoded as pairs + of two-byte numbers, called "surrogate pairs": given a number S in + this range, it is mapped to a pair (H, L) as follows: + + H = (S - 0x10000) / 0x400 + 0xD800 + L = (S - 0x10000) % 0x400 + 0xDC00 + + Two-byte values in the D800...DFFF range are ill-formed except as a + component of a surrogate pair. Even if the encoding within a + two-byte value is little-endian, the H member of the surrogate pair + comes first. + + There is no way to encode values in the 00110000-7FFFFFFF range, + which is not currently a problem as there are no assigned code + points in that range; however, the author expects that it will + eventually become necessary to abandon UTF-16 due to this + limitation. Note also that, because of these pairs, UTF-16 does + not meet the requirements of the C standard for a wide character + encoding (see 3.7.3 and 6.4.4.4p11). + + UTF-8 encoding looks like this: + + value range encoded as + 00000000-0000007F 0xxxxxxx + 00000080-000007FF 110xxxxx 10xxxxxx + 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx + 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + + Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid, + which means that three-byte sequences ED xx yy, with A0 <= xx <= BF, + never occur. Note also that any value that can be encoded by a + given row of the table can also be encoded by all successive rows, + but this is not done; only the shortest possible encoding for any + given value is valid. For instance, the character 07C0 could be + encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or + FC 80 80 80 9F 80. Only the first is valid. + + An implementation note: the transformation from UTF-16 to UTF-8, or + vice versa, is easiest done by using UTF-32 as an intermediary. */ + +/* Internal primitives which go from an UTF-8 byte stream to native-endian + UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal + operation in several places below. */ +static inline int +one_utf8_to_cppchar (const uchar **inbufp, size_t *inbytesleftp, + cppchar_t *cp) +{ + static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 }; + static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + + cppchar_t c; + const uchar *inbuf = *inbufp; + size_t nbytes, i; + + if (*inbytesleftp < 1) + return EINVAL; + + c = *inbuf; + if (c < 0x80) + { + *cp = c; + *inbytesleftp -= 1; + *inbufp += 1; + return 0; + } + + /* The number of leading 1-bits in the first byte indicates how many + bytes follow. */ + for (nbytes = 2; nbytes < 7; nbytes++) + if ((c & ~masks[nbytes-1]) == patns[nbytes-1]) + goto found; + return EILSEQ; + found: + + if (*inbytesleftp < nbytes) + return EINVAL; + + c = (c & masks[nbytes-1]); + inbuf++; + for (i = 1; i < nbytes; i++) + { + cppchar_t n = *inbuf++; + if ((n & 0xC0) != 0x80) + return EILSEQ; + c = ((c << 6) + (n & 0x3F)); + } + + /* Make sure the shortest possible encoding was used. */ + if (c <= 0x7F && nbytes > 1) return EILSEQ; + if (c <= 0x7FF && nbytes > 2) return EILSEQ; + if (c <= 0xFFFF && nbytes > 3) return EILSEQ; + if (c <= 0x1FFFFF && nbytes > 4) return EILSEQ; + if (c <= 0x3FFFFFF && nbytes > 5) return EILSEQ; + + /* Make sure the character is valid. */ + if (c > 0x7FFFFFFF || (c >= 0xD800 && c <= 0xDFFF)) return EILSEQ; + + *cp = c; + *inbufp = inbuf; + *inbytesleftp -= nbytes; + return 0; +} + +static inline int +one_cppchar_to_utf8 (cppchar_t c, uchar **outbufp, size_t *outbytesleftp) +{ + static const uchar masks[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + static const uchar limits[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE }; + size_t nbytes; + uchar buf[6], *p = &buf[6]; + uchar *outbuf = *outbufp; + + nbytes = 1; + if (c < 0x80) + *--p = c; + else + { + do + { + *--p = ((c & 0x3F) | 0x80); + c >>= 6; + nbytes++; + } + while (c >= 0x3F || (c & limits[nbytes-1])); + *--p = (c | masks[nbytes-1]); + } + + if (*outbytesleftp < nbytes) + return E2BIG; + + while (p < &buf[6]) + *outbuf++ = *p++; + *outbytesleftp -= nbytes; + *outbufp = outbuf; + return 0; +} + +/* The following four functions transform one character between the two + encodings named in the function name. All have the signature + int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp, + uchar **outbufp, size_t *outbytesleftp) + + BIGEND must have the value 0 or 1, coerced to (iconv_t); it is + interpreted as a boolean indicating whether big-endian or + little-endian encoding is to be used for the member of the pair + that is not UTF-8. + + INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they + do for iconv. + + The return value is either 0 for success, or an errno value for + failure, which may be E2BIG (need more space), EILSEQ (ill-formed + input sequence), ir EINVAL (incomplete input sequence). */ + +static inline int +one_utf8_to_utf32 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp, + uchar **outbufp, size_t *outbytesleftp) +{ + uchar *outbuf; + cppchar_t s = 0; + int rval; + + /* Check for space first, since we know exactly how much we need. */ + if (*outbytesleftp < 4) + return E2BIG; + + rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s); + if (rval) + return rval; + + outbuf = *outbufp; + outbuf[bigend ? 3 : 0] = (s & 0x000000FF); + outbuf[bigend ? 2 : 1] = (s & 0x0000FF00) >> 8; + outbuf[bigend ? 1 : 2] = (s & 0x00FF0000) >> 16; + outbuf[bigend ? 0 : 3] = (s & 0xFF000000) >> 24; + + *outbufp += 4; + *outbytesleftp -= 4; + return 0; +} + +static inline int +one_utf32_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp, + uchar **outbufp, size_t *outbytesleftp) +{ + cppchar_t s; + int rval; + const uchar *inbuf; + + if (*inbytesleftp < 4) + return EINVAL; + + inbuf = *inbufp; + + s = inbuf[bigend ? 0 : 3] << 24; + s += inbuf[bigend ? 1 : 2] << 16; + s += inbuf[bigend ? 2 : 1] << 8; + s += inbuf[bigend ? 3 : 0]; + + if (s >= 0x7FFFFFFF || (s >= 0xD800 && s <= 0xDFFF)) + return EILSEQ; + + rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp); + if (rval) + return rval; + + *inbufp += 4; + *inbytesleftp -= 4; + return 0; +} + +static inline int +one_utf8_to_utf16 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp, + uchar **outbufp, size_t *outbytesleftp) +{ + int rval; + cppchar_t s = 0; + const uchar *save_inbuf = *inbufp; + size_t save_inbytesleft = *inbytesleftp; + uchar *outbuf = *outbufp; + + rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s); + if (rval) + return rval; + + if (s > 0x0010FFFF) + { + *inbufp = save_inbuf; + *inbytesleftp = save_inbytesleft; + return EILSEQ; + } + + if (s <= 0xFFFF) + { + if (*outbytesleftp < 2) + { + *inbufp = save_inbuf; + *inbytesleftp = save_inbytesleft; + return E2BIG; + } + outbuf[bigend ? 1 : 0] = (s & 0x00FF); + outbuf[bigend ? 0 : 1] = (s & 0xFF00) >> 8; + + *outbufp += 2; + *outbytesleftp -= 2; + return 0; + } + else + { + cppchar_t hi, lo; + + if (*outbytesleftp < 4) + { + *inbufp = save_inbuf; + *inbytesleftp = save_inbytesleft; + return E2BIG; + } + + hi = (s - 0x10000) / 0x400 + 0xD800; + lo = (s - 0x10000) % 0x400 + 0xDC00; + + /* Even if we are little-endian, put the high surrogate first. + ??? Matches practice? */ + outbuf[bigend ? 1 : 0] = (hi & 0x00FF); + outbuf[bigend ? 0 : 1] = (hi & 0xFF00) >> 8; + outbuf[bigend ? 3 : 2] = (lo & 0x00FF); + outbuf[bigend ? 2 : 3] = (lo & 0xFF00) >> 8; + + *outbufp += 4; + *outbytesleftp -= 4; + return 0; + } +} + +static inline int +one_utf16_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp, + uchar **outbufp, size_t *outbytesleftp) +{ + cppchar_t s; + const uchar *inbuf = *inbufp; + int rval; + + if (*inbytesleftp < 2) + return EINVAL; + s = inbuf[bigend ? 0 : 1] << 8; + s += inbuf[bigend ? 1 : 0]; + + /* Low surrogate without immediately preceding high surrogate is invalid. */ + if (s >= 0xDC00 && s <= 0xDFFF) + return EILSEQ; + /* High surrogate must have a following low surrogate. */ + else if (s >= 0xD800 && s <= 0xDBFF) + { + cppchar_t hi = s, lo; + if (*inbytesleftp < 4) + return EINVAL; + + lo = inbuf[bigend ? 2 : 3] << 8; + lo += inbuf[bigend ? 3 : 2]; + + if (lo < 0xDC00 || lo > 0xDFFF) + return EILSEQ; + + s = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000; + } + + rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp); + if (rval) + return rval; + + /* Success - update the input pointers (one_cppchar_to_utf8 has done + the output pointers for us). */ + if (s <= 0xFFFF) + { + *inbufp += 2; + *inbytesleftp -= 2; + } + else + { + *inbufp += 4; + *inbytesleftp -= 4; + } + return 0; +} + +/* Helper routine for the next few functions. The 'const' on + one_conversion means that we promise not to modify what function is + pointed to, which lets the inliner see through it. */ + +static inline bool +conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *, + uchar **, size_t *), + iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to) +{ + const uchar *inbuf; + uchar *outbuf; + size_t inbytesleft, outbytesleft; + int rval; + + inbuf = from; + inbytesleft = flen; + outbuf = to->text + to->len; + outbytesleft = to->asize - to->len; + + for (;;) + { + do + rval = one_conversion (cd, &inbuf, &inbytesleft, + &outbuf, &outbytesleft); + while (inbytesleft && !rval); + + if (__builtin_expect (inbytesleft == 0, 1)) + { + to->len = to->asize - outbytesleft; + return true; + } + if (rval != E2BIG) + { + errno = rval; + return false; + } + + outbytesleft += OUTBUF_BLOCK_SIZE; + to->asize += OUTBUF_BLOCK_SIZE; + to->text = XRESIZEVEC (uchar, to->text, to->asize); + outbuf = to->text + to->asize - outbytesleft; + } +} + + +/* These functions convert entire strings between character sets. + They all have the signature + + bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to); + + The input string FROM is converted as specified by the function + name plus the iconv descriptor CD (which may be fake), and the + result appended to TO. On any error, false is returned, otherwise true. */ + +/* These four use the custom conversion code above. */ +static bool +convert_utf8_utf16 (iconv_t cd, const uchar *from, size_t flen, + struct _cpp_strbuf *to) +{ + return conversion_loop (one_utf8_to_utf16, cd, from, flen, to); +} + +static bool +convert_utf8_utf32 (iconv_t cd, const uchar *from, size_t flen, + struct _cpp_strbuf *to) +{ + return conversion_loop (one_utf8_to_utf32, cd, from, flen, to); +} + +static bool +convert_utf16_utf8 (iconv_t cd, const uchar *from, size_t flen, + struct _cpp_strbuf *to) +{ + return conversion_loop (one_utf16_to_utf8, cd, from, flen, to); +} + +static bool +convert_utf32_utf8 (iconv_t cd, const uchar *from, size_t flen, + struct _cpp_strbuf *to) +{ + return conversion_loop (one_utf32_to_utf8, cd, from, flen, to); +} + +/* Identity conversion, used when we have no alternative. */ +static bool +convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED, + const uchar *from, size_t flen, struct _cpp_strbuf *to) +{ + if (to->len + flen > to->asize) + { + to->asize = to->len + flen; + to->asize += to->asize / 4; + to->text = XRESIZEVEC (uchar, to->text, to->asize); + } + memcpy (to->text + to->len, from, flen); + to->len += flen; + return true; +} + +/* And this one uses the system iconv primitive. It's a little + different, since iconv's interface is a little different. */ +#if HAVE_ICONV + +#define CONVERT_ICONV_GROW_BUFFER \ + do { \ + outbytesleft += OUTBUF_BLOCK_SIZE; \ + to->asize += OUTBUF_BLOCK_SIZE; \ + to->text = XRESIZEVEC (uchar, to->text, to->asize); \ + outbuf = (char *)to->text + to->asize - outbytesleft; \ + } while (0) + +static bool +convert_using_iconv (iconv_t cd, const uchar *from, size_t flen, + struct _cpp_strbuf *to) +{ + ICONV_CONST char *inbuf; + char *outbuf; + size_t inbytesleft, outbytesleft; + + /* Reset conversion descriptor and check that it is valid. */ + if (iconv (cd, 0, 0, 0, 0) == (size_t)-1) + return false; + + inbuf = (ICONV_CONST char *)from; + inbytesleft = flen; + outbuf = (char *)to->text + to->len; + outbytesleft = to->asize - to->len; + + for (;;) + { + iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); + if (__builtin_expect (inbytesleft == 0, 1)) + { + /* Close out any shift states, returning to the initial state. */ + if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1) + { + if (errno != E2BIG) + return false; + + CONVERT_ICONV_GROW_BUFFER; + if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1) + return false; + } + + to->len = to->asize - outbytesleft; + return true; + } + if (errno != E2BIG) + return false; + + CONVERT_ICONV_GROW_BUFFER; + } +} +#else +#define convert_using_iconv 0 /* prevent undefined symbol error below */ +#endif + +/* Arrange for the above custom conversion logic to be used automatically + when conversion between a suitable pair of character sets is requested. */ + +#define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \ + CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO) + +struct cpp_conversion +{ + const char *pair; + convert_f func; + iconv_t fake_cd; +}; +static const struct cpp_conversion conversion_tab[] = { + { "UTF-8/UTF-32LE", convert_utf8_utf32, (iconv_t)0 }, + { "UTF-8/UTF-32BE", convert_utf8_utf32, (iconv_t)1 }, + { "UTF-8/UTF-16LE", convert_utf8_utf16, (iconv_t)0 }, + { "UTF-8/UTF-16BE", convert_utf8_utf16, (iconv_t)1 }, + { "UTF-32LE/UTF-8", convert_utf32_utf8, (iconv_t)0 }, + { "UTF-32BE/UTF-8", convert_utf32_utf8, (iconv_t)1 }, + { "UTF-16LE/UTF-8", convert_utf16_utf8, (iconv_t)0 }, + { "UTF-16BE/UTF-8", convert_utf16_utf8, (iconv_t)1 }, +}; + +/* Subroutine of cpp_init_iconv: initialize and return a + cset_converter structure for conversion from FROM to TO. If + iconv_open() fails, issue an error and return an identity + converter. Silently return an identity converter if FROM and TO + are identical. + + PFILE is only used for generating diagnostics; setting it to NULL + suppresses diagnostics. */ + +static struct cset_converter +init_iconv_desc (cpp_reader *pfile, const char *to, const char *from) +{ + struct cset_converter ret; + char *pair; + size_t i; + + ret.to = to; + ret.from = from; + + if (!strcasecmp (to, from)) + { + ret.func = convert_no_conversion; + ret.cd = (iconv_t) -1; + ret.width = -1; + return ret; + } + + pair = (char *) alloca(strlen(to) + strlen(from) + 2); + + strcpy(pair, from); + strcat(pair, "/"); + strcat(pair, to); + for (i = 0; i < ARRAY_SIZE (conversion_tab); i++) + if (!strcasecmp (pair, conversion_tab[i].pair)) + { + ret.func = conversion_tab[i].func; + ret.cd = conversion_tab[i].fake_cd; + ret.width = -1; + return ret; + } + + /* No custom converter - try iconv. */ + if (HAVE_ICONV) + { + ret.func = convert_using_iconv; + ret.cd = iconv_open (to, from); + ret.width = -1; + + if (ret.cd == (iconv_t) -1) + { + if (pfile) + { + if (errno == EINVAL) + cpp_error (pfile, CPP_DL_ERROR, /* FIXME should be DL_SORRY */ + "conversion from %s to %s not supported by iconv", + from, to); + else + cpp_errno (pfile, CPP_DL_ERROR, "iconv_open"); + } + ret.func = convert_no_conversion; + } + } + else + { + if (pfile) + { + cpp_error (pfile, CPP_DL_ERROR, /* FIXME: should be DL_SORRY */ + "no iconv implementation, cannot convert from %s to %s", + from, to); + } + ret.func = convert_no_conversion; + ret.cd = (iconv_t) -1; + ret.width = -1; + } + + return ret; +} + +/* If charset conversion is requested, initialize iconv(3) descriptors + for conversion from the source character set to the execution + character sets. If iconv is not present in the C library, and + conversion is requested, issue an error. */ + +void +cpp_init_iconv (cpp_reader *pfile) +{ + const char *ncset = CPP_OPTION (pfile, narrow_charset); + const char *wcset = CPP_OPTION (pfile, wide_charset); + const char *default_wcset; + + bool be = CPP_OPTION (pfile, bytes_big_endian); + + if (CPP_OPTION (pfile, wchar_precision) >= 32) + default_wcset = be ? "UTF-32BE" : "UTF-32LE"; + else if (CPP_OPTION (pfile, wchar_precision) >= 16) + default_wcset = be ? "UTF-16BE" : "UTF-16LE"; + else + /* This effectively means that wide strings are not supported, + so don't do any conversion at all. */ + default_wcset = SOURCE_CHARSET; + + if (!ncset) + ncset = SOURCE_CHARSET; + if (!wcset) + wcset = default_wcset; + + pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET); + pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision); + pfile->utf8_cset_desc = init_iconv_desc (pfile, "UTF-8", SOURCE_CHARSET); + pfile->utf8_cset_desc.width = CPP_OPTION (pfile, char_precision); + pfile->char16_cset_desc = init_iconv_desc (pfile, + be ? "UTF-16BE" : "UTF-16LE", + SOURCE_CHARSET); + pfile->char16_cset_desc.width = 16; + pfile->char32_cset_desc = init_iconv_desc (pfile, + be ? "UTF-32BE" : "UTF-32LE", + SOURCE_CHARSET); + pfile->char32_cset_desc.width = 32; + pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET); + pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision); +} + +/* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */ +void +_cpp_destroy_iconv (cpp_reader *pfile) +{ + if (HAVE_ICONV) + { + if (pfile->narrow_cset_desc.func == convert_using_iconv) + iconv_close (pfile->narrow_cset_desc.cd); + if (pfile->utf8_cset_desc.func == convert_using_iconv) + iconv_close (pfile->utf8_cset_desc.cd); + if (pfile->char16_cset_desc.func == convert_using_iconv) + iconv_close (pfile->char16_cset_desc.cd); + if (pfile->char32_cset_desc.func == convert_using_iconv) + iconv_close (pfile->char32_cset_desc.cd); + if (pfile->wide_cset_desc.func == convert_using_iconv) + iconv_close (pfile->wide_cset_desc.cd); + } +} + +/* Utility routine for use by a full compiler. C is a character taken + from the *basic* source character set, encoded in the host's + execution encoding. Convert it to (the target's) execution + encoding, and return that value. + + Issues an internal error if C's representation in the narrow + execution character set fails to be a single-byte value (C99 + 5.2.1p3: "The representation of each member of the source and + execution character sets shall fit in a byte.") May also issue an + internal error if C fails to be a member of the basic source + character set (testing this exactly is too hard, especially when + the host character set is EBCDIC). */ +cppchar_t +cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c) +{ + uchar sbuf[1]; + struct _cpp_strbuf tbuf; + + /* This test is merely an approximation, but it suffices to catch + the most important thing, which is that we don't get handed a + character outside the unibyte range of the host character set. */ + if (c > LAST_POSSIBLY_BASIC_SOURCE_CHAR) + { + cpp_error (pfile, CPP_DL_ICE, + "character 0x%lx is not in the basic source character set\n", + (unsigned long)c); + return 0; + } + + /* Being a character in the unibyte range of the host character set, + we can safely splat it into a one-byte buffer and trust that that + is a well-formed string. */ + sbuf[0] = c; + + /* This should never need to reallocate, but just in case... */ + tbuf.asize = 1; + tbuf.text = XNEWVEC (uchar, tbuf.asize); + tbuf.len = 0; + + if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf)) + { + cpp_errno (pfile, CPP_DL_ICE, "converting to execution character set"); + return 0; + } + if (tbuf.len != 1) + { + cpp_error (pfile, CPP_DL_ICE, + "character 0x%lx is not unibyte in execution character set", + (unsigned long)c); + return 0; + } + c = tbuf.text[0]; + free(tbuf.text); + return c; +} + + + +/* cpp_substring_ranges's constructor. */ + +cpp_substring_ranges::cpp_substring_ranges () : + m_ranges (NULL), + m_num_ranges (0), + m_alloc_ranges (8) +{ + m_ranges = XNEWVEC (source_range, m_alloc_ranges); +} + +/* cpp_substring_ranges's destructor. */ + +cpp_substring_ranges::~cpp_substring_ranges () +{ + free (m_ranges); +} + +/* Add RANGE to the vector of source_range information. */ + +void +cpp_substring_ranges::add_range (source_range range) +{ + if (m_num_ranges >= m_alloc_ranges) + { + m_alloc_ranges *= 2; + m_ranges + = (source_range *)xrealloc (m_ranges, + sizeof (source_range) * m_alloc_ranges); + } + m_ranges[m_num_ranges++] = range; +} + +/* Read NUM ranges from LOC_READER, adding them to the vector of source_range + information. */ + +void +cpp_substring_ranges::add_n_ranges (int num, + cpp_string_location_reader &loc_reader) +{ + for (int i = 0; i < num; i++) + add_range (loc_reader.get_next ()); +} + + + +/* Utility routine that computes a mask of the form 0000...111... with + WIDTH 1-bits. */ +static inline size_t +width_to_mask (size_t width) +{ + width = MIN (width, BITS_PER_CPPCHAR_T); + if (width >= CHAR_BIT * sizeof (size_t)) + return ~(size_t) 0; + else + return ((size_t) 1 << width) - 1; +} + +/* A large table of unicode character information. */ +enum { + /* Valid in a C99 identifier? */ + C99 = 1, + /* Valid in a C99 identifier, but not as the first character? */ + N99 = 2, + /* Valid in a C++ identifier? */ + CXX = 4, + /* Valid in a C11/C++11 identifier? */ + C11 = 8, + /* Valid in a C11/C++11 identifier, but not as the first character? */ + N11 = 16, + /* Valid in a C++23 identifier? */ + CXX23 = 32, + /* Valid in a C++23 identifier, but not as the first character? */ + NXX23 = 64, + /* NFC representation is not valid in an identifier? */ + CID = 128, + /* Might be valid NFC form? */ + NFC = 256, + /* Might be valid NFKC form? */ + NKC = 512, + /* Certain preceding characters might make it not valid NFC/NKFC form? */ + CTX = 1024 +}; + +struct ucnrange { + /* Bitmap of flags above. */ + unsigned short flags; + /* Combining class of the character. */ + unsigned char combine; + /* Last character in the range described by this entry. */ + unsigned int end; +}; +#include "ucnid.h" + +/* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */ +#define UCS_LIMIT 0x10FFFF + +/* Returns 1 if C is valid in an identifier, 2 if C is valid except at + the start of an identifier, and 0 if C is not valid in an + identifier. We assume C has already gone through the checks of + _cpp_valid_ucn. Also update NST for C if returning nonzero. The + algorithm is a simple binary search on the table defined in + ucnid.h. */ + +static int +ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c, + struct normalize_state *nst) +{ + int mn, mx, md; + unsigned short valid_flags, invalid_start_flags; + + if (c > UCS_LIMIT) + return 0; + + mn = 0; + mx = ARRAY_SIZE (ucnranges) - 1; + while (mx != mn) + { + md = (mn + mx) / 2; + if (c <= ucnranges[md].end) + mx = md; + else + mn = md + 1; + } + + /* When -pedantic, we require the character to have been listed by + the standard for the current language. Otherwise, we accept the + union of the acceptable sets for all supported language versions. */ + valid_flags = C99 | CXX | C11 | CXX23; + if (CPP_PEDANTIC (pfile)) + { + if (CPP_OPTION (pfile, cplusplus)) + valid_flags = CXX23; + else if (CPP_OPTION (pfile, c11_identifiers)) + valid_flags = C11; + else if (CPP_OPTION (pfile, c99)) + valid_flags = C99; + } + if (! (ucnranges[mn].flags & valid_flags)) + return 0; + + /* Update NST. */ + if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class) + nst->level = normalized_none; + else if (ucnranges[mn].flags & CTX) + { + bool safe; + cppchar_t p = nst->previous; + + /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC, + and are combined algorithmically from a sequence of the form + 1100-1112 1161-1175 11A8-11C2 + (if the third is not present, it is treated as 11A7, which is not + really a valid character). + Unfortunately, C99 allows (only) the NFC form, but C++ allows + only the combining characters. */ + if (c >= 0x1161 && c <= 0x1175) + safe = p < 0x1100 || p > 0x1112; + else if (c >= 0x11A8 && c <= 0x11C2) + safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0); + else + safe = check_nfc (pfile, c, p); + if (!safe) + { + if ((c >= 0x1161 && c <= 0x1175) || (c >= 0x11A8 && c <= 0x11C2)) + nst->level = MAX (nst->level, normalized_identifier_C); + else + nst->level = normalized_none; + } + } + else if (ucnranges[mn].flags & NKC) + ; + else if (ucnranges[mn].flags & NFC) + nst->level = MAX (nst->level, normalized_C); + else if (ucnranges[mn].flags & CID) + nst->level = MAX (nst->level, normalized_identifier_C); + else + nst->level = normalized_none; + if (ucnranges[mn].combine == 0) + nst->previous = c; + nst->prev_class = ucnranges[mn].combine; + + if (!CPP_PEDANTIC (pfile)) + { + /* If not -pedantic, accept as character that may + begin an identifier a union of characters allowed + at that position in each of the character sets. */ + if ((ucnranges[mn].flags & (C99 | N99)) == C99 + || (ucnranges[mn].flags & CXX) != 0 + || (ucnranges[mn].flags & (C11 | N11)) == C11 + || (ucnranges[mn].flags & (CXX23 | NXX23)) == CXX23) + return 1; + return 2; + } + + if (CPP_OPTION (pfile, cplusplus)) + invalid_start_flags = NXX23; + else if (CPP_OPTION (pfile, c11_identifiers)) + invalid_start_flags = N11; + else if (CPP_OPTION (pfile, c99)) + invalid_start_flags = N99; + else + invalid_start_flags = 0; + + /* In C99, UCN digits may not begin identifiers. In C11 and C++11, + UCN combining characters may not begin identifiers. */ + if (ucnranges[mn].flags & invalid_start_flags) + return 2; + + return 1; +} + +/* [lex.charset]: The character designated by the universal character + name \UNNNNNNNN is that character whose character short name in + ISO/IEC 10646 is NNNNNNNN; the character designated by the + universal character name \uNNNN is that character whose character + short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value + for a universal character name corresponds to a surrogate code point + (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed. + Additionally, if the hexadecimal value for a universal-character-name + outside a character or string literal corresponds to a control character + (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a + character in the basic source character set, the program is ill-formed. + + C99 6.4.3: A universal character name shall not specify a character + whose short identifier is less than 00A0 other than 0024 ($), 0040 (@), + or 0060 (`), nor one in the range D800 through DFFF inclusive. + + If the hexadecimal value is larger than the upper bound of the UCS + codespace specified in ISO/IEC 10646, a pedantic warning is issued + in all versions of C and in the C++20 or later versions of C++. + + *PSTR must be preceded by "\u" or "\U"; it is assumed that the + buffer end is delimited by a non-hex digit. Returns false if the + UCN has not been consumed, true otherwise. + + The value of the UCN, whether valid or invalid, is returned in *CP. + Diagnostics are emitted for invalid values. PSTR is updated to point + one beyond the UCN, or to the syntactically invalid character. + + IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of + an identifier, or 2 otherwise. + + If LOC_READER is non-NULL, then position information is + read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */ + +bool +_cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr, + const uchar *limit, int identifier_pos, + struct normalize_state *nst, cppchar_t *cp, + source_range *char_range, + cpp_string_location_reader *loc_reader) +{ + cppchar_t result, c; + unsigned int length; + const uchar *str = *pstr; + const uchar *base = str - 2; + + if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99)) + cpp_error (pfile, CPP_DL_WARNING, + "universal character names are only valid in C++ and C99"); + else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0 + && !CPP_OPTION (pfile, cplusplus)) + cpp_error (pfile, CPP_DL_WARNING, + "C99's universal character names are incompatible with C90"); + else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0) + cpp_warning (pfile, CPP_W_TRADITIONAL, + "the meaning of '\\%c' is different in traditional C", + (int) str[-1]); + + if (str[-1] == 'u') + length = 4; + else if (str[-1] == 'U') + length = 8; + else + { + cpp_error (pfile, CPP_DL_ICE, "In _cpp_valid_ucn but not a UCN"); + length = 4; + } + + result = 0; + do + { + c = *str; + if (!ISXDIGIT (c)) + break; + str++; + if (loc_reader) + { + gcc_assert (char_range); + char_range->m_finish = loc_reader->get_next ().m_finish; + } + result = (result << 4) + hex_value (c); + } + while (--length && str < limit); + + /* Partial UCNs are not valid in strings, but decompose into + multiple tokens in identifiers, so we can't give a helpful + error message in that case. */ + if (length && identifier_pos) + { + *cp = 0; + return false; + } + + *pstr = str; + if (length) + { + cpp_error (pfile, CPP_DL_ERROR, + "incomplete universal character name %.*s", + (int) (str - base), base); + result = 1; + } + /* The C99 standard permits $, @ and ` to be specified as UCNs. We use + hex escapes so that this also works with EBCDIC hosts. + C++0x permits everything below 0xa0 within literals; + ucn_valid_in_identifier will complain about identifiers. */ + else if ((result < 0xa0 + && !CPP_OPTION (pfile, cplusplus) + && (result != 0x24 && result != 0x40 && result != 0x60)) + || (result & 0x80000000) + || (result >= 0xD800 && result <= 0xDFFF)) + { + cpp_error (pfile, CPP_DL_ERROR, + "%.*s is not a valid universal character", + (int) (str - base), base); + result = 1; + } + else if (identifier_pos && result == 0x24 + && CPP_OPTION (pfile, dollars_in_ident)) + { + if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping) + { + CPP_OPTION (pfile, warn_dollars) = 0; + cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number"); + } + NORMALIZE_STATE_UPDATE_IDNUM (nst, result); + } + else if (identifier_pos) + { + int validity = ucn_valid_in_identifier (pfile, result, nst); + + if (validity == 0) + cpp_error (pfile, CPP_DL_ERROR, + "universal character %.*s is not valid in an identifier", + (int) (str - base), base); + else if (validity == 2 && identifier_pos == 1) + cpp_error (pfile, CPP_DL_ERROR, + "universal character %.*s is not valid at the start of an identifier", + (int) (str - base), base); + } + else if (result > UCS_LIMIT + && (!CPP_OPTION (pfile, cplusplus) + || CPP_OPTION (pfile, lang) > CLK_CXX17)) + cpp_error (pfile, CPP_DL_PEDWARN, + "%.*s is outside the UCS codespace", + (int) (str - base), base); + + *cp = result; + return true; +} + +/* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate + it to the execution character set and write the result into TBUF, + if TBUF is non-NULL. + An advanced pointer is returned. Issues all relevant diagnostics. + If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE + contains the location of the character so far: location information + is read from *LOC_READER, and *RANGES is updated accordingly. */ +static const uchar * +convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit, + struct _cpp_strbuf *tbuf, struct cset_converter cvt, + source_range char_range, + cpp_string_location_reader *loc_reader, + cpp_substring_ranges *ranges) +{ + cppchar_t ucn; + uchar buf[6]; + uchar *bufp = buf; + size_t bytesleft = 6; + int rval; + struct normalize_state nst = INITIAL_NORMALIZE_STATE; + + /* loc_reader and ranges must either be both NULL, or both be non-NULL. */ + gcc_assert ((loc_reader != NULL) == (ranges != NULL)); + + from++; /* Skip u/U. */ + + if (loc_reader) + /* The u/U is part of the spelling of this character. */ + char_range.m_finish = loc_reader->get_next ().m_finish; + + _cpp_valid_ucn (pfile, &from, limit, 0, &nst, + &ucn, &char_range, loc_reader); + + rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft); + if (rval) + { + errno = rval; + cpp_errno (pfile, CPP_DL_ERROR, + "converting UCN to source character set"); + } + else + { + if (tbuf) + if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf)) + cpp_errno (pfile, CPP_DL_ERROR, + "converting UCN to execution character set"); + + if (loc_reader) + { + int num_encoded_bytes = 6 - bytesleft; + for (int i = 0; i < num_encoded_bytes; i++) + ranges->add_range (char_range); + } + } + + return from; +} + +/* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded + extended characters rather than UCNs. If the return value is TRUE, then a + character was successfully decoded and stored in *CP; *PSTR has been + updated to point one past the valid UTF-8 sequence. Diagnostics may have + been emitted if the character parsed is not allowed in the current context. + If the return value is FALSE, then *PSTR has not been modified and *CP may + equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it + may, when processing an identifier in C mode, equal a codepoint that was + validly encoded but is not allowed to appear in an identifier. In either + case, no diagnostic is emitted, and the return value of FALSE should cause + a new token to be formed. + + Unlike _cpp_valid_ucn, this will never be called when lexing a string; only + a potential identifier, or a CPP_OTHER token. NST is unused in the latter + case. + + As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for + the start of an identifier, or 2 otherwise. */ + +extern bool +_cpp_valid_utf8 (cpp_reader *pfile, + const uchar **pstr, + const uchar *limit, + int identifier_pos, + struct normalize_state *nst, + cppchar_t *cp) +{ + const uchar *base = *pstr; + size_t inbytesleft = limit - base; + if (one_utf8_to_cppchar (pstr, &inbytesleft, cp)) + { + /* No diagnostic here as this byte will rather become a + new token. */ + *cp = 0; + return false; + } + + if (identifier_pos) + { + switch (ucn_valid_in_identifier (pfile, *cp, nst)) + { + + case 0: + /* In C++, this is an error for invalid character in an identifier + because logically, the UTF-8 was converted to a UCN during + translation phase 1 (even though we don't physically do it that + way). In C, this byte rather becomes grammatically a separate + token. */ + + if (CPP_OPTION (pfile, cplusplus)) + cpp_error (pfile, CPP_DL_ERROR, + "extended character %.*s is not valid in an identifier", + (int) (*pstr - base), base); + else + { + *pstr = base; + return false; + } + + break; + + case 2: + if (identifier_pos == 1) + { + /* This is treated the same way in C++ or C99 -- lexed as an + identifier which is then invalid because an identifier is + not allowed to start with this character. */ + cpp_error (pfile, CPP_DL_ERROR, + "extended character %.*s is not valid at the start of an identifier", + (int) (*pstr - base), base); + } + break; + } + } + + return true; +} + +/* Subroutine of convert_hex and convert_oct. N is the representation + in the execution character set of a numeric escape; write it into the + string buffer TBUF and update the end-of-string pointer therein. WIDE + is true if it's a wide string that's being assembled in TBUF. This + function issues no diagnostics and never fails. */ +static void +emit_numeric_escape (cpp_reader *pfile, cppchar_t n, + struct _cpp_strbuf *tbuf, struct cset_converter cvt) +{ + size_t width = cvt.width; + + if (width != CPP_OPTION (pfile, char_precision)) + { + /* We have to render this into the target byte order, which may not + be our byte order. */ + bool bigend = CPP_OPTION (pfile, bytes_big_endian); + size_t cwidth = CPP_OPTION (pfile, char_precision); + size_t cmask = width_to_mask (cwidth); + size_t nbwc = width / cwidth; + size_t i; + size_t off = tbuf->len; + cppchar_t c; + + if (tbuf->len + nbwc > tbuf->asize) + { + tbuf->asize += OUTBUF_BLOCK_SIZE; + tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize); + } + + for (i = 0; i < nbwc; i++) + { + c = n & cmask; + n >>= cwidth; + tbuf->text[off + (bigend ? nbwc - i - 1 : i)] = c; + } + tbuf->len += nbwc; + } + else + { + /* Note: this code does not handle the case where the target + and host have a different number of bits in a byte. */ + if (tbuf->len + 1 > tbuf->asize) + { + tbuf->asize += OUTBUF_BLOCK_SIZE; + tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize); + } + tbuf->text[tbuf->len++] = n; + } +} + +/* Convert a hexadecimal escape, pointed to by FROM, to the execution + character set and write it into the string buffer TBUF (if non-NULL). + Returns an advanced pointer, and issues diagnostics as necessary. + No character set translation occurs; this routine always produces the + execution-set character with numeric value equal to the given hex + number. You can, e.g. generate surrogate pairs this way. + If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE + contains the location of the character so far: location information + is read from *LOC_READER, and *RANGES is updated accordingly. */ +static const uchar * +convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit, + struct _cpp_strbuf *tbuf, struct cset_converter cvt, + source_range char_range, + cpp_string_location_reader *loc_reader, + cpp_substring_ranges *ranges) +{ + cppchar_t c, n = 0, overflow = 0; + int digits_found = 0; + size_t width = cvt.width; + size_t mask = width_to_mask (width); + + /* loc_reader and ranges must either be both NULL, or both be non-NULL. */ + gcc_assert ((loc_reader != NULL) == (ranges != NULL)); + + if (CPP_WTRADITIONAL (pfile)) + cpp_warning (pfile, CPP_W_TRADITIONAL, + "the meaning of '\\x' is different in traditional C"); + + /* Skip 'x'. */ + from++; + + /* The 'x' is part of the spelling of this character. */ + if (loc_reader) + char_range.m_finish = loc_reader->get_next ().m_finish; + + while (from < limit) + { + c = *from; + if (! hex_p (c)) + break; + from++; + if (loc_reader) + char_range.m_finish = loc_reader->get_next ().m_finish; + overflow |= n ^ (n << 4 >> 4); + n = (n << 4) + hex_value (c); + digits_found = 1; + } + + if (!digits_found) + { + cpp_error (pfile, CPP_DL_ERROR, + "\\x used with no following hex digits"); + return from; + } + + if (overflow | (n != (n & mask))) + { + cpp_error (pfile, CPP_DL_PEDWARN, + "hex escape sequence out of range"); + n &= mask; + } + + if (tbuf) + emit_numeric_escape (pfile, n, tbuf, cvt); + if (ranges) + ranges->add_range (char_range); + + return from; +} + +/* Convert an octal escape, pointed to by FROM, to the execution + character set and write it into the string buffer TBUF. Returns an + advanced pointer, and issues diagnostics as necessary. + No character set translation occurs; this routine always produces the + execution-set character with numeric value equal to the given octal + number. + If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE + contains the location of the character so far: location information + is read from *LOC_READER, and *RANGES is updated accordingly. */ +static const uchar * +convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit, + struct _cpp_strbuf *tbuf, struct cset_converter cvt, + source_range char_range, + cpp_string_location_reader *loc_reader, + cpp_substring_ranges *ranges) +{ + size_t count = 0; + cppchar_t c, n = 0; + size_t width = cvt.width; + size_t mask = width_to_mask (width); + + /* loc_reader and ranges must either be both NULL, or both be non-NULL. */ + gcc_assert ((loc_reader != NULL) == (ranges != NULL)); + + while (from < limit && count++ < 3) + { + c = *from; + if (c < '0' || c > '7') + break; + from++; + if (loc_reader) + char_range.m_finish = loc_reader->get_next ().m_finish; + n = (n << 3) + c - '0'; + } + + if (n != (n & mask)) + { + cpp_error (pfile, CPP_DL_PEDWARN, + "octal escape sequence out of range"); + n &= mask; + } + + if (tbuf) + emit_numeric_escape (pfile, n, tbuf, cvt); + if (ranges) + ranges->add_range (char_range); + + return from; +} + +/* Convert an escape sequence (pointed to by FROM) to its value on + the target, and to the execution character set. Do not scan past + LIMIT. Write the converted value into TBUF, if TBUF is non-NULL. + Returns an advanced pointer. Handles all relevant diagnostics. + If LOC_READER is non-NULL, then RANGES must be non-NULL: location + information is read from *LOC_READER, and *RANGES is updated + accordingly. */ +static const uchar * +convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit, + struct _cpp_strbuf *tbuf, struct cset_converter cvt, + cpp_string_location_reader *loc_reader, + cpp_substring_ranges *ranges) +{ + /* Values of \a \b \e \f \n \r \t \v respectively. */ +#if HOST_CHARSET == HOST_CHARSET_ASCII + static const uchar charconsts[] = { 7, 8, 27, 12, 10, 13, 9, 11 }; +#elif HOST_CHARSET == HOST_CHARSET_EBCDIC + static const uchar charconsts[] = { 47, 22, 39, 12, 21, 13, 5, 11 }; +#else +#error "unknown host character set" +#endif + + uchar c; + + /* Record the location of the backslash. */ + source_range char_range; + if (loc_reader) + char_range = loc_reader->get_next (); + + c = *from; + switch (c) + { + /* UCNs, hex escapes, and octal escapes are processed separately. */ + case 'u': case 'U': + return convert_ucn (pfile, from, limit, tbuf, cvt, + char_range, loc_reader, ranges); + + case 'x': + return convert_hex (pfile, from, limit, tbuf, cvt, + char_range, loc_reader, ranges); + + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + return convert_oct (pfile, from, limit, tbuf, cvt, + char_range, loc_reader, ranges); + + /* Various letter escapes. Get the appropriate host-charset + value into C. */ + case '\\': case '\'': case '"': case '?': break; + + case '(': case '{': case '[': case '%': + /* '\(', etc, can be used at the beginning of a line in a long + string split onto multiple lines with \-newline, to prevent + Emacs or other text editors from getting confused. '\%' can + be used to prevent SCCS from mangling printf format strings. */ + if (CPP_PEDANTIC (pfile)) + goto unknown; + break; + + case 'b': c = charconsts[1]; break; + case 'f': c = charconsts[3]; break; + case 'n': c = charconsts[4]; break; + case 'r': c = charconsts[5]; break; + case 't': c = charconsts[6]; break; + case 'v': c = charconsts[7]; break; + + case 'a': + if (CPP_WTRADITIONAL (pfile)) + cpp_warning (pfile, CPP_W_TRADITIONAL, + "the meaning of '\\a' is different in traditional C"); + c = charconsts[0]; + break; + + case 'e': case 'E': + if (CPP_PEDANTIC (pfile)) + cpp_error (pfile, CPP_DL_PEDWARN, + "non-ISO-standard escape sequence, '\\%c'", (int) c); + c = charconsts[2]; + break; + + default: + unknown: + if (ISGRAPH (c)) + cpp_error (pfile, CPP_DL_PEDWARN, + "unknown escape sequence: '\\%c'", (int) c); + else + { + encoding_rich_location rich_loc (pfile); + + /* diagnostic.c does not support "%03o". When it does, this + code can use %03o directly in the diagnostic again. */ + char buf[32]; + sprintf(buf, "%03o", (int) c); + cpp_error_at (pfile, CPP_DL_PEDWARN, &rich_loc, + "unknown escape sequence: '\\%s'", buf); + } + } + + if (tbuf) + /* Now convert what we have to the execution character set. */ + if (!APPLY_CONVERSION (cvt, &c, 1, tbuf)) + cpp_errno (pfile, CPP_DL_ERROR, + "converting escape sequence to execution character set"); + + if (loc_reader) + { + char_range.m_finish = loc_reader->get_next ().m_finish; + ranges->add_range (char_range); + } + + return from + 1; +} + +/* TYPE is a token type. The return value is the conversion needed to + convert from source to execution character set for the given type. */ +static struct cset_converter +converter_for_type (cpp_reader *pfile, enum cpp_ttype type) +{ + switch (type) + { + default: + return pfile->narrow_cset_desc; + case CPP_UTF8CHAR: + case CPP_UTF8STRING: + return pfile->utf8_cset_desc; + case CPP_CHAR16: + case CPP_STRING16: + return pfile->char16_cset_desc; + case CPP_CHAR32: + case CPP_STRING32: + return pfile->char32_cset_desc; + case CPP_WCHAR: + case CPP_WSTRING: + return pfile->wide_cset_desc; + } +} + +/* FROM is an array of cpp_string structures of length COUNT. These + are to be converted from the source to the execution character set, + escape sequences translated, and finally all are to be + concatenated. WIDE indicates whether or not to produce a wide + string. If TO is non-NULL, the result is written into TO. + If LOC_READERS and OUT are non-NULL, then location information + is read from LOC_READERS (which must be an array of length COUNT), + and location information is written to *RANGES. + + Returns true for success, false for failure. */ + +static bool +cpp_interpret_string_1 (cpp_reader *pfile, const cpp_string *from, size_t count, + cpp_string *to, enum cpp_ttype type, + cpp_string_location_reader *loc_readers, + cpp_substring_ranges *out) +{ + struct _cpp_strbuf tbuf; + const uchar *p, *base, *limit; + size_t i; + struct cset_converter cvt = converter_for_type (pfile, type); + + /* loc_readers and out must either be both NULL, or both be non-NULL. */ + gcc_assert ((loc_readers != NULL) == (out != NULL)); + + if (to) + { + tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len); + tbuf.text = XNEWVEC (uchar, tbuf.asize); + tbuf.len = 0; + } + + cpp_string_location_reader *loc_reader = NULL; + for (i = 0; i < count; i++) + { + if (loc_readers) + loc_reader = &loc_readers[i]; + + p = from[i].text; + if (*p == 'u') + { + p++; + if (loc_reader) + loc_reader->get_next (); + if (*p == '8') + { + p++; + if (loc_reader) + loc_reader->get_next (); + } + } + else if (*p == 'L' || *p == 'U') p++; + if (*p == 'R') + { + const uchar *prefix; + + /* Skip over 'R"'. */ + p += 2; + if (loc_reader) + { + loc_reader->get_next (); + loc_reader->get_next (); + } + prefix = p; + while (*p != '(') + { + p++; + if (loc_reader) + loc_reader->get_next (); + } + p++; + if (loc_reader) + loc_reader->get_next (); + limit = from[i].text + from[i].len; + if (limit >= p + (p - prefix) + 1) + limit -= (p - prefix) + 1; + + /* Raw strings are all normal characters; these can be fed + directly to convert_cset. */ + if (to) + if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf)) + goto fail; + + if (loc_reader) + { + /* If generating source ranges, assume we have a 1:1 + correspondence between bytes in the source encoding and bytes + in the execution encoding (e.g. if we have a UTF-8 to UTF-8 + conversion), so that this run of bytes in the source file + corresponds to a run of bytes in the execution string. + This requirement is guaranteed by an early-reject in + cpp_interpret_string_ranges. */ + gcc_assert (cvt.func == convert_no_conversion); + out->add_n_ranges (limit - p, *loc_reader); + } + + continue; + } + + /* If we don't now have a leading quote, something has gone wrong. + This can occur if cpp_interpret_string_ranges is handling a + stringified macro argument, but should not be possible otherwise. */ + if (*p != '"' && *p != '\'') + { + gcc_assert (out != NULL); + cpp_error (pfile, CPP_DL_ERROR, "missing open quote"); + if (to) + free (tbuf.text); + return false; + } + + /* Skip leading quote. */ + p++; + if (loc_reader) + loc_reader->get_next (); + + limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */ + + for (;;) + { + base = p; + while (p < limit && *p != '\\') + p++; + if (p > base) + { + /* We have a run of normal characters; these can be fed + directly to convert_cset. */ + if (to) + if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf)) + goto fail; + /* Similar to above: assumes we have a 1:1 correspondence + between bytes in the source encoding and bytes in the + execution encoding. */ + if (loc_reader) + { + gcc_assert (cvt.func == convert_no_conversion); + out->add_n_ranges (p - base, *loc_reader); + } + } + if (p >= limit) + break; + + struct _cpp_strbuf *tbuf_ptr = to ? &tbuf : NULL; + p = convert_escape (pfile, p + 1, limit, tbuf_ptr, cvt, + loc_reader, out); + } + } + + if (to) + { + /* NUL-terminate the 'to' buffer and translate it to a cpp_string + structure. */ + emit_numeric_escape (pfile, 0, &tbuf, cvt); + tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len); + to->text = tbuf.text; + to->len = tbuf.len; + } + /* Use the location of the trailing quote as the location of the + NUL-terminator. */ + if (loc_reader) + { + source_range range = loc_reader->get_next (); + out->add_range (range); + } + + return true; + + fail: + cpp_errno (pfile, CPP_DL_ERROR, "converting to execution character set"); + if (to) + free (tbuf.text); + return false; +} + +/* FROM is an array of cpp_string structures of length COUNT. These + are to be converted from the source to the execution character set, + escape sequences translated, and finally all are to be + concatenated. WIDE indicates whether or not to produce a wide + string. The result is written into TO. Returns true for success, + false for failure. */ +bool +cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count, + cpp_string *to, enum cpp_ttype type) +{ + return cpp_interpret_string_1 (pfile, from, count, to, type, NULL, NULL); +} + +/* A "do nothing" diagnostic-handling callback for use by + cpp_interpret_string_ranges, so that it can temporarily suppress + diagnostic-handling. */ + +static bool +noop_diagnostic_cb (cpp_reader *, enum cpp_diagnostic_level, + enum cpp_warning_reason, rich_location *, + const char *, va_list *) +{ + /* no-op. */ + return true; +} + +/* This function mimics the behavior of cpp_interpret_string, but + rather than generating a string in the execution character set, + *OUT is written to with the source code ranges of the characters + in such a string. + FROM and LOC_READERS should both be arrays of length COUNT. + Returns NULL for success, or an error message for failure. */ + +const char * +cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from, + cpp_string_location_reader *loc_readers, + size_t count, + cpp_substring_ranges *out, + enum cpp_ttype type) +{ + /* There are a couple of cases in the range-handling in + cpp_interpret_string_1 that rely on there being a 1:1 correspondence + between bytes in the source encoding and bytes in the execution + encoding, so that each byte in the execution string can correspond + to the location of a byte in the source string. + + This holds for the typical case of a UTF-8 to UTF-8 conversion. + Enforce this requirement by only attempting to track substring + locations if we have source encoding == execution encoding. + + This is a stronger condition than we need, since we could e.g. + have ASCII to EBCDIC (with 1 byte per character before and after), + but it seems to be a reasonable restriction. */ + struct cset_converter cvt = converter_for_type (pfile, type); + if (cvt.func != convert_no_conversion) + return "execution character set != source character set"; + + /* For on-demand strings we have already lexed the strings, so there + should be no diagnostics. However, if we have bogus source location + data (or stringified macro arguments), the attempt to lex the + strings could fail with an diagnostic. Temporarily install an + diagnostic-handler to catch the diagnostic, so that it can lead to this call + failing, rather than being emitted as a user-visible diagnostic. + If an diagnostic does occur, we should see it via the return value of + cpp_interpret_string_1. */ + bool (*saved_diagnostic_handler) (cpp_reader *, enum cpp_diagnostic_level, + enum cpp_warning_reason, rich_location *, + const char *, va_list *) + ATTRIBUTE_FPTR_PRINTF(5,0); + + saved_diagnostic_handler = pfile->cb.diagnostic; + pfile->cb.diagnostic = noop_diagnostic_cb; + + bool result = cpp_interpret_string_1 (pfile, from, count, NULL, type, + loc_readers, out); + + /* Restore the saved diagnostic-handler. */ + pfile->cb.diagnostic = saved_diagnostic_handler; + + if (!result) + return "cpp_interpret_string_1 failed"; + + /* Success. */ + return NULL; +} + +/* Subroutine of do_line and do_linemarker. Convert escape sequences + in a string, but do not perform character set conversion. */ +bool +cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from, + size_t count, cpp_string *to, + enum cpp_ttype type ATTRIBUTE_UNUSED) +{ + struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc; + bool retval; + + pfile->narrow_cset_desc.func = convert_no_conversion; + pfile->narrow_cset_desc.cd = (iconv_t) -1; + pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision); + + retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING); + + pfile->narrow_cset_desc = save_narrow_cset_desc; + return retval; +} + + +/* Subroutine of cpp_interpret_charconst which performs the conversion + to a number, for narrow strings. STR is the string structure returned + by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for + cpp_interpret_charconst. TYPE is the token type. */ +static cppchar_t +narrow_str_to_charconst (cpp_reader *pfile, cpp_string str, + unsigned int *pchars_seen, int *unsignedp, + enum cpp_ttype type) +{ + size_t width = CPP_OPTION (pfile, char_precision); + size_t max_chars = CPP_OPTION (pfile, int_precision) / width; + size_t mask = width_to_mask (width); + size_t i; + cppchar_t result, c; + bool unsigned_p; + + /* The value of a multi-character character constant, or a + single-character character constant whose representation in the + execution character set is more than one byte long, is + implementation defined. This implementation defines it to be the + number formed by interpreting the byte sequence in memory as a + big-endian binary number. If overflow occurs, the high bytes are + lost, and a warning is issued. + + We don't want to process the NUL terminator handed back by + cpp_interpret_string. */ + result = 0; + for (i = 0; i < str.len - 1; i++) + { + c = str.text[i] & mask; + if (width < BITS_PER_CPPCHAR_T) + result = (result << width) | c; + else + result = c; + } + + if (type == CPP_UTF8CHAR) + max_chars = 1; + if (i > max_chars) + { + i = max_chars; + cpp_error (pfile, type == CPP_UTF8CHAR ? CPP_DL_ERROR : CPP_DL_WARNING, + "character constant too long for its type"); + } + else if (i > 1 && CPP_OPTION (pfile, warn_multichar)) + cpp_warning (pfile, CPP_W_MULTICHAR, "multi-character character constant"); + + /* Multichar constants are of type int and therefore signed. */ + if (i > 1) + unsigned_p = 0; + else if (type == CPP_UTF8CHAR && !CPP_OPTION (pfile, cplusplus)) + unsigned_p = 1; + else + unsigned_p = CPP_OPTION (pfile, unsigned_char); + + /* Truncate the constant to its natural width, and simultaneously + sign- or zero-extend to the full width of cppchar_t. + For single-character constants, the value is WIDTH bits wide. + For multi-character constants, the value is INT_PRECISION bits wide. */ + if (i > 1) + width = CPP_OPTION (pfile, int_precision); + if (width < BITS_PER_CPPCHAR_T) + { + mask = ((cppchar_t) 1 << width) - 1; + if (unsigned_p || !(result & (1 << (width - 1)))) + result &= mask; + else + result |= ~mask; + } + *pchars_seen = i; + *unsignedp = unsigned_p; + return result; +} + +/* Subroutine of cpp_interpret_charconst which performs the conversion + to a number, for wide strings. STR is the string structure returned + by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for + cpp_interpret_charconst. TYPE is the token type. */ +static cppchar_t +wide_str_to_charconst (cpp_reader *pfile, cpp_string str, + unsigned int *pchars_seen, int *unsignedp, + enum cpp_ttype type) +{ + bool bigend = CPP_OPTION (pfile, bytes_big_endian); + size_t width = converter_for_type (pfile, type).width; + size_t cwidth = CPP_OPTION (pfile, char_precision); + size_t mask = width_to_mask (width); + size_t cmask = width_to_mask (cwidth); + size_t nbwc = width / cwidth; + size_t off, i; + cppchar_t result = 0, c; + + if (str.len <= nbwc) + { + /* Error recovery, if no errors have been diagnosed previously, + there should be at least two wide characters. Empty literals + are diagnosed earlier and we can get just the zero terminator + only if there were errors diagnosed during conversion. */ + *pchars_seen = 0; + *unsignedp = 0; + return 0; + } + + /* This is finicky because the string is in the target's byte order, + which may not be our byte order. Only the last character, ignoring + the NUL terminator, is relevant. */ + off = str.len - (nbwc * 2); + result = 0; + for (i = 0; i < nbwc; i++) + { + c = bigend ? str.text[off + i] : str.text[off + nbwc - i - 1]; + result = (result << cwidth) | (c & cmask); + } + + /* Wide character constants have type wchar_t, and a single + character exactly fills a wchar_t, so a multi-character wide + character constant is guaranteed to overflow. */ + if (str.len > nbwc * 2) + cpp_error (pfile, (CPP_OPTION (pfile, cplusplus) + && (type == CPP_CHAR16 || type == CPP_CHAR32)) + ? CPP_DL_ERROR : CPP_DL_WARNING, + "character constant too long for its type"); + + /* Truncate the constant to its natural width, and simultaneously + sign- or zero-extend to the full width of cppchar_t. */ + if (width < BITS_PER_CPPCHAR_T) + { + if (type == CPP_CHAR16 || type == CPP_CHAR32 + || CPP_OPTION (pfile, unsigned_wchar) + || !(result & (1 << (width - 1)))) + result &= mask; + else + result |= ~mask; + } + + if (type == CPP_CHAR16 || type == CPP_CHAR32 + || CPP_OPTION (pfile, unsigned_wchar)) + *unsignedp = 1; + else + *unsignedp = 0; + + *pchars_seen = 1; + return result; +} + +/* Interpret a (possibly wide) character constant in TOKEN. + PCHARS_SEEN points to a variable that is filled in with the number + of characters seen, and UNSIGNEDP to a variable that indicates + whether the result has signed type. */ +cppchar_t +cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token, + unsigned int *pchars_seen, int *unsignedp) +{ + cpp_string str = { 0, 0 }; + bool wide = (token->type != CPP_CHAR && token->type != CPP_UTF8CHAR); + int u8 = 2 * int(token->type == CPP_UTF8CHAR); + cppchar_t result; + + /* An empty constant will appear as L'', u'', U'', u8'', or '' */ + if (token->val.str.len == (size_t) (2 + wide + u8)) + { + cpp_error (pfile, CPP_DL_ERROR, "empty character constant"); + *pchars_seen = 0; + *unsignedp = 0; + return 0; + } + else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, + token->type)) + { + *pchars_seen = 0; + *unsignedp = 0; + return 0; + } + + if (wide) + result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp, + token->type); + else + result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp, + token->type); + + if (str.text != token->val.str.text) + free ((void *)str.text); + + return result; +} + +/* Convert an identifier denoted by ID and LEN, which might contain + UCN escapes or UTF-8 multibyte chars, to the source character set, + either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually + a valid identifier. */ +cpp_hashnode * +_cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len) +{ + /* It turns out that a UCN escape always turns into fewer characters + than the escape itself, so we can allocate a temporary in advance. */ + uchar * buf = (uchar *) alloca (len + 1); + uchar * bufp = buf; + size_t idp; + + for (idp = 0; idp < len; idp++) + if (id[idp] != '\\') + *bufp++ = id[idp]; + else + { + unsigned length = id[idp+1] == 'u' ? 4 : 8; + cppchar_t value = 0; + size_t bufleft = len - (bufp - buf); + int rval; + + idp += 2; + while (length && idp < len && ISXDIGIT (id[idp])) + { + value = (value << 4) + hex_value (id[idp]); + idp++; + length--; + } + idp--; + + /* Special case for EBCDIC: if the identifier contains + a '$' specified using a UCN, translate it to EBCDIC. */ + if (value == 0x24) + { + *bufp++ = '$'; + continue; + } + + rval = one_cppchar_to_utf8 (value, &bufp, &bufleft); + if (rval) + { + errno = rval; + cpp_errno (pfile, CPP_DL_ERROR, + "converting UCN to source character set"); + break; + } + } + + return CPP_HASHNODE (ht_lookup (pfile->hash_table, + buf, bufp - buf, HT_ALLOC)); +} + + +/* Utility to strip a UTF-8 byte order marking from the beginning + of a buffer. Returns the number of bytes to skip, which currently + will be either 0 or 3. */ +int +cpp_check_utf8_bom (const char *data, size_t data_length) +{ + +#if HOST_CHARSET == HOST_CHARSET_ASCII + const unsigned char *udata = (const unsigned char *) data; + if (data_length >= 3 && udata[0] == 0xef && udata[1] == 0xbb + && udata[2] == 0xbf) + return 3; +#endif + + return 0; +} + + +/* Convert an input buffer (containing the complete contents of one + source file) from INPUT_CHARSET to the source character set. INPUT + points to the input buffer, SIZE is its allocated size, and LEN is + the length of the meaningful data within the buffer. The + translated buffer is returned, *ST_SIZE is set to the length of + the meaningful data within the translated buffer, and *BUFFER_START + is set to the start of the returned buffer. *BUFFER_START may + differ from the return value in the case of a BOM or other ignored + marker information. + + INPUT is expected to have been allocated with xmalloc. This + function will either set *BUFFER_START to INPUT, or free it and set + *BUFFER_START to a pointer to another xmalloc-allocated block of + memory. + + PFILE is only used to generate diagnostics; setting it to NULL suppresses + diagnostics, and causes a return of NULL if there was any error instead. */ + +uchar * +_cpp_convert_input (cpp_reader *pfile, const char *input_charset, + uchar *input, size_t size, size_t len, + const unsigned char **buffer_start, off_t *st_size) +{ + struct cset_converter input_cset; + struct _cpp_strbuf to; + unsigned char *buffer; + + input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset); + if (input_cset.func == convert_no_conversion) + { + to.text = input; + to.asize = size; + to.len = len; + } + else + { + to.asize = MAX (65536, len); + to.text = XNEWVEC (uchar, to.asize); + to.len = 0; + + const bool ok = APPLY_CONVERSION (input_cset, input, len, &to); + free (input); + + /* Clean up the mess. */ + if (input_cset.func == convert_using_iconv) + iconv_close (input_cset.cd); + + /* Handle conversion failure. */ + if (!ok) + { + if (!pfile) + { + XDELETEVEC (to.text); + *buffer_start = NULL; + *st_size = 0; + return NULL; + } + cpp_error (pfile, CPP_DL_ERROR, "failure to convert %s to %s", + input_charset, SOURCE_CHARSET); + } + } + + /* Resize buffer if we allocated substantially too much, or if we + haven't enough space for the \n-terminator or following + 15 bytes of padding (used to quiet warnings from valgrind or + Address Sanitizer, when the optimized lexer accesses aligned + 16-byte memory chunks, including the bytes after the malloced, + area, and stops lexing on '\n'). */ + if (to.len + 4096 < to.asize || to.len + 16 > to.asize) + to.text = XRESIZEVEC (uchar, to.text, to.len + 16); + + memset (to.text + to.len, '\0', 16); + + /* If the file is using old-school Mac line endings (\r only), + terminate with another \r, not an \n, so that we do not mistake + the \r\n sequence for a single DOS line ending and erroneously + issue the "No newline at end of file" diagnostic. */ + if (to.len && to.text[to.len - 1] == '\r') + to.text[to.len] = '\r'; + else + to.text[to.len] = '\n'; + + buffer = to.text; + *st_size = to.len; + + /* Ignore a UTF-8 BOM if we see one and the source charset is UTF-8. Note + that glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a + BOM -- however, even if it did, we would still need this code due + to the 'convert_no_conversion' case. */ + const int bom_len = cpp_check_utf8_bom ((const char *) to.text, to.len); + *st_size -= bom_len; + buffer += bom_len; + + *buffer_start = to.text; + return buffer; +} + +/* Decide on the default encoding to assume for input files. */ +const char * +_cpp_default_encoding (void) +{ + const char *current_encoding = NULL; + + /* We disable this because the default codeset is 7-bit ASCII on + most platforms, and this causes conversion failures on every + file in GCC that happens to have one of the upper 128 characters + in it -- most likely, as part of the name of a contributor. + We should definitely recognize in-band markers of file encoding, + like: + - the appropriate Unicode byte-order mark (FE FF) to recognize + UTF16 and UCS4 (in both big-endian and little-endian flavors) + and UTF8 + - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to + distinguish ASCII and EBCDIC. + - now we can parse something like "#pragma GCC encoding <xyz> + on the first line, or even Emacs/VIM's mode line tags (there's + a problem here in that VIM uses the last line, and Emacs has + its more elaborate "local variables" convention). + - investigate whether Java has another common convention, which + would be friendly to support. + (Zack Weinberg and Paolo Bonzini, May 20th 2004) */ +#if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0 + setlocale (LC_CTYPE, ""); + current_encoding = nl_langinfo (CODESET); +#endif + if (current_encoding == NULL || *current_encoding == '\0') + current_encoding = SOURCE_CHARSET; + + return current_encoding; +} + +/* Check if the configured input charset requires no conversion, other than + possibly stripping a UTF-8 BOM. */ +bool cpp_input_conversion_is_trivial (const char *input_charset) +{ + return !strcasecmp (input_charset, SOURCE_CHARSET); +} + +/* Implementation of class cpp_string_location_reader. */ + +/* Constructor for cpp_string_location_reader. */ + +cpp_string_location_reader:: +cpp_string_location_reader (location_t src_loc, + line_maps *line_table) +{ + src_loc = get_range_from_loc (line_table, src_loc).m_start; + + /* SRC_LOC might be a macro location. It only makes sense to do + column-by-column calculations on ordinary maps, so get the + corresponding location in an ordinary map. */ + m_loc + = linemap_resolve_location (line_table, src_loc, + LRK_SPELLING_LOCATION, NULL); + + const line_map_ordinary *map + = linemap_check_ordinary (linemap_lookup (line_table, m_loc)); + m_offset_per_column = (1 << map->m_range_bits); +} + +/* Get the range of the next source byte. */ + +source_range +cpp_string_location_reader::get_next () +{ + source_range result; + result.m_start = m_loc; + result.m_finish = m_loc; + if (m_loc <= LINE_MAP_MAX_LOCATION_WITH_COLS) + m_loc += m_offset_per_column; + return result; +} + +cpp_display_width_computation:: +cpp_display_width_computation (const char *data, int data_length, + const cpp_char_column_policy &policy) : + m_begin (data), + m_next (m_begin), + m_bytes_left (data_length), + m_policy (policy), + m_display_cols (0) +{ + gcc_assert (policy.m_tabstop > 0); + gcc_assert (policy.m_width_cb); +} + + +/* The main implementation function for class cpp_display_width_computation. + m_next points on entry to the start of the UTF-8 encoding of the next + character, and is updated to point just after the last byte of the encoding. + m_bytes_left contains on entry the remaining size of the buffer into which + m_next points, and this is also updated accordingly. If m_next does not + point to a valid UTF-8-encoded sequence, then it will be treated as a single + byte with display width 1. m_cur_display_col is the current display column, + relative to which tab stops should be expanded. Returns the display width of + the codepoint just processed. + If OUT is non-NULL, it is populated. */ + +int +cpp_display_width_computation::process_next_codepoint (cpp_decoded_char *out) +{ + cppchar_t c; + int next_width; + + if (out) + out->m_start_byte = m_next; + + if (*m_next == '\t') + { + ++m_next; + --m_bytes_left; + next_width = m_policy.m_tabstop - (m_display_cols % m_policy.m_tabstop); + if (out) + { + out->m_ch = '\t'; + out->m_valid_ch = true; + } + } + else if (one_utf8_to_cppchar ((const uchar **) &m_next, &m_bytes_left, &c) + != 0) + { + /* Input is not convertible to UTF-8. This could be fine, e.g. in a + string literal, so don't complain. Just treat it as if it has a width + of one. */ + ++m_next; + --m_bytes_left; + next_width = m_policy.m_undecoded_byte_width; + if (out) + out->m_valid_ch = false; + } + else + { + /* one_utf8_to_cppchar() has updated m_next and m_bytes_left for us. */ + next_width = m_policy.m_width_cb (c); + if (out) + { + out->m_ch = c; + out->m_valid_ch = true; + } + } + + if (out) + out->m_next_byte = m_next; + + m_display_cols += next_width; + return next_width; +} + +/* Utility to advance the byte stream by the minimum amount needed to consume + N display columns. Returns the number of display columns that were + actually skipped. This could be less than N, if there was not enough data, + or more than N, if the last character to be skipped had a sufficiently large + display width. */ +int +cpp_display_width_computation::advance_display_cols (int n) +{ + const int start = m_display_cols; + const int target = start + n; + while (m_display_cols < target && !done ()) + process_next_codepoint (NULL); + return m_display_cols - start; +} + +/* For the string of length DATA_LENGTH bytes that begins at DATA, compute + how many display columns are occupied by the first COLUMN bytes. COLUMN + may exceed DATA_LENGTH, in which case the phantom bytes at the end are + treated as if they have display width 1. Tabs are expanded to the next tab + stop, relative to the start of DATA, and non-printable-ASCII characters + will be escaped as per POLICY. */ + +int +cpp_byte_column_to_display_column (const char *data, int data_length, + int column, + const cpp_char_column_policy &policy) +{ + const int offset = MAX (0, column - data_length); + cpp_display_width_computation dw (data, column - offset, policy); + while (!dw.done ()) + dw.process_next_codepoint (NULL); + return dw.display_cols_processed () + offset; +} + +/* For the string of length DATA_LENGTH bytes that begins at DATA, compute + the least number of bytes that will result in at least DISPLAY_COL display + columns. The return value may exceed DATA_LENGTH if the entire string does + not occupy enough display columns. Non-printable-ASCII characters + will be escaped as per POLICY. */ + +int +cpp_display_column_to_byte_column (const char *data, int data_length, + int display_col, + const cpp_char_column_policy &policy) +{ + cpp_display_width_computation dw (data, data_length, policy); + const int avail_display = dw.advance_display_cols (display_col); + return dw.bytes_processed () + MAX (0, display_col - avail_display); +} + +/* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc, + because that will inspect the user's locale, and in particular in an ASCII + locale, it will not return anything useful for extended characters. But GCC + in other respects (see e.g. _cpp_default_encoding()) behaves as if + everything is UTF-8. We also make some tweaks that are useful for the way + GCC needs to use this data, e.g. tabs and other control characters should be + treated as having width 1. The lookup tables are generated from + contrib/unicode/gen_wcwidth.py and were made by simply calling glibc + wcwidth() on all codepoints, then applying the small tweaks. These tables + are not highly optimized, but for the present purpose of outputting + diagnostics, they are sufficient. */ + +#include "generated_cpp_wcwidth.h" +int cpp_wcwidth (cppchar_t c) +{ + if (__builtin_expect (c <= wcwidth_range_ends[0], true)) + return wcwidth_widths[0]; + + /* Binary search the tables. */ + int begin = 1; + static const int end + = sizeof wcwidth_range_ends / sizeof (*wcwidth_range_ends); + int len = end - begin; + do + { + int half = len/2; + int middle = begin + half; + if (c > wcwidth_range_ends[middle]) + { + begin = middle + 1; + len -= half + 1; + } + else + len = half; + } while (len); + + if (__builtin_expect (begin != end, true)) + return wcwidth_widths[begin]; + return 1; +} |