cpplex.c: Remove conditional #undef of MULTIBYTE_CHARS.

2002-04-24  Matt Hiller  <hiller@redhat.com>

	* cpplex.c: Remove conditional #undef of MULTIBYTE_CHARS.
	* c-lex.c: Ditto.

	* cpplex.c (skip_line_comment): Process comment one multibyte
	character at a time rather than one char at a time, if
	appropriate.
	(parse_string): Process string one multibyte character at a time
	rather than one char at a time, if appropriate.
	* c-lex.c (lex_string): Lex and copy multibyte strings
	appropriately.
	* cpplib.h (cppchar_t): Change to unsigned.

From-SVN: r52737

3 files changed, 76 insertions, 22 deletions

diff --git a/gcc/c-lex.c b/gcc/c-lex.c
index 8a76dd3..f2f8c8c 100644
--- a/gcc/c-lex.c
+++ b/gcc/c-lex.c
@@ -40,13 +40,6 @@ Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include "splay-tree.h"
 #include "debug.h"
 
-/* MULTIBYTE_CHARS support only works for native compilers.
-   ??? Ideally what we want is to model widechar support after
-   the current floating point support.  */
-#ifdef CROSS_COMPILE
-#undef MULTIBYTE_CHARS
-#endif
-
 #ifdef MULTIBYTE_CHARS
 #include "mbchar.h"
 #include <locale.h>
@@ -1282,8 +1275,8 @@ lex_string (str, len, wide)
 	  c = cpp_parse_escape (parse_in, &p, limit, mask);
 	}
 	
-      /* Add this single character into the buffer either as a wchar_t
-	 or as a single byte.  */
+      /* Add this single character into the buffer either as a wchar_t,
+	 a multibyte sequence, or as a single byte.  */
       if (wide)
 	{
 	  unsigned charwidth = TYPE_PRECISION (char_type_node);
@@ -1304,6 +1297,16 @@ lex_string (str, len, wide)
 	    }
 	  q += WCHAR_BYTES;
 	}
+#ifdef MULTIBYTE_CHARS
+      else if (char_len > 1)
+	{
+	  /* We're dealing with a multibyte character. */
+	  for ( ; char_len >0; --char_len)
+	    {
+	      *q++ = *(p - char_len);
+	    }
+	}
+#endif
       else
 	{
 	  *q++ = c;
diff --git a/gcc/cpplex.c b/gcc/cpplex.c
index eed9ffd..6a44b86 100644
--- a/gcc/cpplex.c
+++ b/gcc/cpplex.c
@@ -25,13 +25,6 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
 #include "cpplib.h"
 #include "cpphash.h"
 
-/* MULTIBYTE_CHARS support only works for native compilers.
-   ??? Ideally what we want is to model widechar support after
-   the current floating point support.  */
-#ifdef CROSS_COMPILE
-#undef MULTIBYTE_CHARS
-#endif
-
 #ifdef MULTIBYTE_CHARS
 #include "mbchar.h"
 #include <locale.h>
@@ -312,14 +305,39 @@ skip_line_comment (pfile)
   cpp_buffer *buffer = pfile->buffer;
   unsigned int orig_line = pfile->line;
   cppchar_t c;
+#ifdef MULTIBYTE_CHARS
+  wchar_t wc;
+  int char_len;
+#endif
 
   pfile->state.lexing_comment = 1;
+#ifdef MULTIBYTE_CHARS
+  /* Reset multibyte conversion state.  */
+  (void) local_mbtowc (NULL, NULL, 0);
+#endif
   do
     {
       if (buffer->cur == buffer->rlimit)
 	goto at_eof;
 
+#ifdef MULTIBYTE_CHARS
+      char_len = local_mbtowc (&wc, (const char *) buffer->cur,
+			       buffer->rlimit - buffer->cur);
+      if (char_len == -1)
+	{
+	  cpp_error (pfile, DL_WARNING,
+		     "ignoring invalid multibyte character");
+	  char_len = 1;
+	  c = *buffer->cur++;
+	}
+      else
+	{
+	  buffer->cur += char_len;
+	  c = wc;
+	}
+#else
       c = *buffer->cur++;
+#endif
       if (c == '?' || c == '\\')
 	c = skip_escaped_newlines (pfile);
     }
@@ -617,10 +635,18 @@ parse_string (pfile, token, terminator)
   unsigned char *dest, *limit;
   cppchar_t c;
   bool warned_nulls = false;
+#ifdef MULTIBYTE_CHARS
+  wchar_t wc;
+  int char_len;
+#endif
 
   dest = BUFF_FRONT (pfile->u_buff);
   limit = BUFF_LIMIT (pfile->u_buff);
 
+#ifdef MULTIBYTE_CHARS
+  /* Reset multibyte conversion state.  */
+  (void) local_mbtowc (NULL, NULL, 0);
+#endif
   for (;;)
     {
       /* We need room for another char, possibly the terminating NUL.  */
@@ -632,8 +658,26 @@ parse_string (pfile, token, terminator)
 	  limit = BUFF_LIMIT (pfile->u_buff);
 	}
 
-      /* Handle trigraphs, escaped newlines etc.  */
+#ifdef MULTIBYTE_CHARS
+      char_len = local_mbtowc (&wc, (const char *) buffer->cur,
+			       buffer->rlimit - buffer->cur);
+      if (char_len == -1)
+	{
+	  cpp_error (pfile, DL_WARNING,
+		       "ignoring invalid multibyte character");
+	  char_len = 1;
+	  c = *buffer->cur++;
+	}
+      else
+	{
+	  buffer->cur += char_len;
+	  c = wc;
+	}
+#else
       c = *buffer->cur++;
+#endif
+
+      /* Handle trigraphs, escaped newlines etc.  */
       if (c == '?' || c == '\\')
 	c = skip_escaped_newlines (pfile);
 
@@ -666,8 +710,15 @@ parse_string (pfile, token, terminator)
 			 "null character(s) preserved in literal");
 	    }
 	}
-
-      *dest++ = c;
+#ifdef MULTIBYTE_CHARS
+      if (char_len > 1)
+	{
+	  for ( ; char_len > 0; --char_len)
+	    *dest++ = (*buffer->cur - char_len);
+	}
+      else
+#endif
+	*dest++ = c;
     }
 
   *dest = '\0';
diff --git a/gcc/cpplib.h b/gcc/cpplib.h
index 359326e..765d65a 100644
--- a/gcc/cpplib.h
+++ b/gcc/cpplib.h
@@ -187,9 +187,9 @@ struct cpp_token
   } val;
 };
 
-/* A standalone character.  We may want to make it unsigned for the
-   same reason we use unsigned char - to avoid signedness issues.  */
-typedef int cppchar_t;
+/* A standalone character.  It is unsigned for the same reason we use
+   unsigned char - to avoid signedness issues.  */
+typedef unsigned int cppchar_t;
 
 /* Values for opts.dump_macros.
   dump_only means inhibit output of the preprocessed text