Support styling on native MS-Windows console

gdb/ChangeLog:
2019-03-08  Eli Zaretskii  <eliz@gnu.org>

	PR/24315
	* utils.c (can_emit_style_escape) [_WIN32]: Don't disable styling
	on MS-Windows if $TERM is not defined.

	* cli/cli-style.c: Set cli_styling to 1 in the MinGW build.

	* posix-hdep.c (gdb_console_fputs):
	* mingw-hdep.c (rgb_to_16colors, gdb_console_fputs): New
	functions.
	* ui-file.h (gdb_console_fputs): Add prototype.

	* ui-file.c (stdio_file::puts): Call gdb_console_fputs, and fall
	back to fputs only if the former returns zero.

7 files changed, 237 insertions, 1 deletions

diff --git a/gdb/ChangeLog b/gdb/ChangeLog
index de5e74a..bb10c7d 100644
--- a/gdb/ChangeLog
+++ b/gdb/ChangeLog
@@ -1,3 +1,21 @@
+2019-03-08  Eli Zaretskii  <eliz@gnu.org>
+
+	Support styling on native MS-Windows console
+
+	PR/24315
+	* utils.c (can_emit_style_escape) [_WIN32]: Don't disable styling
+	on MS-Windows if $TERM is not defined.
+
+	* cli/cli-style.c: Set cli_styling to 1 in the MinGW build.
+
+	* posix-hdep.c (gdb_console_fputs):
+	* mingw-hdep.c (rgb_to_16colors, gdb_console_fputs): New
+	functions.
+	* ui-file.h (gdb_console_fputs): Add prototype.
+
+	* ui-file.c (stdio_file::puts): Call gdb_console_fputs, and fall
+	back to fputs only if the former returns zero.
+
 2019-03-07  Tom Tromey  <tom@tromey.com>
 
 	* symmisc.c (print_symbol_bcache_statistics): Update.
diff --git a/gdb/cli/cli-style.c b/gdb/cli/cli-style.c
index 234cc41..ec385da 100644
--- a/gdb/cli/cli-style.c
+++ b/gdb/cli/cli-style.c
@@ -24,7 +24,7 @@
 
 /* True if styling is enabled.  */
 
-#if defined(_WIN32) || defined (__CYGWIN__)
+#if defined (__MSDOS__) || defined (__CYGWIN__)
 int cli_styling = 0;
 #else
 int cli_styling = 1;
diff --git a/gdb/mingw-hdep.c b/gdb/mingw-hdep.c
index 1a4b93b..8ed4b44 100644
--- a/gdb/mingw-hdep.c
+++ b/gdb/mingw-hdep.c
@@ -177,3 +177,195 @@ gdb_select (int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
 
   return num_ready;
 }
+
+/* Map COLOR's RGB triplet, with 8 bits per component, into 16 Windows
+   console colors, where each component has just 1 bit, plus a single
+   intensity bit which affects all 3 components.  */
+static int
+rgb_to_16colors (const ui_file_style::color &color)
+{
+  uint8_t rgb[3];
+  color.get_rgb (rgb);
+
+  int retval = 0;
+  for (int i = 0; i < 3; i++)
+    {
+      /* Subdivide 256 possible values of each RGB component into 3
+	 regions: no color, normal color, bright color.  256 / 3 = 85,
+	 but ui-style.c follows xterm and uses 92 for R and G
+	 components of the bright-blue color, so we bias the divisor a
+	 bit to have the bright colors between 9 and 15 identical to
+	 what ui-style.c expects.  */
+      int bits = rgb[i] / 93;
+      retval |= ((bits > 0) << (2 - i)) | ((bits > 1) << 3);
+    }
+
+  return retval;
+}
+
+/* Zero if not yet initialized, 1 if stdout is a console device, else -1.  */
+static int mingw_console_initialized;
+
+/* Handle to stdout . */
+static HANDLE hstdout = INVALID_HANDLE_VALUE;
+
+/* Text attribute to use for normal text (the "none" pseudo-color).  */
+static SHORT  norm_attr;
+
+/* The most recently applied style.  */
+static ui_file_style last_style;
+
+/* Alternative for the libc 'fputs' which handles embedded SGR
+   sequences in support of styling.  */
+
+int
+gdb_console_fputs (const char *linebuf, FILE *fstream)
+{
+  if (!mingw_console_initialized)
+    {
+      hstdout = (HANDLE)_get_osfhandle (fileno (fstream));
+      DWORD cmode;
+      CONSOLE_SCREEN_BUFFER_INFO csbi;
+
+      if (hstdout != INVALID_HANDLE_VALUE
+	  && GetConsoleMode (hstdout, &cmode) != 0
+	  && GetConsoleScreenBufferInfo (hstdout, &csbi))
+	{
+	  norm_attr = csbi.wAttributes;
+	  mingw_console_initialized = 1;
+	}
+      else if (hstdout != INVALID_HANDLE_VALUE)
+	mingw_console_initialized = -1; /* valid, but not a console device */
+    }
+  /* If our stdout is not a console device, let the default 'fputs'
+     handle the task. */
+  if (mingw_console_initialized <= 0)
+    return 0;
+
+  /* Mapping between 8 ANSI colors and Windows console attributes.  */
+  static int fg_color[] = {
+    0,					/* black */
+    FOREGROUND_RED,			/* red */
+    FOREGROUND_GREEN,			/* green */
+    FOREGROUND_GREEN | FOREGROUND_RED,	/* yellow */
+    FOREGROUND_BLUE,			/* blue */
+    FOREGROUND_BLUE | FOREGROUND_RED,	/* magenta */
+    FOREGROUND_BLUE | FOREGROUND_GREEN, /* cyan */
+    FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE /* gray */
+  };
+  static int bg_color[] = {
+    0,					/* black */
+    BACKGROUND_RED,			/* red */
+    BACKGROUND_GREEN,			/* green */
+    BACKGROUND_GREEN | BACKGROUND_RED,	/* yellow */
+    BACKGROUND_BLUE,			/* blue */
+    BACKGROUND_BLUE | BACKGROUND_RED,	/* magenta */
+    BACKGROUND_BLUE | BACKGROUND_GREEN, /* cyan */
+    BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE /* gray */
+  };
+
+  ui_file_style style = last_style;
+  unsigned char c;
+  size_t n_read;
+
+  for ( ; (c = *linebuf) != 0; linebuf += n_read)
+    {
+      if (c == '\033')
+	{
+	  fflush (fstream);
+	  bool parsed = style.parse (linebuf, &n_read);
+	  if (n_read <= 0)	/* should never happen */
+	    n_read = 1;
+	  if (!parsed)
+	    {
+	      /* This means we silently swallow SGR sequences we
+		 cannot parse.  */
+	      continue;
+	    }
+	  /* Colors.  */
+	  const ui_file_style::color &fg = style.get_foreground ();
+	  const ui_file_style::color &bg = style.get_background ();
+	  int fgcolor, bgcolor, bright, inverse;
+	  if (fg.is_none ())
+	    fgcolor = norm_attr & 15;
+	  else if (fg.is_basic ())
+	    fgcolor = fg_color[fg.get_value () & 15];
+	  else
+	    fgcolor = rgb_to_16colors (fg);
+	  if (bg.is_none ())
+	    bgcolor = norm_attr & (15 << 4);
+	  else if (bg.is_basic ())
+	    bgcolor = bg_color[bg.get_value () & 15];
+	  else
+	    bgcolor = rgb_to_16colors (bg) << 4;
+
+	  /* Intensity.  */
+	  switch (style.get_intensity ())
+	    {
+	    case ui_file_style::NORMAL:
+	    case ui_file_style::DIM:
+	      bright = 0;
+	      break;
+	    case ui_file_style::BOLD:
+	      bright = 1;
+	      break;
+	    default:
+	      gdb_assert_not_reached ("invalid intensity");
+	    }
+
+	  /* Inverse video.  */
+	  if (style.is_reverse ())
+	    inverse = 1;
+	  else
+	    inverse = 0;
+
+	  /* Construct the attribute.  */
+	  if (inverse)
+	    {
+	      int t = fgcolor;
+	      fgcolor = (bgcolor >> 4);
+	      bgcolor = (t << 4);
+	    }
+	  if (bright)
+	    fgcolor |= FOREGROUND_INTENSITY;
+
+	  SHORT attr = (bgcolor & (15 << 4)) | (fgcolor & 15);
+
+	  /* Apply the attribute.  */
+	  SetConsoleTextAttribute (hstdout, attr);
+	}
+      else
+	{
+	  /* When we are about to write newline, we need to clear to
+	     EOL with the normal attribute, to avoid spilling the
+	     colors to the next screen line.  We assume here that no
+	     non-default attribute extends beyond the newline.  */
+	  if (c == '\n')
+	    {
+	      DWORD nchars;
+	      COORD start_pos;
+	      DWORD written;
+	      CONSOLE_SCREEN_BUFFER_INFO csbi;
+
+	      fflush (fstream);
+	      GetConsoleScreenBufferInfo (hstdout, &csbi);
+
+	      if (csbi.wAttributes != norm_attr)
+		{
+		  start_pos = csbi.dwCursorPosition;
+		  nchars = csbi.dwSize.X - start_pos.X;
+
+		  FillConsoleOutputAttribute (hstdout, norm_attr, nchars,
+					      start_pos, &written);
+		  FillConsoleOutputCharacter (hstdout, ' ', nchars,
+					      start_pos, &written);
+		}
+	    }
+	  fputc (c, fstream);
+	  n_read = 1;
+	}
+    }
+
+  last_style = style;
+  return 1;
+}
diff --git a/gdb/posix-hdep.c b/gdb/posix-hdep.c
index 7911bc5..d0f467e 100644
--- a/gdb/posix-hdep.c
+++ b/gdb/posix-hdep.c
@@ -30,3 +30,11 @@ gdb_select (int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
 {
   return select (n, readfds, writefds, exceptfds, timeout);
 }
+
+/* Host-dependent console fputs method.  POSIX platforms always return
+   zero, to use the default C 'fputs'.  */
+int
+gdb_console_fputs (const char *buf, FILE *f)
+{
+  return 0;
+}
diff --git a/gdb/ui-file.c b/gdb/ui-file.c
index 690fc62..77f6b31 100644
--- a/gdb/ui-file.c
+++ b/gdb/ui-file.c
@@ -236,6 +236,12 @@ stdio_file::write_async_safe (const char *buf, long length_buf)
 void
 stdio_file::puts (const char *linebuffer)
 {
+  /* This host-dependent function (with implementations in
+     posix-hdep.c and mingw-hdep.c) is given the opportunity to
+     process the output first in host-dependent way.  If it does, it
+     should return non-zero, to avoid calling fputs below.  */
+  if (gdb_console_fputs (linebuffer, m_file))
+    return;
   /* Calling error crashes when we are called from the exception framework.  */
   if (fputs (linebuffer, m_file))
     {
diff --git a/gdb/ui-file.h b/gdb/ui-file.h
index 26cf2bf..6e6ca1c 100644
--- a/gdb/ui-file.h
+++ b/gdb/ui-file.h
@@ -101,6 +101,8 @@ extern void ui_file_write_async_safe (struct ui_file *file, const char *buf,
 
 extern long ui_file_read (struct ui_file *file, char *buf, long length_buf);
 
+extern int gdb_console_fputs (const char *, FILE *);
+
 /* A std::string-based ui_file.  Can be used as a scratch buffer for
    collecting output.  */
 
diff --git a/gdb/utils.c b/gdb/utils.c
index 258614a..840779a 100644
--- a/gdb/utils.c
+++ b/gdb/utils.c
@@ -1445,8 +1445,18 @@ can_emit_style_escape (struct ui_file *stream)
       || !ui_file_isatty (stream))
     return false;
   const char *term = getenv ("TERM");
+  /* Windows doesn't by default define $TERM, but can support styles
+     regardless.  */
+#ifndef _WIN32
   if (term == nullptr || !strcmp (term, "dumb"))
     return false;
+#else
+  /* But if they do define $TERM, let us behave the same as on Posix
+     platforms, for the benefit of programs which invoke GDB as their
+     back-end.  */
+  if (term && !strcmp (term, "dumb"))
+    return false;
+#endif
   return true;
 }