PR23881, pdp11 binutils fails if too much debug data

The PR testcase overflows one of the exec header fields, e_syms (the
size of the symbol table), leading to the string table offset being
wrong.  Things go downhill from there.  Fixed by checking for
overflow.  This happens to trigger in the ld testsuite, so xfail that
test.

	PR 23881
bfd/
	* libaout.h (swap_exec_header_out): Return a bool.
	* aoutx.h (swap_exec_header_out): Check for overflow in exec
	header.
	* pdp11.c (swap_exec_header_out): Likewise.
	* i386lynx.c (WRITE_HEADERS): Adjust.
ld/
	* testsuite/ld-scripts/map-address.exp: xfail pdp11.

5 files changed, 61 insertions, 7 deletions

diff --git a/bfd/aoutx.h b/bfd/aoutx.h
index 545285c..c8aaa14 100644
--- a/bfd/aoutx.h
+++ b/bfd/aoutx.h
@@ -407,7 +407,7 @@ FUNCTION
 	aout_@var{size}_swap_exec_header_out
 
 SYNOPSIS
-	void aout_@var{size}_swap_exec_header_out
+	bool aout_@var{size}_swap_exec_header_out
 	  (bfd *abfd,
 	   struct internal_exec *execp,
 	   struct external_exec *raw_bytes);
@@ -416,11 +416,37 @@ DESCRIPTION
 	Swap the information in an internal exec header structure
 	@var{execp} into the buffer @var{raw_bytes} ready for writing to disk.
 */
-void
+bool
 NAME (aout, swap_exec_header_out) (bfd *abfd,
 				   struct internal_exec *execp,
 				   struct external_exec *bytes)
 {
+  const char *err = NULL;
+  uint64_t val;
+#define MAXVAL(x) ((UINT64_C (1) << (8 * sizeof (x) - 1) << 1) - 1)
+  if ((val = execp->a_text) > MAXVAL (bytes->e_text))
+    err = "e_text";
+  else if ((val = execp->a_data) > MAXVAL (bytes->e_data))
+    err = "e_data";
+  else if ((val = execp->a_bss) > MAXVAL (bytes->e_bss))
+    err = "e_bss";
+  else if ((val = execp->a_syms) > MAXVAL (bytes->e_syms))
+    err = "e_syms";
+  else if ((val = execp->a_entry) > MAXVAL (bytes->e_entry))
+    err = "e_entry";
+  else if ((val = execp->a_trsize) > MAXVAL (bytes->e_trsize))
+    err = "e_trsize";
+  else if ((val = execp->a_drsize) > MAXVAL (bytes->e_drsize))
+    err = "e_drsize";
+#undef MAXVAL
+  if (err)
+    {
+      _bfd_error_handler (_("%pB: %#" PRIx64 " overflows header %s field"),
+			  abfd, val, err);
+      bfd_set_error (bfd_error_file_too_big);
+      return false;
+    }
+
   /* Now fill in fields in the raw data, from the fields in the exec struct.  */
   H_PUT_32 (abfd, execp->a_info  , bytes->e_info);
   PUT_WORD (abfd, execp->a_text  , bytes->e_text);
@@ -430,6 +456,7 @@ NAME (aout, swap_exec_header_out) (bfd *abfd,
   PUT_WORD (abfd, execp->a_entry , bytes->e_entry);
   PUT_WORD (abfd, execp->a_trsize, bytes->e_trsize);
   PUT_WORD (abfd, execp->a_drsize, bytes->e_drsize);
+  return true;
 }
 
 /* Make all the section for an a.out file.  */
diff --git a/bfd/i386lynx.c b/bfd/i386lynx.c
index cd06a64..1d4c411 100644
--- a/bfd/i386lynx.c
+++ b/bfd/i386lynx.c
@@ -46,7 +46,8 @@
 		       * obj_reloc_entry_size (abfd));			\
     execp->a_drsize = ((obj_datasec (abfd)->reloc_count)		\
 		       * obj_reloc_entry_size (abfd));			\
-    NAME (aout, swap_exec_header_out) (abfd, execp, &exec_bytes);	\
+    if (!NAME (aout, swap_exec_header_out) (abfd, execp, &exec_bytes))	\
+      return false;							\
 									\
     if (bfd_seek (abfd, 0, SEEK_SET) != 0				\
 	|| bfd_write (&exec_bytes, EXEC_BYTES_SIZE,			\
diff --git a/bfd/libaout.h b/bfd/libaout.h
index 11a6f70..91033d4 100644
--- a/bfd/libaout.h
+++ b/bfd/libaout.h
@@ -570,7 +570,7 @@ extern bool NAME (aout, adjust_sizes_and_vmas)
 extern void NAME (aout, swap_exec_header_in)
   (bfd *, struct external_exec *, struct internal_exec *);
 
-extern void NAME (aout, swap_exec_header_out)
+extern bool NAME (aout, swap_exec_header_out)
   (bfd *, struct internal_exec *, struct external_exec *);
 
 extern struct bfd_hash_entry * NAME (aout, link_hash_newfunc)
@@ -631,7 +631,8 @@ extern bool NAME (aout, bfd_free_cached_info)
 		       * obj_reloc_entry_size (abfd));			\
     execp->a_drsize = ((obj_datasec (abfd)->reloc_count)		\
 		       * obj_reloc_entry_size (abfd));			\
-    NAME (aout, swap_exec_header_out) (abfd, execp, &exec_bytes);	\
+    if (!NAME (aout, swap_exec_header_out) (abfd, execp, &exec_bytes))	\
+      return false;							\
 									\
     if (bfd_seek (abfd, 0, SEEK_SET) != 0				\
 	|| bfd_write (&exec_bytes, EXEC_BYTES_SIZE,			\
diff --git a/bfd/pdp11.c b/bfd/pdp11.c
index 7d93904..e83a485 100644
--- a/bfd/pdp11.c
+++ b/bfd/pdp11.c
@@ -365,7 +365,8 @@ pdp11_aout_write_headers (bfd *abfd, struct internal_exec *execp)
       execp->a_drsize = 0;
     }
 
-  NAME (aout, swap_exec_header_out) (abfd, execp, & exec_bytes);
+  if (!NAME (aout, swap_exec_header_out) (abfd, execp, & exec_bytes))
+    return false;
 
   if (bfd_seek (abfd, 0, SEEK_SET) != 0)
     return false;
@@ -456,11 +457,33 @@ NAME (aout, swap_exec_header_in) (bfd *abfd,
 
 /*  Swap the information in an internal exec header structure
     "execp" into the buffer "bytes" ready for writing to disk.  */
-void
+bool
 NAME (aout, swap_exec_header_out) (bfd *abfd,
 				   struct internal_exec *execp,
 				   struct external_exec *bytes)
 {
+  const char *err = NULL;
+  uint64_t val;
+#define MAXVAL(x) ((UINT64_C (1) << (8 * sizeof (x) - 1) << 1) - 1)
+  if ((val = execp->a_text) > MAXVAL (bytes->e_text))
+    err = "e_text";
+  else if ((val = execp->a_data) > MAXVAL (bytes->e_data))
+    err = "e_data";
+  else if ((val = execp->a_bss) > MAXVAL (bytes->e_bss))
+    err = "e_bss";
+  else if ((val = execp->a_syms) > MAXVAL (bytes->e_syms))
+    err = "e_syms";
+  else if ((val = execp->a_entry) > MAXVAL (bytes->e_entry))
+    err = "e_entry";
+#undef MAXVAL
+  if (err)
+    {
+      _bfd_error_handler (_("%pB: %#" PRIx64 " overflows header %s field"),
+			  abfd, val, err);
+      bfd_set_error (bfd_error_file_too_big);
+      return false;
+    }
+
   /* Now fill in fields in the raw data, from the fields in the exec struct.  */
   PUT_MAGIC (abfd, execp->a_info,		bytes->e_info);
   PUT_WORD (abfd, execp->a_text,		bytes->e_text);
@@ -482,6 +505,7 @@ NAME (aout, swap_exec_header_out) (bfd *abfd,
       fprintf (stderr, "BFD:%s:%d: internal error\n", __FILE__, __LINE__);
       PUT_WORD (abfd, 0,			bytes->e_flag);
     }
+  return true;
 }
 
 /* Make all the section for an a.out file.  */
diff --git a/ld/testsuite/ld-scripts/map-address.exp b/ld/testsuite/ld-scripts/map-address.exp
index 3944ec1..bc1f5a6 100644
--- a/ld/testsuite/ld-scripts/map-address.exp
+++ b/ld/testsuite/ld-scripts/map-address.exp
@@ -32,6 +32,7 @@ if { [is_pecoff_format] } then {
     set IMAGE_BASE ""
 }
 
+setup_xfail "pdp11-*-*"
 if {![ld_link $ld tmpdir/map-address \
 	 "$LDFLAGS -T $srcdir/$subdir/map-address.t \
 	  $IMAGE_BASE tmpdir/map-address.o \