Fix PR C++/19351: integer overflow in operator new[]

2012-08-20  Florian Weimer  <fweimer@redhat.com>

	PR c++/19351
	* call.c (build_operator_new_call): Add size_check argument and
	evaluate it.
	* cp-tree.h (build_operator_new_call): Adjust declaration.
	* init.c (build_new_1): Compute array size check and apply it.

2012-08-10  Florian Weimer  <fweimer@redhat.com>

	PR c++/19351
	* g++.dg/init/new38.C: New test.
	* g++.dg/init/new39.C: New test.

From-SVN: r190546

3 files changed, 128 insertions, 0 deletions

diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 0db9e17..c31dca5 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,9 @@
+2012-08-20  Florian Weimer  <fweimer@redhat.com>
+
+	PR c++/19351
+	* g++.dg/init/new38.C: New test.
+	* g++.dg/init/new39.C: New test.
+
 2012-08-20  Oleg Endo  <olegendo@gcc.gnu.org>
 
 	PR target/50489
diff --git a/gcc/testsuite/g++.dg/init/new38.C b/gcc/testsuite/g++.dg/init/new38.C
new file mode 100644
index 0000000..1672f22
--- /dev/null
+++ b/gcc/testsuite/g++.dg/init/new38.C
@@ -0,0 +1,54 @@
+// { dg-do compile }
+
+void
+large_array_char(int n)
+{
+  new char[n]
+    [1ULL << (sizeof(void *) * 4)]
+    [1ULL << (sizeof(void *) * 4)]; // { dg-error "size of array" }
+}
+
+template <typename T>
+void
+large_array_char_template(int n)
+{
+  new char[n]
+    [1ULL << (sizeof(void *) * 4)]
+    [1ULL << (sizeof(void *) * 4)]; // { dg-error "size of array" }
+}
+
+
+template <typename T>
+void
+large_array_template1(int n)
+{
+  new T[n] // { dg-error "size of array is too large" }
+    [(1ULL << (sizeof(void *) * 4)) / sizeof(T)]
+    [1ULL << (sizeof(void *) * 4)];
+}
+
+template <typename T>
+void
+large_array_template2(int n)
+{
+  new T[n] // { dg-error "size of array is too large" }
+    [(1ULL << (sizeof(void *) * 4)) / sizeof(T)]
+    [1ULL << (sizeof(void *) * 4)];
+}
+
+template <typename T>
+void
+large_array_template3(int n)
+{
+  new T[n] // { dg-error "size of array is too large" }
+    [(1ULL << (sizeof(void *) * 4)) / sizeof(T)]
+    [1ULL << (sizeof(void *) * 4)];
+}
+
+void
+call_large_array_template(int n)
+{
+  large_array_template1<char>(n);
+  large_array_template2<int>(n);
+  large_array_template3<double>(n);
+}
diff --git a/gcc/testsuite/g++.dg/init/new39.C b/gcc/testsuite/g++.dg/init/new39.C
new file mode 100644
index 0000000..f274ebb
--- /dev/null
+++ b/gcc/testsuite/g++.dg/init/new39.C
@@ -0,0 +1,68 @@
+// Testcase for overflow handling in operator new[].
+// { dg-do run }
+
+#include <stdlib.h>
+#include <stdexcept>
+
+struct without_new {
+  char bar[256];
+};
+
+struct with_new {
+  char bar[256];
+  void *operator new[] (size_t sz)
+  {
+    if (sz != -1)
+      abort ();
+    throw std::bad_alloc();
+  }
+};
+
+template <typename T>
+inline void
+test (size_t s)
+{
+  try {
+    new T[s];
+    abort ();
+  } catch (std::bad_alloc &) {
+  }
+}
+
+template <typename T>
+void
+test_noopt (size_t s) __attribute__((noinline));
+
+template <typename T>
+void
+test_noopt (size_t s)
+{
+  __asm__ ("");
+  test<T> (s);
+}
+
+template <typename T>
+void
+all_tests ()
+{
+  test<T>(-1);
+  test<T>(size_t(-1) / sizeof (T) + 1);
+  test<T>(size_t(-1) / sizeof (T) + 2);
+  test_noopt<T>(-1);
+  test_noopt<T>(size_t(-1) / sizeof (T) + 1);
+  test_noopt<T>(size_t(-1) / sizeof (T) + 2);
+}
+
+int
+main ()
+{
+  try {
+    ::operator new(size_t(-1));
+    abort ();
+  } catch (std::bad_alloc &) {
+  }
+  all_tests<without_new> ();
+  all_tests<with_new> ();
+  return 0;
+}
+