[libFuzzer] add exeprimental -rss_limit_mb flag to fight against OOMs

llvm-svn: 268807

1 files changed, 11 insertions, 0 deletions

diff --git a/llvm/lib/Fuzzer/FuzzerLoop.cpp b/llvm/lib/Fuzzer/FuzzerLoop.cpp
index 3b0b339..e02ebcf 100644
--- a/llvm/lib/Fuzzer/FuzzerLoop.cpp
+++ b/llvm/lib/Fuzzer/FuzzerLoop.cpp
@@ -176,6 +176,17 @@ void Fuzzer::AlarmCallback() {
   }
 }
 
+void Fuzzer::RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb) {
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
+         GetPid(), RssPeakMb, RssLimitMb);
+  Printf("*****************************************************************\n");
+  Printf("** Experimental! TODO: dump the stack trace and the reproducer **\n");
+  Printf("*****************************************************************\n");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 void Fuzzer::PrintStats(const char *Where, const char *End) {
   size_t ExecPerSec = execPerSec();
   if (Options.OutputCSV) {