[libFuzzer] Handle unstable edges by using minimum hit counts

Summary:
Created unstable_handle flag that takes 1 or 2, depending on the handling type.
Modified RunOne to accommodate the following heuristic:
    Use the first CollectFeatures to count how many features there are.
    If no new features, CollectFeatures like before.
    If there is new feature, we run CB 2 more times,
        Check which edges are unstable per input and we store the least amount of hit counts for each edge.
        Apply these hit counts back to inline8bitcounters so that CollectFeatures can work as intended.
Modified UnstableCounters to 8int_t and created a bitset UnstableSet to tell which edges are unstable.

Patch by Kyungtak Woo (@kevinwkt).

Reviewers: Dor1s, metzman, morehouse

Reviewed By: Dor1s, morehouse

Subscribers: delcypher, #sanitizers, llvm-commits, kcc

Differential Revision: https://reviews.llvm.org/D49525

llvm-svn: 337696

1 files changed, 15 insertions, 6 deletions

diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
index 983b474..859ff63 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
@@ -75,17 +75,26 @@ void TracePC::IterateInline8bitCounters(CallBack CB) const {
 // counters.
 void TracePC::InitializeUnstableCounters() {
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
-    if (UnstableCounters[UnstableIdx] != kUnstableCounter)
-      UnstableCounters[UnstableIdx] = ModuleCounters[i].Start[j];
+    UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j];
   });
 }
 
 // Compares the current counters with counters from previous runs
 // and records differences as unstable edges.
-void TracePC::UpdateUnstableCounters() {
+void TracePC::UpdateUnstableCounters(int UnstableMode) {
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
-    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx])
-      UnstableCounters[UnstableIdx] = kUnstableCounter;
+    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter)
+      UnstableCounters[UnstableIdx].IsUnstable = true;
+    if (UnstableMode &&
+        ModuleCounters[i].Start[j] < UnstableCounters[UnstableIdx].Counter)
+      UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j];
+  });
+}
+
+// Moves the minimum hit counts to ModuleCounters.
+void TracePC::ApplyUnstableCounters() {
+  IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
+    ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter;
   });
 }
 
@@ -340,7 +349,7 @@ void TracePC::DumpCoverage() {
 void TracePC::PrintUnstableStats() {
   size_t count = 0;
   for (size_t i = 0; i < NumInline8bitCounters; i++)
-    if (UnstableCounters[i] == kUnstableCounter)
+    if (UnstableCounters[i].IsUnstable)
       count++;
   Printf("stat::stability_rate: %.2f\n",
          100 - static_cast<float>(count * 100) / NumInline8bitCounters);