[dumpling] Implement differential fuzzing

Bug: 441467877
Change-Id: I7278380605e40ca79b4dc889cb8b6734aa7c4327
Reviewed-on: https://chrome-internal-review.googlesource.com/c/v8/fuzzilli/+/8908076
Reviewed-by: Matthias Liedtke <mliedtke@google.com>
Commit-Queue: Danylo Mocherniuk <mdanylo@google.com>

Author: Danylo Mocherniuk

Sources/Fuzzilli/Base/Contributor.swift

@@ -28,6 +28,8 @@ public class Contributor: Hashable {
     private var timedOutSamples = 0
     // Number of crashing programs produced.
     private var crashingSamples = 0
+    // The number of programs resulting in valid differential produced.
+    private var differentialSamples = 0
     // The number of times this contributor has been invoked
     private(set) var invocationCount = 0
     // The number of times this contributor has actually emitted more than 0 instructions
@@ -46,6 +48,10 @@ public class Contributor: Hashable {
         validSamples += 1
     }
 
+    func generatedDifferentialSample() {
+        differentialSamples += 1
+    }
+
     func generatedInterestingSample() {
         interestingSamples += 1
     }
@@ -82,8 +88,12 @@ public class Contributor: Hashable {
         return crashingSamples
     }
 
+    public var differentialsFound: Int {
+        return differentialSamples
+    }
+
     public var totalSamples: Int {
-        return validSamples + interestingSamples + invalidSamples + timedOutSamples + crashingSamples
+        return validSamples + interestingSamples + invalidSamples + timedOutSamples + crashingSamples + differentialSamples
     }
 
     // If this is low, that means the CodeGenerator has dynamic requirements that are not met most of the time.
@@ -151,4 +161,8 @@ extension Contributors {
     public func generatedTimeOutSample() {
         forEach { $0.generatedTimeOutSample() }
     }
+
+    public func generatedDifferentialSample() {
+        forEach { $0.generatedDifferentialSample() }
+    }
 }

Sources/Fuzzilli/Base/Events.swift

@@ -55,6 +55,9 @@ public class Events {
     /// Signals that a crashing program has been found. Dispatched after the crashing program has been minimized.
     public let CrashFound = Event<(program: Program, behaviour: CrashBehaviour, isUnique: Bool, origin: ProgramOrigin)>()
 
+    /// Signals that a differential program was found. Dispatched after the differential program has been minimized.
+    public let DifferentialFound = Event<(program: Program, behaviour: CrashBehaviour, isUnique: Bool, origin: ProgramOrigin)>()
+
     /// Signals that a program causing a timeout has been found.
     public let TimeOutFound = Event<Program>()
 
@@ -140,4 +143,15 @@ public enum ExecutionPurpose {
     case runtimeAssistedMutation
     /// Any other reason.
     case other
+
+    // This variable is added because we currently don't want to differentially execute
+    // a sample if the purpose is unknown (`.other`), `.startup` or `.runtimeAssistedMutation`.
+    public var supportsDifferentialRun: Bool {
+        switch self {
+        case .fuzzing, .checkForDeterministicBehavior, .programImport, .minimization:
+            return true
+        case .startup, .runtimeAssistedMutation, .other:
+            return false
+        }
+    }
 }

Sources/Fuzzilli/Configuration.swift

@@ -12,7 +12,36 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// TODO(mdanylo): this should be part of the protocol between Fuzzilli and V8.
+public struct DifferentialConfig {
+    public let dumpFilenamePattern: String
+
+    public init(dumpFilenamePattern: String) {
+        self.dumpFilenamePattern = dumpFilenamePattern
+    }
+
+    public func getDumpFilename(isOptimized: Bool) -> String {
+        let subDirectory = isOptimized ? "optimizedDump" : "unoptimizedDump"
+        return String(format: dumpFilenamePattern, subDirectory)
+    }
+
+    public func getDumpFilenameParameter(isOptimized: Bool) -> String {
+        return "--dump-out-filename=\(getDumpFilename(isOptimized: isOptimized))"
+    }
+
+    public static func create(for instanceId: Int, storagePath: String) -> DifferentialConfig {
+        // TODO(mdanylo): it would be cool to add random hash to the filename if we store dumps
+        // in the filesystem for debugging.
+        let fileName = "output_dump_\(instanceId).txt"
+        let pattern = "\(storagePath)/%@/\(fileName)"
+        return DifferentialConfig(dumpFilenamePattern: pattern)
+    }
+}
+
 public struct Configuration {
+    /// The config specific to differential fuzzing
+    public let diffConfig: DifferentialConfig?
+
     /// The commandline arguments used by this instance.
     public let arguments: [String]
 
@@ -91,7 +120,9 @@ public struct Configuration {
                 tag: String? = nil,
                 isWasmEnabled: Bool = false,
                 storagePath: String? = nil,
-                forDifferentialFuzzing: Bool = false) {
+                forDifferentialFuzzing: Bool = false,
+                instanceId: Int = -1,
+                dumplingEnabled: Bool = false) {
         self.arguments = arguments
         self.timeout = timeout
         self.logLevel = logLevel
@@ -106,6 +137,11 @@ public struct Configuration {
         self.isWasmEnabled = isWasmEnabled
         self.storagePath = storagePath
         self.forDifferentialFuzzing = forDifferentialFuzzing
+        self.diffConfig = dumplingEnabled ? DifferentialConfig.create(for: instanceId, storagePath: storagePath!) : nil
+    }
+
+    public func getInstanceSpecificArguments(forReferenceRunner: Bool) -> [String] {
+        return diffConfig.map { [$0.getDumpFilenameParameter(isOptimized: !forReferenceRunner)] } ?? []
     }
 }
 

Sources/Fuzzilli/Engines/FuzzEngine.swift

@@ -44,6 +44,10 @@ public class FuzzEngine: ComponentBase {
                 fuzzer.processCrash(program, withSignal: termsig, withStderr: execution.stderr, withStdout: execution.stdout, origin: .local, withExectime: execution.execTime)
                 program.contributors.generatedCrashingSample()
 
+            case .differential:
+                fuzzer.processDifferential(program, withStderr: execution.stderr, withStdout: execution.stdout, origin: .local)
+                program.contributors.generatedDifferentialSample()
+
             case .succeeded:
                 fuzzer.dispatchEvent(fuzzer.events.ValidProgramFound, data: program)
                 var isInteresting = false

Sources/Fuzzilli/Engines/HybridEngine.swift

@@ -21,6 +21,7 @@ public class HybridEngine: FuzzEngine {
     // The different outcomes of one fuzzing iterations.
     private enum CodeGenerationOutcome: String, CaseIterable {
         case success = "Success"
+        case generatedCodeDifferential = "Generated code differential"
         case generatedCodeFailed = "Generated code failed"
         case generatedCodeTimedOut = "Generated code timed out"
         case generatedCodeCrashed = "Generated code crashed"
@@ -112,6 +113,8 @@ public class HybridEngine: FuzzEngine {
             return recordOutcome(.generatedCodeTimedOut)
         case .crashed:
             return recordOutcome(.generatedCodeCrashed)
+        case .differential:
+            return recordOutcome(.generatedCodeDifferential)
         }
 
         // Now perform one round of fixup to improve the generated program based on runtime information and in particular remove all try-catch guards that are not needed.

Sources/Fuzzilli/Evaluation/ProgramCoverageEvaluator.swift

@@ -192,8 +192,8 @@ public class ProgramCoverageEvaluator: ComponentBase, ProgramEvaluator {
             return false
         }
 
-        if execution.outcome.isCrash() {
-            // For crashes, we don't care about the edges that were triggered, just about the outcome itself.
+        if execution.outcome.isCrash() || execution.outcome.isDifferential() {
+            // For crashes and differentials, we don't care about the edges that were triggered, just about the outcome itself.
             return true
         }
 

Sources/Fuzzilli/Execution/Execution.swift

@@ -20,6 +20,12 @@ public enum ExecutionOutcome: CustomStringConvertible, Equatable, Hashable {
     case failed(Int)
     case succeeded
     case timedOut
+    // This outcome is added to support native differential fuzzing.
+    // It should get very similar treatment to crashed -> if the run resulted
+    // in a differential, most likely there's a bug.
+    // Please note that this feature is unstable yet, so the statement above
+    // might not always be the case.
+    case differential
 
     public var description: String {
         switch self {
@@ -31,6 +37,8 @@ public enum ExecutionOutcome: CustomStringConvertible, Equatable, Hashable {
             return "Succeeded"
         case .timedOut:
             return "TimedOut"
+        case .differential:
+            return "Differential"
         }
     }
 
@@ -49,6 +57,14 @@ public enum ExecutionOutcome: CustomStringConvertible, Equatable, Hashable {
             return false
         }
     }
+
+    public func isDifferential() -> Bool {
+        if case .differential = self {
+            return true
+        } else {
+            return false
+        }
+    }
 }
 
 /// The result of executing a program.
@@ -59,3 +75,54 @@ public protocol Execution {
     var fuzzout: String { get }
     var execTime: TimeInterval { get }
 }
+
+/// Struct to capture result of exection in differential mode
+struct DiffExecution: Execution {
+    let outcome: ExecutionOutcome
+    let execTime: TimeInterval
+    let stdout: String
+    let stderr: String
+    let fuzzout: String
+
+    private init(
+        outcome: ExecutionOutcome,
+        execTime: TimeInterval,
+        stdout: String,
+        stderr: String,
+        fuzzout: String
+    ) {
+        self.outcome = outcome
+        self.execTime = execTime
+        self.stdout = stdout
+        self.stderr = stderr
+        self.fuzzout = fuzzout
+    }
+
+    // TODO(mdanylo): we shouldn't pass dump outputs as a separate parameter,
+    // instead we should rather make them a part of a REPRL protocol between Fuzzilli and V8.
+    static func diff(optExec: Execution, unoptExec: Execution,
+            optDumpOut: String, unoptDumpOut: String) -> Execution {
+
+        assert(optExec.outcome == .succeeded && unoptExec.outcome == .succeeded)
+
+        func formatDiff(label: String, optData: String, unoptData: String) -> String {
+            return """
+            === OPT \(label) ===
+            \(optData)
+
+            === UNOPT \(label) ===
+            \(unoptData)
+            """
+        }
+
+        let relateOutcome = DiffOracle.relate(optDumpOut, with: unoptDumpOut)
+
+        return DiffExecution(
+            outcome: relateOutcome ? .succeeded : .differential,
+            execTime: optExec.execTime,
+            stdout: formatDiff(label: "STDOUT", optData: optExec.stdout, unoptData: unoptExec.stdout),
+            stderr: formatDiff(label: "STDERR", optData: optExec.stderr, unoptData: unoptExec.stderr),
+            fuzzout: formatDiff(label: "FUZZOUT", optData: optExec.fuzzout, unoptData: unoptExec.fuzzout)
+        )
+    }
+}