Merge pull request #97 from Flamefire/benchmark

Improve readability of benchmark script and fix CInstrumenter benchmark

Author: AndreasGocht

benchmark/benchmark.py

@@ -7,20 +7,29 @@
 import benchmark_helper
 import pickle
 
-bench = benchmark_helper.BenchmarkEnv(repetitions=51)
 tests = ["bm_baseline.py", "bm_simplefunc.py"]
-results = {}
 
-reps_x = {}
-reps_x["bm_baseline.py"] = ["1000000", "2000000", "3000000", "4000000", "5000000"]
-reps_x["bm_simplefunc.py"] = ["100000", "200000", "300000", "400000", "500000"]
+instrumenters = ["profile", "trace", "dummy", "None"]
+if sys.version_info.major >= 3:
+    instrumenters.extend(["cProfile", "cTrace"])
+
+# How many times the instrumented code is run during 1 test run
+reps_x = {
+    "bm_baseline.py": ["1000000", "2000000", "3000000", "4000000", "5000000"],
+    "bm_simplefunc.py": ["100000", "200000", "300000", "400000", "500000"],
+}
+# How many times a test invocation is repeated (number of timings per test instance)
+test_repetitions = 51
+
+bench = benchmark_helper.BenchmarkEnv(repetitions=test_repetitions)
+results = {}
 
 for test in tests:
-    results[test] = {"profile": {}, "trace": {}, "dummy": {}, "None": {}}
-    if sys.version_info.major >= 3:
-        results.update({"cProfile": {}, "cTrace": {}})
+    results[test] = {}
+
+    for instrumenter in instrumenters:
+        results[test][instrumenter] = {}
 
-    for instrumenter in results[test]:
         if instrumenter == "None":
             enable_scorep = False
             scorep_settings = []
@@ -35,8 +44,8 @@
             times = bench.call(test, [reps],
                                enable_scorep,
                                scorep_settings=scorep_settings)
-            results[test][instrumenter][reps] = times
             print("{:<8}: {}".format(reps, times))
+            results[test][instrumenter][reps] = times
 
 with open("results.pkl", "wb") as f:
     pickle.dump(results, f)

benchmark/benchmark_helper.py

@@ -31,18 +31,16 @@ def call(self, script="", ops=[], enable_scorep=True, scorep_settings=[]):
             arguments.extend(scorep_settings)
         arguments.append(script)
         arguments.extend(ops)
+        print(arguments)
 
         runtimes = []
-        for i in range(self.repetitions):
+        for _ in range(self.repetitions):
             begin = time.time()
-            print(arguments)
             out = subprocess.run(
                 arguments,
-                env=self.env,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE)
+                env=self.env)
             end = time.time()
-            assert(out.returncode == 0)
+            assert out.returncode == 0
 
             runtime = end - begin
             runtimes.append(runtime)