add overhead eval data analysis pipeline

Essoz · Essoz · commit ab9b7b11dc7b · 2025-01-08T11:18:14.000-05:00
diff --git a/.github/workflows/bench-instr-e2e.yml b/.github/workflows/bench-instr-e2e.yml
diff --git a/eval_scripts/perf_benchmark/analysis.xsh b/eval_scripts/perf_benchmark/analysis.xsh
@@ -0,0 +1,52 @@
+import argparse
+import pandas as pd
+import numpy as np
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--res_folder", type=str, required=True)
+args = parser.parse_args()
+
+res_folder = args.res_folder
+
+# only need to handle the marco benchmark results
+# list all the files in the folder
+files = $(ls @(res_folder)).split()
+files = [f for f in files if f.startswith("e2e_")]
+
+"""
+FORMAT OF THE DATA TO PRODUCE FOR E2E
+
+task,method,overhead
+MNIST,systrace,549.57
+ResNet18,systrace,338.43
+Transformer,systrace,205.34
+MNIST,monkey-patch,148.22
+ResNet18,monkey-patch,29.62
+Transformer,monkey-patch,63.12
+MNIST,selective,1.61
+ResNet18,selective,1.07
+Transformer,selective,1.17
+"""
+
+all_results = {}
+for f in files:
+    series = np.loadtxt(f"{res_folder}/{f}")
+    task = f.split("_")[1]
+    method = f.split("_")[2].split(".")[0]
+    if task not in all_results:
+        all_results[task] = {}
+    all_results[task][method] = series.mean()
+
+overhead_results = []
+for task in all_results:
+    assert "naive" in all_results[task], f"naive (base situtation) not found in {task}"
+    for method in all_results[task]:
+        if method == "naive":
+            continue
+        overhead = all_results[task][method] / all_results[task]["naive"]
+        overhead_results.append([task, method, overhead])
+        
+        
+df = pd.DataFrame(overhead_results, columns=["task", "method", "overhead"])
+# dump to csv
+df.to_csv(f"{res_folder}/overhead_e2e.csv", index=False)
diff --git a/eval_scripts/perf_benchmark/run_all.xsh b/eval_scripts/perf_benchmark/run_all.xsh
@@ -1,13 +1,23 @@
 import os
 import subprocess
 
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--res_folder", type=str, required=False)
+args = parser.parse_args()
+
 # configs
 $RAISE_SUBPROC_ERROR = True
 os.environ["PYTHONUNBUFFERED"] = "1"
 
 SELC_INV_FILE = "sampled_100_invariants.json"
-COMMIT = $(git rev-parse --short HEAD)
-RES_FOLDER = f"perf_eval_res_{COMMIT}"
+COMMIT = $(git rev-parse --short HEAD).strip()
+
+if args.res_folder:
+    RES_FOLDER = args.res_folder
+else:
+    RES_FOLDER = f"perf_eval_res_{COMMIT}"
 
 MICRO_FOLDER = "overhead-micro"
 E2E_FOLDER = "overhead-e2e"
@@ -59,20 +69,20 @@ def run_exp(kill_sec: int = 100, workload: str = "mnist"):
     print("Running settrace setup")
     run_cmd(cmd_settrace, kill_sec)
     rm api_calls.log
-    cp iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_settrace.txt")
+    cp iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_systrace.txt")
     rm iteration_times.txt
 
     # 3. traincheck proxy instrumentation
     print("Running traincheck proxy instrumentation")
     run_cmd(CMD_TRAINCHECK, kill_sec)
-    cp traincheck/iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_traincheck.txt")
+    cp traincheck/iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_monkey-patch.txt")
     rm -rf traincheck
     # rm iteration_times.txt
 
     # 4. traincheck selective instrumentation
     print("Running traincheck selective instrumentation")
     run_cmd(CMD_TRAINCHECK_SELECTIVE, kill_sec)
-    cp traincheck-selective/iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_traincheck_selective.txt")
+    cp traincheck-selective/iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_selective.txt")
     rm -rf traincheck-selective
 
     cd ../..
