fix: workaround xonsh bug that if terminated a subprocess not in a separate shell any later subprocess fails silently

Essoz · Essoz · commit 0cb25a952b36 · 2025-01-08T17:52:05.000-05:00
diff --git a/.github/workflows/eval-overhead-e2e.yml b/.github/workflows/eval-overhead-e2e.yml
@@ -25,7 +25,7 @@ jobs:
   benchmark:
     runs-on: self-hosted
 
-    name: Run pytest-benchmark benchmark example
+    name: Run Overhead Benchmarks
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
diff --git a/eval_scripts/perf_benchmark/run_all.xsh b/eval_scripts/perf_benchmark/run_all.xsh
@@ -1,17 +1,16 @@
+import argparse
 import os
+import signal
 import subprocess
 
-import argparse
+# configs
+$RAISE_SUBPROC_ERROR = True
+os.environ["PYTHONUNBUFFERED"] = "1"
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--res_folder", type=str, required=False)
 args = parser.parse_args()
 
-# configs
-$RAISE_SUBPROC_ERROR = True
-$XONSH_SHOW_TRACEBACK = True
-os.environ["PYTHONUNBUFFERED"] = "1"
-
 SELC_INV_FILE = "sampled_100_invariants.json"
 COMMIT = $(git rev-parse --short HEAD).strip()
 
@@ -35,16 +34,17 @@ mv @(MICRO_FOLDER)/wrapper_overhead_micro.csv @(RES_FOLDER)/
 
 def run_cmd(cmd: str, kill_sec: int):
     with open("cmd_output.log", "w") as f:
-        p = subprocess.Popen(cmd.split(), stdout=f, stderr=f)
+        p = subprocess.Popen(cmd, shell=True, stdout=f, stderr=f)
         try:
             output, _ = p.communicate(timeout=kill_sec)
         except subprocess.TimeoutExpired:
             print(f"Timeout: {kill_sec} seconds, killing the process")
-            p.terminate()
-            try:
-                p.wait(timeout=5)
-            except subprocess.TimeoutExpired:
-                p.kill()
+            # os.kill(
+            #     p.pid, signal.SIGTERM
+            # )  # send SIGTERM to the process group NOTE: the signal will be delivered here again
+            # p.kill()
+            p.terminate() # sends SIGTERM
+            print("Killed the running process...")
 
 # run e2e benchmark
 def run_exp(kill_sec: int = 100, workload: str = "mnist"):
@@ -63,7 +63,7 @@ def run_exp(kill_sec: int = 100, workload: str = "mnist"):
 
     cd f"{E2E_FOLDER}/{workload}"
 
-    # run five setups
+    # run four setups
 
     # 1. naive running
     print("Running naive setup")
@@ -79,11 +79,14 @@ def run_exp(kill_sec: int = 100, workload: str = "mnist"):
     rm iteration_times.txt
 
     # 3. traincheck proxy instrumentation
-    print("Running traincheck proxy instrumentation")
-    run_cmd(CMD_TRAINCHECK, 30)
+    print("Running traincheck instrumentation")
+    run_cmd(CMD_TRAINCHECK, kill_sec)
+    print("Trying to copy")
+    print(os.listdir("traincheck"))
+    # shutil.copy("traincheck/iteration_times.txt", f"../../{RES_FOLDER}/e2e_{workload}_monkey-patch.txt")
     cp traincheck/iteration_times.txt @(f"../../{RES_FOLDER}/e2e_{workload}_monkey-patch.txt")
+    print("Copied")
     rm -rf traincheck
-    # rm iteration_times.txt
 
     # 4. traincheck selective instrumentation
     print("Running traincheck selective instrumentation")
