CI: add pytest durations and skip patterngen seeds under sanitizer (NVIDIA#1400)

Andy-Jost · web-flow · commit 083315c89bc2 · 2025-12-18T00:50:06.000Z
- Add --durations=0 to CI pytest invocations to report per-test timings.
- Add cuda_python_test_helpers.under_compute_sanitizer() and use it to skip
  the UVM-heavy PatternGen seed test under compute-sanitizer.
diff --git a/ci/tools/run-tests b/ci/tools/run-tests
@@ -32,7 +32,7 @@ if [[ "${test_module}" == "pathfinder" ]]; then
   echo "Running pathfinder tests with " \
       "LD:${CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS} " \
       "FH:${CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS}"
-  pytest -ra -s -v tests/ |& tee /tmp/pathfinder_test_log.txt
+  pytest -ra -s -v --durations=0 tests/ |& tee /tmp/pathfinder_test_log.txt
   # Fail if no "INFO test_" lines are found; capture line count otherwise
   line_count=$(grep '^INFO test_' /tmp/pathfinder_test_log.txt | wc -l)
   echo "Number of \"INFO test_\" lines: $line_count"
@@ -46,9 +46,9 @@ elif [[ "${test_module}" == "bindings" ]]; then
     pip install $(ls "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)[all] --group test
   fi
   echo "Running bindings tests"
-  ${SANITIZER_CMD} pytest -rxXs -v tests/
+  ${SANITIZER_CMD} pytest -rxXs -v --durations=0 tests/
   if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
-    ${SANITIZER_CMD} pytest -rxXs -v tests/cython
+    ${SANITIZER_CMD} pytest -rxXs -v --durations=0 tests/cython
   fi
   popd
 elif [[ "${test_module}" == "core" ]]; then
@@ -79,11 +79,11 @@ elif [[ "${test_module}" == "core" ]]; then
     pip install $(ls "${CUDA_CORE_ARTIFACTS_DIR}"/*.whl)["cu${TEST_CUDA_MAJOR}"] --group "test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}"
   fi
   echo "Running core tests"
-  ${SANITIZER_CMD} pytest -rxXs -v tests/
+  ${SANITIZER_CMD} pytest -rxXs -v --durations=0 tests/
   # Currently our CI always installs the latest bindings (from either major version).
   # This is not compatible with the test requirements.
   if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
-    ${SANITIZER_CMD} pytest -rxXs -v tests/cython
+    ${SANITIZER_CMD} pytest -rxXs -v --durations=0 tests/cython
   fi
   popd
 fi
diff --git a/cuda_core/tests/test_helpers.py b/cuda_core/tests/test_helpers.py
@@ -11,6 +11,8 @@
 from helpers.latch import LatchKernel
 from helpers.logging import TimestampedLogger
 
+from cuda_python_test_helpers import under_compute_sanitizer
+
 ENABLE_LOGGING = False  # Set True for test debugging and development
 NBYTES = 64
 
@@ -45,6 +47,10 @@ def test_latchkernel():
     log("done")
 
 
+@pytest.mark.skipif(
+    under_compute_sanitizer(),
+    reason="Too slow under compute-sanitizer (UVM-heavy test).",
+)
 def test_patterngen_seeds():
     """Test PatternGen with seed argument."""
     device = Device()
diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/__init__.py b/cuda_python_test_helpers/cuda_python_test_helpers/__init__.py
@@ -16,6 +16,7 @@
     "IS_WSL",
     "libc",
     "supports_ipc_mempool",
+    "under_compute_sanitizer",
 ]
 
 
@@ -37,6 +38,38 @@ def _detect_wsl() -> bool:
     libc = ctypes.CDLL("libc.so.6")
 
 
+def under_compute_sanitizer() -> bool:
+    """Return True if the current process is likely running under compute-sanitizer.
+
+    This is best-effort and primarily intended for CI, where the environment
+    is configured by wrapper scripts.
+    """
+    # Explicit override (if we ever want to set this directly in CI).
+    if os.environ.get("CUDA_PYTHON_UNDER_SANITIZER") == "1":
+        return True
+
+    # CI sets these when compute-sanitizer is enabled.
+    if os.environ.get("SETUP_SANITIZER") == "1":
+        return True
+
+    cmd = os.environ.get("SANITIZER_CMD", "")
+    if "compute-sanitizer" in cmd or "cuda-memcheck" in cmd:
+        return True
+
+    # Secondary signals: depending on how tests are invoked, the wrapper name may
+    # appear in argv (e.g. `compute-sanitizer pytest ...`). This is not reliable
+    # in general (often argv0 is `python`/`pytest`), but it's cheap and harmless.
+    argv0 = os.path.basename(sys.argv[0]) if sys.argv else ""
+    if argv0 in ("compute-sanitizer", "cuda-memcheck"):
+        return True
+    if any(("compute-sanitizer" in a or "cuda-memcheck" in a) for a in sys.argv):
+        return True
+
+    # Another common indicator: sanitizer injectors are configured via env vars.
+    inj = os.environ.get("CUDA_INJECTION64_PATH", "")
+    return "compute-sanitizer" in inj or "cuda-memcheck" in inj
+
+
 @functools.cache
 def supports_ipc_mempool(device_id: Union[int, object]) -> bool:
     """Return True if mempool IPC via POSIX file descriptor is supported.
