Run build dispatches under a parallelism cap with dry-run

claude · luhenry · commit 4b95a0f1c75e · 2026-04-05T17:02:20.000+02:00
Replaces the serial 'max_dispatches' input with 'max_parallelism' (default
10) on check-releases.yml. The Python script now:

- Uses a ThreadPoolExecutor sized to MAX_PARALLELISM to keep that many
  build-python.yml runs in flight at once.
- After each dispatch, locates the newly-created workflow_run and prints
  its html_url, then polls until it completes before releasing the pool
  slot.
- Supports a DRY_RUN env flag (surfaced as a 'dry_run' boolean input on
  the workflow) that lists queued candidates without triggering any
  builds.
diff --git a/.github/scripts/check-releases.py b/.github/scripts/check-releases.py
@@ -1,19 +1,27 @@
 #!/usr/bin/env python3
 """Scan python/cpython tags and dispatch build-python.yml for any not yet released.
 
-Reads ``GH_TOKEN``, ``GITHUB_REPOSITORY`` and ``MAX_DISPATCHES`` from the
-environment. For each CPython ``v3.Y.Z[a|b|rc]N`` tag from 3.10 onward, checks
-whether a matching release (tag ``<normalised>-<run_id>``) already exists here,
-and if not, dispatches the ``build-python.yml`` workflow. Free-threaded is
-enabled automatically for minor >= 13.
+Reads ``GH_TOKEN``, ``GITHUB_REPOSITORY``, ``MAX_PARALLELISM`` and ``DRY_RUN``
+from the environment. For each CPython ``v3.Y.Z[a|b|rc]N`` tag from 3.10
+onward, checks whether a matching release (tag ``<normalised>-<run_id>``)
+already exists here, and if not, dispatches the ``build-python.yml`` workflow.
+Free-threaded is enabled automatically for minor >= 13.
+
+``MAX_PARALLELISM`` is the number of build-python.yml runs to have in flight
+concurrently. Each worker dispatches a run, waits for it to finish, then moves
+on to the next candidate.
 """
 
 from __future__ import annotations
 
+import concurrent.futures as cf
+import datetime
 import json
 import os
 import re
 import sys
+import threading
+import time
 import urllib.error
 import urllib.request
 
@@ -28,31 +36,58 @@
     (re.compile(r"b([0-9]+)$"), r"-beta.\1"),
     (re.compile(r"rc([0-9]+)$"), r"-rc.\1"),
 )
+BUILD_WORKFLOW = "build-python.yml"
+
+# Poll cadence (seconds)
+RUN_DISCOVERY_POLL = 5
+RUN_DISCOVERY_ATTEMPTS = 60  # up to 5 minutes
+RUN_STATUS_POLL = 30
+
+_claim_lock = threading.Lock()
+_claimed_run_ids: set[int] = set()
+_print_lock = threading.Lock()
+
 
+def log(msg: str) -> None:
+    with _print_lock:
+        print(msg, flush=True)
 
-def gh_get(path: str, token: str) -> list[dict]:
+
+def _request(url: str, token: str, method: str = "GET", body: bytes | None = None) -> urllib.request.Request:
+    return urllib.request.Request(
+        url,
+        data=body,
+        method=method,
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+            "Content-Type": "application/json",
+            "User-Agent": "check-releases-script",
+        },
+    )
+
+
+def gh_get_paginated(path: str, token: str) -> list[dict]:
     """Paginate a GitHub REST API list endpoint."""
     results: list[dict] = []
     url: str | None = f"{API_ROOT}{path}"
     sep = "&" if "?" in url else "?"
     url = f"{url}{sep}per_page=100"
     while url:
-        req = urllib.request.Request(
-            url,
-            headers={
-                "Authorization": f"Bearer {token}",
-                "Accept": "application/vnd.github+json",
-                "X-GitHub-Api-Version": "2022-11-28",
-                "User-Agent": "check-releases-script",
-            },
-        )
-        with urllib.request.urlopen(req) as resp:
+        with urllib.request.urlopen(_request(url, token)) as resp:
             results.extend(json.load(resp))
             link = resp.headers.get("Link", "")
         url = _next_link(link)
     return results
 
 
+def gh_get_json(path: str, token: str) -> dict:
+    """Single GET returning parsed JSON."""
+    with urllib.request.urlopen(_request(f"{API_ROOT}{path}", token)) as resp:
+        return json.load(resp)
+
+
 def _next_link(link_header: str) -> str | None:
     for part in link_header.split(","):
         section = part.strip()
@@ -95,36 +130,81 @@ def dispatch_workflow(repo: str, token: str, cpython_tag: str, ft: bool) -> None
             },
         }
     ).encode("utf-8")
-    req = urllib.request.Request(
-        f"{API_ROOT}/repos/{repo}/actions/workflows/build-python.yml/dispatches",
-        data=body,
-        method="POST",
-        headers={
-            "Authorization": f"Bearer {token}",
-            "Accept": "application/vnd.github+json",
-            "X-GitHub-Api-Version": "2022-11-28",
-            "Content-Type": "application/json",
-            "User-Agent": "check-releases-script",
-        },
-    )
-    with urllib.request.urlopen(req) as resp:
+    url = f"{API_ROOT}/repos/{repo}/actions/workflows/{BUILD_WORKFLOW}/dispatches"
+    with urllib.request.urlopen(_request(url, token, method="POST", body=body)) as resp:
         resp.read()
 
 
+def find_dispatched_run(repo: str, token: str, baseline: set[int], after_ts: str) -> dict | None:
+    """Find a workflow run dispatched after ``after_ts`` that isn't baselined or claimed.
+
+    Called once per dispatch. Claims the oldest eligible run so that concurrent
+    workers each grab a distinct run_id.
+    """
+    path = f"/repos/{repo}/actions/workflows/{BUILD_WORKFLOW}/runs?per_page=50&event=workflow_dispatch"
+    for _ in range(RUN_DISCOVERY_ATTEMPTS):
+        time.sleep(RUN_DISCOVERY_POLL)
+        data = gh_get_json(path, token)
+        runs = sorted(data.get("workflow_runs", []), key=lambda r: r["created_at"])
+        with _claim_lock:
+            for r in runs:
+                rid = r["id"]
+                if rid in baseline:
+                    continue
+                if rid in _claimed_run_ids:
+                    continue
+                if r["created_at"] < after_ts:
+                    continue
+                _claimed_run_ids.add(rid)
+                return r
+    return None
+
+
+def wait_run_complete(repo: str, token: str, run_id: int) -> str | None:
+    path = f"/repos/{repo}/actions/runs/{run_id}"
+    while True:
+        time.sleep(RUN_STATUS_POLL)
+        data = gh_get_json(path, token)
+        if data.get("status") == "completed":
+            return data.get("conclusion")
+
+
+def run_build(repo: str, token: str, tag: str, ft: bool, baseline: set[int]) -> tuple[str, str | None]:
+    log(f"[{tag}] dispatching build-python.yml (freethreaded={str(ft).lower()})")
+    after_ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    dispatch_workflow(repo, token, tag, ft)
+    run = find_dispatched_run(repo, token, baseline, after_ts)
+    if run is None:
+        log(f"[{tag}] WARN: could not locate dispatched run within timeout")
+        return tag, None
+    log(f"[{tag}] run {run['id']}: {run['html_url']}")
+    conclusion = wait_run_complete(repo, token, run["id"])
+    log(f"[{tag}] run {run['id']} completed: {conclusion}")
+    return tag, conclusion
+
+
+def parse_bool(value: str | None) -> bool:
+    return (value or "").strip().lower() in {"1", "true", "yes", "y"}
+
+
 def main() -> int:
     token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")
     repo = os.environ.get("GITHUB_REPOSITORY")
     if not token or not repo:
         print("GH_TOKEN and GITHUB_REPOSITORY must be set", file=sys.stderr)
         return 1
     try:
-        max_dispatches = int(os.environ.get("MAX_DISPATCHES", "20"))
+        max_parallelism = int(os.environ.get("MAX_PARALLELISM", "4"))
     except ValueError:
-        print("MAX_DISPATCHES must be an integer", file=sys.stderr)
+        print("MAX_PARALLELISM must be an integer", file=sys.stderr)
         return 1
+    if max_parallelism < 1:
+        print("MAX_PARALLELISM must be >= 1", file=sys.stderr)
+        return 1
+    dry_run = parse_bool(os.environ.get("DRY_RUN"))
 
     # Fetch all v* tags from python/cpython
-    refs = gh_get("/repos/python/cpython/git/matching-refs/tags/v", token)
+    refs = gh_get_paginated("/repos/python/cpython/git/matching-refs/tags/v", token)
     candidates: list[str] = []
     for entry in refs:
         ref = entry.get("ref", "")
@@ -133,42 +213,73 @@ def main() -> int:
         tag = ref[len(REF_PREFIX):]
         if CPYTHON_TAG_RE.match(tag):
             candidates.append(tag)
-    # Sort newest first
     candidates.sort(key=version_sort_key, reverse=True)
 
-    # Fetch all existing release tags on this repo
-    releases = gh_get(f"/repos/{repo}/releases", token)
+    # Existing release tags
+    releases = gh_get_paginated(f"/repos/{repo}/releases", token)
     existing_tags = {r.get("tag_name") or "" for r in releases}
 
-    dispatched = 0
-    dispatched_list: list[str] = []
-
+    # Decide what to dispatch
+    to_dispatch: list[tuple[str, bool]] = []
     for tag in candidates:
         match = CPYTHON_TAG_RE.match(tag)
         assert match is not None
         minor = int(match.group("full_minor").split(".")[1])
         normalised = normalise(tag)
-        print(f"Tag {tag} -> release {normalised}")
+        log(f"Tag {tag} -> release {normalised}")
 
         version_re = re.compile(rf"^{re.escape(normalised)}-[0-9]+$")
         if any(version_re.match(t) for t in existing_tags):
-            print("  already released, skipping")
-            continue
-
-        if dispatched >= max_dispatches:
-            print("  would dispatch (cap reached)")
+            log("  already released, skipping")
             continue
 
         ft = minor >= 13
-        print(f"  dispatching build-python.yml (freethreaded={str(ft).lower()})")
-        dispatch_workflow(repo, token, tag, ft)
-        dispatched += 1
-        dispatched_list.append(f"{tag} -> {normalised} (ft={str(ft).lower()})")
-
-    print("")
-    print(f"=== Dispatched {dispatched} build(s) ===")
-    for line in dispatched_list:
-        print(line)
+        log(f"  queued for dispatch (freethreaded={str(ft).lower()})")
+        to_dispatch.append((tag, ft))
+
+    log("")
+    log(f"=== {len(to_dispatch)} candidate(s) to build ===")
+    for tag, ft in to_dispatch:
+        log(f"  {tag} (ft={str(ft).lower()})")
+
+    if dry_run:
+        log("")
+        log("DRY_RUN=true: not dispatching anything")
+        return 0
+
+    if not to_dispatch:
+        return 0
+
+    # Baseline of existing build-python.yml runs so we can identify newly-created ones
+    baseline_data = gh_get_json(
+        f"/repos/{repo}/actions/workflows/{BUILD_WORKFLOW}/runs?per_page=100", token
+    )
+    baseline = {r["id"] for r in baseline_data.get("workflow_runs", [])}
+
+    log("")
+    log(f"=== Running up to {max_parallelism} build(s) in parallel ===")
+    failures: list[str] = []
+    with cf.ThreadPoolExecutor(max_workers=max_parallelism) as pool:
+        futures = [
+            pool.submit(run_build, repo, token, tag, ft, baseline)
+            for tag, ft in to_dispatch
+        ]
+        for fut in cf.as_completed(futures):
+            try:
+                tag, conclusion = fut.result()
+            except Exception as exc:  # noqa: BLE001
+                log(f"task raised: {exc}")
+                failures.append(str(exc))
+                continue
+            if conclusion != "success":
+                failures.append(f"{tag}: {conclusion}")
+
+    log("")
+    log(f"=== Finished: {len(to_dispatch) - len(failures)}/{len(to_dispatch)} succeeded ===")
+    if failures:
+        for f in failures:
+            log(f"  FAIL {f}")
+        return 1
     return 0
 
 
diff --git a/.github/workflows/check-releases.yml b/.github/workflows/check-releases.yml
@@ -5,11 +5,16 @@ on:
     - cron: '0 6 * * *'
   workflow_dispatch:
     inputs:
-      max_dispatches:
-        description: "Maximum number of build dispatches (0 = dry run)"
+      max_parallelism:
+        description: "Maximum number of build-python.yml runs in flight at once"
         required: false
-        default: "20"
+        default: "10"
         type: string
+      dry_run:
+        description: "List candidates without dispatching any builds"
+        required: false
+        default: false
+        type: boolean
 
 permissions:
   contents: read
@@ -25,7 +30,8 @@ jobs:
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_REPOSITORY: ${{ github.repository }}
-      MAX_DISPATCHES: ${{ inputs.max_dispatches || '20' }}
+      MAX_PARALLELISM: ${{ inputs.max_parallelism || '10' }}
+      DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
