Ask a cluster for a machine when nothing here will do

Slurm is not a machine that attaches and stays; it is a queue somebody else
owns. So nothing here submits a node to it. It submits a job whose payload is an
ordinary worker dialling back in, and everything downstream — the protocol, the
artifacts, cancellation, the books — already worked and did not have to learn
what Slurm is.

The alternative, which Covalent takes, is to stage a serialized call and a
runner onto the login node, poll squeue and copy the result back: a second way
of running a node beside the one that exists. The cost of not doing that is one
assumption, that a compute node can open a connection outward. Where that is
false, _payload is the single method a staged variant would replace.

Clusters are configured in provisioners.json beside the alerts, since this is
infrastructure an operator writes rather than anything a flow says. The script
is generated with the system ssh and no new dependency, and prerun owns the
environment — deliberately no pip install, because what is on a cluster is
somebody's decision.

One outstanding request per profile, cancelled if it never attaches and on the
way out. Nothing autoscales.

The run gate needed the same hook: a run held before it starts never reaches the
placer's own wait, so it would have queued forever on a machine nothing had
asked for.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01A6HeySA27EkGANZN95QySW

backend/tests/flow/test_provision.py

@@ -0,0 +1,219 @@
+"""Asking a batch scheduler for a machine, and what comes back.
+
+The point of doing it this way is that almost nothing here is new: what a
+cluster gives back is an ordinary worker on the ordinary socket. So what is
+worth checking is the asking — that it happens once, that the script says what
+it should, and that a job nobody ever attached does not sit in the queue.
+"""
+
+import subprocess
+import threading
+
+from fluksio.flow.placement import Placer
+from fluksio.flow.provision import SlurmProfile, SlurmProvisioner, load_provisioners
+from fluksio.flow.resources import ResourceAccountant
+from fluksio.flow.schemas import Resources
+
+
+class FakeSsh:
+    """Stands in for the login node: records argv, answers like sbatch."""
+
+    def __init__(self, returncode: int = 0, stdout: str = "4711\n") -> None:
+        self.calls: list[tuple[list[str], str]] = []
+        self.returncode = returncode
+        self.stdout = stdout
+        self.ran = threading.Event()
+
+    def __call__(self, argv, **kwargs):
+        self.calls.append((argv, kwargs.get("input") or ""))
+        self.ran.set()
+        return subprocess.CompletedProcess(
+            argv, self.returncode, stdout=self.stdout, stderr="sbatch: bad partition"
+        )
+
+
+def cluster(events=None, **kwargs) -> SlurmProvisioner:
+    return SlurmProvisioner(
+        name="hpc",
+        login="me@login.cluster",
+        engine_url="wss://engine.example.com/api/v1/workers/attach",
+        profiles=[
+            SlurmProfile(name="cpu", cpus=16, ram_mb=32768),
+            SlurmProfile(
+                name="gpu-small",
+                cpus=8,
+                gpus=1,
+                ram_mb=65536,
+                labels=["gpu"],
+                sbatch=["--partition=gpu", "--gres=gpu:1", "--time=04:00:00"],
+                prerun=["module load cuda/12", "source ~/venvs/flux/bin/activate"],
+            ),
+        ],
+        events=events,
+        **kwargs,
+    )
+
+
+def test_the_smallest_machine_that_would_do_is_the_one_asked_for(monkeypatch):
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster()
+
+    hpc.provision(cpus=2, gpus=0, ram_mb=0)
+    assert ssh.ran.wait(5)
+    argv, script = ssh.calls[0]
+    assert argv[:3] == ["ssh", "-o", "BatchMode=yes"]
+    assert argv[-2:] == ["sbatch", "--parsable"]
+    assert "--job-name=fluksio-cpu" in script
+
+
+def test_the_script_starts_a_worker_that_dials_back_and_stops_when_done(monkeypatch):
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster(max_idle_s=120)
+
+    hpc.provision(cpus=4, gpus=1, ram_mb=0, label="gpu")
+    assert ssh.ran.wait(5)
+    _, script = ssh.calls[0]
+
+    assert "#SBATCH --gres=gpu:1" in script
+    # The environment is the cluster's business, not this module's.
+    assert "module load cuda/12" in script
+    assert "pip install" not in script
+    assert "exec fluksio-worker" in script
+    assert "--url wss://engine.example.com/api/v1/workers/attach" in script
+    assert "--gpus 1" in script
+    assert "--labels gpu" in script
+    # Without this the job holds its allocation until the walltime.
+    assert "--max-idle 120" in script
+
+
+def test_one_job_per_profile_however_often_it_is_asked_for(monkeypatch):
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster()
+
+    for _ in range(5):
+        hpc.provision(cpus=2, gpus=0, ram_mb=0)
+    assert ssh.ran.wait(5)
+    assert len(ssh.calls) == 1
+    assert hpc.status()["outstanding"][0]["job"] == "4711"
+
+
+def test_a_machine_that_arrives_clears_the_way_for_the_next_ask(monkeypatch):
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster()
+
+    hpc.provision(cpus=2, gpus=0, ram_mb=0)
+    assert ssh.ran.wait(5)
+    arrived = hpc.status()["outstanding"][0]["worker"]
+
+    hpc.reconcile({arrived})
+    assert hpc.status()["outstanding"] == []
+    # No scancel: it did what it was asked to do.
+    assert [argv for argv, _ in ssh.calls if "scancel" in argv] == []
+
+
+def test_a_job_that_never_attaches_is_cancelled(monkeypatch):
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster(provision_timeout_s=0)
+
+    hpc.provision(cpus=2, gpus=0, ram_mb=0)
+    assert ssh.ran.wait(5)
+    hpc.reconcile(set())
+
+    assert hpc.status()["outstanding"] == []
+    assert ["scancel", "4711"] == ssh.calls[-1][0][-2:]
+
+
+def test_a_refused_submission_says_what_the_cluster_said(monkeypatch):
+    ssh = FakeSsh(returncode=1)
+    monkeypatch.setattr(subprocess, "run", ssh)
+    events: list[dict] = []
+    hpc = cluster(
+        events=type("Bus", (), {"publish": lambda self, e: events.append(e)})()
+    )
+
+    hpc.provision(cpus=2, gpus=0, ram_mb=0)
+    assert ssh.ran.wait(5)
+    for _ in range(100):
+        if events:
+            break
+        threading.Event().wait(0.02)
+
+    assert "bad partition" in hpc.last_error
+    assert events[-1]["type"] == "worker_provision_failed"
+    # And it is not left outstanding, so asking again actually asks.
+    assert hpc.status()["outstanding"] == []
+
+
+def test_a_node_waiting_for_a_machine_asks_for_one_once(monkeypatch):
+    """The placer asks on every pass of its wait; the cluster is asked once."""
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster()
+    placer = Placer(local=ResourceAccountant(cpus=4, gpus=0))
+    placer.provisioners = [hpc]
+    ran = threading.Event()
+
+    def wants_a_card() -> None:
+        with placer.claim(Resources(cpus=2, gpus=1), node="study.fit"):
+            ran.set()
+
+    thread = threading.Thread(target=wants_a_card, daemon=True)
+    thread.start()
+    # It waits rather than being cut down to what is here, because something
+    # can start a machine that would take it.
+    assert not ran.wait(0.5)
+    assert len(ssh.calls) == 1
+
+    placer.provisioners = []
+    placer.wake()
+    thread.join(timeout=5)
+    assert ran.is_set()
+
+
+def test_an_installation_with_nowhere_to_start_one_has_no_file(tmp_path):
+    assert load_provisioners(tmp_path / "provisioners.json") == []
+
+
+def test_a_configured_cluster_is_read(tmp_path):
+    path = tmp_path / "provisioners.json"
+    path.write_text(
+        """[{"type": "slurm", "name": "hpc", "login": "me@login",
+             "engine_url": "wss://e/api/v1/workers/attach",
+             "profiles": [{"name": "gpu-small", "cpus": 8, "gpus": 1,
+                           "labels": ["gpu"], "sbatch": ["--gres=gpu:1"]}]}]"""
+    )
+    found = load_provisioners(path)
+
+    assert [p.name for p in found] == ["hpc"]
+    assert found[0].covers(cpus=4, gpus=1, ram_mb=0, label="gpu")
+    assert not found[0].covers(cpus=4, gpus=2, ram_mb=0)
+
+
+def test_a_file_that_cannot_be_read_is_not_a_failure_to_start(tmp_path):
+    path = tmp_path / "provisioners.json"
+    path.write_text("{ not json")
+    assert load_provisioners(path) == []
+
+
+def test_a_run_held_before_it_starts_still_gets_a_machine_asked_for(monkeypatch):
+    """The gate holds the run above the placer, so the asking has to happen there.
+
+    Without this the run waits on a machine nothing ever requested, which is a
+    queue entry that never moves and no error anywhere.
+    """
+    ssh = FakeSsh()
+    monkeypatch.setattr(subprocess, "run", ssh)
+    hpc = cluster()
+    placer = Placer(local=ResourceAccountant(cpus=4, gpus=0))
+    placer.provisioners = [hpc]
+    needs = {"cpus": 2, "gpus": 1, "ram_mb": 0, "device": ""}
+
+    assert placer.satisfiable(needs) == "1 gpu(s) and 2 cpu(s)"
+    placer.provision_for(needs)
+    assert ssh.ran.wait(5)
+    assert "--gpus 1" in ssh.calls[0][1]