stroblmeandClaude Opus 5
058f16ec1d
Each was a loose end recorded under `### SDK` in the notepad. `serve` takes its own pidfile down on SIGTERM. uvicorn restores the handler it found and re-raises the signal it stopped on, so the default handler ended the process without unwinding and the `finally` never ran — which is what a stop sends, and what left `serve.pid` behind. `serve.log` is cut back past 5 MB by the engine rather than by the screen that started it, so an adopted engine is bounded too. Gated on its own stdout being an appended regular file, which is what makes the cut safe: the kernel then puts the next write at the new end. Cards are counted from `/dev/nvidia[0-9]*`, so `FLOW_GPUS`/`--gpus` of 0 means "work it out" the way `FLOW_CPUS` always has. The engine counts, not the accountant — a remote worker builds one of those from its own inventory, and detecting there would hand it the engine host's cards. The worker counts last: what a batch job says it was granted still wins. `GET /runs/metrics/names` is the distinct over a selection that `--list` and the terminal's metric picker were approximating by reading the newest run that had measured anything, which missed a name only an older run ever wrote. `MetricSink` announces each batch it has written (`run_metric`, carrying the names). Not a per-point event: one covers up to 500 points or two seconds of them, and the rows stay the record. The terminal comparison fills in as the first readings land instead of staying blank until reopened, and the browser refetches the run and any comparison rather than the list behind them. `retry --group` pages the list route by `before` instead of stopping at 500. The terminal dashboard takes the terminal's colours (`ansi-dark`), and the web UI can re-pair from Settings without disconnecting first. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PRQ9bmTvCbqCwXo9mxZzzV
959 lines
33 KiB
Python
959 lines
33 KiB
Python
"""The command line, and the one ordering it depends on."""
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from fluksio.cli import load_or_create_secret_key
|
|
|
|
|
|
def test_importing_the_cli_does_not_build_the_settings() -> None:
|
|
"""`_configure_environment` has to run before anything reads a setting.
|
|
|
|
The settings are built on the first import of `fluksio.core.config`, and
|
|
every engine module reaches it within an import or two. If importing the
|
|
CLI pulled it in, `DATA_DIR` would be fixed at whatever directory the
|
|
command was run from — which is how the database ends up in the cwd.
|
|
"""
|
|
leaked = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-c",
|
|
"import fluksio.cli, sys; print('fluksio.core.config' in sys.modules)",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
assert leaked.stdout.strip() == "False", leaked.stdout
|
|
|
|
|
|
def test_the_secret_key_is_kept_rather_than_regenerated(tmp_path: Path) -> None:
|
|
"""A new key each start would sign out every session and orphan secrets.enc."""
|
|
path = tmp_path / "secret_key"
|
|
first = load_or_create_secret_key(path)
|
|
assert load_or_create_secret_key(path) == first
|
|
assert path.stat().st_mode & 0o777 == 0o600
|
|
|
|
|
|
def test_the_store_works_without_git(tmp_path: Path, monkeypatch) -> None:
|
|
"""A pip install on a locked-down host may have no git.
|
|
|
|
Flows are files, and that is what the store is for; the history is the part
|
|
that needs git. Losing it must not be a refusal to start.
|
|
"""
|
|
import subprocess as sp
|
|
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
real_run = sp.run
|
|
|
|
def no_git(cmd, *args, **kwargs):
|
|
if cmd and cmd[0] == "git":
|
|
raise FileNotFoundError(2, "No such file or directory", "git")
|
|
return real_run(cmd, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(sp, "run", no_git)
|
|
monkeypatch.setattr(FlowStore, "_warned_no_git", False)
|
|
|
|
store = FlowStore(tmp_path / "flows")
|
|
assert store.head() == ""
|
|
store.write_requirements("numpy\n")
|
|
assert store.read_requirements() == "numpy\n"
|
|
|
|
|
|
def test_run_arguments_are_typed_by_the_flow_they_are_for() -> None:
|
|
"""`--lr 0.05` is a float because the flow says `lr` is one."""
|
|
from fluksio.sdk import SyncError
|
|
from fluksio.sdk.cli import _params
|
|
|
|
definition = {
|
|
"inputs": [
|
|
{"spec": {"name": "lr", "dtype": "float"}},
|
|
{"spec": {"name": "epochs", "dtype": "int"}},
|
|
{"spec": {"name": "resume", "dtype": "bool"}},
|
|
]
|
|
}
|
|
|
|
assert _params(definition, ["--lr", "0.05", "--epochs", "3", "--resume"]) == {
|
|
"lr": 0.05,
|
|
"epochs": 3,
|
|
"resume": True,
|
|
}
|
|
assert _params(definition, ["--lr=1e-4"]) == {"lr": 0.0001}
|
|
|
|
import pytest
|
|
|
|
with pytest.raises(SyncError, match="not an input of this flow"):
|
|
_params(definition, ["--nonesuch", "1"])
|
|
|
|
|
|
def test_a_sweeps_param_spelling_is_refused_by_name() -> None:
|
|
"""`run --param lr=0.002` is a name this flow has not got, and says so."""
|
|
import pytest
|
|
|
|
from fluksio.sdk import SyncError
|
|
from fluksio.sdk.cli import _params
|
|
|
|
definition = {"inputs": [{"spec": {"name": "lr", "dtype": "float"}}]}
|
|
|
|
# Not a JSONDecodeError over `lr=0.002`, which is what reading the value
|
|
# before the name used to give.
|
|
with pytest.raises(SyncError, match="sweep --param"):
|
|
_params(definition, ["--param", "lr=0.002"])
|
|
|
|
with pytest.raises(SyncError, match="'lr' takes float"):
|
|
_params(definition, ["--lr", "fast"])
|
|
|
|
|
|
def test_serve_uses_the_instance_the_directory_belongs_to(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
"""A repository with its own venv gets its own engine, not the machine's."""
|
|
from fluksio import cli
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
monkeypatch.setattr(cli, "DEFAULT_HOME", tmp_path / "home" / ".fluksio")
|
|
|
|
# Nothing above it yet: one is made here rather than in the home directory.
|
|
made = cli._data_dir(None)
|
|
assert made == (tmp_path / ".fluksio").resolve()
|
|
assert (made / ".gitignore").exists()
|
|
|
|
# And is then found again from anywhere below it.
|
|
deep = tmp_path / "src" / "deep"
|
|
deep.mkdir(parents=True)
|
|
monkeypatch.chdir(deep)
|
|
assert cli._data_dir(None) == made
|
|
|
|
# `--global` asks for the shared one even so.
|
|
assert (
|
|
cli._data_dir(None, shared=True) == (tmp_path / "home" / ".fluksio").resolve()
|
|
)
|
|
|
|
# And `--data-dir` still names any directory outright.
|
|
named = cli._data_dir(str(tmp_path / "named"))
|
|
assert named == (tmp_path / "named").resolve()
|
|
|
|
|
|
def test_enrolling_needs_only_a_claim_code() -> None:
|
|
"""The default portal is what makes first-run one flag rather than two."""
|
|
from fluksio.cli import DEFAULT_PORTAL, _parser
|
|
|
|
args = _parser().parse_args(["enroll", "ABC-123"])
|
|
|
|
assert args.portal == DEFAULT_PORTAL
|
|
assert DEFAULT_PORTAL.startswith("https://")
|
|
|
|
# And a portal of your own still wins.
|
|
mine = _parser().parse_args(["enroll", "ABC-123", "--portal", "https://hub.me"])
|
|
assert mine.portal == "https://hub.me"
|
|
|
|
|
|
def test_run_syncs_by_default_and_can_be_told_not_to() -> None:
|
|
from fluksio.cli import _parser
|
|
|
|
assert _parser().parse_args(["run", "train"]).no_sync is False
|
|
assert _parser().parse_args(["run", "train", "--no-sync"]).no_sync is True
|
|
|
|
|
|
def test_a_sweep_is_the_product_of_the_parameters_given() -> None:
|
|
"""`--param lr=0.1,0.01 --param epochs=1,2` is four runs, typed by the flow."""
|
|
import pytest
|
|
|
|
from fluksio.sdk import SyncError
|
|
from fluksio.sdk.cli import _grid
|
|
|
|
definition = {
|
|
"inputs": [
|
|
{"spec": {"name": "lr", "dtype": "float"}},
|
|
{"spec": {"name": "epochs", "dtype": "int"}},
|
|
]
|
|
}
|
|
|
|
grid = _grid(definition, ["lr=0.1,0.01", "epochs=1,2"], seed=7)
|
|
assert [entry["params"] for entry in grid] == [
|
|
{"lr": 0.1, "epochs": 1},
|
|
{"lr": 0.1, "epochs": 2},
|
|
{"lr": 0.01, "epochs": 1},
|
|
{"lr": 0.01, "epochs": 2},
|
|
]
|
|
assert all(entry["seed"] == 7 for entry in grid)
|
|
|
|
with pytest.raises(SyncError, match="not an input of this flow"):
|
|
_grid(definition, ["nonesuch=1"], seed=None)
|
|
with pytest.raises(SyncError, match="name=value"):
|
|
_grid(definition, ["lr"], seed=None)
|
|
|
|
|
|
def test_the_local_engine_is_asked_for_rather_than_guessed() -> None:
|
|
from fluksio.cli import _parser
|
|
|
|
parser = _parser()
|
|
assert parser.parse_args(["run", "train"]).local is False
|
|
assert parser.parse_args(["run", "train", "--local"]).local is True
|
|
assert parser.parse_args(["runs", "--local"]).local is True
|
|
assert parser.parse_args(["sweep", "train", "--param", "lr=1"]).local is False
|
|
|
|
|
|
def test_a_local_run_always_waits(monkeypatch) -> None:
|
|
"""The engine is this process, so a run nobody waits for is thrown away."""
|
|
from contextlib import contextmanager
|
|
|
|
from fluksio.cli import _parser
|
|
from fluksio.sdk import cli
|
|
|
|
submitted: dict[str, object] = {}
|
|
|
|
class FakeHandle:
|
|
id = "run-1"
|
|
status = "ok"
|
|
result: dict[str, object] = {}
|
|
|
|
def wait(self, timeout: float = 0.0) -> "FakeHandle":
|
|
submitted["waited"] = True
|
|
return self
|
|
|
|
class FakeClient:
|
|
def get_flow(self, name: str) -> dict[str, object]:
|
|
return {"definition": {"inputs": []}}
|
|
|
|
def submit(self, flow, params, seed=None, no_cache=False, cause="sdk"):
|
|
submitted["flow"] = flow
|
|
submitted["no_cache"] = no_cache
|
|
submitted["cause"] = cause
|
|
return FakeHandle()
|
|
|
|
def run(self, run_id: str) -> dict[str, object]:
|
|
return {"nodes": [{"status": "cached"}, {"status": "ok"}]}
|
|
|
|
@contextmanager
|
|
def fake_engine():
|
|
yield FakeClient()
|
|
|
|
monkeypatch.setattr(cli, "_engine_client", fake_engine)
|
|
|
|
args = _parser().parse_args(["run", "train", "--local", "--no-sync", "--no-cache"])
|
|
assert cli.cmd_run(args, []) == 0
|
|
# "cli" rather than the SDK's default: the history says which asked.
|
|
assert submitted == {
|
|
"flow": "train",
|
|
"no_cache": True,
|
|
"waited": True,
|
|
"cause": "cli",
|
|
}
|
|
|
|
|
|
def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
|
|
"""Interrupting means stop the run, not walk away leaving it going."""
|
|
from contextlib import contextmanager
|
|
|
|
from fluksio.cli import _parser
|
|
from fluksio.sdk import cli
|
|
|
|
cancelled: list[str] = []
|
|
|
|
class FakeHandle:
|
|
id = "run-1"
|
|
status = "running"
|
|
result: dict[str, object] = {}
|
|
|
|
def wait(self, timeout: float = 0.0):
|
|
raise KeyboardInterrupt
|
|
|
|
class FakeClient:
|
|
def get_flow(self, name: str) -> dict[str, object]:
|
|
return {"definition": {"inputs": []}}
|
|
|
|
def submit(self, flow, params, seed=None, no_cache=False, cause="sdk"):
|
|
return FakeHandle()
|
|
|
|
def cancel(self, run_id: str) -> None:
|
|
cancelled.append(run_id)
|
|
|
|
@contextmanager
|
|
def fake_engine():
|
|
yield FakeClient()
|
|
|
|
monkeypatch.setattr(cli, "_engine_client", fake_engine)
|
|
|
|
args = _parser().parse_args(["run", "train", "--local", "--no-sync"])
|
|
assert cli.cmd_run(args, []) == 130
|
|
assert cancelled == ["run-1"]
|
|
|
|
|
|
def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
|
|
"""Otherwise the traceback is stored and nothing at a terminal shows it."""
|
|
from contextlib import contextmanager
|
|
|
|
from fluksio.cli import _parser
|
|
from fluksio.sdk import cli
|
|
|
|
class FakeHandle:
|
|
id = "run-1"
|
|
status = "error"
|
|
result: dict[str, object] = {}
|
|
failures = [
|
|
{
|
|
"node": "study.train",
|
|
"error": "ValueError: no convergence",
|
|
"logs": 'Traceback (most recent call last):\n File "<node>"\n',
|
|
}
|
|
]
|
|
|
|
def wait(self, timeout: float = 0.0) -> "FakeHandle":
|
|
return self
|
|
|
|
class FakeClient:
|
|
def get_flow(self, name: str) -> dict[str, object]:
|
|
return {"definition": {"inputs": []}}
|
|
|
|
def submit(self, flow, params, seed=None, no_cache=False, cause="sdk"):
|
|
return FakeHandle()
|
|
|
|
def run(self, run_id: str) -> dict[str, object]:
|
|
return {"nodes": []}
|
|
|
|
@contextmanager
|
|
def fake_engine():
|
|
yield FakeClient()
|
|
|
|
monkeypatch.setattr(cli, "_engine_client", fake_engine)
|
|
|
|
args = _parser().parse_args(["run", "train", "--local", "--no-sync"])
|
|
assert cli.cmd_run(args, []) == 1
|
|
|
|
printed = capsys.readouterr().out
|
|
assert "study.train" in printed
|
|
assert "ValueError: no convergence" in printed
|
|
assert "Traceback (most recent call last):" in printed
|
|
|
|
|
|
def test_an_artifact_input_may_be_named_rather_than_pasted() -> None:
|
|
"""The engine resolves either spelling; the CLI just stops mangling them."""
|
|
import json
|
|
|
|
from fluksio.sdk.cli import _coerce
|
|
|
|
assert _coerce("@run:123-abc.dataset", "artifact") == "@run:123-abc.dataset"
|
|
digest = "sha256:" + "a1" * 32
|
|
assert _coerce(digest, "artifact") == digest
|
|
# A reference a script already holds still arrives as the object it is.
|
|
reference = {"digest": digest, "size": 3}
|
|
assert _coerce(json.dumps(reference), "artifact") == reference
|
|
|
|
|
|
def test_the_run_prompt_keeps_declared_values_for_anything_left_blank() -> None:
|
|
"""Enter through the lot is what running with the defaults looks like."""
|
|
from unittest.mock import patch
|
|
|
|
from fluksio.sdk.cli import _ask_params
|
|
|
|
definition = {
|
|
"inputs": [
|
|
{"spec": {"name": "lr", "dtype": "float"}, "initial": 0.05},
|
|
{"spec": {"name": "epochs", "dtype": "int"}, "initial": 10},
|
|
]
|
|
}
|
|
|
|
with patch("builtins.input", side_effect=["", ""]):
|
|
# Nothing sent: an input the caller leaves out keeps what it declares,
|
|
# which is the same contract the browser's dialog has.
|
|
assert _ask_params(definition) == {}
|
|
|
|
with patch("builtins.input", side_effect=["0.01", "50"]):
|
|
assert _ask_params(definition) == {"lr": 0.01, "epochs": 50}
|
|
|
|
|
|
def test_the_run_prompt_names_an_answer_of_the_wrong_type() -> None:
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from fluksio.sdk import SyncError
|
|
from fluksio.sdk.cli import _ask_params
|
|
|
|
definition = {"inputs": [{"spec": {"name": "lr", "dtype": "float"}}]}
|
|
with patch("builtins.input", side_effect=["fast"]):
|
|
with pytest.raises(SyncError, match="lr"):
|
|
_ask_params(definition)
|
|
|
|
|
|
def test_an_export_is_parsed_with_its_selection() -> None:
|
|
"""`export` is a group of two tables, and both take the same filters."""
|
|
from fluksio.cli import _parser
|
|
from fluksio.sdk.cli import cmd_export_metrics, cmd_export_runs
|
|
|
|
parser = _parser()
|
|
metrics = parser.parse_args(
|
|
[
|
|
"export",
|
|
"metrics",
|
|
"--flow",
|
|
"train",
|
|
"--run",
|
|
"a",
|
|
"--run",
|
|
"b",
|
|
"--name",
|
|
"loss,val",
|
|
"--stride",
|
|
"5",
|
|
]
|
|
)
|
|
assert metrics.func is cmd_export_metrics
|
|
assert metrics.run == ["a", "b"]
|
|
assert metrics.stride == 5
|
|
assert metrics.format == "csv"
|
|
|
|
runs = parser.parse_args(["export", "runs", "--params", "lr", "--format", "jsonl"])
|
|
assert runs.func is cmd_export_runs
|
|
assert runs.params == "lr"
|
|
assert runs.format == "jsonl"
|
|
|
|
|
|
def test_the_metric_names_are_asked_for_rather_than_guessed(capsys) -> None:
|
|
"""A name is flow-qualified, so `--list` is what says what would match.
|
|
|
|
The engine answers it over the whole selection, so a name only an older run
|
|
ever recorded is listed — which reading the newest run that measured
|
|
anything could not do.
|
|
"""
|
|
from fluksio.cli import _parser
|
|
from fluksio.sdk.cli import _list_names
|
|
|
|
parser = _parser()
|
|
assert parser.parse_args(["export", "metrics"]).list_names is False
|
|
assert parser.parse_args(["export", "metrics", "--list"]).list_names is True
|
|
|
|
asked = {}
|
|
|
|
class Engine:
|
|
def metric_names(self, ids=(), flow="", **filters):
|
|
asked.update({"ids": list(ids), "flow": flow, **filters})
|
|
return ["train.train_loss", "train.val_loss"]
|
|
|
|
args = parser.parse_args(
|
|
["export", "metrics", "--flow", "train", "--status", "ok", "--list"]
|
|
)
|
|
assert _list_names(Engine(), args) == 0
|
|
assert asked == {"ids": [], "flow": "train", "status": "ok"}
|
|
assert capsys.readouterr().out.split() == ["train.train_loss", "train.val_loss"]
|
|
|
|
|
|
def test_a_runs_artifact_is_listed_and_downloaded(tmp_path, monkeypatch) -> None:
|
|
"""`save_artifact` had no counterpart: the bytes were API-only."""
|
|
from fluksio.cli import _parser
|
|
from fluksio.sdk.cli import _artifacts
|
|
|
|
row = {
|
|
"name": "weights",
|
|
"node": "fit",
|
|
"digest": "sha256:abc",
|
|
"size": 3,
|
|
"media_type": "application/octet-stream",
|
|
"filename": "weights.npz",
|
|
}
|
|
|
|
class Engine:
|
|
def run(self, run_id):
|
|
assert run_id == "r-1"
|
|
return {"id": run_id, "status": "ok", "artifacts": [row]}
|
|
|
|
def download(self, digest):
|
|
assert digest == "sha256:abc"
|
|
return b"abc"
|
|
|
|
parser = _parser()
|
|
monkeypatch.chdir(tmp_path)
|
|
|
|
assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1"])) == 0
|
|
|
|
# Written under the name the node saved it as, not the message's.
|
|
assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1", "weights"])) == 0
|
|
assert (tmp_path / "weights.npz").read_bytes() == b"abc"
|
|
|
|
args = parser.parse_args(["artifacts", "r-1", "weights", "-o", "here.bin"])
|
|
assert _artifacts(Engine(), args) == 0
|
|
assert (tmp_path / "here.bin").read_bytes() == b"abc"
|
|
|
|
|
|
def test_an_engine_without_the_route_is_named_rather_than_404() -> None:
|
|
"""A client ships ahead of the engine; a flat 404 does not say so."""
|
|
from fluksio.sdk.cli import _too_old
|
|
|
|
class Old:
|
|
def summary(self):
|
|
return {"status": "ok", "version": "0.1.4"}
|
|
|
|
class Ancient:
|
|
def summary(self):
|
|
# Older than the field itself.
|
|
return {"status": "ok"}
|
|
|
|
assert "engine is 0.1.4" in _too_old(Old())
|
|
assert "engine is older" in _too_old(Ancient())
|
|
assert "pip install -U fluksio" in _too_old(Ancient())
|
|
|
|
|
|
def test_a_study_in_a_subfolder_is_found(tmp_path) -> None:
|
|
"""One directory per study is a layout; naming each is bookkeeping."""
|
|
from fluksio.sdk.cli import _below
|
|
|
|
(tmp_path / "dev" / "s1").mkdir(parents=True)
|
|
(tmp_path / "dev" / "s2" / "inner").mkdir(parents=True)
|
|
(tmp_path / "results").mkdir()
|
|
(tmp_path / "pkg").mkdir()
|
|
(tmp_path / ".hidden").mkdir()
|
|
(tmp_path / "dev" / "s1" / "study.py").write_text("")
|
|
(tmp_path / "dev" / "s2" / "inner" / "probe.py").write_text("")
|
|
(tmp_path / "results" / "notes.py").write_text("")
|
|
(tmp_path / "pkg" / "__init__.py").write_text("")
|
|
(tmp_path / "pkg" / "deep.py").write_text("")
|
|
(tmp_path / ".hidden" / "skip.py").write_text("")
|
|
|
|
found = {path.relative_to(tmp_path).as_posix() for path in _below(tmp_path)}
|
|
|
|
# However deep, plus packages whole — and nothing under a dot directory.
|
|
assert found == {
|
|
"dev/s1/study.py",
|
|
"dev/s2/inner/probe.py",
|
|
"results/notes.py",
|
|
"pkg",
|
|
}
|
|
|
|
|
|
def test_a_study_per_directory_imports_under_its_own_name(tmp_path) -> None:
|
|
"""One `study.py` per folder is a layout people have, and it works.
|
|
|
|
Named for where each sits under the directory being synced, so nothing
|
|
collides and no `__init__.py` has to be added — which would break the
|
|
bare `from study import ...` a test beside it does.
|
|
"""
|
|
import sys
|
|
|
|
from fluksio.sdk.cli import discover
|
|
|
|
for study in ("s1", "s2"):
|
|
(tmp_path / "dev" / study).mkdir(parents=True)
|
|
(tmp_path / "dev" / study / "study.py").write_text(f"VALUE = {study!r}\n")
|
|
|
|
try:
|
|
discover([str(tmp_path / "dev")])
|
|
assert sys.modules["s1.study"].VALUE == "s1"
|
|
assert sys.modules["s2.study"].VALUE == "s2"
|
|
finally:
|
|
for name in ("s1.study", "s2.study", "s1", "s2"):
|
|
sys.modules.pop(name, None)
|
|
sys.path[:] = [entry for entry in sys.path if entry != str(tmp_path / "dev")]
|
|
|
|
|
|
def test_two_files_of_one_name_are_refused(tmp_path) -> None:
|
|
"""Python keeps one module per name, and a node's body imports by it.
|
|
|
|
Unreachable from one sync of a directory now that a file is named for
|
|
where it sits; this is the spelling that still gets there — two files
|
|
named on the command line, each rooted at its own directory.
|
|
"""
|
|
import pytest
|
|
|
|
from fluksio.sdk import SyncError
|
|
from fluksio.sdk.cli import _import, _module_of
|
|
|
|
for study in ("s1", "s2"):
|
|
(tmp_path / study).mkdir()
|
|
(tmp_path / study / "study.py").write_text("VALUE = 1\n")
|
|
|
|
first = tmp_path / "s1" / "study.py"
|
|
second = tmp_path / "s2" / "study.py"
|
|
_import(*_module_of(first), first)
|
|
with pytest.raises(SyncError, match="both import as 'study'"):
|
|
_import(*_module_of(second), second)
|
|
|
|
|
|
def test_how_long_ago_reads_like_a_duration() -> None:
|
|
"""A failure with no time on it says nothing about whether it is current."""
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from fluksio.sdk.cli import _ago
|
|
|
|
def then(**delta):
|
|
return (datetime.now(UTC) - timedelta(**delta)).isoformat()
|
|
|
|
assert _ago(then(seconds=5)) == "5s ago"
|
|
assert _ago(then(minutes=3)) == "3min ago"
|
|
assert _ago(then(hours=2)) == "2h ago"
|
|
assert _ago(then(days=3)) == "3d ago"
|
|
# What the engine stores is UTC whether or not the spelling says so.
|
|
assert _ago(datetime.now(UTC).replace(tzinfo=None).isoformat()) == "0s ago"
|
|
assert _ago(None) == ""
|
|
|
|
|
|
def test_serve_says_when_a_flow_wants_a_card_nobody_declared(
|
|
tmp_path, monkeypatch, capsys
|
|
) -> None:
|
|
"""The clamp warning goes to the log; this is said while someone is reading.
|
|
|
|
A card behind a driver whose device nodes are not NVIDIA's is not counted,
|
|
so an install that forgets `--gpus` there clamps a GPU node to zero and
|
|
runs them all at once. Detection is stubbed either way: whether this test
|
|
says anything must not depend on the machine running it.
|
|
"""
|
|
from fluksio import cli
|
|
from fluksio.core.config import settings
|
|
from fluksio.flow import resources
|
|
from fluksio.flow.schemas import FlowDef, NodeDef, Resources
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
monkeypatch.setattr(resources, "machine_gpus", lambda: 0)
|
|
|
|
store = FlowStore(tmp_path / "flows")
|
|
store.write_flow(
|
|
FlowDef(
|
|
name="finetune",
|
|
mode="batch",
|
|
nodes=[NodeDef(id="fit", type="python", resources=Resources(gpus=1))],
|
|
)
|
|
)
|
|
monkeypatch.setattr(settings, "FLOWS_DIR", tmp_path / "flows")
|
|
|
|
monkeypatch.setattr(settings, "FLOW_GPUS", 0)
|
|
cli._mention_undeclared_cards()
|
|
said = capsys.readouterr().out
|
|
assert "finetune asks for one" in said
|
|
assert "--gpus" in said
|
|
|
|
# Told how many there are, it has nothing to say.
|
|
monkeypatch.setattr(settings, "FLOW_GPUS", 1)
|
|
cli._mention_undeclared_cards()
|
|
assert capsys.readouterr().out == ""
|
|
|
|
# Nor when it found them itself, which is the ordinary case on a box with
|
|
# NVIDIA's driver: nobody has to say what the device nodes already do.
|
|
monkeypatch.setattr(settings, "FLOW_GPUS", 0)
|
|
monkeypatch.setattr(resources, "machine_gpus", lambda: 1)
|
|
cli._mention_undeclared_cards()
|
|
assert capsys.readouterr().out == ""
|
|
|
|
|
|
def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None:
|
|
"""These are written into the environment before the settings are built."""
|
|
import pytest
|
|
|
|
from fluksio.cli import _parser
|
|
|
|
parser = _parser()
|
|
assert parser.parse_args(["serve", "--max-workers", "2"]).max_workers == 2
|
|
# A machine may genuinely have no card, so zero is a number here.
|
|
assert parser.parse_args(["serve", "--gpus", "0"]).gpus == 0
|
|
assert parser.parse_args(["serve"]).gpus is None
|
|
|
|
for flag, value in (("--max-workers", "0"), ("--gpus", "-1")):
|
|
with pytest.raises(SystemExit):
|
|
parser.parse_args(["serve", flag, value])
|
|
assert "at least" in capsys.readouterr().err
|
|
|
|
|
|
def test_run_and_sweep_take_what_to_sync() -> None:
|
|
from fluksio.cli import _parser
|
|
|
|
parser = _parser()
|
|
assert parser.parse_args(["sweep", "train", "--sync", "dev/s1"]).sync == ["dev/s1"]
|
|
args, _ = parser.parse_known_args(["run", "train", "--sync", "dev/s1"])
|
|
assert args.sync == ["dev/s1"]
|
|
# Nothing named means this directory, downwards.
|
|
assert parser.parse_args(["sweep", "train"]).sync == []
|
|
|
|
|
|
def test_the_dashboard_runs_the_engine_as_a_child_of_itself(monkeypatch) -> None:
|
|
"""At a terminal `serve` is a dashboard; the engine is a plain serve.
|
|
|
|
Every flag is passed through, so what the child runs with is what serve
|
|
was asked for — and `--plain` is what stops it opening a second one.
|
|
"""
|
|
import sys
|
|
|
|
from fluksio import cli
|
|
from fluksio.tui import child_argv
|
|
|
|
argv = child_argv(["serve", "--port", "8123", "--gpus", "1"])
|
|
assert argv[:3] == [sys.executable, "-m", "fluksio.cli"]
|
|
assert argv[3:] == ["serve", "--port", "8123", "--gpus", "1", "--plain"]
|
|
# Already plain: told once, not twice.
|
|
assert child_argv(["serve", "--plain"])[3:] == ["serve", "--plain"]
|
|
|
|
opened: list[str] = []
|
|
monkeypatch.setattr(
|
|
"fluksio.tui.run_tui", lambda args: opened.append("tui") or 0, raising=False
|
|
)
|
|
monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
|
|
monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
|
|
|
|
parser = cli._parser()
|
|
assert cli.cmd_serve(parser.parse_args(["serve"])) == 0
|
|
assert opened == ["tui"]
|
|
|
|
# `--plain` goes past it, which is what the child and every container does.
|
|
# Nothing else of serve runs here, so it fails on the data directory it is
|
|
# given rather than opening a dashboard.
|
|
opened.clear()
|
|
monkeypatch.setattr(
|
|
cli, "_data_dir", lambda *a, **k: (_ for _ in ()).throw(SystemExit(3))
|
|
)
|
|
with __import__("pytest").raises(SystemExit):
|
|
cli.cmd_serve(parser.parse_args(["serve", "--plain"]))
|
|
assert opened == []
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_the_dashboard_is_three_tabs_and_lets_go_of_its_threads(
|
|
tmp_path, monkeypatch
|
|
) -> None:
|
|
"""The tabs, and what `q` has to do before the screen goes.
|
|
|
|
Its two readers are worker threads on the event loop's own executor, which
|
|
is joined while the loop closes — so a reader still blocked on the log file
|
|
or the websocket holds the terminal after the dashboard has gone, which is
|
|
what `q` used to do.
|
|
"""
|
|
from textual.widgets import TabbedContent
|
|
|
|
from fluksio import cli
|
|
from fluksio.tui.app import ServeApp
|
|
|
|
# Nothing is started under this screen: the engine has its own tests.
|
|
monkeypatch.setattr(ServeApp, "start_engine", lambda self: None)
|
|
args = cli._parser().parse_args(["serve", "--data-dir", str(tmp_path / ".fluksio")])
|
|
app = ServeApp(args)
|
|
async with app.run_test() as pilot:
|
|
tabs = app.query_one(TabbedContent)
|
|
assert tabs.active == "overview-tab"
|
|
await pilot.press("2")
|
|
assert tabs.active == "runs-tab"
|
|
# The run keys live on the table, so they are in the footer only here.
|
|
assert "app.pick" in {binding[1] for binding in app.query_one("#runs").BINDINGS}
|
|
await pilot.press("3")
|
|
assert tabs.active == "logs-tab"
|
|
await pilot.press("q")
|
|
|
|
assert app.stop_log.is_set()
|
|
assert app.stop_stream.is_set()
|
|
|
|
|
|
def test_a_serving_engine_records_itself_until_it_stops(tmp_path) -> None:
|
|
"""A pid nobody is running is the same as no pidfile at all."""
|
|
import os
|
|
|
|
from fluksio.cli import read_pidfile, write_pidfile
|
|
|
|
assert read_pidfile(tmp_path) is None
|
|
|
|
written = write_pidfile(tmp_path, 8000)
|
|
assert read_pidfile(tmp_path) == {"pid": os.getpid(), "port": 8000}
|
|
|
|
# Killed outright: the file outlives the process it names.
|
|
written.write_text('{"pid": 2147483646, "port": 8000}')
|
|
assert read_pidfile(tmp_path) is None
|
|
|
|
written.write_text("not json")
|
|
assert read_pidfile(tmp_path) is None
|
|
|
|
|
|
def test_the_pidfile_goes_with_the_process_however_it_ends(tmp_path) -> None:
|
|
"""A SIGTERM used to leave `serve.pid` behind, which is what a stop sends.
|
|
|
|
uvicorn puts back the handler it found and re-raises the signal it stopped
|
|
on; the default handler ends the process without unwinding, so the `finally`
|
|
that removes the file never ran. A stale file costs the dashboard a probe
|
|
and, if a pid is ever recycled, points at a stranger.
|
|
"""
|
|
import signal
|
|
|
|
from fluksio.cli import _hold_pidfile, write_pidfile
|
|
|
|
was = signal.getsignal(signal.SIGTERM)
|
|
try:
|
|
written = write_pidfile(tmp_path, 8000)
|
|
assert written.exists()
|
|
with pytest.raises(SystemExit):
|
|
_hold_pidfile(written, lambda: signal.raise_signal(signal.SIGTERM))
|
|
assert not written.exists()
|
|
|
|
# And the ordinary way out, which always worked.
|
|
written = write_pidfile(tmp_path, 8000)
|
|
_hold_pidfile(written, lambda: None)
|
|
assert not written.exists()
|
|
finally:
|
|
signal.signal(signal.SIGTERM, was)
|
|
|
|
|
|
def test_the_engine_cuts_its_own_log_when_it_grows(tmp_path) -> None:
|
|
"""The dashboard cut it only when *it* started the engine, so an adopted
|
|
one — or one whose screen was closed — wrote without a bound.
|
|
|
|
Only an appended file is cut: the kernel then puts the next write at the
|
|
new end, where a `>` redirect would keep its offset and leave a hole.
|
|
"""
|
|
from fluksio.cli import _appended_log, _trim_log
|
|
|
|
path = tmp_path / "serve.log"
|
|
with path.open("ab", buffering=0) as handle:
|
|
fd = handle.fileno()
|
|
handle.write(b"0123456789")
|
|
assert _appended_log(fd) is True
|
|
assert _trim_log(fd, limit=100) is False
|
|
assert path.stat().st_size == 10
|
|
assert _trim_log(fd, limit=5) is True
|
|
assert path.stat().st_size == 0
|
|
# It keeps writing into the same descriptor afterwards.
|
|
handle.write(b"after")
|
|
assert path.read_bytes() == b"after"
|
|
|
|
with path.open("wb", buffering=0) as handle:
|
|
# Not appended, so not this engine's to cut.
|
|
handle.write(b"0123456789")
|
|
assert _appended_log(handle.fileno()) is False
|
|
|
|
|
|
def test_a_sweep_larger_than_a_page_is_retried_whole() -> None:
|
|
"""`retry --group` read one page of the list route and stopped there.
|
|
|
|
Pages come newest first and `before` is the cursor, so the next page is
|
|
taken from the last row's own timestamp rather than from an offset that
|
|
shifts under a run submitted meanwhile.
|
|
"""
|
|
from fluksio.sdk.cli import PAGE, _group_runs
|
|
|
|
asked = []
|
|
|
|
class Engine:
|
|
def runs(self, flow="", limit=0, **filters):
|
|
asked.append(filters)
|
|
page = 0 if "before" not in filters else 1
|
|
if page:
|
|
return [{"id": "last", "created_at": "2026-09-02T00:00:00Z"}]
|
|
return [
|
|
{"id": f"r{index}", "created_at": f"2026-09-02T00:00:{index:02d}Z"}
|
|
for index in range(PAGE)
|
|
]
|
|
|
|
rows = list(_group_runs(Engine(), "sweep-1"))
|
|
|
|
assert len(rows) == PAGE + 1
|
|
assert asked[0] == {"group": "sweep-1"}
|
|
# The second page starts where the first ended.
|
|
assert asked[1]["before"] == rows[PAGE - 1]["created_at"]
|
|
|
|
|
|
def test_who_holds_the_port_is_told_apart_by_the_token() -> None:
|
|
"""Only this directory's own engine may be reported as already up.
|
|
|
|
The token is signed with this directory's secret key, so an engine that
|
|
accepts it is one reading this directory's database. Another
|
|
instance's Fluksio answers the health check and refuses it.
|
|
"""
|
|
import httpx
|
|
|
|
from fluksio.cli import probe_engine
|
|
|
|
def engine(health: int, summary: int):
|
|
def handle(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("/health-check/"):
|
|
return httpx.Response(health)
|
|
return httpx.Response(summary)
|
|
|
|
return httpx.Client(transport=httpx.MockTransport(handle))
|
|
|
|
with engine(200, 200) as client:
|
|
assert probe_engine("http://x", "t", client) == "ours"
|
|
with engine(200, 401) as client:
|
|
assert probe_engine("http://x", "t", client) == "foreign"
|
|
# A directory with no credential yet cannot prove anything is its own, and
|
|
# `Bearer ` is not a legal header value — so it asks without one.
|
|
with engine(200, 401) as client:
|
|
assert probe_engine("http://x", "", client) == "foreign"
|
|
# Somebody else's dev server, or nothing listening at all.
|
|
with engine(404, 404) as client:
|
|
assert probe_engine("http://x", "t", client) == "other"
|
|
assert probe_engine("http://127.0.0.1:1", "t") == "other"
|
|
|
|
|
|
def test_a_second_engine_for_one_directory_is_refused(tmp_path, monkeypatch, capsys):
|
|
"""One SQLite file, one engine — whatever port the second was asked for.
|
|
|
|
The refusal is worth more than the duplicate it prevents: the second
|
|
engine signs in on the way up, so `client.json` would point at a port
|
|
that dies with it and every later command would reach nothing.
|
|
"""
|
|
import json
|
|
import os
|
|
|
|
from fluksio import cli
|
|
|
|
data_dir = tmp_path / ".fluksio"
|
|
data_dir.mkdir()
|
|
(data_dir / "client.json").write_text(
|
|
json.dumps({"url": "http://127.0.0.1:8000", "token": "the-first-one"})
|
|
)
|
|
cli.write_pidfile(data_dir, 8000)
|
|
monkeypatch.setattr(cli, "probe_engine", lambda *a, **k: "ours")
|
|
|
|
args = cli._parser().parse_args(
|
|
["serve", "--plain", "--port", "8123", "--data-dir", str(data_dir)]
|
|
)
|
|
assert cli.cmd_serve(args) == 0
|
|
said = capsys.readouterr().out
|
|
assert "already serving" in said and str(os.getpid()) in said
|
|
# The credential still names the engine that is actually up.
|
|
stored = json.loads((data_dir / "client.json").read_text())
|
|
assert stored["url"] == "http://127.0.0.1:8000"
|
|
|
|
# A pid that is alive but is not an engine of ours is not a refusal.
|
|
monkeypatch.setattr(cli, "probe_engine", lambda *a, **k: "foreign")
|
|
assert cli.already_serving(data_dir, "127.0.0.1") == ""
|
|
|
|
|
|
def test_serve_moves_off_a_port_that_is_taken() -> None:
|
|
"""A first start should not die on somebody else's dev server."""
|
|
import socket
|
|
|
|
from fluksio.cli import DEFAULT_PORT, _free_port, _parser
|
|
|
|
with socket.socket() as held:
|
|
held.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
held.bind(("127.0.0.1", 0))
|
|
held.listen(1)
|
|
taken = held.getsockname()[1]
|
|
|
|
assert _free_port("127.0.0.1", taken) == taken + 1
|
|
|
|
# A port that was asked for is not moved off: that is what asking means.
|
|
assert _parser().parse_args(["serve"]).port is None
|
|
assert _parser().parse_args(["serve", "--port", "9000"]).port == 9000
|
|
assert DEFAULT_PORT == 8000
|
|
|
|
|
|
def test_a_duration_reads_the_way_an_age_does() -> None:
|
|
"""One notation for both, rather than one per place that printed one."""
|
|
from fluksio.sdk.cli import _dur
|
|
|
|
assert _dur(None) == "0.0s"
|
|
assert _dur(1240) == "1.2s"
|
|
assert _dur(90_000) == "1.5min"
|
|
# A run measured in hours used to print four digits of seconds.
|
|
assert _dur(7_200_000) == "2.0h"
|
|
assert _dur(172_800_000) == "2.0d"
|
|
|
|
|
|
def test_the_dashboard_subscribes_where_the_browser_does() -> None:
|
|
"""The same socket, and the token where a handshake can carry it."""
|
|
from fluksio.sdk.stream import socket_url
|
|
|
|
assert socket_url("http://127.0.0.1:8000", "abc") == (
|
|
"ws://127.0.0.1:8000/api/v1/flows/ws?token=abc"
|
|
)
|
|
# TLS on the one side is TLS on the other.
|
|
assert socket_url("https://engine.example.com", "t").startswith("wss://")
|