Sync only the declaring module before a run; skip git-ignored paths

`fluksio run` walked the working directory and imported every module in
it, which ran a research repository's analysis scripts before the run was
submitted. A flow's origin now records the file and module name that
declared it, and run/sweep import just that, under the same name so the
generated node bodies do not change. A flow with no record still walks,
now skipping what .gitignore covers.

Every code-defined flow reports `flow` changed once on its next sync,
since the origin gained two fields.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QgG6TJw5qBZzaLghQSRVrq
This commit is contained in:
2026-09-16 16:55:02 +02:00
co-authored by Claude Opus 5
parent 7dfe646e2b
commit fccf89f849
9 changed files with 246 additions and 22 deletions
+60 -14
View File
@@ -35,6 +35,7 @@ from fluksio.sdk.client import (
Client,
RunHandle,
data_dir,
ignored,
login,
origin_of,
repo_root,
@@ -152,22 +153,28 @@ def _walkable(entry: Path) -> bool:
)
def _below(root: Path) -> Iterator[Path]:
def _below(root: Path, skip: frozenset[Path] = frozenset()) -> Iterator[Path]:
"""Every module and package under a plain directory, however deep.
One directory per study — `dev/s1_baseline/study.py` — is a layout people
have, and naming each of them on the command line is bookkeeping the tool
can do. A package is yielded whole and not descended into: its own walk
imports its submodules under the right names.
``skip`` is what the repository ignores. Importing a module runs it, and
what a repository ignores — `mlruns/`, `results/` — is output rather than
studies, and often large enough that walking it is most of the cost.
"""
for entry in sorted(root.iterdir()):
if entry in skip:
continue
if entry.is_file() and entry.suffix == ".py":
yield entry
elif _walkable(entry):
if (entry / "__init__.py").exists():
yield entry
else:
yield from _below(entry)
yield from _below(entry, skip)
def _import_package(directory: Path) -> None:
@@ -207,7 +214,7 @@ def discover(targets: list[str], keep_going: bool = False) -> list[Flow]:
# A plain directory — a repository root, or a directory of studies.
# Its own modules, the packages inside it, and the same again all the
# way down, so one folder per study needs no naming.
for entry in _below(path):
for entry in _below(path, ignored(path)):
try:
if entry.is_dir():
_import_package(entry)
@@ -496,7 +503,32 @@ def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
return params
def _sync_first(client: Client, targets: list[str] | None = None) -> None:
def _declaration(client: Client, flow: str) -> tuple[str, str, Path] | None:
"""Where a flow's last sync found it: import root, module name, file.
None when there is no such record to go on: a flow never synced, one drawn
on the canvas, an engine older than the field, or a checkout where that
file is not where the name says.
"""
if not flow:
return None
stored = client.get_flow(flow) or {}
origin = (stored.get("definition") or {}).get("origin") or {}
declared_in = str(origin.get("declared_in") or "")
module = str(origin.get("module") or "")
if not declared_in or not module:
return None
path = Path(repo_root(".")) / declared_in
stem = path.parent if path.name == "__init__.py" else path.with_suffix("")
parts = module.split(".")
if not path.is_file() or list(stem.parts[-len(parts) :]) != parts:
return None
return str(stem.parents[len(parts) - 1]), module, path
def _sync_first(
client: Client, targets: list[str] | None = None, flow: str = ""
) -> None:
"""Upload what the working directory declares, before running it.
The reason a run exists is usually the edit that came before it, and
@@ -504,24 +536,38 @@ def _sync_first(client: Client, targets: list[str] | None = None) -> None:
done. So `run` syncs by default — including the worker refresh, which is
what makes an edit to your own package take effect at all.
The whole directory, downwards: from a repository root the flow is usually
in a study folder below, and the alternative is syncing by hand and then
running with `--no-sync`. The upload is already a no-op for a flow nothing
changed in, so the cost is importing the other studies — which `--sync`
narrows when that is not free.
What is synced is the module that declared the flow, as the last sync
recorded it. Importing a module runs it, so walking the working directory
would run every analysis script in it before the run was even submitted.
A flow with no such record — never synced, or an engine older than the
field — falls back to the whole directory, downwards, skipping what the
repository ignores; a study there that will not import is a warning.
`--sync` names the targets outright.
A directory that declares nothing is not an error: a flow drawn on the
canvas is run the same way, and has nothing to upload.
"""
declared = None if targets else _declaration(client, flow)
try:
flows = discover(targets or ["."], keep_going=True)
except (ImportError, SyncError) as exc:
if declared is None:
flows = discover(targets or ["."], keep_going=True)
else:
# Under the name it was synced with, so the node bodies it
# generates are the ones already stored.
_import(*declared)
flows = list(FLOWS.values())
except SyncError as exc:
# Do not fail a run for a module the run may not even need.
_say(f"warning: nothing synced — {exc}")
return
except Exception as exc:
# A module named outright is imported without the walk's forgiveness,
# and importing it runs it: this is whatever it raised at its top.
_say(f"warning: nothing synced — {type(exc).__name__}: {exc}")
return
if not flows:
return
repo = repo_root((targets or ["."])[0])
repo = repo_root(declared[2] if declared else (targets or ["."])[0])
reports = sync(flows, client, origin=origin_of(repo))
changed = [r for r in reports if not r.unchanged]
if changed:
@@ -606,7 +652,7 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
try:
with _client_for(args) as client:
if not args.no_sync:
_sync_first(client, args.sync)
_sync_first(client, args.sync, args.flow)
stored = client.get_flow(args.flow)
if stored is None:
return _fail(f"no flow '{args.flow}' on that engine")
@@ -1145,7 +1191,7 @@ def cmd_sweep(args: argparse.Namespace) -> int:
try:
with _client_for(args) as client:
if not args.no_sync:
_sync_first(client, args.sync)
_sync_first(client, args.sync, args.flow)
stored = client.get_flow(args.flow)
if stored is None:
return _fail(f"no flow '{args.flow}' on that engine")