Sync only the declaring module before a run; skip git-ignored paths
`fluksio run` walked the working directory and imported every module in it, which ran a research repository's analysis scripts before the run was submitted. A flow's origin now records the file and module name that declared it, and run/sweep import just that, under the same name so the generated node bodies do not change. A flow with no record still walks, now skipping what .gitignore covers. Every code-defined flow reports `flow` changed once on its next sync, since the origin gained two fields. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QgG6TJw5qBZzaLghQSRVrq
This commit is contained in:
+60
-14
@@ -35,6 +35,7 @@ from fluksio.sdk.client import (
|
||||
Client,
|
||||
RunHandle,
|
||||
data_dir,
|
||||
ignored,
|
||||
login,
|
||||
origin_of,
|
||||
repo_root,
|
||||
@@ -152,22 +153,28 @@ def _walkable(entry: Path) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _below(root: Path) -> Iterator[Path]:
|
||||
def _below(root: Path, skip: frozenset[Path] = frozenset()) -> Iterator[Path]:
|
||||
"""Every module and package under a plain directory, however deep.
|
||||
|
||||
One directory per study — `dev/s1_baseline/study.py` — is a layout people
|
||||
have, and naming each of them on the command line is bookkeeping the tool
|
||||
can do. A package is yielded whole and not descended into: its own walk
|
||||
imports its submodules under the right names.
|
||||
|
||||
``skip`` is what the repository ignores. Importing a module runs it, and
|
||||
what a repository ignores — `mlruns/`, `results/` — is output rather than
|
||||
studies, and often large enough that walking it is most of the cost.
|
||||
"""
|
||||
for entry in sorted(root.iterdir()):
|
||||
if entry in skip:
|
||||
continue
|
||||
if entry.is_file() and entry.suffix == ".py":
|
||||
yield entry
|
||||
elif _walkable(entry):
|
||||
if (entry / "__init__.py").exists():
|
||||
yield entry
|
||||
else:
|
||||
yield from _below(entry)
|
||||
yield from _below(entry, skip)
|
||||
|
||||
|
||||
def _import_package(directory: Path) -> None:
|
||||
@@ -207,7 +214,7 @@ def discover(targets: list[str], keep_going: bool = False) -> list[Flow]:
|
||||
# A plain directory — a repository root, or a directory of studies.
|
||||
# Its own modules, the packages inside it, and the same again all the
|
||||
# way down, so one folder per study needs no naming.
|
||||
for entry in _below(path):
|
||||
for entry in _below(path, ignored(path)):
|
||||
try:
|
||||
if entry.is_dir():
|
||||
_import_package(entry)
|
||||
@@ -496,7 +503,32 @@ def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
|
||||
return params
|
||||
|
||||
|
||||
def _sync_first(client: Client, targets: list[str] | None = None) -> None:
|
||||
def _declaration(client: Client, flow: str) -> tuple[str, str, Path] | None:
|
||||
"""Where a flow's last sync found it: import root, module name, file.
|
||||
|
||||
None when there is no such record to go on: a flow never synced, one drawn
|
||||
on the canvas, an engine older than the field, or a checkout where that
|
||||
file is not where the name says.
|
||||
"""
|
||||
if not flow:
|
||||
return None
|
||||
stored = client.get_flow(flow) or {}
|
||||
origin = (stored.get("definition") or {}).get("origin") or {}
|
||||
declared_in = str(origin.get("declared_in") or "")
|
||||
module = str(origin.get("module") or "")
|
||||
if not declared_in or not module:
|
||||
return None
|
||||
path = Path(repo_root(".")) / declared_in
|
||||
stem = path.parent if path.name == "__init__.py" else path.with_suffix("")
|
||||
parts = module.split(".")
|
||||
if not path.is_file() or list(stem.parts[-len(parts) :]) != parts:
|
||||
return None
|
||||
return str(stem.parents[len(parts) - 1]), module, path
|
||||
|
||||
|
||||
def _sync_first(
|
||||
client: Client, targets: list[str] | None = None, flow: str = ""
|
||||
) -> None:
|
||||
"""Upload what the working directory declares, before running it.
|
||||
|
||||
The reason a run exists is usually the edit that came before it, and
|
||||
@@ -504,24 +536,38 @@ def _sync_first(client: Client, targets: list[str] | None = None) -> None:
|
||||
done. So `run` syncs by default — including the worker refresh, which is
|
||||
what makes an edit to your own package take effect at all.
|
||||
|
||||
The whole directory, downwards: from a repository root the flow is usually
|
||||
in a study folder below, and the alternative is syncing by hand and then
|
||||
running with `--no-sync`. The upload is already a no-op for a flow nothing
|
||||
changed in, so the cost is importing the other studies — which `--sync`
|
||||
narrows when that is not free.
|
||||
What is synced is the module that declared the flow, as the last sync
|
||||
recorded it. Importing a module runs it, so walking the working directory
|
||||
would run every analysis script in it before the run was even submitted.
|
||||
A flow with no such record — never synced, or an engine older than the
|
||||
field — falls back to the whole directory, downwards, skipping what the
|
||||
repository ignores; a study there that will not import is a warning.
|
||||
`--sync` names the targets outright.
|
||||
|
||||
A directory that declares nothing is not an error: a flow drawn on the
|
||||
canvas is run the same way, and has nothing to upload.
|
||||
"""
|
||||
declared = None if targets else _declaration(client, flow)
|
||||
try:
|
||||
flows = discover(targets or ["."], keep_going=True)
|
||||
except (ImportError, SyncError) as exc:
|
||||
if declared is None:
|
||||
flows = discover(targets or ["."], keep_going=True)
|
||||
else:
|
||||
# Under the name it was synced with, so the node bodies it
|
||||
# generates are the ones already stored.
|
||||
_import(*declared)
|
||||
flows = list(FLOWS.values())
|
||||
except SyncError as exc:
|
||||
# Do not fail a run for a module the run may not even need.
|
||||
_say(f"warning: nothing synced — {exc}")
|
||||
return
|
||||
except Exception as exc:
|
||||
# A module named outright is imported without the walk's forgiveness,
|
||||
# and importing it runs it: this is whatever it raised at its top.
|
||||
_say(f"warning: nothing synced — {type(exc).__name__}: {exc}")
|
||||
return
|
||||
if not flows:
|
||||
return
|
||||
repo = repo_root((targets or ["."])[0])
|
||||
repo = repo_root(declared[2] if declared else (targets or ["."])[0])
|
||||
reports = sync(flows, client, origin=origin_of(repo))
|
||||
changed = [r for r in reports if not r.unchanged]
|
||||
if changed:
|
||||
@@ -606,7 +652,7 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
||||
try:
|
||||
with _client_for(args) as client:
|
||||
if not args.no_sync:
|
||||
_sync_first(client, args.sync)
|
||||
_sync_first(client, args.sync, args.flow)
|
||||
stored = client.get_flow(args.flow)
|
||||
if stored is None:
|
||||
return _fail(f"no flow '{args.flow}' on that engine")
|
||||
@@ -1145,7 +1191,7 @@ def cmd_sweep(args: argparse.Namespace) -> int:
|
||||
try:
|
||||
with _client_for(args) as client:
|
||||
if not args.no_sync:
|
||||
_sync_first(client, args.sync)
|
||||
_sync_first(client, args.sync, args.flow)
|
||||
stored = client.get_flow(args.flow)
|
||||
if stored is None:
|
||||
return _fail(f"no flow '{args.flow}' on that engine")
|
||||
|
||||
Reference in New Issue
Block a user