Sync only the declaring module before a run; skip git-ignored paths
`fluksio run` walked the working directory and imported every module in it, which ran a research repository's analysis scripts before the run was submitted. A flow's origin now records the file and module name that declared it, and run/sweep import just that, under the same name so the generated node bodies do not change. A flow with no record still walks, now skipping what .gitignore covers. Every code-defined flow reports `flow` changed once on its next sync, since the origin gained two fields. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QgG6TJw5qBZzaLghQSRVrq
This commit is contained in:
@@ -253,6 +253,12 @@ class FlowOrigin(BaseModel):
|
||||
#: a run stamped with a dirty commit names code that was never stored.
|
||||
commit: str = ""
|
||||
dirty: bool = False
|
||||
#: The file that declared the flow, relative to ``repo``, and the module
|
||||
#: name it was imported under. What ``fluksio run`` imports by default,
|
||||
#: instead of every script below the working directory — by that name, or
|
||||
#: the generated node bodies would change with how the flow was synced.
|
||||
declared_in: str = ""
|
||||
module: str = ""
|
||||
|
||||
|
||||
class FlowInput(BaseModel):
|
||||
|
||||
@@ -582,6 +582,35 @@ def _code_of(spec: NodeSpec) -> dict[str, Any]:
|
||||
return {"code_files": files, "code_digest": digest_of(files)}
|
||||
|
||||
|
||||
def _declared_at() -> tuple[str, str]:
|
||||
"""The file and module name of whatever constructed the flow.
|
||||
|
||||
Empty for a declaration no import could reach again: a script run
|
||||
directly, or a notebook cell, which is not a file on disk.
|
||||
"""
|
||||
frame = inspect.currentframe()
|
||||
# Out of this helper, then out of `Flow.__init__`.
|
||||
for _ in range(2):
|
||||
frame = frame.f_back if frame is not None else None
|
||||
if frame is None:
|
||||
return "", ""
|
||||
module = str(frame.f_globals.get("__name__") or "")
|
||||
path = Path(frame.f_code.co_filename)
|
||||
if module == "__main__" or not path.is_file():
|
||||
return "", ""
|
||||
return str(path.resolve()), module
|
||||
|
||||
|
||||
def _relative(path: str, repo: str) -> str:
|
||||
"""``path`` as seen from ``repo``, or empty when it is not inside it."""
|
||||
if not path or not repo:
|
||||
return ""
|
||||
try:
|
||||
return Path(path).relative_to(Path(repo).resolve()).as_posix()
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
def _serialisable(value: Any) -> bool:
|
||||
try:
|
||||
json.dumps(value)
|
||||
@@ -619,6 +648,9 @@ class Flow:
|
||||
self.inputs = list(inputs)
|
||||
self.outputs = list(outputs)
|
||||
self.nodes = [_spec_of(entry) for entry in nodes]
|
||||
#: Where this flow was declared: the file, and the module name it was
|
||||
#: imported under. What `fluksio run` imports by default.
|
||||
self.declared_in, self.module = _declared_at()
|
||||
self._resolve_types()
|
||||
self._validate()
|
||||
self._nodes = [_check(spec, mode) for spec in self.nodes]
|
||||
@@ -719,7 +751,11 @@ class Flow:
|
||||
"outputs": self.outputs,
|
||||
}
|
||||
if origin is not None:
|
||||
doc["origin"] = origin
|
||||
declared_in = _relative(self.declared_in, str(origin.get("repo") or ""))
|
||||
doc["origin"] = origin | {
|
||||
"declared_in": declared_in,
|
||||
"module": self.module if declared_in else "",
|
||||
}
|
||||
return doc
|
||||
|
||||
def shims(self) -> dict[str, str]:
|
||||
|
||||
+60
-14
@@ -35,6 +35,7 @@ from fluksio.sdk.client import (
|
||||
Client,
|
||||
RunHandle,
|
||||
data_dir,
|
||||
ignored,
|
||||
login,
|
||||
origin_of,
|
||||
repo_root,
|
||||
@@ -152,22 +153,28 @@ def _walkable(entry: Path) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _below(root: Path) -> Iterator[Path]:
|
||||
def _below(root: Path, skip: frozenset[Path] = frozenset()) -> Iterator[Path]:
|
||||
"""Every module and package under a plain directory, however deep.
|
||||
|
||||
One directory per study — `dev/s1_baseline/study.py` — is a layout people
|
||||
have, and naming each of them on the command line is bookkeeping the tool
|
||||
can do. A package is yielded whole and not descended into: its own walk
|
||||
imports its submodules under the right names.
|
||||
|
||||
``skip`` is what the repository ignores. Importing a module runs it, and
|
||||
what a repository ignores — `mlruns/`, `results/` — is output rather than
|
||||
studies, and often large enough that walking it is most of the cost.
|
||||
"""
|
||||
for entry in sorted(root.iterdir()):
|
||||
if entry in skip:
|
||||
continue
|
||||
if entry.is_file() and entry.suffix == ".py":
|
||||
yield entry
|
||||
elif _walkable(entry):
|
||||
if (entry / "__init__.py").exists():
|
||||
yield entry
|
||||
else:
|
||||
yield from _below(entry)
|
||||
yield from _below(entry, skip)
|
||||
|
||||
|
||||
def _import_package(directory: Path) -> None:
|
||||
@@ -207,7 +214,7 @@ def discover(targets: list[str], keep_going: bool = False) -> list[Flow]:
|
||||
# A plain directory — a repository root, or a directory of studies.
|
||||
# Its own modules, the packages inside it, and the same again all the
|
||||
# way down, so one folder per study needs no naming.
|
||||
for entry in _below(path):
|
||||
for entry in _below(path, ignored(path)):
|
||||
try:
|
||||
if entry.is_dir():
|
||||
_import_package(entry)
|
||||
@@ -496,7 +503,32 @@ def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
|
||||
return params
|
||||
|
||||
|
||||
def _sync_first(client: Client, targets: list[str] | None = None) -> None:
|
||||
def _declaration(client: Client, flow: str) -> tuple[str, str, Path] | None:
|
||||
"""Where a flow's last sync found it: import root, module name, file.
|
||||
|
||||
None when there is no such record to go on: a flow never synced, one drawn
|
||||
on the canvas, an engine older than the field, or a checkout where that
|
||||
file is not where the name says.
|
||||
"""
|
||||
if not flow:
|
||||
return None
|
||||
stored = client.get_flow(flow) or {}
|
||||
origin = (stored.get("definition") or {}).get("origin") or {}
|
||||
declared_in = str(origin.get("declared_in") or "")
|
||||
module = str(origin.get("module") or "")
|
||||
if not declared_in or not module:
|
||||
return None
|
||||
path = Path(repo_root(".")) / declared_in
|
||||
stem = path.parent if path.name == "__init__.py" else path.with_suffix("")
|
||||
parts = module.split(".")
|
||||
if not path.is_file() or list(stem.parts[-len(parts) :]) != parts:
|
||||
return None
|
||||
return str(stem.parents[len(parts) - 1]), module, path
|
||||
|
||||
|
||||
def _sync_first(
|
||||
client: Client, targets: list[str] | None = None, flow: str = ""
|
||||
) -> None:
|
||||
"""Upload what the working directory declares, before running it.
|
||||
|
||||
The reason a run exists is usually the edit that came before it, and
|
||||
@@ -504,24 +536,38 @@ def _sync_first(client: Client, targets: list[str] | None = None) -> None:
|
||||
done. So `run` syncs by default — including the worker refresh, which is
|
||||
what makes an edit to your own package take effect at all.
|
||||
|
||||
The whole directory, downwards: from a repository root the flow is usually
|
||||
in a study folder below, and the alternative is syncing by hand and then
|
||||
running with `--no-sync`. The upload is already a no-op for a flow nothing
|
||||
changed in, so the cost is importing the other studies — which `--sync`
|
||||
narrows when that is not free.
|
||||
What is synced is the module that declared the flow, as the last sync
|
||||
recorded it. Importing a module runs it, so walking the working directory
|
||||
would run every analysis script in it before the run was even submitted.
|
||||
A flow with no such record — never synced, or an engine older than the
|
||||
field — falls back to the whole directory, downwards, skipping what the
|
||||
repository ignores; a study there that will not import is a warning.
|
||||
`--sync` names the targets outright.
|
||||
|
||||
A directory that declares nothing is not an error: a flow drawn on the
|
||||
canvas is run the same way, and has nothing to upload.
|
||||
"""
|
||||
declared = None if targets else _declaration(client, flow)
|
||||
try:
|
||||
flows = discover(targets or ["."], keep_going=True)
|
||||
except (ImportError, SyncError) as exc:
|
||||
if declared is None:
|
||||
flows = discover(targets or ["."], keep_going=True)
|
||||
else:
|
||||
# Under the name it was synced with, so the node bodies it
|
||||
# generates are the ones already stored.
|
||||
_import(*declared)
|
||||
flows = list(FLOWS.values())
|
||||
except SyncError as exc:
|
||||
# Do not fail a run for a module the run may not even need.
|
||||
_say(f"warning: nothing synced — {exc}")
|
||||
return
|
||||
except Exception as exc:
|
||||
# A module named outright is imported without the walk's forgiveness,
|
||||
# and importing it runs it: this is whatever it raised at its top.
|
||||
_say(f"warning: nothing synced — {type(exc).__name__}: {exc}")
|
||||
return
|
||||
if not flows:
|
||||
return
|
||||
repo = repo_root((targets or ["."])[0])
|
||||
repo = repo_root(declared[2] if declared else (targets or ["."])[0])
|
||||
reports = sync(flows, client, origin=origin_of(repo))
|
||||
changed = [r for r in reports if not r.unchanged]
|
||||
if changed:
|
||||
@@ -606,7 +652,7 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
||||
try:
|
||||
with _client_for(args) as client:
|
||||
if not args.no_sync:
|
||||
_sync_first(client, args.sync)
|
||||
_sync_first(client, args.sync, args.flow)
|
||||
stored = client.get_flow(args.flow)
|
||||
if stored is None:
|
||||
return _fail(f"no flow '{args.flow}' on that engine")
|
||||
@@ -1145,7 +1191,7 @@ def cmd_sweep(args: argparse.Namespace) -> int:
|
||||
try:
|
||||
with _client_for(args) as client:
|
||||
if not args.no_sync:
|
||||
_sync_first(client, args.sync)
|
||||
_sync_first(client, args.sync, args.flow)
|
||||
stored = client.get_flow(args.flow)
|
||||
if stored is None:
|
||||
return _fail(f"no flow '{args.flow}' on that engine")
|
||||
|
||||
@@ -708,6 +708,26 @@ def repo_root(start: str | Path) -> str:
|
||||
return found or str(directory)
|
||||
|
||||
|
||||
def ignored(directory: Path) -> frozenset[Path]:
|
||||
"""What the repository ignores below a directory, as git itself reads it.
|
||||
|
||||
Directories come back whole rather than file by file, so an ignored
|
||||
`mlruns/` costs one entry instead of a walk. Empty outside a repository.
|
||||
"""
|
||||
listed = _git(
|
||||
str(directory),
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--ignored",
|
||||
"--exclude-standard",
|
||||
"--directory",
|
||||
"-z",
|
||||
)
|
||||
return frozenset(
|
||||
directory / entry.rstrip("/") for entry in (listed or "").split("\0") if entry
|
||||
)
|
||||
|
||||
|
||||
def sync(
|
||||
flows: Iterable[Flow],
|
||||
client: Client,
|
||||
|
||||
Reference in New Issue
Block a user