Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a Python SDK: flows declared in your own repository

Browse Source 
A data scientist keeps their code where it is and decorates it: `@node`
declares a function's ports beside the function, `Flow(name, nodes=[...])`
says which of them make a flow, and `use(fn, wire=..., **settings)` rebinds
one for a single flow. `fluksio sync` uploads the document plus a generated
import shim per node, so the store still holds a complete, runnable,
git-versioned definition while the code it imports stays theirs.

`fluksio login|run|runs` and `flow.submit().wait()` are the client half, over
the run endpoints that already existed. Runs record the user repository's
commit beside the store's, so "what code produced this number" is answerable
on the side that now holds the code.

- `fluksio/sdk/`: ports, decorators, the flow builder and its checks, the shim
  generator, an HTTP client and sync. Standard library only at import, so
  `from fluksio import node` in a training script pulls in no engine.
- `FlowDef.origin` marks a flow code-defined; `Run.origin_commit` carries the
  repository's commit; `POST /modules/refresh` retires the workers without an
  install, which every sync calls — a worker holds the imported package in
  memory, so an edit to it is invisible until the process goes.
- The canvas shows a generated body read-only and names the repository to edit
  instead; a body edited there stops the next sync rather than being discarded.
- The worker's reporter carries inert `Port`, `node`, `use` and `Flow`, since
  the shim imports a module whose first line declares them.
- `examples/myresearch` is the worked example, `make sync-example` uploads it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU
This commit is contained in:
stroblme
2026-08-23 20:16:08 +02:00
co-authored by Claude Fable 5
parent 775d151307
commit a38e2745eb
35 changed files with 2693 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/sdk/cli.py
+306
View File
@@ -0,0 +1,306 @@
"""The `fluksio sync`, `run`, `runs` and `login` commands.
Kept beside the SDK rather than in `fluksio.cli`: these are the client half of
the tool, and none of them needs the engine to be importable.
"""
from __future__ import annotations
import argparse
import getpass
import importlib
import json
import pkgutil
import sys
from pathlib import Path
from typing import Any
from fluksio.sdk import FLOWS, Flow, SyncError
from fluksio.sdk.client import (
ApiError,
Client,
config_path,
login,
origin_of,
repo_root,
sync,
)
__all__ = ["add_parsers", "discover"]
def _say(message: str = "") -> None:
print(message)
def _fail(message: str) -> int:
print(f"fluksio: {message}", file=sys.stderr)
return 1
# ---------------------------------------------------------------------------
# Discovery
# ---------------------------------------------------------------------------
def _package_of(directory: Path) -> tuple[str, str]:
"""The path root and dotted name of a package directory."""
parts = [directory.name]
parent = directory.parent
while (parent / "__init__.py").exists():
parts.append(parent.name)
parent = parent.parent
return str(parent), ".".join(reversed(parts))
def _module_of(path: Path) -> tuple[str, str]:
"""The path root and dotted name of a module file."""
parts = [path.stem]
directory = path.parent
while (directory / "__init__.py").exists():
parts.append(directory.name)
directory = directory.parent
return str(directory), ".".join(reversed(parts))
def _import(root: str, dotted: str) -> None:
if root not in sys.path:
sys.path.insert(0, root)
importlib.import_module(dotted)
def discover(targets: list[str]) -> list[Flow]:
"""Import what was named and hand back the flows it declared.
Imported by dotted name with its root on the path, never from a file
location: the generated node bodies import the same way, and a module
loaded under a different name would generate an import that does not
resolve.
"""
for target in targets:
path = Path(target)
if not path.exists():
_import(str(Path.cwd()), target)
continue
path = path.resolve()
if path.is_file():
_import(*_module_of(path))
continue
if (path / "__init__.py").exists():
root, dotted = _package_of(path)
_import(root, dotted)
package = sys.modules[dotted]
for info in pkgutil.walk_packages(package.__path__, f"{dotted}."):
importlib.import_module(info.name)
continue
for module in sorted(path.glob("*.py")):
_import(*_module_of(module))
return list(FLOWS.values())
# ---------------------------------------------------------------------------
# Commands
# ---------------------------------------------------------------------------
def cmd_login(args: argparse.Namespace) -> int:
email = args.email or input("Email: ")
password = args.password or getpass.getpass("Password: ")
try:
login(args.url, email, password)
except ApiError as exc:
return _fail(f"could not log in: {exc.detail}")
_say(f"Logged in to {args.url}; the token is in {config_path()}.")
return 0
def cmd_sync(args: argparse.Namespace) -> int:
targets = args.targets or ["."]
try:
flows = discover(targets)
except (ImportError, SyncError) as exc:
return _fail(str(exc))
if not flows:
return _fail(
f"no flows declared in {', '.join(targets)} — a flow is a `flow(...)` "
"call at module level"
)
repo = repo_root(targets[0])
origin = origin_of(repo)
if origin["dirty"]:
_say(f"warning: {repo} has uncommitted changes, so the stamp says -dirty")
if not origin["commit"]:
_say(f"warning: {repo} is not a git repository, so runs cannot name a commit")
if args.dry_run:
for target in flows:
_say(f"=== flow {target.name}")
_say(json.dumps(target.document(origin), indent=2))
for node_id, code in target.shims().items():
_say(f"=== {target.name}.{node_id}")
_say(code)
return 0
try:
client = Client(url=args.url, token=args.token)
reports = sync(
flows,
client,
origin=origin,
publish=not args.no_publish,
force=args.force,
)
except (SyncError, ApiError) as exc:
return _fail(str(exc))
for report in reports:
if report.unchanged:
_say(f" {report.flow}: unchanged")
continue
what = "created" if report.created else "updated"
detail = ", ".join(report.changed)
state = "published" if report.published else "draft"
_say(f" {report.flow}: {what} ({detail}) — {state}")
stamp = origin["commit"][:7] + ("-dirty" if origin["dirty"] else "")
_say(f"Stamped with {stamp or 'no commit'} from {repo}.")
return 0
def _coerce(value: str, dtype: str) -> Any:
if dtype == "int":
return int(value)
if dtype == "float":
return float(value)
if dtype == "bool":
return value.lower() in ("true", "1", "yes", "on")
if dtype == "str":
return value
return json.loads(value)
def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
"""Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
types = {
str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float"))
for entry in definition.get("inputs") or []
}
params: dict[str, Any] = {}
pending: str | None = None
for token in rest:
if token.startswith("--"):
if pending is not None:
# A flag with no value is a flag: `--resume` means true.
params[pending] = True
name, sep, value = token[2:].partition("=")
# Only the name is spelled with dashes; a value may hold one, and
# `--lr=1e-4` is the case that says so.
pending = name.replace("-", "_")
if sep:
params[pending] = _coerce(value, types.get(pending, "json"))
pending = None
continue
if pending is None:
raise SyncError(f"unexpected argument '{token}'")
params[pending] = _coerce(token, types.get(pending, "json"))
pending = None
if pending is not None:
params[pending] = True
unknown = sorted(set(params) - set(types))
if unknown:
raise SyncError(
f"'{unknown[0]}' is not an input of this flow (it takes "
f"{', '.join(sorted(types)) or 'none'})"
)
return params
def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
try:
client = Client(url=args.url, token=args.token)
stored = client.get_flow(args.flow)
if stored is None:
return _fail(f"no flow '{args.flow}' on that engine")
params = _params(stored.get("definition") or {}, rest)
handle = client.submit(args.flow, params, seed=args.seed)
except (SyncError, ApiError) as exc:
return _fail(str(exc))
_say(f"{handle.id} queued {json.dumps(params)}")
if not args.wait:
return 0
handle.wait(timeout=args.timeout)
_say(f"{handle.id} {handle.status} {json.dumps(handle.result)}")
return 0 if handle.status == "ok" else 1
def cmd_runs(args: argparse.Namespace) -> int:
try:
rows = Client(url=args.url, token=args.token).runs(
flow=args.flow, limit=args.limit
)
except (SyncError, ApiError) as exc:
return _fail(str(exc))
for row in rows:
commit = (row.get("origin_commit") or "")[:7]
_say(
f"{row['id']} {row['status']:<9} {row['flow']:<16} "
f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {json.dumps(row['params'])}"
)
return 0
# ---------------------------------------------------------------------------
# Wiring
# ---------------------------------------------------------------------------
def add_parsers(subparsers: Any) -> None:
"""Register the client commands on `fluksio`'s parser."""
def with_engine(sub: argparse.ArgumentParser) -> None:
sub.add_argument(
"--url", default="", help="the engine (default: the last login)"
)
sub.add_argument("--token", default="", help="override the stored token")
parser = subparsers.add_parser("login", help="store a token for an engine")
parser.add_argument("--url", default="http://localhost:8000")
parser.add_argument("--email", default="")
parser.add_argument("--password", default="")
parser.set_defaults(func=cmd_login)
parser = subparsers.add_parser(
"sync", help="upload the flows declared in your own code"
)
parser.add_argument(
"targets",
nargs="*",
help="modules, packages or directories to import (default: .)",
)
parser.add_argument(
"--dry-run", action="store_true", help="print what would be uploaded"
)
parser.add_argument(
"--no-publish", action="store_true", help="leave the changes as a draft"
)
parser.add_argument(
"--force", action="store_true", help="overwrite work done on the canvas"
)
with_engine(parser)
parser.set_defaults(func=cmd_sync)
parser = subparsers.add_parser(
"run", help="start a run, passing the flow's inputs as --name value"
)
parser.add_argument("flow")
parser.add_argument("--seed", type=int, default=None)
parser.add_argument("--wait", action="store_true", help="block until it finishes")
parser.add_argument("--timeout", type=float, default=0.0)
with_engine(parser)
parser.set_defaults(func=cmd_run)
parser = subparsers.add_parser("runs", help="the runs an engine has recorded")
parser.add_argument("--flow", default="")
parser.add_argument("--limit", type=int, default=20)
with_engine(parser)
parser.set_defaults(func=cmd_runs)