stroblme
1e62e77aed
It claimed to run the pipeline with no engine involved, and could not: the node bodies it called save and load artifacts, which raise outside a node by design. Each node is now a thin wrapper over a plain function — make_rows, train_curve, score — and __main__ calls those, which is the split the sandbox already demonstrates and the one worth copying.
43 lines
1.4 KiB
Python
43 lines
1.4 KiB
Python
"""Where the data comes from."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import fluksio
|
|
from fluksio import Port, node
|
|
|
|
|
|
def make_rows(source: str = "builtin", limit: int = 512) -> dict:
|
|
"""The training set itself, with no Fluksio in it."""
|
|
return {
|
|
"source": source,
|
|
"rows": [{"x": index / limit, "y": (index % 7) / 7} for index in range(limit)],
|
|
}
|
|
|
|
|
|
@node(provides=[Port("dataset", "artifact"), Port("rows", "int")])
|
|
def prepare(source: str = "builtin", limit: int = 512):
|
|
"""Make the training set and store it, returning a reference to it.
|
|
|
|
`source` and `limit` have defaults and are not ports, so they become the
|
|
node's settings: the canvas can tune them without touching this file.
|
|
"""
|
|
payload = make_rows(source, limit)
|
|
blob = json.dumps(payload).encode()
|
|
return {
|
|
"dataset": fluksio.save_artifact(blob, "dataset.json", "application/json"),
|
|
"rows": len(payload["rows"]),
|
|
}
|
|
|
|
|
|
@node(requires=["dataset"], provides=[Port("augmented", "artifact")])
|
|
def augment(dataset, factor: int = 2):
|
|
"""Grow the training set, so `finetune` has something of its own."""
|
|
payload = json.loads(open(fluksio.load_artifact(dataset)).read())
|
|
payload["rows"] = payload["rows"] * factor
|
|
blob = json.dumps(payload).encode()
|
|
return {
|
|
"augmented": fluksio.save_artifact(blob, "augmented.json", "application/json")
|
|
}
|