Export every recorded input, not only the ones that vary
`export runs` dropped a `param.*` column whose value was constant across the exported runs, so a downstream filter broke depending on which runs the selection happened to hold. Every input the selection recorded is a column now; `--params` still narrows it to a sweep's axis. The metrics default is unchanged. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019Hra4ndWMCLU5F3KjUuVAc
This commit is contained in:
@@ -422,25 +422,6 @@ def _dig(record: dict[str, Any], path: str) -> Any:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _varying(runs: list[Run]) -> list[str]:
|
|
||||||
"""The inputs that differ across these runs — the axis of a sweep.
|
|
||||||
|
|
||||||
What a reader comparing arms wants as columns, compared leaf by leaf: two
|
|
||||||
configurations differing in one field give that field as a column rather
|
|
||||||
than two blobs that are not the same. Under two runs nothing can differ,
|
|
||||||
and a table of one run with none of its inputs in it is not worth reading,
|
|
||||||
so all of them are kept.
|
|
||||||
"""
|
|
||||||
keys = sorted({path for run in runs for path, _ in _leaves(run.params)})
|
|
||||||
if len(runs) < 2:
|
|
||||||
return keys
|
|
||||||
return [
|
|
||||||
key
|
|
||||||
for key in keys
|
|
||||||
if len({json.dumps(_dig(run.params, key), sort_keys=True) for run in runs}) > 1
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _scored(runs: list[Run]) -> list[str]:
|
def _scored(runs: list[Run]) -> list[str]:
|
||||||
"""A run's final numbers: every number its declared outputs carry, however
|
"""A run's final numbers: every number its declared outputs carry, however
|
||||||
deep it sits. A flag is not a number, and neither is a label."""
|
deep it sits. A flag is not a number, and neither is a label."""
|
||||||
@@ -557,9 +538,9 @@ def export_runs(
|
|||||||
"""One row per run: what it was given, what it scored, what code it ran.
|
"""One row per run: what it was given, what it scored, what code it ran.
|
||||||
|
|
||||||
The arm-comparison table. Inputs are columns rather than one JSON blob —
|
The arm-comparison table. Inputs are columns rather than one JSON blob —
|
||||||
by default the ones that vary across the selection, which is the sweep
|
every input the selection recorded, so the schema is the same whichever
|
||||||
axis; ``params`` names them instead. ``metrics`` narrows the final numbers
|
runs are asked for; ``params`` narrows it. ``metrics`` narrows the final
|
||||||
to a few of a run's declared outputs.
|
numbers to a few of a run's declared outputs.
|
||||||
|
|
||||||
Both take dotted paths into a record a node returned:
|
Both take dotted paths into a record a node returned:
|
||||||
``metrics=final_metrics.train_loss,test_metrics.known.perfect`` selects
|
``metrics=final_metrics.train_loss,test_metrics.known.perfect`` selects
|
||||||
@@ -567,7 +548,9 @@ def export_runs(
|
|||||||
depth.
|
depth.
|
||||||
"""
|
"""
|
||||||
runs = _selected(session, flow, status, group, ids, since, until)
|
runs = _selected(session, flow, status, group, ids, since, until)
|
||||||
inputs = [part for part in params.split(",") if part] or _varying(runs)
|
inputs = [part for part in params.split(",") if part] or sorted(
|
||||||
|
{path for run in runs for path, _ in _leaves(run.params)}
|
||||||
|
)
|
||||||
scores = [part for part in metrics.split(",") if part] or _scored(runs)
|
scores = [part for part in metrics.split(",") if part] or _scored(runs)
|
||||||
columns = [
|
columns = [
|
||||||
*RUN_COLUMNS,
|
*RUN_COLUMNS,
|
||||||
|
|||||||
@@ -1394,7 +1394,7 @@ def add_parsers(subparsers: Any) -> None:
|
|||||||
metavar="A,B",
|
metavar="A,B",
|
||||||
help=(
|
help=(
|
||||||
"the inputs to put in columns, dotted into a record "
|
"the inputs to put in columns, dotted into a record "
|
||||||
"(default: the ones that vary)"
|
"(default: every input the runs recorded)"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
sub.add_argument(
|
sub.add_argument(
|
||||||
|
|||||||
@@ -428,9 +428,9 @@ class Client:
|
|||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
"""One row per run: its inputs as columns, its final numbers, its code.
|
"""One row per run: its inputs as columns, its final numbers, its code.
|
||||||
|
|
||||||
The arm-comparison table. The inputs kept are the ones that vary
|
The arm-comparison table. Every input the selection recorded is a
|
||||||
across the selection unless ``params`` names them, which is the axis a
|
column, so the schema does not depend on which runs were asked for;
|
||||||
sweep is read along.
|
``params`` narrows it to the axis a sweep is read along.
|
||||||
"""
|
"""
|
||||||
query: dict[str, Any] = {}
|
query: dict[str, Any] = {}
|
||||||
if params:
|
if params:
|
||||||
|
|||||||
@@ -873,10 +873,10 @@ def test_an_export_strides_each_series_and_names_its_run(
|
|||||||
assert all(steps == [0, 2] for steps in curves.values())
|
assert all(steps == [0, 2] for steps in curves.values())
|
||||||
|
|
||||||
|
|
||||||
def test_an_exported_run_row_carries_the_inputs_that_vary(
|
def test_an_exported_run_row_carries_every_recorded_input(
|
||||||
client, superuser_token_headers, exported
|
client, superuser_token_headers, exported
|
||||||
):
|
):
|
||||||
"""The sweep axis becomes columns; what every run shares stays out of them."""
|
"""Every input is a column, so the schema does not move with the selection."""
|
||||||
|
|
||||||
def export(**extra):
|
def export(**extra):
|
||||||
answer = client.get(
|
answer = client.get(
|
||||||
@@ -890,12 +890,15 @@ def test_an_exported_run_row_carries_the_inputs_that_vary(
|
|||||||
rows = _lines(export(format="jsonl"))
|
rows = _lines(export(format="jsonl"))
|
||||||
assert [row["id"] for row in rows] == ["exp-1", "exp-0"]
|
assert [row["id"] for row in rows] == ["exp-1", "exp-0"]
|
||||||
assert {row["param.lr"] for row in rows} == {0.1, 0.01}
|
assert {row["param.lr"] for row in rows} == {0.1, 0.01}
|
||||||
# `epochs` is the same on both runs, so it is not what they differ by, and
|
# `epochs` is the same on both runs and stays a column anyway: which runs
|
||||||
# neither is the model inside the config — but the depth beside it is.
|
# were asked for is not something a downstream filter should have to know.
|
||||||
assert "param.epochs" not in rows[0]
|
assert rows[0]["param.epochs"] == 10
|
||||||
assert "param.config.model" not in rows[0]
|
assert rows[0]["param.config.model"] == "mlp"
|
||||||
assert {row["param.config.depth"] for row in rows} == {1, 2}
|
assert {row["param.config.depth"] for row in rows} == {1, 2}
|
||||||
assert "param.epochs" in _lines(export(format="jsonl", params="epochs"))[0]
|
|
||||||
|
narrowed = _lines(export(format="jsonl", params="epochs"))[0]
|
||||||
|
assert "param.epochs" in narrowed
|
||||||
|
assert "param.lr" not in narrowed
|
||||||
|
|
||||||
# A number inside a record is a column of its own, however deep; a string
|
# A number inside a record is a column of its own, however deep; a string
|
||||||
# is not one of the run's numbers wherever it sits.
|
# is not one of the run's numbers wherever it sits.
|
||||||
@@ -912,7 +915,7 @@ def test_an_exported_run_row_carries_the_inputs_that_vary(
|
|||||||
header = export().text.splitlines()[0]
|
header = export().text.splitlines()[0]
|
||||||
assert header.startswith("id,flow,status,")
|
assert header.startswith("id,flow,status,")
|
||||||
assert header.endswith(
|
assert header.endswith(
|
||||||
"param.config.depth,param.lr,"
|
"param.config.depth,param.config.model,param.epochs,param.lr,"
|
||||||
"metric.acc,metric.final_metrics.train_loss,"
|
"metric.acc,metric.final_metrics.train_loss,"
|
||||||
"metric.test_metrics.known.perfect"
|
"metric.test_metrics.known.perfect"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user