Modules resolve the manifest with uv pip compile before syncing, so listed packages bring their dependencies
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx
This commit is contained in:
@@ -6,9 +6,12 @@ into a venv on the data volume rather than into the engine's environment: a
|
|||||||
pin here can never shadow — or be shadowed by — what the app itself runs on,
|
pin here can never shadow — or be shadowed by — what the app itself runs on,
|
||||||
and the worker processes that import them need nothing from the app.
|
and the worker processes that import them need nothing from the app.
|
||||||
|
|
||||||
``uv pip sync`` rather than install, so a line taken out of the manifest is
|
The manifest is resolved with ``uv pip compile`` first, so a line naming a
|
||||||
uninstalled. The manifest is written only after a sync succeeds, which is all
|
package brings its dependencies along, and a manifest that already pins
|
||||||
the rollback a failed resolve needs.
|
everything resolves to itself. The result goes to ``uv pip sync`` rather than
|
||||||
|
install, so a line taken out of the manifest is uninstalled. The manifest is
|
||||||
|
written only after a sync succeeds, which is all the rollback a failed resolve
|
||||||
|
needs.
|
||||||
|
|
||||||
None of that applies to an *adopted* venv. A data scientist makes a venv,
|
None of that applies to an *adopted* venv. A data scientist makes a venv,
|
||||||
installs what they work with, and then installs Fluksio into it too — at which
|
installs what they work with, and then installs Fluksio into it too — at which
|
||||||
@@ -179,35 +182,52 @@ def sync(requirements: str) -> tuple[bool, str]:
|
|||||||
|
|
||||||
|
|
||||||
def _sync(requirements: str) -> tuple[bool, str]:
|
def _sync(requirements: str) -> tuple[bool, str]:
|
||||||
"""The install itself, under `_sync_lock`."""
|
"""The resolve and install, under `_sync_lock`."""
|
||||||
manifest = ""
|
|
||||||
try:
|
try:
|
||||||
ensure_venv()
|
ensure_venv()
|
||||||
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as handle:
|
python = venv_python()
|
||||||
handle.write(requirements)
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
manifest = handle.name
|
manifest = Path(directory, "requirements.in")
|
||||||
result = subprocess.run(
|
lock = Path(directory, "requirements.txt")
|
||||||
# Empty means empty: without the flag uv refuses to clear a venv,
|
manifest.write_text(requirements)
|
||||||
# so deleting the last line would leave the package installed.
|
for command in (
|
||||||
|
# The emit flags carry the manifest's index, find-links and
|
||||||
|
# binary options into the lock, which is all sync reads.
|
||||||
|
[
|
||||||
|
uv_bin(),
|
||||||
|
"pip",
|
||||||
|
"compile",
|
||||||
|
"--quiet",
|
||||||
|
"--emit-index-url",
|
||||||
|
"--emit-find-links",
|
||||||
|
"--emit-build-options",
|
||||||
|
"--python",
|
||||||
|
python,
|
||||||
|
str(manifest),
|
||||||
|
"--output-file",
|
||||||
|
str(lock),
|
||||||
|
],
|
||||||
|
# Empty means empty: without the flag uv refuses to clear a
|
||||||
|
# venv, so deleting the last line would leave the package
|
||||||
|
# installed.
|
||||||
[
|
[
|
||||||
uv_bin(),
|
uv_bin(),
|
||||||
"pip",
|
"pip",
|
||||||
"sync",
|
"sync",
|
||||||
"--allow-empty-requirements",
|
"--allow-empty-requirements",
|
||||||
"--python",
|
"--python",
|
||||||
venv_python(),
|
python,
|
||||||
manifest,
|
str(lock),
|
||||||
],
|
],
|
||||||
capture_output=True,
|
):
|
||||||
text=True,
|
result = subprocess.run(
|
||||||
timeout=SYNC_TIMEOUT,
|
command, capture_output=True, text=True, timeout=SYNC_TIMEOUT
|
||||||
)
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
break
|
||||||
except (OSError, subprocess.SubprocessError) as exc:
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
detail = str(getattr(exc, "stderr", None) or exc).strip()
|
detail = str(getattr(exc, "stderr", None) or exc).strip()
|
||||||
return False, f"uv could not run: {detail}"
|
return False, f"uv could not run: {detail}"
|
||||||
finally:
|
|
||||||
if manifest:
|
|
||||||
Path(manifest).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
output = (result.stdout + result.stderr).strip()
|
output = (result.stdout + result.stderr).strip()
|
||||||
if result.returncode == 0:
|
if result.returncode == 0:
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -77,6 +78,40 @@ def test_a_manifest_that_does_not_resolve_leaves_the_venv_alone(venv: Path):
|
|||||||
assert (venv / ".applied").read_text() == modules._digest("")
|
assert (venv / ".applied").read_text() == modules._digest("")
|
||||||
|
|
||||||
|
|
||||||
|
def _wheel(directory: Path, name: str, requires: str = "") -> None:
|
||||||
|
"""A wheel with nothing in it but a name and, optionally, one dependency."""
|
||||||
|
info = f"{name}-1.0.dist-info"
|
||||||
|
metadata = f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n"
|
||||||
|
if requires:
|
||||||
|
metadata += f"Requires-Dist: {requires}\n"
|
||||||
|
with zipfile.ZipFile(directory / f"{name}-1.0-py3-none-any.whl", "w") as wheel:
|
||||||
|
wheel.writestr(f"{name}/__init__.py", "")
|
||||||
|
wheel.writestr(f"{info}/METADATA", metadata)
|
||||||
|
wheel.writestr(
|
||||||
|
f"{info}/WHEEL",
|
||||||
|
"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n",
|
||||||
|
)
|
||||||
|
wheel.writestr(f"{info}/RECORD", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_listed_package_brings_its_dependencies(
|
||||||
|
venv: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
):
|
||||||
|
wheels = tmp_path / "wheels"
|
||||||
|
wheels.mkdir()
|
||||||
|
_wheel(wheels, "fluksio_top", requires="fluksio_dep")
|
||||||
|
_wheel(wheels, "fluksio_dep")
|
||||||
|
monkeypatch.setenv("UV_OFFLINE", "1")
|
||||||
|
|
||||||
|
ok, output = modules.sync(f"--find-links {wheels}\nfluksio-top\n")
|
||||||
|
|
||||||
|
assert ok, output
|
||||||
|
imported = subprocess.run(
|
||||||
|
[modules.venv_python(), "-c", "import fluksio_dep"], check=False
|
||||||
|
)
|
||||||
|
assert imported.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Adopting the venv Fluksio was installed into
|
# Adopting the venv Fluksio was installed into
|
||||||
#
|
#
|
||||||
|
|||||||
+12
-12
@@ -247,17 +247,17 @@ Binding the speaking node's `text` straight to `said` makes the panel repeat
|
|||||||
what it heard, which tests the loop before there is any logic in it.
|
what it heard, which tests the loop before there is any logic in it.
|
||||||
|
|
||||||
Both packages go into the node's environment, as described in
|
Both packages go into the node's environment, as described in
|
||||||
[Packages](#packages). A managed environment installs its manifest with
|
[Packages](#packages). In a managed environment, list them in **Modules**:
|
||||||
`uv pip sync`, which skips dependencies, so paste the fully resolved list into
|
|
||||||
**Modules**:
|
|
||||||
|
|
||||||
```sh
|
```text
|
||||||
printf 'faster-whisper\nav<19\npiper-tts\n' | uv pip compile --python-version 3.13 -
|
faster-whisper
|
||||||
|
av<19
|
||||||
|
piper-tts
|
||||||
```
|
```
|
||||||
|
|
||||||
The Docker image runs Python 3.13. `av<19` holds PyAV back, since faster-whisper
|
`av<19` holds PyAV back, since faster-whisper 1.2.1 cannot open files with
|
||||||
1.2.1 cannot open files with PyAV 19. In your own venv or on a remote worker,
|
PyAV 19. In your own venv or on a remote worker,
|
||||||
`pip install faster-whisper "av<19" piper-tts` is enough.
|
`pip install faster-whisper "av<19" piper-tts` does the same.
|
||||||
|
|
||||||
The models land in `~/.cache/voice` on the machine running the node. The Docker
|
The models land in `~/.cache/voice` on the machine running the node. The Docker
|
||||||
image keeps its home directory inside the container, so a recreated container
|
image keeps its home directory inside the container, so a recreated container
|
||||||
@@ -343,10 +343,10 @@ already has a managed one. Without an existing venv, it creates a managed
|
|||||||
environment. `NODE_VENV=managed` always requests one; the Docker image uses
|
environment. `NODE_VENV=managed` always requests one; the Docker image uses
|
||||||
this setting.
|
this setting.
|
||||||
|
|
||||||
In managed mode, the Modules manifest is installed with `uv pip sync` and
|
In managed mode, the Modules manifest is resolved with `uv pip compile`,
|
||||||
versioned with the flows. Sync installs only the packages listed, not their
|
installed with `uv pip sync` and versioned with the flows. Listing a package
|
||||||
dependencies, so list those too, for example as resolved by `uv pip compile`.
|
brings its dependencies along. Applying it refreshes local workers. The API
|
||||||
Applying it refreshes local workers. The API accepts the same manifest:
|
accepts the same manifest:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
curl -X POST "$FLUKSIO/modules/apply" -H "Authorization: Bearer $TOKEN" \
|
curl -X POST "$FLUKSIO/modules/apply" -H "Authorization: Bearer $TOKEN" \
|
||||||
|
|||||||
@@ -32,11 +32,10 @@ the flow repository. Using references keeps credentials out of saved flows.
|
|||||||
packages and Python version. There are two setups:
|
packages and Python version. There are two setups:
|
||||||
|
|
||||||
- **Managed environment:** Docker uses a separate node environment. A
|
- **Managed environment:** Docker uses a separate node environment. A
|
||||||
superuser edits **Requirements** and presses **Apply**. Packages are synced
|
superuser edits **Requirements** and presses **Apply**. The list is resolved
|
||||||
with `uv pip sync`; deleting a requirement removes it. Sync installs only
|
with `uv pip compile` and synced with `uv pip sync`, so each package brings
|
||||||
what is listed, so list the dependencies too, for example the output of
|
its dependencies and deleting a requirement removes it. The manifest is
|
||||||
`uv pip compile`. The manifest is versioned with the flows, and successful
|
versioned with the flows, and successful installs refresh the workers.
|
||||||
installs refresh the workers.
|
|
||||||
- **Adopted environment:** a fresh pip installation inside your own virtual
|
- **Adopted environment:** a fresh pip installation inside your own virtual
|
||||||
environment can use that environment for nodes too. Modules is read-only.
|
environment can use that environment for nodes too. Modules is read-only.
|
||||||
Install dependencies there with pip or uv, then sync or restart Fluksio.
|
Install dependencies there with pip or uv, then sync or restart Fluksio.
|
||||||
|
|||||||
Reference in New Issue
Block a user