diff --git a/backend/fluksio/flow/modules.py b/backend/fluksio/flow/modules.py index cbb9598..ebee5a7 100644 --- a/backend/fluksio/flow/modules.py +++ b/backend/fluksio/flow/modules.py @@ -6,9 +6,12 @@ into a venv on the data volume rather than into the engine's environment: a pin here can never shadow — or be shadowed by — what the app itself runs on, and the worker processes that import them need nothing from the app. -``uv pip sync`` rather than install, so a line taken out of the manifest is -uninstalled. The manifest is written only after a sync succeeds, which is all -the rollback a failed resolve needs. +The manifest is resolved with ``uv pip compile`` first, so a line naming a +package brings its dependencies along, and a manifest that already pins +everything resolves to itself. The result goes to ``uv pip sync`` rather than +install, so a line taken out of the manifest is uninstalled. The manifest is +written only after a sync succeeds, which is all the rollback a failed resolve +needs. None of that applies to an *adopted* venv. A data scientist makes a venv, installs what they work with, and then installs Fluksio into it too — at which @@ -179,35 +182,52 @@ def sync(requirements: str) -> tuple[bool, str]: def _sync(requirements: str) -> tuple[bool, str]: - """The install itself, under `_sync_lock`.""" - manifest = "" + """The resolve and install, under `_sync_lock`.""" try: ensure_venv() - with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as handle: - handle.write(requirements) - manifest = handle.name - result = subprocess.run( - # Empty means empty: without the flag uv refuses to clear a venv, - # so deleting the last line would leave the package installed. - [ - uv_bin(), - "pip", - "sync", - "--allow-empty-requirements", - "--python", - venv_python(), - manifest, - ], - capture_output=True, - text=True, - timeout=SYNC_TIMEOUT, - ) + python = venv_python() + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory, "requirements.in") + lock = Path(directory, "requirements.txt") + manifest.write_text(requirements) + for command in ( + # The emit flags carry the manifest's index, find-links and + # binary options into the lock, which is all sync reads. + [ + uv_bin(), + "pip", + "compile", + "--quiet", + "--emit-index-url", + "--emit-find-links", + "--emit-build-options", + "--python", + python, + str(manifest), + "--output-file", + str(lock), + ], + # Empty means empty: without the flag uv refuses to clear a + # venv, so deleting the last line would leave the package + # installed. + [ + uv_bin(), + "pip", + "sync", + "--allow-empty-requirements", + "--python", + python, + str(lock), + ], + ): + result = subprocess.run( + command, capture_output=True, text=True, timeout=SYNC_TIMEOUT + ) + if result.returncode != 0: + break except (OSError, subprocess.SubprocessError) as exc: detail = str(getattr(exc, "stderr", None) or exc).strip() return False, f"uv could not run: {detail}" - finally: - if manifest: - Path(manifest).unlink(missing_ok=True) output = (result.stdout + result.stderr).strip() if result.returncode == 0: diff --git a/backend/tests/flow/test_modules.py b/backend/tests/flow/test_modules.py index 1d3d991..c2401cd 100644 --- a/backend/tests/flow/test_modules.py +++ b/backend/tests/flow/test_modules.py @@ -2,6 +2,7 @@ import shutil import subprocess +import zipfile from pathlib import Path import pytest @@ -77,6 +78,40 @@ def test_a_manifest_that_does_not_resolve_leaves_the_venv_alone(venv: Path): assert (venv / ".applied").read_text() == modules._digest("") +def _wheel(directory: Path, name: str, requires: str = "") -> None: + """A wheel with nothing in it but a name and, optionally, one dependency.""" + info = f"{name}-1.0.dist-info" + metadata = f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n" + if requires: + metadata += f"Requires-Dist: {requires}\n" + with zipfile.ZipFile(directory / f"{name}-1.0-py3-none-any.whl", "w") as wheel: + wheel.writestr(f"{name}/__init__.py", "") + wheel.writestr(f"{info}/METADATA", metadata) + wheel.writestr( + f"{info}/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr(f"{info}/RECORD", "") + + +def test_a_listed_package_brings_its_dependencies( + venv: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + wheels = tmp_path / "wheels" + wheels.mkdir() + _wheel(wheels, "fluksio_top", requires="fluksio_dep") + _wheel(wheels, "fluksio_dep") + monkeypatch.setenv("UV_OFFLINE", "1") + + ok, output = modules.sync(f"--find-links {wheels}\nfluksio-top\n") + + assert ok, output + imported = subprocess.run( + [modules.venv_python(), "-c", "import fluksio_dep"], check=False + ) + assert imported.returncode == 0 + + # --------------------------------------------------------------------------- # Adopting the venv Fluksio was installed into # diff --git a/docs/code/nodes.md b/docs/code/nodes.md index f65c9a3..cee3609 100644 --- a/docs/code/nodes.md +++ b/docs/code/nodes.md @@ -247,17 +247,17 @@ Binding the speaking node's `text` straight to `said` makes the panel repeat what it heard, which tests the loop before there is any logic in it. Both packages go into the node's environment, as described in -[Packages](#packages). A managed environment installs its manifest with -`uv pip sync`, which skips dependencies, so paste the fully resolved list into -**Modules**: +[Packages](#packages). In a managed environment, list them in **Modules**: -```sh -printf 'faster-whisper\nav<19\npiper-tts\n' | uv pip compile --python-version 3.13 - +```text +faster-whisper +av<19 +piper-tts ``` -The Docker image runs Python 3.13. `av<19` holds PyAV back, since faster-whisper -1.2.1 cannot open files with PyAV 19. In your own venv or on a remote worker, -`pip install faster-whisper "av<19" piper-tts` is enough. +`av<19` holds PyAV back, since faster-whisper 1.2.1 cannot open files with +PyAV 19. In your own venv or on a remote worker, +`pip install faster-whisper "av<19" piper-tts` does the same. The models land in `~/.cache/voice` on the machine running the node. The Docker image keeps its home directory inside the container, so a recreated container @@ -343,10 +343,10 @@ already has a managed one. Without an existing venv, it creates a managed environment. `NODE_VENV=managed` always requests one; the Docker image uses this setting. -In managed mode, the Modules manifest is installed with `uv pip sync` and -versioned with the flows. Sync installs only the packages listed, not their -dependencies, so list those too, for example as resolved by `uv pip compile`. -Applying it refreshes local workers. The API accepts the same manifest: +In managed mode, the Modules manifest is resolved with `uv pip compile`, +installed with `uv pip sync` and versioned with the flows. Listing a package +brings its dependencies along. Applying it refreshes local workers. The API +accepts the same manifest: ```sh curl -X POST "$FLUKSIO/modules/apply" -H "Authorization: Bearer $TOKEN" \ diff --git a/docs/interface/operations.md b/docs/interface/operations.md index 8a57d22..9db94d8 100644 --- a/docs/interface/operations.md +++ b/docs/interface/operations.md @@ -32,11 +32,10 @@ the flow repository. Using references keeps credentials out of saved flows. packages and Python version. There are two setups: - **Managed environment:** Docker uses a separate node environment. A - superuser edits **Requirements** and presses **Apply**. Packages are synced - with `uv pip sync`; deleting a requirement removes it. Sync installs only - what is listed, so list the dependencies too, for example the output of - `uv pip compile`. The manifest is versioned with the flows, and successful - installs refresh the workers. + superuser edits **Requirements** and presses **Apply**. The list is resolved + with `uv pip compile` and synced with `uv pip sync`, so each package brings + its dependencies and deleting a requirement removes it. The manifest is + versioned with the flows, and successful installs refresh the workers. - **Adopted environment:** a fresh pip installation inside your own virtual environment can use that environment for nodes too. Modules is read-only. Install dependencies there with pip or uv, then sync or restart Fluksio.