Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Init-login-byok-or-hosted · L-260918-58e1d3 (#1367)

Louis Choquela6f7141d6bfe
diff --git a/.badges/tests.json b/.badges/tests.json
index f2e0817cb..70e0da708 100644
--- a/.badges/tests.json
+++ b/.badges/tests.json
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"label": "tests",
- "message": "15816",
+ "message": "16496",
"color": "blue",
"cacheSeconds": 300
}
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml
index 4a4ce2cbd..078ce8e17 100644
--- a/.drift/acks/cli-docs.toml
+++ b/.drift/acks/cli-docs.toml
@@ -1,29 +1,29 @@
contract = "cli-docs"
-digest = "sha256:35f1ecf9aa9cfa54726f3badc961f2efe548559cb1b2c8966f1bf22a25f4e0de"
+digest = "sha256:1753075a19fa8a93ee26f9bb162c2f085ebb07d90d565e0690a8c9f356bb026f"
reviewed_by = "Claude (Opus 5.5)"
-reviewed_at = "2026-10-08T12:45:01Z"
-rationale = "clean-pass: merged tree of origin/dev into this branch. The open trigger changes are all this branch's hosted run surface, the run commands of both CLIs, _run_hosted.py, _hosted_sources.py and the removed _run_core_api.py, already answered on the branch in docs/tools/cli/run.md, run-by-address.md, agent-cli.md and pipelex/cli/agent_cli/CLAUDE.md. Dev touched no file under docs/tools/cli and no run command; its one CLI trigger, init_config delegating to copy_kit_templates, keeps the review dev recorded, and docs/tools/cli/init.md still describes it. The merged docs name --hosted, --local, --base-url and --runner local or hosted, and nothing under docs/tools/cli still names the removed --runner api path except the warning that says it is gone."
+reviewed_at = "2026-10-08T13:14:35Z"
+rationale = "clean-pass: merge of the reviewed tip of feature/Pipelex-runs-hosted-via into this branch, the ack file taken from that tip. The merge changes no trigger file: every trigger blob equals the one this branch last acked, and the open changes are this branch's own work since that tip, the login command, init's question of where runs execute with its setup_path module, and the former-release cleanup across migrate, doctor, init, the CLI factories, agent_output and error_handlers. The merged docs answer each: login.md covers the browser and paste flows, the 0600 home .env and PIPELEX_APP_URL; init.md describes the hosted or local question written to the run execution setting and the former-release cleanup it offers first; migrate.md and agent-cli.md describe the cleanup, FormerReleaseConfigError and the former_release key, and agent-cli.md and the agent CLI CLAUDE.md list execution in the init config and the hosted doctor verdict. Nothing under docs/tools/cli is stale against the merged CLI."
[trigger_files]
"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
-"pipelex/cli/_cli.py" = "blob:1ef9cd89b6044b9b510fad81661872a43d9cc21e"
+"pipelex/cli/_cli.py" = "blob:404604dc7bb89f0eb5e89773b9f4f631dc0c6fac"
"pipelex/cli/agent_cli/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/agent_cli/_agent_cli.py" = "blob:d3aa51f4472da74c48de39d5b15ee3f6a2fe16a8"
"pipelex/cli/agent_cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/agent_cli/commands/agent_cli_factory.py" = "blob:f96268c610cdd3b7328919768d1aef1b8ba4e084"
-"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:a3651bb1023350e358d7865c3e9d401b3408b5c7"
+"pipelex/cli/agent_cli/commands/agent_cli_factory.py" = "blob:947ac4b789f2800811d653dc30a63419336c1b5d"
+"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:e37898df47cfbb5b6d0ec42468be31463f82bff1"
"pipelex/cli/agent_cli/commands/bundle_path_resolver.py" = "blob:a1904517c731afb72af7d49adf6d217dc50b2660"
"pipelex/cli/agent_cli/commands/check_model_cmd.py" = "blob:3a9c93daecadf266b3eba466904b3d005d448d40"
"pipelex/cli/agent_cli/commands/codegen/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/agent_cli/commands/codegen/app.py" = "blob:56130b89fe9efec5ad3d014ed6bc16bef1bd1645"
"pipelex/cli/agent_cli/commands/codegen/check_cmd.py" = "blob:9e8beb084a9f04f39e8acca9e6ae2554d40cb667"
"pipelex/cli/agent_cli/commands/codegen/types_cmd.py" = "blob:5da4131cb459383ab4007c2cdb15f3cfde74510f"
-"pipelex/cli/agent_cli/commands/doctor_cmd.py" = "blob:ad2de92b98c2ab4a26e74d6154da1f594b5820b6"
+"pipelex/cli/agent_cli/commands/doctor_cmd.py" = "blob:948be63cf4fab165846c83b4bd2517bd4a46be5e"
"pipelex/cli/agent_cli/commands/fix/__init__.py" = "blob:65662c86f17f610ad55ecfc3e27a022b9aa26e8b"
"pipelex/cli/agent_cli/commands/fix/app.py" = "blob:e6190d9bde6a05755c04bd21806f699461038ac0"
"pipelex/cli/agent_cli/commands/fix/bundle_cmd.py" = "blob:cfd9e645986636c44c02a6a04a0fc67ed60d444b"
"pipelex/cli/agent_cli/commands/fmt_cmd.py" = "blob:97d6fb6c328e0b718b1293bd6b73cba03edcb7dc"
-"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:535f77e3bdac38b77c35125ea48b5f2b47eb9b3f"
+"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:e8ff262f4c0a0e1bd1cf75d4008276b8d5552169"
"pipelex/cli/agent_cli/commands/inputs/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/agent_cli/commands/inputs/_inputs_core.py" = "blob:4161029643f3a302de54fa637d2676521e10d873"
"pipelex/cli/agent_cli/commands/inputs/app.py" = "blob:7e19284687b334c684fb03381a8d9d20af1aa152"
@@ -31,7 +31,7 @@ rationale = "clean-pass: merged tree of origin/dev into this branch. The open tr
"pipelex/cli/agent_cli/commands/inputs/method_cmd.py" = "blob:604e91b2e65d951fd9a6eeed924a01f04fe2096f"
"pipelex/cli/agent_cli/commands/inputs/pipe_cmd.py" = "blob:5c9c233b00d626599dea4907b148b4f08cdd58a0"
"pipelex/cli/agent_cli/commands/lint_cmd.py" = "blob:91ad9cb4d618ba4c0b7da366cde2f74f38d350df"
-"pipelex/cli/agent_cli/commands/migrate_cmd.py" = "blob:128de81f63aee55a9901e1df7a11ce15a402c536"
+"pipelex/cli/agent_cli/commands/migrate_cmd.py" = "blob:dbe1f689607ccea0b1f5f1dadca20f19f65ce400"
"pipelex/cli/agent_cli/commands/models_cmd.py" = "blob:f48a921fa89d8275c585b3e15f135f5585ac01aa"
"pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:a1c530d110482485ce462f199b9e79fac29f8896"
"pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
@@ -50,7 +50,7 @@ rationale = "clean-pass: merged tree of origin/dev into this branch. The open tr
"pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:0f4cf76d99b39422610c5daf6b5a0500de234818"
"pipelex/cli/agent_cli/commands/validate/pipe_cmd.py" = "blob:9a315ab32c89430c8ea111a0a50e629ed1acd6bf"
"pipelex/cli/bundle_target_resolution.py" = "blob:460aa997ea0886c6faee0f3a42bd55231c480240"
-"pipelex/cli/cli_factory.py" = "blob:315842c19e173ccb7f57d0c5fee955154624c0dd"
+"pipelex/cli/cli_factory.py" = "blob:310ad1a2d055469cdc4f4728c82b9e83a9ff5852"
"pipelex/cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/build/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/build/app.py" = "blob:d3205e1dc0268a9d32a517e4f9a9a3b4647e8889"
@@ -74,7 +74,7 @@ rationale = "clean-pass: merged tree of origin/dev into this branch. The open tr
"pipelex/cli/commands/codegen/inputs_cmd.py" = "blob:28b22cfbca717d3f5ee75162b68e132439655deb"
"pipelex/cli/commands/codegen/types_cmd.py" = "blob:27d7a06d571ad832b449d21b6f9eb66c017fb197"
"pipelex/cli/commands/crate_loading.py" = "blob:3d7d651931ecf1a15a3e2d72411b3212b55d74ef"
-"pipelex/cli/commands/doctor_cmd.py" = "blob:33937d62ee5168690dae5146611ade0d6c44c126"
+"pipelex/cli/commands/doctor_cmd.py" = "blob:c5944ed93b9a33cac01db9e8fa996a763cc8048b"
"pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/fix/_diff_sandbox.py" = "blob:d39c0df92b95567f74881deb01f8c1b3119fc8ef"
"pipelex/cli/commands/fix/_fix_core.py" = "blob:aa698305e6b60dfb7caf133698dc1d5c1f5f68f1"
@@ -83,18 +83,24 @@ rationale = "clean-pass: merged tree of origin/dev into this branch. The open tr
"pipelex/cli/commands/graph_cmd.py" = "blob:d54737af7efc9ab9bbf3f978050d017af0e27a41"
"pipelex/cli/commands/init/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/init/backends.py" = "blob:04a1a06f1003ed0c7157ec4ca6c8659e4e09fe6b"
-"pipelex/cli/commands/init/command.py" = "blob:f17d58fe1813d4bb9552cc418914ea4aebae2d2a"
-"pipelex/cli/commands/init/config_files.py" = "blob:8d1ea8e03e8117de5264d4bc046e5bc259fb1342"
-"pipelex/cli/commands/init/credentials.py" = "blob:a9c3ee9e8bdcc5d0f3148f055ab6e2ff34a940fc"
+"pipelex/cli/commands/init/command.py" = "blob:f5cbca22f8ea19957f271af2d9844d9191263520"
+"pipelex/cli/commands/init/config_files.py" = "blob:8345ae46e7f77a6f49f526ed20c8dd8263f118f3"
+"pipelex/cli/commands/init/credentials.py" = "blob:1817fd9b1988e4fea528a3977259b57f7274b69b"
"pipelex/cli/commands/init/ide_extension.py" = "blob:9f0ec1494c07062246ecc906bc55b7f2cf95c715"
"pipelex/cli/commands/init/routing.py" = "blob:c73927deb99804556cb545cdaa6eabf8d1fc3450"
+"pipelex/cli/commands/init/setup_path.py" = "blob:6485fae3ee6dd1b112badffdc956f1b734dc1b10"
"pipelex/cli/commands/init/telemetry.py" = "blob:07a33b6540e1fc4146d0777463c93abd4ea70231"
"pipelex/cli/commands/init/ui/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/init/ui/backends_ui.py" = "blob:a5250ebf7f2865da6ca12fc292b75fa7f5e8a690"
-"pipelex/cli/commands/init/ui/general_ui.py" = "blob:4496bb2b88cf42e09efa4b6ba44c592373a314c6"
+"pipelex/cli/commands/init/ui/general_ui.py" = "blob:ef412cbb03d1c6c97295131c381eaf84895193cd"
"pipelex/cli/commands/init/ui/routing_ui.py" = "blob:640bd0d11d1879ead83fd9f3d1736d2735c1abfa"
+"pipelex/cli/commands/init/ui/setup_path_ui.py" = "blob:49a2a1333eca05aa58589e1f2a4b9b9af47eb740"
"pipelex/cli/commands/init/ui/types.py" = "blob:e263802d8e9c9f9687435de1eb992afa5336544f"
-"pipelex/cli/commands/migrate_cmd.py" = "blob:897f9cbf93acace8f61091a9924895f0fda76443"
+"pipelex/cli/commands/login/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
+"pipelex/cli/commands/login/api_key_store.py" = "blob:3a64ee921b54131dcba8c829880c060b5195e2ed"
+"pipelex/cli/commands/login/command.py" = "blob:ee5f422b91acadaeadfae7f3a9b1541f1621de94"
+"pipelex/cli/commands/login/loopback.py" = "blob:b66eaa021caa1270a64237169f5a2585267f488c"
+"pipelex/cli/commands/migrate_cmd.py" = "blob:3839424ccbf1c7f8e7af8a7f124a062cba18cece"
"pipelex/cli/commands/plugins_cmd.py" = "blob:9657b713c3650ff4a929e92ae68af453d896393e"
"pipelex/cli/commands/resolve_cmd.py" = "blob:88237df0f048bbf5a02e585baef932bfc355e1ee"
"pipelex/cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
@@ -119,7 +125,7 @@ rationale = "clean-pass: merged tree of origin/dev into this branch. The open tr
"pipelex/cli/commands/validate/pipe_cmd.py" = "blob:791b9a693f4ec52f727b42dab4b102af46a4ee22"
"pipelex/cli/commands/which_cmd.py" = "blob:ed45019f8acd3678340fe395c49d065d6fc6c04d"
"pipelex/cli/deck_notice.py" = "blob:542877125e89e35b02bc969cfee952155b0d7c2c"
-"pipelex/cli/error_handlers.py" = "blob:dd01a16295c17cc06378d3290ad5e60c27de986f"
+"pipelex/cli/error_handlers.py" = "blob:92d233152460052a423f021e8252fbce872a9de4"
"pipelex/cli/exceptions.py" = "blob:1d51c3a53ddd3f6cf8de566fa2ade3fa75ab7a3a"
"pipelex/cli/installed_methods.py" = "blob:f7818b7857a8350389cf196e32630e7b3c51818c"
"pipelex/cli/method_resolver.py" = "blob:afaf77b2d8b6a3467d6ab1f355b1ee78a71eafa7"
diff --git a/.env.example b/.env.example
index 9fc01baea..b558c1f73 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,7 @@
# The hosted Pipelex API: `pipelex run --hosted` (or `[run] execution = "hosted"`) runs on it with this key alone,
-# no provider key needed. Create one on app.pipelex.com.
-PIPELEX_API_KEY=
+# no provider key needed. `pipelex login` gets one through your browser and saves it to ~/.pipelex/.env; set it here
+# only to use another key in this directory, since this file is loaded after that one and wins, even when left empty.
+# PIPELEX_API_KEY=
# Override the hosted API's origin (scheme://host[:port]), for a staging plane or a self-hosted Pipelex API server;
# unset means https://api.pipelex.com, and --base-url on the command overrides it in turn.
# PIPELEX_BASE_URL=https://api.pipelex.com
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cd358c6cc..fc2d70bf5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,9 @@
### Added
- **Runs on the hosted Pipelex API, `--runner api` removed (Breaking)**: `pipelex run method|pipe|bundle` and `pipelex-agent run` take `--hosted` / `--local` and `--base-url <scheme://host[:port]>` (over `PIPELEX_BASE_URL`, over `https://api.pipelex.com`), and the new `[run] execution = "local" | "hosted"` setting (default `local`) chooses when no flag does; a hosted run boots nothing locally, needs only `PIPELEX_API_KEY`, sends what a local run would load (`PIPELEXPATH` included) or a published address or a catalog id (`mt_…`) for the hosted API to resolve, uploads the local files named at document and image inputs (reading the method's signature only when the inputs name an existing local file), saves its outputs where a local run does, prints the run id as soon as the run is acknowledged and names it in any failure met after that, Ctrl-C included, and reports a connection lost after the run request was sent as a run that may exist rather than as a network failure. The agent CLI's global `--runner pipelex|api` and its mthds `MthdsAPIClient` path are removed: `pipelex-agent run … --runner local|hosted` replaces it, on pipelex-sdk's `PipelexAPIClient`, which `pipelex` now requires (pinned exactly, like `mthds`) and which Python callers use directly to run on the hosted API.
+- **`pipelex login`**: gets a Pipelex API key through the browser, with a one-time `state` value the app must echo, or from a hidden prompt with `--paste`, checks it against the hosted API, and saves it to `~/.pipelex/.env` as `PIPELEX_API_KEY` (mode 0600, other lines kept, a symlinked file written through, the key never printed), naming a working-directory `.env` whose own `PIPELEX_API_KEY` line would override it, and exiting 1 with the path and the error when the file cannot be written; `PIPELEX_APP_URL` aims it at another Pipelex app.
+- **`pipelex init` asks where runs execute**: Enter takes the hosted Pipelex API, which signs you in unless a key is already set and asks for no provider key, and the other answer runs today's backend, routing and credential steps; the answer is written to `[run] execution`, with a warning when the working directory's project overrides it. `pipelex doctor --fix` keeps a `[run] execution` already set, keeps a `pipelex.toml` that sets none local, and takes the hosted default only for a brand-new home, `pipelex doctor` and `pipelex-agent doctor` judge a hosted setup by its Pipelex API key and report provider credentials as needed only for `--local` runs, and `pipelex-agent init` takes `"execution": "hosted"` in `--config` and reports `execution`.
+- **`pipelex migrate` cleans up what a former release left for the Pipelex Gateway**: a configuration set up by a release that ran on the Pipelex Gateway is refused at boot with the new `FormerReleaseConfigError`, which names what stops it and `pipelex migrate`. That command and `pipelex-agent migrate` (under the new `former_release` key) remove the Gateway's backend, routing profiles, routes and files and leave Pipelex Manifold's, which is not retired; they keep a copy of each file they change or remove, and report what they leave for a hand edit or what would still stop Pipelex from starting. `pipelex init` and `pipelex doctor --fix` offer the same cleanup, and `pipelex doctor` lists the files.
### Fixed
diff --git a/README.md b/README.md
index e2a2c3e5f..9682d5a8d 100644
--- a/README.md
+++ b/README.md
@@ -86,7 +86,7 @@ Prefer to run it yourself? This repository is the Pipelex runtime — its own in
## Run it yourself
-This repository is the Pipelex runtime: the Python package that reads a `.mthds` file and runs it. Install it and everything happens on your own machine, against the model providers you choose.
+This repository is the Pipelex runtime: the Python package that reads a `.mthds` file and runs it. Install it and run methods from your terminal, either on the hosted Pipelex API, with no provider account of your own, or on your own machine, against the model providers you choose.
### Install
@@ -96,7 +96,12 @@ pipelex init
pipelex doctor
```
-`pipelex init` writes your `~/.pipelex` configuration and offers to install the editor extension; `pipelex doctor` reports what is configured and what is missing. The `cli` extra installs Rich, which the `pipelex` and `pipelex-agent` commands render their output through.
+`pipelex init` writes your `~/.pipelex` configuration, offers to install the editor extension, and asks where your runs execute:
+
+- **On the hosted Pipelex API**, the default that Enter takes: no provider account is needed. `pipelex init` signs you in through your browser and saves a Pipelex API key to `~/.pipelex/.env`. `pipelex login` does that again on its own, and `pipelex login --paste` takes a key on a machine without a browser.
+- **On this machine, with your own provider keys**: you choose your AI providers and enter their keys, or point Pipelex at a local model.
+
+`pipelex doctor` reports what is configured and what is missing. The `cli` extra installs Rich, which the `pipelex` and `pipelex-agent` commands render their output through.
Some providers and features need an extra:
@@ -118,6 +123,7 @@ uv tool install "pipelex[cli,anthropic,google,google-genai,mistralai,bedrock,fal
### Configure AI Access
+- **The hosted API** — Choose it in `pipelex init`, which signs you in and makes it the default, to run with no provider key on this machine. On a machine set up for local runs, `pipelex login` and `--hosted` send one run there.
- **Bring Your Own Keys** — Use existing API keys from OpenAI, Anthropic, Google, Mistral, etc. See [Configure AI Providers](https://docs.pipelex.com/latest/setup/configure-ai-providers/).
- **Local AI** — Ollama, vLLM, LM Studio, or llama.cpp — no API keys required. See [Configure AI Providers](https://docs.pipelex.com/latest/setup/configure-ai-providers/).
@@ -132,14 +138,14 @@ Save the method shown under [What a method looks like](#what-a-method-looks-like
}
```
-The method names no model, so it runs on the deck's `default-general` alias, which `pipelex init` points at an OpenAI model. With a provider other than OpenAI or Azure OpenAI, point that alias at one of your provider's models first, by adding it to `~/.pipelex/inference/deck/x_custom_llm_deck.toml` — the models each provider serves are listed under `~/.pipelex/inference/backends/`:
+On the hosted Pipelex API, it runs as it is. On this machine, the method names no model, so it runs on the deck's `default-general` alias, which `pipelex init` points at an OpenAI model. With a provider other than OpenAI or Azure OpenAI, point that alias at one of your provider's models first, by adding it to `~/.pipelex/inference/deck/x_custom_llm_deck.toml` — the models each provider serves are listed under `~/.pipelex/inference/backends/`:
```toml
[llm.aliases]
default-general = "claude-5-sonnet" # an Anthropic model, for example
```
-Then run it:
+Then run it, where `pipelex init` set your runs to execute:
```bash
pipelex run bundle summarize.mthds --inputs inputs.json
diff --git a/docs/configuration/config-practical/run-config.md b/docs/configuration/config-practical/run-config.md
index f12f7f908..ab8fac4df 100644
--- a/docs/configuration/config-practical/run-config.md
+++ b/docs/configuration/config-practical/run-config.md
@@ -26,6 +26,8 @@ class RunConfig(ConfigModel):
A flag on the command always wins: with `execution = "hosted"`, `pipelex run … --local` still runs on this machine, and with `"local"`, `--hosted` sends that one run to the hosted API.
+[`pipelex init`](../../tools/cli/init.md#where-your-runs-execute) writes this setting from your answer to "Where should your runs execute?", and `pipelex-agent init` from the `execution` of its `--config`.
+
## Example Configuration
```toml
@@ -39,7 +41,7 @@ Put it in your project's `.pipelex/pipelex.toml` to make hosted runs the default
A hosted run reads its key and its origin from the environment, never from this file:
-- `PIPELEX_API_KEY` holds the Pipelex API key (`plx_sk_…`). Export it in your shell, or save it in `~/.pipelex/.env`, which Pipelex loads at startup.
+- `PIPELEX_API_KEY` holds the Pipelex API key (`plx_sk_…`). [`pipelex login`](../../tools/cli/login.md) gets one through your browser and saves it in `~/.pipelex/.env`, which Pipelex loads at startup; you can also export it in your shell.
- `PIPELEX_BASE_URL` overrides the origin, for a staging plane or a self-hosted Pipelex API server. `--base-url` on the command overrides it in turn. Either must be `scheme://host[:port]`, with no path such as `/v1`.
## Related Documentation
diff --git a/docs/configuration/config-technical/inference-backend-config.md b/docs/configuration/config-technical/inference-backend-config.md
index 5b34b4fd7..ca29b2e79 100644
--- a/docs/configuration/config-technical/inference-backend-config.md
+++ b/docs/configuration/config-technical/inference-backend-config.md
@@ -171,6 +171,8 @@ A variable may stand only in a value that a call sends: `api_key`, `endpoint` an
An enabled backend must declare at least one model in its file under `backends/` (see [Model Specifications](#model-specifications)). Pipelex refuses to boot when one declares none, because routing would send models to a backend that cannot serve them: disable the backend, or list the models it serves. The `internal` backend is exempt, since plugins add its models at boot. A backend table also no longer accepts a `model_specs_section` key; an enabled backend that still carries one is refused, and the fix is to list its models in `backends/<name>.toml` and remove the key, or to disable the backend.
+The `pipelex_gateway` backend that releases up to v0.72 shipped is gone. A configuration those releases set up, with it enabled, a routing profile such as `all_pipelex_gateway` active, or a `model_specs_section` key left behind, is refused with one error naming [`pipelex migrate`](../../tools/cli/migrate.md#a-configuration-a-former-release-set-up), which removes what they left, keeps a copy of each file it changes, and makes `all_enabled_backends` the active profile. The `pipelex_manifold` backend is not retired: what those releases left for it, a table they shipped disabled, its file under `backends/` and the `all_pipelex_manifold` profile, stays as it is.
+
Judgment models are served by their own backend alone, so the default routing profile sends them to their own backend through an optional route (`"jev-*" = "typesafe"`), which applies only while that backend is enabled. With a `TYPESAFE_API_KEY` set, `@default-judgment` works under the default profile with no routing edit.
### Model Specifications
diff --git a/docs/errors/former-release-config-error.md b/docs/errors/former-release-config-error.md
new file mode 100644
index 000000000..542c78f0b
--- /dev/null
+++ b/docs/errors/former-release-config-error.md
@@ -0,0 +1,21 @@
+---
+title: "Configuration left by a former release"
+description: "Reference for the `FormerReleaseConfigError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Configuration left by a former release
+
+The inference configuration still carries what a former release wrote for the Pipelex Gateway.
+
+| Field | Value |
+|---|---|
+| `error_type` | `FormerReleaseConfigError` |
+| `title` | Configuration left by a former release |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/former-release-config-error/` |
+| `error_domain` | _(inherited from parent)_ |
+| Defined in | `pipelex.migration.exceptions` |
+| Parent class | [`PipelexSetupError`](pipelex-setup-error.md) |
+
+[Back to Error Reference](index.md)
diff --git a/docs/errors/platform-and-tooling.md b/docs/errors/platform-and-tooling.md
index b1d35e67c..7c8ed9500 100644
--- a/docs/errors/platform-and-tooling.md
+++ b/docs/errors/platform-and-tooling.md
@@ -130,6 +130,7 @@ own page. Classes are grouped by subsystem.
## Migration
+- [`FormerReleaseConfigError`](former-release-config-error.md) — Configuration left by a former release
- [`MigrationError`](migration-error.md) — Migration error
- [`MigrationGoldenError`](migration-golden-error.md) — Migration golden error
- [`MigrationLedgerError`](migration-ledger-error.md) — Migration ledger error
diff --git a/docs/features/cli.md b/docs/features/cli.md
index 087289103..5466ddce6 100644
--- a/docs/features/cli.md
+++ b/docs/features/cli.md
@@ -15,9 +15,10 @@ The `pipelex` CLI is the primary tool for working with Pipelex methods. It cover
| Command | Description |
|---------|-------------|
-| **`pipelex init`** | Initialize configuration, backends, credentials, routing, and telemetry |
+| **`pipelex init`** | Initialize configuration, choose where runs execute (the hosted Pipelex API or this machine), and set up backends, credentials, routing, and telemetry |
+| **`pipelex login`** | Get a Pipelex API key through your browser, or paste one with `--paste`, and save it for runs on the hosted Pipelex API |
| **`pipelex update`** | Refresh the model deck and `backends/internal.toml` to match the installed pipelex version |
-| **`pipelex doctor`** | Check configuration health and suggest fixes |
+| **`pipelex doctor`** | Check configuration health and suggest fixes; a setup whose runs execute on the hosted Pipelex API needs its Pipelex API key and no provider key |
| **`pipelex build`** | Generate the structures of your concepts and example inputs and outputs for a pipe |
| **`pipelex validate`** | Check pipeline syntax, structure, and run dry-run validation |
| **`pipelex fix`** | Apply deterministic safe fixes to a bundle and re-validate (with `--diff` preview) |
@@ -46,7 +47,7 @@ The `pipelex-agent` CLI is a machine-first interface designed for automated envi
| Command | Description |
|---------|-------------|
-| `init` | Non-interactive configuration setup (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
+| `init` | Non-interactive configuration setup; `"execution": "hosted"` in `--config` sets up hosted runs (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
| `run` | Execute a pipeline (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
| `validate` | Validate pipes, bundles, or methods (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
| `fix` | Apply deterministic safe fixes to a bundle in place and re-validate (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
diff --git a/docs/get-started/configure-ai-providers.md b/docs/get-started/configure-ai-providers.md
index bc1dbd7e5..a559428fb 100644
--- a/docs/get-started/configure-ai-providers.md
+++ b/docs/get-started/configure-ai-providers.md
@@ -6,7 +6,7 @@ description: "Set up API keys for OpenAI, Anthropic, Mistral, OpenRouter and oth
## Configure API Access
-To run pipelines on your own machine, Pipelex needs access to AI models. **You have two options**: your own provider keys, or models running locally. Nothing reports to Pipelex either way. If you would rather hold no provider keys at all, the [Quick Start](./quick-start.md) runs methods on the hosted Pipelex API instead.
+To run pipelines on your own machine, Pipelex needs access to AI models. **You have two options**: your own provider keys, or models running locally. Nothing reports to Pipelex either way. This is the setup `pipelex init` runs when you answer **On this machine, with your own provider keys** to its question "Where should your runs execute?". If you would rather hold no provider keys at all, take its default answer, the hosted Pipelex API, which signs you in for a Pipelex API key instead (see [Run It Yourself](./run-it-yourself.md) and [`pipelex login`](../tools/cli/login.md)).
### Option 1: Bring Your Own API Keys
@@ -59,12 +59,14 @@ You only need to add keys for the providers you plan to use. A single OpenRouter
When using your own keys, enable the corresponding backends:
-1. Initialize configuration:
+1. Initialize configuration, and answer **2) On this machine, with your own provider keys** when it asks where your runs execute. It then asks which backends you have access to and prompts for their keys, saved to `~/.pipelex/.env`:
```bash
pipelex init
```
+ On a machine already set up, `pipelex init inference` reruns the backend selection alone, without changing where your runs execute.
+
2. Edit `~/.pipelex/inference/backends.toml` (or `.pipelex/inference/backends.toml` in your project root if you have a project-local config — the project file fully overrides the global one). To run on a backend of your own choosing without touching a tracked file, write the same keys in a git-ignored `backends_override.toml` beside it instead: see [Personal overrides](../configuration/config-technical/inference-backend-config.md#personal-overrides).
```toml
@@ -114,11 +116,11 @@ To set up Pipelex configuration files, run:
pipelex init
```
-By default, this creates the global `~/.pipelex/` directory with:
+By default, this creates the global `~/.pipelex/` directory, with the same files whichever answer you give to where your runs execute; the `inference/` files are what runs on this machine use:
```
~/.pipelex/
-├── pipelex.toml # Feature flags, logging, cost reporting
+├── pipelex.toml # Feature flags, logging, cost reporting, where runs execute
├── plxt.toml # MTHDS/TOML formatting and linting configuration
├── telemetry.toml # AI trace destinations (PostHog, Langfuse, OTLP)
└── inference/ # LLM configuration and model presets
diff --git a/docs/get-started/mthds-language-tutorial.md b/docs/get-started/mthds-language-tutorial.md
index 05b71a57e..b6304a050 100644
--- a/docs/get-started/mthds-language-tutorial.md
+++ b/docs/get-started/mthds-language-tutorial.md
@@ -16,7 +16,7 @@ pipelex init
The `cli` extra installs Rich, which the `pipelex` command renders its output through.
-`pipelex init` creates your project configuration. To run your methods, add your own provider keys to `~/.pipelex/.env`, or point Pipelex at a local model: see [Configure AI Providers](./configure-ai-providers.md).
+`pipelex init` creates your configuration and asks where your runs execute. Press Enter to run them on the hosted Pipelex API: it signs you in through your browser and saves a Pipelex API key, so you need no provider key of your own. To run them on this machine instead, answer `2`, then add your own provider keys or point Pipelex at a local model: see [Configure AI Providers](./configure-ai-providers.md).
!!! tip "VS Code Extension"
We **highly** recommend installing the Pipelex extension for `.mthds` syntax highlighting and flowchart visualization:
diff --git a/docs/get-started/run-it-yourself.md b/docs/get-started/run-it-yourself.md
index e307b70f6..030506a44 100644
--- a/docs/get-started/run-it-yourself.md
+++ b/docs/get-started/run-it-yourself.md
@@ -1,11 +1,11 @@
---
title: "Run It Yourself"
-description: "Install the Pipelex runtime and run MTHDS methods on your own machine, against the model providers you choose."
+description: "Install the Pipelex runtime and run MTHDS methods from your terminal: on the hosted Pipelex API with a Pipelex API key, or on your own machine against the model providers you choose."
---
# Run It Yourself
-The Pipelex runtime is the Python package that reads a `.mthds` file and runs it. Install it and everything happens on your own machine, against the model providers you choose. If you would rather not run anything yourself, the [Quick Start](./quick-start.md) builds methods with your coding agent and runs them on the hosted API.
+The Pipelex runtime is the Python package that reads a `.mthds` file and runs it. Install it and run methods from your terminal, either on the hosted Pipelex API, with no provider account of your own, or on your own machine, against the model providers you choose. If you would rather not install anything, the [Quick Start](./quick-start.md) builds methods with your coding agent and runs them on the hosted API.
## Install
@@ -15,7 +15,12 @@ pipelex init
pipelex doctor
```
-`pipelex init` writes your `~/.pipelex` configuration and offers to install the editor extension; `pipelex doctor` reports what is configured and what is missing. The `cli` extra installs Rich, which the `pipelex` and `pipelex-agent` commands render their output through.
+`pipelex init` writes your `~/.pipelex` configuration, offers to install the editor extension, and asks where your runs execute:
+
+- **On the hosted Pipelex API**, the default that Enter takes: no provider account is needed. `pipelex init` signs you in through your browser and saves a Pipelex API key to `~/.pipelex/.env`. [`pipelex login`](../tools/cli/login.md) does that again on its own, and `pipelex login --paste` takes a key on a machine without a browser.
+- **On this machine, with your own provider keys**: you choose your AI providers and enter their keys, or point Pipelex at a local model.
+
+`pipelex doctor` reports what is configured and what is missing. The `cli` extra installs Rich, which the `pipelex` and `pipelex-agent` commands render their output through.
Some providers and features need an extra:
@@ -37,9 +42,11 @@ uv tool install "pipelex[cli,anthropic,google,google-genai,mistralai,bedrock,fal
## Configure AI access
+- **The hosted API** — Choose it in `pipelex init`, which writes `[run] execution = "hosted"` and signs you in, to run on the hosted Pipelex API with no provider key on this machine. On a machine set up for local runs, `pipelex login` and `--hosted` send one run there. See [Running on the Hosted API](../tools/cli/run.md#running-on-the-hosted-api).
- **Bring Your Own Keys** — Use existing API keys from OpenAI, Anthropic, Google, Mistral, etc. See [Configure AI Providers](./configure-ai-providers.md).
- **Local AI** — Ollama, vLLM, LM Studio, or llama.cpp — no API keys required. See [Configure AI Providers](./configure-ai-providers.md).
-- **The hosted API** — Set `PIPELEX_API_KEY` and add `--hosted` to a run, or set `[run] execution = "hosted"`, to run on the hosted Pipelex API with no provider key on this machine. See [Running on the Hosted API](../tools/cli/run.md#running-on-the-hosted-api).
+
+A single run can always go the other way: `--hosted` sends it to the hosted API, `--local` keeps it on this machine.
## Run a method
@@ -66,14 +73,14 @@ Save its inputs as `inputs.json`:
}
```
-The method names no model, so it runs on the deck's `default-general` alias, which `pipelex init` points at an OpenAI model. With a provider other than OpenAI or Azure OpenAI, point that alias at one of your provider's models first, by adding it to `~/.pipelex/inference/deck/x_custom_llm_deck.toml` — the models each provider serves are listed under `~/.pipelex/inference/backends/`:
+On the hosted Pipelex API, it runs as it is. On this machine, the method names no model, so it runs on the deck's `default-general` alias, which `pipelex init` points at an OpenAI model. With a provider other than OpenAI or Azure OpenAI, point that alias at one of your provider's models first, by adding it to `~/.pipelex/inference/deck/x_custom_llm_deck.toml` — the models each provider serves are listed under `~/.pipelex/inference/backends/`:
```toml
[llm.aliases]
default-general = "claude-5-sonnet" # an Anthropic model, for example
```
-Then run it:
+Then run it, where `pipelex init` set your runs to execute:
```bash
pipelex run bundle summarize.mthds --inputs inputs.json
diff --git a/docs/index.md b/docs/index.md
index 8893b940a..03aa6df7e 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -25,7 +25,7 @@ Describe how the work gets done in plain English, and your coding agent builds i
- :material-laptop: **[Run It Yourself](./get-started/run-it-yourself.md)**
- Install the Pipelex runtime and run methods on your own machine, against the model providers you choose.
+ Install the Pipelex runtime and run methods from your terminal: on the hosted Pipelex API with one sign-in, or on your own machine against the model providers you choose.
- :material-school: **[MTHDS Language Tutorial](./get-started/mthds-language-tutorial.md)**
diff --git a/docs/migration-ledger.md b/docs/migration-ledger.md
index c84f589c8..147c0a0d1 100644
--- a/docs/migration-ledger.md
+++ b/docs/migration-ledger.md
@@ -486,6 +486,7 @@ There is one file-blocked reason per **state the file is in**, rather than one p
| `unsupported_schema_version` | It declares a `[meta] schema_version` below its ledger's floor, so the entries that would carry it forward are no longer there. Nothing was written. |
| `unwritable` | It needed a change and the run could not make it — the backup would not go down, or the replacement would not. The file is exactly as it was found. |
| `changed_during_run` | It was removed or edited between the read and the write, so the run refused to write over work it had not seen. |
+| `needs_a_hand_edit` | It is valid, and holds a shape the run will not rewrite on its own, since doing it safely would restructure what the user wrote — routing profiles written as one inline table, which the former-release cleanup cannot add a profile to. Nothing was written; the detail says what to change by hand. |
| `state_uncertain` | The write could not be confirmed: the transaction could not describe what it left behind, and the file does not hold what the run wrote. |
`state_uncertain` is the only one that cannot promise the file is as it was found, which is why it is not folded into `unwritable`: the next move is to compare the file against the rescue copy the plan names, not to fix a permission and run again. It is also, for the single-file commit a migration performs, the only transaction failure that reaches a plan at all — a replacement that fails re-raises its own error, because rolling back nothing is trivially complete.
@@ -628,8 +629,51 @@ A run is often both at once, which is the ordinary shape on a machine that has d
**Nothing inside a file is rendered here either.** The row reports paths and counts. It is the fourth channel the rendering rule covers, beside the command's output, its structured plan, and the block on a validation error.
+**The row reports the cleanup of a [former release](#a-former-releases-configuration) too**, since that is the command's first step: `former_release_files` names the files it would rewrite or remove, and `former_release_blocks_boot` says whether what they hold stops this machine's boot. That verdict is the boot's own check, `former_release_boot_blockers()` over the files the boot merges, never a reading of one directory: a project that boots on bases of its own is not told it cannot start because of what the home still carries, and an `active` in one directory naming a retired profile the other defines is not missed. A failure reading those files leaves the row unchecked, like any failure inside the scan. The finding is `pending` whenever there are such files, and `--fix` runs the cleanup before the migration, as the command does. When the cleanup's own check finds that a boot would still not start once it has run, the row says so and recommends `pipelex migrate --dry-run`, which says why, and with nothing left to clean it needs attention rather than reading as up to date. `--fix` has nothing to write for it.
+
**A failure inside the scan costs more here than anywhere else**, which is why it is caught rather than raised: an exception escaping this probe reaches the doctor's own outer handler, which prints one line and exits — so a broken packaged ledger would replace *every row the user came for* with "Unexpected error". The catch stays narrow (`MigrationError`, `OSError`), so a bug in our applier still surfaces as itself.
+## A former release's configuration
+
+Releases up to v0.72 ran models through the Pipelex Gateway, and the configuration they set up still names it: a `pipelex_gateway` table in `inference/backends.toml`, its per-backend file in `inference/backends/`, routing profiles that send models to it (`active = "all_pipelex_gateway"` above all), a `model_specs_section` key naming specs the remote configuration used to serve, the `pipelex_service.toml` that recorded the Gateway's terms acceptance, and the `pipelex_gateway_models*.md` model lists. None of that is a shape change of a surface — the files are valid; what they name is gone — so it is no ledger entry. It is a step of its own, which `pipelex migrate` runs first.
+
+**Pipelex Manifold is not retired.** Those releases also offered it as a private beta, with a `pipelex_manifold` table shipped disabled and still naming `model_specs_section`, its per-backend file, and an `all_pipelex_manifold` profile. The hosted plane runs every model through it, with a plugin of its own, and declares that backend and profile itself, enabled and active. So nothing here reads the `pipelex_manifold` table, its file, its profile or a route to it (`MANIFOLD_BACKEND_NAME`): on a user's machine what the release left for it is inert while the table is disabled, and the hosted plane's configuration is never refused, reported or rewritten. An enabled table still naming `model_specs_section` is refused by the backend library's own load, whose message says what to change. `tests/data/migration/former_release/hosted_manifold/` holds the hosted plane's inference trees, which every command here must find clean.
+
+**One reading of that state, across directories.** `detect_former_release_across(config_dirs=)` (`pipelex/migration/former_release.py`) is a pure read of the home and project configuration directories together, and every consumer asks it or `former_release_boot_blockers()`, which reads the same way: the boot, `pipelex doctor`, `pipelex init`'s inspect stage, and the cleanup. It reads the documents as the boot merges them. A boot reads one base `backends.toml` and one base `routing_profiles.toml` — the project's when it has one, the home's otherwise — then the home's override, then the project's; a boot started outside any project reads the home's base and override alone. `inference_merge_sequences()` derives those sequences from the directories, and a test holds it to the loader's own. A finding is decided over every sequence its file is read in, never over its own directory alone, because a profile one directory defines can be made active by a file in the other.
+
+Each finding says which file it is in and whether it stops a boot reading it. The shapes that do are the ones the boot refuses: a retired backend, or one still naming `model_specs_section`, left enabled; and an active routing profile that sends models to a retired backend, by its default or by a route it must honour (an optional route is inert while its backend is absent, and so is `fallback_order`), or a retired profile that no file defines. A base that names a retired active profile under an override that picked another is still found, as inert, since it is refused the day the override goes. Whether a given machine's boot is refused is `former_release_boot_blockers()` over the exact files that boot merges, which is the verdict the doctor and `pipelex init` report. A document that does not parse is skipped rather than raised: whether it parses is the boot's and the doctor's to say.
+
+**The boot refuses first, with one error.** `ModelManager.setup` runs `former_release_boot_blockers()` over exactly the sequences it is about to merge, across the home and project directories, before the backend library loads. On a blocker it raises `FormerReleaseConfigError`, which names each blocking finding with its file and both remedies, `pipelex migrate` and `pipelex init`, instead of whichever lower refusal would have come first: the Gateway's missing key, its backend file declaring no model, or `RoutingProfileDisabledBackendError` once the Gateway is switched off and `active` still names its profile. Reading the boot's own sequences is what keeps it from refusing a boot that would succeed: a project override that disables the Gateway lifts the home base's finding, as it lifts the boot's refusal.
+
+**The cleanup removes what was found, and nothing else.** `clean_former_release(config_dirs=, dry_run=)` (`pipelex/migration/former_release_cleanup.py`) takes the home directory and the project's, in that order, and turns each finding into one change on its file:
+
+| Finding | Change |
+|---|---|
+| a retired backend table, in `backends.toml` or its override | the table is deleted |
+| `model_specs_section` on another backend, `pipelex_manifold` aside | the key is deleted; the backend stays |
+| a profile table whose own default is a retired backend, which nothing else keeps alive | the profile is deleted from that file |
+| a profile table whose own default is a retired backend, while a file read before it keeps the profile or a file read after it gives it a live default | only the `default` key is deleted, so the user's routes stay; a table left empty goes with it |
+| the part of a profile an override holds, with no default and no description of its own, when every definition before it goes | that part is deleted too, since on its own it is a profile the boot refuses |
+| a route to a retired backend in a profile that otherwise stays, a catch-all `"*"` route included | the route is deleted, from the route table that holds it |
+| an `active` in a base file the plan moves | `active` moves to the profile the kit makes active (`all_enabled_backends`), which is added first among the profiles, as the kit ships it, when the file does not define it |
+| an `active` in a personal override the plan moves | `active` is deleted, so the file read before it decides |
+| the project's override, when the file the project's boot reads `active` from has to change for the boot outside the project alone | it is given an `active` of its own, naming the profile the project's boot starts on |
+| a retired backend's file, a model list, `pipelex_service.toml` | the file is removed |
+
+**The routing profiles are planned per boot, and checked before anything is written.** A machine has two boots, the one outside any project and the project's, and some files are read by both: the home's override always, and the home's base when the project has none of its own. Each profile table's fate is decided in each sequence that reads its file, and a file both boots read keeps whatever either needs: a profile one boot still uses is never deleted from under it, and a table one boot keeps is never left without its description in the other, whose own definition then keeps its description and loses only the retired `default`. Then every way of moving the `active` keys is tried on each boot's library as the cleanup would leave it. An `active` may move when it names what a former release left, or a profile the cleanup removes or leaves with nothing to route by — no `default`, no route and no `fallback_order` — which counts as gone; in a boot the release stops, so may an earlier `active` that the boot would read next, when it names a profile that is missing or routes nothing. No plan may stop a boot that starts today, change the profile it starts on, or leave its profile with nothing to route by; of the rest, the plan leaving the most boots with a profile that routes something wins, then the one leaving no `active` naming a profile that is gone, then the one without the project's own `active`, then the one moving the fewest. So when the home's base has to move off a profile that the project's override keeps alive by retargeting it, the project's override is given that profile as its own `active`, and the project goes on starting on it.
+
+**A file no change can suit both boots with is left for a hand edit.** When a boot the release stops could only be fixed by changing a file the other boot reads, and that change would switch the other boot's profile, and the project's override cannot make up the difference because there is none, the shared file is left exactly as it was, reported `needs_a_hand_edit` with the conflict and the edit that settles it: give the project an `active` of its own in its override, then make the shared file's suit the boot outside the project. The plan is then made again without touching that file. A last check reads the texts the cleanup is about to write as each boot would, and a rewrite that would still stop a boot that starts today, whatever produced it, is left for a hand edit too. `tests/unit/pipelex/migration/test_former_release_invariants.py` holds the cleanup to these rules over seeded random layouts of a home and a project.
+
+The kit's profile goes in before the deletions so that the banner heading the profiles stays at the head of them rather than leaving with the first profile it introduced. When it cannot go in, because the file writes its profiles as one inline table, the file is left exactly as it was and reported `needs_a_hand_edit`, with what to change: moving `active` to a profile the file lacks would stop the boot as surely as the retired one. Each key and table is deleted where the document spells it, through the applier's `delete_literally()`, so the comment introducing it goes with it, and a route or a profile spelled `"*"` is that one key, never the "every key" a migration op reads `*` as; the document is re-read between edits that applied. A change the cleanup cannot express on a file leaves that file as it was, reported `needs_a_hand_edit`, and never stops the run, the other files or the doctor's row. A rewritten file also loses the comment paragraphs at the head of the document that are, line for line, the ones the former release shipped there — the Gateway's instructions above the routing profiles; every other comment, a user's note naming the Gateway included, stays.
+
+**The cleanup checks its own work, on every run.** `what_stops_the_boot()` reads the files every boot of the machine merges, the home's alone and the project's, as the cleanup leaves them — after a write pass, the files on disk; in a dry run, the texts it would write — and reports what still stops one: what a former release left that a boot refuses, or a routing profile library the boot cannot load, above all an `active` naming a profile no file defines. It runs when there is nothing to clean too, so a cleanup left half done, by a run from another project or a file it could not write, is never reported as a machine with nothing to do. Anything it finds goes into the report's `still_blocking`, which makes the report need attention and never clean, so the command exits non-zero rather than calling the machine fixed.
+
+**The replay never walks a file the cleanup removes.** The ledger replay that follows the cleanup leaves those files out, in the rehearsal and in the write pass alike, in both commands, the doctor's row and its `--fix`: a retired backend's file carrying a key the ledger would migrate, or one it cannot explain, is neither counted among the files to migrate nor reported as needing a look, since it is about to go.
+
+**Every file gets the replay's guarantees.** A rewrite goes through the same per-file transaction as a migrated file (`runner.write_file_with_backup`), and a removal through its twin (`runner.remove_file_with_backup`): one `.bak.<stamp>` copy, taken first and kept, older copies pruned, and a file changed or unwritable mid-run left exactly as it was and reported, with the copy taken for it taken back — except the copy of a file that is gone by then, which stays, since a concurrent run of the same second may have removed the file after adopting that very copy as its own. A removal removes the path itself, so a symbolic link goes and the file it names stays where it lives. The transaction is per file, not across the files: a write that fails after others went through leaves those written, so a boot the plan kept on its profile through a change in another file, such as the project's own `active`, can start on another profile until that file is edited, and the report names it as blocked. A dry run reports the same changes and writes nothing, not even the `.gitignore`. Running it twice is running it once: a cleaned directory has no findings, and a backup's name matches none of the patterns the detector looks for.
+
+**Goldens.** `tests/data/migration/former_release/v0_72/` is the v0.72 kit's own files, copied from its wheel; `v0_72_cleaned/` holds the two documents the cleanup rewrites, byte for byte as it must leave them. Both are `plxt`-stable, since `make agent-check` formats every TOML file in the repository.
+
## Applying
> **Operations apply to the user's file, and a template is never the remedy.** "Delete your configuration and re-initialize it" is the failure this project exists to remove, not a fallback it may reach for: re-initializing throws away every choice the user made, and it is exactly what a structural change should not cost them.
diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.md
index 346f5d182..f26db4096 100644
--- a/docs/tools/cli/agent-cli.md
+++ b/docs/tools/cli/agent-cli.md
@@ -160,11 +160,12 @@ These commands do not have subcommands:
| Command | Description |
|---------|-------------|
+| `init` | Initialize configuration non-interactively from `--config`: `execution` (`hosted` or `local`), `backends`, `primary_backend`; see [Non-Interactive Init](init.md#non-interactive-init-pipelex-agent-init) (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
| `fmt` | Format a `.mthds`/`.toml`/`.plx` file in-place (delegates to `plxt`) |
| `lint` | Lint a `.mthds`/`.toml`/`.plx` file for errors (delegates to `plxt`) |
| `models` | List available model presets, aliases, and waterfalls (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
| `check-model` | Check a model reference for a model category (`check-model <reference> --type <category>`, the category being `llm`, `extract`, `img_gen`, `search`, `judgment` or `doc_gen`) and suggest alternatives when it is not valid. It applies the rule a validation applies, as [`GET /v1/models/check`](../../api-server/models.md#check-a-model-reference) does: a sigiled reference names that kind of binding, and a bare name is valid when it is a model of the category, an alias of the category, or a waterfall of the category while model fallback is on; a `handle:` reference writes a bare name without reading a sigil, so `handle:@best-gpt` is the bare name `@best-gpt`, not the alias `best-gpt`. `--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value |
-| `doctor` | Check config, credentials, and model health (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
+| `doctor` | Check config, credentials, and model health; a hosted setup (`[run] execution = "hosted"`) is judged by its Pipelex API key, reported in `checks.pipelex_api_key`, with `execution` at the top level and the provider credentials and models rows marked `informational` (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |
## Output Contract
@@ -203,7 +204,7 @@ For the structured commands, exactly one such object is written to stderr per fa
**A failed run** (`run pipe`, `run bundle`, `run method`) reports `"error_type": "PipelineExecutionError"` and adds four fields: `pipe_code` and `pipe_stack` name the pipe that failed and its path from the entry pipe, and `cause_type` and `cause_message` give the class and own message of the root fault, the innermost Pipelex error behind the failure. The `message` is that root fault's message prefixed with the failing pipe and its path (`Pipe 'summarize' failed (two_steps → summarize): …`), and the classification fields are the root fault's, so branch on `cause_type` to tell, say, a model that is not available from a failure to combine a parallel's results.
-**A run the hosted API refuses** (`--runner hosted`, when the hosted API answers with a non-2xx status) reports what its problem document says, in the same envelope and the same markdown. `error_type` is the runner's error class: `ValidateBundleError` for a bundle refused before any pipe ran, as a local run reports it, or the root fault of a run that failed, such as `ModelNotFoundError` or `StuffFactoryError`, which a local run reports as `cause_type` under the `error_type` `PipelineExecutionError`. There is no `cause_type` on this path, so a consumer of a refused hosted run branches on `error_type`, which is `ApiResponseError` when the answer names no class, as a gateway's error page does not. The `message` quotes the request, the status and the runner's reason (`API POST /v1/start failed (422): Pipe 'condense_article' failed (digest_article → condense_article): …`), so on a failed run it names the failing pipe and its path. The `hint` is the runner's next step; `error_domain`, `error_category` and `retryable` are the runner's own; `validation_errors` holds the runner's items, rendered in markdown as the same grouped prose `validate` prints; `pipe_code` and `pipe_stack` appear when the problem document carries them; `error_code` is the problem's `code`, the stable code the hosted plane gives the refusals it authors itself (`rate_limited`, `unauthorized`); `retry_after_seconds` is the delay of a `Retry-After` header; and `http_status` and `request_id` are what to quote when the fault is the runner's. When the answer advised no next step, the `hint` follows its status, never the local advice kept for the runner's class: a 401 or a 403 points at `PIPELEX_API_KEY`; a 404 points at the method's address or id and at the origin (`--base-url`, else `PIPELEX_BASE_URL`, else `https://api.pipelex.com`, with no path such as `/v1`); a 429 says to wait and run again and marks the error `retryable` unless the answer said otherwise; a 503 says to wait too but warns that running again can repeat a paid run, because the service in front of the runner can answer it after the run completed, so it is `retryable` only when the answer says so; any other 4xx says to change what the message names; and any other status says to report the `request_id`, or the `http_status` when there is none.
+**A run the hosted API refuses** (`--runner hosted`, when the hosted API answers with a non-2xx status) reports what its problem document says, in the same envelope and the same markdown. `error_type` is the runner's error class: `ValidateBundleError` for a bundle refused before any pipe ran, as a local run reports it, or the root fault of a run that failed, such as `ModelNotFoundError` or `StuffFactoryError`, which a local run reports as `cause_type` under the `error_type` `PipelineExecutionError`. There is no `cause_type` on this path, so a consumer of a refused hosted run branches on `error_type`, which is `ApiResponseError` when the answer names no class, as a gateway's error page does not. The `message` quotes the request, the status and the runner's reason (`API POST /v1/start failed (422): Pipe 'condense_article' failed (digest_article → condense_article): …`), so on a failed run it names the failing pipe and its path. The `hint` is the runner's next step; `error_domain`, `error_category` and `retryable` are the runner's own; `validation_errors` holds the runner's items, rendered in markdown as the same grouped prose `validate` prints; `pipe_code` and `pipe_stack` appear when the problem document carries them; `error_code` is the problem's `code`, the stable code the hosted plane gives the refusals it authors itself (`rate_limited`, `unauthorized`); `retry_after_seconds` is the delay of a `Retry-After` header; and `http_status` and `request_id` are what to quote when the fault is the runner's. When the answer advised no next step, the `hint` follows its status, never the local advice kept for the runner's class: a 401 or a 403 points at `PIPELEX_API_KEY` and `pipelex login`; a 404 points at the method's address or id and at the origin (`--base-url`, else `PIPELEX_BASE_URL`, else `https://api.pipelex.com`, with no path such as `/v1`); a 429 says to wait and run again and marks the error `retryable` unless the answer said otherwise; a 503 says to wait too but warns that running again can repeat a paid run, because the service in front of the runner can answer it after the run completed, so it is `retryable` only when the answer says so; any other 4xx says to change what the message names; and any other status says to report the `request_id`, or the `http_status` when there is none.
**A hosted run that started and failed** reports its stored error report: `error_type` is the runner's class, `message` says the run's final status and reason, `hint` is the runner's next step, `error_domain`, `error_category`, `retryable`, `model`, `provider` and `validation_errors` are the report's when it carries them, and `pipeline_run_id` and `run_status` name the run. `retryable` is the runner's verdict whenever it gave one, `false` included, never the local default for its class. **A hosted run that failed on the way** carries its own class as `error_type`, with a next step for each: `ApiUnreachableError` (the network or the origin, `"error_domain": "config"`, only for a failure before the run request left this machine), `HostedRunOutcomeUnknownError` (the connection failed after the run request was sent, so a run may exist with no id returned: check the run history before running again), `RunTimeoutError` (the run keeps going on the hosted API past the wait), `PipelineExecuteTimeoutError` (a server that keeps no runs cut the blocking route before the result came back; the run may still be going, so it is `"retryable": false` and its hint points at the proxy's timeout or a base URL that keeps runs), `MissingMainStuffError` (the run completed and delivered no main output), `HostedRunPollingError` (the run started, then reading its result failed; it may still be running and is paid for, so look it up rather than run again), `HostedMethodInvalidError` (the method does not load, refused before any run starts when the inputs name a local file, its `validation_errors` naming each faulty file in `source`), `HostedLocalFileUploadUnavailableError` (the inputs name a local file and the method calls another one by its address: give the file as an `https://` URL), `InvalidLocalSourceError` (a local file in the inputs cannot be read), `UploadTransportError` (`config` when the upload never reached the hosted API, `runtime` when the hosted API failed it), `UploadAuthenticationError`, `RejectedAssetError` and the other upload failures, and `HostedBaseUrlError` for a base URL that is not an origin, refused before anything is sent. Every failure met once the hosted API acknowledged the run carries its `pipeline_run_id`. **An interrupted hosted run** (Ctrl-C) exits with `130` and the envelope `HostedRunInterruptedError`, whose `pipeline_run_id` names the run the hosted API had acknowledged: the run keeps going and is paid for. Interrupted before the acknowledgement, the envelope has no `pipeline_run_id` and its hint says to check the run history before running again. A configuration that cannot be read while deciding where the run executes gives the hint a failed local boot gives, and any other failure still leaves as this envelope, with the exception's class as `error_type`.
@@ -211,7 +212,9 @@ For the structured commands, exactly one such object is written to stderr per fa
`TelemetryConfigValidationError` carries it too, on the same terms: absent means the `telemetry.toml` is wrong and the fields the message names are what to correct, present with `would_write: true` means it is old, and present with `would_write: false` means it is both reported on and unrepairable by the command.
-**A failure is not the only way to learn this, and for an agent it is not even the reliable one.** A configuration the migration history explains boots with a warning rather than an error — and `pipelex-agent` silences logging process-wide before anything can emit one, so no error arrives and the machine is still stale. `pipelex-agent doctor` is the channel that always answers: its `checks.pending_migrations` row carries a `finding` (`up_to_date`, `pending`, `needs_attention`, `unavailable`), the `migratable_files` a migration would rewrite, and the `attention_files` it will not repair on its own. Branch on `finding`; `unavailable` means the check could not run and is not a claim that the machine is current. The row answers for both configuration directories even under `--global`, because it describes `pipelex migrate`, which has no such flag.
+**`FormerReleaseConfigError`** — a boot on a configuration set up by a release that ran on the Pipelex Gateway, with its backend enabled or one of its routing profiles active, is refused with this error, which names what stops it with its file and carries no `migration` object: what is there is no stale shape but what that release left. The loop is the same, `pipelex-agent migrate --dry-run --format json`, whose `former_release` key lists each file the cleanup rewrites or removes and its changes, show the user, then `--yes`. On every run, `former_release.still_blocking` lists, one sentence each, whatever would still stop the boot once the cleanup is done: after `--yes`, read off the files written, and in a dry run, off the files as they would be written, so the plan already says when the cleanup alone will not be enough. When it is not empty, `needs_attention` is true and `is_clean` false, even with no file left to clean, and the machine is not fixed yet.
+
+**A failure is not the only way to learn this, and for an agent it is not even the reliable one.** A configuration the migration history explains boots with a warning rather than an error — and `pipelex-agent` silences logging process-wide before anything can emit one, so no error arrives and the machine is still stale. `pipelex-agent doctor` is the channel that always answers: its `checks.pending_migrations` row carries a `finding` (`up_to_date`, `pending`, `needs_attention`, `unavailable`), the `migratable_files` a migration would rewrite, the `attention_files` it will not repair on its own, and the `former_release_files` its first step cleans up, with `former_release_blocks_boot` saying whether they stop this machine's boot, read off the files the boot merges in the current directory rather than off each directory alone. When the cleanup's own check finds that a boot would still not start once it has run, the row says so in its `message` and `recommended_actions` names `pipelex migrate --dry-run`, which says why; with nothing left to clean its `finding` is `needs_attention`, and that action is the row's only one. Branch on `finding`; `unavailable` means the check could not run and is not a claim that the machine is current. The row answers for both configuration directories even under `--global`, because it describes `pipelex migrate`, which has no such flag.
`fmt` and `lint` are raw passthroughs to the `plxt` binary and bypass the JSON output contract, producing native `plxt` output instead.
diff --git a/docs/tools/cli/index.md b/docs/tools/cli/index.md
index e18059085..c3610053b 100644
--- a/docs/tools/cli/index.md
+++ b/docs/tools/cli/index.md
@@ -23,7 +23,8 @@ The Pipelex CLI is organized into several command groups:
| Command | Description |
|---------|-------------|
-| [**init**](init.md) | Initialize Pipelex configuration |
+| [**init**](init.md) | Initialize Pipelex configuration and choose where your runs execute: the hosted Pipelex API or this machine |
+| [**login**](login.md) | Get a Pipelex API key through your browser, or paste one, and save it for runs on the hosted Pipelex API |
| [**update**](update.md) | Refresh the model deck and `backends/internal.toml` to match the installed pipelex version |
| [**migrate**](migrate.md) | Bring your configuration files up to the schema the installed version expects |
| [**validate**](validate.md) | Validate configuration and pipelines |
@@ -70,8 +71,9 @@ Package manifest management currently lives in the `mthds` CLI:
1. **Initial Setup**
- - Run `pipelex init` to create configuration files and select your backends
- - Configure your AI providers in `.pipelex/inference/backends.toml`
+ - Run `pipelex init` to create configuration files and choose where your runs execute
+ - On the hosted Pipelex API, `pipelex init` signs you in for a Pipelex API key; `pipelex login` does it again on its own
+ - On this machine, it asks for your backends and their keys; configure them further in `.pipelex/inference/backends.toml`
2. **Development Workflow**
diff --git a/docs/tools/cli/init.md b/docs/tools/cli/init.md
index c11ccc751..410cfcc21 100644
--- a/docs/tools/cli/init.md
+++ b/docs/tools/cli/init.md
@@ -1,10 +1,10 @@
---
-description: "Initialize your Pipelex project configuration — create .pipelex directory structure, set up backends, and configure telemetry destinations."
+description: "Initialize Pipelex — create the .pipelex directory, choose where your runs execute (the hosted Pipelex API or this machine), set up backends, and configure telemetry destinations."
---
# Init Commands
-Initialize Pipelex configuration files and related setup flows.
+Initialize Pipelex configuration files and choose where your methods run: on the hosted Pipelex API, with a Pipelex API key, or on this machine, with your own provider keys.
## Initialize Configuration
@@ -21,10 +21,10 @@ By default, `pipelex init` writes to the home configuration directory (`~/.pipel
**Arguments:**
- `FOCUS` - What to initialize (optional):
- - `all` (default) - Initialize everything
- - `config` - Only configuration files
+ - `all` (default) - Initialize everything, asking where your runs execute
+ - `config` - Only configuration files; on a first setup, where no `inference/backends.toml` exists yet, it also asks where your runs execute and sets that up
- `credentials` - Prompt for missing credentials only
- - `inference` - Only inference backend setup
+ - `inference` - Only the inference backends of runs on this machine; where runs execute is left as it is
- `routing` - Only routing profile setup
- `telemetry` - Only telemetry configuration
@@ -50,11 +50,28 @@ pipelex init inference
pipelex init telemetry
```
+## Where Your Runs Execute
+
+After you confirm, `pipelex init` asks one question:
+
+```
+Where should your runs execute?
+ [1] On the hosted Pipelex API, with a Pipelex API key default
+ [2] On this machine, with your own provider keys
+```
+
+Enter takes the hosted Pipelex API. Your answer is written to `[run] execution` in the target `pipelex.toml`, the default every `pipelex run` and `pipelex-agent run` takes when the command passes neither `--hosted` nor `--local` (see [Run Configuration](../../configuration/config-practical/run-config.md)). A single run can always go the other way with one of those flags.
+
+- **The hosted Pipelex API** writes `execution = "hosted"` and then gets you a key: it runs [`pipelex login`](login.md), which opens the Pipelex app in your browser and saves the key it returns to `~/.pipelex/.env`. When a Pipelex API key is already set, in `PIPELEX_API_KEY` or saved in that file, it is kept and nothing opens; run `pipelex login` to replace it. An empty `PIPELEX_API_KEY=` counts as no key, and a `.env` in the working directory that sets another value is named, since Pipelex loads it after the home one. No provider key is asked for, since a hosted run uses none, and the inference files are the kit's defaults: like every reset, a re-init copies the kit's backends, model deck and routing over customised ones, and on this path nothing reconfigures them afterwards. Run `pipelex init inference` to set up this machine's backends again for `--local` runs. If the login does not complete, or its key cannot be saved, the rest of the setup is kept, and `pipelex login` (or `pipelex login --paste`) gets the key later.
+- **This machine** writes `execution = "local"` and runs the backend selection, routing and credential prompts described below.
+
+`pipelex init inference` configures the backends of runs on this machine and never asks this question, so it does not change where your runs execute. To switch, run `pipelex init` again, or set `[run] execution` yourself.
+
## What Gets Initialized
This command creates or resets a Pipelex config directory with:
-- **pipelex.toml** - Main configuration file for logging, reporting, etc.
+- **pipelex.toml** - Main configuration file for logging, reporting, where runs execute, etc.
- **inference/** - AI backend and routing configuration
- `backends.toml` - Backend provider settings
- `routing_profiles.toml` - Model routing rules
@@ -70,11 +87,35 @@ This command creates or resets a Pipelex config directory with:
When you run `pipelex init`, Pipelex can guide you through:
-1. **Config reset** - Recreate the selected config files
-2. **Backend selection** - Choose which AI providers to enable. The prompt pre-selects OpenAI, whose one key serves every default language-model and image-generation tier of the shipped model deck
-3. **Credential prompts** - Fill in missing keys when relevant
-4. **Routing configuration** - Set up how models are routed to backends
-5. **Telemetry setup** - Configure observability and analytics
+1. **What a former release left** - On a machine set up by a release that ran models through the Pipelex Gateway, the cleanup that release's files need, offered first (see [below](#a-machine-a-former-release-set-up))
+2. **Config reset** - Recreate the selected config files
+3. **Where your runs execute** - The hosted Pipelex API (the default) or this machine
+4. On the hosted Pipelex API: **sign-in** - [`pipelex login`](login.md) gets a Pipelex API key, unless one is already set. It runs last, once every file is written
+5. On this machine:
+ 1. **Backend selection** - Choose which AI providers to enable. The prompt pre-selects OpenAI, whose one key serves every default language-model and image-generation tier of the shipped model deck
+ 2. **Routing configuration** - Set up how models are routed to backends
+ 3. **Credential prompts** - Fill in missing keys when relevant
+6. **Telemetry setup** - Configure observability and analytics
+
+## A Machine a Former Release Set Up
+
+Releases up to v0.72 ran models through the Pipelex Gateway. It no longer exists, and Pipelex refuses to start on the files those releases wrote: a `pipelex_gateway` backend left enabled, or a routing profile such as `all_pipelex_gateway` left active.
+
+`pipelex init` looks for what such a release left, in the home configuration directory and in the project's `.pipelex/`, before it asks anything else, reading both together the way Pipelex merges them when it starts. When it finds some, it shows which files it is in and, when it stops Pipelex from starting here, what does: that is judged from the files Pipelex actually starts on in the current directory, so a project with configuration files of its own is told that what the home directory still carries no longer stops it, and is still offered the cleanup. Then it asks:
+
+```
+Clean it up now? [y/n] (y):
+```
+
+Yes runs the cleanup that [`pipelex migrate`](migrate.md#a-configuration-a-former-release-set-up) runs: the retired backends, routing profiles and files are removed, a copy of each file it changes or removes is kept beside it, and an active routing profile of that release's moves to `all_enabled_backends`, without switching Pipelex to another profile where it starts today as long as every file can be written. Then the setup goes on to the question of where your runs execute; if the cleanup leaves a file for you to edit by hand, or Pipelex still cannot start once it ran, `pipelex init` says so and names what to look at. No leaves the files as they are and goes on with the setup, which replaces the target directory's inference files but not the files beside them nor those of the other directory; `pipelex migrate` cleans them up whenever you are ready, and `pipelex init` adds that Pipelex will not start until then only when it would not.
+
+`pipelex doctor --fix` runs `pipelex init` with nobody there to answer, and that run never cleans up: the doctor asks about the cleanup itself, in its **Configuration Migrations** row, and only a yes there removes or rewrites anything.
+
+`pipelex init config` on an existing setup resets `pipelex.toml` without asking the question again, so it keeps the `[run] execution` it found, and asks for no provider key when that is `"hosted"`.
+
+`pipelex doctor --fix` installs missing configuration files with nobody there to answer, and never moves runs off this machine. On a first setup it keeps the `[run] execution` the target `pipelex.toml` already sets; a `pipelex.toml` that sets none, such as one written before the setting existed, runs on this machine, as the package default says, and stays there. Only a brand-new home, with no `pipelex.toml` yet, takes the hosted Pipelex API, writing `execution = "hosted"` and printing `pipelex login` instead of opening a browser.
+
+**A project can override the global choice.** A project's `.pipelex/pipelex.toml` wins over the one in `~/.pipelex/`, and the one `pipelex init --local` copies sets `execution = "local"`. When you set up the home configuration from inside a project whose own file sets the other value, `pipelex init` warns you, naming that file: runs started in that project execute where it says, until you change it there or pass `--hosted` or `--local` on a run.
## Non-Interactive Init (`pipelex-agent init`)
@@ -84,6 +125,8 @@ For automated or agent-driven setups, use the `pipelex-agent` CLI:
pipelex-agent init [--config/-c JSON] [--global/-g]
```
+It never opens a browser: a hosted setup reads its key from `PIPELEX_API_KEY`, which a person gets with `pipelex login`.
+
**Target directory:**
- **Default:** project-level `.pipelex/` at the detected project root (looks for `.git`, `pyproject.toml`, etc.). Errors out if no project root is found.
@@ -93,6 +136,7 @@ pipelex-agent init [--config/-c JSON] [--global/-g]
```json
{
+ "execution": "local",
"backends": ["openai", "anthropic"],
"primary_backend": "openai"
}
@@ -102,9 +146,12 @@ All fields are optional:
| Field | Type | Description |
|-------|------|-------------|
+| `execution` | `"hosted"` or `"local"` | Where runs execute by default, written to `[run] execution`. `"hosted"` configures no backend or routing, leaves the kit's inference files as written, and cannot be combined with `backends` or `primary_backend`. `"local"`, or no `execution` at all, configures the backends below. |
| `backends` | `list[str]` | Backend keys to enable (e.g. `openai`, `anthropic`, `openrouter`). Omit to keep the template's enabled backends and its routing profile, which sends each model to the first of them that serves it. |
| `primary_backend` | `str` | Required when 2+ backends are named. Named without `backends`, it routes the template's backends to it first. |
+**Output:** `success`, `target_dir`, `config_files_copied` and `execution`. A local setup adds `backends_enabled` and `routing_profile`; a hosted one adds `api_key_set`, whether a Pipelex API key is set in `PIPELEX_API_KEY` or saved in `~/.pipelex/.env`, never the key itself. A `warnings` list is added when the project around the working directory sets another `[run] execution` in its own `.pipelex/pipelex.toml`, which wins over the one just written.
+
Telemetry is not configured via `--config`: init seeds a `telemetry.toml` from a template (a global init writes an active one; a project init drops a commented-out one).
**Examples:**
@@ -115,10 +162,15 @@ pipelex-agent init --config '{"backends": ["openai"]}'
# Initialize globally with OpenAI, whose one key serves every default language-model and image-generation tier of the model deck
pipelex-agent init -g --config '{"backends": ["openai"]}'
+
+# Run on the hosted Pipelex API by default, with the key in PIPELEX_API_KEY
+pipelex-agent init -g --config '{"execution": "hosted"}'
```
## Related Configuration
+- [Login](login.md)
+- [Run Configuration](../../configuration/config-practical/run-config.md)
- [Configure AI Providers](../../get-started/configure-ai-providers.md)
- [Inference Backend Configuration](../../configuration/config-technical/inference-backend-config.md)
- [Telemetry Configuration](../../configuration/config-practical/telemetry-config.md)
diff --git a/docs/tools/cli/login.md b/docs/tools/cli/login.md
new file mode 100644
index 000000000..a0a5d4569
--- /dev/null
+++ b/docs/tools/cli/login.md
@@ -0,0 +1,72 @@
+---
+description: "Get a Pipelex API key through your browser, or paste one, and save it so methods run on the hosted Pipelex API."
+---
+
+# Login Command
+
+`pipelex login` gets a Pipelex API key (`plx_sk_…`) and saves it where a hosted run reads it, so `pipelex run … --hosted`, and every run when [`[run] execution = "hosted"`](../../configuration/config-practical/run-config.md), works with nothing exported in your shell.
+
+```bash
+pipelex login
+pipelex login --paste
+```
+
+[`pipelex init`](init.md) runs the same login when you choose to run on the hosted Pipelex API, so you only need this command to sign in on its own, to replace a key, or on a machine without a browser.
+
+## What It Does
+
+1. It opens the Pipelex app in your browser, on a page that creates a key for the command line, and prints the same link in case the browser does not open. When no browser can be opened at all, it says so at once: open the link in a browser on this machine, or press Ctrl-C and run `pipelex login --paste`. Sign in, or create an account, on that page.
+2. The app hands the new key back to `pipelex login`, which waits for it on a port of `127.0.0.1` it opened for this one login. It waits up to five minutes.
+3. It checks the key: the key must start with `plx_sk_`, and the hosted API must accept it, which it asks by reading the account the key belongs to.
+4. It saves the key as `PIPELEX_API_KEY` in `~/.pipelex/.env` (or in the `.env` of the directory `PIPELEX_HOME` names), the file Pipelex loads into its environment at startup.
+
+The key is never printed. The `.env` file is readable and writable by you only (mode `0600`), and every other line in it, your provider keys and comments included, is kept as it was. A `PIPELEX_API_KEY` already in the file is replaced.
+
+**The check decides what is saved:**
+
+- When the hosted API accepts the key, it is saved, and the command says which account it belongs to.
+- When the hosted API refuses it (HTTP 401 or 403), nothing is saved and the command exits with code `1`, naming the status.
+- When the key cannot be checked, because the hosted API cannot be reached or answers anything else, it is saved anyway, with a warning saying why it was not checked.
+- A value that does not start with `plx_sk_` is refused before any check.
+
+**A `.env` in the working directory wins.** Pipelex loads the `.env` of the directory a command runs in after `~/.pipelex/.env`, so a `PIPELEX_API_KEY` line there, even an empty one, is what commands run in that directory send. When that file sets another value than the key just saved, `pipelex login` names it, without printing either value: remove the line from it.
+
+**The handover is protected.** Each login sends the app a random `state` value along with the port, and the app sends it back with the key. A request to the port that does not carry this login's `state` is refused and its key discarded: any page open in your browser can send a request to a local port, so a key that arrives without it did not come from the page this login opened. The command says so in the terminal and keeps waiting.
+
+When no key arrives in time, the command exits with code `1` and names `pipelex login --paste`.
+
+## Paste a Key Instead
+
+```bash
+pipelex login --paste
+```
+
+On a machine where no browser can reach a local port, such as a remote server over SSH or a CI job, create a key in the Pipelex app on any machine, then paste it at the prompt. The key is not shown as you type. It goes through the same check and is saved the same way.
+
+In CI, you can also skip the file entirely and set `PIPELEX_API_KEY` as a secret of the job.
+
+## Other Apps and APIs
+
+| Variable | What it changes | Default |
+| --- | --- | --- |
+| `PIPELEX_APP_URL` | The Pipelex app the browser opens, given as an origin (`scheme://host[:port]`, no path) | `https://app.pipelex.com` |
+| `PIPELEX_BASE_URL` | The hosted API the key is checked against, and that hosted runs go to | `https://api.pipelex.com` |
+
+A value that is not an origin is refused before anything opens. Most people never set either; they point the command at a development or staging plane, for instance:
+
+```bash
+PIPELEX_APP_URL=https://app-dev.pipelex.com PIPELEX_BASE_URL=https://api-dev.pipelex.com pipelex login
+```
+
+Set them in `~/.pipelex/.env` to keep them for every command.
+
+## Exit Codes
+
+- `0`: a key was saved, checked or not.
+- `1`: nothing was saved: no key arrived in time, none was pasted, the value was not a Pipelex API key, the hosted API refused it, `PIPELEX_APP_URL` is not an origin, or the `.env` could not be written. In that last case the command names the file and the error; the key it received went unused, so revoke it in the Pipelex app and run `pipelex login` again once the file can be written.
+
+## Related
+
+- [Init](init.md): the first-run setup, which asks where your runs execute and runs this login for the hosted Pipelex API
+- [Running on the Hosted API](run.md#running-on-the-hosted-api): what a hosted run sends, uploads and saves
+- [Run Configuration](../../configuration/config-practical/run-config.md): the `[run] execution` default
diff --git a/docs/tools/cli/migrate.md b/docs/tools/cli/migrate.md
index 523482e35..3b7fef868 100644
--- a/docs/tools/cli/migrate.md
+++ b/docs/tools/cli/migrate.md
@@ -24,7 +24,7 @@ Two directories, and only those:
Within each, it looks at the configuration files themselves — `pipelex.toml` and its `pipelex_*.toml` tiers, `telemetry.toml` and its tiers — and at the inference backend definitions in `inference/backends/`.
-It goes one level deep, and only into a directory some configuration family owns. `inference/backends/` is such a directory, so every `*.toml` in it is repaired the same way a `pipelex.toml` is. `inference/deck/` is not one, and neither is `inference/backends.toml`, which sits beside the backends directory rather than in it — neither is ever entered or rewritten by this command, and the model deck has its own `pipelex update`.
+It goes one level deep, and only into a directory some configuration family owns. `inference/backends/` is such a directory, so every `*.toml` in it is repaired the same way a `pipelex.toml` is. `inference/deck/` is not one, and is never entered: the model deck has its own `pipelex update`. Neither is `inference/backends.toml`, which sits beside the backends directory rather than in it, nor `inference/routing_profiles.toml`: the one thing this command ever does to them is remove what a [former release](#a-configuration-a-former-release-set-up) left in them.
Within a file it repairs, it only ever undoes a change *we* made. A key you added yourself — a misspelling, or a setting from a plugin — is reported rather than removed, because the migration history describes our renames and removals and has nothing to say about your keys.
@@ -36,6 +36,29 @@ Those copies are ours, not part of your project, so the command keeps them out o
Running it twice is the same as running it once. Nothing is skipped on the basis of a version record, because there is no version record; every run replays the whole history and leaves alone whatever is already current. A file that is already up to date comes back byte for byte identical.
+## A configuration a former release set up
+
+Releases up to v0.72 ran models through the Pipelex Gateway. It no longer exists, and the files those releases wrote still name it. Pipelex refuses to start on some of them, and says so with one error that names this command and [`pipelex init`](init.md#a-machine-a-former-release-set-up):
+
+- a `pipelex_gateway` backend left enabled in `inference/backends.toml`, or another backend still naming `model_specs_section`, whose model specs are no longer downloaded;
+- an active routing profile that sends models to it: `all_pipelex_gateway`, which those releases made the default, or a profile routing some models there.
+
+The cleanup of what those releases left is this command's first step, before any file is migrated. It reads the home directory and the project's together, the way Pipelex merges them when it starts, so a profile defined in one and made active from the other is cleaned up as one:
+
+- the `pipelex_gateway` table is removed from `inference/backends.toml` and its personal override, and a `model_specs_section` key from any other backend, which stays;
+- a routing profile whose default is the Gateway is removed, and so is a route that sends a model to it, from a profile that otherwise stays, a catch-all `"*"` route included;
+- when a profile of your own is only pointed at the Gateway by a personal override, only that `default` is removed from the override, so your profile and its routes stay as you wrote them;
+- an active routing profile that is removed, or left with nothing to route models by, is replaced in the files that name it, in either directory. In a base file it moves to `all_enabled_backends`, the profile a fresh install makes active, which routes each model to the first enabled backend that serves it; when the file does not define that profile, it is added as the current release ships it. A personal override that names it stops naming any, so the file read before it decides;
+- Pipelex starting outside any project and Pipelex starting in your project read some files in common, your home's override above all, and the cleanup plans its changes so that none of them switches the other to another routing profile or stops it from starting. When your project needs a different `active` from the one outside it, your project's personal override is given one of its own. That holds when every file can be written: the files are written one by one, so if one fails after others were written, the report names it, and until you make its change by hand Pipelex may start on another profile where that file is read;
+- `inference/backends/pipelex_gateway.toml`, the Gateway's model lists `inference/backends/pipelex_gateway_models*.md` and `pipelex_service.toml`, which recorded the Gateway's terms acceptance, are removed;
+- the paragraph of instructions about the Pipelex Gateway those releases wrote at the head of `routing_profiles.toml` goes with them; every other comment stays, including a note of yours that mentions the Gateway.
+
+Nothing else is touched: your other backends, your own profiles and routes, and your keys stay as they are. Pipelex Manifold, which those releases offered as a private beta, is not retired, so what they left for it — a `pipelex_manifold` table shipped disabled, `inference/backends/pipelex_manifold.toml` and the `all_pipelex_manifold` profile — stays too. Each file the cleanup rewrites or removes is copied first, exactly as a migrated file is (below), so a removed file is one rename away from being back. The dry run shows each change, file by file, and the question that follows counts the files of both steps. A file the cleanup removes is never also counted among the files to migrate, nor reported as needing a look.
+
+Some cases end with something for you to do, and the command exits non-zero for each. A file the cleanup cannot rewrite safely is left as it is, and the report says what to change by hand, then run the command again: a routing profiles file written as one inline `profiles = { … }` table, which the cleanup cannot add a profile to without restructuring it, and a file read both outside your project and in it whose `active` would have to change for one and not the other, when your project has no personal override to hold its own. And on every run, a dry run and a run with nothing left to clean included, the command reads the files Pipelex starts on as the cleanup leaves them: if they still stop it — an active routing profile that no file defines, for instance, left by a cleanup run from another project — it says so under **Pipelex cannot start** or **After the cleanup, Pipelex still cannot start** instead of reporting success or that there is nothing to do.
+
+`pipelex doctor` lists these files in its **Configuration Migrations** row and says when they stop Pipelex from starting, and `pipelex doctor --fix` runs the cleanup on a yes. When Pipelex would still not start once the cleanup has run, the row says so and recommends `pipelex migrate --dry-run`, which says why: that is left for you to fix, so `--fix` has nothing to write for it. Both judge that from the files Pipelex actually starts on in the current directory: a project with configuration files of its own is not told it cannot start because of what your home directory still carries, though the files are listed and the cleanup is offered. `pipelex init` finds them too, and offers the same cleanup before it asks anything else.
+
## When it cannot do the whole job
Some changes cannot be made for you. A setting whose accepted values narrowed, for instance, needs someone who knows what the value was *meant* to be — so Pipelex names the key and leaves it to you rather than guessing. The report says which file, which key, and what to look at. Everything it *can* do is still done, and one file it cannot process never stops the others.
@@ -60,7 +83,9 @@ Nothing that Pipelex tells you about an out-of-date file will ever suggest delet
## For agents
-`pipelex-agent migrate` is the machine-facing counterpart. It writes only when passed `--yes`, since it cannot ask, and answers with a structured plan — `--format json` for the contract, `--format markdown` to read. Branch on the `needs_attention` field, never on the exit code.
+`pipelex-agent migrate` is the machine-facing counterpart. It writes only when passed `--yes`, since it cannot ask, and answers with a structured plan — `--format json` for the contract, `--format markdown` to read. Branch on the `needs_attention` field, never on the exit code. The cleanup of a former release is under `former_release`: each file it rewrites or removes, with its `action` (`rewrite` or `remove`), its `changes` in words, the `backup_path` once applied, and why it was left when it could not be cleaned (`needs_a_hand_edit` for an inline profiles table, a file both boots read that no change suits, or a change the cleanup cannot express, with what to change in `blocked_detail`). `former_release.still_blocking` lists what would still stop Pipelex from starting once the cleanup is done, one sentence each, on every run: after `--yes`, read off the files written, and in a dry run, off the files as they would be written; it is empty when the machine can start. Both count toward `needs_attention` and `is_clean` like the migration itself. The files the cleanup removes are never among the `plans`.
+
+An agent whose boot fails with `FormerReleaseConfigError` runs the same loop: `pipelex-agent migrate --dry-run --format json`, show the user what would be removed, then `--yes`.
An agent usually meets this command through a failure rather than by choosing it. A configuration error carries a `migration` field when — and only when — a scan of the machine found something; its presence is the signal that the configuration is *old* rather than *wrong*. The loop from there is `pipelex-agent migrate --dry-run --format json`, show the user what would change, then `--yes` on confirmation. Never hand-edit a configuration file.
diff --git a/docs/tools/cli/run.md b/docs/tools/cli/run.md
index 60e7e8b4a..6b3cff935 100644
--- a/docs/tools/cli/run.md
+++ b/docs/tools/cli/run.md
@@ -182,10 +182,10 @@ A run executes on this machine by default, with the inference backends configure
Where a run executes is decided in this order:
1. `--hosted` or `--local` on the command.
-2. `[run] execution` in `.pipelex/pipelex.toml` (`"local"` or `"hosted"`; the project's file over the one in `~/.pipelex/`), see [Run Configuration](../../configuration/config-practical/run-config.md).
+2. `[run] execution` in `.pipelex/pipelex.toml` (`"local"` or `"hosted"`; the project's file over the one in `~/.pipelex/`), which [`pipelex init`](init.md#where-your-runs-execute) writes from your answer, see [Run Configuration](../../configuration/config-practical/run-config.md).
3. Local.
-The key is read from `PIPELEX_API_KEY`, exported in your shell or saved in `~/.pipelex/.env` (or a `.env` in the working directory), which Pipelex loads at startup. The run goes to `--base-url` when given, else to `PIPELEX_BASE_URL`, else to `https://api.pipelex.com`. Either value must be an origin, `scheme://host[:port]` with `http` or `https` and no path such as `/v1`; anything else is refused before a request is sent, naming the setting it came from. A self-hosted Pipelex API server is reached the same way, by pointing the base URL at it.
+The key is read from `PIPELEX_API_KEY`, exported in your shell or saved in `~/.pipelex/.env` (or a `.env` in the working directory), which Pipelex loads at startup. [`pipelex login`](login.md) gets a key through your browser and saves it there. The run goes to `--base-url` when given, else to `PIPELEX_BASE_URL`, else to `https://api.pipelex.com`. Either value must be an origin, `scheme://host[:port]` with `http` or `https` and no path such as `/v1`; anything else is refused before a request is sent, naming the setting it came from. A self-hosted Pipelex API server is reached the same way, by pointing the base URL at it.
**What a hosted run sends** is what a local run would load:
@@ -203,7 +203,7 @@ The method's signature, which says where a file sits, is read from the hosted AP
**What it refuses**: `--dry-run`, `--mock-inputs`, `--mock-usage`, `--orchestrator`, `--save-csv`, `--costs` / `--no-costs` and `--graph-full-data` / `--graph-no-data` steer this machine's runtime, which a hosted run never boots, so each is refused with a message pointing at `--local`. `--base-url` on a run that executes locally is refused too.
-**When it fails**, the message gives the hosted API's reason and a next step: the one the server advised when it gave one, else one that follows the status (a refused key points at `PIPELEX_API_KEY`, an unknown method or route at the method's address and the base URL, a rate limit says to wait). A bundle the hosted API refuses lists its validation errors, and so does a run that started and failed on them. Once the hosted API has acknowledged the run, the failure also prints the run id: a run that failed, one that outlived the wait, one that completed without a main output, and one whose result could not be read, because the network or the hosted API failed while the run was being followed. That last one may still be running and is paid for, so look it up by its id on app.pipelex.com rather than running the command again. A connection lost after the request that runs the method was sent is not reported as a network failure either: the hosted API may have created the run before the connection failed, and its id never arrived, so check the run history before running again. Only a failure before the request left this machine, a connection that could not be made, points at the network and the base URL. A server that keeps no runs to poll, such as a self-hosted runner, is run on the blocking route, and when the connection is cut before its result comes back, the run may still be going there: running the command again runs the method again, so raise the timeout of the proxy in front of that server, or point the base URL at a server that keeps runs. The exit code is `1`.
+**When it fails**, the message gives the hosted API's reason and a next step: the one the server advised when it gave one, else one that follows the status (a refused key points at `PIPELEX_API_KEY` and `pipelex login`, an unknown method or route at the method's address and the base URL, a rate limit says to wait). A bundle the hosted API refuses lists its validation errors, and so does a run that started and failed on them. Once the hosted API has acknowledged the run, the failure also prints the run id: a run that failed, one that outlived the wait, one that completed without a main output, and one whose result could not be read, because the network or the hosted API failed while the run was being followed. That last one may still be running and is paid for, so look it up by its id on app.pipelex.com rather than running the command again. A connection lost after the request that runs the method was sent is not reported as a network failure either: the hosted API may have created the run before the connection failed, and its id never arrived, so check the run history before running again. Only a failure before the request left this machine, a connection that could not be made, points at the network and the base URL. A server that keeps no runs to poll, such as a self-hosted runner, is run on the blocking route, and when the connection is cut before its result comes back, the run may still be going there: running the command again runs the method again, so raise the timeout of the proxy in front of that server, or point the base URL at a server that keeps runs. The exit code is `1`.
## Input File Formats
diff --git a/docs/under-the-hood/init-cli-flows.md b/docs/under-the-hood/init-cli-flows.md
index 70777eaf9..84d80faaa 100644
--- a/docs/under-the-hood/init-cli-flows.md
+++ b/docs/under-the-hood/init-cli-flows.md
@@ -1,11 +1,11 @@
---
title: "Init CLI Flows"
-description: "Inside the pipelex init command — how focus-based dispatch sets up config files, backends, routing profiles, and telemetry independently."
+description: "Inside the pipelex init command — its inspect, choose and execute stages, where runs execute, and how focus-based dispatch sets up config files, backends, routing profiles, and telemetry independently."
---
# Init CLI Flows
-`pipelex init` sets up the `.pipelex/` configuration directory. It handles four independent concerns — config files, inference backends, routing profiles, and telemetry — through a focus-based dispatch system. Each concern owns its own file-copying and customization logic, so they can be run together or individually without interference.
+`pipelex init` sets up the `.pipelex/` configuration directory and decides where runs execute: on the hosted Pipelex API, with a Pipelex API key, or on this machine, with the user's own provider keys. It handles four independent concerns — config files, inference backends, routing profiles, and telemetry — through a focus-based dispatch system, in three stages: inspect, choose, execute. Each concern owns its own file-copying and customization logic, so they can be run together or individually without interference.
---
@@ -26,9 +26,9 @@ These two categories are managed by separate steps. `init_config()` copies only
| Command | Focus | What It Does |
|---------|-------|--------------|
-| `pipelex init` | `all` | Full setup: config + inference + routing + telemetry |
-| `pipelex init config` | `config` | Copy config templates, then trigger inference if first-time |
-| `pipelex init inference` | `inference` | Interactive backend selection + routing |
+| `pipelex init` | `all` | Full setup: config, where runs execute, then inference + routing + credentials (this machine) or sign-in (hosted), then telemetry |
+| `pipelex init config` | `config` | Copy config templates, then ask where runs execute and set it up if first-time |
+| `pipelex init inference` | `inference` | Interactive backend selection + routing for runs on this machine; never asks where runs execute |
| `pipelex init routing` | `routing` | Routing profile customization only |
| `pipelex init telemetry` | `telemetry` | Telemetry config template copy |
| `pipelex init credentials` | `credentials` | Credential setup for enabled backends |
@@ -38,21 +38,21 @@ All commands except `credentials` perform a **full reset** (overwrite existing f
### Inputs
- **`focus`** (`InitFocus` enum): Determines which steps run. Derived from the CLI subcommand.
-- **`skip_confirmation`** (`bool`): When `True`, skips the interactive confirmation prompt. Used when called from `pipelex doctor --fix`.
+- **`skip_confirmation`** (`bool`): When `True`, asks nothing: skips the confirmation, keeps the `[run] execution` the target already sets (local for a `pipelex.toml` that sets none), takes the hosted Pipelex API only for a brand-new home with no `pipelex.toml`, and prints `pipelex login` instead of opening a browser. Used when called from `pipelex doctor --fix`.
- **`local`** (`bool`): When `True`, targets the project-level `.pipelex/` directory instead of the global `~/.pipelex/`. Maps to the `--local` CLI flag.
### Outputs / Side Effects
| Artifact | Produced By | Target Dir | Path |
|----------|-------------|------------|------|
-| `pipelex.toml` | `init_config()` | Project or global | `.pipelex/pipelex.toml` |
+| `pipelex.toml` | `init_config()`, then the setup path writes `[run] execution` | Project or global | `.pipelex/pipelex.toml` |
| `plxt.toml` | `init_config()` | Project or global | `.pipelex/plxt.toml` |
| `backends.toml` | Inference step | Project or global | `.pipelex/inference/backends.toml` |
| `backends/*.toml` | Inference step | Project or global | `.pipelex/inference/backends/` |
| `deck/*.toml` | Inference step | Project or global | `.pipelex/inference/deck/` |
| `routing_profiles.toml` | Inference step | Project or global | `.pipelex/inference/routing_profiles.toml` |
| `telemetry.toml` | Telemetry step | Project or global | `.pipelex/telemetry.toml` |
-| `.env` | Credentials step | **Always global** | `~/.pipelex/.env`, or `.env` under `PIPELEX_HOME` (mode 0600) |
+| `.env` | Credentials step, or the sign-in (`PIPELEX_API_KEY`) | **Always global** | `~/.pipelex/.env`, or `.env` under `PIPELEX_HOME` (mode 0600) |
!!! info "Project vs global"
Most files are written to the target directory chosen at init time (project `.pipelex/` or global `~/.pipelex/`). The exception is `.env` (credentials), which is **always** written to and read from the global directory. Wherever this page says `~/.pipelex/`, the `PIPELEX_HOME` environment variable relocates it ([Configuration](../configuration/index.md#the-home-configuration-directory-pipelex_home)).
@@ -61,6 +61,24 @@ All commands except `credentials` perform a **full reset** (overwrite existing f
## Architecture
+### Three Stages
+
+`init_cmd()` runs every focus but `credentials` through three functions:
+
+1. **Inspect** — `inspect_initialization(focus=, local=)` resolves the target directory, reads what is on disk and returns an `InitInspection`: which steps are needed, whether this run asks where runs execute (`asks_setup_path`), the `[run] execution` the target `pipelex.toml` sets now (`configured_execution`), and what a former release left (`former_release_findings` and `former_release_boot_blockers`, see below). It asks nothing and writes nothing.
+2. **Choose** — `choose_initialization(console=, inspection=, skip_confirmation=)` first offers the cleanup of what a former release left, when the inspection found any and `skip_confirmation` is false. Then it shows the confirmation panel, then, when `asks_setup_path`, the question `prompt_setup_path()` asks. It returns an `InitChoices`: the `SetupPath` (`HOSTED` or `LOCAL`, `None` when this run does not decide it) and whether anyone answers prompts.
+3. **Execute** — `execute_initialization(console=, inspection=, choices=)` writes the files and runs the steps the choices call for.
+
+`asks_setup_path` is true when the run sets up inference and its focus is `all` or `config`: the full setup always, and a configuration reset only on a first setup. `inference` configures this machine's backends on purpose and never asks.
+
+### A Former Release's Configuration
+
+A machine set up by a release that ran models through the Pipelex Gateway still carries what that took: its backend table, routing profiles that send models to it, its per-backend file and model lists, `model_specs_section` keys and `pipelex_service.toml`. What those releases left for Pipelex Manifold, which is not retired, stays as it is. The boot refuses such a machine with `FormerReleaseConfigError`, naming `pipelex migrate` and `pipelex init`.
+
+The inspect stage runs `detect_former_release_across()` (`pipelex/migration/former_release.py`) over the configuration directories a boot reads (`config_manager.existing_config_dirs`, the home's then the project's, kept in `former_release_config_dirs`), reading them together as the boot merges them, and keeps the directories with findings in `former_release_findings`. Whether what they hold stops this machine's boot is a separate reading, `former_release_boot_blockers()` over `config_manager.backends_file_paths()` and `routing_profiles_file_paths()`, the files the boot itself merges, kept in `former_release_boot_blockers`: a project booting on bases of its own is not told it cannot start because of the home's, and a profile one directory activates from the other is not missed. The choose stage then calls `offer_former_release_cleanup()`, before the confirmation: it shows a panel naming what stops the boot, when anything does, and the files it is in, asks "Clean it up now?" (default yes), and on a yes runs `apply_former_release_cleanup()` over `former_release_config_dirs`, the write pass of `pipelex migrate`'s first step, with the same copies kept. When that pass leaves a file for a hand edit, or its check after writing finds that the boot still cannot start, init says so and the setup goes on. A no leaves the files as they are and the setup goes on, with "Pipelex will not start until then" added only when `former_release_boot_blockers` is not empty. Under `skip_confirmation` (`pipelex doctor --fix`) the offer is skipped and the cleanup never runs: nobody is there to answer, and the doctor asks about the cleanup in its own migrations row.
+
+The cleanup is needed even though a full reset rewrites the target's `backends.toml` and `routing_profiles.toml`: the reset copies the kit's backend files over the target's but removes none, so the retired backend files, the model lists and `pipelex_service.toml` would stay, and so would everything in the other directory and in the personal overrides.
+
### Overall Flow
```mermaid
@@ -70,34 +88,40 @@ flowchart TD
FOCUS -- credentials --> CREDS_DIRECT["prompt_credentials<br/>Prompt for missing API keys"]
CREDS_DIRECT --> DONE
- FOCUS -- "all / config / inference / routing / telemetry" --> NEEDS[determine_needs]
- NEEDS --> CONFIRM{User confirms?}
+ FOCUS -- "all / config / inference / routing / telemetry" --> INSPECT["inspect_initialization<br/>determine_needs, first-time detection,<br/>current run execution, former release"]
+ INSPECT --> FORMER{"former release found<br/>and someone to answer?"}
+ FORMER -- Yes --> CLEANUP["offer_former_release_cleanup<br/>clean up on yes"]
+ FORMER -- No --> SKIP
+ CLEANUP --> SKIP{skip_confirmation?}
+ SKIP -- "Yes (doctor --fix)" --> DEFAULT["Setup path: the configured one, local for a<br/>pipelex.toml without it, hosted for a new home<br/>(when the run asks it)"]
+ SKIP -- No --> CONFIRM{User confirms?}
CONFIRM -- No --> CANCEL([Cancelled])
- CONFIRM -- Yes --> EXEC[execute_initialization]
+ CONFIRM -- Yes --> ASKS{asks_setup_path?}
+ ASKS -- Yes --> QUESTION["prompt_setup_path<br/>1 hosted (Enter) / 2 this machine"]
+ ASKS -- No --> EXEC
+ QUESTION --> EXEC
+ DEFAULT --> EXEC[execute_initialization]
EXEC --> S1{needs_config?}
- S1 -- Yes --> INITCFG["init_config()<br/>Copies non-inference files"]
- INITCFG --> DETECT{backends.toml<br/>existed before?}
- DETECT -- "No (first time)" --> FORCE_INF[Force needs_inference = True]
- DETECT -- Yes --> CHK_INF{check_inference<br/>in focus?}
- CHK_INF -- Yes --> FORCE_INF
- CHK_INF -- No --> S2
-
+ S1 -- Yes --> INITCFG["init_config()<br/>Copies non-inference files<br/>(writes back a kept run execution)"]
S1 -- No --> S2
-
- FORCE_INF --> S2
+ INITCFG --> S2
S2{needs_inference?}
S2 -- Yes --> COPY_INF["Copy inference templates<br/>(backends, deck, routing)"]
- COPY_INF --> CUST_BE[customize_backends_config<br/>Interactive backend selection]
+ COPY_INF --> PATH{setup path?}
+ PATH -- hosted --> HOSTED["Write execution = hosted<br/>Suggest IDE extension"]
+ PATH -- "local / none" --> CUST_BE[customize_backends_config<br/>Interactive backend selection]
CUST_BE --> CHK_RT{check_routing?}
CHK_RT -- "No (auto-route)" --> CUST_RT[customize_routing_profile<br/>Auto-routing based on selection]
- CUST_RT --> S25
- CHK_RT -- "Yes (focus=routing)" --> S25
+ CHK_RT -- "Yes (focus=routing)" --> LOCAL_SET
+ CUST_RT --> LOCAL_SET["Write execution = local<br/>(when the run asked)"]
+ HOSTED --> S25
+ LOCAL_SET --> S25
S2 -- No --> S25
- S25{check_credentials?}
+ S25{check_credentials<br/>and not hosted?}
S25 -- Yes --> CREDS["prompt_credentials<br/>Prompt for missing API keys"]
S25 -- No --> S3
CREDS --> S3
@@ -110,9 +134,13 @@ flowchart TD
S4{needs_telemetry?}
S4 -- Yes --> TELEM[setup_telemetry<br/>Copy telemetry template]
- S4 -- No --> DONE2([Done])
+ S4 -- No --> S5
+ TELEM --> S5
- TELEM --> DONE2
+ S5{setup path hosted?}
+ S5 -- Yes --> KEY["ensure_pipelex_api_key<br/>key set: keep it / interactive: pipelex login /<br/>doctor --fix: print pipelex login"]
+ S5 -- No --> DONE2([Done])
+ KEY --> DONE2
```
---
@@ -121,7 +149,7 @@ flowchart TD
### Determine Needs
-`determine_needs()` evaluates the current state of `.pipelex/` to decide which steps are required:
+`determine_needs()`, called by the inspect stage, evaluates the current state of `.pipelex/` to decide which steps are required:
```python
nb_missing_config_files = init_config(reset=False, dry_run=True) if check_config else 0
@@ -144,6 +172,29 @@ The `check_*` flags are derived from the `focus` parameter:
!!! info "Routing is separate from inference"
`check_routing` is only `True` for `focus=routing`. When `focus=all`, routing is handled automatically as part of the inference step (Step 2), not as a standalone step.
+### First-Time Inference Detection
+
+`init_config()` never copies `inference/`, so a configuration reset on a directory with no `inference/backends.toml` is a first setup. The inspect stage sets up inference too in that case, whatever the focus:
+
+```python
+if needs_config and is_first_time_backends_setup:
+ needs_inference = True
+```
+
+| Condition | Meaning | Action |
+|-----------|---------|--------|
+| `needs_config` and no `backends.toml` | First-time setup (no inference yet) | Force the inference step, and ask where runs execute |
+| `check_inference` | Inference in focus | Re-run inference (reset) |
+
+### Where Runs Execute
+
+`pipelex/cli/commands/init/setup_path.py` holds the choice: `SetupPath` (`HOSTED`, `LOCAL`), `DEFAULT_SETUP_PATH` (hosted, the answer Enter takes, and the one `doctor --fix` takes for a brand-new home with no `pipelex.toml`; otherwise `InitInspection.unattended_setup_path` keeps the configured `[run] execution`, and local for a `pipelex.toml` that sets none), and the tomlkit edit of `[run] execution`, `write_run_execution()` and `read_run_execution()`, which keep every other line of `pipelex.toml`, comments included. The question itself is `prompt_setup_path()` in `ui/setup_path_ui.py`.
+
+- **Hosted** writes `execution = "hosted"` in Step 2 and reconfigures nothing after the reset, so the inference files are the kit's defaults, any customised backends and routing included, until `pipelex init inference` reconfigures them; it skips the credentials step, and, once every file is written (Step 5), calls `ensure_pipelex_api_key()`: a key already in `PIPELEX_API_KEY` or saved in the home `.env` is kept with no login and no network call; otherwise an interactive run calls `login_with_browser()` from `pipelex/cli/commands/login/` (see [`pipelex login`](../tools/cli/login.md)), and a run with nobody to answer prints `pipelex login`. An `OSError` in that step, such as a loopback listener that cannot bind, is reported as a saved setup with no key, naming `pipelex login` and `pipelex login --paste`, since every file is already written.
+- **Local** runs the backend and routing steps unchanged, writes `execution = "local"`, and runs the credentials step.
+
+A configuration reset that does not ask the question (`pipelex init config` on an existing setup) copies the kit's `pipelex.toml`, whose `execution` is `"local"`, so the execute stage writes back the `configured_execution` the inspection read, and skips the credentials step when that is `"hosted"`.
+
### Step 1: Config Step — `init_config()`
Copies the config template tree from `kit/configs/` to `.pipelex/`, with two skip mechanisms:
@@ -164,26 +215,6 @@ if os.path.isdir(src_item):
# recurse...
```
-### First-Time Inference Detection
-
-After `init_config()` runs, `execute_initialization` decides whether inference setup is needed — even if inference was not in the original focus:
-
-```python
-backends_existed_before = path_exists(backends_toml_path)
-init_config(reset=reset)
-backends_exists_now = path_exists(backends_toml_path)
-
-if not backends_existed_before or (check_inference and backends_exists_now):
- needs_inference = True
-```
-
-This handles two scenarios:
-
-| Condition | Meaning | Action |
-|-----------|---------|--------|
-| `not backends_existed_before` | First-time setup (no inference yet) | Force inference step regardless of focus |
-| `check_inference and backends_exists_now` | Inference in focus + existing config | Re-run inference (reset) |
-
### Step 2: Inference Step
When `needs_inference` is `True` and `reset` is `True`, the inference step copies its own template files independently:
@@ -193,16 +224,17 @@ When `needs_inference` is `True` and `reset` is `True`, the inference step copie
3. `deck/*.toml` — model deck configurations
4. `routing_profiles.toml` — routing profile definitions
-Then runs interactive customization:
+A reset therefore puts customised backends and routing back to the kit's defaults, which the confirmation panel says. Then, on the hosted path, it writes `execution = "hosted"` and offers the IDE extension, customizing nothing, so the defaults stay until `pipelex init inference`. On the local path, or when the run does not decide where runs execute (`focus=inference`), it runs interactive customization:
1. `customize_backends_config()` — prompts user to select backends and suggests IDE extension installation via `suggest_extension_install_if_needed()`
2. `customize_routing_profile()` — auto-configures routing based on selected backends (**only when `check_routing` is `False`**, i.e. when routing is not the specific focus)
+3. On the local path, `execution = "local"` is written
When `focus=routing`, the inference step skips routing entirely because Step 3 handles it as a standalone operation.
### Step 2.5: Credentials Step
-When `check_credentials` is `True` (for `focus` in `all`, `config`, `inference`), `prompt_credentials()` checks which backends are enabled and prompts the user for any missing API keys. Credentials are stored in `~/.pipelex/.env` with mode 0600.
+When `check_credentials` is `True` (for `focus` in `all`, `config`, `inference`) and runs do not execute on the hosted Pipelex API, `prompt_credentials()` checks which backends are enabled and prompts the user for any missing API keys. Credentials are stored in `~/.pipelex/.env` with mode 0600.
### Step 3: Routing Step
@@ -217,16 +249,23 @@ Copies a telemetry template and prints instructions. No interactive prompts. Whi
`setup_telemetry()` selects the template via a `for_project: bool` arg passed down from the `--local` flag on `pipelex init`.
+### Step 5: Pipelex API Key
+
+On the hosted path only, `ensure_pipelex_api_key()` runs last, so every file is written before a sign-in that can take minutes. A login that saves nothing leaves the setup as written and names `pipelex login`.
+
---
## Scenario Matrix
| Scenario | Focus | Config Step | Inference Step | Credentials | Routing Step | Telemetry Step |
|----------|-------|:-----------:|:--------------:|:-----------:|:------------:|:--------------:|
-| Fresh project, full init | `all` | Copies config files | Copies templates + interactive selection | Prompted | Auto (part of inference) | Copies template |
-| Fresh project, config only | `config` | Copies config files | Forced (first-time detected) | Prompted | Auto (part of inference) | Skipped |
-| Existing project, full re-init | `all` | Overwrites config files | Resets templates + interactive selection | Prompted | Auto (part of inference) | Overwrites template |
-| Existing project, config only | `config` | Overwrites config files | Skipped (backends already exist) | Prompted | Skipped | Skipped |
+| Fresh project, full init, hosted | `all` | Copies config files | Copies templates, writes `execution = "hosted"` | Skipped (sign-in instead) | Skipped | Copies template |
+| Fresh project, full init, this machine | `all` | Copies config files | Copies templates + interactive selection, writes `execution = "local"` | Prompted | Auto (part of inference) | Copies template |
+| Fresh project, config only | `config` | Copies config files | Forced (first-time detected), asks where runs execute | Prompted on this machine | Auto on this machine | Skipped |
+| `doctor --fix`, brand-new home, no `pipelex.toml` | `config` | Copies config files | Forced, hosted taken without asking | Skipped (prints `pipelex login`) | Skipped | Skipped |
+| `doctor --fix`, no `backends.toml`, `pipelex.toml` present | `config` | Copies config files | Forced, the configured setting kept without asking, local when it sets none | Prompted on this machine | Auto on this machine | Skipped |
+| Existing project, full re-init | `all` | Overwrites config files | Resets templates, asks where runs execute again | Prompted on this machine | Auto on this machine | Overwrites template |
+| Existing project, config only | `config` | Overwrites config files, keeps `[run] execution` | Skipped (backends already exist) | Prompted unless hosted | Skipped | Skipped |
| Existing project, inference only | `inference` | Skipped | Resets templates + interactive selection | Prompted | Auto (part of inference) | Skipped |
| Existing project, routing only | `routing` | Skipped | Skipped | Skipped | Resets template + interactive selection | Skipped |
| Existing project, credentials only | `credentials` | Skipped | Skipped | Prompted | Skipped | Skipped |
@@ -262,13 +301,17 @@ Copies a telemetry template and prints instructions. No interactive prompts. Whi
| Module | Purpose |
|--------|---------|
-| `pipelex/cli/commands/init/command.py` | Orchestration: `init_cmd()`, `execute_initialization()`, `determine_needs()` |
+| `pipelex/cli/commands/init/command.py` | Orchestration: `init_cmd()`, the three stages `inspect_initialization()`, `choose_initialization()`, `execute_initialization()`, `determine_needs()`, and `offer_former_release_cleanup()` |
+| `pipelex/migration/former_release.py`, `pipelex/migration/former_release_cleanup.py` | What a former release left: `detect_former_release_across()`, the boot's verdict `former_release_boot_blockers()`, and the cleanup `clean_former_release()` that `pipelex migrate` runs first, with its check after writing, `what_stops_the_boot()` |
+| `pipelex/cli/commands/init/setup_path.py` | Where runs execute: `SetupPath`, `write_run_execution()`, `read_run_execution()`, `ensure_pipelex_api_key()` |
+| `pipelex/cli/commands/init/ui/setup_path_ui.py` | The question: `prompt_setup_path()` |
+| `pipelex/cli/commands/login/` | `pipelex login`: the loopback listener, the key store, `login_with_browser()` and `login_with_paste()` |
| `pipelex/cli/commands/init/config_files.py` | Config file copying: `init_config()`, skip lists |
| `pipelex/kit/template_copy.py` | The kit template walk `init_config()` shares with the first-boot fill, `copy_kit_templates()`, and the atomic writes `copy_file_atomically()` and `write_text_atomically()` |
| `pipelex/cli/commands/init/backends.py` | Backend customization: `customize_backends_config()`, `get_selected_backend_keys()` |
| `pipelex/cli/commands/init/routing.py` | Routing customization: `customize_routing_profile()` |
| `pipelex/cli/commands/init/telemetry.py` | Telemetry setup: `setup_telemetry()` |
-| `pipelex/cli/commands/init/credentials.py` | Credential prompting: `prompt_credentials()`, `write_env_file()`, `read_env_file()` |
+| `pipelex/cli/commands/init/credentials.py` | Credential prompting: `prompt_credentials()`, `write_env_file()`, `read_env_file()`, and the single-entry `set_env_file_entry()` the sign-in saves its key with |
| `pipelex/cli/commands/init/ide_extension.py` | IDE extension suggestion: `suggest_extension_install_if_needed()` |
| `pipelex/cli/commands/init/ui/types.py` | `InitFocus` enum definition |
@@ -338,7 +381,7 @@ The copy is `copy_kit_templates()` in `pipelex/kit/template_copy.py`, the same w
## Agent CLI Init
-`pipelex-agent init` is the non-interactive counterpart to `pipelex init`. It runs the same steps but takes all inputs from a JSON config argument instead of interactive prompts.
+`pipelex-agent init` is the non-interactive counterpart to `pipelex init`. It runs the same steps but takes all inputs from a JSON config argument instead of interactive prompts, and never signs in: a hosted setup reads its key from `PIPELEX_API_KEY`.
**Source:** `pipelex/cli/agent_cli/commands/init_cmd.py`
@@ -359,6 +402,7 @@ If no project root is found and `--global` is not set, the command fails with an
| Field | Type | Purpose |
|-------|------|---------|
+| `execution` | `"hosted"` or `"local"` | Written to `[run] execution`. `"hosted"` skips the backend and routing steps and refuses `backends` and `primary_backend`; absent means `"local"` |
| `backends` | `list[str]` | Backend keys to enable (e.g. `"openai"`, `"anthropic"`, `"openrouter"`). Omitted, the template's enabled set and its routing profile are kept |
| `primary_backend` | `str` | Required when 2+ backends are named. Named without `backends`, it routes the template's enabled set to it first |
@@ -371,9 +415,13 @@ flowchart TD
RESOLVE --> STEP1["Step 1: init_config()<br/>Copy config templates (skips inference/)"]
STEP1 --> STEP15["Step 1.5: Copy inference templates<br/>(backends.toml, backends/*, deck/*, routing_profiles.toml)"]
STEP15 --> STEP16["Step 1.6: Copy telemetry template<br/>(global: active defaults; project: commented-out)"]
- STEP16 --> STEP2["Step 2: Configure backends<br/>Enable requested backends in backends.toml"]
+ STEP16 --> STEP17["Step 1.7: Write [run] execution"]
+ STEP17 --> EXECUTION{execution?}
+ EXECUTION -- hosted --> HOSTED_OUT["Report execution and api_key_set"]
+ EXECUTION -- "local (or absent)" --> STEP2["Step 2: Configure backends<br/>Enable requested backends in backends.toml"]
STEP2 --> STEP3["Step 3: Configure routing<br/>Auto-derive routing profile<br/>(the template's, when the config names neither field)"]
STEP3 --> OUTPUT["Output result<br/>(Markdown, or JSON via --format json)"]
+ HOSTED_OUT --> OUTPUT
```
!!! note "No credentials step"
@@ -386,6 +434,8 @@ flowchart TD
| Prompts | Interactive (Rich prompts) | None — all input from `--config` JSON |
| Output | Rich console output | Markdown by default (`agent_success_formatted()` / `agent_error()`), structured JSON via `--format json` |
| Credentials | Prompted interactively | Not configured — use `pipelex-agent doctor` or `pipelex init credentials` |
+| Where runs execute | Asked, hosted by default | `execution` in `--config`, local by default |
+| Pipelex API key | `pipelex login` through the browser, unless one is set | Read from `PIPELEX_API_KEY`; reported as `api_key_set` |
| Focus dispatch | Supports individual focus (`config`, `inference`, etc.) | Runs config, inference, routing, telemetry (no credentials) |
| Target dir default | Global (`~/.pipelex/`), use `--local` for project | Project (`{project_root}/.pipelex/`), use `--global` for global |
@@ -395,6 +445,10 @@ flowchart TD
`pipelex doctor` and `pipelex doctor --fix` use the config directory resolution to find and fix configuration issues wherever they live.
+### Where Runs Execute
+
+Once the configuration files are healthy, `resolve_doctor_run_execution()` reads the effective `[run] execution`, local when the configuration does not load. A hosted run boots nothing on this machine, so under `"hosted"` the backend credentials and models rows, which load this machine's backends, are shown as information, needed only for `--local` runs: they count toward neither the overall status nor the exit code, and no provider key is asked for. A Pipelex API Key row counts instead: `check_pipelex_api_key_set()` finds the key where a run reads it, `PIPELEX_API_KEY` and then the home `.env`, healthy when it is set and starts with `plx_sk_`, otherwise naming `pipelex login`. It makes no network call and never prints the key. `pipelex-agent doctor` reports the same: `execution` at the top level, `checks.pipelex_api_key` with a `finding` of `set`, `not_a_pipelex_key` or `missing`, and `informational: true` on the two rows it does not count.
+
**Source:** `pipelex/cli/commands/doctor_cmd.py`
### How Each Check Resolves Files
@@ -407,7 +461,7 @@ flowchart TD
| `check_backend_files()` | `resolve_config_file()` for `inference/backends/`, `backends_file_paths()` for the enabled-backend list | Finds `inference/backends/` wherever it lives, and reads the merged backends document to decide which backends to probe |
| `check_pending_migrations()` | none | **Both directories, always** — see below |
-`check_pending_migrations()` is the one check that takes no `config_dir` at all, and that is deliberate. Every other row reports on a *file* and is scoped to the directory the doctor was pointed at, `--global` included. That one reports on a *command* — it is `pipelex migrate`'s own dry run — and `pipelex migrate` has no `--global`: it walks the global `~/.pipelex/` and the project `.pipelex/` both. Scoping the row narrower would name a command that then rewrites a file the row never mentioned. Every file it reports is named with its full path, so the wider scope stays legible.
+`check_pending_migrations()` is the one check that takes no `config_dir` at all, and that is deliberate. It also reports the command's first step, the cleanup of what a former release left: `former_release_files` names the files the cleanup would rewrite or remove, and `former_release_blocks_boot` says whether what they hold stops this machine's boot, read by `former_release_boot_blockers()` over `config_manager.backends_file_paths()` and `routing_profiles_file_paths()` inside the row's own error handling, so a failure reading them leaves the row unchecked rather than escaping. The dry run of the replay leaves out the files the cleanup removes, as `pipelex migrate` does. While what a former release left stops the boot, `check_models()` does not load anything: the boot would stop on that before any backend, so the row says so and names `pipelex migrate` rather than quoting whichever refusal about one backend or one profile it would meet first. `check_backend_credentials()` skips a retired backend's table, since its key is no remedy. Every other row reports on a *file* and is scoped to the directory the doctor was pointed at, `--global` included. That one reports on a *command* — it is `pipelex migrate`'s own dry run — and `pipelex migrate` has no `--global`: it walks the global `~/.pipelex/` and the project `.pipelex/` both. Scoping the row narrower would name a command that then rewrites a file the row never mentioned. Every file it reports is named with its full path, so the wider scope stays legible.
`check_backend_files()` loads the backend library once per enabled backend, and loads it without resolving credentials (`CredentialResolution.SKIP`, the keyless boot's mode). The row reports on file shape — an unknown key, a spec that is not a table, a missing per-backend file — so a backend's credentials are the Credentials row's finding, not this one's. Resolving them here would report an enabled backend whose key is not set yet as a backend-configuration error, and because the loader stops at the first backend it cannot load, a malformed file listed after that backend in `backends.toml` would go unreported. Skipping credentials skips nothing else: every enabled backend is loaded, a malformed file stays fatal, and so do the refusals of an enabled backend that declares no model or still carries `model_specs_section`. Every probe sees that same first failure, so a failure is charged to the backend the error declares (`backend_name`, stamped by the loader on every error about one backend) or, for an error that declares none, to the backend whose file it names — never to a backend whose name merely appears in the message's prose. `check_models()` attributes the same way and for the same reason: its own load resolves credentials, which the probe's does not, so it can be the first thing to name a broken backend, and it writes into the very reports this row produced.
@@ -424,7 +478,7 @@ replace_backend_file(backend_name, config_dir=resolved_config_dir)
This ensures the fix targets the same directory where the issue was found — if the broken file was in the project `.pipelex/`, the replacement goes there too, not to `~/.pipelex/`.
-The migration fix is the exception that follows from the row above it: it calls `apply_pending_migrations(config_dirs=config_directories_to_migrate())` and so writes to both directories, because that is what the command it is running does. It calls that helper rather than `migrate_cmd` itself — the command exits the process when a run leaves something for a person, which would cut off the doctor's remaining fixes, its remaining rows and its own exit code.
+The migration fix is the exception that follows from the row above it: it calls `apply_former_release_cleanup()` first when the row lists files a former release left, then `apply_pending_migrations(config_dirs=config_directories_to_migrate(), skipped_paths=...)` with the files the cleanup removed left out, and so writes to both directories, because that is what the command it is running does. When the cleanup leaves something for a person — a file it could not clean, or a boot its check after writing finds still refused — the fix says so rather than printing a plain success. It calls that helper rather than `migrate_cmd` itself — the command exits the process when a run leaves something for a person, which would cut off the doctor's remaining fixes, its remaining rows and its own exit code.
---
diff --git a/mkdocs.yml b/mkdocs.yml
index 962e45149..800c03cdd 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -291,6 +291,7 @@ plugins:
Reference:
- tools/cli/index.md: "CLI Reference Overview"
- tools/cli/init.md: "CLI Init"
+ - tools/cli/login.md: "CLI Login"
- tools/cli/update.md: "CLI Update"
- tools/cli/migrate.md: "CLI Migrate"
- tools/cli/validate.md: "CLI Validate"
@@ -532,6 +533,7 @@ nav:
- CLI Reference:
- Overview: tools/cli/index.md
- Init: tools/cli/init.md
+ - Login: tools/cli/login.md
- Update: tools/cli/update.md
- Migrate: tools/cli/migrate.md
- Validate: tools/cli/validate.md
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 1ef9cd89b..404604dc7 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -12,6 +12,7 @@ from pipelex.cli.commands.fix.app import fix_app
from pipelex.cli.commands.graph_cmd import graph_app
from pipelex.cli.commands.init.command import init_cmd
from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.cli.commands.login.command import login_cmd
from pipelex.cli.commands.migrate_cmd import migrate_cmd
from pipelex.cli.commands.plugins_cmd import plugins_app
from pipelex.cli.commands.resolve_cmd import resolve_cmd
@@ -30,6 +31,7 @@ from pipelex.tools.misc.package_utils import get_package_version
# Core commands in display order (natural ordering doesn't work between Typer groups and commands).
_CORE_COMMAND_ORDER: list[str] = [
"init",
+ "login",
"doctor",
"update",
"migrate",
@@ -151,7 +153,7 @@ def app_callback(
"""
)
# Skip checks if no command is being run (e.g., just --help) or if running setup/diagnostic commands
- if ctx.invoked_subcommand is None or ctx.invoked_subcommand in {"init", "doctor", "update", "migrate", "which"}:
+ if ctx.invoked_subcommand is None or ctx.invoked_subcommand in {"init", "login", "doctor", "update", "migrate", "which"}:
return
# Check system readiness (dependencies and venv for dev installs)
@@ -163,7 +165,10 @@ def app_callback(
warn_if_deck_stale()
[email protected](name="init", help="Initialize Pipelex configuration, backends, credentials, routing, and telemetry")
+ name="init",
+ help="Initialize Pipelex: configuration, where runs execute (the hosted Pipelex API or this machine), backends, credentials, and telemetry",
+)
def init_command(
focus: Annotated[
InitFocus,
@@ -187,13 +192,16 @@ def init_command(
Focus options:
- all Full setup: config files, backends, credentials, routing, telemetry (default)
+ all Full setup (default): config files, then where runs execute: on the hosted Pipelex API
+ (signs you in for a Pipelex API key) or on this machine (backends, routing, provider keys);
+ then telemetry
- config Reset configuration files and prompt for missing API keys
+ config Reset configuration files; on a first setup, ask where runs execute as 'all' does
credentials Prompt for missing API keys only (reads enabled backends, saves to the .env in the home configuration directory)
- inference Reset inference backends selection and prompt for missing API keys
+ inference Reset the inference backends runs on this machine use and prompt for missing API keys
+ (where runs execute stays as it is)
routing Reset routing profile
@@ -202,6 +210,25 @@ def init_command(
init_cmd(focus=focus, local=local)
[email protected](name="login", help="Get a Pipelex API key through your browser and save it for runs on the hosted Pipelex API")
+def login_command(
+ paste: Annotated[
+ bool,
+ typer.Option(
+ "--paste",
+ help="Paste a key created in the Pipelex app instead of opening a browser (for a remote machine or CI)",
+ ),
+ ] = False,
+) -> None:
+ """Get a Pipelex API key and save it to the .env in the home configuration directory (~/.pipelex/.env, or PIPELEX_HOME) as PIPELEX_API_KEY.
+
+ Opens the Pipelex app in your browser, which hands a new key back to this command on a local port. The key is checked
+ with the hosted API before it is saved, and never printed. PIPELEX_APP_URL aims the command at another Pipelex app,
+ PIPELEX_BASE_URL at another hosted API.
+ """
+ login_cmd(paste=paste)
+
+
@app.command(name="doctor", help="Check Pipelex configuration health and suggest fixes")
def doctor_command(
fix: Annotated[bool, typer.Option("--fix", "-f", help="Offer to fix detected issues interactively")] = False,
diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.md
index 2ceb8768f..3c6a3dc08 100644
--- a/pipelex/cli/agent_cli/CLAUDE.md
+++ b/pipelex/cli/agent_cli/CLAUDE.md
@@ -20,8 +20,8 @@ Markdown structure per command:
- `run`: `# Pipeline run complete`, a `## Result` section (the rendered `main_stuff` markdown with `--with-memory`, otherwise the concept JSON in a fenced block), and output / graph file paths. With the graph on, the `graph_files` object of the `--with-memory` envelope names the viewer (`graph_html`), the graph on a live run (`graph_spec`, the graphspec renamed `live_run_graph.json`) and the graph's three companions written beside it under their canonical names (`pipe_io_contracts`, `input_form`, `output_form` — the validate artifacts describing the graph's data, keyed by `pipe_ref`); the agent CLI settles its graph inclusion (graphspec on, viewer on) before the run rather than at render time, since the companions are built inside the run's library window under that same flag; a viewer resolves them from the graph file's directory, so the rename never moves them. Cost is machine-first: on the local runner, when costs are on (`--costs`, the default) and the run did reportable work (any tokens or any cost), a best-effort `cost_report` object (`{total_cost, by_model: [...]}`, real USD) rides the JSON `--with-memory` envelope — there is **no** Rich cost table on stderr (unlike the human `pipelex run` CLI). The markdown renderer ignores the `cost_report` key, so it is JSON-only. A free/zero-price model still reports its token usage (with `total_cost` 0). The `cost_report` is **absent** when the run did no reportable work (dry runs), when `--no-costs` is passed, on a hosted run (which assembles no local usage), or if cost-summary aggregation fails (caught and skipped so it never fails an otherwise-successful run) — so consumers must treat it as optional.
- `validate`: `# Validation passed`, the bundle path when relevant, and a list of validated pipes with their status. Every entry in `validated_pipes` (markdown and JSON) identifies its pipe by the namespaced `pipe_ref` (`domain.code`) — the same unambiguous identity across `validate all`, `validate bundle`, and `validate pipe`, so the same pipe is never reported under two identifiers. `validate bundle`, `validate method`, and `validate all` carry `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), namespaced by `pipe_ref` — rendered as a "Pending signatures" section in markdown. They also carry a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown adds an explicit plain-English runnability verdict on those surfaces: a complete bundle/library states it is runnable, while one with pending signatures states it is NOT yet runnable immediately above the verbatim "Pending signatures" heading. These whole-bundle/whole-library surfaces are **strict by default**: they exit non-zero on `not is_runnable` unless `--allow-signatures` is passed (which both tolerates the placeholders for the exit code and mock-runs the signature pipes in the sweep). Single-pipe surfaces make no library-wide runnability claim and never gate the exit code on it: bare `validate pipe <code>` omits the key and emits no verdict, while `validate bundle`/`validate method` with `--pipe` still surface the library-wide `pending_signatures` for information but do not gate on it (the requested slice can be fully implemented even when unrelated placeholders remain elsewhere). The whole-bundle and whole-library surfaces also carry a `warnings` array — the advisory lints, rendered as a "Warnings" section in markdown — assembled by `pipelex/pipeline/advisory_warnings.py`, the ONE composition point every advisory-bearing channel calls (this CLI, the bare CLI and the protocol report). Bare `validate pipe <code>` carries none; `validate bundle` / `validate method` with `--pipe` DO carry them, because they validate the whole bundle and only narrow the dry run, so the array stays bundle-wide there. Add a new advisory family there, never at a call site: the sites used to assemble their own and disagreed about which families they carried.
- `fix`: `# Fix applied - bundle is valid` after one or more fixes, `# Bundle already valid` when the loop had nothing to apply, or `# Bundle valid but not runnable` when pending `PipeSignature` placeholders remain. The JSON success envelope carries `is_valid`, `bundle_path`, `iterations`, `fixes_applied`, `files_written`, `remaining_errors`, and, on successful validation, `pending_signatures` plus `is_runnable`. Whole-bundle fix runs are strict by default: they emit the success envelope but exit 1 on `not is_runnable` unless `--allow-signatures` is passed. A still-invalid verdict exits 1 via `FixBundleError` on stderr with the same structured result fields plus any `bail_reason`.
-- `migrate`: `# Configuration migration`, the mode (applied / dry run) and verdict, the directories walked, a files-walked/changed/written count, then one `## <path>` section per file that is not clean listing the entries applied, the entries blocked with their reason and guidance, and any unexplained paths. Paths, operation kinds and ledger-supplied values only — no value read from the user's file is ever rendered, in either format.
-- `init`: `# Pipelex initialized` with target directory, enabled backends, and routing profile.
+- `migrate`: `# Configuration migration`, the mode (applied / dry run) and verdict, the directories walked, a files-walked/changed/written count and, when a former release left something, its file count, then one `## <path>` section per file the former-release cleanup touches, listing its changes and its backup, a `## Pipelex still cannot start` section when the cleanup's check, run on every run, found something, and one per file that is not clean listing the entries applied, the entries blocked with their reason and guidance, and any unexplained paths. Paths, operation kinds, ledger-supplied values and the names of retired backends and profiles only — no value read from the user's file is ever rendered, in either format.
+- `init`: `# Pipelex initialized` with target directory and execution, then enabled backends and routing profile for a local setup, or whether a Pipelex API key is set for a hosted one.
- error path: `# Error: <error_type>`, the message, the hint as a `> 💡` callout, the `validation_errors` items (when the cause's report, or a hosted refusal, carries any) as the same grouped prose `validate` prints, and a `## Details` section. `error_source` (internal stack frames) is omitted from markdown — it remains in the JSON envelope (`--error-format json`) for programmatic consumers.
## Companion: Agent Skills
@@ -82,7 +82,7 @@ commands/
| Command | Does |
|---------|------|
-| `init` | Initializes Pipelex configuration (non-interactive). Defaults to project `.pipelex/` at detected project root. Use `--global`/`-g` to target the home configuration directory (`~/.pipelex/`, or `PIPELEX_HOME`). Accepts `--config`/`-c` with inline JSON or file path for backends and routing (via `primary_backend`); telemetry is seeded from a template, not `--config`. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
+| `init` | Initializes Pipelex configuration (non-interactive). Defaults to project `.pipelex/` at detected project root. Use `--global`/`-g` to target the home configuration directory (`~/.pipelex/`, or `PIPELEX_HOME`). Accepts `--config`/`-c` with inline JSON or file path for `execution` (`hosted` or `local`, written to `[run] execution` with `pipelex/cli/commands/init/setup_path.py`'s `write_run_execution`), backends and routing (via `primary_backend`); `"execution": "hosted"` skips backends and routing, refuses `backends`/`primary_backend`, and reports `execution` plus `api_key_set` (never the key) instead of `backends_enabled`/`routing_profile`; a `warnings` list is added when the project around the working directory sets another `[run] execution` in its own `pipelex.toml` (`find_project_execution_shadow`); telemetry is seeded from a template, not `--config`. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `run` | Executes a pipeline (pipe\|bundle\|method subcommands), returns main_stuff + working_memory. Graph visualizations on by default (`--no-graph` to disable). `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `validate` | Dry-runs pipes/bundles/methods (pipe\|bundle\|method subcommands), returns validation status per pipe. Bundle subcommand supports `--graph` for graph visualization (with `--graph-format` for the graph renderer). `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `fix` | Applies deterministic safe fixes to a bundle file or directory (`fix bundle`) and re-validates until valid, out of fixes, or max iterations. Supports `--allow-signatures`, `--select`, `--ignore`, `--max-iterations`, and `-L/--library-dir`. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
@@ -92,8 +92,8 @@ commands/
| `codegen` | Agent mirror of the bare `pipelex codegen` family (types\|check subcommands). `types --target <flavor>` resolves the closure into the normalized crate and writes stamped typed artifacts + `codegen.lock` (write-if-changed); `check` is the offline drift check (pure hashing, no Pipelex boot — exit 0 current, 1 drift as a structured `CodegenDriftError` with `drifts[]`, 2 no/unreadable lock). Both: `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `models` | Lists available model presets, aliases, and waterfalls. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value) |
| `check-model` | Validates a model reference by `ModelDeck.is_reference_defined`, the rule a validation and `GET /v1/models/check` apply, and suggests alternatives if invalid. `--type`/`-t` for model category, a `ModelCheckCategory` (the check's categories, `doc_gen` included, which the `models` listing's `ModelCategory` leaves out), `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value) |
-| `migrate` | Migrates this machine's configuration files (the home configuration directory, `~/.pipelex/` or `PIPELEX_HOME`, and project `.pipelex/`, non-recursively) to the current schema by replaying each surface's ledger. Does **not** boot — it is the command for a configuration that cannot load. Writes only with `--yes`; `--dry-run` plans and is the default; the two together are refused with exit 2. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json`. The verdict is the structured `needs_attention`, not the exit code. |
-| `doctor` | Checks config, credentials, models health. `--global`/`-g` for global dir. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). The telemetry check carries a `finding` (`healthy`/`not_found`/`unparseable`/`out_of_date`/`invalid`) beside its `healthy` and `message`; branch on that, and note `out_of_date` is `pipelex migrate`'s and never `pipelex init telemetry`'s — a reset would discard the settings the migration keeps — and that it means the migration would actually rewrite that file, so a file it would only report on is `invalid`. `checks.pending_migrations` is the machine-wide half: a `finding` (`up_to_date`/`pending`/`needs_attention`/`unavailable`) plus `migratable_files` and `attention_files`. It is the only channel a machine has for a pending migration — a tolerated boot warns a person and this CLI silences logging — and it is **not** scoped by `--global`, because it answers for `pipelex migrate`, which walks both directories. `checks.plugins` is present once the runtime setup ran, which discovers the plugins the way boot does: `healthy` and `message`, not healthy when the plugin registry did not build, in which case the message names the plugin error. `checks.secrets_provider` is present once the runtime setup ran, which builds the provider `[runtime.secrets] method` selects before the sink, as boot does: `healthy` and `message`, not healthy when the method names no registered provider, when its factory raised or when the registry did not build. It is built even when logging was configured before the doctor ran, since `checks.models` resolves the backends' credentials through it, and when it did not build the models check says it could not run; a provider that raises on a lookup is reported in `checks.models` rather than failing the command. `checks.log_sink` is present once the runtime setup ran: `healthy` and `message`, not healthy when `[runtime.log] sink` names no registered sink, when the configured sink failed to install (a `${…}` placeholder in its settings that does not resolve among the causes), when the secrets provider did not build, in which case the sink is not attempted, or when the registry did not build. The message says why, and says where the report's own lines went: onto the console sink on stderr, which the doctor installs in place of the one that failed, or onto a sink already recorded, which it leaves alone rather than installing over — a sink is recorded before the held records are replayed, so a failed installation does not always leave nothing behind. The fourth case is its own: a sink that was installed and then failed while the records held since logging was configured were replayed through it, where the report runs on that sink, which stays installed, and the message says what it raised. A failure the `secrets_provider` and `log_sink` messages quote is scrubbed with the log redaction patterns, the configured `extra_patterns` included, because the envelope reaches its reader without passing through any sink, and so is a failure of the backends' setup that the `models` message quotes, where a secrets provider's own text can appear; the backend-file findings and the gateway and plugin-registry failures it quotes are Pipelex's own text and are quoted as raised. |
+| `migrate` | Migrates this machine's configuration files (the home configuration directory, `~/.pipelex/` or `PIPELEX_HOME`, and project `.pipelex/`, non-recursively) to the current schema by replaying each surface's ledger. Does **not** boot — it is the command for a configuration that cannot load. Its first step is the cleanup of what a former release that ran on the Pipelex Gateway left (the boot's `FormerReleaseConfigError` names this command): the JSON carries it under `former_release` (`is_clean`, `needs_attention`, `files`, each with `file_path`, `action` `rewrite`/`remove`, `changes`, `backup_path`, `was_applied`, `blocked_reason`, `blocked_detail`, and `still_blocking`, what would still stop the boot once the cleanup is done, read on every run: off the files a `--yes` run wrote, or as a dry run would write them) and counts it in `summary` (`former_release_files`, `former_release_files_cleaned`, `former_release_files_blocked`), `needs_attention` and `is_clean`. Writes only with `--yes`; `--dry-run` plans and is the default; the two together are refused with exit 2. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json`. The verdict is the structured `needs_attention`, not the exit code. |
+| `doctor` | Checks config, credentials, models health. `--global`/`-g` for global dir. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). The telemetry check carries a `finding` (`healthy`/`not_found`/`unparseable`/`out_of_date`/`invalid`) beside its `healthy` and `message`; branch on that, and note `out_of_date` is `pipelex migrate`'s and never `pipelex init telemetry`'s — a reset would discard the settings the migration keeps — and that it means the migration would actually rewrite that file, so a file it would only report on is `invalid`. `checks.pending_migrations` is the machine-wide half: a `finding` (`up_to_date`/`pending`/`needs_attention`/`unavailable`) plus `migratable_files` and `attention_files`, and `former_release_files` with `former_release_blocks_boot` for the command's first step, the cleanup of what a former release left, the verdict read off the files the boot merges; when a boot would still not start once the command has run, `recommended_actions` names `pipelex migrate --dry-run`, which says why, and with nothing left to clean that is the row's only action and its `finding` is `needs_attention`; while that stops the boot, `checks.models` is not run and its message names `pipelex migrate`, and `checks.backend_credentials` never lists a retired backend. It is the only channel a machine has for a pending migration — a tolerated boot warns a person and this CLI silences logging — and it is **not** scoped by `--global`, because it answers for `pipelex migrate`, which walks both directories. `checks.plugins` is present once the runtime setup ran, which discovers the plugins the way boot does: `healthy` and `message`, not healthy when the plugin registry did not build, in which case the message names the plugin error. `checks.secrets_provider` is present once the runtime setup ran, which builds the provider `[runtime.secrets] method` selects before the sink, as boot does: `healthy` and `message`, not healthy when the method names no registered provider, when its factory raised or when the registry did not build. It is built even when logging was configured before the doctor ran, since `checks.models` resolves the backends' credentials through it, and when it did not build the models check says it could not run; a provider that raises on a lookup is reported in `checks.models` rather than failing the command. `checks.log_sink` is present once the runtime setup ran: `healthy` and `message`, not healthy when `[runtime.log] sink` names no registered sink, when the configured sink failed to install (a `${…}` placeholder in its settings that does not resolve among the causes), when the secrets provider did not build, in which case the sink is not attempted, or when the registry did not build. The message says why, and says where the report's own lines went: onto the console sink on stderr, which the doctor installs in place of the one that failed, or onto a sink already recorded, which it leaves alone rather than installing over — a sink is recorded before the held records are replayed, so a failed installation does not always leave nothing behind. The fourth case is its own: a sink that was installed and then failed while the records held since logging was configured were replayed through it, where the report runs on that sink, which stays installed, and the message says what it raised. A failure the `secrets_provider` and `log_sink` messages quote is scrubbed with the log redaction patterns, the configured `extra_patterns` included, because the envelope reaches its reader without passing through any sink, and so is a failure of the backends' setup that the `models` message quotes, where a secrets provider's own text can appear; the backend-file findings and the gateway and plugin-registry failures it quotes are Pipelex's own text and are quoted as raised. `execution` at the top level is the effective `[run] execution` (`resolve_doctor_run_execution()`, local when the configuration does not load). Under `hosted`, `checks.pipelex_api_key` is added (`healthy`, `finding` `set`/`not_a_pipelex_key`/`missing`, `message`; presence and shape only, no network call, never the key), it counts toward `all_healthy`, and `checks.backend_credentials` and `checks.models` carry `informational: true` and count toward neither `all_healthy` nor `recommended_actions`, since a hosted run boots nothing locally. |
## Key Patterns
@@ -106,6 +106,6 @@ commands/
- **Validate refusals are verdicts**: the `validate` commands run their library load inside `translate_to_validate_bundle_error` (`pipelex/pipeline/validate_bundle_translation.py`), so every input-domained refusal raised while loading a bundle, the unknown model (`PipeOperatorModelChoiceError`, an `unknown_model` item) included, reaches the command as a `ValidateBundleError` and leaves with the invalid-verdict envelope and exit 1. No `validate` command keeps an arm of its own for a model-choice error; exit 2 stays for a failure of the tool or its environment (config, runtime, security).
- **Run refusals are verdicts too**: a run loads its bundle inside the same translation (`acquire_library`), so an invalid bundle leaves `run pipe` / `run bundle` / `run method` as a `ValidateBundleError` with exit 1, through the generic `agent_error` arm, and `_assemble_error_payload` copies the report's `validation_errors` into the envelope for any cause that carries them. `run_pipeline_core` sets `library_dirs_are_callers=True`, so a refusal in a `-L` directory (or the bundle's own directory) is the same verdict; `run bundle`'s own parse of a bundle without `--pipe` goes through the translation too. So no `run` command keeps an arm of its own for a model-choice error either: an unknown model reaches it as the verdict's `unknown_model` item.
- **Hosted runs**: every `run` subcommand takes `--runner local|hosted`, `--hosted/--local` and `--base-url`; `resolve_agent_run_execution()` (`run/_run_hosted.py`) settles them through `pipelex.hosted.execution.resolve_run_execution` (the flag, then `[run] execution`, then local), refusing a `--runner` that contradicts the flag pair and a `--base-url` on a local run; a `PipelexConfigError` there carries `AGENT_INIT_FAILURE_HINT`, the hint a failed boot gives (`agent_cli_factory.py`). A hosted run boots nothing: it builds a `HostedRunRequest` from what `pipelex/cli/commands/run/_hosted_sources.py` resolves (the bundle and its library's `.mthds` files, a pipe's library, or a method's files; an address or a catalog id as `method_ref` / `method_id`, never fetched) and `run_hosted_for_agent()` runs it through `pipelex.hosted.hosted_run.run_hosted`, which reads the signature and uploads local input files (only when some input string names an existing local file, so a method with an address-based dependency runs with any other inputs), then drives the lifecycle `start_and_wait` drives step by step (the `/v1/version` handshake, `start`, `wait_for_result`, and the blocking `execute` on a server with no run store), so a failure while following an acknowledged run is a `HostedRunPollingError` naming the run, and a transport failure after the run request was sent is a `HostedRunOutcomeUnknownError` rather than an `ApiUnreachableError`. `run_hosted`'s `on_started` callback records the acknowledged run's id, so Ctrl-C (`KeyboardInterrupt`, or the `CancelledError` of a cancelled task) leaves as a `HostedRunInterruptedError` envelope carrying `pipeline_run_id`, with exit code 130. `run bundle` on a file with no `-L` sends `PIPELEXPATH`'s files (`hosted_library_dirs`), and an installed method named like a catalog id is that method. An exception no hosted arm names leaves through the command-boundary catch-all as the envelope. Its success envelope is `build_run_output()`'s, with the main output's content as `json`, an empty `markdown`, the working memory dump and `pipeline_run_id`; nothing is written to disk. `--dry-run` / `--mock-inputs` are refused on it. There is no global `--runner` any more, and no `ctx.obj`.
-- **Hosted refusals**: `agent_error_hosted()` routes pipelex-sdk's `ApiResponseError` (every `PipelexAPIClient` route raises it on a non-2xx answer) to `agent_error_api_response()`. `api_response_error_payload()` builds the envelope from the problem document alone: `error_type` is the runner's class (`ValidateBundleError` at load, as a local run reports it, or the root fault of a failed run, which a local run reports as `cause_type` under `PipelineExecutionError`; this path has no `cause_type`), `message` is the error's own `API POST /v1/start failed (<status>): <reason>` without the next step it appends, `hint` is `user_action.detail`, and `error_domain` / `error_category` / `retryable` / `validation_errors` / `pipe_code` / `pipe_stack` / `request_id` are the runner's, plus `error_code` (the problem's `code`), `retry_after_seconds` (the `Retry-After` header) and `http_status`. It deliberately bypasses `agent_error()`'s lookups: the dict hints keyed on a class name local commands (`doctor`, `check-model`) that cannot fix a remote refusal, and the domain is only the runner's to say, so one with no `error_domain` carries none. A refusal with no `user_action` — the hosted plane's own answers in front of the runner: the gateway's 403 for a bad key, the rate limiter's 429, the unreachable runner's 503 — gets its status's next step from `pipelex.hosted.error_rendering` (`HOSTED_REFUSAL_NEXT_STEPS_BY_STATUS`, the table the human CLI reads too; 401 and 403 point at `PIPELEX_API_KEY`, 404 at the method and the origin), else `HOSTED_REFUSAL_CALLER_NEXT_STEP` for any other 4xx and `HOSTED_SERVER_FAULT_NEXT_STEP` for the rest. Only a 429 that did not say otherwise is inferred `retryable`: the platform can answer a 503 after the run completed, when it failed to record the result, so its hint warns that a retry can repeat a paid run. Every other failure, the SDK's and pipelex's own `HostedRunError` subclasses alike, goes through `agent_error()` with what `describe_hosted_error()` reads off it, the view the human CLI prints too: the class, message, next step, domain and `retryable`, the `pipeline_run_id` of any failure met once the run was acknowledged (`RunFailedError`, `RunTimeoutError`, `MissingMainStuffError`, `HostedRunPollingError`, `HostedRunInterruptedError`), a `retryable` passed whenever the hosted API gave a verdict, `false` included, so `RETRYABLE_ERROR_TYPES` keyed on the runner's class never overrides it, and the `validation_errors` of a failed run or of a `HostedMethodInvalidError` (pipe-io's verdict on a method that does not load, whose items name the faulty file in `source`); a `RunFailedError` adds `run_status` and its report's category, model and provider. The markdown renderer keeps a runner item this pipelex's `ValidationErrorItem` cannot type (a newer `error_type`, an unknown field) as JSON under Details instead of failing.
+- **Hosted refusals**: `agent_error_hosted()` routes pipelex-sdk's `ApiResponseError` (every `PipelexAPIClient` route raises it on a non-2xx answer) to `agent_error_api_response()`. `api_response_error_payload()` builds the envelope from the problem document alone: `error_type` is the runner's class (`ValidateBundleError` at load, as a local run reports it, or the root fault of a failed run, which a local run reports as `cause_type` under `PipelineExecutionError`; this path has no `cause_type`), `message` is the error's own `API POST /v1/start failed (<status>): <reason>` without the next step it appends, `hint` is `user_action.detail`, and `error_domain` / `error_category` / `retryable` / `validation_errors` / `pipe_code` / `pipe_stack` / `request_id` are the runner's, plus `error_code` (the problem's `code`), `retry_after_seconds` (the `Retry-After` header) and `http_status`. It deliberately bypasses `agent_error()`'s lookups: the dict hints keyed on a class name local commands (`doctor`, `check-model`) that cannot fix a remote refusal, and the domain is only the runner's to say, so one with no `error_domain` carries none. A refusal with no `user_action` — the hosted plane's own answers in front of the runner: the gateway's 403 for a bad key, the rate limiter's 429, the unreachable runner's 503 — gets its status's next step from `pipelex.hosted.error_rendering` (`HOSTED_REFUSAL_NEXT_STEPS_BY_STATUS`, the table the human CLI reads too; 401 and 403 point at `PIPELEX_API_KEY` and `pipelex login`, 404 at the method and the origin), else `HOSTED_REFUSAL_CALLER_NEXT_STEP` for any other 4xx and `HOSTED_SERVER_FAULT_NEXT_STEP` for the rest. Only a 429 that did not say otherwise is inferred `retryable`: the platform can answer a 503 after the run completed, when it failed to record the result, so its hint warns that a retry can repeat a paid run. Every other failure, the SDK's and pipelex's own `HostedRunError` subclasses alike, goes through `agent_error()` with what `describe_hosted_error()` reads off it, the view the human CLI prints too: the class, message, next step, domain and `retryable`, the `pipeline_run_id` of any failure met once the run was acknowledged (`RunFailedError`, `RunTimeoutError`, `MissingMainStuffError`, `HostedRunPollingError`, `HostedRunInterruptedError`), a `retryable` passed whenever the hosted API gave a verdict, `false` included, so `RETRYABLE_ERROR_TYPES` keyed on the runner's class never overrides it, and the `validation_errors` of a failed run or of a `HostedMethodInvalidError` (pipe-io's verdict on a method that does not load, whose items name the faulty file in `source`); a `RunFailedError` adds `run_status` and its report's category, model and provider. The markdown renderer keeps a runner item this pipelex's `ValidationErrorItem` cannot type (a newer `error_type`, an unknown field) as JSON under Details instead of failing.
- **Method-reference failures**: the `run`/`validate`/`inputs` method commands call `resolve_method_target(..., raise_ref_errors=True)` and shape the typed `MethodRefError` subclasses into `agent_error(...)` themselves (validate uses `exit_code=2`, its no-verdict convention) — the human CLI's red-text rendering stays in `resolve_method_target`'s default arm.
- **TOML handling**: Uses `tomlkit` (not `tomllib`) to preserve formatting and inline tables.
diff --git a/pipelex/cli/agent_cli/commands/agent_cli_factory.py b/pipelex/cli/agent_cli/commands/agent_cli_factory.py
index f96268c61..947ac4b78 100644
--- a/pipelex/cli/agent_cli/commands/agent_cli_factory.py
+++ b/pipelex/cli/agent_cli/commands/agent_cli_factory.py
@@ -9,6 +9,7 @@ from typing import Any
from pipelex.cli.agent_cli.commands.agent_output import agent_error
from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+from pipelex.migration.exceptions import FormerReleaseConfigError
from pipelex.pipelex import Pipelex
from pipelex.runtime_hub import RuntimeHub
from pipelex.system.console_target import ConsoleTarget
@@ -206,6 +207,10 @@ def make_pipelex_for_agent_cli(*, library_dirs: list[str] | list[Path] | None =
)
except TelemetryConfigValidationError as exc:
agent_error(exc.message, error_type="TelemetryConfigValidationError", cause=exc)
+ except FormerReleaseConfigError as exc:
+ # Its own hint rather than the generic one, which offers a reset: the cleanup keeps everything but what a former
+ # release left, and an agent can run it.
+ agent_error(exc.message, error_type="FormerReleaseConfigError", cause=exc)
except ModelDeckPresetValidatonError as exc:
agent_error(
exc.message,
diff --git a/pipelex/cli/agent_cli/commands/agent_output.py b/pipelex/cli/agent_cli/commands/agent_output.py
index a3651bb10..e37898df4 100644
--- a/pipelex/cli/agent_cli/commands/agent_output.py
+++ b/pipelex/cli/agent_cli/commands/agent_output.py
@@ -130,6 +130,13 @@ AGENT_ERROR_HINTS: dict[str, str] = {
"False means the migration would write nothing and its 'plans' name what to correct by hand. "
"Without a 'migration' field, correct the telemetry.toml settings named in the message."
),
+ # A configuration a former release set up for the Pipelex Gateway: the cleanup is the loop, and
+ # a reset would discard every other setting the cleanup keeps.
+ "FormerReleaseConfigError": (
+ "This configuration still carries what a former release left for the Pipelex Gateway. "
+ "Run 'pipelex-agent migrate --dry-run --format json' to see what the cleanup would remove (its 'former_release' key), "
+ "show the user, then run 'pipelex-agent migrate --yes'. It keeps a copy of each file it changes or removes."
+ ),
"BinaryNotFoundError": "Install pipelex-tools: uv tool install pipelex-tools",
# Graph errors
"GraphSpecParseError": "Validate graphspec.json structure; ensure it matches the expected GraphSpec schema",
diff --git a/pipelex/cli/agent_cli/commands/doctor_cmd.py b/pipelex/cli/agent_cli/commands/doctor_cmd.py
index ad2de92b9..948be63cf 100644
--- a/pipelex/cli/agent_cli/commands/doctor_cmd.py
+++ b/pipelex/cli/agent_cli/commands/doctor_cmd.py
@@ -12,6 +12,7 @@ from pipelex.cli.agent_cli.commands.agent_cli_factory import (
)
from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_error, agent_success, set_agent_cli_error_format
from pipelex.cli.commands.doctor_cmd import (
+ LOCAL_RUNS_ONLY_NOTE,
BackendFileReport,
ConfigLocationInfo,
LogSinkCheck,
@@ -25,11 +26,15 @@ from pipelex.cli.commands.doctor_cmd import (
check_config_files,
check_models,
check_pending_migrations,
+ check_pipelex_api_key_set,
check_telemetry_config,
gather_config_location,
+ resolve_doctor_run_execution,
setup_doctor_runtime,
)
from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.hosted.error_rendering import PIPELEX_LOGIN_COMMAND
+from pipelex.hosted.run_config import RunExecution
from pipelex.system.configuration.config_loader import config_manager
from pipelex.tools.log.log import log
@@ -39,8 +44,15 @@ def _status_icon(*, healthy: bool) -> str:
return "\u2705" if healthy else "\u26a0\ufe0f"
+def _informational_or_status_icon(*, check: dict[str, Any]) -> str:
+ """The row's status icon, or an information sign for a row a hosted setup reports without counting it."""
+ if check.get("informational") and not check["healthy"]:
+ return "\u2139\ufe0f"
+ return _status_icon(healthy=check["healthy"])
+
+
def _pending_migrations_actions(*, check: PendingMigrationsCheck) -> list[str]:
- """What to tell an agent about the pending-migrations row — none, one move, or two.
+ """What to tell an agent about the pending-migrations row: nothing when it is up to date, else one action per thing it found.
A machine consumer never learns of a pending migration from a boot: a stale configuration the
ledger can explain boots with a warning, and the agent CLI silences its own logging before one
@@ -49,23 +61,35 @@ def _pending_migrations_actions(*, check: PendingMigrationsCheck) -> list[str]:
Two actions rather than one when a run both migrates some files and leaves others behind,
which is the ordinary shape on a machine that has drifted: the command is worth running *and*
something is still owed afterwards, and an agent that only heard the first would stop early.
+ The cleanup of what a former release left is the command's first step, and has its own action. A boot the command
+ would leave refused has one too, and it can be the only one: with nothing left to clean, the row lists no file.
"""
actions: list[str] = []
match check.finding:
case PendingMigrationsFinding.UP_TO_DATE:
return actions
- case PendingMigrationsFinding.PENDING:
- migratable = ", ".join(check.migratable_files)
- actions.append(f"Run '{MIGRATE_COMMAND}' to bring these configuration files up to date (it keeps the settings in them): {migratable}")
- case PendingMigrationsFinding.NEEDS_ATTENTION:
+ case PendingMigrationsFinding.PENDING | PendingMigrationsFinding.NEEDS_ATTENTION:
pass
case PendingMigrationsFinding.UNAVAILABLE:
actions.append(f"{check.message}. Run '{MIGRATE_COMMAND} --dry-run' to check by hand.")
+ if check.former_release_files:
+ stops = ", which stops every boot until it is gone" if check.former_release_blocks_boot else ""
+ actions.append(
+ f"Run '{MIGRATE_COMMAND}' to clean up what a former release left for the Pipelex Gateway{stops} "
+ f"(it keeps a copy of each file): {', '.join(check.former_release_files)}"
+ )
+ if check.migratable_files:
+ migratable = ", ".join(check.migratable_files)
+ actions.append(f"Run '{MIGRATE_COMMAND}' to bring these configuration files up to date (it keeps the settings in them): {migratable}")
if check.attention_files:
attention = ", ".join(check.attention_files)
actions.append(
f"Run '{MIGRATE_COMMAND} --dry-run' to see what these configuration files carry that the migration will not do on its own: {attention}"
)
+ if check.boot_still_blocked:
+ actions.append(
+ f"Run '{MIGRATE_COMMAND} --dry-run' to see what still stops Pipelex from starting, which the command leaves for a person to fix"
+ )
return actions
@@ -109,6 +133,9 @@ def _format_doctor_markdown(result: dict[str, Any]) -> str:
f"**Status:** {status_text}",
f"**Config location:** {config_location['config_dir']} ({location_type})",
]
+ execution = result.get("execution")
+ if execution is not None:
+ lines.append(f"**Execution:** `{execution}`")
# Config Files
config_check = checks["config_files"]
@@ -119,6 +146,8 @@ def _format_doctor_markdown(result: dict[str, Any]) -> str:
migrations_check = checks["pending_migrations"]
lines.append(f"\n## Configuration Migrations \u2014 {_status_icon(healthy=migrations_check['healthy'])}\n")
lines.append(migrations_check["message"])
+ for file_path in migrations_check.get("former_release_files", []):
+ lines.append(f"- `{file_path}`: left by a former release")
for file_path in migrations_check["migratable_files"]:
lines.append(f"- `{file_path}`: out of date")
for file_path in migrations_check["attention_files"]:
@@ -147,10 +176,18 @@ def _format_doctor_markdown(result: dict[str, Any]) -> str:
lines.append(f"\n## Log Sink \u2014 {_status_icon(healthy=log_sink_check['healthy'])}\n")
lines.append(log_sink_check["message"])
+ # Pipelex API Key, present only when runs execute on the hosted Pipelex API
+ api_key_check = checks.get("pipelex_api_key")
+ if api_key_check is not None:
+ lines.append(f"\n## Pipelex API Key \u2014 {_status_icon(healthy=api_key_check['healthy'])}\n")
+ lines.append(api_key_check["message"])
+
# Backend Credentials
creds_check = checks["backend_credentials"]
- lines.append(f"\n## Backend Credentials \u2014 {_status_icon(healthy=creds_check['healthy'])}\n")
+ lines.append(f"\n## Backend Credentials \u2014 {_informational_or_status_icon(check=creds_check)}\n")
lines.append(creds_check["message"])
+ if creds_check.get("informational"):
+ lines.append(LOCAL_RUNS_ONLY_NOTE + ".")
for backend_entry in creds_check.get("backends", []):
name = backend_entry["backend_name"]
if backend_entry["all_credentials_valid"]:
@@ -170,9 +207,11 @@ def _format_doctor_markdown(result: dict[str, Any]) -> str:
models_skipped_flag = models_check.get("skipped", False)
# Skipped state renders with the warn icon so consumers don't confuse "deferred until
# config is fixed" with a genuine models failure.
- models_icon = "\u26a0\ufe0f" if models_skipped_flag else _status_icon(healthy=models_check["healthy"])
+ models_icon = "\u26a0\ufe0f" if models_skipped_flag else _informational_or_status_icon(check=models_check)
lines.append(f"\n## Models \u2014 {models_icon}\n")
lines.append(models_check["message"])
+ if models_check.get("informational"):
+ lines.append(LOCAL_RUNS_ONLY_NOTE + ".")
for file_entry in models_check.get("backend_files", []):
name = file_entry["backend_name"]
if file_entry["is_valid"]:
@@ -316,12 +355,19 @@ def _do_agent_doctor_cmd(*, global_: bool, output_format: CliOutputFormat, error
# installed a hub or configured log — the helper guards both internally).
apply_agent_cli_output_discipline()
+ # Where runs execute decides what the setup needs: a hosted run boots nothing here, so it needs a Pipelex API key
+ # and none of the provider keys or local models the rows above check, which then count only for --local runs.
+ run_execution = resolve_doctor_run_execution() if config_healthy else RunExecution.LOCAL
+ local_inference_counts = not run_execution.is_hosted
+ pipelex_api_key_check = None if local_inference_counts else check_pipelex_api_key_set()
+
all_healthy = (
config_healthy
and pending_migrations_check.is_healthy
and telemetry_check.is_healthy
- and backends_healthy
- and models_healthy
+ and (backends_healthy or not local_inference_counts)
+ and (models_healthy or not local_inference_counts)
+ and (pipelex_api_key_check is None or pipelex_api_key_check.is_healthy)
and (log_sink_check is None or log_sink_check.is_healthy)
and (plugins_check is None or plugins_check.is_healthy)
and (secrets_provider_check is None or secrets_provider_check.is_healthy)
@@ -364,11 +410,15 @@ def _do_agent_doctor_cmd(*, global_: bool, output_format: CliOutputFormat, error
recommended_telemetry_action = _telemetry_action(
check=telemetry_check,
config_location=config_location,
- migration_already_recommended=pending_migrations_check.finding.is_repaired_by_migrating,
+ migration_already_recommended=bool(pending_migrations_check.migratable_files),
)
if recommended_telemetry_action is not None:
recommended_actions.append(recommended_telemetry_action)
- if not backends_healthy:
+ if pipelex_api_key_check is not None and not pipelex_api_key_check.is_healthy:
+ recommended_actions.append(
+ f"Have a person run '{PIPELEX_LOGIN_COMMAND}' to get a Pipelex API key for hosted runs, or set PIPELEX_API_KEY to one"
+ )
+ if not backends_healthy and local_inference_counts:
for report in backend_credential_reports.values():
if report.missing_vars:
for var_name in report.missing_vars:
@@ -376,7 +426,7 @@ def _do_agent_doctor_cmd(*, global_: bool, output_format: CliOutputFormat, error
if report.placeholder_vars:
for var_name in report.placeholder_vars:
recommended_actions.append(f"Replace placeholder value for environment variable: {var_name}")
- if not models_healthy:
+ if not models_healthy and local_inference_counts:
for file_report in backend_file_reports.values():
if not file_report.is_valid and file_report.has_kit_template:
recommended_actions.append(f"Run 'pipelex doctor --fix' to replace outdated backend config: {file_report.backend_name}")
@@ -398,6 +448,7 @@ def _do_agent_doctor_cmd(*, global_: bool, output_format: CliOutputFormat, error
result: dict[str, Any] = {
"success": True,
"all_healthy": all_healthy,
+ "execution": run_execution.value,
"config_location": config_location.model_dump(),
"checks": {
"config_files": {
@@ -411,6 +462,8 @@ def _do_agent_doctor_cmd(*, global_: bool, output_format: CliOutputFormat, error
"message": pending_migrations_check.message,
"migratable_files": pending_migrations_check.migratable_files,
"attention_files": pending_migrations_check.attention_files,
+ "former_release_files": pending_migrations_check.former_release_files,
+ "former_release_blocks_boot": pending_migrations_check.former_release_blocks_boot,
},
"telemetry": {
"healthy": telemetry_check.is_healthy,
@@ -430,6 +483,15 @@ def _do_agent_doctor_cmd(*, global_: bool, output_format: CliOutputFormat, error
},
},
}
+ if pipelex_api_key_check is not None:
+ # A hosted run boots nothing here: the provider keys and the local models are reported, not counted.
+ result["checks"]["backend_credentials"]["informational"] = True
+ result["checks"]["models"]["informational"] = True
+ result["checks"]["pipelex_api_key"] = {
+ "healthy": pipelex_api_key_check.is_healthy,
+ "finding": pipelex_api_key_check.finding.value,
+ "message": pipelex_api_key_check.message,
+ }
if plugins_check is not None:
result["checks"]["plugins"] = {"healthy": plugins_check.is_healthy, "message": plugins_check.message}
if secrets_provider_check is not None:
diff --git a/pipelex/cli/agent_cli/commands/init_cmd.py b/pipelex/cli/agent_cli/commands/init_cmd.py
index 535f77e3b..e8ff262f4 100644
--- a/pipelex/cli/agent_cli/commands/init_cmd.py
+++ b/pipelex/cli/agent_cli/commands/init_cmd.py
@@ -16,8 +16,12 @@ from pipelex.cli.agent_cli.commands.agent_output import (
)
from pipelex.cli.commands.init.backends import get_selected_backend_keys, update_backends_in_toml
from pipelex.cli.commands.init.config_files import init_config
+from pipelex.cli.commands.init.setup_path import describe_project_execution_shadow, find_project_execution_shadow, write_run_execution
from pipelex.cli.commands.init.ui.backends_ui import get_backend_options_from_toml
+from pipelex.cli.commands.login.api_key_store import find_pipelex_api_key
from pipelex.cogt.models.deck_manifest import stamp_kit_manifests
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from pipelex.hosted.run_config import RunExecution
from pipelex.kit.paths import get_kit_configs_dir
from pipelex.system.configuration.config_loader import config_manager
from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME, TELEMETRY_PROJECT_TEMPLATE_FILE_NAME
@@ -59,19 +63,67 @@ def _parse_config_arg(config_arg: str | None) -> dict[str, Any]:
def _format_init_markdown(result: dict[str, Any]) -> str:
"""Render an init result dict as agent-readable markdown."""
- backends_enabled: list[str] = result.get("backends_enabled") or []
lines: list[str] = [
"# Pipelex initialized",
"",
f"**Target directory:** `{result['target_dir']}`",
"",
- f"**Backends enabled:** {', '.join(backends_enabled) or 'none'}",
- "",
- f"**Routing profile:** `{result['routing_profile']}`",
+ f"**Execution:** `{result['execution']}`",
]
+ warning_lines = [f"**Warning:** {warning}" for warning in result.get("warnings", [])]
+ if warning_lines:
+ lines.extend(["", *warning_lines])
+ if RunExecution(result["execution"]).is_hosted:
+ key_line = (
+ f"set in `{PIPELEX_API_KEY_ENV_KEY}`"
+ if result.get("api_key_set")
+ else f"not set: set `{PIPELEX_API_KEY_ENV_KEY}` to a Pipelex API key (`plx_sk_…`) before a hosted run, "
+ "or have a person run `pipelex login`"
+ )
+ lines.extend(["", f"**Pipelex API key:** {key_line}"])
+ return "\n".join(lines)
+ backends_enabled: list[str] = result.get("backends_enabled") or []
+ lines.extend(
+ [
+ "",
+ f"**Backends enabled:** {', '.join(backends_enabled) or 'none'}",
+ "",
+ f"**Routing profile:** `{result['routing_profile']}`",
+ ]
+ )
return "\n".join(lines)
+#: The `--config` keys that configure this machine's backends, which a hosted setup does not.
+_LOCAL_ONLY_CONFIG_KEYS = ("backends", "primary_backend")
+
+
+def _resolve_execution(*, config: dict[str, Any]) -> RunExecution:
+ """Where runs execute, from the config's `execution`: `hosted` or `local`, `local` when absent.
+
+ A hosted setup configures no backend, so it refuses the keys that only configure one.
+ """
+ requested: Any = config.get("execution")
+ if requested is None:
+ return RunExecution.LOCAL
+ try:
+ execution = RunExecution(requested)
+ except ValueError:
+ agent_error(
+ f"Unknown execution: {requested!r}. Available: {', '.join(RunExecution)}",
+ error_type="ArgumentError",
+ )
+ if execution.is_hosted:
+ local_only_keys = [key for key in _LOCAL_ONLY_CONFIG_KEYS if key in config]
+ if local_only_keys:
+ agent_error(
+ f"{', '.join(local_only_keys)} configure the backends of runs on this machine, which a hosted setup does not: "
+ 'drop them, or set "execution": "local"',
+ error_type="ArgumentError",
+ )
+ return execution
+
+
def _resolve_target_dir(*, global_: bool) -> Path:
"""Resolve the target directory for initialization.
@@ -300,8 +352,10 @@ def agent_init_cmd(
"-c",
help=(
"Inline JSON string or path to a JSON file. "
- 'Schema: {"backends": list[str], "primary_backend": str}. '
+ 'Schema: {"execution": "hosted" | "local", "backends": list[str], "primary_backend": str}. '
"All fields are optional. "
+ "execution: where runs execute by default, written to [run] execution; 'hosted' runs on the hosted Pipelex API "
+ "with the key in PIPELEX_API_KEY and configures no backend; 'local' (the default) runs on this machine. "
"backends: backend keys to enable (e.g. 'openai', 'anthropic', 'openrouter'). Omit to keep template defaults "
"and the template's routing profile. "
"primary_backend: required when 2+ backends are named."
@@ -327,9 +381,10 @@ def agent_init_cmd(
) -> None:
"""Initialize Pipelex configuration (non-interactive).
- Sets up config files, inference backends, routing profile, and telemetry.
+ Sets up config files, where runs execute, inference backends, routing profile, and telemetry.
Credentials are NOT configured — use 'pipelex-agent doctor' to check credential
- health, and 'pipelex init credentials' if needed.
+ health, and 'pipelex init credentials' if needed. A hosted setup reads its key from
+ PIPELEX_API_KEY, which a person gets with 'pipelex login'.
Target directory: project .pipelex/ at detected project root by default.
Use --global/-g to force the home configuration directory (~/.pipelex/, or PIPELEX_HOME).
@@ -337,10 +392,15 @@ def agent_init_cmd(
Config JSON schema::
{
+ "execution": "local",
"backends": ["openrouter", "openai"],
"primary_backend": "openai"
}
+ - execution: where runs execute by default, written to `[run] execution` in the target
+ pipelex.toml. "hosted" runs on the hosted Pipelex API and configures no backend or
+ routing, so it refuses backends and primary_backend. "local", or no execution at all,
+ runs on this machine and configures the backends below.
- backends: list of backend keys to enable. Omit to keep all template defaults and the
template's routing profile, which routes among every backend it enables.
- primary_backend: required when 2+ backends are named. Auto-derived when only
@@ -355,6 +415,7 @@ def agent_init_cmd(
try:
# Parse config
parsed_config = _parse_config_arg(config)
+ execution = _resolve_execution(config=parsed_config)
# Resolve target directory
target_dir = _resolve_target_dir(global_=global_)
@@ -370,23 +431,40 @@ def agent_init_cmd(
# the user's global telemetry settings during layered loading.
_copy_telemetry_template(target_dir, for_project=not global_)
- # Step 2: Configure backends
- template_backends_path = Path(str(get_kit_configs_dir() / "inference" / "backends.toml"))
- backends_toml_path = target_dir / "inference" / "backends.toml"
- backends_enabled = _configure_backends(
- config=parsed_config, backends_toml_path=backends_toml_path, template_backends_path=template_backends_path
- )
-
- # Step 3: Configure routing
- routing_profile = _configure_routing(backends_enabled, config=parsed_config, target_dir=target_dir)
+ # Step 1.7: Write where runs execute
+ write_run_execution(pipelex_toml_path=target_dir / "pipelex.toml", execution=execution)
result_payload: dict[str, Any] = {
"success": True,
"target_dir": str(target_dir),
"config_files_copied": config_files_copied,
- "backends_enabled": backends_enabled,
- "routing_profile": routing_profile,
+ "execution": execution.value,
}
+ match execution:
+ case RunExecution.HOSTED:
+ # Hosted runs use none of this machine's backends: the kit's inference files stay as written. The key
+ # comes from the environment, since an agent cannot sign in through a browser.
+ result_payload["api_key_set"] = find_pipelex_api_key() is not None
+ case RunExecution.LOCAL:
+ # Step 2: Configure backends
+ template_backends_path = Path(str(get_kit_configs_dir() / "inference" / "backends.toml"))
+ backends_toml_path = target_dir / "inference" / "backends.toml"
+ backends_enabled = _configure_backends(
+ config=parsed_config, backends_toml_path=backends_toml_path, template_backends_path=template_backends_path
+ )
+
+ # Step 3: Configure routing
+ routing_profile = _configure_routing(backends_enabled, config=parsed_config, target_dir=target_dir)
+
+ result_payload["backends_enabled"] = backends_enabled
+ result_payload["routing_profile"] = routing_profile
+
+ # A project's own setting wins over the one just written, so a global choice it overrides is reported
+ shadow = find_project_execution_shadow(
+ target_config_dir=target_dir, execution=execution, project_config_dir=config_manager.project_config_dir
+ )
+ if shadow is not None:
+ result_payload["warnings"] = [describe_project_execution_shadow(shadow=shadow)]
# Output result
agent_success_formatted(result_payload, markdown_renderer=_format_init_markdown, output_format=output_format)
diff --git a/pipelex/cli/agent_cli/commands/migrate_cmd.py b/pipelex/cli/agent_cli/commands/migrate_cmd.py
index 128de81f6..dbe1f6896 100644
--- a/pipelex/cli/agent_cli/commands/migrate_cmd.py
+++ b/pipelex/cli/agent_cli/commands/migrate_cmd.py
@@ -18,6 +18,13 @@ changing.
**This command must run when nothing else does**, exactly as its human sibling: no boot, no model
deck, no credentials, no network. A broken configuration is the reason to reach for it.
+**Its first step is the cleanup of a former release**, as in the human command: what a release that
+ran on the Pipelex Gateway left, reported under ``former_release`` and counted in
+the verdict, so a machine whose boot names this command ends the loop able to boot. The replay never
+walks a file the cleanup removes, and what would still stop the boot once the cleanup is done — read on every
+run, off the files as a write left them or as a dry run would leave them, and on a machine with nothing to clean —
+is listed under ``former_release.still_blocking``, and makes the verdict ``needs_attention`` and never ``is_clean``.
+
See ``docs/migration-ledger.md``.
"""
@@ -33,6 +40,7 @@ from pipelex.cli.agent_cli.commands.agent_output import (
set_agent_cli_error_format,
)
from pipelex.cli.commands.migrate_cmd import describe_op
+from pipelex.migration.former_release_cleanup import FormerReleaseCleanup, FormerReleaseFileCleanup, clean_former_release
from pipelex.migration.plan import MigrationPlan, MigrationReport
from pipelex.migration.run import config_directories_to_migrate, migrate_config_directories
@@ -71,24 +79,27 @@ def agent_migrate_cmd(
)
config_dirs = config_directories_to_migrate()
- report = migrate_config_directories(config_dirs=config_dirs, dry_run=not yes)
- result = _result_payload(report=report, config_dirs=[str(directory) for directory in config_dirs], applied=yes)
+ # The cleanup first: the replay leaves the files it removes out of its walk.
+ cleanup = clean_former_release(config_dirs=config_dirs, dry_run=not yes)
+ report = migrate_config_directories(config_dirs=config_dirs, dry_run=not yes, skipped_paths=cleanup.removed_paths)
+ result = _result_payload(report=report, cleanup=cleanup, config_dirs=[str(directory) for directory in config_dirs], applied=yes)
agent_success_formatted(result, markdown_renderer=_render_markdown, output_format=output_format)
- if report.needs_attention:
+ if report.needs_attention or cleanup.needs_attention:
raise typer.Exit(1)
-def _result_payload(*, report: MigrationReport, config_dirs: list[str], applied: bool) -> dict[str, Any]:
- """The structured answer: the verdict, the arithmetic, and every plan the run produced.
+def _result_payload(*, report: MigrationReport, cleanup: FormerReleaseCleanup, config_dirs: list[str], applied: bool) -> dict[str, Any]:
+ """The structured answer: the verdict, the arithmetic, the cleanup, and every plan the run produced.
Every file the walk visited is here, clean ones included. A report *is* the set of files this
run looked at, and an agent deciding whether its configuration directory was even reached
- needs the ones that had nothing to say as much as the ones that did.
+ needs the ones that had nothing to say as much as the ones that did. The cleanup lists only
+ the files it touches: a former release's leftovers are found, not walked.
"""
return {
"applied": applied,
- "needs_attention": report.needs_attention,
- "is_clean": report.is_clean,
+ "needs_attention": report.needs_attention or cleanup.needs_attention,
+ "is_clean": report.is_clean and cleanup.is_clean,
"config_dirs": config_dirs,
"summary": {
"files_walked": len(report.plans),
@@ -97,6 +108,15 @@ def _result_payload(*, report: MigrationReport, config_dirs: list[str], applied:
"files_blocked": len([plan for plan in report.plans if plan.blocked_reason is not None]),
"entries_blocked": sum(len(plan.blocked) for plan in report.plans),
"unexplained_paths": sum(len(plan.unexplained) for plan in report.plans),
+ "former_release_files": len(cleanup.files),
+ "former_release_files_cleaned": len(cleanup.applied_files),
+ "former_release_files_blocked": len(cleanup.blocked_files),
+ },
+ "former_release": {
+ "is_clean": cleanup.is_clean,
+ "needs_attention": cleanup.needs_attention,
+ "files": [file.model_dump(mode="json") for file in cleanup.files],
+ "still_blocking": cleanup.still_blocking,
},
"plans": [plan.model_dump(mode="json") for plan in report.plans],
}
@@ -118,11 +138,23 @@ def _render_markdown(result: dict[str, Any]) -> str:
lines += [f"**Mode:** {mode}", f"**Verdict:** {verdict}", "", "**Directories walked:**"]
lines += [f"- `{directory}`" for directory in config_dirs]
lines += ["", f"**Files:** {summary['files_walked']} walked, {summary['files_changed']} changed, {summary['files_written']} written."]
+ if summary["former_release_files"]:
+ cleaned = f", {summary['former_release_files_cleaned']} cleaned up" if applied else ""
+ lines.append(f"**Left by a former release:** {summary['former_release_files']} file(s){cleaned}.")
if result["is_clean"]:
lines += ["", "Every configuration file walked is at the current schema."]
return "\n".join(lines)
+ for file_dict in result["former_release"]["files"]:
+ file = FormerReleaseFileCleanup.model_validate(file_dict)
+ lines += ["", f"## `{file.file_path}`", "", "Left by a former release that ran on the Pipelex Gateway.", ""]
+ lines += _former_release_lines(file=file, applied=applied)
+ still_blocking: list[str] = result["former_release"]["still_blocking"]
+ if still_blocking:
+ lines += ["", "## Pipelex still cannot start", "", "Once the cleanup is done, a boot of this machine is still refused:", ""]
+ lines += [f"- {problem}" for problem in still_blocking]
+
for plan_dict in result["plans"]:
plan = MigrationPlan.model_validate(plan_dict)
if plan.is_clean:
@@ -132,6 +164,16 @@ def _render_markdown(result: dict[str, Any]) -> str:
return "\n".join(lines)
+def _former_release_lines(*, file: FormerReleaseFileCleanup, applied: bool) -> list[str]:
+ if file.blocked_reason is not None:
+ return [f"- **This file could not be cleaned up** (`{file.blocked_reason}`): {file.blocked_detail}"]
+ verb = "Done" if applied else "Would do"
+ lines = [f"- **{verb}:** {change}" for change in file.changes]
+ if file.backup_path is not None:
+ lines.append(f"- Backup of the original: `{file.backup_path}`")
+ return lines
+
+
def _file_lines(*, plan: MigrationPlan, applied: bool) -> list[str]:
if plan.blocked_reason is not None:
return [f"- **This file could not be processed** (`{plan.blocked_reason}`): {plan.blocked_detail}"]
diff --git a/pipelex/cli/cli_factory.py b/pipelex/cli/cli_factory.py
index 315842c19..310ad1a2d 100644
--- a/pipelex/cli/cli_factory.py
+++ b/pipelex/cli/cli_factory.py
@@ -4,10 +4,12 @@ from pathlib import Path
from pipelex.cli.error_handlers import (
ErrorContext,
+ handle_former_release_config_error,
handle_model_deck_preset_error,
handle_telemetry_config_validation_error,
)
from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+from pipelex.migration.exceptions import FormerReleaseConfigError
from pipelex.pipelex import Pipelex
from pipelex.system.runtime import IntegrationMode
from pipelex.system.telemetry.exceptions import TelemetryConfigValidationError
@@ -49,3 +51,5 @@ def make_pipelex_for_cli(
handle_telemetry_config_validation_error(exc)
except ModelDeckPresetValidatonError as exc:
handle_model_deck_preset_error(exc, context=context)
+ except FormerReleaseConfigError as exc:
+ handle_former_release_config_error(exc=exc)
diff --git a/pipelex/cli/commands/doctor_cmd.py b/pipelex/cli/commands/doctor_cmd.py
index 33937d62e..c5944ed93 100644
--- a/pipelex/cli/commands/doctor_cmd.py
+++ b/pipelex/cli/commands/doctor_cmd.py
@@ -20,7 +20,9 @@ from pipelex.base_exceptions import PipelexConfigError
from pipelex.cli.commands.init.command import init_cmd
from pipelex.cli.commands.init.config_files import init_config
from pipelex.cli.commands.init.ui.types import InitFocus
-from pipelex.cli.commands.migrate_cmd import apply_pending_migrations
+from pipelex.cli.commands.login.api_key_store import find_pipelex_api_key
+from pipelex.cli.commands.login.command import LOGIN_PASTE_COMMAND
+from pipelex.cli.commands.migrate_cmd import apply_former_release_cleanup, apply_pending_migrations
from pipelex.cli.commands.update_cmd import update_cmd
from pipelex.cli.exceptions import PipelexCLIError
from pipelex.cogt.exceptions import (
@@ -50,9 +52,16 @@ from pipelex.cogt.models.deck_manifest import (
from pipelex.cogt.models.model_manager import ModelManager
from pipelex.config import get_config
from pipelex.core.validation import MIGRATE_COMMAND, raise_config_setup_error, report_validation_error
+from pipelex.hosted.api_key_check import PIPELEX_API_KEY_PREFIX, is_well_formed_pipelex_api_key
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from pipelex.hosted.error_rendering import PIPELEX_LOGIN_COMMAND
+from pipelex.hosted.execution import configured_run_execution
+from pipelex.hosted.run_config import RunExecution
from pipelex.interpreter_plugins.builtins import BUILTIN_PLUGINS, CORE_UNCONDITIONAL_PLUGIN_NAMES, ENTRY_POINT_GROUPS
from pipelex.kit.paths import get_kit_configs_dir
-from pipelex.migration.exceptions import MigrationError
+from pipelex.migration.exceptions import FormerReleaseConfigError, MigrationError
+from pipelex.migration.former_release import RETIRED_BACKEND_NAMES, former_release_boot_blockers
+from pipelex.migration.former_release_cleanup import clean_former_release
from pipelex.migration.run import config_directories_to_migrate, migrate_config_directories, scan_config_surface
from pipelex.plugins.discovery import build_registrar
from pipelex.plugins.exceptions import PluginError
@@ -340,6 +349,18 @@ class PendingMigrationsCheck(BaseModel):
A file can be on both lists: an entry that conflicts partway through is blocked, and the
operations of it that applied before the conflict are still written."""
+ former_release_files: list[str] = Field(default_factory=list[str])
+ """The files the command cleans up first: what a release that ran on the Pipelex Gateway left, each
+ rewritten without it or removed, a copy of each kept."""
+
+ former_release_blocks_boot: bool = False
+ """Whether what that release left stops this machine's boot until it is cleaned up: read off the files that boot
+ merges, the project's own bases included, never off one directory alone."""
+
+ boot_still_blocked: bool = False
+ """Whether a boot of this machine would still be refused once the command has run: the cleanup's own check, on the
+ files as it would leave them. The command cannot fix this, so its dry run, which says why, is the remedy."""
+
@property
def is_healthy(self) -> bool:
return self.finding.is_healthy
@@ -368,40 +389,144 @@ def check_pending_migrations() -> PendingMigrationsCheck:
user's files and never as an exception — an exception here reaches `doctor_cmd`'s outer
handler, which prints one line and exits, so a broken packaged ledger would replace every row
the user came for. The catch stays narrow, so a bug in our applier still surfaces as itself.
+
+ **The command's first step is the cleanup of a former release**, and the row reports it too: the
+ files a release that ran on the Pipelex Gateway left, which the command
+ removes or rewrites, and whether what is in them stops the boot. That verdict is the boot's own
+ check over the files it merges, home and project together: a project booting on bases of its own
+ is not told it cannot start because of the home's, and a profile one directory activates from
+ the other is not missed. The replay leaves the files the cleanup removes out of its walk, as the
+ command does.
"""
+ config_dirs = config_directories_to_migrate()
try:
- report = migrate_config_directories(config_dirs=config_directories_to_migrate(), dry_run=True)
+ cleanup = clean_former_release(config_dirs=config_dirs, dry_run=True)
+ report = migrate_config_directories(config_dirs=config_dirs, dry_run=True, skipped_paths=cleanup.removed_paths)
+ former_release_blocks_boot = bool(
+ former_release_boot_blockers(
+ backends_library_paths=config_manager.backends_file_paths(),
+ routing_profile_library_paths=config_manager.routing_profiles_file_paths(),
+ )
+ )
except (MigrationError, OSError) as exc:
return PendingMigrationsCheck(
finding=PendingMigrationsFinding.UNAVAILABLE,
message=f"Could not check for pending migrations: {exc}",
)
- if report.is_clean:
+ if report.is_clean and cleanup.is_clean:
return PendingMigrationsCheck(
finding=PendingMigrationsFinding.UP_TO_DATE,
message="Every configuration file is at the current schema",
)
+ former_release_files = [str(file.file_path) for file in cleanup.files if not file.is_blocked]
migratable_files = [str(plan.file_path) for plan in report.changed_plans]
# In the order the run visited them, and deduplicated: one file can be both blocked and
# carrying a path the schema cannot explain.
attention_paths = {plan.file_path for plan in report.blocked_plans} | {plan.file_path for plan in report.unexplained_plans}
- attention_files = [str(plan.file_path) for plan in report.plans if plan.file_path in attention_paths]
+ attention_files = [str(file.file_path) for file in cleanup.blocked_files]
+ attention_files += [str(plan.file_path) for plan in report.plans if plan.file_path in attention_paths]
sentences: list[str] = []
+ if former_release_files:
+ sentence = (
+ f"{len(former_release_files)} file(s) carry what a former release left for the Pipelex Gateway, which '{MIGRATE_COMMAND}' cleans up"
+ )
+ if former_release_blocks_boot:
+ sentence += " — Pipelex cannot start until it does"
+ sentences.append(sentence)
if migratable_files:
sentences.append(f"{len(migratable_files)} configuration file(s) can be brought up to date by '{MIGRATE_COMMAND}'")
if attention_files:
sentences.append(f"{len(attention_files)} configuration file(s) need a look — run '{MIGRATE_COMMAND} --dry-run' for the detail")
+ if cleanup.still_blocking:
+ # The cleanup's own check, run on the files as it would leave them: what a cleanup run from another project
+ # left half done, above all, which no file here carries any more.
+ sentences.append(f"Pipelex would still not start once '{MIGRATE_COMMAND}' has run — '{MIGRATE_COMMAND} --dry-run' says why")
+ has_work = bool(migratable_files or former_release_files)
return PendingMigrationsCheck(
- finding=PendingMigrationsFinding.PENDING if migratable_files else PendingMigrationsFinding.NEEDS_ATTENTION,
+ finding=PendingMigrationsFinding.PENDING if has_work else PendingMigrationsFinding.NEEDS_ATTENTION,
message="; ".join(sentences),
migratable_files=migratable_files,
attention_files=attention_files,
+ former_release_files=former_release_files,
+ former_release_blocks_boot=former_release_blocks_boot,
+ boot_still_blocked=bool(cleanup.still_blocking),
+ )
+
+
+class PipelexApiKeyFinding(StrEnum):
+ """What the Pipelex API key row found, for a setup whose runs execute on the hosted Pipelex API."""
+
+ SET = "set"
+ NOT_A_PIPELEX_KEY = "not_a_pipelex_key"
+ MISSING = "missing"
+
+ @property
+ def is_healthy(self) -> bool:
+ match self:
+ case PipelexApiKeyFinding.SET:
+ return True
+ case PipelexApiKeyFinding.NOT_A_PIPELEX_KEY | PipelexApiKeyFinding.MISSING:
+ return False
+
+
+class PipelexApiKeyCheck(BaseModel):
+ """The Pipelex API key row of the health report: whether hosted runs have a key to send, never the key itself."""
+
+ model_config = ConfigDict(frozen=True)
+
+ finding: PipelexApiKeyFinding
+ message: str
+
+ @property
+ def is_healthy(self) -> bool:
+ return self.finding.is_healthy
+
+
+def check_pipelex_api_key_set() -> PipelexApiKeyCheck:
+ """Whether a Pipelex API key is set where a hosted run reads it: `PIPELEX_API_KEY`, else the one saved in the home `.env`.
+
+ Only its presence and its shape are checked, with no call to the hosted API, and the key is never quoted.
+ """
+ api_key = find_pipelex_api_key()
+ if api_key is None:
+ return PipelexApiKeyCheck(
+ finding=PipelexApiKeyFinding.MISSING,
+ message=f"No Pipelex API key is set ({PIPELEX_API_KEY_ENV_KEY}): run '{PIPELEX_LOGIN_COMMAND}' to get one",
+ )
+ if not is_well_formed_pipelex_api_key(api_key=api_key):
+ return PipelexApiKeyCheck(
+ finding=PipelexApiKeyFinding.NOT_A_PIPELEX_KEY,
+ message=(
+ f"{PIPELEX_API_KEY_ENV_KEY} does not look like a Pipelex API key, which starts with {PIPELEX_API_KEY_PREFIX}: "
+ f"run '{PIPELEX_LOGIN_COMMAND}' to replace it"
+ ),
+ )
+ return PipelexApiKeyCheck(
+ finding=PipelexApiKeyFinding.SET,
+ message=f"A Pipelex API key is set ({PIPELEX_API_KEY_ENV_KEY}); the hosted API checks it when a run sends it",
)
+def resolve_doctor_run_execution() -> RunExecution:
+ """Where runs execute by default, which decides what this setup needs; local when the configuration does not load.
+
+ A hosted run boots nothing on this machine, so a hosted setup needs a Pipelex API key and no provider key.
+ """
+ try:
+ return configured_run_execution()
+ except PipelexConfigError:
+ return RunExecution.LOCAL
+
+
+#: The setting that makes runs execute on the hosted Pipelex API, as the report quotes it.
+HOSTED_EXECUTION_SETTING = '[run] execution = "hosted"'
+#: How the report says that a row only matters for runs on this machine.
+LOCAL_RUNS_ONLY_NOTE = "Needed only for --local runs: runs execute on the hosted Pipelex API by default"
+
+
def check_backend_credentials(*, config_dir: Path | None = None) -> tuple[bool, dict[str, BackendCredentialsReport], str]:
"""Check if backend credentials are properly configured.
@@ -423,8 +548,9 @@ def check_backend_credentials(*, config_dir: Path | None = None) -> tuple[bool,
all_backends_valid = True
for backend_name, backend_dict in backends_dict.items():
- # Skip internal backend
- if backend_name == "internal":
+ # Skip internal backend, and a retired one: its table is what a former release left, which the migrations
+ # row names the cleanup for, and its key is no remedy, since this release has no such backend to reach.
+ if backend_name == "internal" or backend_name in RETIRED_BACKEND_NAMES:
continue
# Only check enabled backends
@@ -687,6 +813,8 @@ def display_health_report(
log_sink_check: LogSinkCheck | None = None,
plugins_check: PluginsCheck | None = None,
secrets_provider_check: SecretsProviderCheck | None = None,
+ run_execution: RunExecution = RunExecution.LOCAL,
+ pipelex_api_key_check: PipelexApiKeyCheck | None = None,
) -> None:
"""Display a comprehensive health report.
@@ -719,16 +847,23 @@ def display_health_report(
never did, in which case the row is not rendered.
secrets_provider_check: What the runtime setup found when it built the secrets provider;
None when it never did, in which case the row is not rendered.
+ run_execution: Where runs execute by default. On the hosted Pipelex API, the backend credentials and the
+ models rows are reported as needed only for `--local` runs, and the Pipelex API key row counts instead.
+ pipelex_api_key_check: What the Pipelex API key row found; None when runs execute on this machine, in which
+ case the row is not rendered.
"""
log_sink_healthy = log_sink_check is None or log_sink_check.is_healthy
plugins_healthy = plugins_check is None or plugins_check.is_healthy
secrets_provider_healthy = secrets_provider_check is None or secrets_provider_check.is_healthy
+ local_inference_counts = not run_execution.is_hosted
+ pipelex_api_key_healthy = pipelex_api_key_check is None or pipelex_api_key_check.is_healthy
all_healthy = (
config_healthy
and pending_migrations_check.is_healthy
and telemetry_check.is_healthy
- and backends_healthy
- and models_healthy
+ and (backends_healthy or not local_inference_counts)
+ and (models_healthy or not local_inference_counts)
+ and pipelex_api_key_healthy
and deck_healthy
and internal_backend_healthy
and log_sink_healthy
@@ -816,10 +951,24 @@ def display_health_report(
console.print(f" [red]✗[/red] {escape(log_sink_check.message)}")
console.print()
+ # Pipelex API Key section: what a hosted run sends, and the only credential a hosted setup needs
+ if pipelex_api_key_check is not None:
+ console.print("[bold]Pipelex API Key[/bold]")
+ if pipelex_api_key_check.is_healthy:
+ console.print(f" [green]✓[/green] {escape(pipelex_api_key_check.message)}")
+ else:
+ console.print(f" [red]✗[/red] {escape(pipelex_api_key_check.message)}")
+ console.print(f" [dim]Runs execute on the hosted Pipelex API by default ({escape(HOSTED_EXECUTION_SETTING)})[/dim]")
+ console.print()
+
# Backend Credentials section
console.print("[bold]Backend Credentials[/bold]")
if backends_healthy:
console.print(f" [green]✓[/green] {escape(backends_message)}")
+ elif not local_inference_counts:
+ # A hosted run boots nothing here, so the provider keys it lacks are information, not a fault.
+ console.print(f" [dim]ℹ {escape(backends_message)}[/dim]")
+ console.print(f" [dim]{LOCAL_RUNS_ONLY_NOTE}.[/dim]")
elif not backend_credential_reports:
# No backends were checked (e.g., file not found)
console.print(f" [red]✗[/red] {escape(backends_message)}")
@@ -853,6 +1002,10 @@ def display_health_report(
# Skipped reads as advisory, not failure — the Config Files row is the real issue.
console.print(f" [yellow]⚠[/yellow] {escape(models_message)}")
console.print(" [dim]Models check deferred until config errors are fixed.[/dim]")
+ elif not local_inference_counts:
+ # The models row loads this machine's backends, which a hosted run never boots.
+ console.print(f" [dim]ℹ {escape(models_message.splitlines()[0] if models_message else models_message)}[/dim]")
+ console.print(f" [dim]{LOCAL_RUNS_ONLY_NOTE}.[/dim]")
else:
console.print(f" [red]✗[/red] {escape(models_message)}")
@@ -886,7 +1039,8 @@ def display_health_report(
# Check what can be auto-fixed
can_auto_fix_config = not config_healthy and config_missing_count > 0
can_auto_fix_telemetry = telemetry_check.finding.is_repaired_by_initializing
- can_migrate = pending_migrations_check.finding.is_repaired_by_migrating
+ can_migrate = bool(pending_migrations_check.migratable_files)
+ can_clean_up_former_release = bool(pending_migrations_check.former_release_files)
telemetry_is_out_of_date = telemetry_check.finding.is_out_of_date
# The migration row names the same command and covers every surface, so the telemetry-only
# bullet under it would be the same advice twice. It still appears on its own — a telemetry
@@ -895,13 +1049,16 @@ def display_health_report(
# Read off the list rather than off the finding: a run can both migrate some files and
# leave others for a person, and that combination is the ordinary one on a stale machine.
migrations_need_a_look = bool(pending_migrations_check.attention_files)
+ # Its own bullet, since it can be the row's only one: with nothing left to clean the row lists no file.
+ boot_stays_stopped = pending_migrations_check.boot_still_blocked
has_telemetry_validation_error = not telemetry_check.is_healthy and not can_auto_fix_telemetry and not telemetry_is_out_of_date
- # Check for backend file issues
+ # Check for backend file issues, which only runs on this machine meet
has_backend_file_issues = False
can_auto_fix_backends = False
has_custom_backend_issues = False
- if backend_file_reports:
+ has_backend_credential_issues = local_inference_counts and not backends_healthy and bool(backend_credential_reports)
+ if backend_file_reports and local_inference_counts:
invalid_backends = {name: report for name, report in backend_file_reports.items() if not report.is_valid}
if invalid_backends:
has_backend_file_issues = True
@@ -919,13 +1076,16 @@ def display_health_report(
or not secrets_provider_healthy
or can_auto_fix_telemetry
or can_migrate
+ or can_clean_up_former_release
or migrations_need_a_look
+ or boot_stays_stopped
or pending_migrations_check.finding.is_uncheckable
or telemetry_is_out_of_date
or has_telemetry_validation_error
- or (not backends_healthy and backend_credential_reports)
+ or has_backend_credential_issues
or has_backend_file_issues
or has_deck_drift
+ or not pipelex_api_key_healthy
)
if has_recommendations:
@@ -937,6 +1097,18 @@ def display_health_report(
if can_auto_fix_telemetry:
console.print(" • Run [cyan]pipelex init telemetry[/cyan] to configure telemetry preferences")
+ if not pipelex_api_key_healthy:
+ console.print(
+ f" • Run [cyan]{PIPELEX_LOGIN_COMMAND}[/cyan] to get a Pipelex API key for hosted runs "
+ f"([cyan]{LOGIN_PASTE_COMMAND}[/cyan] on a machine without a browser)"
+ )
+
+ if can_clean_up_former_release:
+ console.print(
+ f" • Run [cyan]{MIGRATE_COMMAND}[/cyan] to clean up "
+ f"{len(pending_migrations_check.former_release_files)} file(s) a former release left"
+ )
+
if can_migrate:
console.print(
f" • Run [cyan]{MIGRATE_COMMAND}[/cyan] to bring "
@@ -949,6 +1121,12 @@ def display_health_report(
f"{len(pending_migrations_check.attention_files)} configuration file(s) carry that the migration will not do on its own"
)
+ if boot_stays_stopped:
+ console.print(
+ f" • Run [cyan]{MIGRATE_COMMAND} --dry-run[/cyan] to see what still stops Pipelex from starting, "
+ "which the command leaves for you to fix"
+ )
+
if pending_migrations_check.finding.is_uncheckable:
console.print(f" • Run [cyan]{MIGRATE_COMMAND} --dry-run[/cyan] to check for pending migrations — this report could not")
@@ -996,7 +1174,7 @@ def display_health_report(
backend_file = f"{escape(config_location.config_dir)}/inference/backends/{escape(backend_name)}.toml"
console.print(f" • Manually fix backend configuration in [cyan]{backend_file}[/cyan]")
- if not backends_healthy and backend_credential_reports:
+ if has_backend_credential_issues:
# Collect all missing and placeholder vars
all_missing_vars: set[str] = set()
all_placeholder_vars: set[str] = set()
@@ -1025,7 +1203,6 @@ def display_health_report(
# Show Discord support for manual-fix issues (regardless of --fix flag)
has_config_validation_error = not config_healthy and config_missing_count == 0
- has_backend_credential_issues = not backends_healthy and backend_credential_reports
if has_config_validation_error or has_backend_credential_issues or has_telemetry_validation_error:
console.print("[dim]If you need help with manual fixes:[/dim]")
console.print(" [cyan]https://docs.pipelex.com[/cyan] - Documentation")
@@ -1058,6 +1235,8 @@ def _print_pending_migrations(*, check: PendingMigrationsCheck) -> None:
console.print(f" [green]✓[/green] {escape(check.message)}")
case PendingMigrationsFinding.PENDING | PendingMigrationsFinding.NEEDS_ATTENTION:
console.print(f" [yellow]⚠[/yellow] {escape(check.message)}")
+ for file_path in check.former_release_files:
+ console.print(f" [dim]{escape(file_path)}[/dim] — left by a former release")
for file_path in check.migratable_files:
console.print(f" [dim]{escape(file_path)}[/dim] — out of date")
for file_path in check.attention_files:
@@ -1422,6 +1601,22 @@ def setup_doctor_runtime(*, log_config_overrides: Mapping[str, Any] | None = Non
return runtime_setup
+FORMER_RELEASE_STOPS_THE_MODELS_CHECK = (
+ f"Not checked: what a former release left for the Pipelex Gateway stops the boot — run '{MIGRATE_COMMAND}' to clean it up"
+)
+
+
+def _migration_fix_question(*, former_release_count: int, migratable_count: int) -> str:
+ """The `--fix` question for `pipelex migrate`'s write pass, counting the files of each of its steps: the cleanup, then the migration."""
+ migration = f"{migratable_count} configuration file(s) to the current schema"
+ if not former_release_count:
+ return f"Migrate {migration}?"
+ cleanup = f"Clean up {former_release_count} file(s) a former release left"
+ if not migratable_count:
+ return f"{cleanup}?"
+ return f"{cleanup} and migrate {migration}?"
+
+
def check_models(
*, secrets_provider: SecretsProviderAbstract | None, config_dir: Path | None = None
) -> tuple[bool, str, dict[str, BackendFileReport]]:
@@ -1440,6 +1635,14 @@ def check_models(
Returns:
Tuple of (is_healthy, message, backend_file_reports)
"""
+ # What a former release left stops the boot before any backend loads, so it is named here rather than whichever
+ # refusal about one backend or one profile the setup below would meet first. The migrations row lists the files.
+ if former_release_boot_blockers(
+ backends_library_paths=config_manager.backends_file_paths(config_dir=config_dir),
+ routing_profile_library_paths=config_manager.routing_profiles_file_paths(config_dir=config_dir),
+ ):
+ return False, FORMER_RELEASE_STOPS_THE_MODELS_CHECK, {}
+
# The backends resolve their credentials through the configured provider, so without it no backend
# loads the way boot would load it; the secrets provider row already says why it did not build.
if secrets_provider is None:
@@ -1510,6 +1713,7 @@ def check_models(
ModelDeckValidationError,
InferenceBackendCredentialsError,
PluginModelDeclarationError,
+ FormerReleaseConfigError,
) as exc:
return False, f"Error checking models: {redacted_failure(exc=exc, log_config=log_config)}", backend_file_reports
except Exception as exc: # ruff: ignore[blind-except]
@@ -1614,6 +1818,13 @@ def do_doctor_cmd(
deck_healthy, deck_report, deck_message = check_deck_sync()
internal_backend_healthy, internal_backend_report, internal_backend_message = check_internal_backend_sync()
+ # Where runs execute decides what the setup needs: a hosted run boots nothing here, so it needs a Pipelex API key
+ # and none of the provider keys or local models the rows above check, which then count only for --local runs.
+ run_execution = resolve_doctor_run_execution() if config_healthy else RunExecution.LOCAL
+ local_inference_counts = not run_execution.is_hosted
+ pipelex_api_key_check = None if local_inference_counts else check_pipelex_api_key_set()
+ pipelex_api_key_healthy = pipelex_api_key_check is None or pipelex_api_key_check.is_healthy
+
# Display report
display_health_report(
config_healthy=config_healthy,
@@ -1639,14 +1850,17 @@ def do_doctor_cmd(
log_sink_check=log_sink_check,
plugins_check=plugins_check,
secrets_provider_check=secrets_provider_check,
+ run_execution=run_execution,
+ pipelex_api_key_check=pipelex_api_key_check,
)
all_healthy = (
config_healthy
and pending_migrations_check.is_healthy
and telemetry_check.is_healthy
- and backends_healthy
- and models_healthy
+ and (backends_healthy or not local_inference_counts)
+ and (models_healthy or not local_inference_counts)
+ and pipelex_api_key_healthy
and deck_healthy
and internal_backend_healthy
and (log_sink_check is None or log_sink_check.is_healthy)
@@ -1668,7 +1882,7 @@ def do_doctor_cmd(
# Check for backend file issues that can be auto-fixed
can_fix_backends = False
fixable_backends: list[tuple[str, BackendFileReport]] = []
- if backend_file_reports:
+ if backend_file_reports and local_inference_counts:
invalid_backends = [(name, report) for name, report in backend_file_reports.items() if not report.is_valid]
fixable_backends = [(name, report) for name, report in invalid_backends if report.has_kit_template]
can_fix_backends = len(fixable_backends) > 0
@@ -1684,7 +1898,7 @@ def do_doctor_cmd(
has_config_validation_error = not config_healthy and config_missing_count == 0
# A telemetry finding a person has to resolve: neither a fresh file nor a migration gets there.
has_telemetry_validation_error = not telemetry_check.is_healthy and not can_fix_telemetry and not telemetry_check.finding.is_out_of_date
- has_backend_credential_issues = not backends_healthy and backend_credential_reports
+ has_backend_credential_issues = local_inference_counts and not backends_healthy and bool(backend_credential_reports)
# If --fix flag is provided, offer to fix auto-fixable issues
if fix and has_auto_fixable_issues:
@@ -1709,12 +1923,28 @@ def do_doctor_cmd(
# implementation of it, and it is offered before the rows that report on file *contents*
# because migrating can be what resolves them.
if can_fix_migrations:
+ former_release_count = len(pending_migrations_check.former_release_files)
migratable_count = len(pending_migrations_check.migratable_files)
- if Confirm.ask(f"[bold]Migrate {migratable_count} configuration file(s) to the current schema?[/bold]", default=True):
+ question = _migration_fix_question(former_release_count=former_release_count, migratable_count=migratable_count)
+ if Confirm.ask(f"[bold]{question}[/bold]", default=True):
try:
console.print()
- applied = apply_pending_migrations(config_dirs=config_directories_to_migrate())
- console.print(f"[green]✓[/green] Migrated {len(applied.written_plans)} configuration file(s)")
+ config_dirs = config_directories_to_migrate()
+ # The cleanup first, as `pipelex migrate` runs it: the replay leaves the files it removes out of its walk.
+ removed_paths: frozenset[Path] = frozenset()
+ if former_release_count:
+ cleaned = apply_former_release_cleanup(config_dirs=config_dirs)
+ removed_paths = cleaned.removed_paths
+ if cleaned.needs_attention:
+ console.print(
+ f"[yellow]⚠[/yellow] Cleaned up {len(cleaned.applied_files)} file(s) a former release left, "
+ "and something is left for you to look at, as marked above"
+ )
+ else:
+ console.print(f"[green]✓[/green] Cleaned up {len(cleaned.applied_files)} file(s) a former release left")
+ if migratable_count:
+ applied = apply_pending_migrations(config_dirs=config_dirs, skipped_paths=removed_paths)
+ console.print(f"[green]✓[/green] Migrated {len(applied.written_plans)} configuration file(s)")
# The rows below were measured before this ran, so a file this just repaired can
# still be reported as broken further down.
console.print("[dim]Re-run[/dim] [cyan]pipelex doctor[/cyan] [dim]for an updated report.[/dim]")
@@ -1778,11 +2008,22 @@ def do_doctor_cmd(
console.print()
# Handle issues that can't be auto-fixed
- if has_config_validation_error or has_telemetry_validation_error or has_backend_credential_issues:
+ if has_config_validation_error or has_telemetry_validation_error or has_backend_credential_issues or not pipelex_api_key_healthy:
console = get_console()
console.print("[bold yellow]Manual Fixes Required[/bold yellow]")
console.print()
+ # The Pipelex API key, the one credential a hosted setup needs
+ if pipelex_api_key_check is not None and not pipelex_api_key_check.is_healthy:
+ console.print("[bold]Pipelex API key:[/bold]")
+ console.print(f" {escape(pipelex_api_key_check.message)}")
+ console.print()
+ console.print(
+ f"Run [cyan]{PIPELEX_LOGIN_COMMAND}[/cyan] to get one through your browser, "
+ f"or [cyan]{LOGIN_PASTE_COMMAND}[/cyan] on a machine without one."
+ )
+ console.print()
+
# Config validation errors
if has_config_validation_error:
console.print("[bold]Configuration validation error:[/bold]")
diff --git a/pipelex/cli/commands/init/command.py b/pipelex/cli/commands/init/command.py
index f17d58fe1..f5cbca22f 100644
--- a/pipelex/cli/commands/init/command.py
+++ b/pipelex/cli/commands/init/command.py
@@ -1,29 +1,71 @@
-"""Main command orchestration for the init command."""
+"""Main command orchestration for the init command.
+
+`init_cmd` runs in three stages:
+
+1. **Inspect** (`inspect_initialization`): read what is on disk and what the focus asks for, and decide which steps
+ are needed. It asks nothing and writes nothing. It also finds what a former release that ran on the Pipelex Gateway
+ left in the configuration directories a boot reads, read together as the boot merges them, and
+ whether it stops this machine's boot, which is the boot's own check over the files it merges.
+2. **Choose** (`choose_initialization`): first, when the inspection found what a former release left and someone is
+ there to answer, say so and offer the cleanup `pipelex migrate` runs, then run it on a yes; then the
+ confirmation, then where runs execute, the hosted Pipelex API or this machine. An unattended run (`pipelex doctor
+ --fix`) never runs the cleanup: the doctor asks about it itself.
+3. **Execute** (`execute_initialization`): write the files and run the steps the choices call for. Only the local
+ path's own steps (backends, routing, credentials) and the hosted path's sign-in ask anything more.
+"""
import shutil
from pathlib import Path
import typer
+from pydantic import BaseModel, ConfigDict
from rich.console import Console
from rich.markup import escape
+from rich.panel import Panel
from rich.prompt import Confirm
+from pipelex import log
from pipelex.cli.commands.init.backends import (
customize_backends_config,
get_selected_backend_keys,
)
from pipelex.cli.commands.init.config_files import init_config
from pipelex.cli.commands.init.credentials import prompt_credentials
+from pipelex.cli.commands.init.ide_extension import suggest_extension_install_if_needed
from pipelex.cli.commands.init.routing import customize_routing_profile
+from pipelex.cli.commands.init.setup_path import (
+ DEFAULT_SETUP_PATH,
+ SetupPath,
+ apply_setup_path_setting,
+ ensure_pipelex_api_key,
+ read_run_execution,
+ write_run_execution,
+)
from pipelex.cli.commands.init.telemetry import setup_telemetry
from pipelex.cli.commands.init.ui.general_ui import build_initialization_panel
+from pipelex.cli.commands.init.ui.setup_path_ui import prompt_setup_path
from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.cli.commands.login.command import LOGIN_PASTE_COMMAND
+from pipelex.cli.commands.migrate_cmd import apply_former_release_cleanup
from pipelex.cogt.models.deck_manifest import stamp_kit_manifests
+from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.hosted.run_config import RunExecution
from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.migration.former_release import (
+ FormerReleaseFinding,
+ FormerReleaseFindings,
+ detect_former_release_across,
+ former_release_boot_blockers,
+ kit_default_routing_profile_name,
+)
from pipelex.runtime_hub import get_console
from pipelex.system.configuration.config_loader import config_manager
from pipelex.system.telemetry.telemetry_config import TELEMETRY_CONFIG_FILE_NAME
+#: The focuses whose run asks where runs execute when it sets up inference: the full setup, and a first setup reached
+#: through `config`. `inference` configures this machine's backends on purpose and leaves the setting as it is.
+SETUP_PATH_FOCUSES: frozenset[InitFocus] = frozenset({InitFocus.ALL, InitFocus.CONFIG})
+
def determine_needs(
*,
@@ -62,6 +104,235 @@ def determine_needs(
return needs_config, needs_inference, needs_routing, needs_telemetry
+class InitInspection(BaseModel):
+ """What an init run found on disk and what it will do, decided before anything is asked or written."""
+
+ model_config = ConfigDict(frozen=True)
+
+ focus: InitFocus
+ target_config_dir: Path
+ for_project: bool
+ reset: bool
+ backends_toml_path: Path
+ routing_profiles_toml_path: Path
+ telemetry_config_path: Path
+ pipelex_toml_path: Path
+ #: Whether the target `pipelex.toml` exists: a home without one is brand new.
+ pipelex_toml_exists: bool
+ is_first_time_backends_setup: bool
+ #: The `[run] execution` the target `pipelex.toml` sets now, kept across a reset that does not ask again.
+ configured_execution: RunExecution | None
+ check_credentials: bool
+ check_inference: bool
+ check_routing: bool
+ needs_config: bool
+ needs_inference: bool
+ needs_routing: bool
+ needs_telemetry: bool
+ #: The configuration directories a boot reads, the home's then the project's, which the former-release cleanup runs over.
+ former_release_config_dirs: list[Path]
+ #: What a former release left in each of them, read as the boot merges them; empty when nothing.
+ former_release_findings: list[FormerReleaseFindings]
+ #: What of it stops this machine's boot, read off the files that boot merges; empty when the boot starts.
+ former_release_boot_blockers: list[FormerReleaseFinding]
+
+ @property
+ def asks_setup_path(self) -> bool:
+ """Whether this run decides where runs execute: when it sets up inference for the full setup, or for a first one."""
+ return self.needs_inference and self.focus in SETUP_PATH_FOCUSES
+
+ @property
+ def kept_execution(self) -> RunExecution | None:
+ """The setting to write back after the configuration files are reset without asking where runs execute."""
+ if self.needs_config and not self.asks_setup_path:
+ return self.configured_execution
+ return None
+
+ @property
+ def keeps_hosted_runs(self) -> bool:
+ """Whether a reset that does not ask again keeps runs on the hosted Pipelex API, so asks for no provider key."""
+ return self.kept_execution is not None and self.kept_execution.is_hosted
+
+ @property
+ def unattended_setup_path(self) -> SetupPath | None:
+ """The path taken when nobody is asked (`pipelex doctor --fix`).
+
+ None when this run does not decide where runs execute. Otherwise installing missing files never moves runs off
+ this machine: the target's `[run] execution` is kept when it sets one, a `pipelex.toml` that sets none, such as
+ one written before `[run]` existed, runs locally as the package default says and stays local, and the hosted
+ default is taken only for a brand-new home, with no `pipelex.toml` yet.
+ """
+ if not self.asks_setup_path:
+ return None
+ if self.configured_execution is not None:
+ return SetupPath.from_run_execution(execution=self.configured_execution)
+ if self.pipelex_toml_exists:
+ return SetupPath.LOCAL
+ return DEFAULT_SETUP_PATH
+
+
+class InitChoices(BaseModel):
+ """What the person chose, or what was taken for them when nobody was asked."""
+
+ model_config = ConfigDict(frozen=True)
+
+ #: Where runs execute, when this run decides it; `None` when it does not.
+ setup_path: SetupPath | None
+ #: Whether someone answers prompts: `False` for `pipelex doctor --fix`.
+ interactive: bool
+
+
+def inspect_initialization(*, focus: InitFocus, local: bool) -> InitInspection:
+ """Stage 1: find the target directory, read what is there, and decide which steps the focus needs.
+
+ Args:
+ focus: What to initialize.
+ local: Target the project's `.pipelex/` at the detected project root instead of the home configuration directory.
+
+ Returns:
+ The inspection the later stages work from. Nothing is asked or written.
+ """
+ # Config updates are not yet supported - always reset
+ reset = True
+
+ if local:
+ # --local: create at project root, fall back to CWD
+ project_root = config_manager.project_root
+ if project_root is not None:
+ target_config_dir = project_root / ".pipelex"
+ else:
+ target_config_dir = Path.cwd() / ".pipelex"
+ else:
+ # Default: create the global config in the home configuration directory
+ target_config_dir = config_manager.global_config_dir
+
+ telemetry_config_path = target_config_dir / TELEMETRY_CONFIG_FILE_NAME
+ backends_toml_path = target_config_dir / "inference" / "backends.toml"
+ routing_profiles_toml_path = target_config_dir / "inference" / "routing_profiles.toml"
+ pipelex_toml_path = target_config_dir / "pipelex.toml"
+
+ check_config = focus in {InitFocus.ALL, InitFocus.CONFIG}
+ check_credentials = focus in {InitFocus.ALL, InitFocus.CONFIG, InitFocus.INFERENCE}
+ check_inference = focus in {InitFocus.ALL, InitFocus.INFERENCE}
+ check_routing = focus == InitFocus.ROUTING
+ check_telemetry = focus in {InitFocus.ALL, InitFocus.TELEMETRY}
+
+ is_first_time_backends_setup = not backends_toml_path.exists()
+
+ needs_config, needs_inference, needs_routing, needs_telemetry = determine_needs(
+ reset=reset,
+ check_config=check_config,
+ check_inference=check_inference,
+ check_routing=check_routing,
+ check_telemetry=check_telemetry,
+ backends_toml_path=backends_toml_path,
+ routing_profiles_toml_path=routing_profiles_toml_path,
+ telemetry_config_path=telemetry_config_path,
+ target_config_dir=target_config_dir,
+ )
+ # `init_config` never copies `inference/`, so a configuration reset on a directory with no `backends.toml` is a
+ # first setup, and sets up inference too, whatever the focus.
+ if needs_config and is_first_time_backends_setup:
+ needs_inference = True
+
+ # Every directory a boot reads, not only the target, read together: a profile one of them activates can be defined in the
+ # other, and a former release's files there stop the boot as surely.
+ former_release_config_dirs = list(config_manager.existing_config_dirs)
+ former_release_findings = [findings for findings in detect_former_release_across(config_dirs=former_release_config_dirs) if not findings.is_clean]
+ boot_blockers = former_release_boot_blockers(
+ backends_library_paths=config_manager.backends_file_paths(),
+ routing_profile_library_paths=config_manager.routing_profiles_file_paths(),
+ )
+
+ return InitInspection(
+ focus=focus,
+ target_config_dir=target_config_dir,
+ for_project=local,
+ reset=reset,
+ backends_toml_path=backends_toml_path,
+ routing_profiles_toml_path=routing_profiles_toml_path,
+ telemetry_config_path=telemetry_config_path,
+ pipelex_toml_path=pipelex_toml_path,
+ pipelex_toml_exists=pipelex_toml_path.is_file(),
+ is_first_time_backends_setup=is_first_time_backends_setup,
+ configured_execution=read_run_execution(pipelex_toml_path=pipelex_toml_path),
+ check_credentials=check_credentials,
+ check_inference=check_inference,
+ check_routing=check_routing,
+ needs_config=needs_config,
+ needs_inference=needs_inference,
+ needs_routing=needs_routing,
+ needs_telemetry=needs_telemetry,
+ former_release_config_dirs=former_release_config_dirs,
+ former_release_findings=former_release_findings,
+ former_release_boot_blockers=boot_blockers,
+ )
+
+
+def describe_former_release_findings(*, findings: list[FormerReleaseFindings], blockers: list[FormerReleaseFinding]) -> str:
+ """What a former release left, in words: what of it stops the boot, the files it is in, and what the cleanup does.
+
+ The findings that stop the boot are spelled out; the rest are counted by file, since `pipelex migrate --dry-run`
+ lists every change for whoever wants each one.
+
+ Args:
+ findings: What the former release left, per directory.
+ blockers: What of it stops this machine's boot, as `former_release_boot_blockers` reads the files the boot merges.
+ """
+ lead = "This machine was set up by a former Pipelex release, which ran models through the Pipelex Gateway, and this release no longer has it."
+ if blockers:
+ lines = [f"{lead} Pipelex cannot start until what that release left is cleaned up:", *(f"• {finding.description}" for finding in blockers)]
+ else:
+ lines = [f"{lead} What that release left no longer stops Pipelex from starting, and is still worth removing."]
+ file_count = sum(len(directory_findings.file_paths) for directory_findings in findings)
+ lines += ["", f"It is in {file_count} file(s):"]
+ for directory_findings in findings:
+ relative_paths = [path.relative_to(directory_findings.config_dir).as_posix() for path in directory_findings.file_paths]
+ lines.append(f"• in '{directory_findings.config_dir}': {', '.join(relative_paths)}")
+ lines += [
+ "",
+ (
+ "The cleanup removes it and keeps a copy of each file it changes or removes; when the active routing profile is one of that "
+ f"release's, it moves it to '{kit_default_routing_profile_name()}'. Run `{MIGRATE_COMMAND} --dry-run` to see each change."
+ ),
+ ]
+ return "\n".join(lines)
+
+
+def offer_former_release_cleanup(*, console: Console, inspection: InitInspection) -> None:
+ """Say what a former release left, and clean it up on a yes.
+
+ The cleanup is `pipelex migrate`'s first step, the same write pass with the same copies kept. A no leaves the files
+ as they are and the setup goes on: what it writes into the target replaces that directory's inference files, but
+ not the files beside them nor the other directory's, which go on stopping the boot until the cleanup runs. Only
+ asked of a person: nothing removes or rewrites a user's files without a yes.
+ """
+ console.print()
+ console.print(
+ Panel(
+ escape(describe_former_release_findings(findings=inspection.former_release_findings, blockers=inspection.former_release_boot_blockers)),
+ title="[bold yellow]Left by a former release[/bold yellow]",
+ border_style="yellow",
+ )
+ )
+ if not Confirm.ask("[bold]Clean it up now?[/bold]", default=True):
+ until = " Pipelex will not start until then." if inspection.former_release_boot_blockers else ""
+ console.print(f"[yellow]Left as it is.[/yellow] [cyan]{MIGRATE_COMMAND}[/cyan] cleans it up whenever you are ready.{until}")
+ return
+ cleanup = apply_former_release_cleanup(config_dirs=inspection.former_release_config_dirs)
+ if cleanup.still_blocking:
+ console.print(
+ f"[yellow]⚠ Pipelex still cannot start after the cleanup, as marked above. Fix what is named there, then run "
+ f"[cyan]{MIGRATE_COMMAND}[/cyan] to check.[/yellow]"
+ )
+ elif cleanup.needs_attention:
+ console.print(
+ f"[yellow]⚠ Some files could not be cleaned up, as marked above. Run [cyan]{MIGRATE_COMMAND}[/cyan] once they can be written.[/yellow]"
+ )
+ else:
+ console.print(f"[green]✓[/green] Cleaned up {len(cleanup.applied_files)} file(s); a copy of each original is beside it.")
+
+
def confirm_initialization(
*,
console: Console,
@@ -72,6 +343,7 @@ def confirm_initialization(
check_credentials: bool,
reset: bool,
focus: InitFocus,
+ asks_setup_path: bool = False,
) -> bool:
"""Ask user to confirm initialization.
@@ -84,6 +356,7 @@ def confirm_initialization(
check_credentials: Whether credential prompting will happen.
reset: Whether this is a reset operation.
focus: The initialization focus area.
+ asks_setup_path: Whether the run will ask where runs execute.
Returns:
True if user confirms, False otherwise.
@@ -100,6 +373,7 @@ def confirm_initialization(
needs_telemetry=needs_telemetry,
reset=reset,
check_credentials=check_credentials,
+ asks_setup_path=asks_setup_path,
)
)
@@ -118,127 +392,173 @@ def confirm_initialization(
return True
-def execute_initialization(
- *,
- console: Console,
- needs_config: bool,
- needs_inference: bool,
- needs_routing: bool,
- needs_telemetry: bool,
- check_credentials: bool,
- reset: bool,
- check_inference: bool,
- check_routing: bool,
- backends_toml_path: Path,
- telemetry_config_path: Path,
- is_first_time_backends_setup: bool,
- target_config_dir: Path | None = None,
- for_project: bool = False,
-):
- """Execute the initialization steps.
+def choose_initialization(*, console: Console, inspection: InitInspection, skip_confirmation: bool) -> InitChoices:
+ """Stage 2: offer the cleanup of what a former release left, confirm, then ask where runs execute when this run decides it.
Args:
- console: Rich Console instance for output.
- needs_config: Whether to initialize config files.
- needs_inference: Whether to set up inference backends.
- needs_routing: Whether to set up routing profiles.
- needs_telemetry: Whether to set up telemetry.
- check_credentials: Whether to prompt for missing credentials.
- reset: Whether this is a reset operation.
- check_inference: Whether inference was in focus.
- check_routing: Whether routing was in focus.
- backends_toml_path: Path to backends.toml file.
- telemetry_config_path: Path to telemetry config file.
- is_first_time_backends_setup: Whether backends.toml didn't exist before this run.
- target_config_dir: Explicit target .pipelex directory. If None, uses config_manager.pipelex_config_dir.
- for_project: True when initializing a project's .pipelex/; False when initializing
- the home configuration directory. Selects which telemetry template gets copied.
+ console: Rich Console instance for user interaction.
+ inspection: What stage 1 found.
+ skip_confirmation: Ask nothing (`pipelex doctor --fix`): what a former release left is not offered and not
+ cleaned up, since the doctor asks about it itself; the confirmation is skipped, and where runs execute is
+ the one the target's `[run] execution` already sets, local for a `pipelex.toml` that sets none, and the
+ hosted Pipelex API only for a brand-new home with no `pipelex.toml` yet.
+
+ Returns:
+ The choices stage 3 executes.
+ Raises:
+ typer.Exit: If the person cancels at the confirmation.
"""
- # Step 1: Initialize config if needed
- if needs_config:
- # Check if backends.toml exists before copying
- backends_existed_before = backends_toml_path.exists()
+ if inspection.former_release_findings and not skip_confirmation:
+ offer_former_release_cleanup(console=console, inspection=inspection)
+ if skip_confirmation:
console.print()
- init_config(reset=reset, target_dir=target_config_dir)
+ return InitChoices(setup_path=inspection.unattended_setup_path, interactive=False)
+
+ confirm_initialization(
+ console=console,
+ needs_config=inspection.needs_config,
+ needs_inference=inspection.needs_inference,
+ needs_routing=inspection.needs_routing,
+ needs_telemetry=inspection.needs_telemetry,
+ # A reset that keeps hosted runs asks for no provider key, as the execute stage skips that step.
+ check_credentials=inspection.check_credentials and not inspection.keeps_hosted_runs,
+ reset=inspection.reset,
+ focus=inspection.focus,
+ asks_setup_path=inspection.asks_setup_path,
+ )
+ setup_path: SetupPath | None = None
+ if inspection.asks_setup_path:
+ console.print()
+ setup_path = prompt_setup_path(console=console, default=DEFAULT_SETUP_PATH)
+ return InitChoices(setup_path=setup_path, interactive=True)
+
+
+def _reset_inference_files(*, console: Console, inspection: InitInspection) -> None:
+ """Copy the kit's inference files over the target's (`init_config` skips `inference/`)."""
+ template_inference_dir = Path(str(get_kit_configs_dir())) / "inference"
+ target_inference_dir = inspection.target_config_dir / "inference"
+
+ # Reset backends.toml
+ template_backends_path = template_inference_dir / "backends.toml"
+ inspection.backends_toml_path.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(template_backends_path, inspection.backends_toml_path)
+
+ # Reset all individual backend files in backends/ directory
+ template_backends_dir = template_inference_dir / "backends"
+ target_backends_dir = target_inference_dir / "backends"
+ target_backends_dir.mkdir(parents=True, exist_ok=True)
+ for backend_file in template_backends_dir.iterdir():
+ if backend_file.suffix in {".toml", ".md"}:
+ dst_path = target_backends_dir / backend_file.name
+ shutil.copy2(backend_file, dst_path)
+
+ # Reset deck/ directory files (model deck configurations)
+ template_deck_dir = template_inference_dir / "deck"
+ target_deck_dir = target_inference_dir / "deck"
+ target_deck_dir.mkdir(parents=True, exist_ok=True)
+ for deck_file in template_deck_dir.iterdir():
+ if deck_file.suffix == ".toml":
+ dst_path = target_deck_dir / deck_file.name
+ shutil.copy2(deck_file, dst_path)
+
+ # Stamp the kit manifests of the deck and of backends/internal.toml so future updates can
+ # detect drift and `pipelex update` knows the exact kit version this install came from.
+ stamp_kit_manifests(inference_dir=target_inference_dir)
+
+ # Reset routing_profiles.toml
+ template_routing_path = template_inference_dir / "routing_profiles.toml"
+ target_routing_path = target_inference_dir / "routing_profiles.toml"
+ if template_routing_path.exists():
+ shutil.copy2(template_routing_path, target_routing_path)
+ console.print("✅ Reset routing_profiles.toml from template")
+
+
+def _suggest_extension(*, console: Console) -> None:
+ """Offer the IDE extension, as the local path's backend step does."""
+ try:
+ suggest_extension_install_if_needed(console=console)
+ except EOFError as exc:
+ # No stdin available for the install prompt — skip the optional IDE extension suggestion.
+ log.debug(f"IDE extension suggestion skipped: {exc}")
- # init_config skips the inference/ directory (handled independently by the inference step).
- # Detect first-time setup: if backends.toml didn't exist before, inference needs to be set up.
- backends_exists_now = backends_toml_path.exists()
- if not backends_existed_before or (check_inference and backends_exists_now):
- needs_inference = True
+def execute_initialization(*, console: Console, inspection: InitInspection, choices: InitChoices) -> None:
+ """Stage 3: write the files and run the steps the inspection and the choices call for.
- # Determine if this is truly a first-time setup
- first_time_setup = is_first_time_backends_setup
+ The local path runs the backends, routing and credentials steps; the hosted path leaves the kit's inference files
+ as written, writes the hosted default, and, once every file is written, makes sure hosted runs have a key.
- # Step 2: Set up inference backends if needed
- if needs_inference:
- console.print()
+ Args:
+ console: Rich Console instance for output.
+ inspection: What stage 1 found.
+ choices: What stage 2 settled.
+ """
+ target_config_dir = inspection.target_config_dir
+ backends_toml_path = inspection.backends_toml_path
+ setup_path = choices.setup_path
+ kept_execution = inspection.kept_execution
- # Copy the inference template files when resetting (init_config skips inference/)
- if reset:
- template_inference_dir = Path(str(get_kit_configs_dir())) / "inference"
- effective_config_dir = target_config_dir or config_manager.pipelex_config_dir
- target_inference_dir = effective_config_dir / "inference"
-
- # Reset backends.toml
- template_backends_path = template_inference_dir / "backends.toml"
- backends_toml_path.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(template_backends_path, backends_toml_path)
-
- # Reset all individual backend files in backends/ directory
- template_backends_dir = template_inference_dir / "backends"
- target_backends_dir = target_inference_dir / "backends"
- target_backends_dir.mkdir(parents=True, exist_ok=True)
- for backend_file in template_backends_dir.iterdir():
- if backend_file.suffix in {".toml", ".md"}:
- dst_path = target_backends_dir / backend_file.name
- shutil.copy2(backend_file, dst_path)
-
- # Reset deck/ directory files (model deck configurations)
- template_deck_dir = template_inference_dir / "deck"
- target_deck_dir = target_inference_dir / "deck"
- target_deck_dir.mkdir(parents=True, exist_ok=True)
- for deck_file in template_deck_dir.iterdir():
- if deck_file.suffix == ".toml":
- dst_path = target_deck_dir / deck_file.name
- shutil.copy2(deck_file, dst_path)
-
- # Stamp the kit manifests of the deck and of backends/internal.toml so future updates can
- # detect drift and `pipelex update` knows the exact kit version this install came from.
- stamp_kit_manifests(inference_dir=target_inference_dir)
-
- # Reset routing_profiles.toml
- template_routing_path = template_inference_dir / "routing_profiles.toml"
- target_routing_path = target_inference_dir / "routing_profiles.toml"
- if template_routing_path.exists():
- shutil.copy2(template_routing_path, target_routing_path)
- console.print("✅ Reset routing_profiles.toml from template")
+ # Step 1: Initialize config if needed (init_config skips inference/, which the inference step handles)
+ if inspection.needs_config:
+ console.print()
+ init_config(reset=inspection.reset, target_dir=target_config_dir)
+ if kept_execution is not None:
+ # The reset copied the kit's default over where runs execute, a choice this run did not ask about again.
+ write_run_execution(pipelex_toml_path=inspection.pipelex_toml_path, execution=kept_execution)
- first_time_setup = True # Treat as first-time setup since we just replaced the files
+ first_time_setup = inspection.is_first_time_backends_setup
- customize_backends_config(is_first_time_setup=first_time_setup, target_config_dir=target_config_dir)
+ # Step 2: Set up inference if needed, and where runs execute when this run decides it
+ if inspection.needs_inference:
+ console.print()
- # Automatically set up routing after backends (unless routing is the specific focus)
- if not check_routing:
- selected_backend_keys = get_selected_backend_keys(backends_toml_path)
- if selected_backend_keys:
- customize_routing_profile(selected_backend_keys, target_config_dir=target_config_dir)
+ if inspection.reset:
+ _reset_inference_files(console=console, inspection=inspection)
+ first_time_setup = True # Treat as first-time setup since we just replaced the files
- # Step 2.5: Prompt for missing credentials
- if check_credentials:
+ match setup_path:
+ case SetupPath.HOSTED:
+ # Hosted runs use none of this machine's backends: the kit's inference files stay as written.
+ apply_setup_path_setting(
+ console=console,
+ setup_path=setup_path,
+ pipelex_toml_path=inspection.pipelex_toml_path,
+ project_config_dir=config_manager.project_config_dir,
+ )
+ if choices.interactive:
+ _suggest_extension(console=console)
+ case SetupPath.LOCAL | None:
+ customize_backends_config(is_first_time_setup=first_time_setup, target_config_dir=target_config_dir)
+
+ # Automatically set up routing after backends (unless routing is the specific focus)
+ if not inspection.check_routing:
+ selected_backend_keys = get_selected_backend_keys(backends_toml_path)
+ if selected_backend_keys:
+ customize_routing_profile(selected_backend_keys, target_config_dir=target_config_dir)
+
+ if setup_path is not None:
+ console.print()
+ apply_setup_path_setting(
+ console=console,
+ setup_path=setup_path,
+ pipelex_toml_path=inspection.pipelex_toml_path,
+ project_config_dir=config_manager.project_config_dir,
+ )
+
+ # Step 2.5: Prompt for missing credentials, which only runs on this machine use
+ runs_hosted = (setup_path is not None and setup_path.is_hosted) or inspection.keeps_hosted_runs
+ if inspection.check_credentials and not runs_hosted:
prompt_credentials(console=console, backends_toml_path=backends_toml_path)
# Step 3: Set up routing profile if specifically requested
- if needs_routing:
+ if inspection.needs_routing:
console.print()
# If reset is True, copy the template file first
- if reset:
- effective_config_dir_for_routing = target_config_dir or config_manager.pipelex_config_dir
- routing_profiles_toml_path = effective_config_dir_for_routing / "inference" / "routing_profiles.toml"
+ if inspection.reset:
+ routing_profiles_toml_path = inspection.routing_profiles_toml_path
template_routing_path = Path(str(get_kit_configs_dir())) / "inference" / "routing_profiles.toml"
routing_profiles_toml_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(template_routing_path, routing_profiles_toml_path)
@@ -251,8 +571,17 @@ def execute_initialization(
console.print("[yellow]⚠ Warning: No backends enabled. Please run 'pipelex init inference' first.[/yellow]")
# Step 4: Set up telemetry if needed
- if needs_telemetry:
- setup_telemetry(console=console, telemetry_config_path=telemetry_config_path, for_project=for_project)
+ if inspection.needs_telemetry:
+ setup_telemetry(console=console, telemetry_config_path=inspection.telemetry_config_path, for_project=inspection.for_project)
+
+ # Step 5: Hosted runs need a key. Last, so every file is written before a sign-in that can take minutes.
+ if setup_path is not None and setup_path.is_hosted:
+ try:
+ ensure_pipelex_api_key(console=console, interactive=choices.interactive)
+ except OSError as exc:
+ # Every file is written by now, so the setup stands; only the key is missing, and the login is its remedy.
+ console.print(f"[yellow]⚠ Your setup is saved, but no Pipelex API key was: {escape(str(exc))}[/yellow]")
+ console.print(f"Run [cyan]pipelex login[/cyan] (or [cyan]{LOGIN_PASTE_COMMAND}[/cyan]) to get one.")
console.print()
@@ -263,14 +592,17 @@ def init_cmd(
skip_confirmation: bool = False,
local: bool = False,
):
- """Initialize Pipelex configuration, inference backends, credentials, routing, and telemetry.
+ """Initialize Pipelex: configuration files, where runs execute, inference backends, credentials, routing, and telemetry.
Note: Config updates are not yet supported. This command always performs a full reset
of the configuration, overwriting any existing files.
Args:
focus: What to initialize - 'all', 'config', 'credentials', 'inference', 'routing', or 'telemetry'
- skip_confirmation: If True, skip the confirmation prompt (used when called from doctor --fix)
+ skip_confirmation: If True, ask nothing (used when called from doctor --fix): leave what a former release left
+ to the doctor's own question, skip the confirmation, keep the `[run] execution` already set (local for a
+ `pipelex.toml` that sets none), take the hosted Pipelex API only for a brand-new home with no
+ `pipelex.toml`, and print `pipelex login` instead of opening a browser.
local: If True, create project-level .pipelex/ at the detected project root.
Otherwise, create the home configuration directory (~/.pipelex/, or PIPELEX_HOME).
"""
@@ -288,89 +620,17 @@ def init_cmd(
console.print()
return
- # Config updates are not yet supported - always reset
- reset = True
-
- # Determine target directory
- if local:
- # --local: create at project root, fall back to CWD
- project_root = config_manager.project_root
- if project_root is not None:
- target_config_dir = project_root / ".pipelex"
- else:
- target_config_dir = Path.cwd() / ".pipelex"
- else:
- # Default: create the global config in the home configuration directory
- target_config_dir = config_manager.global_config_dir
- console.print(f"[dim]Target directory: {escape(str(target_config_dir))}[/dim]")
-
- pipelex_config_dir = target_config_dir
- telemetry_config_path = pipelex_config_dir / TELEMETRY_CONFIG_FILE_NAME
- backends_toml_path = pipelex_config_dir / "inference" / "backends.toml"
- routing_profiles_toml_path = pipelex_config_dir / "inference" / "routing_profiles.toml"
-
- # Determine what to check based on focus parameter
- check_config = focus in {InitFocus.ALL, InitFocus.CONFIG}
- check_credentials = focus in {InitFocus.ALL, InitFocus.CONFIG, InitFocus.INFERENCE}
- check_inference = focus in {InitFocus.ALL, InitFocus.INFERENCE}
- check_routing = focus == InitFocus.ROUTING
- check_telemetry = focus in {InitFocus.ALL, InitFocus.TELEMETRY}
-
- # Track if backends.toml existed before we start
- is_first_time_backends_setup = not backends_toml_path.exists()
-
- # Check what needs to be initialized
- needs_config, needs_inference, needs_routing, needs_telemetry = determine_needs(
- reset=reset,
- check_config=check_config,
- check_inference=check_inference,
- check_routing=check_routing,
- check_telemetry=check_telemetry,
- backends_toml_path=backends_toml_path,
- routing_profiles_toml_path=routing_profiles_toml_path,
- telemetry_config_path=telemetry_config_path,
- target_config_dir=pipelex_config_dir,
- )
+ inspection = inspect_initialization(focus=focus, local=local)
+ console.print(f"[dim]Target directory: {escape(str(inspection.target_config_dir))}[/dim]")
# Show info message if config already exists
- if not is_first_time_backends_setup and not skip_confirmation:
+ if not inspection.is_first_time_backends_setup and not skip_confirmation:
console.print()
console.print("[dim]ℹ Config update requires running a full reset.[/dim]")
try:
- # Show unified initialization prompt (skip if skip_confirmation is True)
- if not skip_confirmation:
- confirm_initialization(
- console=console,
- needs_config=needs_config,
- needs_inference=needs_inference,
- needs_routing=needs_routing,
- needs_telemetry=needs_telemetry,
- check_credentials=check_credentials,
- reset=reset,
- focus=focus,
- )
- else:
- # skip_confirmation is True, just add a blank line for spacing
- console.print()
-
- # Execute initialization steps
- execute_initialization(
- console=console,
- needs_config=needs_config,
- needs_inference=needs_inference,
- needs_routing=needs_routing,
- needs_telemetry=needs_telemetry,
- check_credentials=check_credentials,
- reset=reset,
- check_inference=check_inference,
- check_routing=check_routing,
- backends_toml_path=backends_toml_path,
- telemetry_config_path=telemetry_config_path,
- is_first_time_backends_setup=is_first_time_backends_setup,
- target_config_dir=pipelex_config_dir,
- for_project=local,
- )
+ choices = choose_initialization(console=console, inspection=inspection, skip_confirmation=skip_confirmation)
+ execute_initialization(console=console, inspection=inspection, choices=choices)
except typer.Exit:
# Re-raise Exit exceptions
@@ -378,6 +638,6 @@ def init_cmd(
except Exception as exc: # ruff: ignore[blind-except]
# Command-level boundary: any unexpected init failure is reported as a warning and the command returns without crashing.
console.print(f"\n[red]⚠ Warning: Initialization failed: {escape(str(exc))}[/red]", style="bold")
- if needs_config:
+ if inspection.needs_config:
console.print("[red]Please run 'pipelex init config' manually.[/red]")
return
diff --git a/pipelex/cli/commands/init/config_files.py b/pipelex/cli/commands/init/config_files.py
index 8d1ea8e03..8345ae46e 100644
--- a/pipelex/cli/commands/init/config_files.py
+++ b/pipelex/cli/commands/init/config_files.py
@@ -32,9 +32,8 @@ def init_config(*, reset: bool = False, dry_run: bool = False, target_dir: Path
config_template_dir = Path(str(get_kit_configs_dir()))
target_config_dir = target_dir or config_manager.pipelex_config_dir
- target_config_dir.mkdir(parents=True, exist_ok=True)
-
if not dry_run:
+ target_config_dir.mkdir(parents=True, exist_ok=True)
# Before anything is copied in, so the directory is never briefly a source of untracked
# noise. `pipelex migrate` ensures the same rule for a directory that predates it.
ensure_config_dir_gitignore(directory=target_config_dir)
diff --git a/pipelex/cli/commands/init/credentials.py b/pipelex/cli/commands/init/credentials.py
index a9c3ee9e8..1817fd9b1 100644
--- a/pipelex/cli/commands/init/credentials.py
+++ b/pipelex/cli/commands/init/credentials.py
@@ -5,6 +5,7 @@ import stat
from pathlib import Path
from typing import Any, cast
+from dotenv import dotenv_values, set_key
from rich.console import Console
from rich.markup import escape
from rich.prompt import Prompt
@@ -56,7 +57,7 @@ def write_env_file(env_path: Path, *, entries: dict[str, str]) -> None:
env_path.parent.mkdir(parents=True, exist_ok=True)
lines: list[str] = [
- "# Pipelex credentials — managed by 'pipelex init'",
+ "# Pipelex credentials — managed by 'pipelex init' and 'pipelex login'",
"# You can also edit this file manually.",
"",
]
@@ -68,6 +69,36 @@ def write_env_file(env_path: Path, *, entries: dict[str, str]) -> None:
env_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
+def set_env_file_entry(*, env_path: Path, key: str, value: str) -> None:
+ """Set one entry of a .env file, keeping every other line as it is, comments included.
+
+ Creates the file and its parent directories when missing. Every assignment of `key` already in the file, an
+ `export` one included, is replaced by `key=value`, so no later line can shadow it. The file is readable and
+ writable by its owner only (mode 0600), before and after the write, so the value never sits in a file others can read.
+ A `.env` that is a symlink, as a dotfiles setup makes it, stays one: the write goes to the file it points to.
+
+ Args:
+ env_path: Path to the .env file.
+ key: The variable to set.
+ value: Its value, written unquoted: the caller passes a value with no whitespace, quote, `#` or backslash.
+ """
+ env_path.parent.mkdir(parents=True, exist_ok=True)
+ owner_only = stat.S_IRUSR | stat.S_IWUSR
+ if env_path.is_file():
+ # The rewrite keeps the mode it finds, so tighten it first rather than after.
+ env_path.chmod(owner_only)
+ set_key(env_path, key, value, quote_mode="never", follow_symlinks=True)
+ env_path.chmod(owner_only)
+
+
+def read_env_file_value(*, env_path: Path, key: str) -> str | None:
+ """The value a .env file gives `key`, read as the runtime's own `.env` loading reads it, or `None` when it gives none."""
+ if not env_path.is_file():
+ return None
+ value = dotenv_values(env_path).get(key)
+ return value or None
+
+
def get_required_vars_for_enabled_backends(backends_toml_path: Path) -> dict[str, list[str]]:
"""Determine which env vars are required by each enabled backend.
diff --git a/pipelex/cli/commands/init/setup_path.py b/pipelex/cli/commands/init/setup_path.py
new file mode 100644
index 000000000..6485fae3e
--- /dev/null
+++ b/pipelex/cli/commands/init/setup_path.py
@@ -0,0 +1,210 @@
+"""Where runs execute, as `pipelex init` sets it up: on the hosted Pipelex API, or on this machine.
+
+The answer is written to `[run] execution` in the target `pipelex.toml`, the default `pipelex run` and
+`pipelex-agent run` take when a command passes neither `--hosted` nor `--local`. The hosted path then needs a
+Pipelex API key, which `pipelex login` obtains; the local path is the backends, routing and credentials steps the
+init command runs itself.
+"""
+
+from collections.abc import Mapping
+from enum import StrEnum
+from pathlib import Path
+from typing import Any, NamedTuple, cast
+
+import tomlkit
+from rich.console import Console
+from rich.markup import escape
+
+from pipelex.cli.commands.login.api_key_store import find_pipelex_api_key
+from pipelex.cli.commands.login.command import LOGIN_PASTE_COMMAND, login_with_browser, warn_about_shadowing_env_file
+from pipelex.cli.exceptions import PipelexCLIError
+from pipelex.hosted.api_key_check import PIPELEX_API_KEY_PREFIX, is_well_formed_pipelex_api_key
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from pipelex.hosted.run_config import RunExecution
+from pipelex.tools.misc.exceptions import TomlError
+from pipelex.tools.misc.toml_utils import load_toml_from_path, load_toml_with_tomlkit, save_toml_to_path
+
+#: The section of `pipelex.toml` holding the run commands' defaults, and the key naming where runs execute.
+RUN_SECTION = "run"
+RUN_EXECUTION_KEY = "execution"
+
+
+class SetupPath(StrEnum):
+ """The two ways `pipelex init` sets up running methods."""
+
+ HOSTED = "hosted"
+ LOCAL = "local"
+
+ @property
+ def run_execution(self) -> RunExecution:
+ """The `[run] execution` this path writes."""
+ match self:
+ case SetupPath.HOSTED:
+ return RunExecution.HOSTED
+ case SetupPath.LOCAL:
+ return RunExecution.LOCAL
+
+ @property
+ def is_hosted(self) -> bool:
+ return self.run_execution.is_hosted
+
+ @classmethod
+ def from_run_execution(cls, *, execution: RunExecution) -> "SetupPath":
+ """The path that writes this `[run] execution`."""
+ match execution:
+ case RunExecution.HOSTED:
+ return SetupPath.HOSTED
+ case RunExecution.LOCAL:
+ return SetupPath.LOCAL
+
+
+#: The path Enter takes, and the one taken when nobody is asked (`pipelex doctor --fix`).
+DEFAULT_SETUP_PATH = SetupPath.HOSTED
+
+
+def read_run_execution(*, pipelex_toml_path: Path) -> RunExecution | None:
+ """The `[run] execution` a `pipelex.toml` sets, or `None` when the file is missing, unreadable or sets none."""
+ if not pipelex_toml_path.is_file():
+ return None
+ try:
+ document = load_toml_from_path(pipelex_toml_path)
+ except (OSError, TomlError):
+ return None
+ run_section: Any = document.get(RUN_SECTION)
+ if not isinstance(run_section, Mapping):
+ return None
+ value: Any = cast("Mapping[str, Any]", run_section).get(RUN_EXECUTION_KEY)
+ if not isinstance(value, str):
+ return None
+ try:
+ return RunExecution(value)
+ except ValueError:
+ return None
+
+
+def write_run_execution(*, pipelex_toml_path: Path, execution: RunExecution) -> None:
+ """Set `[run] execution` in a `pipelex.toml`, keeping every other line, comments included.
+
+ Creates the file, or the `[run]` table, when missing.
+
+ Raises:
+ PipelexCLIError: If the file holds a `run` that is not a table.
+ """
+ if pipelex_toml_path.is_file():
+ document = load_toml_with_tomlkit(pipelex_toml_path)
+ else:
+ pipelex_toml_path.parent.mkdir(parents=True, exist_ok=True)
+ document = tomlkit.document()
+ if RUN_SECTION not in document:
+ document[RUN_SECTION] = tomlkit.table()
+ run_section = document[RUN_SECTION]
+ if not isinstance(run_section, dict):
+ msg = f"'{RUN_SECTION}' in {pipelex_toml_path} is not a table, so where runs execute cannot be written there"
+ raise PipelexCLIError(msg)
+ run_section[RUN_EXECUTION_KEY] = execution.value
+ save_toml_to_path(document, path=pipelex_toml_path)
+
+
+def describe_where_runs_execute(*, execution: RunExecution) -> str:
+ """Where runs with this execution run, in words."""
+ match execution:
+ case RunExecution.HOSTED:
+ return "on the hosted Pipelex API"
+ case RunExecution.LOCAL:
+ return "on this machine"
+
+
+def _setting_text(*, execution: RunExecution) -> str:
+ return f'[{RUN_SECTION}] {RUN_EXECUTION_KEY} = "{execution}"'
+
+
+class ProjectExecutionShadow(NamedTuple):
+ """A project's `pipelex.toml` setting another `[run] execution` than the one just written to the home directory."""
+
+ pipelex_toml_path: Path
+ execution: RunExecution
+
+
+def find_project_execution_shadow(
+ *, target_config_dir: Path, execution: RunExecution, project_config_dir: Path | None
+) -> ProjectExecutionShadow | None:
+ """The project's `pipelex.toml` when it sets another `[run] execution` than the one written to `target_config_dir`.
+
+ The project's setting wins over the home one, so a choice written globally does not apply to runs started in that
+ project. Nothing is found when there is no project configuration, when it is the target itself, or when it sets the
+ same execution or none.
+
+ Args:
+ target_config_dir: The configuration directory the setting was written to.
+ execution: The setting written there.
+ project_config_dir: The `.pipelex/` of the project around the working directory, when there is one.
+ """
+ if project_config_dir is None or project_config_dir.resolve() == target_config_dir.resolve():
+ return None
+ project_pipelex_toml_path = project_config_dir / "pipelex.toml"
+ project_execution = read_run_execution(pipelex_toml_path=project_pipelex_toml_path)
+ if project_execution is None or project_execution == execution:
+ return None
+ return ProjectExecutionShadow(pipelex_toml_path=project_pipelex_toml_path, execution=project_execution)
+
+
+def describe_project_execution_shadow(*, shadow: ProjectExecutionShadow) -> str:
+ """The warning for a project that overrides the global choice, as plain text."""
+ return (
+ f"{shadow.pipelex_toml_path} sets {_setting_text(execution=shadow.execution)}, and a project's setting wins over the "
+ f"global one: runs started in that project execute {describe_where_runs_execute(execution=shadow.execution)}. "
+ "Change it there, or pass --hosted or --local on a run."
+ )
+
+
+def apply_setup_path_setting(*, console: Console, setup_path: SetupPath, pipelex_toml_path: Path, project_config_dir: Path | None) -> None:
+ """Write where runs execute for this setup path, say so, and warn when the working directory's project overrides it.
+
+ Args:
+ console: Where to say it.
+ setup_path: The path chosen.
+ pipelex_toml_path: The `pipelex.toml` to write the setting to.
+ project_config_dir: The `.pipelex/` of the project around the working directory, when there is one.
+ """
+ execution = setup_path.run_execution
+ write_run_execution(pipelex_toml_path=pipelex_toml_path, execution=execution)
+ setting = escape(_setting_text(execution=execution))
+ console.print(
+ f"[green]✓[/green] Runs execute {describe_where_runs_execute(execution=execution)} by default "
+ f"[dim]({setting} in {escape(str(pipelex_toml_path))})[/dim]"
+ )
+ shadow = find_project_execution_shadow(target_config_dir=pipelex_toml_path.parent, execution=execution, project_config_dir=project_config_dir)
+ if shadow is not None:
+ console.print(f"[yellow]⚠ {escape(describe_project_execution_shadow(shadow=shadow))}[/yellow]")
+
+
+def ensure_pipelex_api_key(*, console: Console, interactive: bool) -> None:
+ """Make sure hosted runs have a Pipelex API key: keep the one already set, else sign in, else say how.
+
+ A key already in `PIPELEX_API_KEY` or saved in the home `.env` is kept without a login or a network call, so a
+ machine or a test that sets one goes through without waiting. With no key and nobody to answer, it prints
+ `pipelex login` instead of opening a browser.
+
+ Args:
+ console: Where to say what happened.
+ interactive: Whether someone is there to sign in through the browser.
+ """
+ console.print()
+ existing_key = find_pipelex_api_key()
+ if existing_key:
+ console.print(f"[green]✓[/green] A Pipelex API key is already set ({PIPELEX_API_KEY_ENV_KEY}); hosted runs will use it.")
+ if not is_well_formed_pipelex_api_key(api_key=existing_key):
+ console.print(f"[yellow]⚠ It does not look like a Pipelex API key, which starts with {PIPELEX_API_KEY_PREFIX}.[/yellow]")
+ warn_about_shadowing_env_file(console=console)
+ console.print("[dim]To replace it, run[/dim] [cyan]pipelex login[/cyan][dim].[/dim]")
+ return
+ if not interactive:
+ console.print("[yellow]Hosted runs need a Pipelex API key.[/yellow] Run [cyan]pipelex login[/cyan] to get one.")
+ return
+ console.print("[bold]Hosted runs need a Pipelex API key: let's get one.[/bold]")
+ outcome = login_with_browser(console=console)
+ if not outcome.is_saved:
+ console.print(
+ "[yellow]No key was saved.[/yellow] Hosted runs need one: run [cyan]pipelex login[/cyan] "
+ f"(or [cyan]{LOGIN_PASTE_COMMAND}[/cyan]) when you are ready."
+ )
diff --git a/pipelex/cli/commands/init/ui/general_ui.py b/pipelex/cli/commands/init/ui/general_ui.py
index 4496bb2b8..ef412cbb0 100644
--- a/pipelex/cli/commands/init/ui/general_ui.py
+++ b/pipelex/cli/commands/init/ui/general_ui.py
@@ -5,6 +5,9 @@ from rich.panel import Panel
from pipelex.cli.commands.init.credentials import get_global_env_path
+#: A reset copies the kit's inference files over the target's, so customised backends and routing are lost.
+_INFERENCE_RESET_LINE = "• [yellow]Reset[/yellow] inference backends, model deck and routing to the kit's defaults"
+
def build_initialization_panel(
*,
@@ -14,6 +17,7 @@ def build_initialization_panel(
needs_telemetry: bool,
reset: bool,
check_credentials: bool = False,
+ asks_setup_path: bool = False,
) -> Panel:
"""Build the initialization confirmation panel.
@@ -24,6 +28,8 @@ def build_initialization_panel(
needs_telemetry: Whether telemetry setup is needed.
reset: Whether this is a reset operation.
check_credentials: Whether credential prompting will happen.
+ asks_setup_path: Whether the run asks where runs execute, the hosted Pipelex API or this machine, whose
+ answer decides the inference and credential steps.
Returns:
A Panel containing the initialization confirmation message.
@@ -31,11 +37,29 @@ def build_initialization_panel(
# Build message based on what's being initialized
message_parts: list[str] = []
credentials_file = escape(str(get_global_env_path()))
- if reset:
+ if asks_setup_path:
+ if needs_config:
+ verb = "[yellow]Reset and reconfigure[/yellow]" if reset else "Create"
+ message_parts.append(f"• {verb} configuration files in [cyan].pipelex/[/cyan]")
+ if reset and needs_inference:
+ message_parts.append(_INFERENCE_RESET_LINE)
+ message_parts.append("• Ask where your runs execute:")
+ message_parts.append(
+ " on the hosted Pipelex API: sign in to get a Pipelex API key (default); "
+ "[cyan]pipelex init inference[/cyan] reconfigures the backends later"
+ )
+ message_parts.append(
+ f" on this machine: choose your inference backends and enter their API keys (saved to [cyan]{credentials_file}[/cyan])"
+ )
+ message_parts.append("• Suggest IDE extension for [cyan].mthds[/cyan] syntax highlighting")
+ if needs_telemetry:
+ verb = "[yellow]Reset and reconfigure[/yellow]" if reset else "Choose"
+ message_parts.append(f"• {verb} telemetry preferences")
+ elif reset:
if needs_config:
message_parts.append("• [yellow]Reset and reconfigure[/yellow] configuration files in [cyan].pipelex/[/cyan]")
if needs_inference:
- message_parts.append("• [yellow]Reset and reconfigure[/yellow] inference backends")
+ message_parts.append(f"{_INFERENCE_RESET_LINE}, then ask you to choose your inference backends")
message_parts.append("• Suggest IDE extension for [cyan].mthds[/cyan] syntax highlighting")
if check_credentials:
message_parts.append(f"• Prompt for missing API keys (saved to [cyan]{credentials_file}[/cyan])")
diff --git a/pipelex/cli/commands/init/ui/setup_path_ui.py b/pipelex/cli/commands/init/ui/setup_path_ui.py
new file mode 100644
index 000000000..49a2a1333
--- /dev/null
+++ b/pipelex/cli/commands/init/ui/setup_path_ui.py
@@ -0,0 +1,70 @@
+"""The question `pipelex init` asks first: where should runs execute?"""
+
+from rich.console import Console, Group
+from rich.markup import escape
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
+from rich.text import Text
+
+from pipelex.cli.commands.init.setup_path import SetupPath
+
+#: The question, as the panel titles it.
+SETUP_PATH_QUESTION = "Where should your runs execute?"
+
+# The options in the order they are listed and numbered, the default first.
+_SETUP_PATH_OPTIONS: tuple[tuple[SetupPath, str], ...] = (
+ (SetupPath.HOSTED, "On the hosted Pipelex API, with a Pipelex API key"),
+ (SetupPath.LOCAL, "On this machine, with your own provider keys"),
+)
+
+
+def _option_number(*, setup_path: SetupPath) -> str:
+ for index, (option, _) in enumerate(_SETUP_PATH_OPTIONS, start=1):
+ if option == setup_path:
+ return str(index)
+ msg = f"No option lists the setup path {setup_path!r}"
+ raise ValueError(msg)
+
+
+def build_setup_path_panel(*, default: SetupPath) -> Panel:
+ """The two places a run can execute, numbered, the default marked."""
+ table = Table(show_header=False, box=None, padding=(0, 2))
+ table.add_column(style="bold cyan", justify="right", width=4)
+ table.add_column(style="bold")
+ table.add_column(style="dim")
+ for index, (setup_path, label) in enumerate(_SETUP_PATH_OPTIONS, start=1):
+ table.add_row(f"[{index}]", label, "default" if setup_path == default else "")
+
+ description = Text(
+ "The hosted API runs your methods with no provider account of your own: you sign in once and get a Pipelex API key.\n"
+ "This machine runs them on the AI providers you choose, with your own keys or local models.\n"
+ "Either way, a single run can still go the other way with --hosted or --local.",
+ style="dim",
+ )
+ return Panel(
+ Group(description, Text(""), table),
+ title=f"[bold yellow]{SETUP_PATH_QUESTION}[/bold yellow]",
+ border_style="yellow",
+ padding=(1, 2),
+ )
+
+
+def prompt_setup_path(*, console: Console, default: SetupPath = SetupPath.HOSTED) -> SetupPath:
+ """Ask where runs execute, Enter (an empty answer) taking the default.
+
+ Accepts the option's number, or its name (`hosted`, `local`). Asks again on anything else.
+ """
+ console.print(build_setup_path_panel(default=default))
+ by_answer: dict[str, SetupPath] = {}
+ for index, (setup_path, _) in enumerate(_SETUP_PATH_OPTIONS, start=1):
+ by_answer[str(index)] = setup_path
+ by_answer[setup_path.value] = setup_path
+ while True:
+ answer = Prompt.ask("[bold]Enter your choice[/bold]", default=_option_number(setup_path=default), console=console)
+ if not answer.strip():
+ return default
+ chosen = by_answer.get(answer.strip().lower())
+ if chosen is not None:
+ return chosen
+ console.print(f"[red]Invalid choice: {escape(repr(answer))}.[/red] Enter 1 for the hosted Pipelex API or 2 for this machine.\n")
diff --git a/pipelex/cli/commands/login/__init__.py b/pipelex/cli/commands/login/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/cli/commands/login/api_key_store.py b/pipelex/cli/commands/login/api_key_store.py
new file mode 100644
index 000000000..3a64ee921
--- /dev/null
+++ b/pipelex/cli/commands/login/api_key_store.py
@@ -0,0 +1,82 @@
+"""Where `pipelex login` keeps the Pipelex API key, and where `pipelex init` looks for one already set.
+
+The key is saved as `PIPELEX_API_KEY` in the `.env` of the home configuration directory (`~/.pipelex/.env`, or under
+`PIPELEX_HOME`), the file the runtime loads into the environment at import, so a hosted run finds it with nothing
+exported. The file keeps its other lines and is readable by its owner only.
+
+The runtime then loads the working directory's `.env`, which wins: a `PIPELEX_API_KEY` line there, even an empty one,
+is what every command run in that directory sends, whatever the home file holds.
+"""
+
+import os
+from pathlib import Path
+from typing import NamedTuple
+
+from dotenv import dotenv_values
+
+from pipelex.cli.commands.init.credentials import get_global_env_path, read_env_file_value, set_env_file_entry
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+
+#: The working directory's `.env`, which the runtime loads after the home one (see `pipelex.system.environment`).
+WORKING_DIRECTORY_ENV_FILE_NAME = ".env"
+
+
+def save_pipelex_api_key(*, api_key: str) -> Path:
+ """Save the key as `PIPELEX_API_KEY` in the home `.env`, and set it in this process's environment.
+
+ Args:
+ api_key: A Pipelex API key, already checked to be well formed.
+
+ Returns:
+ The file it was saved to.
+ """
+ env_path = get_global_env_path()
+ set_env_file_entry(env_path=env_path, key=PIPELEX_API_KEY_ENV_KEY, value=api_key)
+ # The rest of this command, init's later steps included, sees the key as a new process would.
+ os.environ[PIPELEX_API_KEY_ENV_KEY] = api_key
+ return env_path
+
+
+def read_saved_pipelex_api_key() -> str | None:
+ """The key saved in the home `.env`, or `None` when it holds none."""
+ return read_env_file_value(env_path=get_global_env_path(), key=PIPELEX_API_KEY_ENV_KEY)
+
+
+def find_pipelex_api_key() -> str | None:
+ """The key a hosted run would use: `PIPELEX_API_KEY` in the environment, else the one saved in the home `.env`.
+
+ A variable present in the environment decides, even when empty: an empty value is what the next process sends, so it
+ is no key, and the saved one is not looked at. The runtime loads the home `.env` into the environment at import, so
+ the saved key only matters when the file was written after that, or under a `PIPELEX_HOME` set since.
+ """
+ if PIPELEX_API_KEY_ENV_KEY in os.environ:
+ return os.environ[PIPELEX_API_KEY_ENV_KEY] or None
+ return read_saved_pipelex_api_key()
+
+
+class ShadowingEnvFile(NamedTuple):
+ """The working directory's `.env`, when its `PIPELEX_API_KEY` line overrides the saved key."""
+
+ path: Path
+ #: Whether it sets the variable to an empty value, rather than to another key.
+ sets_empty_value: bool
+
+
+def find_shadowing_env_file() -> ShadowingEnvFile | None:
+ """The working directory's `.env` when it sets `PIPELEX_API_KEY` to something other than the key saved in the home `.env`.
+
+ The runtime loads that file after the home one, both overriding, so its value, an empty one included, is what every
+ command run in this directory sends. A key set there alone, with none saved, is that directory's key and shadows
+ nothing. Neither value is returned.
+ """
+ working_env_path = Path(WORKING_DIRECTORY_ENV_FILE_NAME).resolve()
+ if not working_env_path.is_file() or working_env_path == get_global_env_path().resolve():
+ return None
+ values = dotenv_values(working_env_path)
+ if PIPELEX_API_KEY_ENV_KEY not in values:
+ return None
+ working_value = values[PIPELEX_API_KEY_ENV_KEY] or ""
+ saved_key = read_saved_pipelex_api_key()
+ if working_value and (saved_key is None or working_value == saved_key):
+ return None
+ return ShadowingEnvFile(path=working_env_path, sets_empty_value=not working_value)
diff --git a/pipelex/cli/commands/login/command.py b/pipelex/cli/commands/login/command.py
new file mode 100644
index 000000000..ee5f422b9
--- /dev/null
+++ b/pipelex/cli/commands/login/command.py
@@ -0,0 +1,281 @@
+"""`pipelex login`: get a Pipelex API key through the browser, or have one pasted, check it, and save it.
+
+The browser flow opens the Pipelex app's `/auth/cli` page with the port of a one-shot loopback listener and a fresh
+`state` value; the app mints a key for the signed-in account and sends the browser back to the listener with the key
+and the same `state`. `--paste` takes a key created in the app instead, for a machine with no local browser. Either way
+the key must start with `plx_sk_`, is checked against the hosted API (`GET /v1/me`), and is saved to the home `.env` as
+`PIPELEX_API_KEY`: refused when the hosted API refuses it, saved with a warning when it could not be checked. The key is
+never printed.
+"""
+
+import webbrowser
+from enum import StrEnum
+from pathlib import Path
+from urllib.parse import urlencode, urlsplit
+
+import typer
+from rich.console import Console
+from rich.markup import escape
+from rich.prompt import Prompt
+
+from pipelex.cli.commands.init.credentials import get_global_env_path
+from pipelex.cli.commands.login.api_key_store import find_shadowing_env_file, save_pipelex_api_key
+from pipelex.cli.commands.login.loopback import CallbackRefusal, LoopbackListener
+from pipelex.cli.exceptions import PipelexCLIError
+from pipelex.hosted.api_key_check import PIPELEX_API_KEY_PREFIX, ApiKeyVerdict, check_pipelex_api_key, is_well_formed_pipelex_api_key
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY, PIPELEX_BASE_URL_ENV_KEY, redact_url_for_display
+from pipelex.runtime_hub import get_console
+from pipelex.system.environment import get_optional_env
+from pipelex.urls import URLs
+
+#: How long the browser flow waits for the key: long enough to create an account on the way.
+LOGIN_TIMEOUT_SECONDS = 300
+#: The variable aiming `pipelex login` at another Pipelex app, such as a development one.
+PIPELEX_APP_URL_ENV_KEY = "PIPELEX_APP_URL"
+#: The app's page that mints a key for the command line.
+CLI_AUTH_PATH = "/auth/cli"
+#: The command for a machine with no local browser.
+LOGIN_PASTE_COMMAND = "pipelex login --paste"
+
+
+class LoginOutcome(StrEnum):
+ """How a login ended."""
+
+ SAVED = "saved"
+ SAVED_UNCHECKED = "saved_unchecked"
+ REFUSED = "refused"
+ NO_KEY = "no_key"
+ #: The key was good, but the home `.env` could not be written.
+ NOT_SAVED = "not_saved"
+
+ @property
+ def is_saved(self) -> bool:
+ match self:
+ case LoginOutcome.SAVED | LoginOutcome.SAVED_UNCHECKED:
+ return True
+ case LoginOutcome.REFUSED | LoginOutcome.NO_KEY | LoginOutcome.NOT_SAVED:
+ return False
+
+
+def _is_origin(*, value: str) -> bool:
+ """Whether a value is `scheme://host[:port]`, with http or https and no credentials, path, query or fragment."""
+ try:
+ parts = urlsplit(value)
+ _ = parts.port
+ except ValueError:
+ return False
+ if parts.scheme not in {"http", "https"} or not parts.hostname:
+ return False
+ has_credentials = parts.username is not None or parts.password is not None
+ return not has_credentials and not (parts.path or parts.query or parts.fragment)
+
+
+def resolve_app_origin() -> str:
+ """The origin of the Pipelex app: `PIPELEX_APP_URL` when it is set, else https://app.pipelex.com.
+
+ Raises:
+ PipelexCLIError: If `PIPELEX_APP_URL` is set to something other than `scheme://host[:port]`.
+ """
+ configured = get_optional_env(PIPELEX_APP_URL_ENV_KEY)
+ if configured is None:
+ return URLs.app
+ origin = configured.strip().rstrip("/")
+ if not _is_origin(value=origin):
+ msg = (
+ f"{PIPELEX_APP_URL_ENV_KEY} is not an origin: {redact_url_for_display(url=configured)!r}. "
+ f"Give scheme://host[:port], with http or https and no path, such as {URLs.app}, or unset it."
+ )
+ raise PipelexCLIError(msg)
+ return origin
+
+
+def build_cli_auth_url(*, app_origin: str, callback_port: int, state: str) -> str:
+ """The app page that mints a key and sends it back to the listener on `callback_port` with `state`."""
+ query = urlencode({"callback_port": callback_port, "state": state})
+ return f"{app_origin}{CLI_AUTH_PATH}?{query}"
+
+
+def _save_or_report(*, console: Console, api_key: str) -> Path | None:
+ """Save the key to the home `.env`, or say why it could not be, never quoting the key.
+
+ Returns:
+ The file it was saved to, or `None` when the file could not be written.
+ """
+ try:
+ return save_pipelex_api_key(api_key=api_key)
+ except OSError as exc:
+ console.print(f"[red]The key could not be saved to {escape(str(get_global_env_path()))}: {escape(str(exc))}. Nothing was saved.[/red]")
+ console.print(
+ "[dim]The key this login received went unused: revoke it in the Pipelex app, make that file writable, "
+ "then run[/dim] [cyan]pipelex login[/cyan] [dim]again.[/dim]"
+ )
+ return None
+
+
+def check_and_save_api_key(*, console: Console, api_key: str) -> LoginOutcome:
+ """Check a key's format, then ask the hosted API about it, and save it unless it is malformed or refused.
+
+ Args:
+ console: Where to say what happened, never quoting the key.
+ api_key: The key the browser handed back or the person pasted.
+
+ Returns:
+ `SAVED` when the hosted API accepted it, `SAVED_UNCHECKED` when it could not be checked, `REFUSED` when it is
+ not a Pipelex API key or the hosted API refused it, `NOT_SAVED` when the home `.env` could not be written
+ (nothing is saved in those two cases).
+ """
+ api_key = api_key.strip()
+ if not is_well_formed_pipelex_api_key(api_key=api_key):
+ console.print(
+ f"[red]That is not a Pipelex API key: a Pipelex API key starts with {PIPELEX_API_KEY_PREFIX} "
+ "and holds no spaces or quotes. Nothing was saved.[/red]"
+ )
+ return LoginOutcome.REFUSED
+
+ console.print("[dim]Checking the key with the hosted API…[/dim]")
+ check = check_pipelex_api_key(api_key=api_key)
+ shown_base_url = escape(redact_url_for_display(url=check.base_url))
+ match check.verdict:
+ case ApiKeyVerdict.REFUSED:
+ console.print(f"[red]The hosted API at {shown_base_url} refused the key (HTTP {check.http_status}). Nothing was saved.[/red]")
+ console.print(
+ f"[dim]Check that the key is current and belongs to the API {PIPELEX_BASE_URL_ENV_KEY} names "
+ "(https://api.pipelex.com when it is unset), then run pipelex login again.[/dim]"
+ )
+ return LoginOutcome.REFUSED
+ case ApiKeyVerdict.ACCEPTED:
+ env_path = _save_or_report(console=console, api_key=api_key)
+ if env_path is None:
+ return LoginOutcome.NOT_SAVED
+ account = f" as {escape(check.account_email)}" if check.account_email else ""
+ console.print(
+ f"[green]✓[/green] Logged in{account}. Your Pipelex API key is saved to {escape(str(env_path))} as {PIPELEX_API_KEY_ENV_KEY}."
+ )
+ warn_about_shadowing_env_file(console=console)
+ return LoginOutcome.SAVED
+ case ApiKeyVerdict.UNCHECKED:
+ env_path = _save_or_report(console=console, api_key=api_key)
+ if env_path is None:
+ return LoginOutcome.NOT_SAVED
+ console.print(
+ f"[yellow]⚠ Could not check the key: {escape(check.reason or 'no answer')}. "
+ f"It is saved to {escape(str(env_path))} as {PIPELEX_API_KEY_ENV_KEY} anyway: "
+ "run pipelex login again if a hosted run refuses it.[/yellow]"
+ )
+ warn_about_shadowing_env_file(console=console)
+ return LoginOutcome.SAVED_UNCHECKED
+
+
+def warn_about_shadowing_env_file(*, console: Console) -> None:
+ """Say so when the working directory's `.env` sets `PIPELEX_API_KEY` to something other than the saved key.
+
+ The runtime loads that file after the home one, so commands run in this directory would send its value instead. Neither
+ value is printed.
+ """
+ shadow = find_shadowing_env_file()
+ if shadow is None:
+ return
+ what = "to an empty value" if shadow.sets_empty_value else "to another key"
+ console.print(
+ f"[yellow]⚠ {escape(str(shadow.path))} also sets {PIPELEX_API_KEY_ENV_KEY} ({what}) and is loaded after "
+ f"{escape(str(get_global_env_path()))}, so commands run in this directory send that value instead. "
+ f"Remove the {PIPELEX_API_KEY_ENV_KEY} line from it.[/yellow]"
+ )
+
+
+def _say_refusal(*, console: Console, refusal: CallbackRefusal) -> None:
+ match refusal:
+ case CallbackRefusal.STATE_MISMATCH:
+ console.print("[yellow]Refused a sign-in that did not come from the page this login opened; still waiting.[/yellow]")
+ case CallbackRefusal.NO_API_KEY:
+ console.print("[yellow]A sign-in came back without an API key; still waiting.[/yellow]")
+
+
+def login_with_browser(*, console: Console, timeout_seconds: float | None = None) -> LoginOutcome:
+ """Open the app's key page in the browser, wait for the key on the loopback listener, then check and save it.
+
+ Args:
+ console: Where to print the link, the wait and the outcome.
+ timeout_seconds: How long to wait for the browser; `LOGIN_TIMEOUT_SECONDS` when `None`.
+
+ Returns:
+ How the login ended; on `NO_KEY` the console names `pipelex login --paste`.
+ """
+ if timeout_seconds is None:
+ timeout_seconds = LOGIN_TIMEOUT_SECONDS
+ try:
+ app_origin = resolve_app_origin()
+ except PipelexCLIError as exc:
+ console.print(f"[red]{escape(exc.message)}[/red]")
+ return LoginOutcome.NO_KEY
+
+ with LoopbackListener() as listener:
+ auth_url = build_cli_auth_url(app_origin=app_origin, callback_port=listener.port, state=listener.state)
+ console.print("[bold]Opening your browser to sign in to Pipelex…[/bold]")
+ console.print(f"[dim]If it does not open, visit:[/dim] {escape(auth_url)}")
+ if not webbrowser.open(auth_url):
+ # The link is printed above, so a machine with no usable browser still has a way through.
+ console.print(
+ "[yellow]No browser could be opened here.[/yellow] Open the link above in a browser on this machine, "
+ f"or press Ctrl-C and run [cyan]{LOGIN_PASTE_COMMAND}[/cyan]."
+ )
+ console.print(f"[dim]Waiting up to {int(timeout_seconds)} seconds for the key…[/dim]")
+ api_key = listener.wait_for_api_key(
+ timeout_seconds=timeout_seconds,
+ on_refusal=lambda refusal: _say_refusal(console=console, refusal=refusal),
+ )
+
+ if api_key is None:
+ console.print(f"[red]No key arrived within {int(timeout_seconds)} seconds.[/red]")
+ console.print(
+ f"[dim]Run[/dim] [cyan]pipelex login[/cyan] [dim]again, or create a key in the Pipelex app ({escape(app_origin)}) "
+ f"and run[/dim] [cyan]{LOGIN_PASTE_COMMAND}[/cyan][dim].[/dim]"
+ )
+ return LoginOutcome.NO_KEY
+ return check_and_save_api_key(console=console, api_key=api_key)
+
+
+def login_with_paste(*, console: Console) -> LoginOutcome:
+ """Ask for a key created in the app, without echoing it, then check and save it.
+
+ Returns:
+ How the login ended: `NO_KEY` when nothing was pasted.
+ """
+ try:
+ app_origin = resolve_app_origin()
+ except PipelexCLIError as exc:
+ console.print(f"[red]{escape(exc.message)}[/red]")
+ return LoginOutcome.NO_KEY
+ console.print(
+ f"Paste a Pipelex API key ({PIPELEX_API_KEY_PREFIX}…) created in the Pipelex app ({escape(app_origin)}). It is not shown as you type."
+ )
+ try:
+ api_key = Prompt.ask("[bold]Pipelex API key[/bold]", password=True, console=console, default="", show_default=False)
+ except EOFError:
+ api_key = ""
+ if not api_key.strip():
+ console.print("[red]No key was entered. Nothing was saved.[/red]")
+ return LoginOutcome.NO_KEY
+ return check_and_save_api_key(console=console, api_key=api_key)
+
+
+def login_cmd(*, paste: bool = False) -> None:
+ """Get a Pipelex API key, check it, and save it to the home `.env` as `PIPELEX_API_KEY`.
+
+ Args:
+ paste: Ask for a key created in the app instead of opening the browser.
+
+ Raises:
+ typer.Exit: With code 1 when no key was saved: none arrived, none was pasted, or it was malformed or refused.
+ """
+ console = get_console()
+ console.print()
+ outcome = login_with_paste(console=console) if paste else login_with_browser(console=console)
+ console.print()
+ if not outcome.is_saved:
+ raise typer.Exit(code=1)
+ console.print(
+ "[dim]Run a method on the hosted API with[/dim] [cyan]pipelex run … --hosted[/cyan][dim], or make it the default with[/dim] "
+ "[cyan]pipelex init[/cyan][dim].[/dim]"
+ )
+ console.print()
diff --git a/pipelex/cli/commands/login/loopback.py b/pipelex/cli/commands/login/loopback.py
new file mode 100644
index 000000000..b66eaa021
--- /dev/null
+++ b/pipelex/cli/commands/login/loopback.py
@@ -0,0 +1,175 @@
+"""The one-shot loopback listener `pipelex login` waits on for the browser to hand back a Pipelex API key.
+
+The listener binds `127.0.0.1` on a port the system picks and accepts one thing: `GET /callback` carrying an `api_key`
+and the `state` value this login generated. Any other request is answered and ignored, and the wait goes on: a callback
+whose `state` is missing or different did not come from the page this login opened (any page open in the browser can
+send a request to a local port), so its key is discarded, never saved. The pages it serves tell the browser not to
+cache them nor send them on as a referrer, since the callback's URL carries the key.
+"""
+
+import secrets
+import socketserver
+import time
+from collections.abc import Callable
+from enum import StrEnum
+from http import HTTPStatus
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from types import TracebackType
+from typing import cast
+from urllib.parse import parse_qs, urlsplit
+
+from typing_extensions import Self, override
+
+#: The interface the listener binds: the loopback address itself, not `localhost`, which can resolve elsewhere (RFC 8252 §7.3).
+LOOPBACK_HOST = "127.0.0.1"
+#: The one path the listener accepts.
+CALLBACK_PATH = "/callback"
+#: The query parameter carrying the key.
+API_KEY_PARAM = "api_key"
+#: The query parameter carrying the value this login generated, which the app echoes back.
+STATE_PARAM = "state"
+# How long one connection may take to send its request: a client that connects and sends nothing must not hold the
+# listener past the login's own deadline.
+_REQUEST_READ_TIMEOUT_SECONDS = 10
+
+_PAGE_TEMPLATE = (
+ "<!doctype html><html><head><meta charset='utf-8'><title>Pipelex login</title></head>"
+ "<body style='font-family:system-ui,sans-serif;text-align:center;padding:60px'>"
+ "<h1>{heading}</h1><p>{body}</p></body></html>"
+)
+
+
+def generate_state() -> str:
+ """A fresh unguessable value for one login, which only the page this login opened can echo back."""
+ return secrets.token_urlsafe(32)
+
+
+class CallbackRefusal(StrEnum):
+ """Why the listener turned a `/callback` request away."""
+
+ STATE_MISMATCH = "state_mismatch"
+ NO_API_KEY = "no_api_key"
+
+
+#: Called with each refusal while the listener waits, so the terminal can say what the browser was told.
+RefusalCallback = Callable[[CallbackRefusal], None]
+
+
+class _LoopbackServer(HTTPServer):
+ """The listener's socket server, holding what this login expects and what it received."""
+
+ def __init__(self, *, expected_state: str) -> None:
+ self.expected_state = expected_state
+ self.api_key: str | None = None
+ self.refusals: list[CallbackRefusal] = []
+ super().__init__((LOOPBACK_HOST, 0), _CallbackHandler)
+
+ @override
+ def server_bind(self) -> None:
+ # `HTTPServer.server_bind` resolves the host's fully qualified name, a reverse lookup that can stall for seconds
+ # on some networks and tells a loopback listener nothing.
+ socketserver.TCPServer.server_bind(self)
+ self.server_name = LOOPBACK_HOST
+ self.server_port = int(self.server_address[1])
+
+
+class _CallbackHandler(BaseHTTPRequestHandler):
+ """Answers each request the listener receives, recording a key only from a callback carrying this login's `state`."""
+
+ timeout = _REQUEST_READ_TIMEOUT_SECONDS
+
+ def do_GET(self) -> None:
+ server = cast("_LoopbackServer", self.server)
+ parsed = urlsplit(self.path)
+ if parsed.path != CALLBACK_PATH:
+ self._send_page(status=HTTPStatus.NOT_FOUND, heading="Not found", body="This address only answers the Pipelex login callback.")
+ return
+ params = parse_qs(parsed.query)
+ states = params.get(STATE_PARAM, [])
+ if len(states) != 1 or not secrets.compare_digest(states[0].encode(), server.expected_state.encode()):
+ server.refusals.append(CallbackRefusal.STATE_MISMATCH)
+ self._send_page(
+ status=HTTPStatus.BAD_REQUEST,
+ heading="Login refused",
+ body="This sign-in did not come from the page pipelex login opened, so its key was discarded. Use the link your terminal printed.",
+ )
+ return
+ api_keys = params.get(API_KEY_PARAM, [])
+ if len(api_keys) != 1 or not api_keys[0]:
+ server.refusals.append(CallbackRefusal.NO_API_KEY)
+ self._send_page(
+ status=HTTPStatus.BAD_REQUEST,
+ heading="No key received",
+ body="The sign-in came back without an API key. Try again from the link your terminal printed.",
+ )
+ return
+ server.api_key = api_keys[0]
+ self._send_page(
+ status=HTTPStatus.OK,
+ heading="Logged in",
+ body="pipelex login received your key. You can close this tab and return to your terminal.",
+ )
+
+ def _send_page(self, *, status: HTTPStatus, heading: str, body: str) -> None:
+ content = _PAGE_TEMPLATE.format(heading=heading, body=body).encode("utf-8")
+ self.send_response(status)
+ self.send_header("Content-Type", "text/html; charset=utf-8")
+ self.send_header("Content-Length", str(len(content)))
+ self.send_header("Cache-Control", "no-store")
+ self.send_header("Referrer-Policy", "no-referrer")
+ self.send_header("Connection", "close")
+ self.end_headers()
+ self.wfile.write(content)
+
+ @override
+ def log_message(self, format: str, *args: object) -> None:
+ """Keep the listener silent: the default writes every request line, key included, to stderr."""
+
+
+class LoopbackListener:
+ """A one-shot listener on `127.0.0.1`, open from construction until `close()` or the end of a `with` block.
+
+ Construct it, send the browser to a URL naming `port` and `state`, then `wait_for_api_key`.
+ """
+
+ def __init__(self) -> None:
+ self.state = generate_state()
+ self._server = _LoopbackServer(expected_state=self.state)
+
+ @property
+ def port(self) -> int:
+ """The port the system gave the listener."""
+ return self._server.server_port
+
+ def wait_for_api_key(self, *, timeout_seconds: float, on_refusal: RefusalCallback | None = None) -> str | None:
+ """Answer requests until a callback carrying this login's `state` hands over a key, or the time runs out.
+
+ Args:
+ timeout_seconds: How long to wait in all.
+ on_refusal: Called with each refused callback, as it is refused.
+
+ Returns:
+ The key, or `None` when none arrived in time.
+ """
+ deadline = time.monotonic() + timeout_seconds
+ while self._server.api_key is None:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return None
+ self._server.timeout = remaining
+ refusals_before = len(self._server.refusals)
+ self._server.handle_request()
+ if on_refusal is not None:
+ for refusal in self._server.refusals[refusals_before:]:
+ on_refusal(refusal)
+ return self._server.api_key
+
+ def close(self) -> None:
+ """Stop listening."""
+ self._server.server_close()
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None) -> None:
+ self.close()
diff --git a/pipelex/cli/commands/migrate_cmd.py b/pipelex/cli/commands/migrate_cmd.py
index 897f9cbf9..3839424cc 100644
--- a/pipelex/cli/commands/migrate_cmd.py
+++ b/pipelex/cli/commands/migrate_cmd.py
@@ -22,6 +22,17 @@ What the per-file transaction refuses over is narrower than that, and worth not
compares against the reading *that pass* took, so what it catches is a writer landing between this
run's own read and its replace, and not an edit made while the confirmation was pending.
+**The first step is the cleanup of a former release.** A machine set up by a release that ran models
+through the Pipelex Gateway carries tables, profiles and files the runtime no
+longer has, and the boot refuses it with an error naming this command. That cleanup is no ledger
+entry — it removes what a release left rather than reshaping a file — so it runs beside the replay,
+first, under the same two passes, the same question and the same backups
+(`pipelex/migration/former_release_cleanup.py`). The replay leaves the files the cleanup removes out
+of its walk in both passes, so a file about to go is never counted, migrated or reported on. On
+every run, a dry run and one with nothing to clean included, the cleanup reads the files each boot
+merges as it leaves them, and a machine that still cannot start is reported as such, with a non-zero
+exit, never as a success or as nothing to do.
+
See `docs/migration-ledger.md`.
"""
@@ -34,13 +45,16 @@ from rich.markup import escape
from rich.panel import Panel
from rich.prompt import Confirm
+from pipelex.migration.former_release_cleanup import clean_former_release
from pipelex.migration.run import config_directories_to_migrate, ensure_config_dir_gitignores, migrate_config_directories
from pipelex.runtime_hub import get_console
from pipelex.suggested_fix import WILDCARD_SEGMENT, DeleteKeyOp, DeleteTableOp, MoveKeyOp, RemapValueOp, RenameTableKeyOp
if TYPE_CHECKING:
+ from collections.abc import Collection
from pathlib import Path
+ from pipelex.migration.former_release_cleanup import FormerReleaseCleanup
from pipelex.migration.plan import BlockedEntry, MigrationReport, UnexplainedPath
from pipelex.suggested_fix import MigrationOp
@@ -71,10 +85,12 @@ def migrate_cmd(*, dry_run: bool = False, yes: bool = False) -> None:
for directory in config_dirs:
console.print(f" [dim]{escape(str(directory))}[/dim]")
- rehearsal = migrate_config_directories(config_dirs=config_dirs, dry_run=True)
+ former_release = clean_former_release(config_dirs=config_dirs, dry_run=True)
+ print_former_release_cleanup(cleanup=former_release)
+ rehearsal = migrate_config_directories(config_dirs=config_dirs, dry_run=True, skipped_paths=former_release.removed_paths)
_print_report(report=rehearsal, wrote=False)
- if rehearsal.is_clean:
+ if rehearsal.is_clean and former_release.is_clean:
if not dry_run:
# The write pass is what normally leaves the config-directory `.gitignore` behind, and
# this return means it is never going to run. A machine with nothing to carry forward
@@ -88,10 +104,10 @@ def migrate_cmd(*, dry_run: bool = False, yes: bool = False) -> None:
if dry_run:
console.print("[dim]Dry run — nothing was written.[/dim]")
console.print()
- _exit_on_attention(report=rehearsal)
+ _exit_on_attention(report=rehearsal, cleanup=former_release)
return
- if not rehearsal.changed_plans:
+ if not rehearsal.changed_plans and not former_release.files:
# The clean return's twin, and it needs the rule for the same reason: `--dry-run` has
# already returned above, so this run is authorized to write, and nothing was declined
# because nothing was asked. Unguarded on purpose — the guard would be dead code, and it
@@ -101,32 +117,105 @@ def migrate_cmd(*, dry_run: bool = False, yes: bool = False) -> None:
# when there are none to apply is a prompt with one honest answer.
console.print(_panel(message="Nothing here can be migrated automatically — see the notes above.", style="yellow"))
console.print()
- _exit_on_attention(report=rehearsal)
+ _exit_on_attention(report=rehearsal, cleanup=former_release)
return
- if not yes and not Confirm.ask(f"[bold]Migrate {len(rehearsal.changed_plans)} file(s)?[/bold]", default=True):
+ question = _question(cleaned=len(former_release.files), migrated=len(rehearsal.changed_plans))
+ if not yes and not Confirm.ask(f"[bold]{question}[/bold]", default=True):
console.print("[yellow]Migration cancelled — nothing was written.[/yellow]")
console.print()
# The "no" answered "shall I write?"; what the rehearsal left for a person is still there.
- _exit_on_attention(report=rehearsal)
+ _exit_on_attention(report=rehearsal, cleanup=former_release)
return
- applied = apply_pending_migrations(config_dirs=config_dirs)
- written = len(applied.written_plans)
- unconfirmed = any(plan.blocked_reason is not None and plan.blocked_reason.leaves_the_write_unconfirmed for plan in applied.plans)
+ # The cleanup first: it removes files the replay leaves out of its walk, and rewrites files the replay never claims.
+ cleaned = apply_former_release_cleanup(config_dirs=config_dirs)
+ applied = apply_pending_migrations(config_dirs=config_dirs, skipped_paths=cleaned.removed_paths)
+ done = _done_in_words(cleaned=len(cleaned.applied_files), migrated=len(applied.written_plans))
+ unconfirmed = any(plan.blocked_reason is not None and plan.blocked_reason.leaves_the_write_unconfirmed for plan in applied.plans) or any(
+ file.blocked_reason is not None and file.blocked_reason.leaves_the_write_unconfirmed for file in cleaned.files
+ )
if unconfirmed:
# Tested first: a file in an unknown state is the verdict of the run, however many others landed.
- lead = f"Migrated {written} file(s), but no write could be confirmed for every file" if written else "No write could be confirmed"
+ lead = f"{done}, but no write could be confirmed for every file" if done else "No write could be confirmed"
console.print(_panel(message=f"{lead} — compare each file marked above against the rescue copy named there.", style="red"))
- elif written:
- console.print(_panel(message=f"Migrated {written} file(s); a copy of each original is beside it.", style="green"))
+ elif cleaned.still_blocking:
+ lead = f"{done}, but Pipelex still cannot start" if done else "Pipelex still cannot start"
+ console.print(_panel(message=f"{lead} — see the lines marked ✗ above.", style="red"))
+ elif done:
+ console.print(_panel(message=f"{done}; a copy of each original is beside it.", style="green"))
else:
console.print(_panel(message="Nothing was written.", style="yellow"))
console.print()
- _exit_on_attention(report=applied)
+ _exit_on_attention(report=applied, cleanup=cleaned)
+
+
+def _question(*, cleaned: int, migrated: int) -> str:
+ """The confirmation, counting the files of each step the run is about to write."""
+ if not cleaned:
+ return f"Migrate {migrated} file(s)?"
+ if not migrated:
+ return f"Clean up {cleaned} file(s)?"
+ return f"Clean up {cleaned} file(s) and migrate {migrated}?"
+
+
+def _done_in_words(*, cleaned: int, migrated: int) -> str:
+ """What the write pass did, or the empty string when it wrote nothing."""
+ if cleaned and migrated:
+ return f"Cleaned up {cleaned} file(s) a former release left and migrated {migrated}"
+ if cleaned:
+ return f"Cleaned up {cleaned} file(s) a former release left"
+ if migrated:
+ return f"Migrated {migrated} file(s)"
+ return ""
+
+
+def apply_former_release_cleanup(*, config_dirs: list[Path]) -> FormerReleaseCleanup:
+ """Remove what a former release left, render what it did, and hand the report back.
+
+ The cleanup's write pass on its own, like `apply_pending_migrations` beside it, for the callers that have
+ already shown the findings and asked their own question: `pipelex doctor --fix` and `pipelex init`.
+ """
+ cleanup = clean_former_release(config_dirs=config_dirs, dry_run=False)
+ print_former_release_cleanup(cleanup=cleanup)
+ return cleanup
+
+def print_former_release_cleanup(*, cleanup: FormerReleaseCleanup) -> None:
+ """Each file the cleanup touches, with its changes: rehearsed (`→`) or made (`✓`), and the copy kept of it.
-def apply_pending_migrations(*, config_dirs: list[Path]) -> MigrationReport:
+ Then what would still stop the boot, the check the cleanup runs on every run: once it has written, once a dry run
+ would have, or as the files stand when there was nothing to clean.
+ """
+ console = get_console()
+ for file in cleanup.files:
+ console.print()
+ console.print(f" [cyan]{escape(str(file.file_path))}[/cyan] [dim](left by a former release)[/dim]")
+ if file.blocked_reason is not None:
+ console.print(f" [red]✗[/red] {file.blocked_reason}: {escape(file.blocked_detail or '')}")
+ continue
+ marker = "[green]✓[/green]" if file.was_applied else "[green]→[/green]"
+ for change in file.changes:
+ console.print(f" {marker} {escape(change)}")
+ if file.backup_path is not None:
+ console.print(f" [dim]backup: {escape(str(file.backup_path))}[/dim]")
+ if cleanup.still_blocking:
+ console.print()
+ console.print(f" [bold]{_still_blocking_heading(cleanup=cleanup)}[/bold]")
+ for problem in cleanup.still_blocking:
+ console.print(f" [red]✗[/red] {escape(problem)}")
+
+
+def _still_blocking_heading(*, cleanup: FormerReleaseCleanup) -> str:
+ """The heading over what still stops the boot, in the tense of the run."""
+ if not cleanup.files:
+ return "Pipelex cannot start:"
+ if cleanup.dry_run:
+ return "After the cleanup, Pipelex would still not start:"
+ return "After the cleanup, Pipelex still cannot start:"
+
+
+def apply_pending_migrations(*, config_dirs: list[Path], skipped_paths: Collection[Path] = ()) -> MigrationReport:
"""Write what a migration would write, render what it did, and hand the report back.
The second of this command's two passes on its own, with no exit code attached to it. A
@@ -135,20 +224,23 @@ def apply_pending_migrations(*, config_dirs: list[Path]) -> MigrationReport:
pending-migrations row *is* the dry run. Calling `migrate_cmd` there instead would end the
doctor's own run: the command exits the process when something is left for a person, and the
doctor still has rows to render and an exit code of its own to set.
+
+ `skipped_paths` are the files the former-release cleanup removes, which the walk leaves out.
"""
- applied = migrate_config_directories(config_dirs=config_dirs, dry_run=False)
+ applied = migrate_config_directories(config_dirs=config_dirs, dry_run=False, skipped_paths=skipped_paths)
_print_written(report=applied)
return applied
-def _exit_on_attention(*, report: MigrationReport) -> None:
+def _exit_on_attention(*, report: MigrationReport, cleanup: FormerReleaseCleanup) -> None:
"""Leave a non-zero exit code behind when something needs a human.
The exit code is presentation, not the verdict — `pipelex-agent migrate` carries the same
verdict as a structured field. What it means here is narrow and worth keeping narrow: *this
- run left something a person has to look at*, never *this run failed*.
+ run left something a person has to look at*, never *this run failed*. A file the cleanup could
+ not clean is one of those, as much as a file the replay could not migrate.
"""
- if report.needs_attention:
+ if report.needs_attention or cleanup.needs_attention:
sys.exit(1)
diff --git a/pipelex/cli/error_handlers.py b/pipelex/cli/error_handlers.py
index dd01a1629..92d233152 100644
--- a/pipelex/cli/error_handlers.py
+++ b/pipelex/cli/error_handlers.py
@@ -12,6 +12,7 @@ from rich.traceback import Traceback
from pipelex.base_exceptions import ValidationErrorCategory, ValidationErrorItem, error_domain_is_input, iter_cause_chain
from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
+from pipelex.migration.exceptions import FormerReleaseConfigError
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipeline.exceptions import ValidateBundleError
from pipelex.pipeline.validation_render import build_fix_command, count_applicable_fixes, suggested_fix_label, validation_item_title
@@ -433,3 +434,21 @@ def handle_telemetry_config_validation_error(exc: TelemetryConfigValidationError
console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")
raise typer.Exit(1) from exc
+
+
+def handle_former_release_config_error(*, exc: FormerReleaseConfigError) -> NoReturn:
+ """Handle the boot's refusal of a configuration a former release set up for the Pipelex Gateway.
+
+ The message is the whole answer: it names what stops the boot, file by file, and the remedies, `pipelex migrate`
+ and `pipelex init`. A traceback would bury it, and any advice added here would repeat it.
+
+ Args:
+ exc: The former release configuration error.
+ """
+ console = get_console()
+ print_traceback_if_requested(console=console)
+ console.print("\n[bold red]❌ Configuration left by a former release[/bold red]\n")
+ console.print(escape(exc.message))
+ console.print()
+ console.print(f"[dim]Join our Discord for help: {URLs.discord}[/dim]\n")
+ raise typer.Exit(1) from exc
diff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.py
index a0e9d2a40..64dcddc70 100644
--- a/pipelex/cogt/model_backends/backend_library.py
+++ b/pipelex/cogt/model_backends/backend_library.py
@@ -39,6 +39,7 @@ from pipelex.cogt.model_backends.model_spec_factory import (
from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndex
from pipelex.cogt.model_backends.model_spec_keys import describe_rejected_keys, split_model_spec_keys
from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.migration.former_release import MODEL_SPECS_SECTION_KEY
from pipelex.migration.plan import MigrationPlan
from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations
from pipelex.system.configuration.config_loader import config_manager
@@ -68,9 +69,6 @@ if TYPE_CHECKING:
InferenceBackendLibraryRoot = dict[str, InferenceBackend]
-# A `backends.toml` key that named the remote-config section holding a backend's model specs.
-RETIRED_MODEL_SPECS_SECTION_KEY = "model_specs_section"
-
# Where a keyless load records the variables a backend's own file references, beside the fields of
# its `backends.toml` table (see `InferenceBackend.unresolved_credentials`).
UNRESOLVED_MODEL_SPECS_KEY = "model_specs"
@@ -219,14 +217,14 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
enabled = inference_backend_blueprint_dict_raw.get("enabled", True)
if not enabled and not include_disabled:
continue
- if enabled and RETIRED_MODEL_SPECS_SECTION_KEY in backend_table:
+ if enabled and MODEL_SPECS_SECTION_KEY in backend_table:
# The key once pointed at specs the remote config served. Read as an extra key, it would
# boot the backend with whatever its own file lists — for a backend whose file was a
# comment-only template, nothing — and every model routed to it would go missing with no
# word on why. Fatal in both modes, like any document that is wrong.
msg = (
f"Invalid inference backend '{backend_name}' in {library_paths_description}: "
- f"'{RETIRED_MODEL_SPECS_SECTION_KEY}' is no longer supported, because model specs are no longer "
+ f"'{MODEL_SPECS_SECTION_KEY}' is no longer supported, because model specs are no longer "
f"downloaded. List the backend's models in 'backends/{backend_name}.toml' and remove the key, "
f"or disable the backend."
)
diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.py
index 0920f5d44..6d24e8120 100644
--- a/pipelex/cogt/models/model_manager.py
+++ b/pipelex/cogt/models/model_manager.py
@@ -26,6 +26,8 @@ from pipelex.cogt.models.model_deck import ModelDeck, ModelDeckBlueprint
from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprint
from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
from pipelex.config import get_config
+from pipelex.migration.exceptions import FormerReleaseConfigError
+from pipelex.migration.former_release import describe_former_release_boot_refusal, former_release_boot_blockers
from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations
from pipelex.system.configuration.config_loader import config_manager
from pipelex.tools.misc.file_utils import find_files_in_dir
@@ -82,9 +84,20 @@ class ModelManager(ModelManagerAbstract):
# sequences — the base file, then the personal override files — see
# `ConfigLoader.backends_file_paths`.
resolved_backends_dir_path = backends_dir_path or str(config_manager.backends_dir_path)
+ resolved_backends_library_paths = backends_library_paths or config_manager.backends_file_paths()
+ resolved_routing_profile_library_paths = routing_profile_library_paths or config_manager.routing_profiles_file_paths()
+ # What a former release left for the Pipelex Gateway is refused here, by one error naming the
+ # cleanup, ahead of the refusals below: the Gateway's unset key, its backend file declaring no model, or a
+ # routing profile sending models to a backend that is not enabled — none of which says why, or what to run.
+ if blockers := former_release_boot_blockers(
+ backends_library_paths=resolved_backends_library_paths,
+ routing_profile_library_paths=resolved_routing_profile_library_paths,
+ ):
+ msg = describe_former_release_boot_refusal(blockers=blockers)
+ raise FormerReleaseConfigError(msg)
self.inference_backend_library.load(
secrets_provider=secrets_provider,
- backends_library_paths=backends_library_paths or config_manager.backends_file_paths(),
+ backends_library_paths=resolved_backends_library_paths,
backends_dir_path=resolved_backends_dir_path,
# A keyless boot knows every enabled backend's models and resolves no credential; the boot
# that needs inference resolves every one and refuses to start without it.
@@ -104,7 +117,7 @@ class ModelManager(ModelManagerAbstract):
)
enabled_backends = self.inference_backend_library.all_enabled_backends()
self._routing_profile = load_active_routing_profile(
- routing_profile_library_paths=routing_profile_library_paths or config_manager.routing_profiles_file_paths(),
+ routing_profile_library_paths=resolved_routing_profile_library_paths,
enabled_backends=enabled_backends,
)
model_deck_paths = ModelManager.get_model_deck_paths(deck_dir_path=deck_dir_path or str(config_manager.model_decks_dir_path))
diff --git a/pipelex/fix_ops/applier.py b/pipelex/fix_ops/applier.py
index 869e753f8..d79303954 100644
--- a/pipelex/fix_ops/applier.py
+++ b/pipelex/fix_ops/applier.py
@@ -826,6 +826,24 @@ class _OpResult(NamedTuple):
detail: str | None = None
+def delete_literally(*, toml_doc: TOMLDocument, table_path: Sequence[str], key: str) -> bool:
+ """Delete one key or table, at a path read literally, taking its comments as ``delete_key`` does.
+
+ The ops read a ``*`` segment as "every key of this table" and refuse it as a key, which is what a ledger means
+ by it. A document can still spell a key ``*`` — a catch-all route ``"*" = "..."`` above all — and a caller that
+ read the key off the document and means that one key deletes it here: no segment is expanded, and a ``*`` is
+ looked up as the key it is.
+
+ Returns:
+ Whether the key was there and went; a path or key the document lacks changes nothing.
+ """
+ parent_table = _resolve_table(toml_doc=toml_doc, table_path=list(table_path))
+ if parent_table is None or key not in parent_table:
+ return False
+ _lift_out(toml_doc=toml_doc, parent_table=parent_table, table_path=list(table_path), key=key)
+ return True
+
+
def _expand_table_paths(*, toml_doc: TOMLDocument, table_path: list[str]) -> list[list[str]]:
"""Resolve ``table_path`` into the concrete paths it addresses in this document.
diff --git a/pipelex/hosted/api_key_check.py b/pipelex/hosted/api_key_check.py
new file mode 100644
index 000000000..1f2bb9855
--- /dev/null
+++ b/pipelex/hosted/api_key_check.py
@@ -0,0 +1,106 @@
+"""Whether a Pipelex API key looks like one, and whether the hosted API accepts it.
+
+`pipelex login` checks a key before it saves it, and `pipelex init` reads whether one is already set, so both go
+through here: the format check needs no network, and the hosted check asks the hosted API who the key belongs to
+(`GET /v1/me`) through the one client factory. No CLI import, so either CLI can use it.
+"""
+
+import asyncio
+import re
+from enum import StrEnum
+from typing import NamedTuple
+
+from mthds.protocol.exceptions import PipelineRequestError
+from pipelex_sdk.client import PipelexAPIClient
+from pipelex_sdk.errors import ApiResponseError, ApiUnreachableError
+from pipelex_sdk.product_models import UserProfile
+
+from pipelex.hosted.client_factory import make_hosted_client, redact_url_for_display
+from pipelex.hosted.exceptions import HostedBaseUrlError
+
+#: Every Pipelex API key starts with this.
+PIPELEX_API_KEY_PREFIX = "plx_sk_"
+# The prefix, then URL-safe characters only: a key is written unquoted to a `.env` file, so a character a `.env`
+# parser reads as a quote, a comment or a separator must never be part of one.
+_PIPELEX_API_KEY_PATTERN = re.compile(rf"{PIPELEX_API_KEY_PREFIX}[A-Za-z0-9._~-]+")
+#: The statuses by which the hosted API says the key itself is refused.
+API_KEY_REFUSAL_STATUSES: frozenset[int] = frozenset({401, 403})
+
+
+def is_well_formed_pipelex_api_key(*, api_key: str) -> bool:
+ """Whether the value has the shape of a Pipelex API key: `plx_sk_` followed by URL-safe characters."""
+ return _PIPELEX_API_KEY_PATTERN.fullmatch(api_key) is not None
+
+
+class ApiKeyVerdict(StrEnum):
+ """What the hosted API said of a key."""
+
+ ACCEPTED = "accepted"
+ REFUSED = "refused"
+ UNCHECKED = "unchecked"
+
+
+class ApiKeyCheck(NamedTuple):
+ """The outcome of asking the hosted API who a key belongs to.
+
+ `http_status` is the refusal's status (401 or 403) when the key was refused, and the status of an answer that
+ neither accepted nor refused it. `reason` says why the key could not be checked, for a warning; it never quotes
+ the key. `account_email` names the account an accepted key belongs to, when the hosted API said.
+ """
+
+ verdict: ApiKeyVerdict
+ base_url: str
+ http_status: int | None = None
+ reason: str | None = None
+ account_email: str | None = None
+
+
+async def _get_profile(*, client: PipelexAPIClient) -> UserProfile:
+ async with client:
+ return await client.get_me()
+
+
+def check_pipelex_api_key(*, api_key: str) -> ApiKeyCheck:
+ """Ask the hosted API whether it accepts this key, with `GET /v1/me`.
+
+ The hosted API is the one `PIPELEX_BASE_URL` names, else `https://api.pipelex.com`. Only a 401 or a 403 refuses
+ the key; any other failure (the network, a bad base URL, another status, an answer that is not a profile) leaves it
+ unchecked, with the reason.
+
+ Args:
+ api_key: The key to check, sent as the bearer token of this one request.
+
+ Returns:
+ The verdict, the base URL asked, and the status, the reason or the account.
+ """
+ try:
+ client = make_hosted_client(api_key=api_key)
+ except HostedBaseUrlError as exc:
+ return ApiKeyCheck(verdict=ApiKeyVerdict.UNCHECKED, base_url="", reason=exc.message)
+ base_url = client.base_url
+ try:
+ profile = asyncio.run(_get_profile(client=client))
+ except ApiResponseError as exc:
+ if exc.status in API_KEY_REFUSAL_STATUSES:
+ return ApiKeyCheck(verdict=ApiKeyVerdict.REFUSED, base_url=base_url, http_status=exc.status)
+ return ApiKeyCheck(
+ verdict=ApiKeyVerdict.UNCHECKED,
+ base_url=base_url,
+ http_status=exc.status,
+ reason=f"the hosted API at {redact_url_for_display(url=base_url)} answered HTTP {exc.status}",
+ )
+ except ApiUnreachableError:
+ return ApiKeyCheck(
+ verdict=ApiKeyVerdict.UNCHECKED,
+ base_url=base_url,
+ reason=f"the hosted API at {redact_url_for_display(url=base_url)} could not be reached",
+ )
+ except (PipelineRequestError, ValueError) as exc:
+ # A non-2xx answer the SDK maps to its plainer error, or a 2xx body that is not JSON or not a profile (a JSON
+ # decoding error and a pydantic validation error are both `ValueError`s): whatever answered is not the hosted API.
+ return ApiKeyCheck(
+ verdict=ApiKeyVerdict.UNCHECKED,
+ base_url=base_url,
+ reason=f"the hosted API at {redact_url_for_display(url=base_url)} did not answer with an account ({type(exc).__name__})",
+ )
+ return ApiKeyCheck(verdict=ApiKeyVerdict.ACCEPTED, base_url=base_url, account_email=profile.email)
diff --git a/pipelex/hosted/client_factory.py b/pipelex/hosted/client_factory.py
index 72a1d895e..964211d87 100644
--- a/pipelex/hosted/client_factory.py
+++ b/pipelex/hosted/client_factory.py
@@ -57,12 +57,14 @@ def redact_url_for_display(*, url: str) -> str:
return shown
-def make_hosted_client(*, base_url: str | None = None) -> PipelexAPIClient:
+def make_hosted_client(*, base_url: str | None = None, api_key: str | None = None) -> PipelexAPIClient:
"""Build a hosted API client, the key and the base URL left to the SDK's own resolution.
Args:
base_url: The origin to call, `scheme://host[:port]`, as `--base-url` gives it. `None` lets the SDK read
`PIPELEX_BASE_URL`, then fall back to the hosted API.
+ api_key: The key to send instead of `PIPELEX_API_KEY`, for `pipelex login` checking a key it has not saved
+ yet. `None` lets the SDK read `PIPELEX_API_KEY`.
Returns:
A client not yet started: use it as an async context manager.
@@ -71,7 +73,7 @@ def make_hosted_client(*, base_url: str | None = None) -> PipelexAPIClient:
HostedBaseUrlError: If the base URL is not host-only, naming the setting it came from.
"""
try:
- return PipelexAPIClient(base_url=base_url, app_info=AppInfo(name=_APP_NAME, version=get_package_version()))
+ return PipelexAPIClient(api_key=api_key, base_url=base_url, app_info=AppInfo(name=_APP_NAME, version=get_package_version()))
except PipelineRequestError:
# The SDK's constructor raises this for one reason only: a base URL that is not an origin.
source = "--base-url" if base_url is not None else PIPELEX_BASE_URL_ENV_KEY
diff --git a/pipelex/hosted/error_rendering.py b/pipelex/hosted/error_rendering.py
index 4c6f47b44..692db549f 100644
--- a/pipelex/hosted/error_rendering.py
+++ b/pipelex/hosted/error_rendering.py
@@ -33,9 +33,12 @@ from pipelex.base_exceptions import ErrorDomain
from pipelex.hosted.client_factory import HOSTED_API_DEFAULT_BASE_URL, PIPELEX_API_KEY_ENV_KEY, PIPELEX_BASE_URL_ENV_KEY
from pipelex.hosted.exceptions import HostedMethodInvalidError, HostedRunError, HostedRunInterruptedError, HostedRunPollingError
-#: Where a hosted run takes its key from, and where a person gets one.
+#: The command that gets a Pipelex API key and saves it where a hosted run reads it.
+PIPELEX_LOGIN_COMMAND = "pipelex login"
+#: Where a hosted run takes its key from, and how a person gets one.
HOSTED_API_KEY_NEXT_STEP = (
- f"Set {PIPELEX_API_KEY_ENV_KEY} to a Pipelex API key (plx_sk_…) in your shell or in ~/.pipelex/.env; create one on app.pipelex.com"
+ f"Run {PIPELEX_LOGIN_COMMAND} to get a Pipelex API key (plx_sk_…) and save it to ~/.pipelex/.env as {PIPELEX_API_KEY_ENV_KEY}, "
+ f"or set {PIPELEX_API_KEY_ENV_KEY} to one in your shell"
)
#: Where a hosted run takes its origin from.
HOSTED_BASE_URL_NEXT_STEP = (
@@ -56,7 +59,8 @@ HOSTED_REFUSAL_NEXT_STEPS_BY_STATUS: dict[int, str] = {
401: f"The hosted API refused the credentials. {HOSTED_API_KEY_NEXT_STEP}",
403: (
f"The hosted API refused the request: check that {PIPELEX_API_KEY_ENV_KEY} is a valid key for the API at "
- f"{PIPELEX_BASE_URL_ENV_KEY} (or --base-url); when it is, the message says what this request may not do"
+ f"{PIPELEX_BASE_URL_ENV_KEY} (or --base-url), and run {PIPELEX_LOGIN_COMMAND} to replace it when it is not; "
+ "when it is, the message says what this request may not do"
),
404: f"The hosted API has no such method or route: check the method's address or catalog id. {HOSTED_BASE_URL_NEXT_STEP}",
429: _RATE_LIMITED_NEXT_STEP,
diff --git a/pipelex/kit/paths.py b/pipelex/kit/paths.py
index efba84380..4668bb01c 100644
--- a/pipelex/kit/paths.py
+++ b/pipelex/kit/paths.py
@@ -1,6 +1,11 @@
from importlib.resources import files
from importlib.resources.abc import Traversable
+#: The file releases up to v0.72 wrote into a configuration directory to record the Pipelex Gateway's terms
+#: acceptance. Nothing reads it any more; this is the one spelling of its name, which the migration's surfaces and the
+#: former-release cleanup derive theirs from.
+RETIRED_SERVICE_FILE_NAME = "pipelex_service.toml"
+
# Git-ignored config files that should not be synced between .pipelex and kit/configs.
# These are personal override files that differ per developer/environment:
# - pipelex_service.toml: retired, written by earlier releases; a developer's checkout may still hold one
@@ -18,7 +23,7 @@ from importlib.resources.abc import Traversable
GIT_IGNORED_CONFIG_FILES: frozenset[str] = frozenset(
{
".DS_Store",
- "pipelex_service.toml",
+ RETIRED_SERVICE_FILE_NAME,
"pipelex_override.toml",
"telemetry_override.toml",
"backends_override.toml",
diff --git a/pipelex/migration/exceptions.py b/pipelex/migration/exceptions.py
index 1e7b02b2d..bb913aa56 100644
--- a/pipelex/migration/exceptions.py
+++ b/pipelex/migration/exceptions.py
@@ -1,4 +1,4 @@
-from pipelex.base_exceptions import ErrorDomain, PipelexError
+from pipelex.base_exceptions import ErrorDomain, PipelexError, PipelexSetupError
class MigrationError(PipelexError):
@@ -34,3 +34,16 @@ class MigrationSnapshotRefusedError(MigrationError):
"""
_declared_title = "Migration snapshot refused"
+
+
+class FormerReleaseConfigError(PipelexSetupError):
+ """The inference configuration still carries what a former release wrote for the Pipelex Gateway.
+
+ Raised at boot, before the backend and routing profile libraries load, when a retired backend is enabled, a backend
+ still names `model_specs_section`, or the active routing profile sends models to a retired backend. The message
+ names each of them with its file, and the remedies: `pipelex migrate`, which removes what that release left and
+ keeps a copy of each file it changes, and `pipelex init`, which offers the same cleanup and then sets Pipelex up
+ again, on the hosted Pipelex API or on this machine.
+ """
+
+ _declared_title = "Configuration left by a former release"
diff --git a/pipelex/migration/former_release.py b/pipelex/migration/former_release.py
new file mode 100644
index 000000000..4ddefc5ba
--- /dev/null
+++ b/pipelex/migration/former_release.py
@@ -0,0 +1,1241 @@
+"""What a former release left in the configuration directories, found by reading them and nothing else.
+
+Releases up to v0.72 ran models through the Pipelex Gateway. The installations they set up still carry what that
+took: a `pipelex_gateway` table in `inference/backends.toml` with a per-backend file beside the others, routing profiles
+that send models to it (`active = "all_pipelex_gateway"` above all), a `model_specs_section` key naming specs the
+remote configuration used to serve, the `pipelex_service.toml` that recorded the Gateway's terms acceptance, and the
+`pipelex_gateway_models*.md` model lists. The runtime no longer has that backend, so an enabled table or an active
+profile pointing at it stops every boot with a refusal about one file that says nothing of the release behind it.
+
+Pipelex Manifold, which those releases offered as a private beta, is not retired: the hosted plane runs on it and
+declares its backend and profile itself. What a former release left for it — a table shipped disabled, its per-backend
+file, the `all_pipelex_manifold` profile — is inert on a user's machine and is left as it is (`MANIFOLD_BACKEND_NAME`).
+
+`detect_former_release_across` is the one reading of that state. The boot, `pipelex doctor`, `pipelex init`'s inspect
+stage and the `pipelex migrate` cleanup (`former_release_cleanup.py`) all ask it or `former_release_boot_blockers`,
+which reads the same way, so what one of them calls a former release's configuration is what every other one finds and
+removes. It reads, and it never writes or imports anything from the CLI.
+
+**The documents are read as the boot merges them, across directories.** A boot reads one base `backends.toml` and one
+base `routing_profiles.toml` — the project's when it has one, the home's otherwise — then the home's override, then the
+project's (`inference_merge_sequences`). A profile one directory defines can be made active by a file in the other, and
+an override can retarget a profile its base defines, so each finding is decided over every sequence its file is read
+in, never over its own directory alone.
+
+**A finding says which file it is in, and whether it stops the boot.** The shapes that do are the ones the boot
+refuses: a retired backend, or a backend still naming `model_specs_section`, left enabled; and an active routing
+profile that sends models to a retired backend, or a retired profile no file defines. Everything else — a disabled
+table, a profile nobody activates, the files beside the backends, the service file — is inert, and is found so that the
+cleanup leaves nothing of that release behind. Whether a machine's boot is refused is `former_release_boot_blockers`
+over the exact sequences that boot reads: a per-directory reading can be wrong both ways, since a project's own base
+hides the home's, and an `active` in one directory can name a profile defined in the other.
+
+**What the cleanup does to the routing profiles is planned per boot, and checked before anything is written.** A
+profile goes from a file whose own `default` is a retired backend, unless the boot reading it still has the profile
+alive — a file before keeps it, or a file after gives it a live `default` — and then only that `default` goes. A file
+both boots read keeps whatever either boot needs. Then each way of moving the `active` keys the release spoiled is tried
+on each boot's library as the cleanup would leave it (`_routing_plan`): no plan may stop a boot that starts today or
+change the profile it starts on, a profile left with nothing to route by counts as gone, and the project's own override
+may be given an `active` of its own so that a change one boot needs does not reach the other. Where nothing can suit
+both boots, the shared file is left as it is, for a hand edit, and the findings say so (`FormerReleaseFindings.hand_edits`).
+
+See `docs/migration-ledger.md` → "A former release's configuration".
+"""
+
+import copy
+import itertools
+from collections.abc import Mapping, Sequence
+from enum import StrEnum
+from pathlib import Path
+from typing import Any, NamedTuple, cast
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from pipelex.base_exceptions import PipelexUnexpectedError
+from pipelex.cogt.model_routing.routing_profile_factory import RoutingProfileLibraryBlueprint
+from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.kit.paths import RETIRED_SERVICE_FILE_NAME, get_kit_configs_dir
+from pipelex.system.configuration.config_loader import (
+ BACKENDS_DIR_NAME,
+ BACKENDS_FILE_NAME,
+ BACKENDS_OVERRIDE_FILE_NAME,
+ INFERENCE_DIR_NAME,
+ ROUTING_PROFILES_FILE_NAME,
+ ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+)
+from pipelex.tools.misc.exceptions import TomlError
+from pipelex.tools.misc.json_utils import deep_update
+from pipelex.tools.misc.toml_utils import load_toml_from_content, load_toml_from_path
+
+#: The command that sets Pipelex up, which offers the cleanup first when it finds what a former release left.
+INIT_COMMAND = "pipelex init"
+
+#: The backends a former release shipped that this one no longer has: the Pipelex Gateway.
+RETIRED_BACKEND_NAMES: frozenset[str] = frozenset({"pipelex_gateway"})
+
+#: The routing profile a former release shipped to send every model to it. Recognized by name only where the
+#: file read holds no definition of the profile: an override naming one over a base in another directory.
+RETIRED_ROUTING_PROFILE_NAMES: frozenset[str] = frozenset({"all_pipelex_gateway"})
+
+#: Pipelex Manifold's backend, which is not retired: the hosted plane runs every model through it, with a plugin of its
+#: own, and declares the `pipelex_manifold` table and the `all_pipelex_manifold` profile itself. A former release shipped
+#: the table disabled and still naming `model_specs_section`, which is inert while it is disabled, so nothing here
+#: reports or rewrites that table, and the hosted plane's is never touched. Enabled and still naming the key, the table
+#: is refused by the backend library's own load, whose message says what to change.
+MANIFOLD_BACKEND_NAME = "pipelex_manifold"
+
+#: The `backends.toml` key that named the remote-configuration section holding a backend's model specs.
+MODEL_SPECS_SECTION_KEY = "model_specs_section"
+
+#: The model lists a former release copied beside the backend files, a Markdown one and its plain-text twin.
+GATEWAY_MODELS_REFERENCE_GLOB = "pipelex_gateway_models*.md"
+
+#: The routing profile tables that route a pattern to a backend. `fallback_order` is not one: a disabled or absent
+#: backend in it is skipped, so it never stops a boot and is left as written.
+ROUTE_TABLE_KEYS: tuple[str, ...] = ("routes", "optional_routes")
+
+#: The keys of a routing profile document the cleanup addresses.
+ROUTING_PROFILES_KEY = "profiles"
+ACTIVE_KEY = "active"
+
+_ROUTES_KEY = "routes"
+_DEFAULT_KEY = "default"
+_DESCRIPTION_KEY = "description"
+_FALLBACK_ORDER_KEY = "fallback_order"
+_ENABLED_KEY = "enabled"
+
+_OVERRIDE_FILE_NAMES = frozenset({BACKENDS_OVERRIDE_FILE_NAME, ROUTING_PROFILES_OVERRIDE_FILE_NAME})
+
+
+class FormerReleaseFindingKind(StrEnum):
+ """One kind of thing a former release left behind."""
+
+ RETIRED_BACKEND_TABLE = "retired_backend_table"
+ """A `pipelex_gateway` table in `backends.toml` or its override. Stops the boot when enabled."""
+
+ MODEL_SPECS_SECTION_KEY = "model_specs_section_key"
+ """A `model_specs_section` key on another backend's table, Pipelex Manifold's aside. Stops the boot when that backend
+ is enabled."""
+
+ RETIRED_BACKEND_FILE = "retired_backend_file"
+ """`inference/backends/pipelex_gateway.toml`, the retired backend's per-model file."""
+
+ GATEWAY_MODELS_REFERENCE = "gateway_models_reference"
+ """A `pipelex_gateway_models*.md` list of the models the Gateway served, beside the backend files."""
+
+ SERVICE_FILE = "service_file"
+ """The `pipelex_service.toml` that recorded the Gateway's terms acceptance."""
+
+ RETIRED_ROUTING_PROFILE = "retired_routing_profile"
+ """A routing profile whose default backend is a retired one, or the part an override holds of one, which goes."""
+
+ RETIRED_PROFILE_DEFAULT = "retired_profile_default"
+ """The `default` of a routing profile that otherwise stays, naming a retired backend: an override retargeting a
+ profile its base defines, or a base whose profile another file retargets to a live backend. Only the key goes."""
+
+ ROUTE_TO_RETIRED_BACKEND = "route_to_retired_backend"
+ """A route of a profile that otherwise stays, sending a pattern to a retired backend."""
+
+ ACTIVE_ROUTING_PROFILE = "active_routing_profile"
+ """The active routing profile sends models to a retired backend, by default or by a route, or is a retired profile
+ no file defines. Stops the boot that reads it."""
+
+ PROJECT_ACTIVE_ROUTING_PROFILE = "project_active_routing_profile"
+ """Not something the release left, but the change that keeps the project's boot whole: the project's override is
+ given an `active` of its own, because the file the project's boot reads it from is read by the boot outside the
+ project too, and has to change for one boot and not the other."""
+
+ @property
+ def is_a_file_of_its_own(self) -> bool:
+ """Whether the finding is a whole file, which the cleanup removes, rather than something inside a document."""
+ match self:
+ case (
+ FormerReleaseFindingKind.RETIRED_BACKEND_FILE
+ | FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE
+ | FormerReleaseFindingKind.SERVICE_FILE
+ ):
+ return True
+ case (
+ FormerReleaseFindingKind.RETIRED_BACKEND_TABLE
+ | FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY
+ | FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE
+ | FormerReleaseFindingKind.RETIRED_PROFILE_DEFAULT
+ | FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND
+ | FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE
+ | FormerReleaseFindingKind.PROJECT_ACTIVE_ROUTING_PROFILE
+ ):
+ return False
+
+
+class FormerReleaseFinding(BaseModel):
+ """One thing a former release left, in one file.
+
+ Every part of it is a path, a key, or one of the retired names this module knows: no value read from the file is
+ carried, the rule every migration report holds to.
+ """
+
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ kind: FormerReleaseFindingKind = Field(strict=False)
+ file_path: Path
+ subject: str | None = None
+ """The backend or routing profile the finding is about; `None` for a file found by its name alone."""
+
+ route_pattern: str | None = None
+ """The model pattern of a route to a retired backend: the route itself, or the one that makes an active profile retired."""
+
+ route_table: str | None = None
+ """Which of the profile's route tables holds that route, `routes` or `optional_routes`."""
+
+ retired_backend: str | None = None
+ """Which retired backend a routing profile or a route sends models to."""
+
+ blocks_boot: bool = False
+ """Whether this stops a boot that reads the file: the runtime refuses it rather than ignoring it."""
+
+ profile_is_removed: bool = False
+ """For an active routing profile: whether the cleanup moves `active` off the profile it names, because a boot reading
+ the file would otherwise be left on a profile that is gone, or has nothing left to route by. When the profile stays
+ whole for every boot reading the file, the `active` is left as it is, and so it is when moving it would change
+ the profile another boot starts on."""
+
+ @property
+ def description(self) -> str:
+ """The finding as one sentence, for a report a person reads."""
+ where = f"'{self.file_path}'"
+ match self.kind:
+ case FormerReleaseFindingKind.RETIRED_BACKEND_TABLE:
+ state = "enables" if self.blocks_boot else "declares"
+ return f"{where} {state} the '{self.subject}' backend, which this release no longer has"
+ case FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY:
+ state = "enabled " if self.blocks_boot else ""
+ return (
+ f"the {state}'{self.subject}' backend in {where} still names '{MODEL_SPECS_SECTION_KEY}', "
+ "whose model specs are no longer downloaded"
+ )
+ case FormerReleaseFindingKind.RETIRED_BACKEND_FILE:
+ return f"{where} holds the model settings of the '{self.subject}' backend, which this release no longer has"
+ case FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE:
+ return f"{where} lists the models the Pipelex Gateway served"
+ case FormerReleaseFindingKind.SERVICE_FILE:
+ return f"{where} recorded the Pipelex Gateway's terms acceptance, and nothing reads it any more"
+ case FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE:
+ return f"{where} defines the routing profile '{self.subject}', which sends models to '{self.retired_backend}'"
+ case FormerReleaseFindingKind.RETIRED_PROFILE_DEFAULT:
+ return f"the routing profile '{self.subject}' in {where} sets its default backend to '{self.retired_backend}'"
+ case FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND:
+ return f"the routing profile '{self.subject}' in {where} routes '{self.route_pattern}' to '{self.retired_backend}'"
+ case FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE:
+ if self.route_pattern is not None:
+ return (
+ f"{where} makes '{self.subject}' the active routing profile, and it routes '{self.route_pattern}' to '{self.retired_backend}'"
+ )
+ target = f"'{self.retired_backend}'" if self.retired_backend else "a backend this release no longer has"
+ return f"{where} makes '{self.subject}' the active routing profile, and it sends models to {target}"
+ case FormerReleaseFindingKind.PROJECT_ACTIVE_ROUTING_PROFILE:
+ return (
+ f"the project's boot is to start on the routing profile '{self.subject}', and the file it reads 'active' from is "
+ f"read by the boot outside the project too: {where} is given an 'active' of its own"
+ )
+
+
+class FormerReleaseFindings(BaseModel):
+ """Everything a former release left in one configuration directory."""
+
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ config_dir: Path
+ findings: list[FormerReleaseFinding] = Field(default_factory=list[FormerReleaseFinding])
+ hand_edits: dict[Path, str] = Field(default_factory=dict[Path, str])
+ """The files here the cleanup leaves as they are, each with what to change by hand and why: a file both boots
+ read, whose `active` would have to change for one boot and stay for the other."""
+
+ @property
+ def is_clean(self) -> bool:
+ return not self.findings
+
+ @property
+ def blocks_boot(self) -> bool:
+ """Whether a finding here stops one of the sequences it was read in.
+
+ Never a machine's verdict: whether its boot is refused is `former_release_boot_blockers` over the files that
+ boot merges, which a project's own base or the other directory's override can change both ways.
+ """
+ return any(finding.blocks_boot for finding in self.findings)
+
+ @property
+ def file_paths(self) -> list[Path]:
+ """The files the findings are in, each once, in the order they were found."""
+ seen: dict[Path, None] = {}
+ for finding in self.findings:
+ seen.setdefault(finding.file_path, None)
+ return list(seen)
+
+
+def backend_library_paths_in(*, config_dir: Path) -> list[Path]:
+ """One directory's `backends.toml` and its personal override, in merge order."""
+ inference_dir = config_dir / INFERENCE_DIR_NAME
+ return [inference_dir / BACKENDS_FILE_NAME, inference_dir / BACKENDS_OVERRIDE_FILE_NAME]
+
+
+def routing_profile_library_paths_in(*, config_dir: Path) -> list[Path]:
+ """One directory's `routing_profiles.toml` and its personal override, in merge order."""
+ inference_dir = config_dir / INFERENCE_DIR_NAME
+ return [inference_dir / ROUTING_PROFILES_FILE_NAME, inference_dir / ROUTING_PROFILES_OVERRIDE_FILE_NAME]
+
+
+def distinct_config_dirs(*, config_dirs: Sequence[Path]) -> list[Path]:
+ """The directories, each once, in the order given: one named twice, or through a link, is read once."""
+ seen: set[Path] = set()
+ directories: list[Path] = []
+ for config_dir in config_dirs:
+ resolved = config_dir.resolve()
+ if resolved in seen:
+ continue
+ seen.add(resolved)
+ directories.append(config_dir)
+ return directories
+
+
+def inference_merge_sequences(*, config_dirs: Sequence[Path], file_name: str, override_file_name: str) -> list[list[Path]]:
+ """The merge sequences a boot reads one inference document from, over the home and a project directory.
+
+ `config_dirs` is in tier order, as `ConfigLoader.existing_config_dirs` gives it: the home configuration directory,
+ then the project's. A boot started outside the project reads the home's base and override; one started in it reads
+ the project's base when the project has one and the home's otherwise, then the home's override, then the project's.
+ That is `ConfigLoader._inference_file_paths`, which a test holds this derivation to. Over one directory, the one
+ sequence is its base and its override.
+
+ Args:
+ config_dirs: The home directory, then the project's; a directory named twice counts once.
+ file_name: The base file's name, `backends.toml` or `routing_profiles.toml`.
+ override_file_name: Its override's name.
+
+ Returns:
+ One sequence per kind of boot, each base first: the home's alone, then the project's when there is one.
+ """
+ directories = distinct_config_dirs(config_dirs=config_dirs)
+ if not directories:
+ return []
+ if len(directories) > 2:
+ msg = f"a boot reads the home and one project directory, and was handed {len(directories)} directories — caller bug"
+ raise PipelexUnexpectedError(msg)
+ home_dir = directories[0]
+ home_sequence = [home_dir / INFERENCE_DIR_NAME / file_name, home_dir / INFERENCE_DIR_NAME / override_file_name]
+ if len(directories) == 1:
+ return [home_sequence]
+ project_inference_dir = directories[1] / INFERENCE_DIR_NAME
+ project_base = project_inference_dir / file_name
+ base = project_base if project_base.exists() else home_sequence[0]
+ return [home_sequence, [base, home_sequence[1], project_inference_dir / override_file_name]]
+
+
+def detect_former_release(*, config_dir: Path) -> FormerReleaseFindings:
+ """Find what a former release left in one configuration directory standing alone.
+
+ The reading of a machine with that one directory; `detect_former_release_across` reads the home and a project
+ together, as their boot does, and is what the cleanup and the commands ask.
+ """
+ return detect_former_release_across(config_dirs=[config_dir])[0]
+
+
+def detect_former_release_across(*, config_dirs: Sequence[Path]) -> list[FormerReleaseFindings]:
+ """Find what a former release left in the home and project configuration directories, read as their boots read them.
+
+ A pure read: nothing is written, and a directory or a file that is not there is simply clean. A document that does
+ not parse is skipped rather than raised, because whether it parses is for the boot and the doctor to report, and
+ the rest can still be judged.
+
+ Args:
+ config_dirs: The home directory, then the project's, as `ConfigLoader.existing_config_dirs` lists them.
+
+ Returns:
+ The findings of each distinct directory, clean ones included, in the order given; each in a stable order:
+ the backend library, the files beside the backends, the service file, then the routing profile library.
+ """
+ directories = distinct_config_dirs(config_dirs=config_dirs)
+ backend_findings = backend_library_findings(
+ sequences=inference_merge_sequences(config_dirs=directories, file_name=BACKENDS_FILE_NAME, override_file_name=BACKENDS_OVERRIDE_FILE_NAME)
+ )
+ routing_plan = _routing_plan(
+ sequences=inference_merge_sequences(
+ config_dirs=directories, file_name=ROUTING_PROFILES_FILE_NAME, override_file_name=ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ ),
+ contents=None,
+ )
+ all_findings: list[FormerReleaseFindings] = []
+ for config_dir in directories:
+ backend_files = set(backend_library_paths_in(config_dir=config_dir))
+ routing_files = set(routing_profile_library_paths_in(config_dir=config_dir))
+ findings = [finding for finding in backend_findings if finding.file_path in backend_files]
+ findings.extend(_backend_directory_findings(backends_dir=config_dir / INFERENCE_DIR_NAME / BACKENDS_DIR_NAME))
+ service_file = config_dir / RETIRED_SERVICE_FILE_NAME
+ if service_file.is_file():
+ findings.append(FormerReleaseFinding(kind=FormerReleaseFindingKind.SERVICE_FILE, file_path=service_file))
+ findings.extend(finding for finding in routing_plan.findings if finding.file_path in routing_files)
+ hand_edits = {path: hand_edit for path, hand_edit in routing_plan.hand_edits.items() if path in routing_files}
+ all_findings.append(FormerReleaseFindings(config_dir=config_dir, findings=findings, hand_edits=hand_edits))
+ return all_findings
+
+
+def former_release_boot_blockers(
+ *,
+ backends_library_paths: Sequence[Path],
+ routing_profile_library_paths: Sequence[Path],
+ contents: Mapping[Path, str | None] | None = None,
+) -> list[FormerReleaseFinding]:
+ """What a former release left that would stop a boot reading exactly these files.
+
+ The boot merges a base file with the overrides of the home and the project directories, so a base in one directory
+ can be lifted by an override in the other; reading the same sequences the boot reads is what keeps this from
+ refusing a boot that would have succeeded, or passing one that will not. It reads, and plans nothing: the boot
+ runs it on every start.
+
+ Args:
+ backends_library_paths: The `backends.toml` merge sequence the boot reads, base first.
+ routing_profile_library_paths: The `routing_profiles.toml` merge sequence the boot reads, base first.
+ contents: Documents to read in place of the files at their paths, `None` for a file read as absent: the
+ files as a cleanup would leave them, for a check before it writes.
+
+ Returns:
+ The blocking findings, the backend library's first; empty when nothing a former release left stops the boot.
+ """
+ backend_findings = backend_library_findings(sequences=[list(backends_library_paths)], contents=contents)
+ routing_sequences = [list(routing_profile_library_paths)]
+ routing_documents = _read_documents(sequences=routing_sequences, contents=contents)
+ verdicts = _active_verdicts(sequences=_read_sequences(sequences=routing_sequences, documents=routing_documents), documents=routing_documents)
+ return [finding for finding in [*backend_findings, *verdicts.values()] if finding.blocks_boot]
+
+
+def kit_routing_profile_library_path() -> Path:
+ """The kit's `routing_profiles.toml`, the one `pipelex init` copies."""
+ return Path(str(get_kit_configs_dir())) / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME
+
+
+def kit_default_routing_profile_name() -> str:
+ """The routing profile the kit makes active, which the cleanup moves a retired active profile to.
+
+ Read from the kit rather than spelled here, so the cleanup lands on whatever `pipelex init` would set up.
+ """
+ active = load_toml_from_path(kit_routing_profile_library_path()).get(ACTIVE_KEY)
+ if not isinstance(active, str):
+ msg = f"the kit's routing profile library '{kit_routing_profile_library_path()}' names no active profile — packaging bug"
+ raise PipelexUnexpectedError(msg)
+ return active
+
+
+def describe_former_release_boot_refusal(*, blockers: Sequence[FormerReleaseFinding]) -> str:
+ """The message of the one error a boot raises on what a former release left: what stops it, and the remedies.
+
+ Args:
+ blockers: The findings that stop the boot, as `former_release_boot_blockers` returns them.
+ """
+ lines = [
+ (
+ "This configuration was set up by a former Pipelex release, which ran models through the Pipelex Gateway. "
+ "This release no longer has it, so it cannot start on these files:"
+ ),
+ *(f"- {finding.description}" for finding in blockers),
+ "",
+ (
+ f"Run `{MIGRATE_COMMAND}` to clean it up. It removes what that release left in the home configuration directory and "
+ f"in the project's `.pipelex/`, keeps a copy of each file it changes or removes, and moves the active routing profile to "
+ f"'{kit_default_routing_profile_name()}', which sends each model to the first enabled backend that serves it."
+ ),
+ (
+ f"Or run `{INIT_COMMAND}`, which offers the same cleanup and then sets Pipelex up again: on the hosted Pipelex API "
+ "with a Pipelex API key, or on this machine with your own provider keys."
+ ),
+ ]
+ return "\n".join(lines)
+
+
+def backend_library_findings(*, sequences: Sequence[Sequence[Path]], contents: Mapping[Path, str | None] | None = None) -> list[FormerReleaseFinding]:
+ """The retired backend tables and `model_specs_section` keys in the `backends.toml` merge sequences.
+
+ Each file holding one is reported once; whether it stops the boot is read off the merged document of each sequence
+ the file is read in, so an override that disables a table lifts the finding in the base too. `contents` stands in
+ for the files at its paths, `None` for a file read as absent.
+ """
+ documents = _read_documents(sequences=sequences, contents=contents)
+ read_sequences = _read_sequences(sequences=sequences, documents=documents)
+ merged_documents = [(set(sequence), _merged(documents=[documents[path] for path in sequence])) for sequence in read_sequences]
+ findings: list[FormerReleaseFinding] = []
+ for path, document in documents.items():
+ for backend_name, backend_table in document.items():
+ if not isinstance(backend_table, dict):
+ continue
+ if backend_name in RETIRED_BACKEND_NAMES:
+ kind = FormerReleaseFindingKind.RETIRED_BACKEND_TABLE
+ elif MODEL_SPECS_SECTION_KEY in backend_table and backend_name != MANIFOLD_BACKEND_NAME:
+ kind = FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY
+ else:
+ continue
+ blocks_boot = any(_is_enabled(document=merged, backend_name=backend_name) for members, merged in merged_documents if path in members)
+ findings.append(FormerReleaseFinding(kind=kind, file_path=path, subject=backend_name, blocks_boot=blocks_boot))
+ return findings
+
+
+def retired_routes_of(*, profile: dict[str, Any]) -> list[tuple[str, str, str]]:
+ """The `(route table, pattern, backend)` routes and optional routes of one profile that name a retired backend, in file order."""
+ retired: list[tuple[str, str, str]] = []
+ for route_table_key in ROUTE_TABLE_KEYS:
+ route_table = profile.get(route_table_key)
+ if not isinstance(route_table, dict):
+ continue
+ for route_pattern, backend_name in cast("dict[str, Any]", route_table).items():
+ if isinstance(backend_name, str) and backend_name in RETIRED_BACKEND_NAMES:
+ retired.append((route_table_key, route_pattern, backend_name))
+ return retired
+
+
+class RoutingBoot(NamedTuple):
+ """One boot's routing profile library, judged as the boot would load it."""
+
+ loads: bool
+ """The library validates and defines the profile it makes active."""
+
+ active: str | None
+ """The profile `active` names, whether or not the library defines it."""
+
+ routes: bool
+ """The active profile routes something: a `default`, a route, or a `fallback_order`."""
+
+ refused: bool
+ """What a former release left stops it: the active profile sends models to a retired backend by its default or a
+ route, or it is a retired profile no file defines."""
+
+ @property
+ def starts(self) -> bool:
+ """The boot gets past its routing profile library."""
+ return self.loads and not self.refused
+
+ @property
+ def is_whole(self) -> bool:
+ """The boot starts, and its active profile sends a model somewhere."""
+ return self.starts and self.routes
+
+ def regresses_to(self, *, after: "RoutingBoot") -> bool:
+ """Whether a boot that started on this library would start differently on `after`: not at all, on another
+ active profile, or on one that no longer routes anything.
+ """
+ if not self.starts:
+ return False
+ return not after.starts or after.active != self.active or (self.routes and not after.routes)
+
+
+def routing_boots(*, sequences: Sequence[Sequence[Path]], contents: Mapping[Path, str | None] | None = None) -> list[RoutingBoot]:
+ """Each boot's routing profile library judged as the boot would load it: the files at the paths, or `contents`.
+
+ A file that is not there, or does not parse, is read as absent, as the cleanup's planning reads it.
+ """
+ documents = _read_documents(sequences=sequences, contents=contents)
+ return [_boot_of(documents=[documents[path] for path in sequence]) for sequence in _read_sequences(sequences=sequences, documents=documents)]
+
+
+class _ProfileFate(StrEnum):
+ """What the cleanup does to one profile table in one file."""
+
+ KEPT = "kept"
+ LOSES_ITS_DEFAULT = "loses_its_default"
+ REMOVED = "removed"
+
+ @property
+ def preservation(self) -> int:
+ """How much of the table stays: the order in which a file read by two boots takes the fate either gives it."""
+ match self:
+ case _ProfileFate.REMOVED:
+ return 0
+ case _ProfileFate.LOSES_ITS_DEFAULT:
+ return 1
+ case _ProfileFate.KEPT:
+ return 2
+
+
+_Fates = dict[tuple[Path, str], tuple[_ProfileFate, str | None]]
+
+#: The boots a machine has, in the order `inference_merge_sequences` lists their sequences.
+_BOOT_NAMES = ("the boot outside the project", "the project's boot")
+
+
+class _Libraries(NamedTuple):
+ """The routing profile documents every boot of the machine reads, and what each boot made of them before."""
+
+ sequences: list[list[Path]]
+ documents: dict[Path, dict[str, Any]]
+ before: list[RoutingBoot]
+ verdicts: dict[Path, FormerReleaseFinding]
+ """Each file whose `active` names a profile a former release left, as `_active_verdicts` reads it."""
+
+ project_override: Path | None
+ """The project's override when it exists: the one file only the project's boot reads after the files both boots
+ read, where the project is given an `active` of its own."""
+
+ project_override_path: Path | None
+ """Where the project's override is, or would be: the file a hand edit gives the project its own `active` in."""
+
+ @property
+ def shared(self) -> frozenset[Path]:
+ """The files more than one boot reads."""
+ if len(self.sequences) < 2:
+ return frozenset()
+ return frozenset(self.sequences[0]) & frozenset(self.sequences[1])
+
+
+class _ActivePlan(NamedTuple):
+ """What the cleanup does to the `active` keys: the files it moves off their profile, and the project override it pins."""
+
+ moves: frozenset[Path]
+ pin: tuple[Path, str] | None
+
+ def edit_of(self, *, path: Path) -> str:
+ if path in self.moves:
+ return "move"
+ if self.pin is not None and self.pin[0] == path:
+ return f"pin {self.pin[1]}"
+ return "keep"
+
+
+class _Candidate(NamedTuple):
+ """One way of moving the `active` keys, with each boot's library as it would leave it."""
+
+ plan: _ActivePlan
+ boots: list[RoutingBoot]
+ regressions: list[int]
+ score: tuple[int, int, int, int]
+
+
+class _RoutingPlan(NamedTuple):
+ findings: list[FormerReleaseFinding]
+ hand_edits: dict[Path, str]
+
+
+def _routing_plan(*, sequences: Sequence[Sequence[Path]], contents: Mapping[Path, str | None] | None) -> _RoutingPlan:
+ """What the cleanup does to the routing profile files, decided per boot and checked before anything is written.
+
+ Each profile table's fate is decided in each sequence that reads its file, and a file two boots read keeps what
+ either needs (`_profile_fates`). Then every way of moving the `active` keys a former release spoiled is tried on
+ each boot's library as the cleanup would leave it (`_choose_actives`): none may stop a boot that starts today, or
+ change the profile it starts on; of the rest, the one that leaves the most boots whole wins. Where a file both boots
+ read would have to change for one boot and stay for the other, and the project's override cannot make up the
+ difference, that file is left as it is, for a hand edit, and the plan is made again without touching it.
+ """
+ documents = _read_documents(sequences=sequences, contents=contents)
+ read_sequences = _read_sequences(sequences=sequences, documents=documents)
+ project_override_path = sequences[1][-1] if len(sequences) > 1 else None
+ project_override = (
+ project_override_path
+ if project_override_path is not None and project_override_path in documents and project_override_path not in read_sequences[0]
+ else None
+ )
+ libraries = _Libraries(
+ sequences=read_sequences,
+ documents=documents,
+ before=[_boot_of(documents=[documents[path] for path in sequence]) for sequence in read_sequences],
+ verdicts=_active_verdicts(sequences=read_sequences, documents=documents),
+ project_override=project_override,
+ project_override_path=project_override_path,
+ )
+ first_fates = _profile_fates(libraries=libraries, frozen={})
+ fates = first_fates
+ frozen: dict[Path, str] = {}
+ while True:
+ plan, conflicts = _choose_actives(libraries=libraries, fates=fates, frozen=frozen)
+ if not conflicts:
+ break
+ frozen.update(conflicts)
+ fates = _profile_fates(libraries=libraries, frozen=frozen)
+ findings = _plan_findings(libraries=libraries, fates=fates, first_fates=first_fates, frozen=frozen, plan=plan)
+ return _RoutingPlan(findings=findings, hand_edits=frozen)
+
+
+def _plan_findings(
+ *, libraries: _Libraries, fates: _Fates, first_fates: _Fates, frozen: Mapping[Path, str], plan: _ActivePlan
+) -> list[FormerReleaseFinding]:
+ """The findings the plan makes: each `active`, the project's own one, then each profile table, file by file.
+
+ A file left for a hand edit is reported with what the release left in it, as the plan would have removed it had
+ the file been free; the cleanup then leaves it as it is, with the reason.
+ """
+ findings: list[FormerReleaseFinding] = []
+ for path, document in libraries.documents.items():
+ verdict = libraries.verdicts.get(path)
+ active = document.get(ACTIVE_KEY)
+ if verdict is not None:
+ findings.append(verdict.model_copy(update={"profile_is_removed": path in plan.moves}))
+ elif isinstance(active, str) and (path in plan.moves or path in frozen):
+ findings.append(
+ FormerReleaseFinding(
+ kind=FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE, file_path=path, subject=active, profile_is_removed=path in plan.moves
+ )
+ )
+ if plan.pin is not None and plan.pin[0] == path:
+ findings.append(FormerReleaseFinding(kind=FormerReleaseFindingKind.PROJECT_ACTIVE_ROUTING_PROFILE, file_path=path, subject=plan.pin[1]))
+ for path, document in libraries.documents.items():
+ file_fates = first_fates if path in frozen else fates
+ for profile_name, profile in _profiles_of(document=document).items():
+ fate, retired_backend = file_fates[path, profile_name]
+ if fate is _ProfileFate.REMOVED:
+ findings.append(
+ FormerReleaseFinding(
+ kind=FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE, file_path=path, subject=profile_name, retired_backend=retired_backend
+ )
+ )
+ continue
+ if fate is _ProfileFate.LOSES_ITS_DEFAULT:
+ findings.append(
+ FormerReleaseFinding(
+ kind=FormerReleaseFindingKind.RETIRED_PROFILE_DEFAULT, file_path=path, subject=profile_name, retired_backend=retired_backend
+ )
+ )
+ for route_table, route_pattern, backend_name in retired_routes_of(profile=profile):
+ findings.append(
+ FormerReleaseFinding(
+ kind=FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND,
+ file_path=path,
+ subject=profile_name,
+ route_pattern=route_pattern,
+ route_table=route_table,
+ retired_backend=backend_name,
+ )
+ )
+ return findings
+
+
+def _profile_fates(*, libraries: _Libraries, frozen: Mapping[Path, str]) -> _Fates:
+ """What becomes of each profile table of each file, with the retired backend that decides it.
+
+ Decided in each sequence that reads the file (`_fate_in_sequence`); a file read by both boots takes the fate that
+ keeps the most of the table, since a profile one boot still needs cannot go from a file it reads, and a profile left
+ in a boot that does not need it, without a retired default, is harmless there. A file left for a hand edit keeps
+ every table. Then no boot may be left with a part of a profile and none of its description, which the boot
+ refuses: the table that held it, which a sequence removed while a shared file kept another part, keeps it and loses
+ only its retired `default` (`_descriptions_lost`), and the fates are decided again.
+ """
+ forced: set[tuple[Path, str]] = set()
+ while True:
+ fates = _fates_once(libraries=libraries, frozen=frozen, forced=forced)
+ lost = _descriptions_lost(libraries=libraries, fates=fates) - forced
+ if not lost:
+ return fates
+ forced |= lost
+
+
+def _fates_once(*, libraries: _Libraries, frozen: Mapping[Path, str], forced: set[tuple[Path, str]]) -> _Fates:
+ fates: _Fates = {}
+ for path in _in_reading_order(sequences=libraries.sequences):
+ for profile_name, profile in _profiles_of(document=libraries.documents[path]).items():
+ if path in frozen:
+ fates[path, profile_name] = (_ProfileFate.KEPT, None)
+ continue
+ readings = [
+ _fate_in_sequence(
+ path=path, profile_name=profile_name, profile=profile, sequence=sequence, documents=libraries.documents, fates=fates
+ )
+ for sequence in libraries.sequences
+ if path in sequence
+ ]
+ fate, retired_backend = max(readings, key=lambda reading: reading[0].preservation)
+ if fate is _ProfileFate.REMOVED and (path, profile_name) in forced:
+ fate = _ProfileFate.LOSES_ITS_DEFAULT
+ fates[path, profile_name] = (fate, retired_backend)
+ return fates
+
+
+def _fate_in_sequence(
+ *, path: Path, profile_name: str, profile: dict[str, Any], sequence: list[Path], documents: dict[Path, dict[str, Any]], fates: _Fates
+) -> tuple[_ProfileFate, str | None]:
+ """What one sequence asks of one profile table, the files it reads before this one being decided already.
+
+ A table whose own `default` is a retired backend goes, unless the profile stays alive without it: a file read
+ before this one keeps a definition of it, or a file read after gives it a live `default`. Then only the `default`
+ goes, and the routes the user tuned stay. A table with no `default` of its own and no `description`, whose every
+ earlier definition goes, is a part of a profile that no longer exists, and goes too: left alone, it would be a
+ profile with no description, which the boot refuses.
+ """
+ index = sequence.index(path)
+ earlier = [neighbour for neighbour in sequence[:index] if profile_name in _profiles_of(document=documents[neighbour])]
+ later = [neighbour for neighbour in sequence[index + 1 :] if profile_name in _profiles_of(document=documents[neighbour])]
+ kept_earlier = any(fates[neighbour, profile_name][0] is not _ProfileFate.REMOVED for neighbour in earlier)
+ own_default = profile.get(_DEFAULT_KEY)
+ if _names_a_retired_backend(value=own_default):
+ later_live_default = any(
+ _names_a_live_backend(value=_profiles_of(document=documents[neighbour])[profile_name].get(_DEFAULT_KEY)) for neighbour in later
+ )
+ fate = _ProfileFate.LOSES_ITS_DEFAULT if kept_earlier or later_live_default else _ProfileFate.REMOVED
+ return fate, cast("str", own_default)
+ if earlier and not kept_earlier and own_default is None and _DESCRIPTION_KEY not in profile:
+ return _ProfileFate.REMOVED, fates[earlier[0], profile_name][1]
+ return _ProfileFate.KEPT, None
+
+
+def _descriptions_lost(*, libraries: _Libraries, fates: _Fates) -> set[tuple[Path, str]]:
+ """The tables to keep for their description: in some sequence, a profile left by a part with no description of its own.
+
+ That happens when a file two boots read keeps a part of the profile for one boot, while the other boot's own
+ definition of it goes. The definition holding the description then stays, losing only its retired `default`: the
+ one in a file only that boot reads, the last read, when there is one.
+ """
+ shared = libraries.shared
+ lost: set[tuple[Path, str]] = set()
+ for sequence in libraries.sequences:
+ names = dict.fromkeys(name for path in sequence for name in _profiles_of(document=libraries.documents[path]))
+ for profile_name in names:
+ tables = [
+ (path, _profiles_of(document=libraries.documents[path])[profile_name])
+ for path in sequence
+ if profile_name in _profiles_of(document=libraries.documents[path])
+ ]
+ staying = [table for path, table in tables if _table_stays(profile=table, fate=fates[path, profile_name][0])]
+ if not staying or any(_DESCRIPTION_KEY in table for table in staying):
+ continue
+ holders = [path for path, table in tables if _DESCRIPTION_KEY in table]
+ if holders:
+ unshared = [path for path in holders if path not in shared]
+ lost.add(((unshared or holders)[-1], profile_name))
+ return lost
+
+
+def _table_stays(*, profile: dict[str, Any], fate: _ProfileFate) -> bool:
+ """Whether a table is still in its file once cleaned: one that loses its `default` and holds nothing else goes."""
+ match fate:
+ case _ProfileFate.KEPT:
+ return True
+ case _ProfileFate.LOSES_ITS_DEFAULT:
+ return any(key != _DEFAULT_KEY for key in profile)
+ case _ProfileFate.REMOVED:
+ return False
+
+
+def _choose_actives(*, libraries: _Libraries, fates: _Fates, frozen: Mapping[Path, str]) -> tuple[_ActivePlan, dict[Path, str]]:
+ """The `active` keys to move, and the files to leave for a hand edit because no move suits both boots.
+
+ A file whose `active` names a profile the release left, or one the cleanup removes or leaves with nothing to route
+ by, may move off it: a base to the profile the kit makes active, an override by losing the key so the file read
+ before it decides. The project's override may be given an `active` of its own, when the project's boot reads one
+ the cleanup has to change, or one that names what a former release left. Every combination is tried on each boot's
+ library as it would be left, and the plans that would stop a boot that starts today, or change the profile it
+ starts on, are out. Of the rest, the one leaving the most boots whole wins, then the one leaving no `active` naming
+ a profile that is gone, then the one without the project's own `active`, then the one moving the fewest.
+
+ A boot the former release stops that the chosen plan leaves stopped, while another plan would have started it, is a
+ conflict: the files both boots read where the two plans differ are returned for a hand edit.
+ """
+ no_change = _ActivePlan(moves=frozenset(), pin=None)
+ after_fates = _documents_after(libraries=libraries, fates=fates, frozen=frozen, plan=no_change, kit=None)
+ setters = {path: document[ACTIVE_KEY] for path, document in libraries.documents.items() if isinstance(document.get(ACTIVE_KEY), str)}
+ movable = [
+ path
+ for path, active in setters.items()
+ if path not in frozen
+ and (
+ _is_implicated(libraries=libraries, path=path, active=active, after_fates=after_fates)
+ or _is_exposed(libraries=libraries, path=path, active=active, after_fates=after_fates)
+ )
+ ]
+ kit = _kit_default_profile() if movable else None
+ candidates = [
+ _evaluate(libraries=libraries, fates=fates, frozen=frozen, plan=plan, kit=kit, setters=setters)
+ for plan in _possible_plans(libraries=libraries, setters=setters, movable=movable, kit=kit)
+ ]
+ acceptable = [candidate for candidate in candidates if not candidate.regressions]
+ if not acceptable:
+ return no_change, _untouchable(libraries=libraries, candidate=candidates[0], after_fates=after_fates, frozen=frozen)
+ best = max(acceptable, key=lambda candidate: candidate.score)
+ conflicts: dict[Path, str] = {}
+ for index, sequence in enumerate(libraries.sequences):
+ if best.boots[index].is_whole or not libraries.before[index].refused:
+ continue
+ fixing = [candidate for candidate in candidates if candidate.boots[index].is_whole]
+ if not fixing:
+ continue
+ fix = max(fixing, key=lambda candidate: candidate.score)
+ for path in sequence:
+ if path in libraries.shared and fix.plan.edit_of(path=path) != best.plan.edit_of(path=path):
+ conflicts[path] = _conflict_hand_edit(libraries=libraries, path=path, stopped=index)
+ return best.plan, conflicts
+
+
+def _possible_plans(
+ *, libraries: _Libraries, setters: Mapping[Path, str], movable: list[Path], kit: tuple[str, dict[str, Any]] | None
+) -> list[_ActivePlan]:
+ """Every combination of moves, each with and without the project's own `active`, fewest moves first."""
+ pin_values = _pin_values(libraries=libraries, setters=setters, kit=kit)
+ project_sequence = libraries.sequences[1] if len(libraries.sequences) > 1 else []
+ project_setters = [path for path in project_sequence if path in setters]
+ project_reads_a_spoiled_active = bool(project_setters) and project_setters[-1] in movable
+ plans: list[_ActivePlan] = []
+ for size in range(len(movable) + 1):
+ for moves in itertools.combinations(movable, size):
+ plans.append(_ActivePlan(moves=frozenset(moves), pin=None))
+ if libraries.project_override is None or libraries.project_override in moves:
+ continue
+ moves_what_the_project_reads = any(path in libraries.shared and path in project_sequence for path in moves)
+ if not (project_reads_a_spoiled_active or moves_what_the_project_reads):
+ continue
+ plans.extend(_ActivePlan(moves=frozenset(moves), pin=(libraries.project_override, value)) for value in pin_values)
+ return plans
+
+
+def _pin_values(*, libraries: _Libraries, setters: Mapping[Path, str], kit: tuple[str, dict[str, Any]] | None) -> list[str]:
+ """What the project's own `active` may name: what its boot starts on today, what a file it reads names, the kit's."""
+ if libraries.project_override is None:
+ return []
+ values = [libraries.before[1].active, *(setters[path] for path in libraries.sequences[1] if path in setters)]
+ if kit is not None:
+ values.append(kit[0])
+ current = setters.get(libraries.project_override)
+ return [value for value in dict.fromkeys(values) if value is not None and value != current]
+
+
+def _evaluate(
+ *,
+ libraries: _Libraries,
+ fates: _Fates,
+ frozen: Mapping[Path, str],
+ plan: _ActivePlan,
+ kit: tuple[str, dict[str, Any]] | None,
+ setters: Mapping[Path, str],
+) -> _Candidate:
+ after = _documents_after(libraries=libraries, fates=fates, frozen=frozen, plan=plan, kit=kit)
+ boots = [_boot_of(documents=[after[path] for path in sequence]) for sequence in libraries.sequences]
+ regressions = [index for index, (before, boot) in enumerate(zip(libraries.before, boots, strict=True)) if before.regresses_to(after=boot)]
+ dangling = sum(1 for path in setters if _names_a_profile_gone_everywhere(libraries=libraries, path=path, after=after))
+ score = (sum(1 for boot in boots if boot.is_whole), -dangling, -int(plan.pin is not None), -len(plan.moves))
+ return _Candidate(plan=plan, boots=boots, regressions=regressions, score=score)
+
+
+def _names_a_profile_gone_everywhere(*, libraries: _Libraries, path: Path, after: dict[Path, dict[str, Any]]) -> bool:
+ """Whether a file's `active`, as the plan leaves it, names a profile no boot reading the file defines any more."""
+ active = after[path].get(ACTIVE_KEY)
+ if not isinstance(active, str):
+ return False
+ return all(
+ active not in _profiles_of(document=_merged(documents=[after[member] for member in sequence]))
+ for sequence in libraries.sequences
+ if path in sequence
+ )
+
+
+def _is_implicated(*, libraries: _Libraries, path: Path, active: str, after_fates: dict[Path, dict[str, Any]]) -> bool:
+ """Whether a file's `active` is the cleanup's to move: it names what a former release left, or a profile the cleanup
+ removes, or leaves with nothing to route by, in a boot that reads it.
+ """
+ if path in libraries.verdicts:
+ return True
+ for sequence in libraries.sequences:
+ if path not in sequence:
+ continue
+ profiles_before = _profiles_of(document=_merged(documents=[libraries.documents[member] for member in sequence]))
+ profiles_after = _profiles_of(document=_merged(documents=[after_fates[member] for member in sequence]))
+ if active not in profiles_before:
+ continue
+ if active not in profiles_after or (
+ _routes_something(profile=profiles_before[active]) and not _routes_something(profile=profiles_after[active])
+ ):
+ return True
+ return False
+
+
+def _is_exposed(*, libraries: _Libraries, path: Path, active: str, after_fates: dict[Path, dict[str, Any]]) -> bool:
+ """Whether a file's `active` names a profile no file defines, or one with nothing to route by, in a boot the former
+ release stops.
+
+ Such an `active` is shadowed today by a later one naming what the release left, or is what the boot reads once the
+ release's routes go; once the later one moves, it is the one the boot reads, so it may move too, and a profile with
+ nothing to route by counts as gone for that boot. In a boot the release does not stop, such an `active` is the
+ user's own, and stays for them to see.
+ """
+ for index, sequence in enumerate(libraries.sequences):
+ if path not in sequence or not libraries.before[index].refused:
+ continue
+ profiles = _profiles_of(document=_merged(documents=[after_fates[member] for member in sequence]))
+ if active not in profiles or not _routes_something(profile=profiles[active]):
+ return True
+ return False
+
+
+def _untouchable(
+ *, libraries: _Libraries, candidate: _Candidate, after_fates: dict[Path, dict[str, Any]], frozen: Mapping[Path, str]
+) -> dict[Path, str]:
+ """The files to leave as they are when even keeping every `active` would stop a boot that starts today: the ones the
+ cleanup would change in that boot's sequence.
+ """
+ untouchable: dict[Path, str] = {}
+ for index in candidate.regressions:
+ for path in libraries.sequences[index]:
+ if path not in frozen and after_fates[path] != libraries.documents[path]:
+ untouchable[path] = (
+ f"cleaning this file would stop {_BOOT_NAMES[index] if len(libraries.sequences) > 1 else 'the boot'} from starting as it does "
+ f"now. By hand, remove what a former release left in it, keeping each routing profile you use whole, then run "
+ f"'{MIGRATE_COMMAND}' again"
+ )
+ return untouchable
+
+
+def _conflict_hand_edit(*, libraries: _Libraries, path: Path, stopped: int) -> str:
+ """Why a file both boots read is left for a hand edit, and the edit that settles it."""
+ active = libraries.documents[path].get(ACTIVE_KEY)
+ started = 1 - stopped
+ return (
+ f"both {_BOOT_NAMES[0]} and {_BOOT_NAMES[1]} read this file, and its 'active', '{active}', cannot change for one without "
+ f"changing for the other: {_BOOT_NAMES[stopped]} cannot run a model with it once the former release's profiles and routes "
+ f"are gone, and {_BOOT_NAMES[started]} would no longer start as it does. By hand, give the project an 'active' of its own "
+ f"in '{libraries.project_override_path}', then make this file's suit {_BOOT_NAMES[0]}, and run '{MIGRATE_COMMAND}' again"
+ )
+
+
+def _documents_after(
+ *, libraries: _Libraries, fates: _Fates, frozen: Mapping[Path, str], plan: _ActivePlan, kit: tuple[str, dict[str, Any]] | None
+) -> dict[Path, dict[str, Any]]:
+ """Each document as the cleanup would leave it: the same edits `former_release_cleanup.cleaned_text` makes, on the parsed document.
+
+ A table that goes, a retired `default` and every retired route; a table left with nothing once its `default` went;
+ an `active` moved off its profile — a base onto the kit's default profile, which is added when the file did not
+ define it, an override by losing the key — and the project's own `active`. A file left for a hand edit stays as it is.
+ """
+ after: dict[Path, dict[str, Any]] = {}
+ for path, document in libraries.documents.items():
+ if path in frozen:
+ after[path] = document
+ continue
+ cleaned = copy.deepcopy(document)
+ profiles = cast("dict[str, Any]", cleaned.get(ROUTING_PROFILES_KEY))
+ for profile_name, profile in _profiles_of(document=document).items():
+ fate = fates[path, profile_name][0]
+ if fate is _ProfileFate.REMOVED:
+ del profiles[profile_name]
+ continue
+ kept = cast("dict[str, Any]", profiles[profile_name])
+ if fate is _ProfileFate.LOSES_ITS_DEFAULT:
+ kept.pop(_DEFAULT_KEY, None)
+ for route_table, route_pattern, _ in retired_routes_of(profile=profile):
+ cast("dict[str, Any]", kept[route_table]).pop(route_pattern, None)
+ if not _table_stays(profile=profile, fate=fate):
+ del profiles[profile_name]
+ if path in plan.moves:
+ if path.name in _OVERRIDE_FILE_NAMES:
+ cleaned.pop(ACTIVE_KEY, None)
+ elif kit is not None:
+ kit_default, kit_profile = kit
+ existing = cleaned.get(ROUTING_PROFILES_KEY)
+ if kit_default not in _profiles_of(document=document) and (existing is None or isinstance(existing, dict)):
+ cleaned[ROUTING_PROFILES_KEY] = {kit_default: copy.deepcopy(kit_profile), **cast("dict[str, Any]", existing or {})}
+ cleaned[ACTIVE_KEY] = kit_default
+ if plan.pin is not None and plan.pin[0] == path:
+ cleaned[ACTIVE_KEY] = plan.pin[1]
+ after[path] = cleaned
+ return after
+
+
+def _kit_default_profile() -> tuple[str, dict[str, Any]]:
+ """The profile the kit makes active, by name and as the kit defines it."""
+ kit_default = kit_default_routing_profile_name()
+ kit_profiles = _profiles_of(document=load_toml_from_path(kit_routing_profile_library_path()))
+ if kit_default not in kit_profiles:
+ msg = (
+ f"the kit's routing profile library '{kit_routing_profile_library_path()}' does not define its active profile "
+ f"'{kit_default}' — packaging bug"
+ )
+ raise PipelexUnexpectedError(msg)
+ return kit_default, kit_profiles[kit_default]
+
+
+def _boot_of(*, documents: Sequence[dict[str, Any]]) -> RoutingBoot:
+ """One sequence's routing profile library, judged as the boot would load it."""
+ merged = _merged(documents=list(documents))
+ active = merged.get(ACTIVE_KEY)
+ active_name = active if isinstance(active, str) else None
+ profiles = _profiles_of(document=merged)
+ refused = active_name is not None and _is_refused(active=active_name, profiles=profiles)
+ try:
+ RoutingProfileLibraryBlueprint.model_validate(merged)
+ except ValidationError:
+ return RoutingBoot(loads=False, active=active_name, routes=False, refused=refused)
+ if active_name is None or active_name not in profiles:
+ return RoutingBoot(loads=False, active=active_name, routes=False, refused=refused)
+ return RoutingBoot(loads=True, active=active_name, routes=_routes_something(profile=profiles[active_name]), refused=refused)
+
+
+def _is_refused(*, active: str, profiles: dict[str, dict[str, Any]]) -> bool:
+ """Whether the boot refuses this active profile for what a former release left in it: the refusal `_active_verdict` names."""
+ if active not in profiles:
+ return active in RETIRED_ROUTING_PROFILE_NAMES
+ profile = profiles[active]
+ return _names_a_retired_backend(value=profile.get(_DEFAULT_KEY)) or any(route[0] == _ROUTES_KEY for route in retired_routes_of(profile=profile))
+
+
+def _routes_something(*, profile: dict[str, Any]) -> bool:
+ """Whether a profile sends any model somewhere: a `default`, a route, or a `fallback_order`. Optional routes alone
+ reach only backends that happen to be enabled, and are not counted.
+ """
+ return bool(profile.get(_DEFAULT_KEY)) or bool(profile.get(_ROUTES_KEY)) or bool(profile.get(_FALLBACK_ORDER_KEY))
+
+
+def _active_verdicts(*, sequences: list[list[Path]], documents: dict[Path, dict[str, Any]]) -> dict[Path, FormerReleaseFinding]:
+ """Each file whose `active` names a profile sending models to a retired backend, or a retired profile no file defines.
+
+ Every such file is reported, because each is something the cleanup may have to move; only the last file of a
+ sequence to set `active` is the one that boot reads, so only its finding can stop the boot. A file read in two
+ sequences is reported once, with what either says. Optional routes are not counted: one naming a disabled backend
+ is inert, and the boot never refuses it. Whether the cleanup moves the `active` is the plan's to say
+ (`profile_is_removed`).
+ """
+ found: dict[Path, FormerReleaseFinding] = {}
+ for sequence in sequences:
+ merged_profiles = _profiles_of(document=_merged(documents=[documents[path] for path in sequence]))
+ setters = [(path, documents[path][ACTIVE_KEY]) for path in sequence if ACTIVE_KEY in documents[path]]
+ for index, (path, active) in enumerate(setters):
+ if not isinstance(active, str):
+ continue
+ finding = _active_verdict(path=path, active=active, is_read_by_the_boot=index == len(setters) - 1, merged_profiles=merged_profiles)
+ if finding is not None:
+ found[path] = _combined(earlier=found.get(path), later=finding)
+ return {path: found[path] for path in documents if path in found}
+
+
+def _active_verdict(*, path: Path, active: str, is_read_by_the_boot: bool, merged_profiles: dict[str, dict[str, Any]]) -> FormerReleaseFinding | None:
+ """One file's `active`, judged over one sequence: whether it names a profile a former release left."""
+ defined = active in merged_profiles
+ merged_default = merged_profiles[active].get(_DEFAULT_KEY) if defined else None
+ default_is_retired = _names_a_retired_backend(value=merged_default)
+ is_a_retired_name = not defined and active in RETIRED_ROUTING_PROFILE_NAMES
+ # Optional routes are left out: the boot refuses a route to a backend it cannot reach, never an optional one.
+ retired_route = (
+ next((route for route in retired_routes_of(profile=merged_profiles[active]) if route[0] == _ROUTES_KEY), None)
+ if defined and not default_is_retired
+ else None
+ )
+ if not (default_is_retired or is_a_retired_name or retired_route):
+ return None
+ retired_backend = cast("str", merged_default) if default_is_retired else (retired_route[2] if retired_route else None)
+ return FormerReleaseFinding(
+ kind=FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE,
+ file_path=path,
+ subject=active,
+ route_pattern=retired_route[1] if retired_route else None,
+ route_table=retired_route[0] if retired_route else None,
+ retired_backend=retired_backend,
+ blocks_boot=is_read_by_the_boot,
+ )
+
+
+def _combined(*, earlier: FormerReleaseFinding | None, later: FormerReleaseFinding) -> FormerReleaseFinding:
+ """One file's `active` judged over two sequences: the stronger reading, a retired default or name over a route."""
+ if earlier is None:
+ return later
+ primary = later if earlier.route_pattern is not None and later.route_pattern is None else earlier
+ return primary.model_copy(update={"blocks_boot": earlier.blocks_boot or later.blocks_boot})
+
+
+def _backend_directory_findings(*, backends_dir: Path) -> list[FormerReleaseFinding]:
+ """The retired backend's per-model file and the Gateway's model lists, beside the backend files."""
+ findings = [
+ FormerReleaseFinding(
+ kind=FormerReleaseFindingKind.RETIRED_BACKEND_FILE, file_path=backends_dir / f"{backend_name}.toml", subject=backend_name
+ )
+ for backend_name in sorted(RETIRED_BACKEND_NAMES)
+ if (backends_dir / f"{backend_name}.toml").is_file()
+ ]
+ if backends_dir.is_dir():
+ findings.extend(
+ FormerReleaseFinding(kind=FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE, file_path=path)
+ for path in sorted(backends_dir.glob(GATEWAY_MODELS_REFERENCE_GLOB))
+ if path.is_file()
+ )
+ return findings
+
+
+def _read_documents(*, sequences: Sequence[Sequence[Path]], contents: Mapping[Path, str | None] | None) -> dict[Path, dict[str, Any]]:
+ """Each file of the sequences that exists and parses, with its document, each once, in the order first met.
+
+ A path `contents` names is read from there instead: its text, or nothing for a file read as absent.
+ """
+ documents: dict[Path, dict[str, Any]] = {}
+ for sequence in sequences:
+ for path in sequence:
+ if path in documents:
+ continue
+ if contents is not None and path in contents:
+ text = contents[path]
+ if text is None:
+ continue
+ try:
+ documents[path] = load_toml_from_content(text)
+ except TomlError:
+ continue
+ continue
+ if not path.is_file():
+ continue
+ try:
+ documents[path] = load_toml_from_path(path)
+ except (OSError, TomlError, UnicodeDecodeError):
+ continue
+ return documents
+
+
+def _read_sequences(*, sequences: Sequence[Sequence[Path]], documents: dict[Path, dict[str, Any]]) -> list[list[Path]]:
+ """The sequences without the files that are not there or did not parse, which the boot reads as nothing."""
+ return [[path for path in sequence if path in documents] for sequence in sequences]
+
+
+def _in_reading_order(*, sequences: list[list[Path]]) -> list[Path]:
+ """Every file of the sequences, each after every file read before it in any of them."""
+ remaining = list(dict.fromkeys(path for sequence in sequences for path in sequence))
+ ordered: list[Path] = []
+ while remaining:
+ ready = next(path for path in remaining if set(_neighbours(path=path, sequences=sequences, before=True)) <= set(ordered))
+ ordered.append(ready)
+ remaining.remove(ready)
+ return ordered
+
+
+def _neighbours(*, path: Path, sequences: list[list[Path]], before: bool) -> list[Path]:
+ """The files read before (or after) this one, in any sequence that reads it, each once."""
+ neighbours: dict[Path, None] = {}
+ for sequence in sequences:
+ if path not in sequence:
+ continue
+ index = sequence.index(path)
+ for neighbour in sequence[:index] if before else sequence[index + 1 :]:
+ neighbours.setdefault(neighbour, None)
+ return list(neighbours)
+
+
+def _merged(*, documents: list[dict[str, Any]]) -> dict[str, Any]:
+ """The documents deep-merged in order, as the loaders merge a base and its overrides.
+
+ Each is copied first: `deep_update` stores the tables of the first document it is given by reference, so merging
+ the originals would let an override rewrite the base document every per-file finding is read from.
+ """
+ merged: dict[str, Any] = {}
+ for document in documents:
+ deep_update(merged, updates=copy.deepcopy(document))
+ return merged
+
+
+def _profiles_of(*, document: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ profiles = document.get(ROUTING_PROFILES_KEY)
+ if not isinstance(profiles, dict):
+ return {}
+ return {
+ profile_name: cast("dict[str, Any]", profile)
+ for profile_name, profile in cast("dict[str, Any]", profiles).items()
+ if isinstance(profile, dict)
+ }
+
+
+def _names_a_retired_backend(*, value: Any) -> bool:
+ return isinstance(value, str) and value in RETIRED_BACKEND_NAMES
+
+
+def _names_a_live_backend(*, value: Any) -> bool:
+ return isinstance(value, str) and value not in RETIRED_BACKEND_NAMES
+
+
+def _is_enabled(*, document: dict[str, Any], backend_name: str) -> bool:
+ """Whether the backend loader would load this table: it is a table, and `enabled` is not false. It defaults to true."""
+ backend_table = document.get(backend_name)
+ if not isinstance(backend_table, dict):
+ return False
+ return bool(cast("dict[str, Any]", backend_table).get(_ENABLED_KEY, True))
diff --git a/pipelex/migration/former_release_cleanup.py b/pipelex/migration/former_release_cleanup.py
new file mode 100644
index 000000000..d1624d0cf
--- /dev/null
+++ b/pipelex/migration/former_release_cleanup.py
@@ -0,0 +1,815 @@
+"""The cleanup of what a former release left: the `pipelex migrate` step that removes it, with a copy of every file.
+
+`former_release.py` finds what a release that ran on the Pipelex Gateway left in the home and
+project configuration directories, read as their boots merge them; this module removes exactly that, and nothing else.
+Each finding becomes one change:
+
+- a retired backend table, a retired routing profile and a route to a retired backend are deleted, the comment
+ introducing each going with it;
+- a retired `default` of a profile that stays — an override retargeting the user's own profile — is deleted alone,
+ so the profile's routes stay;
+- a `model_specs_section` key is deleted from the backend that carries it, and the backend stays;
+- an `active` naming a profile the cleanup removes moves off it in every file that sets it, whichever directory the
+ profile was defined in: a base moves to the profile the kit makes active (`all_enabled_backends`), which is added to
+ the file as the kit ships it when the file does not define it, and an override stops naming one, so the file read
+ before it decides;
+- a file of its own — a retired backend's per-model file, the Gateway's model lists, `pipelex_service.toml` — is
+ removed.
+
+A file that is rewritten also loses the paragraphs of comments the former release shipped at the head of the
+document, word for word: the Gateway's instructions. Every other comment, a user's note naming the Gateway included,
+stays. A file whose shape the cleanup cannot rewrite safely — routing profiles written as one inline table, where the
+kit's profile cannot be added, or a change the cleanup cannot express — is left as it is and reported, with what to
+change by hand.
+
+**After writing, the cleanup checks its own work.** It reads the files every boot of the machine merges, and anything
+that still stops one — what a former release left, or an active routing profile no file defines — is reported as
+needing attention rather than as a success.
+
+Every file gets the backup the ledger replay gives the files it rewrites, through the same per-file transaction
+(`runner.write_file_with_backup` and `runner.remove_file_with_backup`): one `.bak.<stamp>` copy, taken first and
+kept, older ones pruned, and a file changed or unwritable mid-run left exactly as it was and reported. The
+transaction is per file, not across the files: one that fails after others were written leaves those written, so a
+boot the plan kept on its profile through a change in another file can start on another profile until that file is
+edited. A dry run reads and reports the same changes and writes nothing.
+
+See `docs/migration-ledger.md` → "A former release's configuration".
+"""
+
+import copy
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from enum import StrEnum
+from pathlib import Path
+from typing import Any, NamedTuple, cast
+
+import tomlkit
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from tomlkit import TOMLDocument
+from tomlkit.container import OutOfOrderTableProxy
+from tomlkit.exceptions import TOMLKitError
+from tomlkit.items import Comment, Item, Null, Table, Whitespace
+
+from pipelex.base_exceptions import PipelexUnexpectedError
+from pipelex.cogt.model_routing.routing_profile_factory import RoutingProfileLibraryBlueprint
+from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.fix_ops.applier import apply_fix_ops, delete_literally
+from pipelex.fix_ops.file_transaction import FileSnapshot, read_file_snapshot
+from pipelex.migration.engine import apply_ops_over_text
+from pipelex.migration.former_release import (
+ ACTIVE_KEY,
+ MODEL_SPECS_SECTION_KEY,
+ ROUTE_TABLE_KEYS,
+ ROUTING_PROFILES_KEY,
+ FormerReleaseFinding,
+ FormerReleaseFindingKind,
+ detect_former_release_across,
+ distinct_config_dirs,
+ former_release_boot_blockers,
+ inference_merge_sequences,
+ kit_default_routing_profile_name,
+ kit_routing_profile_library_path,
+ routing_boots,
+)
+from pipelex.migration.gitignore import ensure_config_dir_gitignore
+from pipelex.migration.plan import FileBlockedReason
+from pipelex.migration.runner import FileWriteOutcome, remove_file_with_backup, write_file_with_backup
+from pipelex.suggested_fix import RemapValueOp, SetKeyOp
+from pipelex.system.configuration.config_loader import (
+ BACKENDS_FILE_NAME,
+ BACKENDS_OVERRIDE_FILE_NAME,
+ ROUTING_PROFILES_FILE_NAME,
+ ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+)
+from pipelex.tools.misc.exceptions import TomlError
+from pipelex.tools.misc.json_utils import deep_update
+from pipelex.tools.misc.toml_utils import describe_toml_base_and_overrides, load_toml_from_content, load_toml_from_path
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+#: The paragraphs of comments a former release shipped at the head of its documents about its own backends, line by
+#: line, as v0.72 wrote them: the Gateway's instructions above the routing profiles. A paragraph at the head of a
+#: document goes only when it is one of these exactly; any other, a user's note naming the Gateway included, stays.
+_FORMER_RELEASE_HEAD_PARAGRAPHS: frozenset[tuple[str, ...]] = frozenset(
+ {
+ (
+ '# We recommend using the "all_pipelex_gateway" profile to get a head start with all models.',
+ "# To use the Pipelex Gateway backend:",
+ "# 1. Get your API key at https://app.pipelex.com (free credits included)",
+ "# 2. Add it to your .env file: PIPELEX_GATEWAY_API_KEY=your-key-here",
+ "# 3. Run `pipelex init` and accept the Gateway terms of service",
+ ),
+ }
+)
+
+_DEFAULT_KEY = "default"
+
+_OVERRIDE_FILE_NAMES = frozenset({BACKENDS_OVERRIDE_FILE_NAME, ROUTING_PROFILES_OVERRIDE_FILE_NAME})
+
+
+class FormerReleaseFileAction(StrEnum):
+ """What the cleanup does to one file."""
+
+ REWRITE = "rewrite"
+ """The file stays, without what the former release left in it."""
+
+ REMOVE = "remove"
+ """The file is the former release's own, and goes."""
+
+
+class FormerReleaseFileCleanup(BaseModel):
+ """The cleanup of one file: what it changes, and what became of the write.
+
+ Like every migration report, it carries paths, keys and the names of retired backends and profiles, and never a
+ value read from the file.
+ """
+
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ file_path: Path
+ action: FormerReleaseFileAction = Field(strict=False)
+ changes: list[str]
+ """Each change in words, in the order it is made."""
+
+ backup_path: Path | None = None
+ """Where the copy of the file as it was found is, once the cleanup ran."""
+
+ was_applied: bool = False
+ blocked_reason: FileBlockedReason | None = Field(default=None, strict=False)
+ blocked_detail: str | None = None
+
+ @property
+ def is_blocked(self) -> bool:
+ return self.blocked_reason is not None
+
+
+class FormerReleaseCleanup(BaseModel):
+ """The cleanup of every directory one run was pointed at."""
+
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ dry_run: bool
+ files: list[FormerReleaseFileCleanup] = Field(default_factory=list[FormerReleaseFileCleanup])
+ still_blocking: list[str] = Field(default_factory=list[str])
+ """What would still stop a boot of this machine once the cleanup is done, each in a sentence: read on every run, off
+ the files as the write pass left them, or as a dry run would leave them, and empty when every boot can start."""
+
+ @property
+ def is_clean(self) -> bool:
+ """Whether there was nothing to do and nothing to say: no file a former release left, and no boot still stopped."""
+ return not self.files and not self.still_blocking
+
+ @property
+ def removed_paths(self) -> frozenset[Path]:
+ """The files the cleanup removes, or would: the ledger replay leaves them out of its walk, written or not."""
+ return frozenset(file.file_path for file in self.files if file.action is FormerReleaseFileAction.REMOVE)
+
+ @property
+ def applied_files(self) -> list[FormerReleaseFileCleanup]:
+ return [file for file in self.files if file.was_applied]
+
+ @property
+ def blocked_files(self) -> list[FormerReleaseFileCleanup]:
+ return [file for file in self.files if file.is_blocked]
+
+ @property
+ def needs_attention(self) -> bool:
+ """Whether a file could not be cleaned, or the machine still cannot start once it was: the user has to look."""
+ return bool(self.blocked_files) or bool(self.still_blocking)
+
+
+def clean_former_release(*, config_dirs: Sequence[Path], dry_run: bool, moment: datetime | None = None) -> FormerReleaseCleanup:
+ """Remove what a former release left in each configuration directory, keeping a copy of every file.
+
+ The directories are read together, as their boots merge them, and the cleanup is planned per boot
+ (`former_release._routing_plan`). Every file's new text is worked out before anything is written, and checked:
+ files whose changes would stop a boot that starts today, or change the profile it starts on, are left as they are,
+ for a hand edit (`_without_regressions`). That holds for the run's changes taken together: the files are written one
+ by one, and one that fails after others were written is reported blocked while those stay written. Then, on every
+ run, what would still stop a boot of the machine is read off the files as the cleanup leaves them — after writing,
+ or as a dry run would write them — and reported in `still_blocking` rather than as a success.
+
+ Args:
+ config_dirs: The directories to clean in tier order, `~/.pipelex/` then a project's `.pipelex/`, as
+ `ConfigLoader.existing_config_dirs` lists them; one named twice is cleaned once.
+ dry_run: Report what would change and write nothing, not even the `.gitignore` that hides the backups.
+ moment: The time the backups are stamped with, one for the whole run; now when not given.
+
+ Returns:
+ One entry per file that had something to clean, in the order the directories were given, and what would still
+ stop a boot.
+ """
+ stamp_moment = moment or datetime.now(UTC)
+ kit_default = kit_default_routing_profile_name()
+ directories = distinct_config_dirs(config_dirs=config_dirs)
+ all_findings = detect_former_release_across(config_dirs=directories)
+ prepared: list[_PreparedFile] = []
+ for findings in all_findings:
+ for file_path in findings.file_paths:
+ file_findings = [finding for finding in findings.findings if finding.file_path == file_path]
+ prepared_file = _prepare_file(
+ file_path=file_path, findings=file_findings, kit_default=kit_default, hand_edit=findings.hand_edits.get(file_path)
+ )
+ if prepared_file is not None:
+ prepared.append(prepared_file)
+ prepared = _without_regressions(prepared=prepared, config_dirs=directories)
+ if dry_run:
+ return FormerReleaseCleanup(
+ dry_run=True,
+ files=[prepared_file.report for prepared_file in prepared],
+ still_blocking=what_stops_the_boot(config_dirs=directories, contents=_contents_after(prepared=prepared)),
+ )
+ for findings in all_findings:
+ if not findings.is_clean:
+ ensure_config_dir_gitignore(directory=findings.config_dir)
+ files = [_write(prepared_file=prepared_file, moment=stamp_moment) for prepared_file in prepared]
+ return FormerReleaseCleanup(dry_run=False, files=files, still_blocking=what_stops_the_boot(config_dirs=directories))
+
+
+def what_stops_the_boot(*, config_dirs: Sequence[Path], contents: Mapping[Path, str | None] | None = None) -> list[str]:
+ """What stops a boot of this machine, read off the files each boot merges: the home's alone, and the project's.
+
+ The check the cleanup runs on every run. It names what a former release left that a boot refuses, and a routing
+ profile library the boot cannot load — above all an `active` naming a profile no file of the sequence defines,
+ which is what a cleanup that removed a profile and missed a file activating it would leave, or one run from another
+ project. A sequence without its base file, or with a file that does not parse, is not judged here: those are the
+ boot's and `pipelex init`'s to name.
+
+ Args:
+ config_dirs: The home directory, then the project's.
+ contents: The files as a cleanup would leave them, read in place of the ones at their paths, `None` for a file
+ it would remove: what a dry run checks.
+
+ Returns:
+ Each problem in a sentence, each once; empty when every boot of the machine can start on these files.
+ """
+ backends_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=BACKENDS_FILE_NAME, override_file_name=BACKENDS_OVERRIDE_FILE_NAME
+ )
+ routing_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=ROUTING_PROFILES_FILE_NAME, override_file_name=ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ )
+ problems: list[str] = []
+ for backends_paths, routing_paths in zip(backends_sequences, routing_sequences, strict=True):
+ blockers = former_release_boot_blockers(backends_library_paths=backends_paths, routing_profile_library_paths=routing_paths, contents=contents)
+ problems.extend(finding.description for finding in blockers)
+ if (problem := _routing_library_problem(paths=routing_paths, contents=contents)) is not None:
+ problems.append(problem)
+ return list(dict.fromkeys(problems))
+
+
+def _routing_library_problem(*, paths: Sequence[Path], contents: Mapping[Path, str | None] | None) -> str | None:
+ """Why the boot could not load this routing profile library, or `None` when it could, or when that is not ours to say."""
+ library = _merged_library(paths=paths, contents=contents)
+ if library is None:
+ return None
+ description = describe_toml_base_and_overrides(paths=paths)
+ try:
+ blueprint = RoutingProfileLibraryBlueprint.model_validate(library)
+ except ValidationError as exc:
+ return f"the routing profile library {description} is not valid: {format_pydantic_validation_error(exc)}"
+ if blueprint.active not in blueprint.profiles:
+ return f"the routing profile library {description} makes '{blueprint.active}' the active routing profile, and none of its files defines it"
+ return None
+
+
+def _merged_library(*, paths: Sequence[Path], contents: Mapping[Path, str | None] | None) -> dict[str, Any] | None:
+ """The base and its overrides merged as the boot merges them, `contents` standing in for the files it names; `None`
+ without a base, or with a file that does not parse.
+ """
+ merged: dict[str, Any] = {}
+ for index, path in enumerate(paths):
+ if contents is not None and path in contents:
+ text = contents[path]
+ if text is None:
+ if index == 0:
+ return None
+ continue
+ try:
+ document = load_toml_from_content(text)
+ except TomlError:
+ return None
+ elif path.is_file():
+ try:
+ document = load_toml_from_path(path)
+ except (OSError, TomlError, UnicodeDecodeError):
+ return None
+ elif index == 0:
+ return None
+ else:
+ continue
+ deep_update(merged, updates=document)
+ return merged
+
+
+class _PreparedFile(NamedTuple):
+ """One file's cleanup, worked out and not yet written: its report, and what the write pass does with it."""
+
+ report: FormerReleaseFileCleanup
+ snapshot: FileSnapshot | None = None
+ """The file as read, which the write compares against; `None` for a file already blocked."""
+
+ new_text: str | None = None
+ """The text a rewrite writes; `None` for a removal, or a file already blocked."""
+
+ @property
+ def is_blocked(self) -> bool:
+ return self.report.is_blocked
+
+
+def _prepare_file(*, file_path: Path, findings: list[FormerReleaseFinding], kit_default: str, hand_edit: str | None) -> _PreparedFile | None:
+ """Work one file's cleanup out, or say why it cannot be made; `None` when its findings call for no change of its own."""
+ if any(finding.kind.is_a_file_of_its_own for finding in findings):
+ return _prepare_removal(file_path=file_path, findings=findings)
+ if hand_edit is not None:
+ return _PreparedFile(
+ report=_blocked(
+ file_path=file_path,
+ action=FormerReleaseFileAction.REWRITE,
+ changes=[finding.description for finding in findings],
+ reason=FileBlockedReason.NEEDS_A_HAND_EDIT,
+ detail=hand_edit,
+ )
+ )
+ return _prepare_rewrite(file_path=file_path, findings=findings, kit_default=kit_default)
+
+
+def _prepare_removal(*, file_path: Path, findings: list[FormerReleaseFinding]) -> _PreparedFile:
+ changes = [_removal_in_words(finding=finding) for finding in findings]
+ try:
+ # The path itself, not what a symbolic link names: removing the link is the cleanup, and the file it points
+ # at belongs to wherever it lives.
+ snapshot = read_file_snapshot(file_path)
+ except FileNotFoundError:
+ return _PreparedFile(
+ report=_blocked(
+ file_path=file_path,
+ action=FormerReleaseFileAction.REMOVE,
+ changes=changes,
+ reason=FileBlockedReason.CHANGED_DURING_RUN,
+ detail="the file was removed while the cleanup was being prepared",
+ )
+ )
+ except OSError as exc:
+ return _PreparedFile(
+ report=_blocked(
+ file_path=file_path,
+ action=FormerReleaseFileAction.REMOVE,
+ changes=changes,
+ reason=FileBlockedReason.UNREADABLE,
+ detail=f"the file could not be read: {exc.strerror or exc}",
+ )
+ )
+ return _PreparedFile(
+ report=FormerReleaseFileCleanup(file_path=file_path, action=FormerReleaseFileAction.REMOVE, changes=changes), snapshot=snapshot
+ )
+
+
+def _prepare_rewrite(*, file_path: Path, findings: list[FormerReleaseFinding], kit_default: str) -> _PreparedFile | None:
+ descriptions = [finding.description for finding in findings]
+ try:
+ # Through a symbolic link to the file it names, as the ledger replay rewrites one: replacing the link path
+ # would delete the link and leave the real file as it was.
+ snapshot = read_file_snapshot(file_path.resolve())
+ except FileNotFoundError:
+ return _blocked_rewrite(
+ file_path=file_path,
+ changes=descriptions,
+ reason=FileBlockedReason.CHANGED_DURING_RUN,
+ detail="the file was removed while the cleanup was being prepared",
+ )
+ except OSError as exc:
+ return _blocked_rewrite(
+ file_path=file_path,
+ changes=descriptions,
+ reason=FileBlockedReason.UNREADABLE,
+ detail=f"the file could not be read: {exc.strerror or exc}",
+ )
+ try:
+ text = snapshot.content.decode("utf-8")
+ cleaned = cleaned_text(text=text, findings=findings, kit_default=kit_default, is_override=file_path.name in _OVERRIDE_FILE_NAMES)
+ except (UnicodeDecodeError, TOMLKitError) as exc:
+ # It parsed when it was read for the findings, so this is a file changed since or an operation failing on
+ # valid TOML. Either way it is this file's to report, never a reason to stop the run.
+ return _blocked_rewrite(
+ file_path=file_path, changes=descriptions, reason=FileBlockedReason.UNPARSEABLE, detail=f"the file could not be cleaned: {exc}"
+ )
+ except ValueError as exc:
+ # A change the cleanup could not express on this document — an operation refusing a key, above all. One odd
+ # file is left for a hand edit; it never stops the run, the other files, or the doctor reading a dry run.
+ reason = format_pydantic_validation_error(exc) if isinstance(exc, ValidationError) else str(exc)
+ return _blocked_rewrite(
+ file_path=file_path,
+ changes=descriptions,
+ reason=FileBlockedReason.NEEDS_A_HAND_EDIT,
+ detail=(
+ f"the cleanup could not express its changes to this file ({reason}). By hand, remove what a former release "
+ f"left in it, then run '{MIGRATE_COMMAND}' again"
+ ),
+ )
+ if cleaned.hand_edit is not None:
+ return _blocked_rewrite(file_path=file_path, changes=descriptions, reason=FileBlockedReason.NEEDS_A_HAND_EDIT, detail=cleaned.hand_edit)
+ if cleaned.text == text:
+ return None
+ return _PreparedFile(
+ report=FormerReleaseFileCleanup(file_path=file_path, action=FormerReleaseFileAction.REWRITE, changes=cleaned.changes),
+ snapshot=snapshot,
+ new_text=cleaned.text,
+ )
+
+
+def _blocked_rewrite(*, file_path: Path, changes: list[str], reason: FileBlockedReason, detail: str) -> _PreparedFile:
+ return _PreparedFile(report=_blocked(file_path=file_path, action=FormerReleaseFileAction.REWRITE, changes=changes, reason=reason, detail=detail))
+
+
+def _without_regressions(*, prepared: list[_PreparedFile], config_dirs: Sequence[Path]) -> list[_PreparedFile]:
+ """The prepared files, less the routing profile rewrites that would stop a boot that starts today.
+
+ The plan already keeps every boot that starts as it is; this reads the texts the cleanup is about to write, as the
+ boot would, so that nothing the plan did not foresee — a file whose shape left one of its changes undone — is ever
+ written into a state that stops a boot. A boot that started and would not start the same way, on the same profile,
+ has the rewrites of the files it reads left for a hand edit, and the texts are read again.
+ """
+ routing_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=ROUTING_PROFILES_FILE_NAME, override_file_name=ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ )
+ before = routing_boots(sequences=routing_sequences)
+ while True:
+ after = routing_boots(sequences=routing_sequences, contents=_contents_after(prepared=prepared))
+ stopped = [index for index, (boot, boot_after) in enumerate(zip(before, after, strict=True)) if boot.regresses_to(after=boot_after)]
+ culprits = {path: index for index in stopped for path in routing_sequences[index]}
+ to_block = [prepared_file for prepared_file in prepared if not prepared_file.is_blocked and prepared_file.report.file_path in culprits]
+ if not to_block:
+ return prepared
+ prepared = [
+ _PreparedFile(
+ report=_blocked(
+ file_path=prepared_file.report.file_path,
+ action=prepared_file.report.action,
+ changes=prepared_file.report.changes,
+ reason=FileBlockedReason.NEEDS_A_HAND_EDIT,
+ detail=(
+ f"cleaning this file would stop {_boot_name(index=culprits[prepared_file.report.file_path], count=len(routing_sequences))} "
+ f"from starting as it does now, so it is left as it is. By hand, remove what a former release left in it, keeping "
+ f"each routing profile you start on whole, then run '{MIGRATE_COMMAND}' again"
+ ),
+ )
+ )
+ if prepared_file in to_block
+ else prepared_file
+ for prepared_file in prepared
+ ]
+
+
+def _boot_name(*, index: int, count: int) -> str:
+ if count < 2:
+ return "Pipelex"
+ return ("the boot outside the project", "the project's boot")[index]
+
+
+def _contents_after(*, prepared: Sequence[_PreparedFile]) -> dict[Path, str | None]:
+ """Each file the cleanup writes, as it will be: its new text, or nothing for a file it removes."""
+ contents: dict[Path, str | None] = {}
+ for prepared_file in prepared:
+ if prepared_file.is_blocked:
+ continue
+ if prepared_file.report.action is FormerReleaseFileAction.REMOVE:
+ contents[prepared_file.report.file_path] = None
+ elif prepared_file.new_text is not None:
+ contents[prepared_file.report.file_path] = prepared_file.new_text
+ return contents
+
+
+def _write(*, prepared_file: _PreparedFile, moment: datetime) -> FormerReleaseFileCleanup:
+ """Make one prepared change, with its backup, through the per-file transaction; a blocked file is reported as it is."""
+ report = prepared_file.report
+ if prepared_file.is_blocked or prepared_file.snapshot is None:
+ return report
+ if report.action is FormerReleaseFileAction.REMOVE:
+ outcome = remove_file_with_backup(snapshot=prepared_file.snapshot, moment=moment)
+ elif prepared_file.new_text is not None:
+ outcome = write_file_with_backup(snapshot=prepared_file.snapshot, new_content=prepared_file.new_text, moment=moment)
+ else:
+ return report
+ return _with_outcome(file_path=report.file_path, action=report.action, changes=report.changes, outcome=outcome)
+
+
+class CleanedText(NamedTuple):
+ """One document's cleanup: the text, the changes in words, or why it is left for a person to edit."""
+
+ text: str
+ changes: list[str]
+ hand_edit: str | None = None
+ """What the user has to change by hand, when the cleanup cannot rewrite this document safely; the text is then
+ the document as read."""
+
+
+class _Deletion(NamedTuple):
+ """One key or table deleted where the document spells it: a `*` in the path or the key is a key spelled so."""
+
+ table_path: tuple[str, ...]
+ key: str
+
+
+#: One edit of a document: a deletion, or a migration op writing a value — the moved `active`, or the project's own one.
+_Edit = _Deletion | RemapValueOp | SetKeyOp
+
+
+def _apply_edit(*, text: str, edit: _Edit) -> tuple[str, bool]:
+ """The document with one edit made, re-read from its text, and whether the edit changed anything."""
+ match edit:
+ case _Deletion():
+ document = tomlkit.loads(text)
+ if not delete_literally(toml_doc=document, table_path=edit.table_path, key=edit.key):
+ return text, False
+ return tomlkit.dumps(document), True # pyright: ignore[reportUnknownMemberType]
+ case RemapValueOp():
+ application = apply_ops_over_text(text=text, ops=[edit])
+ return application.text, application.applied_count > 0
+ case SetKeyOp():
+ # Not a migration op, which never writes a value of its own: applied to the document directly.
+ document = tomlkit.loads(text)
+ applications = apply_fix_ops(toml_doc=document, ops=[edit])
+ new_text = tomlkit.dumps(document) # pyright: ignore[reportUnknownMemberType]
+ return new_text, applications[0].outcome.did_apply and new_text != text
+
+
+class _KitProfilePlacement(StrEnum):
+ ADDED = "added"
+ ALREADY_THERE = "already_there"
+ IMPOSSIBLE = "impossible"
+
+
+def cleaned_text(*, text: str, findings: Sequence[FormerReleaseFinding], kit_default: str, is_override: bool) -> CleanedText:
+ """One document without what a former release left in it, and each change in words.
+
+ The kit's default profile goes in first, ahead of the deletions, so that the banner heading the profiles stays at
+ the head of them instead of leaving with the first profile it introduced. When it cannot go in — the profiles are
+ one inline table — the active profile is not moved to it either, and nothing is changed: an `active` naming a
+ profile the file lacks would stop the boot as surely, so the document is left for a hand edit. The comments at
+ the head of the document go last, and only from a document something else changed.
+
+ Args:
+ text: The document as read.
+ findings: What `detect_former_release_across` found in this document.
+ kit_default: The routing profile the kit makes active.
+ is_override: Whether the document is a personal override, whose retired `active` is deleted rather than moved.
+
+ Returns:
+ The cleaned text, the same string when nothing changed, and the changes made, in order; or the document as
+ read and what to edit by hand.
+ """
+ changes: list[str] = []
+ current_text = text
+ edits: list[tuple[_Edit, str]] = []
+ for finding in findings:
+ edits.extend(_ops_for(finding=finding, kit_default=kit_default, is_override=is_override))
+
+ moves_active = any(isinstance(edit, RemapValueOp) for edit, _ in edits)
+ if moves_active:
+ document = tomlkit.loads(current_text)
+ match _add_kit_profile(document=document, kit_default=kit_default):
+ case _KitProfilePlacement.ADDED:
+ current_text = tomlkit.dumps(document) # pyright: ignore[reportUnknownMemberType]
+ changes.append(f"added the routing profile '{kit_default}', as this release ships it")
+ case _KitProfilePlacement.ALREADY_THERE:
+ pass
+ case _KitProfilePlacement.IMPOSSIBLE:
+ return CleanedText(text=text, changes=[], hand_edit=_inline_profiles_hand_edit(kit_default=kit_default))
+
+ for edit, change in edits:
+ current_text, applied = _apply_edit(text=current_text, edit=edit)
+ if applied:
+ changes.append(change)
+ current_text = _without_emptied_profiles(text=current_text, findings=findings)
+ if current_text == text:
+ return CleanedText(text=text, changes=[])
+
+ document = tomlkit.loads(current_text)
+ if _drop_former_release_comments(document=document):
+ current_text = tomlkit.dumps(document) # pyright: ignore[reportUnknownMemberType]
+ changes.append("removed a comment about the Pipelex Gateway")
+ return CleanedText(text=current_text, changes=_deduplicated(changes=changes))
+
+
+def _without_emptied_profiles(*, text: str, findings: Sequence[FormerReleaseFinding]) -> str:
+ """The document without a profile table that losing its retired `default` left empty.
+
+ An override that only retargeted a profile is a `[profiles.<name>]` table holding that one key: once the key goes,
+ the header overrides nothing, and it goes too, with no change line of its own, since the key's says what happened.
+ """
+ retargeted = {finding.subject for finding in findings if finding.kind is FormerReleaseFindingKind.RETIRED_PROFILE_DEFAULT}
+ if not retargeted:
+ return text
+ profiles = load_toml_from_content(text).get(ROUTING_PROFILES_KEY)
+ if not isinstance(profiles, dict):
+ return text
+ emptied = [
+ name
+ for name, profile in cast("dict[str, Any]", profiles).items()
+ if name in retargeted and isinstance(profile, dict) and not cast("dict[str, Any]", profile)
+ ]
+ for name in emptied:
+ text, _ = _apply_edit(text=text, edit=_Deletion(table_path=(ROUTING_PROFILES_KEY,), key=name))
+ return text
+
+
+def _inline_profiles_hand_edit(*, kit_default: str) -> str:
+ return (
+ f"the routing profiles are written as one inline table, where the cleanup cannot add the profile '{kit_default}' to make it "
+ f"active in place of the former release's. By hand, either write each profile as a [profiles.<name>] table, or make one of "
+ f"your own profiles active, then run '{MIGRATE_COMMAND}' again"
+ )
+
+
+def _ops_for(*, finding: FormerReleaseFinding, kit_default: str, is_override: bool) -> list[tuple[_Edit, str]]:
+ """The edits that remove one finding from its document, each with its change in words.
+
+ Every key and table is deleted where the document spells it (`_Deletion`), so a route or a profile spelled `*` is
+ that one key, never the "every key" a migration op reads `*` as.
+ """
+ match finding.kind:
+ case FormerReleaseFindingKind.RETIRED_BACKEND_TABLE:
+ return [(_Deletion(table_path=(), key=_subject_of(finding=finding)), f"removed the '{finding.subject}' backend")]
+ case FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY:
+ return [
+ (
+ _Deletion(table_path=(_subject_of(finding=finding),), key=MODEL_SPECS_SECTION_KEY),
+ f"removed '{MODEL_SPECS_SECTION_KEY}' from the '{finding.subject}' backend",
+ )
+ ]
+ case FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE:
+ return [
+ (_Deletion(table_path=(ROUTING_PROFILES_KEY,), key=_subject_of(finding=finding)), f"removed the routing profile '{finding.subject}'")
+ ]
+ case FormerReleaseFindingKind.RETIRED_PROFILE_DEFAULT:
+ return [
+ (
+ _Deletion(table_path=(ROUTING_PROFILES_KEY, _subject_of(finding=finding)), key=_DEFAULT_KEY),
+ f"removed '{_DEFAULT_KEY}', which named '{finding.retired_backend}', from the routing profile '{finding.subject}'",
+ )
+ ]
+ case FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND:
+ route_table = finding.route_table or ROUTE_TABLE_KEYS[0]
+ route_pattern = finding.route_pattern or ""
+ return [
+ (
+ _Deletion(table_path=(ROUTING_PROFILES_KEY, _subject_of(finding=finding), route_table), key=route_pattern),
+ f"removed the route of '{route_pattern}' to '{finding.retired_backend}' from the routing profile '{finding.subject}'",
+ )
+ ]
+ case FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE:
+ if not finding.profile_is_removed:
+ # The plan keeps this `active`: what made its profile retired — a route, or a retired `default` another
+ # file overrides — is a finding of its own, in the file that holds it, and goes there; or moving it would
+ # change the profile another boot starts on, and the project's override makes up the difference.
+ return []
+ if is_override:
+ return [
+ (
+ _Deletion(table_path=(), key=ACTIVE_KEY),
+ f"removed '{ACTIVE_KEY}', which named the routing profile '{finding.subject}', so the file read before this one decides",
+ )
+ ]
+ return [
+ (
+ RemapValueOp(table_path=[], key=ACTIVE_KEY, mapping={_subject_of(finding=finding): kit_default}),
+ f"moved the active routing profile from '{finding.subject}' to '{kit_default}'",
+ )
+ ]
+ case FormerReleaseFindingKind.PROJECT_ACTIVE_ROUTING_PROFILE:
+ return [
+ (
+ SetKeyOp(table_path=[], key=ACTIVE_KEY, value=_subject_of(finding=finding)),
+ (
+ f"set '{ACTIVE_KEY}' to the routing profile '{finding.subject}' for this project's boot alone, since the file it "
+ f"read '{ACTIVE_KEY}' from changes for the boot outside the project"
+ ),
+ )
+ ]
+ case (
+ FormerReleaseFindingKind.RETIRED_BACKEND_FILE | FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE | FormerReleaseFindingKind.SERVICE_FILE
+ ):
+ return []
+
+
+def _removal_in_words(*, finding: FormerReleaseFinding) -> str:
+ match finding.kind:
+ case FormerReleaseFindingKind.RETIRED_BACKEND_FILE:
+ return f"removed the model settings of the '{finding.subject}' backend"
+ case FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE:
+ return "removed the list of the models the Pipelex Gateway served"
+ case FormerReleaseFindingKind.SERVICE_FILE:
+ return "removed the record of the Pipelex Gateway's terms acceptance"
+ case (
+ FormerReleaseFindingKind.RETIRED_BACKEND_TABLE
+ | FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY
+ | FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE
+ | FormerReleaseFindingKind.RETIRED_PROFILE_DEFAULT
+ | FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND
+ | FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE
+ | FormerReleaseFindingKind.PROJECT_ACTIVE_ROUTING_PROFILE
+ ):
+ return finding.description
+
+
+def _add_kit_profile(*, document: TOMLDocument, kit_default: str) -> _KitProfilePlacement:
+ """Put the kit's default profile first among the document's profiles, as the kit ships it, unless it is there.
+
+ Copied from the kit's own document, so its comments come with it and it lands as `pipelex init` would write it.
+ An inline `profiles` table, which no release wrote, cannot take it without being restructured, and is not.
+ """
+ profiles: Table | None = None
+ if ROUTING_PROFILES_KEY in document:
+ existing = document.item(ROUTING_PROFILES_KEY)
+ if isinstance(existing, OutOfOrderTableProxy):
+ if kit_default in existing:
+ return _KitProfilePlacement.ALREADY_THERE
+ # Profiles spread across the file between other tables: the profile joins them where tomlkit puts it.
+ existing[kit_default] = _kit_profile(kit_default=kit_default)
+ return _KitProfilePlacement.ADDED
+ if not isinstance(existing, Table):
+ if isinstance(existing, dict) and kit_default in existing:
+ return _KitProfilePlacement.ALREADY_THERE
+ return _KitProfilePlacement.IMPOSSIBLE
+ if kit_default in existing:
+ return _KitProfilePlacement.ALREADY_THERE
+ profiles = existing
+ if profiles is None:
+ profiles = tomlkit.table(is_super_table=True)
+ document.append(ROUTING_PROFILES_KEY, profiles)
+ if len(profiles.value) == 0:
+ profiles.append(kit_default, _kit_profile(kit_default=kit_default))
+ else:
+ profiles.value._insert_at(0, kit_default, _kit_profile(kit_default=kit_default)) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ return _KitProfilePlacement.ADDED
+
+
+def _kit_profile(*, kit_default: str) -> Item:
+ kit_path = kit_routing_profile_library_path()
+ kit_profiles = tomlkit.loads(kit_path.read_text(encoding="utf-8")).item(ROUTING_PROFILES_KEY)
+ if not isinstance(kit_profiles, Table) or kit_default not in kit_profiles:
+ msg = f"the kit's routing profile library '{kit_path}' does not define its active profile '{kit_default}' — packaging bug"
+ raise PipelexUnexpectedError(msg)
+ return copy.deepcopy(kit_profiles.value.item(kit_default))
+
+
+def _drop_former_release_comments(*, document: TOMLDocument) -> bool:
+ """Drop each paragraph of comments at the document's head that a former release shipped about its backends.
+
+ The head is the root of the document, ahead of its first table: where a release wrote the instructions for the
+ file, and where a user's notes sit too, above the first table or between `active` and it. So a paragraph goes
+ only when it is, line for line, one the release shipped (`_FORMER_RELEASE_HEAD_PARAGRAPHS`); one that merely
+ names the Gateway is the user's. A paragraph is a run of comment lines between blank ones; it goes whole, with
+ the blank line after it. The items are blanked in place rather than removed, which keeps the container's indexes
+ valid; they render as nothing.
+ """
+ body = document.body
+ dropped = False
+ index = 0
+ while index < len(body):
+ if not isinstance(body[index][1], Comment):
+ index += 1
+ continue
+ end = index
+ while end < len(body) and isinstance(body[end][1], Comment):
+ end += 1
+ paragraph = tuple(item.as_string().strip() for _, item in body[index:end])
+ if paragraph in _FORMER_RELEASE_HEAD_PARAGRAPHS:
+ for position in range(index, end):
+ body[position] = (None, Null())
+ if end < len(body) and isinstance(body[end][1], Whitespace):
+ body[end] = (None, Null())
+ end += 1
+ dropped = True
+ index = end
+ return dropped
+
+
+def _deduplicated(*, changes: list[str]) -> list[str]:
+ """The changes, each once, in order: a profile defined in two halves of a file is one profile removed."""
+ return list(dict.fromkeys(changes))
+
+
+def _subject_of(*, finding: FormerReleaseFinding) -> str:
+ if finding.subject is None:
+ msg = f"a former-release finding of kind '{finding.kind}' names no subject — detector bug"
+ raise PipelexUnexpectedError(msg)
+ return finding.subject
+
+
+def _with_outcome(*, file_path: Path, action: FormerReleaseFileAction, changes: list[str], outcome: FileWriteOutcome) -> FormerReleaseFileCleanup:
+ return FormerReleaseFileCleanup(
+ file_path=file_path,
+ action=action,
+ changes=changes,
+ backup_path=outcome.backup_path,
+ was_applied=outcome.was_written,
+ blocked_reason=outcome.blocked_reason,
+ blocked_detail=outcome.blocked_detail,
+ )
+
+
+def _blocked(
+ *, file_path: Path, action: FormerReleaseFileAction, changes: list[str], reason: FileBlockedReason, detail: str
+) -> FormerReleaseFileCleanup:
+ return FormerReleaseFileCleanup(file_path=file_path, action=action, changes=changes, blocked_reason=reason, blocked_detail=detail)
diff --git a/pipelex/migration/plan.py b/pipelex/migration/plan.py
index 87a33702b..18f166cf5 100644
--- a/pipelex/migration/plan.py
+++ b/pipelex/migration/plan.py
@@ -101,8 +101,8 @@ class FileBlockedReason(StrEnum):
"""Why a file could not be processed at all.
One member per *state the file is in*, because that is what decides what the user does next —
- not one per exception the run happened to catch. Four of the five leave the file exactly as it
- was found; the fifth is the one that does not, and it says so.
+ not one per exception the run happened to catch. Every member but `STATE_UNCERTAIN` leaves the
+ file exactly as it was found; that one does not, and it says so.
"""
UNREADABLE = "unreadable"
@@ -128,6 +128,12 @@ class FileBlockedReason(StrEnum):
"""The file was removed or edited between the read and the write, so the run refused to write
over work it had not seen. The file is whatever that other writer left."""
+ NEEDS_A_HAND_EDIT = "needs_a_hand_edit"
+ """The file is valid, and holds a shape the run will not rewrite on its own: changing it safely
+ would mean restructuring what the user wrote — routing profiles written as one inline table,
+ which the former-release cleanup cannot add a profile to. Nothing was written; the detail says
+ what to change by hand before running again."""
+
STATE_UNCERTAIN = "state_uncertain"
"""The write could not be confirmed: the transaction could not describe what it left behind,
and the file does not hold what this run wrote. The one reason that cannot promise the file is
diff --git a/pipelex/migration/run.py b/pipelex/migration/run.py
index 4be02ad5f..16e1f8163 100644
--- a/pipelex/migration/run.py
+++ b/pipelex/migration/run.py
@@ -30,6 +30,7 @@ user would be told one thing by their boot and another by their tool.
See `docs/migration-ledger.md` → "Surfaces" and "Applying".
"""
+from collections.abc import Collection
from pathlib import Path
from pipelex.migration.gitignore import ensure_config_dir_gitignore
@@ -72,7 +73,7 @@ def ensure_config_dir_gitignores(*, config_dirs: list[Path]) -> None:
ensure_config_dir_gitignore(directory=directory)
-def migrate_config_directories(*, config_dirs: list[Path], dry_run: bool) -> MigrationReport:
+def migrate_config_directories(*, config_dirs: list[Path], dry_run: bool, skipped_paths: Collection[Path] = ()) -> MigrationReport:
"""Replay every surface's shipped ledger over every claimed file in the given directories.
The registry and the ledger directory are the package's own, which is what makes this the
@@ -86,6 +87,10 @@ def migrate_config_directories(*, config_dirs: list[Path], dry_run: bool) -> Mig
This covers the run that *writes*. It is deliberately not the only place the rule is ensured:
a run with nothing to carry forward never reaches this function at all, and that is the state
most machines are in. See `ensure_config_dir_gitignores`.
+
+ `skipped_paths` are the files the former-release cleanup removes, which every caller running
+ that cleanup first hands here, in its rehearsal and its write pass alike: a file about to go is
+ neither migrated nor reported on.
"""
if not dry_run:
ensure_config_dir_gitignores(config_dirs=config_dirs)
@@ -94,6 +99,7 @@ def migrate_config_directories(*, config_dirs: list[Path], dry_run: bool) -> Mig
migration_dir=packaged_migration_dir(),
config_dirs=config_dirs,
dry_run=dry_run,
+ skipped_paths=skipped_paths,
)
diff --git a/pipelex/migration/runner.py b/pipelex/migration/runner.py
index d89ce732e..06aae8e9c 100644
--- a/pipelex/migration/runner.py
+++ b/pipelex/migration/runner.py
@@ -12,8 +12,11 @@ engine replays the whole ledger over every file and the applier skips whatever i
See `docs/migration-ledger.md` → "Applying" and "Per-file transactions".
"""
+import os
+from collections.abc import Collection
from datetime import UTC, datetime
from pathlib import Path
+from typing import Any, NamedTuple
import tomlkit
from tomlkit import TOMLDocument
@@ -21,7 +24,7 @@ from tomlkit.exceptions import TOMLKitError
from pipelex import log
from pipelex.fix_ops.exceptions import FixTransactionError, FixWriteConflictError
-from pipelex.fix_ops.file_transaction import FileSnapshot, PendingFileUpdate, commit_file_updates, read_file_snapshot
+from pipelex.fix_ops.file_transaction import FileSnapshot, PendingFileUpdate, assert_snapshot_unchanged, commit_file_updates, read_file_snapshot
from pipelex.migration.backup import RescuedBackup, WrittenBackup, keep_backup_for_rescue, prune_backups_except, write_backup
from pipelex.migration.diagnosis import diagnose_unexplained_paths
from pipelex.migration.engine import DocumentReplay, replay_ledger_over_text
@@ -153,7 +156,33 @@ def _refuse_a_file_below_the_floor(*, surface: Surface, ledger: MigrationLedger,
)
+class FileWriteOutcome(NamedTuple):
+ """What a backed-up write or removal of one file did: where the copy is, whether it landed, and why not.
+
+ The fields a migration plan carries for the file, so a caller reports a write the same way whatever it wrote.
+ """
+
+ backup_path: Path | None = None
+ was_written: bool = False
+ blocked_reason: FileBlockedReason | None = None
+ blocked_detail: str | None = None
+
+ def as_plan_update(self) -> dict[str, Any]:
+ return {
+ "backup_path": self.backup_path,
+ "was_written": self.was_written,
+ "blocked_reason": self.blocked_reason,
+ "blocked_detail": self.blocked_detail,
+ }
+
+
def _write_migrated_file(*, plan: MigrationPlan, snapshot: FileSnapshot, new_content: str, moment: datetime) -> MigrationPlan:
+ """Back the file up and replace it, reporting what happened on the file's plan. See `write_file_with_backup`."""
+ outcome = write_file_with_backup(snapshot=snapshot, new_content=new_content, moment=moment)
+ return plan.model_copy(update=outcome.as_plan_update())
+
+
+def write_file_with_backup(*, snapshot: FileSnapshot, new_content: str, moment: datetime) -> FileWriteOutcome:
"""Back the file up, replace it atomically, then prune the older backups.
The order is the point. Backing up first means there is never a moment with a rewritten file
@@ -165,19 +194,16 @@ def _write_migrated_file(*, plan: MigrationPlan, snapshot: FileSnapshot, new_con
Nothing raised in here escapes to the caller: this is the per-file boundary, and an exception
crossing it would abort every sibling file after this one — the one thing the per-file scope
- exists to rule out. Whatever goes wrong lands on this file's plan, or, once the file is
- written, in a warning, because a written file is written whatever happens to the housekeeping
- around it.
+ exists to rule out. Whatever goes wrong lands on the outcome, or, once the file is written, in
+ a warning, because a written file is written whatever happens to the housekeeping around it.
+
+ Shared by the ledger replay and the former-release cleanup, so a file either rewrites keeps the
+ same one backup and the same guarantees.
"""
try:
backup = write_backup(snapshot=snapshot, moment=moment)
except OSError as exc:
- return plan.model_copy(
- update={
- "blocked_reason": FileBlockedReason.UNWRITABLE,
- "blocked_detail": f"the backup could not be written: {exc.strerror or exc}",
- }
- )
+ return FileWriteOutcome(blocked_reason=FileBlockedReason.UNWRITABLE, blocked_detail=f"the backup could not be written: {exc.strerror or exc}")
try:
commit_file_updates([PendingFileUpdate(snapshot=snapshot, new_content=new_content)])
@@ -186,12 +212,10 @@ def _write_migrated_file(*, plan: MigrationPlan, snapshot: FileSnapshot, new_con
# taking back the copy it just made, which has nothing to back up. What the primitive
# refused *over* is somebody else's write, and `_discard_backup` is what asks whose.
_discard_backup(backup=backup, snapshot=snapshot, new_content=new_content)
- return plan.model_copy(update={"blocked_reason": FileBlockedReason.CHANGED_DURING_RUN, "blocked_detail": str(exc)})
+ return FileWriteOutcome(blocked_reason=FileBlockedReason.CHANGED_DURING_RUN, blocked_detail=str(exc))
except OSError as exc:
_discard_backup(backup=backup, snapshot=snapshot, new_content=new_content)
- return plan.model_copy(
- update={"blocked_reason": FileBlockedReason.UNWRITABLE, "blocked_detail": f"the file could not be written: {exc.strerror or exc}"}
- )
+ return FileWriteOutcome(blocked_reason=FileBlockedReason.UNWRITABLE, blocked_detail=f"the file could not be written: {exc.strerror or exc}")
except FixTransactionError as exc:
# For the single-file commit this runner performs, a replace that fails re-raises its own
# `OSError` or `FixWriteConflictError` — a rollback of nothing is trivially complete — so
@@ -200,18 +224,60 @@ def _write_migrated_file(*, plan: MigrationPlan, snapshot: FileSnapshot, new_con
# still what landed is the open question, and the file answers it.
if not _carries(path=snapshot.path, content=new_content):
kept = _keep_the_original(backup=backup, path=snapshot.path, moment=moment)
- return plan.model_copy(
- update={
- "blocked_reason": FileBlockedReason.STATE_UNCERTAIN,
- "blocked_detail": (
- f"the write could not be confirmed: the file does not hold what this run wrote, and the transaction could not "
- f"say what it left behind — {_whereabouts_of(kept=kept)}: {exc}"
- ),
- "backup_path": kept.path,
- }
+ return FileWriteOutcome(
+ blocked_reason=FileBlockedReason.STATE_UNCERTAIN,
+ blocked_detail=(
+ f"the write could not be confirmed: the file does not hold what this run wrote, and the transaction could not "
+ f"say what it left behind — {_whereabouts_of(kept=kept)}: {exc}"
+ ),
+ backup_path=kept.path,
)
log.warning(f"'{snapshot.path}' was migrated, but the write left something behind: {exc}")
+ _prune_older_backups(snapshot=snapshot, backup=backup)
+ return FileWriteOutcome(backup_path=backup.path, was_written=True)
+
+
+def remove_file_with_backup(*, snapshot: FileSnapshot, moment: datetime) -> FileWriteOutcome:
+ """Back the file up, remove it, then prune the older backups — the removal twin of `write_file_with_backup`.
+
+ The copy is taken first and kept, so a removed file is one rename away from being back. The file is removed only
+ if it is still exactly what was read: a file changed or removed in between is someone else's work, reported and
+ left alone. The copy this run took of a file still on disk is taken back; the copy of a file that is gone stays,
+ since a concurrent run may have removed it after adopting that copy as its own (`_discard_created_backup`). The
+ path itself is removed, so a symbolic link goes and the file it names, someone's dotfiles, stays.
+
+ Like the write, nothing raised here crosses the per-file boundary.
+ """
+ try:
+ backup = write_backup(snapshot=snapshot, moment=moment)
+ except OSError as exc:
+ return FileWriteOutcome(blocked_reason=FileBlockedReason.UNWRITABLE, blocked_detail=f"the backup could not be written: {exc.strerror or exc}")
+
+ try:
+ assert_snapshot_unchanged(snapshot)
+ snapshot.path.unlink()
+ except FixWriteConflictError as exc:
+ kept = _discard_created_backup(backup=backup, snapshot=snapshot)
+ return FileWriteOutcome(blocked_reason=FileBlockedReason.CHANGED_DURING_RUN, blocked_detail=str(exc), backup_path=kept)
+ except FileNotFoundError:
+ kept = _discard_created_backup(backup=backup, snapshot=snapshot)
+ return FileWriteOutcome(
+ blocked_reason=FileBlockedReason.CHANGED_DURING_RUN,
+ blocked_detail="the file was removed while the cleanup was running",
+ backup_path=kept,
+ )
+ except OSError as exc:
+ kept = _discard_created_backup(backup=backup, snapshot=snapshot)
+ return FileWriteOutcome(
+ blocked_reason=FileBlockedReason.UNWRITABLE, blocked_detail=f"the file could not be removed: {exc.strerror or exc}", backup_path=kept
+ )
+
+ _prune_older_backups(snapshot=snapshot, backup=backup)
+ return FileWriteOutcome(backup_path=backup.path, was_written=True)
+
+
+def _prune_older_backups(*, snapshot: FileSnapshot, backup: WrittenBackup) -> None:
try:
prune_backups_except(path=snapshot.path, keep=backup.path)
except OSError as exc:
@@ -220,7 +286,29 @@ def _write_migrated_file(*, plan: MigrationPlan, snapshot: FileSnapshot, new_con
log.warning(
f"'{snapshot.path}' was migrated and backed up to '{backup.path}', but an older backup could not be pruned: {exc.strerror or exc}"
)
- return plan.model_copy(update={"backup_path": backup.path, "was_written": True})
+
+
+def _discard_created_backup(*, backup: WrittenBackup, snapshot: FileSnapshot) -> Path | None:
+ """Remove the copy this run made of a file it then did not remove, unless it may be the last copy of that file.
+
+ The removal's `_discard_backup`, asking the file the same question. Another run's copy is that run's, and stays. A
+ file still on disk was not removed by anyone — a user edited it, or it would not go — so this run's copy backs
+ up nothing and goes. A file no longer on disk may have been removed by a concurrent run of the same second, which
+ found this copy's name taken and adopted it as its own: that copy is then the only one of the file anywhere, named
+ by a report that is already out, so it stays.
+
+ Returns:
+ The copy that stays and is worth naming, or `None` when there is none.
+ """
+ if not backup.was_created:
+ return None
+ if not os.path.lexists(snapshot.path):
+ return backup.path
+ try:
+ backup.path.unlink(missing_ok=True)
+ except OSError as exc:
+ log.warning(f"the backup '{backup.path}' was made for a removal that did not happen and could not be removed: {exc.strerror or exc}")
+ return None
def _keep_the_original(*, backup: WrittenBackup, path: Path, moment: datetime) -> RescuedBackup:
@@ -300,6 +388,7 @@ def migrate_directories(
config_dirs: list[Path],
dry_run: bool,
only_surface_id: str | None = None,
+ skipped_paths: Collection[Path] = (),
moment: datetime | None = None,
) -> MigrationReport:
"""Migrate every claimed file in every given configuration directory.
@@ -307,6 +396,10 @@ def migrate_directories(
The walk is over the directories a caller names — in practice the global `~/.pipelex/` and the
project's `.pipelex/`, and only those. A directory that does not exist is skipped.
+ `skipped_paths` are files the walk leaves out altogether: the ones the former-release cleanup
+ removes, which `pipelex migrate` runs first. A file about to be deleted is neither counted among
+ the files to migrate nor reported as needing a look, in the rehearsal or in the write pass.
+
`only_surface_id` narrows the answer to one surface, for a caller that is diagnosing a
particular model's refusal rather than migrating the machine. **It narrows the result, not the
registry, and that distinction is the whole point**: which surface owns a file is decided
@@ -315,11 +408,19 @@ def migrate_directories(
it. Arbitration first, then the filter.
"""
stamp = moment if moment is not None else datetime.now(UTC)
+ skipped = {_walk_identity(path=path) for path in skipped_paths}
plans: list[MigrationPlan] = []
for directory in config_dirs:
for surface, file_path in registry.files_by_surface_in_directory(directory=directory):
if only_surface_id is not None and surface.surface_id != only_surface_id:
continue
+ if _walk_identity(path=file_path) in skipped:
+ continue
ledger = load_ledger_cached(migration_dir=migration_dir, surface_id=surface.surface_id)
plans.append(migrate_file(surface=surface, ledger=ledger, file_path=file_path, dry_run=dry_run, moment=stamp))
return MigrationReport(plans=plans)
+
+
+def _walk_identity(*, path: Path) -> Path:
+ """A file as the walk compares it: its directory resolved and its own name kept, so a link is the link itself."""
+ return path.parent.resolve() / path.name
diff --git a/pipelex/migration/surfaces.py b/pipelex/migration/surfaces.py
index 81ce55a0f..f0f7c29ce 100644
--- a/pipelex/migration/surfaces.py
+++ b/pipelex/migration/surfaces.py
@@ -26,6 +26,7 @@ from pipelex.cogt.model_backends.model_spec_document import (
describe_model_spec_document_rejection,
)
from pipelex.cogt.model_backends.model_spec_keys import is_header_shaped, is_legal_header_name
+from pipelex.kit.paths import RETIRED_SERVICE_FILE_NAME
from pipelex.migration.exceptions import MigrationRegistryError
from pipelex.migration.fingerprint import SurfaceFingerprint, compute_fingerprint
from pipelex.suggested_fix import WILDCARD_SEGMENT
@@ -45,7 +46,7 @@ PIPELEX_CONFIG_FILE_NAME = "pipelex.toml"
# installation may still hold one, and it is inert: nothing reads it. It is claimed by no surface,
# which matters because a retired name can still match a live surface's glob — `pipelex_service.toml`
# is a `pipelex_*.toml` — and that surface's ledger must not be replayed over a document it never saw.
-RETIRED_CONFIG_FILE_NAMES: frozenset[str] = frozenset({"pipelex_service.toml"})
+RETIRED_CONFIG_FILE_NAMES: frozenset[str] = frozenset({RETIRED_SERVICE_FILE_NAME})
# The package directory, `pipelex/` — this module sits one level under it.
_PACKAGE_ROOT = Path(__file__).resolve().parent.parent
diff --git a/pipelex/urls.py b/pipelex/urls.py
index 78f5d129a..009d4d570 100644
--- a/pipelex/urls.py
+++ b/pipelex/urls.py
@@ -12,7 +12,7 @@ class URLs:
pipe_func_docs = "https://docs.pipelex.com/latest/building-methods/pipes/pipe-operators/PipeFunc/"
backend_provider_docs = "https://docs.pipelex.com/latest/setup/configure-ai-providers/"
native_concepts_docs = "https://docs.pipelex.com/latest/building-methods/concepts/native-concepts/"
- app_cli_auth = "https://app.pipelex.com/auth/cli"
+ app = "https://app.pipelex.com"
# Base for the RFC 7807 ``type`` URI of every PipelexError class. A stable
# identifier by spec — kept as a constant so PipelexError.type_uri() stays
diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txt
index 248e01489..f2c79d69e 100644
--- a/tests/data/errors/error_identity.txt
+++ b/tests/data/errors/error_identity.txt
@@ -103,6 +103,7 @@ FatalError | Fatal error | https://docs.pipelex.com/latest/errors/fatal-error/
FileTypeError | File type | https://docs.pipelex.com/latest/errors/file-type-error/
FixTransactionError | Fix transaction | https://docs.pipelex.com/latest/errors/fix-transaction-error/
FixWriteConflictError | Fix write conflict | https://docs.pipelex.com/latest/errors/fix-write-conflict-error/
+FormerReleaseConfigError | Configuration left by a former release | https://docs.pipelex.com/latest/errors/former-release-config-error/
FuncRegistryError | Func registry | https://docs.pipelex.com/latest/errors/func-registry-error/
GcpLogSinkCredentialsError | Log sink credentials missing or refused | https://docs.pipelex.com/latest/errors/gcp-log-sink-credentials-error/
GeneratedImageError | Generated image | https://docs.pipelex.com/latest/errors/generated-image-error/
diff --git a/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends.toml b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends.toml
new file mode 100644
index 000000000..1946abc2f
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends.toml
@@ -0,0 +1,29 @@
+# The hosted plane's inference backends: Pipelex Manifold and pipelex's internal backend, nothing else.
+#
+# Hand-maintained on purpose. `pipelex init inference` would write every provider the kit knows, and
+# the hosted plane uses none of them: every model call goes to Pipelex Manifold, the in-VPC inference
+# gateway, and a declared backend is one more thing that can demand a credential at boot. There is no
+# `backends_override.toml` beside this file; this is the whole document.
+#
+# `[pipelex_manifold]` is ours alone: pipelex's kit stopped shipping a block for it in 0.73.0, so this
+# declaration is the only one there is. pipelex reads its model specs from
+# `backends/pipelex_manifold.toml`, part of the model registry's runtime slice: the catalog
+# `models/catalog.toml` without the gateway's two keys, written beside this file by the image build
+# and by `make models` (models/README.md), and gitignored here. Those specs carry no `model_id`, so
+# the model the runtime sends is the handle, and Manifold maps it to the provider. The endpoint (the
+# gateway's origin, no `/v1`) and the service token arrive as the PIPELEX_MANIFOLD_ENDPOINT and
+# PIPELEX_MANIFOLD_API_KEY task env vars.
+#
+# `[internal]` is not an AI provider: it is pipelex's software-only backend, serving the PDF text
+# extractors (`pypdfium2-extract-pdf`, `docling-extract-text`) that the deck's `default-text-from-pdf`
+# and `default-no-inference` presets name. It needs no credential and makes no network call.
+
+[pipelex_manifold]
+display_name = "Pipelex Manifold"
+enabled = true
+endpoint = "${PIPELEX_MANIFOLD_ENDPOINT}"
+api_key = "${PIPELEX_MANIFOLD_API_KEY}"
+auth_header = "x-pipelex-api-key"
+
+[internal] # software-only backend, runs internally, without AI
+enabled = true
diff --git a/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends/internal.toml b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends/internal.toml
new file mode 100644
index 000000000..e5444f72b
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends/internal.toml
@@ -0,0 +1,80 @@
+################################################################################
+# Internal Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for internal software-only models.
+# These models run internally without external APIs or AI services.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Model costs are in USD per million tokens (input/output)
+#
+# Managed by `pipelex update`, which refreshes this file from the kit and backs
+# up a locally edited copy to <file>.bak.<timestamp> first, so local edits will
+# not survive future runs. Declare your own models in a backend of your own.
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+thinking_mode = "none"
+
+################################################################################
+# TEXT EXTRACTION MODELS
+################################################################################
+
+# --- PyPDFium2 Text Extractor -------------------------------------------------
+[pypdfium2-extract-pdf]
+model_type = "text_extractor"
+sdk = "pypdfium2"
+model_id = "extract-text"
+inputs = ["pdf"]
+outputs = ["pages"]
+costs = {}
+
+# --- Docling Text Extractor ---------------------------------------------------
+[docling-extract-text]
+model_type = "text_extractor"
+sdk = "docling_sdk"
+model_id = "extract-text"
+inputs = [
+ "pdf",
+ "docx",
+ "pptx",
+ "xlsx",
+ "html",
+ "md",
+ "csv",
+ "txt",
+ "vtt",
+ "eml",
+ "image",
+]
+outputs = ["pages"]
+costs = {}
+
+################################################################################
+# DOCUMENT GENERATION ENGINES
+################################################################################
+#
+# The engines a PipeDocGen step prints with. Their inputs are the sources they
+# print from: "layout" is the auto-layout of the step's inputs, "html" an HTML
+# template, "template_file" an office template file. Their outputs are the
+# formats they print. reportlab-pdf is built into Pipelex. The other engines
+# come with the Pipelex document generation plugin, pipelex-doc-gen, which
+# declares them itself when it loads: they are never listed in this file.
+
+# --- ReportLab PDF ------------------------------------------------------------
+[reportlab-pdf]
+model_type = "doc_gen"
+sdk = "reportlab"
+model_id = "print-pdf"
+inputs = ["layout"]
+outputs = ["pdf"]
+costs = {}
diff --git a/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends/pipelex_manifold.toml b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends/pipelex_manifold.toml
new file mode 100644
index 000000000..a5c7b7ad8
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/backends/pipelex_manifold.toml
@@ -0,0 +1,34 @@
+################################################################################
+# GENERATED FROM pipelex-server's models/ BY model-registry. DO NOT EDIT.
+#
+# Pipelex Manifold's model specs: the catalog without the gateway's and the registry's keys.
+# Registry version 40a5dcbb3d810817, built from a working tree.
+# Edit models/ and run `make models` at the repository root.
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "manifold_completions"
+structure_method = "instructor/openai_tools"
+
+[gpt-4o-mini]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+sdk = "manifold_responses"
+structure_method = "instructor/openai_responses_tools"
+thinking_mode = "none"
+
+[gpt-4o-mini.costs]
+input = 0.15
+output = 0.6
+
+[gpt-4o]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+sdk = "manifold_responses"
+structure_method = "instructor/openai_responses_tools"
+thinking_mode = "none"
+
+[gpt-4o.costs]
+input = 2.5
+output = 10.0
diff --git a/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/routing_profiles.toml b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/routing_profiles.toml
new file mode 100644
index 000000000..37ac629b8
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/api-hosted/inference/routing_profiles.toml
@@ -0,0 +1,16 @@
+# The hosted plane's routing: every model goes to Pipelex Manifold.
+#
+# Hand-maintained on purpose, beside a `backends.toml` that declares only `pipelex_manifold` and
+# `internal`. There is no `routing_profiles_override.toml`; this is the whole document, and the
+# decision is baked into the image: rolling back is redeploying the previous image tag.
+#
+# The profile names a default backend and no fallback order, so a model Manifold does not serve fails
+# rather than reaching some other backend. The internal backend needs no profile of its own: when the
+# default backend has no spec for a model and no fallback order is set, pipelex looks in `internal`,
+# which is how the PDF text extractors resolve.
+
+active = "all_pipelex_manifold"
+
+[profiles.all_pipelex_manifold]
+description = "Use Pipelex Manifold for all its supported models"
+default = "pipelex_manifold"
diff --git a/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends.toml b/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends.toml
new file mode 100644
index 000000000..38b626f27
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends.toml
@@ -0,0 +1,31 @@
+# This suite's inference backends, the same as the hosted worker and runner ship (worker/.pipelex/inference):
+# Pipelex Manifold and pipelex's internal backend, nothing else, so the suite resolves models through the
+# catalog they run on.
+#
+# Hand-maintained on purpose. `pipelex init inference` would write every provider the kit knows, and
+# the hosted plane uses none of them: every model call goes to Pipelex Manifold, the in-VPC inference
+# gateway, and a declared backend is one more thing that can demand a credential at boot. There is no
+# `backends_override.toml` beside this file; this is the whole document.
+#
+# `[pipelex_manifold]` is ours alone: pipelex's kit stopped shipping a block for it in 0.73.0, so this
+# declaration is the only one there is. pipelex reads its model specs from
+# `backends/pipelex_manifold.toml`, part of the model registry's runtime slice: the catalog
+# `models/catalog.toml` without the gateway's two keys, written beside this file by this suite's
+# rootdir conftest and by `make models` (models/README.md), and gitignored here. Those specs carry no
+# `model_id`, so the model the runtime sends is the handle, and Manifold maps it to the provider. The
+# endpoint (the gateway's origin, no `/v1`) and the service token arrive as the
+# PIPELEX_MANIFOLD_ENDPOINT and PIPELEX_MANIFOLD_API_KEY task env vars.
+#
+# `[internal]` is not an AI provider: it is pipelex's software-only backend, serving the PDF text
+# extractors (`pypdfium2-extract-pdf`, `docling-extract-text`) that the deck's `default-text-from-pdf`
+# and `default-no-inference` presets name. It needs no credential and makes no network call.
+
+[pipelex_manifold]
+display_name = "Pipelex Manifold"
+enabled = true
+endpoint = "${PIPELEX_MANIFOLD_ENDPOINT}"
+api_key = "${PIPELEX_MANIFOLD_API_KEY}"
+auth_header = "x-pipelex-api-key"
+
+[internal] # software-only backend, runs internally, without AI
+enabled = true
diff --git a/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends/internal.toml b/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends/internal.toml
new file mode 100644
index 000000000..e5444f72b
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends/internal.toml
@@ -0,0 +1,80 @@
+################################################################################
+# Internal Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for internal software-only models.
+# These models run internally without external APIs or AI services.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Model costs are in USD per million tokens (input/output)
+#
+# Managed by `pipelex update`, which refreshes this file from the kit and backs
+# up a locally edited copy to <file>.bak.<timestamp> first, so local edits will
+# not survive future runs. Declare your own models in a backend of your own.
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+thinking_mode = "none"
+
+################################################################################
+# TEXT EXTRACTION MODELS
+################################################################################
+
+# --- PyPDFium2 Text Extractor -------------------------------------------------
+[pypdfium2-extract-pdf]
+model_type = "text_extractor"
+sdk = "pypdfium2"
+model_id = "extract-text"
+inputs = ["pdf"]
+outputs = ["pages"]
+costs = {}
+
+# --- Docling Text Extractor ---------------------------------------------------
+[docling-extract-text]
+model_type = "text_extractor"
+sdk = "docling_sdk"
+model_id = "extract-text"
+inputs = [
+ "pdf",
+ "docx",
+ "pptx",
+ "xlsx",
+ "html",
+ "md",
+ "csv",
+ "txt",
+ "vtt",
+ "eml",
+ "image",
+]
+outputs = ["pages"]
+costs = {}
+
+################################################################################
+# DOCUMENT GENERATION ENGINES
+################################################################################
+#
+# The engines a PipeDocGen step prints with. Their inputs are the sources they
+# print from: "layout" is the auto-layout of the step's inputs, "html" an HTML
+# template, "template_file" an office template file. Their outputs are the
+# formats they print. reportlab-pdf is built into Pipelex. The other engines
+# come with the Pipelex document generation plugin, pipelex-doc-gen, which
+# declares them itself when it loads: they are never listed in this file.
+
+# --- ReportLab PDF ------------------------------------------------------------
+[reportlab-pdf]
+model_type = "doc_gen"
+sdk = "reportlab"
+model_id = "print-pdf"
+inputs = ["layout"]
+outputs = ["pdf"]
+costs = {}
diff --git a/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends/pipelex_manifold.toml b/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends/pipelex_manifold.toml
new file mode 100644
index 000000000..a5c7b7ad8
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/temporal/inference/backends/pipelex_manifold.toml
@@ -0,0 +1,34 @@
+################################################################################
+# GENERATED FROM pipelex-server's models/ BY model-registry. DO NOT EDIT.
+#
+# Pipelex Manifold's model specs: the catalog without the gateway's and the registry's keys.
+# Registry version 40a5dcbb3d810817, built from a working tree.
+# Edit models/ and run `make models` at the repository root.
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "manifold_completions"
+structure_method = "instructor/openai_tools"
+
+[gpt-4o-mini]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+sdk = "manifold_responses"
+structure_method = "instructor/openai_responses_tools"
+thinking_mode = "none"
+
+[gpt-4o-mini.costs]
+input = 0.15
+output = 0.6
+
+[gpt-4o]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+sdk = "manifold_responses"
+structure_method = "instructor/openai_responses_tools"
+thinking_mode = "none"
+
+[gpt-4o.costs]
+input = 2.5
+output = 10.0
diff --git a/tests/data/migration/former_release/hosted_manifold/temporal/inference/routing_profiles.toml b/tests/data/migration/former_release/hosted_manifold/temporal/inference/routing_profiles.toml
new file mode 100644
index 000000000..07fc2b94d
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/temporal/inference/routing_profiles.toml
@@ -0,0 +1,15 @@
+# This suite's routing, the same the hosted worker and runner ship with: every model goes to Pipelex Manifold.
+#
+# Hand-maintained on purpose, beside a `backends.toml` that declares only `pipelex_manifold` and
+# `internal`. There is no `routing_profiles_override.toml`; this is the whole document.
+#
+# The profile names a default backend and no fallback order, so a model Manifold does not serve fails
+# rather than reaching some other backend. The internal backend needs no profile of its own: when the
+# default backend has no spec for a model and no fallback order is set, pipelex looks in `internal`,
+# which is how the PDF text extractors resolve.
+
+active = "all_pipelex_manifold"
+
+[profiles.all_pipelex_manifold]
+description = "Use Pipelex Manifold for all its supported models"
+default = "pipelex_manifold"
diff --git a/tests/data/migration/former_release/hosted_manifold/worker/inference/backends.toml b/tests/data/migration/former_release/hosted_manifold/worker/inference/backends.toml
new file mode 100644
index 000000000..1946abc2f
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/worker/inference/backends.toml
@@ -0,0 +1,29 @@
+# The hosted plane's inference backends: Pipelex Manifold and pipelex's internal backend, nothing else.
+#
+# Hand-maintained on purpose. `pipelex init inference` would write every provider the kit knows, and
+# the hosted plane uses none of them: every model call goes to Pipelex Manifold, the in-VPC inference
+# gateway, and a declared backend is one more thing that can demand a credential at boot. There is no
+# `backends_override.toml` beside this file; this is the whole document.
+#
+# `[pipelex_manifold]` is ours alone: pipelex's kit stopped shipping a block for it in 0.73.0, so this
+# declaration is the only one there is. pipelex reads its model specs from
+# `backends/pipelex_manifold.toml`, part of the model registry's runtime slice: the catalog
+# `models/catalog.toml` without the gateway's two keys, written beside this file by the image build
+# and by `make models` (models/README.md), and gitignored here. Those specs carry no `model_id`, so
+# the model the runtime sends is the handle, and Manifold maps it to the provider. The endpoint (the
+# gateway's origin, no `/v1`) and the service token arrive as the PIPELEX_MANIFOLD_ENDPOINT and
+# PIPELEX_MANIFOLD_API_KEY task env vars.
+#
+# `[internal]` is not an AI provider: it is pipelex's software-only backend, serving the PDF text
+# extractors (`pypdfium2-extract-pdf`, `docling-extract-text`) that the deck's `default-text-from-pdf`
+# and `default-no-inference` presets name. It needs no credential and makes no network call.
+
+[pipelex_manifold]
+display_name = "Pipelex Manifold"
+enabled = true
+endpoint = "${PIPELEX_MANIFOLD_ENDPOINT}"
+api_key = "${PIPELEX_MANIFOLD_API_KEY}"
+auth_header = "x-pipelex-api-key"
+
+[internal] # software-only backend, runs internally, without AI
+enabled = true
diff --git a/tests/data/migration/former_release/hosted_manifold/worker/inference/backends/internal.toml b/tests/data/migration/former_release/hosted_manifold/worker/inference/backends/internal.toml
new file mode 100644
index 000000000..e5444f72b
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/worker/inference/backends/internal.toml
@@ -0,0 +1,80 @@
+################################################################################
+# Internal Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for internal software-only models.
+# These models run internally without external APIs or AI services.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Model costs are in USD per million tokens (input/output)
+#
+# Managed by `pipelex update`, which refreshes this file from the kit and backs
+# up a locally edited copy to <file>.bak.<timestamp> first, so local edits will
+# not survive future runs. Declare your own models in a backend of your own.
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+thinking_mode = "none"
+
+################################################################################
+# TEXT EXTRACTION MODELS
+################################################################################
+
+# --- PyPDFium2 Text Extractor -------------------------------------------------
+[pypdfium2-extract-pdf]
+model_type = "text_extractor"
+sdk = "pypdfium2"
+model_id = "extract-text"
+inputs = ["pdf"]
+outputs = ["pages"]
+costs = {}
+
+# --- Docling Text Extractor ---------------------------------------------------
+[docling-extract-text]
+model_type = "text_extractor"
+sdk = "docling_sdk"
+model_id = "extract-text"
+inputs = [
+ "pdf",
+ "docx",
+ "pptx",
+ "xlsx",
+ "html",
+ "md",
+ "csv",
+ "txt",
+ "vtt",
+ "eml",
+ "image",
+]
+outputs = ["pages"]
+costs = {}
+
+################################################################################
+# DOCUMENT GENERATION ENGINES
+################################################################################
+#
+# The engines a PipeDocGen step prints with. Their inputs are the sources they
+# print from: "layout" is the auto-layout of the step's inputs, "html" an HTML
+# template, "template_file" an office template file. Their outputs are the
+# formats they print. reportlab-pdf is built into Pipelex. The other engines
+# come with the Pipelex document generation plugin, pipelex-doc-gen, which
+# declares them itself when it loads: they are never listed in this file.
+
+# --- ReportLab PDF ------------------------------------------------------------
+[reportlab-pdf]
+model_type = "doc_gen"
+sdk = "reportlab"
+model_id = "print-pdf"
+inputs = ["layout"]
+outputs = ["pdf"]
+costs = {}
diff --git a/tests/data/migration/former_release/hosted_manifold/worker/inference/backends/pipelex_manifold.toml b/tests/data/migration/former_release/hosted_manifold/worker/inference/backends/pipelex_manifold.toml
new file mode 100644
index 000000000..a5c7b7ad8
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/worker/inference/backends/pipelex_manifold.toml
@@ -0,0 +1,34 @@
+################################################################################
+# GENERATED FROM pipelex-server's models/ BY model-registry. DO NOT EDIT.
+#
+# Pipelex Manifold's model specs: the catalog without the gateway's and the registry's keys.
+# Registry version 40a5dcbb3d810817, built from a working tree.
+# Edit models/ and run `make models` at the repository root.
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "manifold_completions"
+structure_method = "instructor/openai_tools"
+
+[gpt-4o-mini]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+sdk = "manifold_responses"
+structure_method = "instructor/openai_responses_tools"
+thinking_mode = "none"
+
+[gpt-4o-mini.costs]
+input = 0.15
+output = 0.6
+
+[gpt-4o]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+sdk = "manifold_responses"
+structure_method = "instructor/openai_responses_tools"
+thinking_mode = "none"
+
+[gpt-4o.costs]
+input = 2.5
+output = 10.0
diff --git a/tests/data/migration/former_release/hosted_manifold/worker/inference/routing_profiles.toml b/tests/data/migration/former_release/hosted_manifold/worker/inference/routing_profiles.toml
new file mode 100644
index 000000000..37ac629b8
--- /dev/null
+++ b/tests/data/migration/former_release/hosted_manifold/worker/inference/routing_profiles.toml
@@ -0,0 +1,16 @@
+# The hosted plane's routing: every model goes to Pipelex Manifold.
+#
+# Hand-maintained on purpose, beside a `backends.toml` that declares only `pipelex_manifold` and
+# `internal`. There is no `routing_profiles_override.toml`; this is the whole document, and the
+# decision is baked into the image: rolling back is redeploying the previous image tag.
+#
+# The profile names a default backend and no fallback order, so a model Manifold does not serve fails
+# rather than reaching some other backend. The internal backend needs no profile of its own: when the
+# default backend has no spec for a model and no fallback order is set, pipelex looks in `internal`,
+# which is how the PDF text extractors resolve.
+
+active = "all_pipelex_manifold"
+
+[profiles.all_pipelex_manifold]
+description = "Use Pipelex Manifold for all its supported models"
+default = "pipelex_manifold"
diff --git a/tests/data/migration/former_release/v0_72/inference/backends.toml b/tests/data/migration/former_release/v0_72/inference/backends.toml
new file mode 100644
index 000000000..28a5a7777
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/inference/backends.toml
@@ -0,0 +1,132 @@
+####################################################################################################
+# Pipelex Inference Backends Configuration
+####################################################################################################
+#
+# This file configures the inference backends available to Pipelex.
+# Each backend connects to a different AI service provider (OpenAI, Anthropic, Google, etc.).
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+####################################################################################################
+
+[pipelex_gateway]
+display_name = "⭐ Pipelex Gateway"
+enabled = true # Enable after accepting terms via `pipelex init config`
+api_key = "${PIPELEX_GATEWAY_API_KEY}"
+
+# Pipelex Manifold is a closed private beta: leave it off unless you have been invited.
+[pipelex_manifold]
+display_name = "Pipelex Manifold (private beta)"
+enabled = false
+endpoint = "${PIPELEX_MANIFOLD_ENDPOINT}"
+api_key = "${PIPELEX_MANIFOLD_API_KEY}"
+model_specs_section = "manifold_model_specs"
+auth_header = "x-pipelex-api-key"
+
+[anthropic]
+enabled = true
+api_key = "${ANTHROPIC_API_KEY}"
+
+[azure_openai]
+display_name = "Azure OpenAI"
+enabled = true
+endpoint = "${AZURE_API_BASE}"
+api_key = "${AZURE_API_KEY}"
+api_version = "${AZURE_API_VERSION}"
+
+[bedrock]
+display_name = "Amazon Bedrock"
+enabled = true
+aws_region = "${AWS_REGION}"
+
+[blackboxai]
+display_name = "BlackBox AI"
+enabled = true
+endpoint = "https://api.blackbox.ai/v1"
+api_key = "${BLACKBOX_API_KEY}"
+
+[fal]
+display_name = "FAL"
+enabled = true
+api_key = "${FAL_API_KEY}"
+
+[google]
+display_name = "Google AI"
+enabled = true
+api_key = "${GOOGLE_API_KEY}"
+
+[groq]
+display_name = "Groq"
+enabled = true
+endpoint = "https://api.groq.com/openai/v1"
+api_key = "${GROQ_API_KEY}"
+
+[huggingface]
+display_name = "Hugging Face"
+enabled = true
+api_key = "${HF_TOKEN}"
+
+[linkup]
+display_name = "Linkup"
+enabled = true
+api_key = "${LINKUP_API_KEY}"
+
+[minimax]
+display_name = "MiniMax"
+enabled = true
+endpoint = "https://api.minimax.io/anthropic"
+api_key = "${MINIMAX_API_KEY}"
+
+[mistral]
+display_name = "Mistral AI"
+enabled = true
+api_key = "${MISTRAL_API_KEY}"
+
+[ollama]
+enabled = true
+endpoint = "http://localhost:11434/v1"
+
+[openai]
+display_name = "OpenAI"
+enabled = true
+api_key = "${OPENAI_API_KEY}"
+
+[openrouter]
+display_name = "OpenRouter"
+enabled = true
+endpoint = "https://openrouter.ai/api/v1"
+api_key = "${OPENROUTER_API_KEY}"
+
+[portkey]
+display_name = "Portkey"
+enabled = true
+endpoint = "https://api.portkey.ai/v1"
+api_key = "${PORTKEY_API_KEY}"
+
+[scaleway]
+display_name = "Scaleway"
+enabled = true
+endpoint = "${SCALEWAY_ENDPOINT}"
+api_key = "${SCALEWAY_API_KEY}"
+
+[typesafe]
+display_name = "TypeSafe"
+enabled = true
+api_key = "${TYPESAFE_API_KEY}"
+
+[vertexai]
+display_name = "Google Vertex AI"
+enabled = false # This is the only one we disable because setting it up requires internet access just to get credentials so it fails in CI sandboxes
+gcp_project_id = "${GCP_PROJECT_ID}"
+gcp_location = "${GCP_LOCATION}"
+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+
+[xai]
+display_name = "xAI"
+enabled = true
+endpoint = "https://api.x.ai/v1"
+api_key = "${XAI_API_KEY}"
+
+[internal] # software-only backend, runs internally, without AI
+enabled = true
diff --git a/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway.toml b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway.toml
new file mode 100644
index 000000000..5b02901ee
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway.toml
@@ -0,0 +1,41 @@
+################################################################################
+# Pipelex Gateway Local Overrides
+################################################################################
+#
+# TELEMETRY NOTICE:
+#
+# Using Pipelex Gateway enables identified telemetry tied to your API key
+# (hashed for security). This is independent from your telemetry.toml settings.
+#
+# We collect only technical data (model names, token counts, latency, error rates).
+# We do NOT collect prompts, completions, pipe codes, or business data.
+#
+# This allows us to monitor service quality, enforce fair usage, and support you.
+#
+################################################################################
+#
+# WARNING: USE AT YOUR OWN RISK!
+#
+# The actual model configuration is fetched remotely from Pipelex servers.
+# Any override in this file may cause unexpected behavior or failures,
+# as the remote configuration may change at any time.
+#
+# If you must override, you may ONLY use these keys per model:
+# - sdk
+# - structure_method
+#
+# All other keys will be ignored.
+#
+# If you need custom configurations, consider using your own API keys
+# with direct provider backends (openai, anthropic, etc.) instead.
+#
+# Documentation:
+# https://docs.pipelex.com/latest/configuration/config-technical/inference-backend-config/
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+# Per-model overrides example:
+# [gpt-4o]
+# sdk = "gateway_completions"
+# structure_method = "instructor/openai_tools"
diff --git a/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway_models.md b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway_models.md
new file mode 100644
index 000000000..51d58d05b
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway_models.md
@@ -0,0 +1,3 @@
+# Pipelex Gateway — Available Models
+This file lists the LLMs, document extraction models, and image generation models currently available through Pipelex Gateway.
+For configuration details, see the [documentation](https://docs.pipelex.com/latest/setup/configure-ai-providers/#option-1-pipelex-gateway-easiest-and-most-powerful-for-getting-started).
diff --git a/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway_models_plain.md b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway_models_plain.md
new file mode 100644
index 000000000..1deca8d33
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_gateway_models_plain.md
@@ -0,0 +1,5 @@
+# Pipelex Gateway — Available Models (Plain Text)
+
+This file lists the LLMs, document extraction models, and image generation models currently available through Pipelex Gateway.
+For configuration details, see the [documentation](https://docs.pipelex.com/latest/setup/configure-ai-providers/#option-1-pipelex-gateway-easiest-and-most-powerful-for-getting-started).
+
diff --git a/tests/data/migration/former_release/v0_72/inference/backends/pipelex_manifold.toml b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_manifold.toml
new file mode 100644
index 000000000..1a9adc718
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/inference/backends/pipelex_manifold.toml
@@ -0,0 +1,56 @@
+################################################################################
+# Pipelex Manifold Local Overrides
+################################################################################
+#
+# PRIVATE BETA:
+#
+# Pipelex Manifold is the Pipelex-operated inference service. It is in private
+# beta, it ships disabled in backends.toml, and this file does nothing until you
+# enable it there and set PIPELEX_MANIFOLD_ENDPOINT and PIPELEX_MANIFOLD_API_KEY.
+#
+################################################################################
+#
+# TELEMETRY NOTICE:
+#
+# Enabling pipelex_gateway alongside this backend enables identified telemetry
+# from your machine, tied to your Pipelex Gateway API key (hashed for security).
+# This is independent from your telemetry.toml settings.
+#
+# That client-side telemetry follows the gateway key and nothing else, so an
+# installation running Pipelex Manifold ALONE sends none of it. Manifold is a
+# Pipelex-operated service, though, so your usage of it is recorded server-side
+# the way any hosted platform records the calls it serves.
+#
+# Either way we collect only technical data (model names, token counts, latency,
+# error rates). We do NOT collect prompts, completions, pipe codes, or business
+# data.
+#
+# This allows us to monitor service quality, enforce fair usage, and support you.
+#
+################################################################################
+#
+# WARNING: USE AT YOUR OWN RISK!
+#
+# The actual model configuration is fetched remotely from Pipelex servers.
+# Any override in this file may cause unexpected behavior or failures,
+# as the remote configuration may change at any time.
+#
+# If you must override, you may ONLY use these keys per model:
+# - sdk
+# - structure_method
+#
+# All other keys will be ignored.
+#
+# If you need custom configurations, consider using your own API keys
+# with direct provider backends (openai, anthropic, etc.) instead.
+#
+# Documentation:
+# https://docs.pipelex.com/latest/configuration/config-technical/inference-backend-config/
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+# Per-model overrides example:
+# [gpt-4o]
+# sdk = "manifold_completions"
+# structure_method = "instructor/openai_tools"
diff --git a/tests/data/migration/former_release/v0_72/inference/routing_profiles.toml b/tests/data/migration/former_release/v0_72/inference/routing_profiles.toml
new file mode 100644
index 000000000..25619b1ee
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/inference/routing_profiles.toml
@@ -0,0 +1,182 @@
+# Routing profile library - Routes models to their backends
+# =========================================================================================
+# This file controls which backend serves which model.
+# Simply change the 'active' field to switch profiles,
+# or you can add your own custom profiles.
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+# =========================================================================================
+
+# Note: The internal backend (software-only models) is always available regardless of
+# which routing profile you select. See the documentation for details.
+
+# Which profile to use (change this to switch routing)
+active = "all_pipelex_gateway"
+
+# We recommend using the "all_pipelex_gateway" profile to get a head start with all models.
+# To use the Pipelex Gateway backend:
+# 1. Get your API key at https://app.pipelex.com (free credits included)
+# 2. Add it to your .env file: PIPELEX_GATEWAY_API_KEY=your-key-here
+# 3. Run `pipelex init` and accept the Gateway terms of service
+
+# =========================================================================================
+# Routing Profiles
+# =========================================================================================
+
+[profiles.all_pipelex_gateway]
+description = "Use Pipelex Gateway for all its supported models"
+default = "pipelex_gateway"
+
+# Judgment models are brought by you rather than served by Pipelex, so they go to their own backend
+# whenever it is enabled. An optional route is inert while its backend is disabled.
+[profiles.all_pipelex_gateway.optional_routes]
+"jev-*" = "typesafe"
+
+# The same model names, served by the Pipelex-operated inference service instead. Switching between
+# this profile and the one above needs no other edit: a model's wire id and sdk are per-backend, so
+# decks, aliases and presets keep naming the same handles.
+[profiles.all_pipelex_manifold]
+description = "Use Pipelex Manifold (private beta) for all its supported models"
+default = "pipelex_manifold"
+
+# Judgment models are brought by you rather than served by Pipelex, so they go to their own backend
+# whenever it is enabled. An optional route is inert while its backend is disabled.
+[profiles.all_pipelex_manifold.optional_routes]
+"jev-*" = "typesafe"
+
+[profiles.all_anthropic]
+description = "Use Anthropic backend for all its supported models"
+default = "anthropic"
+
+[profiles.all_azure_openai]
+description = "Use Azure OpenAI backend for all its supported models"
+default = "azure_openai"
+
+[profiles.all_bedrock]
+description = "Use Bedrock backend for all its supported models"
+default = "bedrock"
+
+[profiles.all_blackboxai]
+description = "Use BlackBoxAI backend for all its supported models"
+default = "blackboxai"
+
+[profiles.all_fal]
+description = "Use FAL backend for all its supported models"
+default = "fal"
+
+[profiles.all_google]
+description = "Use Google GenAI backend for all its supported models"
+default = "google"
+
+[profiles.all_groq]
+description = "Use groq backend for all its supported models"
+default = "groq"
+
+[profiles.all_huggingface]
+description = "Use HuggingFace backend for all its supported models"
+default = "huggingface"
+
+[profiles.all_linkup]
+description = "Use Linkup backend for all its supported models"
+default = "linkup"
+
+[profiles.all_minimax]
+description = "Use MiniMax backend for all its supported models"
+default = "minimax"
+
+[profiles.all_mistral]
+description = "Use Mistral backend for all its supported models"
+default = "mistral"
+
+[profiles.all_ollama]
+description = "Use Ollama backend for all its supported models"
+default = "ollama"
+
+[profiles.all_openai]
+description = "Use OpenAI backend for all its supported models"
+default = "openai"
+
+[profiles.all_openrouter]
+description = "Use OpenRouter backend for all its supported models"
+default = "openrouter"
+
+[profiles.all_portkey]
+description = "Use Portkey backend for all its supported models"
+default = "portkey"
+
+[profiles.all_scaleway]
+description = "Use Scaleway backend for all its supported models"
+default = "scaleway"
+
+[profiles.all_typesafe]
+description = "Use TypeSafe backend for all its supported models"
+default = "typesafe"
+
+[profiles.all_vertexai]
+description = "Use Vertex AI backend for all its supported models"
+default = "vertexai"
+
+[profiles.all_xai]
+description = "Use xAI backend for all its supported models"
+default = "xai"
+
+[profiles.all_internal]
+description = "Use internal backend for all its supported models"
+default = "internal"
+
+# =========================================================================================
+# Custom Profiles
+# =========================================================================================
+# Add your own profiles below following the same pattern:
+#
+# [profiles.your_profile_name]
+# description = "What this profile does"
+# default = "backend-name" # Where to route models by default
+# [profiles.your_profile_name.routes]
+# "model-pattern" = "backend-name" # Specific routing rules
+#
+# Pattern matching supports:
+# - Exact names: "gpt-4o-mini"
+# - Wildcards: "claude-*" (matches all models starting with claude-)
+# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
+
+# =========================================================================================
+# Example of a custom routing profile with mostly pattern matching and one specific model
+# =========================================================================================
+[profiles.example_routing_using_patterns]
+description = "Example routing profile using patterns"
+default = "pipelex_gateway"
+
+[profiles.example_routing_using_patterns.routes]
+# Pattern matching: "model-pattern" = "backend-name"
+"gpt-*" = "azure_openai"
+"claude-*" = "bedrock"
+"gemini-*" = "google"
+"grok-*" = "xai"
+"*-sdxl" = "fal"
+"flux-*" = "fal"
+"gpt-image-1" = "openai"
+
+# =========================================================================================
+# Example of a custom routing profile with specific model matching
+# =========================================================================================
+
+[profiles.example_routing_using_specific_models]
+description = "Example routing profile using specific models"
+
+[profiles.example_routing_using_specific_models.routes]
+"gpt-5.4-nano" = "pipelex_gateway"
+"gpt-4o-mini" = "blackboxai"
+"gpt-5.4-mini" = "openai"
+"gpt-5.5" = "azure_openai"
+
+"claude-4-sonnet" = "pipelex_gateway"
+"claude-3.7-sonnet" = "blackboxai"
+
+"gemini-2.5-flash-lite" = "pipelex_gateway"
+"gemini-2.5-flash" = "blackboxai"
+"gemini-2.5-pro" = "vertexai"
+
+"grok-3" = "pipelex_gateway"
+"grok-3-mini" = "xai"
diff --git a/tests/data/migration/former_release/v0_72/pipelex_service.toml b/tests/data/migration/former_release/v0_72/pipelex_service.toml
new file mode 100644
index 000000000..a12571dd1
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72/pipelex_service.toml
@@ -0,0 +1,23 @@
+####################################################################################################
+# Pipelex Service Configuration
+####################################################################################################
+#
+# This file stores settings related to Pipelex managed services.
+# Currently used for Pipelex Gateway terms acceptance.
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+####################################################################################################
+
+[agreement]
+# Set to true after accepting Pipelex terms of service.
+terms_accepted = false
+
+# Note: when using pipelex_gateway, telemetry is enabled to monitor service usage.
+# We collect technical data (model, pipe type...) and quantitative data (token counts...)
+# but NOT your content, pipe codes, or output class names.
+
+[onboarding]
+# Set to true after completing inference setup (gateway or BYOK).
+inference_setup_completed = false
diff --git a/tests/data/migration/former_release/v0_72_cleaned/inference/backends.toml b/tests/data/migration/former_release/v0_72_cleaned/inference/backends.toml
new file mode 100644
index 000000000..340269560
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72_cleaned/inference/backends.toml
@@ -0,0 +1,127 @@
+####################################################################################################
+# Pipelex Inference Backends Configuration
+####################################################################################################
+#
+# This file configures the inference backends available to Pipelex.
+# Each backend connects to a different AI service provider (OpenAI, Anthropic, Google, etc.).
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+####################################################################################################
+
+# Pipelex Manifold is a closed private beta: leave it off unless you have been invited.
+[pipelex_manifold]
+display_name = "Pipelex Manifold (private beta)"
+enabled = false
+endpoint = "${PIPELEX_MANIFOLD_ENDPOINT}"
+api_key = "${PIPELEX_MANIFOLD_API_KEY}"
+model_specs_section = "manifold_model_specs"
+auth_header = "x-pipelex-api-key"
+
+[anthropic]
+enabled = true
+api_key = "${ANTHROPIC_API_KEY}"
+
+[azure_openai]
+display_name = "Azure OpenAI"
+enabled = true
+endpoint = "${AZURE_API_BASE}"
+api_key = "${AZURE_API_KEY}"
+api_version = "${AZURE_API_VERSION}"
+
+[bedrock]
+display_name = "Amazon Bedrock"
+enabled = true
+aws_region = "${AWS_REGION}"
+
+[blackboxai]
+display_name = "BlackBox AI"
+enabled = true
+endpoint = "https://api.blackbox.ai/v1"
+api_key = "${BLACKBOX_API_KEY}"
+
+[fal]
+display_name = "FAL"
+enabled = true
+api_key = "${FAL_API_KEY}"
+
+[google]
+display_name = "Google AI"
+enabled = true
+api_key = "${GOOGLE_API_KEY}"
+
+[groq]
+display_name = "Groq"
+enabled = true
+endpoint = "https://api.groq.com/openai/v1"
+api_key = "${GROQ_API_KEY}"
+
+[huggingface]
+display_name = "Hugging Face"
+enabled = true
+api_key = "${HF_TOKEN}"
+
+[linkup]
+display_name = "Linkup"
+enabled = true
+api_key = "${LINKUP_API_KEY}"
+
+[minimax]
+display_name = "MiniMax"
+enabled = true
+endpoint = "https://api.minimax.io/anthropic"
+api_key = "${MINIMAX_API_KEY}"
+
+[mistral]
+display_name = "Mistral AI"
+enabled = true
+api_key = "${MISTRAL_API_KEY}"
+
+[ollama]
+enabled = true
+endpoint = "http://localhost:11434/v1"
+
+[openai]
+display_name = "OpenAI"
+enabled = true
+api_key = "${OPENAI_API_KEY}"
+
+[openrouter]
+display_name = "OpenRouter"
+enabled = true
+endpoint = "https://openrouter.ai/api/v1"
+api_key = "${OPENROUTER_API_KEY}"
+
+[portkey]
+display_name = "Portkey"
+enabled = true
+endpoint = "https://api.portkey.ai/v1"
+api_key = "${PORTKEY_API_KEY}"
+
+[scaleway]
+display_name = "Scaleway"
+enabled = true
+endpoint = "${SCALEWAY_ENDPOINT}"
+api_key = "${SCALEWAY_API_KEY}"
+
+[typesafe]
+display_name = "TypeSafe"
+enabled = true
+api_key = "${TYPESAFE_API_KEY}"
+
+[vertexai]
+display_name = "Google Vertex AI"
+enabled = false # This is the only one we disable because setting it up requires internet access just to get credentials so it fails in CI sandboxes
+gcp_project_id = "${GCP_PROJECT_ID}"
+gcp_location = "${GCP_LOCATION}"
+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+
+[xai]
+display_name = "xAI"
+enabled = true
+endpoint = "https://api.x.ai/v1"
+api_key = "${XAI_API_KEY}"
+
+[internal] # software-only backend, runs internally, without AI
+enabled = true
diff --git a/tests/data/migration/former_release/v0_72_cleaned/inference/routing_profiles.toml b/tests/data/migration/former_release/v0_72_cleaned/inference/routing_profiles.toml
new file mode 100644
index 000000000..a8b351c37
--- /dev/null
+++ b/tests/data/migration/former_release/v0_72_cleaned/inference/routing_profiles.toml
@@ -0,0 +1,171 @@
+# Routing profile library - Routes models to their backends
+# =========================================================================================
+# This file controls which backend serves which model.
+# Simply change the 'active' field to switch profiles,
+# or you can add your own custom profiles.
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+# =========================================================================================
+
+# Note: The internal backend (software-only models) is always available regardless of
+# which routing profile you select. See the documentation for details.
+
+# Which profile to use (change this to switch routing)
+active = "all_enabled_backends"
+
+# =========================================================================================
+# Routing Profiles
+# =========================================================================================
+
+[profiles.all_enabled_backends]
+description = "Route each model to the first enabled backend that serves it, in this order"
+fallback_order = [
+ "openai",
+ "anthropic",
+ "google",
+ "mistral",
+ "bedrock",
+ "azure_openai",
+ "xai",
+ "groq",
+ "fal",
+ "huggingface",
+ "linkup",
+ "openrouter",
+ "portkey",
+ "blackboxai",
+ "minimax",
+ "scaleway",
+ "ollama",
+ "vertexai",
+]
+
+# Judgment models are served by their own backend alone, so they go to it whenever it is enabled.
+# An optional route is inert while its backend is disabled.
+[profiles.all_enabled_backends.optional_routes]
+"jev-*" = "typesafe"
+
+# The same model names, served by the Pipelex-operated inference service instead. Switching between
+# this profile and the one above needs no other edit: a model's wire id and sdk are per-backend, so
+# decks, aliases and presets keep naming the same handles.
+[profiles.all_pipelex_manifold]
+description = "Use Pipelex Manifold (private beta) for all its supported models"
+default = "pipelex_manifold"
+
+# Judgment models are brought by you rather than served by Pipelex, so they go to their own backend
+# whenever it is enabled. An optional route is inert while its backend is disabled.
+[profiles.all_pipelex_manifold.optional_routes]
+"jev-*" = "typesafe"
+
+[profiles.all_anthropic]
+description = "Use Anthropic backend for all its supported models"
+default = "anthropic"
+
+[profiles.all_azure_openai]
+description = "Use Azure OpenAI backend for all its supported models"
+default = "azure_openai"
+
+[profiles.all_bedrock]
+description = "Use Bedrock backend for all its supported models"
+default = "bedrock"
+
+[profiles.all_blackboxai]
+description = "Use BlackBoxAI backend for all its supported models"
+default = "blackboxai"
+
+[profiles.all_fal]
+description = "Use FAL backend for all its supported models"
+default = "fal"
+
+[profiles.all_google]
+description = "Use Google GenAI backend for all its supported models"
+default = "google"
+
+[profiles.all_groq]
+description = "Use groq backend for all its supported models"
+default = "groq"
+
+[profiles.all_huggingface]
+description = "Use HuggingFace backend for all its supported models"
+default = "huggingface"
+
+[profiles.all_linkup]
+description = "Use Linkup backend for all its supported models"
+default = "linkup"
+
+[profiles.all_minimax]
+description = "Use MiniMax backend for all its supported models"
+default = "minimax"
+
+[profiles.all_mistral]
+description = "Use Mistral backend for all its supported models"
+default = "mistral"
+
+[profiles.all_ollama]
+description = "Use Ollama backend for all its supported models"
+default = "ollama"
+
+[profiles.all_openai]
+description = "Use OpenAI backend for all its supported models"
+default = "openai"
+
+[profiles.all_openrouter]
+description = "Use OpenRouter backend for all its supported models"
+default = "openrouter"
+
+[profiles.all_portkey]
+description = "Use Portkey backend for all its supported models"
+default = "portkey"
+
+[profiles.all_scaleway]
+description = "Use Scaleway backend for all its supported models"
+default = "scaleway"
+
+[profiles.all_typesafe]
+description = "Use TypeSafe backend for all its supported models"
+default = "typesafe"
+
+[profiles.all_vertexai]
+description = "Use Vertex AI backend for all its supported models"
+default = "vertexai"
+
+[profiles.all_xai]
+description = "Use xAI backend for all its supported models"
+default = "xai"
+
+[profiles.all_internal]
+description = "Use internal backend for all its supported models"
+default = "internal"
+
+# =========================================================================================
+# Custom Profiles
+# =========================================================================================
+# Add your own profiles below following the same pattern:
+#
+# [profiles.your_profile_name]
+# description = "What this profile does"
+# default = "backend-name" # Where to route models by default
+# [profiles.your_profile_name.routes]
+# "model-pattern" = "backend-name" # Specific routing rules
+#
+# Pattern matching supports:
+# - Exact names: "gpt-4o-mini"
+# - Wildcards: "claude-*" (matches all models starting with claude-)
+# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
+
+# =========================================================================================
+# Example of a custom routing profile with specific model matching
+# =========================================================================================
+
+[profiles.example_routing_using_specific_models]
+description = "Example routing profile using specific models"
+
+[profiles.example_routing_using_specific_models.routes]
+"gpt-4o-mini" = "blackboxai"
+"gpt-5.4-mini" = "openai"
+"gpt-5.5" = "azure_openai"
+"claude-3.7-sonnet" = "blackboxai"
+"gemini-2.5-flash" = "blackboxai"
+"gemini-2.5-pro" = "vertexai"
+"grok-3-mini" = "xai"
diff --git a/tests/e2e/pipelex/cli/conftest.py b/tests/e2e/pipelex/cli/conftest.py
index 1f8658c2c..007a28e4d 100644
--- a/tests/e2e/pipelex/cli/conftest.py
+++ b/tests/e2e/pipelex/cli/conftest.py
@@ -1,10 +1,47 @@
"""Fixtures for human-CLI E2E tests.
Re-exports the hermetic-HOME subprocess harness of the agent-CLI E2E suite so human-CLI
-subprocess tests (e.g. ``pipelex fix bundle``) run against the same isolated config tree.
+subprocess tests (e.g. ``pipelex fix bundle``) run against the same isolated config tree,
+and builds the empty home the hosted-setup subprocess tests start from.
"""
+from __future__ import annotations
+
+import stat
+from typing import TYPE_CHECKING
+
+import pytest
+
from tests.e2e.agent_cli.conftest import ( # ruff: ignore[unused-import] - fixtures re-exported for this directory
hermetic_home,
offline_subprocess_env,
)
+from tests.helpers.login_browser import TEST_KEY
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+# Stands in for the `code` and `cursor` CLIs, reporting the Pipelex extension as installed, so init's offer to
+# install it neither prompts nor touches a real editor.
+_IDE_SHADOW_SCRIPT = '#!/bin/sh\nif [ "$1" = "--list-extensions" ]; then\n echo "Pipelex.pipelex"\nfi\nexit 0\n'
+
+
+def fresh_home_env(tmp_path: Path) -> dict[str, str]:
+ """An empty `HOME`, no `PIPELEX_HOME`, a Pipelex API key, and an unreachable hosted API should anything call it."""
+ home = tmp_path / "home"
+ home.mkdir()
+ shadow_dir = tmp_path / "ide_shadow"
+ shadow_dir.mkdir()
+ for command in ("code", "cursor"):
+ script = shadow_dir / command
+ script.write_text(_IDE_SHADOW_SCRIPT, encoding="utf-8")
+ script.chmod(stat.S_IRWXU)
+ return {
+ "HOME": str(home),
+ "PATH": f"{shadow_dir}:/usr/bin:/bin",
+ "RUN_MODE": "ci_test",
+ "PIPELEX_API_KEY": TEST_KEY,
+ "PIPELEX_BASE_URL": "http://127.0.0.1:9",
+ "PIPELEX_APP_URL": "http://127.0.0.1:9",
+ }
diff --git a/tests/e2e/pipelex/cli/test_doctor_hosted_subprocess.py b/tests/e2e/pipelex/cli/test_doctor_hosted_subprocess.py
new file mode 100644
index 000000000..1335bffeb
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_doctor_hosted_subprocess.py
@@ -0,0 +1,46 @@
+"""After the default `pipelex init`, which sets up hosted runs, the installed `pipelex doctor` passes with a key and no provider key.
+
+Hosted runs boot nothing on this machine, so the doctor reports the provider credentials and the local model check as
+needed only for `--local` runs, and judges the setup by its Pipelex API key, which it never sends anywhere.
+"""
+
+from __future__ import annotations
+
+import subprocess # ruff: ignore[suspicious-subprocess-import] -- invokes the real pipelex binaries on purpose
+from pathlib import Path
+
+from tests.helpers.login_browser import TEST_KEY
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+VENV_BIN = REPO_ROOT / ".venv" / "bin"
+
+
+def _run(*, args: list[str], env: dict[str, str], stdin: str | None = None) -> subprocess.CompletedProcess[str]:
+ return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]
+ [str(VENV_BIN / "pipelex"), "--no-logo", *args],
+ env=env,
+ cwd=env["HOME"],
+ input=stdin,
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=180,
+ )
+
+
+class TestDoctorHostedSubprocess:
+ def test_a_hosted_setup_passes_with_a_key_and_fails_naming_pipelex_login_without_one(self, fresh_home_env: dict[str, str]) -> None:
+ init = _run(args=["init"], env=fresh_home_env, stdin="\n" * 20)
+ assert init.returncode == 0, init.stdout + init.stderr
+
+ with_key = _run(args=["doctor"], env=fresh_home_env)
+ with_key_transcript = with_key.stdout + with_key.stderr
+ assert with_key.returncode == 0, with_key_transcript
+ assert "Manual Fixes Required" not in with_key_transcript
+ assert TEST_KEY not in with_key_transcript
+
+ without_key_env = {name: value for name, value in fresh_home_env.items() if name != "PIPELEX_API_KEY"}
+ without_key = _run(args=["doctor"], env=without_key_env)
+ without_key_transcript = without_key.stdout + without_key.stderr
+ assert without_key.returncode == 1, without_key_transcript
+ assert "pipelex login" in without_key_transcript
diff --git a/tests/e2e/pipelex/cli/test_former_release_commands.py b/tests/e2e/pipelex/cli/test_former_release_commands.py
new file mode 100644
index 000000000..fa42ce3cc
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_former_release_commands.py
@@ -0,0 +1,179 @@
+"""A machine a former release set up, through the installed binaries: refused at boot, cleaned up, booting again.
+
+The home and a project's `.pipelex/` each carry what the v0.72 release wrote — its `backends.toml`,
+`routing_profiles.toml`, the Gateway's backend file and model lists, `pipelex_service.toml`, and what it left for Pipelex
+Manifold, which stays since Manifold is not retired, copied from that release's wheel — laid over the current kit's
+inference tree, which stands in for the per-backend files the inference-backend ledger carries forward on its own. That is the state every former user
+is in, and each command a remedy names is run on it: the boot names `pipelex migrate` and `pipelex init`, `pipelex
+migrate` and `pipelex-agent migrate` clean both directories so the machine boots, `pipelex doctor` names the cleanup
+and `--fix` runs it, and `pipelex init` offers it before it asks where runs execute.
+
+The boot probe is `pipelex-agent models`, the cheapest command that performs a full Pipelex boot.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess # ruff: ignore[suspicious-subprocess-import] - invokes the real pipelex binaries for E2E coverage
+from pathlib import Path
+
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.migration.former_release import detect_former_release
+from pipelex.system.configuration.config_loader import BACKENDS_DIR_NAME, BACKENDS_FILE_NAME, INFERENCE_DIR_NAME, ROUTING_PROFILES_FILE_NAME
+from tests.e2e.agent_cli.conftest import REPO_ROOT
+
+PIPELEX_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex"
+PIPELEX_AGENT_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex-agent"
+
+V0_72_CONFIG_DIR = REPO_ROOT / "tests" / "data" / "migration" / "former_release" / "v0_72"
+V0_72_CLEANED_DIR = REPO_ROOT / "tests" / "data" / "migration" / "former_release" / "v0_72_cleaned"
+
+PROJECT_DIR_NAME = "workspace"
+
+# The files the cleanup touches in one v0.72 directory, rewritten or removed.
+V0_72_FILES_PER_DIRECTORY = len(detect_former_release(config_dir=V0_72_CONFIG_DIR).file_paths)
+
+# A model table whose key the inference-backend ledger renames.
+MIGRATABLE_MODEL_TABLE = '\n["gpt-4o"]\nprompting_target = "openai"\n'
+
+
+def _plant_a_former_release_machine(*, hermetic_home: Path) -> tuple[Path, Path, Path]:
+ """The v0.72 files over the kit, in the home `hermetic_home` seeded and in a project's `.pipelex/` beside it.
+
+ Returns the project directory, the home configuration directory and the project's configuration directory.
+ """
+ home_config_dir = hermetic_home / ".pipelex"
+ shutil.copytree(V0_72_CONFIG_DIR, home_config_dir, dirs_exist_ok=True)
+ project_dir = hermetic_home / PROJECT_DIR_NAME
+ project_config_dir = project_dir / ".pipelex"
+ # A whole configuration directory, as that release's `pipelex init` wrote one into a project.
+ shutil.copytree(Path(str(get_kit_configs_dir())), project_config_dir)
+ shutil.copytree(V0_72_CONFIG_DIR, project_config_dir, dirs_exist_ok=True)
+ return project_dir, home_config_dir, project_config_dir
+
+
+def _run(*, args: list[str], env: dict[str, str], cwd: Path, answers: str | None = None) -> subprocess.CompletedProcess[str]:
+ # A wide terminal, so Rich never breaks a long temporary path across two lines. `answers` feeds the prompts: every
+ # question these commands ask here defaults to yes, so bare newlines accept each one in turn.
+ return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]
+ args,
+ env={**env, "COLUMNS": "400"},
+ cwd=str(cwd),
+ input=answers,
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=180,
+ )
+
+
+def _boot(*, env: dict[str, str], cwd: Path) -> subprocess.CompletedProcess[str]:
+ return _run(args=[str(PIPELEX_AGENT_BIN), "models", "--format", "json"], env=env, cwd=cwd)
+
+
+def _assert_cleaned(*, config_dirs: tuple[Path, ...]) -> None:
+ for config_dir in config_dirs:
+ assert detect_former_release(config_dir=config_dir).is_clean, f"something a former release left is still in {config_dir}"
+
+
+class TestAFormerReleaseMachine:
+ def test_the_boot_is_refused_with_the_one_error_naming_both_remedies(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:
+ project_dir, _, _ = _plant_a_former_release_machine(hermetic_home=hermetic_home)
+
+ booted = _boot(env=offline_subprocess_env, cwd=project_dir)
+
+ assert booted.returncode != 0
+ output = booted.stdout + booted.stderr
+ assert "FormerReleaseConfigError" in output, output
+ assert "pipelex migrate" in output
+ assert "pipelex init" in output
+ envelope = json.loads(booted.stderr)
+ assert envelope["error_type"] == "FormerReleaseConfigError"
+ assert "pipelex-agent migrate --dry-run --format json" in envelope["hint"], "an agent is handed the loop it can run"
+
+ def test_pipelex_migrate_cleans_both_directories_and_the_machine_boots(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:
+ project_dir, home_config_dir, project_config_dir = _plant_a_former_release_machine(hermetic_home=hermetic_home)
+
+ migrated = _run(args=[str(PIPELEX_BIN), "--no-logo", "migrate", "--yes"], env=offline_subprocess_env, cwd=project_dir)
+
+ assert migrated.returncode == 0, migrated.stdout + migrated.stderr
+ report = migrated.stdout + migrated.stderr
+ assert f"Cleaned up {2 * V0_72_FILES_PER_DIRECTORY} file(s) a former release left" in report
+ _assert_cleaned(config_dirs=(home_config_dir, project_config_dir))
+ for name in (BACKENDS_FILE_NAME, ROUTING_PROFILES_FILE_NAME):
+ golden = (V0_72_CLEANED_DIR / INFERENCE_DIR_NAME / name).read_bytes()
+ assert (home_config_dir / INFERENCE_DIR_NAME / name).read_bytes() == golden
+ assert (project_config_dir / INFERENCE_DIR_NAME / name).read_bytes() == golden
+ booted = _boot(env=offline_subprocess_env, cwd=project_dir)
+ assert booted.returncode == 0, booted.stdout + booted.stderr
+
+ def test_the_agent_loop_plans_the_cleanup_then_applies_it_and_the_machine_boots(
+ self, hermetic_home: Path, offline_subprocess_env: dict[str, str]
+ ) -> None:
+ project_dir, home_config_dir, project_config_dir = _plant_a_former_release_machine(hermetic_home=hermetic_home)
+ # A key the ledger would carry forward, in a file the cleanup removes: the replay never walks it.
+ gateway_file = home_config_dir / INFERENCE_DIR_NAME / BACKENDS_DIR_NAME / "pipelex_gateway.toml"
+ with gateway_file.open("a", encoding="utf-8") as stream:
+ stream.write(MIGRATABLE_MODEL_TABLE)
+
+ planned = _run(args=[str(PIPELEX_AGENT_BIN), "migrate", "--dry-run", "--format", "json"], env=offline_subprocess_env, cwd=project_dir)
+
+ assert planned.returncode == 0, planned.stdout + planned.stderr
+ plan = json.loads(planned.stdout)
+ assert plan["applied"] is False
+ assert plan["is_clean"] is False
+ assert plan["needs_attention"] is False
+ assert plan["summary"]["former_release_files"] == 2 * V0_72_FILES_PER_DIRECTORY
+ assert {file["action"] for file in plan["former_release"]["files"]} == {"rewrite", "remove"}
+ assert str(gateway_file) not in {migration_plan["file_path"] for migration_plan in plan["plans"]}
+ assert not detect_former_release(config_dir=home_config_dir).is_clean, "a dry run writes nothing"
+
+ applied = _run(args=[str(PIPELEX_AGENT_BIN), "migrate", "--yes", "--format", "json"], env=offline_subprocess_env, cwd=project_dir)
+
+ assert applied.returncode == 0, applied.stdout + applied.stderr
+ result = json.loads(applied.stdout)
+ assert result["summary"]["former_release_files_cleaned"] == 2 * V0_72_FILES_PER_DIRECTORY
+ assert all(file["backup_path"] for file in result["former_release"]["files"])
+ assert result["former_release"]["still_blocking"] == []
+ _assert_cleaned(config_dirs=(home_config_dir, project_config_dir))
+ booted = _boot(env=offline_subprocess_env, cwd=project_dir)
+ assert booted.returncode == 0, booted.stdout + booted.stderr
+
+ again = _run(args=[str(PIPELEX_AGENT_BIN), "migrate", "--yes", "--format", "json"], env=offline_subprocess_env, cwd=project_dir)
+ assert json.loads(again.stdout)["former_release"]["is_clean"] is True
+
+ def test_pipelex_doctor_names_the_cleanup_and_fix_runs_it(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:
+ project_dir, home_config_dir, project_config_dir = _plant_a_former_release_machine(hermetic_home=hermetic_home)
+
+ diagnosed = _run(args=[str(PIPELEX_BIN), "--no-logo", "doctor"], env=offline_subprocess_env, cwd=project_dir)
+
+ assert diagnosed.returncode == 1, diagnosed.stdout + diagnosed.stderr
+ report = diagnosed.stdout + diagnosed.stderr
+ assert f"{home_config_dir / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME} — left by a former release" in report
+ assert "Pipelex cannot start until it does" in report
+ assert "Not checked: what a former release left" in report
+
+ fixed = _run(args=[str(PIPELEX_BIN), "--no-logo", "doctor", "--fix"], env=offline_subprocess_env, cwd=project_dir, answers="\n" * 20)
+
+ assert f"Cleaned up {2 * V0_72_FILES_PER_DIRECTORY} file(s) a former release left" in fixed.stdout + fixed.stderr
+ _assert_cleaned(config_dirs=(home_config_dir, project_config_dir))
+
+ def test_pipelex_init_cleans_up_on_yes_then_asks_where_runs_execute(
+ self, hermetic_home: Path, offline_subprocess_env: dict[str, str], fresh_home_env: dict[str, str]
+ ) -> None:
+ project_dir, home_config_dir, project_config_dir = _plant_a_former_release_machine(hermetic_home=hermetic_home)
+ # The hosted path with a key already set, as the hosted init tests take it: no sign-in, no editor touched.
+ env = {
+ **offline_subprocess_env,
+ **{key: fresh_home_env[key] for key in ("PATH", "PIPELEX_API_KEY", "PIPELEX_BASE_URL", "PIPELEX_APP_URL")},
+ }
+
+ initialized = _run(args=[str(PIPELEX_BIN), "--no-logo", "init"], env=env, cwd=project_dir, answers="\n" * 30)
+
+ assert initialized.returncode == 0, initialized.stdout + initialized.stderr
+ transcript = initialized.stdout + initialized.stderr
+ assert "Left by a former release" in transcript
+ cleaned_at = transcript.index("Clean it up now?")
+ assert transcript.index("Where should your runs execute?") > cleaned_at, "the cleanup comes before the setup question"
+ _assert_cleaned(config_dirs=(home_config_dir, project_config_dir))
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py b/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py
index f9b6fb630..c9cabdcc4 100644
--- a/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py
+++ b/tests/e2e/pipelex/cli/test_init_cmd_edge_cases.py
@@ -26,6 +26,7 @@ class TestEdgeCases:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(str(indices[0])) # Select only openai
env.add_confirm_input(True) # Confirm creating profile if needed
@@ -50,6 +51,7 @@ class TestEdgeCases:
# User inputs - no fallback order prompt for exactly 2 backends
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select anthropic, openai
env.add_prompt_input("1") # Primary backend: anthropic (first in selection)
@@ -84,6 +86,7 @@ class TestEdgeCases:
# User inputs
env.add_confirm_input(True) # Confirm reset
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(str(indices[0])) # Select anthropic
env.add_confirm_input(True) # Confirm creating profile if needed
@@ -109,6 +112,7 @@ class TestEdgeCases:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select openai, mistral
env.add_prompt_input("1") # Primary backend
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_first_time.py b/tests/e2e/pipelex/cli/test_init_cmd_first_time.py
index 65c9e738d..3ba792196 100644
--- a/tests/e2e/pipelex/cli/test_init_cmd_first_time.py
+++ b/tests/e2e/pipelex/cli/test_init_cmd_first_time.py
@@ -9,6 +9,7 @@ import typer
from pipelex.cli.commands.init.command import init_cmd
from pipelex.cli.commands.init.ui.backends_ui import RECOMMENDED_INIT_BACKEND
from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.hosted.run_config import RunExecution
from pipelex.kit.paths import get_kit_configs_dir
from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit
from tests.helpers.init_cmd_helpers import MockedInitEnvironment, get_backend_indices_helper
@@ -26,6 +27,7 @@ class TestFirstTimeInitialization:
# User inputs: confirm init, recommended default backend
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("") # Empty = the recommended default
env.setup_mocks()
@@ -41,6 +43,8 @@ class TestFirstTimeInitialization:
env.verify_backends_enabled([RECOMMENDED_INIT_BACKEND])
env.verify_routing(f"all_{RECOMMENDED_INIT_BACKEND}")
env.verify_telemetry("off")
+ env.verify_run_execution(RunExecution.LOCAL)
+ env.mock_login.assert_not_called()
def test_init_with_multiple_backends_and_routing(self, tmp_path: Path, mocker: MockerFixture) -> None:
"""Test Case 1.2: Initialization with multiple backends."""
@@ -55,6 +59,7 @@ class TestFirstTimeInitialization:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select 3 backends
env.add_prompt_input("1") # Primary: first one (anthropic)
env.add_prompt_input("2,1") # Custom fallback order (mistral, anthropic)
@@ -81,6 +86,7 @@ class TestFirstTimeInitialization:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("all") # Select all backends
env.add_prompt_input("1") # Primary backend: the first listed
env.add_prompt_input("") # Accept the default fallback order
@@ -108,6 +114,7 @@ class TestFirstTimeInitialization:
# User inputs: confirm, then quit at backend selection
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("q") # Quit at backend selection
env.setup_mocks()
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_focused.py b/tests/e2e/pipelex/cli/test_init_cmd_focused.py
index 3d243ce8d..e1b542305 100644
--- a/tests/e2e/pipelex/cli/test_init_cmd_focused.py
+++ b/tests/e2e/pipelex/cli/test_init_cmd_focused.py
@@ -22,6 +22,7 @@ class TestFocusedInitialization:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("") # Backend selection: the recommended default
env.setup_mocks()
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_hosted.py b/tests/e2e/pipelex/cli/test_init_cmd_hosted.py
new file mode 100644
index 000000000..4aa6d4779
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_init_cmd_hosted.py
@@ -0,0 +1,125 @@
+"""`pipelex init` on the hosted path: Enter takes it, the kit's inference files stay as written, and a key is obtained or kept."""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+from pipelex.cli.commands.init.command import init_cmd
+from pipelex.cli.commands.init.setup_path import SetupPath, write_run_execution
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.cli.commands.login.command import LoginOutcome
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from pipelex.hosted.run_config import RunExecution
+from pipelex.kit.paths import get_kit_configs_dir
+from tests.helpers.init_cmd_helpers import MockedInitEnvironment
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+
+def _printed(env: MockedInitEnvironment) -> str:
+ return "\n".join(str(call.args[0]) for call in env.mock_console.print.call_args_list if call.args)
+
+
+def _assert_kit_inference_files(env: MockedInitEnvironment) -> None:
+ kit_inference_dir = get_kit_configs_dir() / "inference"
+ for name in ("backends.toml", "routing_profiles.toml"):
+ assert (env.inference_dir / name).read_text(encoding="utf-8") == (kit_inference_dir / name).read_text(encoding="utf-8")
+
+
+class TestInitHostedPath:
+ def test_enter_takes_the_hosted_api_and_signs_in(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+ env.add_confirm_input(True) # Confirm initialization
+ env.add_prompt_input("") # Where runs execute: Enter, the hosted Pipelex API
+ env.setup_mocks()
+ credentials = mocker.patch("pipelex.cli.commands.init.command.prompt_credentials")
+
+ init_cmd(focus=InitFocus.ALL)
+
+ env.verify_run_execution(RunExecution.HOSTED)
+ _assert_kit_inference_files(env)
+ env.verify_file_exists("telemetry.toml")
+ env.mock_login.assert_called_once()
+ credentials.assert_not_called()
+ assert env.prompt_inputs == []
+
+ def test_a_key_already_set_skips_the_login(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+ env.add_confirm_input(True)
+ env.choose_setup_path(SetupPath.HOSTED)
+ env.setup_mocks()
+ os.environ[PIPELEX_API_KEY_ENV_KEY] = "plx_sk_test_not_a_secret"
+
+ init_cmd(focus=InitFocus.ALL)
+
+ env.verify_run_execution(RunExecution.HOSTED)
+ env.mock_login.assert_not_called()
+ env.mock_browser_open.assert_not_called()
+ assert "already set" in _printed(env)
+
+ def test_a_failed_login_keeps_the_setup_and_names_pipelex_login(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+ env.add_confirm_input(True)
+ env.choose_setup_path(SetupPath.HOSTED)
+ env.setup_mocks()
+ env.mock_login.return_value = LoginOutcome.NO_KEY
+
+ init_cmd(focus=InitFocus.ALL)
+
+ env.verify_run_execution(RunExecution.HOSTED)
+ env.verify_file_exists("telemetry.toml")
+ assert "pipelex login" in _printed(env)
+
+ def test_doctor_fix_takes_the_hosted_default_without_asking_or_opening_a_browser(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """`pipelex doctor --fix` installs missing configuration files this way: nobody is there to answer."""
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_empty_dir()
+ env.setup_mocks()
+
+ init_cmd(focus=InitFocus.CONFIG, skip_confirmation=True)
+
+ env.verify_run_execution(RunExecution.HOSTED)
+ _assert_kit_inference_files(env)
+ env.mock_prompt_ask.assert_not_called()
+ env.mock_confirm_ask.assert_not_called()
+ env.mock_login.assert_not_called()
+ env.mock_browser_open.assert_not_called()
+ assert "Run [cyan]pipelex login[/cyan]" in _printed(env)
+
+ def test_a_config_reset_keeps_hosted_runs_and_asks_for_no_provider_key(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """`pipelex init config` on an existing setup resets pipelex.toml without asking again, so it keeps the answer."""
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+ write_run_execution(pipelex_toml_path=env.pipelex_dir / "pipelex.toml", execution=RunExecution.HOSTED)
+ env.add_confirm_input(True)
+ env.setup_mocks()
+ credentials = mocker.patch("pipelex.cli.commands.init.command.prompt_credentials")
+
+ init_cmd(focus=InitFocus.CONFIG)
+
+ env.verify_run_execution(RunExecution.HOSTED)
+ credentials.assert_not_called()
+ env.mock_login.assert_not_called()
+ env.mock_prompt_ask.assert_not_called()
+
+ def test_init_inference_configures_this_machine_without_the_question(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """`pipelex init inference` sets up the local backends on purpose and leaves where runs execute as it is."""
+ env = MockedInitEnvironment(tmp_path, mocker)
+ env.setup_with_configs(include_backends=True, include_routing=True, include_telemetry=True)
+ write_run_execution(pipelex_toml_path=env.pipelex_dir / "pipelex.toml", execution=RunExecution.HOSTED)
+ env.add_confirm_input(True)
+ env.add_prompt_input("") # Backend selection: the recommended default, not the setup question
+ env.setup_mocks()
+
+ init_cmd(focus=InitFocus.INFERENCE)
+
+ env.verify_run_execution(RunExecution.HOSTED)
+ env.mock_login.assert_not_called()
+ assert env.prompt_inputs == []
diff --git a/tests/e2e/pipelex/cli/test_init_cmd_validation.py b/tests/e2e/pipelex/cli/test_init_cmd_validation.py
index b9514cc4b..9c4f66a1b 100644
--- a/tests/e2e/pipelex/cli/test_init_cmd_validation.py
+++ b/tests/e2e/pipelex/cli/test_init_cmd_validation.py
@@ -22,6 +22,7 @@ class TestInputValidation:
# User inputs: invalid index, then valid
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("99") # Invalid index
env.add_prompt_input("") # Valid: the recommended default
@@ -41,6 +42,7 @@ class TestInputValidation:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("abc") # Invalid non-numeric
env.add_prompt_input("") # Valid: the recommended default
@@ -65,6 +67,7 @@ class TestInputValidation:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Space-separated indices
env.add_prompt_input("1") # Primary backend
env.add_prompt_input("") # Accept default fallback order
@@ -85,6 +88,7 @@ class TestInputValidation:
# User inputs: empty string for default
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("") # Empty = the recommended default
env.setup_mocks()
@@ -108,6 +112,7 @@ class TestInputValidation:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select 3 backends
env.add_prompt_input("1") # Primary backend
env.add_prompt_input("1") # Invalid: only 1 index instead of 2
@@ -134,6 +139,7 @@ class TestInputValidation:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select 3 backends
env.add_prompt_input("1") # Primary backend
env.add_prompt_input("1,1") # Invalid: duplicate indices
diff --git a/tests/e2e/pipelex/cli/test_init_hosted_subprocess.py b/tests/e2e/pipelex/cli/test_init_hosted_subprocess.py
new file mode 100644
index 000000000..0d2d5d9db
--- /dev/null
+++ b/tests/e2e/pipelex/cli/test_init_hosted_subprocess.py
@@ -0,0 +1,76 @@
+"""The installed `pipelex init` and `pipelex-agent init` take the hosted path end to end, with no login wait when a key is set.
+
+A machine or a test harness that sets `PIPELEX_API_KEY` and presses Enter at every prompt must see `pipelex init` exit,
+not wait for a browser: Enter takes the hosted Pipelex API, and the key already set is kept without a login.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess # ruff: ignore[suspicious-subprocess-import] -- invokes the real pipelex binaries on purpose
+from pathlib import Path
+
+from pipelex.cli.commands.init.setup_path import read_run_execution
+from pipelex.hosted.run_config import RunExecution
+from tests.helpers.login_browser import TEST_KEY
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+VENV_BIN = REPO_ROOT / ".venv" / "bin"
+
+
+class TestInitHostedSubprocess:
+ def test_pipelex_init_with_a_key_set_takes_the_hosted_default_and_exits(self, fresh_home_env: dict[str, str]) -> None:
+ home = Path(fresh_home_env["HOME"])
+ result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]
+ [str(VENV_BIN / "pipelex"), "--no-logo", "init"],
+ env=fresh_home_env,
+ cwd=str(home),
+ input="\n" * 20,
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=120,
+ )
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ pipelex_dir = home / ".pipelex"
+ assert read_run_execution(pipelex_toml_path=pipelex_dir / "pipelex.toml") == RunExecution.HOSTED
+ assert (pipelex_dir / "inference" / "backends.toml").is_file()
+ assert (pipelex_dir / "telemetry.toml").is_file()
+ # The prompts go to stdout and everything the console prints to stderr
+ transcript = result.stdout + result.stderr
+ assert "Where should your runs execute?" in transcript
+ assert "already set" in transcript
+ assert TEST_KEY not in transcript
+ assert not (pipelex_dir / ".env").exists()
+
+ def test_pipelex_agent_init_hosted_reports_the_execution(self, fresh_home_env: dict[str, str]) -> None:
+ home = Path(fresh_home_env["HOME"])
+ result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]
+ [str(VENV_BIN / "pipelex-agent"), "init", "--global", "--config", '{"execution": "hosted"}', "--format", "json"],
+ env=fresh_home_env,
+ cwd=str(home),
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=120,
+ )
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ payload = json.loads(result.stdout)
+ assert payload["execution"] == "hosted"
+ assert payload["api_key_set"] is True
+ assert read_run_execution(pipelex_toml_path=home / ".pipelex" / "pipelex.toml") == RunExecution.HOSTED
+
+ def test_pipelex_login_help_exits_0(self, fresh_home_env: dict[str, str]) -> None:
+ result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]
+ [str(VENV_BIN / "pipelex"), "--no-logo", "login", "--help"],
+ env=fresh_home_env,
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=60,
+ )
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert "--paste" in result.stdout
diff --git a/tests/e2e/pipelex/cli/test_migrate_commands.py b/tests/e2e/pipelex/cli/test_migrate_commands.py
index 5315953bb..b7fbc2202 100644
--- a/tests/e2e/pipelex/cli/test_migrate_commands.py
+++ b/tests/e2e/pipelex/cli/test_migrate_commands.py
@@ -1145,9 +1145,11 @@ class TestAStaleTelemetryFileIsMigratedNotReset:
assert "pipelex init telemetry" not in printed
-# A backend file earlier kits shipped, whose name matches the main configuration's `pipelex_*.toml` glob.
-LEGACY_PREFIXED_BACKEND_FILE_NAME = "pipelex_gateway.toml"
-LEGACY_PREFIXED_BACKEND_FILE_CONTENT = "# Per-model overrides for the Pipelex Gateway backend.\n"
+# A backend file of the user's own whose name matches the main configuration's `pipelex_*.toml` glob. Earlier kits
+# shipped one, `pipelex_gateway.toml`, but that one is now removed by the cleanup of a former release, which is
+# `pipelex migrate`'s first step, so the specimen is a backend the cleanup has no business with.
+PREFIXED_BACKEND_FILE_NAME = "pipelex_inhouse.toml"
+PREFIXED_BACKEND_FILE_CONTENT = "# Per-model overrides for an in-house backend.\n"
class TestAStaleBackendDirectory:
@@ -1157,7 +1159,8 @@ class TestAStaleBackendDirectory:
directory. These files do not: they are the reason the walk learned to enter a subdirectory at
all, and the reason a file is claimed by `(directory, name)` rather than by name — earlier kits
shipped a `pipelex_gateway.toml` in there, which the main configuration's `pipelex_*.toml` glob
- would otherwise have claimed. It is planted here for that reason, untouched, and the assertions say so.
+ would otherwise have claimed. A backend file of that shape is planted here for that reason,
+ untouched, and the assertions say so.
What a boot *says* is not asserted here and cannot be: the probe is `pipelex-agent`, which cuts
logging off process-wide as its first act. The warning is asserted through the human binary
@@ -1168,10 +1171,9 @@ class TestAStaleBackendDirectory:
"""Everything in and beside the directory that no migration may rewrite.
Excluded by name rather than by extension, and the difference is the whole point of this
- class: earlier kits shipped `pipelex_gateway.toml` in this directory, and machines set up with
- them still carry it, so it is planted here as they left it. The main configuration's
- `pipelex_*.toml` glob would claim it by name alone, so it is the one file a walk that forgot
- about directories would rewrite. An extension filter dropped it — and every other `.toml` in
+ class: a backend file named `pipelex_<name>.toml`, as earlier kits shipped one, is planted here.
+ The main configuration's `pipelex_*.toml` glob would claim it by name alone, so it is the one
+ file a walk that forgot about directories would rewrite. An extension filter dropped it — and every other `.toml` in
here — out of the snapshot, leaving the class's own claim unasserted. Only the files the
fixture aged are excluded now. `inference/backends.toml` is added back from one level up,
which is a different claim entirely: it is a table per backend rather than a table per model,
@@ -1181,7 +1183,7 @@ class TestAStaleBackendDirectory:
it and cannot break the comparison.
"""
backends_dir = _backends_dir(hermetic_home=hermetic_home)
- backends_dir.joinpath(LEGACY_PREFIXED_BACKEND_FILE_NAME).write_text(LEGACY_PREFIXED_BACKEND_FILE_CONTENT, encoding="utf-8")
+ backends_dir.joinpath(PREFIXED_BACKEND_FILE_NAME).write_text(PREFIXED_BACKEND_FILE_CONTENT, encoding="utf-8")
neighbours = [path for path in sorted(backends_dir.iterdir()) if path not in migrated]
neighbours.append(hermetic_home / ".pipelex" / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME)
return {path: path.read_bytes() for path in neighbours}
diff --git a/tests/helpers/init_cmd_helpers.py b/tests/helpers/init_cmd_helpers.py
index dcbdfdcaa..0aedde4fc 100644
--- a/tests/helpers/init_cmd_helpers.py
+++ b/tests/helpers/init_cmd_helpers.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import os
import shutil
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -7,9 +8,17 @@ from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from pytest_mock import MockerFixture
+ from pipelex.hosted.run_config import RunExecution
+
+from pipelex.cli.commands.init.setup_path import SetupPath, read_run_execution
+from pipelex.cli.commands.login.command import LoginOutcome
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
from pipelex.kit.paths import get_kit_configs_dir
from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit
+#: What a person types at "Where should your runs execute?" to pick each path.
+SETUP_PATH_ANSWERS: dict[SetupPath, str] = {SetupPath.HOSTED: "1", SetupPath.LOCAL: "2"}
+
def get_backend_indices_helper(backends_toml_path: str, backend_names: list[str]) -> list[int]:
"""Get 1-based indices for given backend names from backends.toml.
@@ -122,6 +131,8 @@ class MockedInitEnvironment:
self.mock_prompt_ask: Any = None
self.mock_confirm_ask: Any = None
self.mock_config_manager: Any = None
+ self.mock_login: Any = None
+ self.mock_browser_open: Any = None
# Input sequences
self.prompt_inputs: list[str] = []
@@ -194,6 +205,8 @@ class MockedInitEnvironment:
self.mocker.patch("pipelex.cli.commands.init.backends.config_manager", self.mock_config_manager)
self.mocker.patch("pipelex.cli.commands.init.routing.config_manager", self.mock_config_manager)
self.mocker.patch("pipelex.cli.commands.init.config_files.config_manager", self.mock_config_manager)
+ # The credentials file, where `pipelex login` saves its key, lives in the home directory this test stands in
+ self.mocker.patch("pipelex.cli.commands.init.credentials.config_manager", self.mock_config_manager)
def mock_console_outputs(self) -> None:
"""Mock Console handling to suppress output."""
@@ -217,6 +230,18 @@ class MockedInitEnvironment:
"""
self.prompt_inputs.append(value)
+ def choose_setup_path(self, setup_path: SetupPath) -> None:
+ """Answer "Where should your runs execute?" with this path.
+
+ Args:
+ setup_path: The path to pick; Enter would take the hosted Pipelex API.
+ """
+ self.prompt_inputs.append(SETUP_PATH_ANSWERS[setup_path])
+
+ def choose_local(self) -> None:
+ """Answer "Where should your runs execute?" with this machine, the path every backend test walks."""
+ self.choose_setup_path(SetupPath.LOCAL)
+
def add_confirm_input(self, value: bool) -> None:
"""Add a confirm input to the sequence.
@@ -256,6 +281,17 @@ class MockedInitEnvironment:
# The IDE extension suggestion asks to install into the editor it finds on this machine, so it would
# consume a queued answer and install into the real editor wherever the extension is missing
self.mocker.patch("pipelex.cli.commands.init.backends.suggest_extension_install_if_needed")
+ self.mocker.patch("pipelex.cli.commands.init.command.suggest_extension_install_if_needed")
+
+ # The hosted path signs in through the browser: no test opens one or reaches the hosted API. The login
+ # reports a saved key by default; a test sets `mock_login.return_value` for another outcome.
+ self.mock_login = self.mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser", return_value=LoginOutcome.SAVED)
+ self.mock_browser_open = self.mocker.patch("pipelex.cli.commands.login.command.webbrowser.open")
+
+ # Start with no Pipelex API key, whatever the developer's shell exports, and restore the environment
+ # afterwards, since a login sets the key in it directly
+ self.mocker.patch.dict(os.environ)
+ os.environ.pop(PIPELEX_API_KEY_ENV_KEY, None)
def get_backend_indices(self, backend_names: list[str]) -> list[int]:
"""Get 1-based indices for backend names.
@@ -294,6 +330,15 @@ class MockedInitEnvironment:
routing_path = str(self.inference_dir / "routing_profiles.toml")
verify_routing_profile(routing_path, expected_active, expected_default, expected_fallback_order)
+ def verify_run_execution(self, expected: RunExecution | None) -> None:
+ """Verify where runs execute, as pipelex.toml sets it.
+
+ Args:
+ expected: The expected `[run] execution`, or None when the file should set none.
+ """
+ actual = read_run_execution(pipelex_toml_path=self.pipelex_dir / "pipelex.toml")
+ assert actual == expected, f"Expected [run] execution {expected!r}, got {actual!r}"
+
def verify_telemetry(self, expected_mode: str) -> None:
"""Verify telemetry configuration.
diff --git a/tests/helpers/login_browser.py b/tests/helpers/login_browser.py
new file mode 100644
index 000000000..87d1ca610
--- /dev/null
+++ b/tests/helpers/login_browser.py
@@ -0,0 +1,71 @@
+"""Stand-ins for the browser and the key check of `pipelex login`, so a login test opens nothing and calls no API."""
+
+from __future__ import annotations
+
+import http.client
+import threading
+from typing import TYPE_CHECKING
+from urllib.parse import parse_qs, urlencode, urlsplit
+
+from pipelex.cli.commands.init.credentials import read_env_file_value
+from pipelex.cli.commands.login.loopback import CALLBACK_PATH, LOOPBACK_HOST
+from pipelex.hosted.api_key_check import ApiKeyCheck, ApiKeyVerdict
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture, MockType
+
+#: A value shaped like a Pipelex API key, which no hosted API knows.
+TEST_KEY = "plx_sk_test_not_a_secret"
+API_URL = "https://api.test"
+ACCEPTED = ApiKeyCheck(verdict=ApiKeyVerdict.ACCEPTED, base_url=API_URL, account_email="[email protected]")
+
+
+class BrowserStandIn:
+ """Stands in for `webbrowser.open`: records the URL and, when told to, sends the app's callback to the listener."""
+
+ def __init__(self, *, api_key: str | None, echo_state: bool = True, opens: bool = True) -> None:
+ self.opened_urls: list[str] = []
+ self._api_key = api_key
+ self._echo_state = echo_state
+ self._opens = opens
+ self._threads: list[threading.Thread] = []
+
+ def open(self, url: str) -> bool:
+ self.opened_urls.append(url)
+ if self._api_key is None:
+ return self._opens
+ query = parse_qs(urlsplit(url).query)
+ port = int(query["callback_port"][0])
+ state = query["state"][0] if self._echo_state else "forged-state"
+ thread = threading.Thread(target=self._call_back, kwargs={"port": port, "state": state}, daemon=True)
+ thread.start()
+ self._threads.append(thread)
+ return self._opens
+
+ def _call_back(self, *, port: int, state: str) -> None:
+ connection = http.client.HTTPConnection(LOOPBACK_HOST, port, timeout=5)
+ try:
+ connection.request("GET", f"{CALLBACK_PATH}?{urlencode({'api_key': self._api_key, 'state': state})}")
+ connection.getresponse().read()
+ finally:
+ connection.close()
+
+ def join(self) -> None:
+ for thread in self._threads:
+ thread.join(timeout=10)
+
+
+def patch_browser(mocker: MockerFixture, *, browser: BrowserStandIn) -> None:
+ mocker.patch("pipelex.cli.commands.login.command.webbrowser.open", side_effect=browser.open)
+
+
+def patch_check(mocker: MockerFixture, *, check: ApiKeyCheck) -> MockType:
+ return mocker.patch("pipelex.cli.commands.login.command.check_pipelex_api_key", return_value=check)
+
+
+def saved_key(*, home: Path) -> str | None:
+ """The key saved in a home configuration directory's `.env`."""
+ return read_env_file_value(env_path=home / ".env", key=PIPELEX_API_KEY_ENV_KEY)
diff --git a/tests/helpers/pipelex_api_key_env.py b/tests/helpers/pipelex_api_key_env.py
new file mode 100644
index 000000000..20bb721e9
--- /dev/null
+++ b/tests/helpers/pipelex_api_key_env.py
@@ -0,0 +1,22 @@
+"""Hold `PIPELEX_API_KEY` steady for a test that saves or reads one.
+
+`pipelex login` sets the variable in the process environment directly, as a new process would see it, so a test must
+have monkeypatch record the variable before the code under test touches it: `monkeypatch.delenv` on an absent variable
+records nothing, and the key a test saved would leak into every later test of the worker.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+
+if TYPE_CHECKING:
+ import pytest
+
+
+def isolate_pipelex_api_key(monkeypatch: pytest.MonkeyPatch, *, value: str | None = None) -> None:
+ """Set `PIPELEX_API_KEY` to `value`, or unset it, and restore the developer's own at teardown whatever the test wrote."""
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, value if value is not None else "recorded-then-unset")
+ if value is None:
+ monkeypatch.delenv(PIPELEX_API_KEY_ENV_KEY)
diff --git a/tests/helpers/recorded_console.py b/tests/helpers/recorded_console.py
new file mode 100644
index 000000000..e1873c5aa
--- /dev/null
+++ b/tests/helpers/recorded_console.py
@@ -0,0 +1,19 @@
+"""A Rich console whose output a test reads back."""
+
+from io import StringIO
+from typing import NamedTuple
+
+from rich.console import Console
+
+
+class RecordedConsole(NamedTuple):
+ console: Console
+ buffer: StringIO
+
+ def text(self) -> str:
+ return self.buffer.getvalue()
+
+
+def make_recorded_console() -> RecordedConsole:
+ buffer = StringIO()
+ return RecordedConsole(console=Console(file=buffer, width=200), buffer=buffer)
diff --git a/tests/integration/pipelex/cli/test_init_cmd_integration.py b/tests/integration/pipelex/cli/test_init_cmd_integration.py
index ffc15b8c7..7ac720722 100644
--- a/tests/integration/pipelex/cli/test_init_cmd_integration.py
+++ b/tests/integration/pipelex/cli/test_init_cmd_integration.py
@@ -23,6 +23,7 @@ class TestInitCommandIntegration:
# User inputs: confirm init, select all backends
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("all") # Select all backends
env.add_prompt_input("1") # Primary backend: the first listed
env.add_prompt_input("") # Accept the default fallback order
@@ -58,6 +59,7 @@ class TestInitCommandIntegration:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select anthropic, mistral, openai
env.add_prompt_input("1") # Primary backend: first one (anthropic)
env.add_prompt_input("") # Accept default fallback order
@@ -88,6 +90,7 @@ class TestInitCommandIntegration:
# User inputs
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(str(indices[0])) # Select only openai
env.add_confirm_input(True) # Confirm creating profile if needed
@@ -113,6 +116,7 @@ class TestInitCommandIntegration:
# User inputs - CONFIG focus still runs full init on first time
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input("") # Backend selection: the recommended default
env.setup_mocks()
@@ -223,6 +227,7 @@ class TestInitCommandIntegration:
# User inputs
env.add_confirm_input(True) # Confirm reset
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(str(indices[0])) # Select mistral only
env.add_confirm_input(True) # Confirm creating profile if needed
@@ -282,6 +287,7 @@ class TestInitCommandIntegration:
# User inputs - note: no fallback order prompt for 2 backends
env.add_confirm_input(True) # Confirm initialization
+ env.choose_local() # Where runs execute: this machine
env.add_prompt_input(indices_str) # Select anthropic, openai
env.add_prompt_input("1") # Primary backend: anthropic (first in selection)
diff --git a/tests/unit/pipelex/cli/agent_cli/commands/test_agent_init_execution.py b/tests/unit/pipelex/cli/agent_cli/commands/test_agent_init_execution.py
new file mode 100644
index 000000000..8c2dcad71
--- /dev/null
+++ b/tests/unit/pipelex/cli/agent_cli/commands/test_agent_init_execution.py
@@ -0,0 +1,130 @@
+"""`pipelex-agent init` takes `"execution": "hosted"`: it writes the setting, configures no backend, and reports it."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+import pytest
+import typer
+from typer.testing import CliRunner
+
+from pipelex.cli.agent_cli.commands.init_cmd import agent_init_cmd
+from pipelex.cli.commands.init.setup_path import read_run_execution, write_run_execution
+from pipelex.hosted.run_config import RunExecution
+from pipelex.kit.paths import get_kit_configs_dir
+from tests.helpers.pipelex_api_key_env import isolate_pipelex_api_key
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+_app = typer.Typer()
+_app.command()(agent_init_cmd)
+
+
+def target_dir(tmp_path: Path, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch) -> Path:
+ """The project's `.pipelex/`, with the credentials file in a home of the test's own and no key in the environment."""
+ project_root = tmp_path / "project"
+ project_root.mkdir()
+ config_manager = mocker.MagicMock()
+ config_manager.project_root = project_root
+ config_manager.global_config_dir = tmp_path / "home"
+ config_manager.project_config_dir = None
+ mocker.patch("pipelex.cli.agent_cli.commands.init_cmd.config_manager", config_manager)
+ mocker.patch("pipelex.cli.commands.init.credentials.config_manager", config_manager)
+ isolate_pipelex_api_key(monkeypatch)
+ return project_root / ".pipelex"
+
+
+def _init(*, config: dict[str, Any] | None, global_: bool = False) -> tuple[int, dict[str, Any]]:
+ args = ["--format", "json"]
+ if global_:
+ args.append("--global")
+ if config is not None:
+ args.extend(["--config", json.dumps(config)])
+ result = CliRunner().invoke(_app, args)
+ output = result.stdout if result.exit_code == 0 else result.stderr
+ return result.exit_code, json.loads(output)
+
+
+def _kit_text(relative_path: str) -> str:
+ return (get_kit_configs_dir() / relative_path).read_text(encoding="utf-8")
+
+
+class TestAgentInitExecution:
+ def test_hosted_writes_the_setting_and_leaves_the_kit_inference_files(self, target_dir: Path) -> None:
+ exit_code, payload = _init(config={"execution": "hosted"})
+
+ assert exit_code == 0
+ assert payload["execution"] == "hosted"
+ assert payload["api_key_set"] is False
+ assert "backends_enabled" not in payload
+ assert "routing_profile" not in payload
+ assert read_run_execution(pipelex_toml_path=target_dir / "pipelex.toml") == RunExecution.HOSTED
+ for relative_path in ("inference/backends.toml", "inference/routing_profiles.toml"):
+ assert (target_dir / relative_path).read_text(encoding="utf-8") == _kit_text(relative_path)
+
+ @pytest.mark.usefixtures("target_dir")
+ def test_hosted_reports_a_key_in_the_environment(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("PIPELEX_API_KEY", "plx_sk_test_not_a_secret")
+
+ exit_code, payload = _init(config={"execution": "hosted"})
+
+ assert exit_code == 0
+ assert payload["api_key_set"] is True
+ assert "plx_sk_test_not_a_secret" not in json.dumps(payload)
+
+ @pytest.mark.parametrize("config", [None, {}, {"execution": "local"}, {"execution": "local", "backends": ["openai"]}])
+ def test_local_or_absent_configures_the_backends_as_before(self, target_dir: Path, config: dict[str, Any] | None) -> None:
+ exit_code, payload = _init(config=config)
+
+ assert exit_code == 0
+ assert payload["execution"] == "local"
+ assert "backends_enabled" in payload
+ assert "routing_profile" in payload
+ assert read_run_execution(pipelex_toml_path=target_dir / "pipelex.toml") == RunExecution.LOCAL
+
+ def test_an_unknown_execution_is_refused(self, target_dir: Path) -> None:
+ exit_code, payload = _init(config={"execution": "elsewhere"})
+
+ assert exit_code != 0
+ assert payload["error_type"] == "ArgumentError"
+ assert "elsewhere" in payload["message"]
+ assert not target_dir.exists()
+
+ @pytest.mark.parametrize("config", [{"execution": "hosted", "backends": ["openai"]}, {"execution": "hosted", "primary_backend": "openai"}])
+ def test_hosted_refuses_the_backend_keys(self, target_dir: Path, config: dict[str, Any]) -> None:
+ exit_code, payload = _init(config=config)
+
+ assert exit_code != 0
+ assert payload["error_type"] == "ArgumentError"
+ assert not target_dir.exists()
+
+ def test_a_global_init_names_a_project_that_overrides_it(self, target_dir: Path, mocker: MockerFixture, tmp_path: Path) -> None:
+ project_pipelex_toml = target_dir / "pipelex.toml"
+ write_run_execution(pipelex_toml_path=project_pipelex_toml, execution=RunExecution.LOCAL)
+ config_manager = mocker.patch("pipelex.cli.agent_cli.commands.init_cmd.config_manager")
+ config_manager.global_config_dir = tmp_path / "home"
+ config_manager.project_config_dir = target_dir
+
+ exit_code, payload = _init(config={"execution": "hosted"}, global_=True)
+
+ assert exit_code == 0
+ assert read_run_execution(pipelex_toml_path=tmp_path / "home" / "pipelex.toml") == RunExecution.HOSTED
+ (warning,) = payload["warnings"]
+ assert str(project_pipelex_toml) in warning
+ assert 'execution = "local"' in warning
+
+ def test_a_global_init_matching_the_project_warns_nothing(self, target_dir: Path, mocker: MockerFixture, tmp_path: Path) -> None:
+ write_run_execution(pipelex_toml_path=target_dir / "pipelex.toml", execution=RunExecution.HOSTED)
+ config_manager = mocker.patch("pipelex.cli.agent_cli.commands.init_cmd.config_manager")
+ config_manager.global_config_dir = tmp_path / "home"
+ config_manager.project_config_dir = target_dir
+
+ exit_code, payload = _init(config={"execution": "hosted"}, global_=True)
+
+ assert exit_code == 0
+ assert "warnings" not in payload
diff --git a/tests/unit/pipelex/cli/commands/init/conftest.py b/tests/unit/pipelex/cli/commands/init/conftest.py
new file mode 100644
index 000000000..ea584f3f2
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/conftest.py
@@ -0,0 +1,32 @@
+"""A recorded console and a home configuration directory of each test's own, for the init tests."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.system.environment import PIPELEX_HOME_ENV_KEY
+from tests.helpers.pipelex_api_key_env import isolate_pipelex_api_key
+from tests.helpers.recorded_console import RecordedConsole, make_recorded_console
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+def recorded_console() -> RecordedConsole:
+ """A console whose output the test reads back."""
+ return make_recorded_console()
+
+
+def pipelex_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ """A home configuration directory of the test's own, an empty working directory, and no key in the environment."""
+ home = tmp_path / "pipelex_home"
+ working_dir = tmp_path / "work"
+ working_dir.mkdir()
+ monkeypatch.chdir(working_dir)
+ monkeypatch.setenv(PIPELEX_HOME_ENV_KEY, str(home))
+ isolate_pipelex_api_key(monkeypatch)
+ return home
diff --git a/tests/unit/pipelex/cli/commands/init/test_ensure_pipelex_api_key.py b/tests/unit/pipelex/cli/commands/init/test_ensure_pipelex_api_key.py
new file mode 100644
index 000000000..0317bec21
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_ensure_pipelex_api_key.py
@@ -0,0 +1,103 @@
+"""The hosted path keeps a key already set, else signs in through the browser, else names `pipelex login`."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.cli.commands.init.credentials import set_env_file_entry
+from pipelex.cli.commands.init.setup_path import ensure_pipelex_api_key
+from pipelex.cli.commands.login.command import LoginOutcome
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+ from tests.helpers.recorded_console import RecordedConsole
+
+
+class TestEnsurePipelexApiKey:
+ def test_a_key_in_the_environment_skips_the_login(
+ self, mocker: MockerFixture, pipelex_home: Path, monkeypatch: pytest.MonkeyPatch, recorded_console: RecordedConsole
+ ) -> None:
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "plx_sk_test_not_a_secret")
+ login = mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser")
+ check = mocker.patch("pipelex.cli.commands.login.command.check_pipelex_api_key")
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=True)
+
+ login.assert_not_called()
+ check.assert_not_called()
+ assert "already set" in recorded_console.text()
+ assert "plx_sk_test_not_a_secret" not in recorded_console.text()
+ assert not (pipelex_home / ".env").exists()
+
+ def test_a_working_directory_env_that_shadows_the_saved_key_is_named(
+ self, mocker: MockerFixture, pipelex_home: Path, monkeypatch: pytest.MonkeyPatch, recorded_console: RecordedConsole
+ ) -> None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="plx_sk_saved")
+ Path(".env").write_text(f"{PIPELEX_API_KEY_ENV_KEY}=plx_sk_project\n", encoding="utf-8")
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "plx_sk_project")
+ login = mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser")
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=True)
+
+ login.assert_not_called()
+ printed = recorded_console.text()
+ assert "already set" in printed
+ assert str(Path(".env").resolve()) in printed.replace("\n", "")
+ assert "plx_sk_project" not in printed
+ assert "plx_sk_saved" not in printed
+
+ def test_a_saved_key_skips_the_login(self, mocker: MockerFixture, pipelex_home: Path, recorded_console: RecordedConsole) -> None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="plx_sk_saved")
+ login = mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser")
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=True)
+
+ login.assert_not_called()
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_a_key_that_does_not_look_like_one_is_kept_with_a_warning(
+ self, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, recorded_console: RecordedConsole
+ ) -> None:
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "sk-something-else")
+ login = mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser")
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=True)
+
+ login.assert_not_called()
+ assert "does not look like a Pipelex API key" in recorded_console.text()
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_with_no_key_it_signs_in_through_the_browser(self, mocker: MockerFixture, recorded_console: RecordedConsole) -> None:
+ login = mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser", return_value=LoginOutcome.SAVED)
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=True)
+
+ login.assert_called_once_with(console=recorded_console.console)
+ assert "No key was saved" not in recorded_console.text()
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_a_login_that_saved_nothing_names_pipelex_login(self, mocker: MockerFixture, recorded_console: RecordedConsole) -> None:
+ mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser", return_value=LoginOutcome.NO_KEY)
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=True)
+
+ assert "No key was saved" in recorded_console.text()
+ assert "pipelex login" in recorded_console.text()
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_with_nobody_to_answer_it_prints_pipelex_login_and_opens_no_browser(
+ self, mocker: MockerFixture, recorded_console: RecordedConsole
+ ) -> None:
+ login = mocker.patch("pipelex.cli.commands.init.setup_path.login_with_browser")
+ browser_open = mocker.patch("pipelex.cli.commands.login.command.webbrowser.open")
+
+ ensure_pipelex_api_key(console=recorded_console.console, interactive=False)
+
+ login.assert_not_called()
+ browser_open.assert_not_called()
+ assert "Run pipelex login" in recorded_console.text()
diff --git a/tests/unit/pipelex/cli/commands/init/test_hosted_key_step.py b/tests/unit/pipelex/cli/commands/init/test_hosted_key_step.py
new file mode 100644
index 000000000..c23c02714
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_hosted_key_step.py
@@ -0,0 +1,42 @@
+"""The hosted path's last step, getting a Pipelex API key, fails without undoing or misreporting the setup written before it."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pipelex.cli.commands.init.command import InitChoices, execute_initialization, inspect_initialization
+from pipelex.cli.commands.init.setup_path import SetupPath, read_run_execution
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.cli.commands.login.command import LOGIN_PASTE_COMMAND
+from pipelex.hosted.run_config import RunExecution
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+ from tests.helpers.recorded_console import RecordedConsole
+
+
+class TestHostedKeyStep:
+ def test_a_listener_that_cannot_bind_leaves_the_setup_saved_and_names_pipelex_login(
+ self, mocker: MockerFixture, pipelex_home: Path, recorded_console: RecordedConsole
+ ) -> None:
+ mocker.patch("pipelex.cli.commands.init.command.suggest_extension_install_if_needed")
+ mocker.patch("pipelex.cli.commands.login.command.LoopbackListener", side_effect=OSError(48, "Address already in use"))
+ browser_open = mocker.patch("pipelex.cli.commands.login.command.webbrowser.open")
+ inspection = inspect_initialization(focus=InitFocus.ALL, local=False)
+
+ execute_initialization(
+ console=recorded_console.console, inspection=inspection, choices=InitChoices(setup_path=SetupPath.HOSTED, interactive=True)
+ )
+
+ browser_open.assert_not_called()
+ assert read_run_execution(pipelex_toml_path=pipelex_home / "pipelex.toml") == RunExecution.HOSTED
+ assert (pipelex_home / "telemetry.toml").is_file()
+ # Rich wraps long lines at spaces; joining the words back reads the message as written
+ printed = " ".join(recorded_console.text().split())
+ assert "Address already in use" in printed
+ assert "pipelex login" in printed
+ assert LOGIN_PASTE_COMMAND in printed
+ assert "pipelex init config" not in printed
diff --git a/tests/unit/pipelex/cli/commands/init/test_init_stages.py b/tests/unit/pipelex/cli/commands/init/test_init_stages.py
new file mode 100644
index 000000000..370e6864a
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_init_stages.py
@@ -0,0 +1,283 @@
+"""`pipelex init`'s inspect stage decides, before anything is asked, whether the run asks where runs execute.
+
+It also finds what a former release that ran on the Pipelex Gateway left, which the choose stage
+offers to clean up before anything else is asked; the v0.72 kit, copied from that release's wheel, is the specimen.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from rich.console import Console
+
+from pipelex.cli.commands.init.command import InitInspection, choose_initialization, describe_former_release_findings, inspect_initialization
+from pipelex.cli.commands.init.setup_path import SetupPath, write_run_execution
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.hosted.run_config import RunExecution
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.migration.former_release import detect_former_release
+
+if TYPE_CHECKING:
+ from unittest.mock import MagicMock
+
+ from pytest_mock import MockerFixture
+
+
+def config_manager(mocker: MockerFixture) -> MagicMock:
+ return mocker.patch("pipelex.cli.commands.init.command.config_manager")
+
+
+def config_dir(tmp_path: Path, config_manager: MagicMock) -> Path:
+ """The home configuration directory init targets, empty, and the only one the machine has: the boot reads its files."""
+ directory = tmp_path / ".pipelex"
+ config_manager.global_config_dir = directory
+ config_manager.existing_config_dirs = [directory]
+ config_manager.backends_file_paths.return_value = [directory / "inference" / "backends.toml", directory / "inference" / "backends_override.toml"]
+ config_manager.routing_profiles_file_paths.return_value = [
+ directory / "inference" / "routing_profiles.toml",
+ directory / "inference" / "routing_profiles_override.toml",
+ ]
+ return directory
+
+
+V0_72_CONFIG_DIR = "tests/data/migration/former_release/v0_72"
+
+
+def _install_a_former_release(config_dir: Path) -> None:
+ shutil.copytree(V0_72_CONFIG_DIR, config_dir)
+
+
+def _snapshot(*, directory: Path) -> dict[str, bytes]:
+ return {path.relative_to(directory).as_posix(): path.read_bytes() for path in sorted(directory.rglob("*")) if path.is_file()}
+
+
+def _install_backends(config_dir: Path) -> None:
+ inference_dir = config_dir / "inference"
+ inference_dir.mkdir(parents=True)
+ shutil.copy2(str(get_kit_configs_dir() / "inference" / "backends.toml"), inference_dir / "backends.toml")
+
+
+def _inspect(*, focus: InitFocus) -> InitInspection:
+ return inspect_initialization(focus=focus, local=False)
+
+
+class TestInitStages:
+ @pytest.mark.parametrize("focus", [InitFocus.ALL, InitFocus.CONFIG])
+ def test_a_first_setup_asks_where_runs_execute(self, config_dir: Path, focus: InitFocus) -> None:
+ inspection = _inspect(focus=focus)
+
+ assert inspection.needs_inference
+ assert inspection.asks_setup_path
+ assert inspection.target_config_dir == config_dir
+ assert not config_dir.exists(), "inspecting writes nothing"
+
+ def test_a_full_reset_asks_again(self, config_dir: Path) -> None:
+ _install_backends(config_dir)
+ assert _inspect(focus=InitFocus.ALL).asks_setup_path
+
+ def test_a_config_reset_of_an_existing_setup_keeps_the_setting(self, config_dir: Path) -> None:
+ _install_backends(config_dir)
+ write_run_execution(pipelex_toml_path=config_dir / "pipelex.toml", execution=RunExecution.HOSTED)
+
+ inspection = _inspect(focus=InitFocus.CONFIG)
+
+ assert not inspection.asks_setup_path
+ assert inspection.configured_execution == RunExecution.HOSTED
+ assert inspection.kept_execution == RunExecution.HOSTED
+ assert inspection.keeps_hosted_runs
+
+ @pytest.mark.parametrize("focus", [InitFocus.INFERENCE, InitFocus.ROUTING, InitFocus.TELEMETRY])
+ def test_the_focused_setups_never_ask(self, config_dir: Path, focus: InitFocus) -> None:
+ _install_backends(config_dir)
+ inspection = _inspect(focus=focus)
+
+ assert not inspection.asks_setup_path
+ assert inspection.kept_execution is None
+
+ @pytest.mark.usefixtures("config_dir")
+ def test_without_anyone_to_answer_a_brand_new_home_takes_the_hosted_default(self, mocker: MockerFixture) -> None:
+ prompt = mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path")
+ confirm = mocker.patch("pipelex.cli.commands.init.command.Confirm.ask")
+
+ choices = choose_initialization(console=Console(quiet=True), inspection=_inspect(focus=InitFocus.CONFIG), skip_confirmation=True)
+
+ assert choices.setup_path == SetupPath.HOSTED
+ assert not choices.interactive
+ prompt.assert_not_called()
+ confirm.assert_not_called()
+
+ @pytest.mark.parametrize("content", ["# written before [run] existed\n[log]\nlevel = 'info'\n", "", "[run\nbroken"])
+ def test_without_anyone_to_answer_an_existing_pipelex_toml_without_the_setting_stays_local(
+ self, config_dir: Path, mocker: MockerFixture, content: str
+ ) -> None:
+ """A `pipelex.toml` older than `[run]` runs on this machine, the package default, so a repair keeps it there."""
+ config_dir.mkdir(parents=True)
+ (config_dir / "pipelex.toml").write_text(content, encoding="utf-8")
+ mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path")
+ inspection = _inspect(focus=InitFocus.CONFIG)
+ assert inspection.asks_setup_path
+ assert inspection.configured_execution is None
+
+ choices = choose_initialization(console=Console(quiet=True), inspection=inspection, skip_confirmation=True)
+
+ assert choices.setup_path == SetupPath.LOCAL
+
+ @pytest.mark.parametrize("configured", [RunExecution.LOCAL, RunExecution.HOSTED])
+ def test_without_anyone_to_answer_a_configured_setting_is_kept(self, config_dir: Path, mocker: MockerFixture, configured: RunExecution) -> None:
+ """`pipelex doctor --fix` installs what is missing; it never turns runs on this machine into hosted ones."""
+ write_run_execution(pipelex_toml_path=config_dir / "pipelex.toml", execution=configured)
+ prompt = mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path")
+ inspection = _inspect(focus=InitFocus.CONFIG)
+ assert inspection.asks_setup_path
+
+ choices = choose_initialization(console=Console(quiet=True), inspection=inspection, skip_confirmation=True)
+
+ assert choices.setup_path is not None
+ assert choices.setup_path == SetupPath.from_run_execution(execution=configured)
+ assert choices.setup_path.run_execution == configured
+ prompt.assert_not_called()
+
+ @pytest.mark.usefixtures("config_dir")
+ def test_the_question_follows_the_confirmation(self, mocker: MockerFixture) -> None:
+ calls: list[str] = []
+
+ def confirm(*_args: object, **_kwargs: object) -> bool:
+ calls.append("confirm")
+ return True
+
+ def ask_setup_path(**_kwargs: object) -> SetupPath:
+ calls.append("setup_path")
+ return SetupPath.LOCAL
+
+ mocker.patch("pipelex.cli.commands.init.command.Confirm.ask", side_effect=confirm)
+ mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path", side_effect=ask_setup_path)
+
+ choices = choose_initialization(console=Console(quiet=True), inspection=_inspect(focus=InitFocus.ALL), skip_confirmation=False)
+
+ assert calls == ["confirm", "setup_path"]
+ assert choices.setup_path == SetupPath.LOCAL
+ assert choices.interactive
+
+ def test_a_run_that_does_not_decide_it_asks_nothing_about_it(self, config_dir: Path, mocker: MockerFixture) -> None:
+ _install_backends(config_dir)
+ mocker.patch("pipelex.cli.commands.init.command.Confirm.ask", return_value=True)
+ prompt = mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path")
+
+ choices = choose_initialization(console=Console(quiet=True), inspection=_inspect(focus=InitFocus.TELEMETRY), skip_confirmation=False)
+
+ assert choices.setup_path is None
+ prompt.assert_not_called()
+
+ def test_the_inspection_finds_what_a_former_release_left_and_writes_nothing(self, config_dir: Path) -> None:
+ _install_a_former_release(config_dir)
+ before = _snapshot(directory=config_dir)
+
+ inspection = _inspect(focus=InitFocus.ALL)
+
+ assert [findings.config_dir for findings in inspection.former_release_findings] == [config_dir]
+ assert inspection.former_release_boot_blockers
+ assert _snapshot(directory=config_dir) == before
+
+ def test_a_project_booting_on_a_base_of_its_own_is_not_told_it_cannot_start(
+ self, config_dir: Path, config_manager: MagicMock, tmp_path: Path
+ ) -> None:
+ """The verdict is the boot's: the files it merges are the project's bases, which a former release never touched."""
+ _install_a_former_release(config_dir)
+ project_inference = tmp_path / "project" / ".pipelex" / "inference"
+ shutil.copytree(Path(str(get_kit_configs_dir())) / "inference", project_inference)
+ config_manager.backends_file_paths.return_value = [project_inference / "backends.toml", config_dir / "inference" / "backends_override.toml"]
+ config_manager.routing_profiles_file_paths.return_value = [
+ project_inference / "routing_profiles.toml",
+ config_dir / "inference" / "routing_profiles_override.toml",
+ ]
+
+ inspection = _inspect(focus=InitFocus.ALL)
+
+ assert inspection.former_release_findings, "the home still carries what the release left"
+ assert inspection.former_release_boot_blockers == []
+ description = describe_former_release_findings(findings=inspection.former_release_findings, blockers=inspection.former_release_boot_blockers)
+ assert "no longer stops Pipelex from starting" in description
+
+ def test_a_clean_target_has_no_former_release_findings(self, config_dir: Path) -> None:
+ _install_backends(config_dir)
+ assert _inspect(focus=InitFocus.ALL).former_release_findings == []
+
+ def test_a_former_release_is_cleaned_up_on_yes_before_the_setup_question(self, config_dir: Path, mocker: MockerFixture) -> None:
+ _install_a_former_release(config_dir)
+ calls: list[str] = []
+
+ def confirm(prompt: str, **_kwargs: object) -> bool:
+ calls.append("clean_up" if "Clean it up" in prompt else "confirm")
+ return True
+
+ def ask_setup_path(**_kwargs: object) -> SetupPath:
+ calls.append("setup_path")
+ return SetupPath.LOCAL
+
+ mocker.patch("pipelex.cli.commands.init.command.Confirm.ask", side_effect=confirm)
+ mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path", side_effect=ask_setup_path)
+ inspection = _inspect(focus=InitFocus.ALL)
+
+ choose_initialization(console=Console(quiet=True), inspection=inspection, skip_confirmation=False)
+
+ assert calls == ["clean_up", "confirm", "setup_path"]
+ assert detect_former_release(config_dir=config_dir).is_clean
+
+ def test_a_no_leaves_the_files_as_they_are_and_the_setup_goes_on(self, config_dir: Path, mocker: MockerFixture) -> None:
+ _install_a_former_release(config_dir)
+ before = _snapshot(directory=config_dir)
+
+ def decline_the_cleanup_only(prompt: str, **_kwargs: object) -> bool:
+ return "Clean it up" not in prompt
+
+ mocker.patch("pipelex.cli.commands.init.command.Confirm.ask", side_effect=decline_the_cleanup_only)
+ prompt = mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path", return_value=SetupPath.LOCAL)
+
+ choices = choose_initialization(console=Console(quiet=True), inspection=_inspect(focus=InitFocus.ALL), skip_confirmation=False)
+
+ assert _snapshot(directory=config_dir) == before
+ prompt.assert_called_once()
+ assert choices.setup_path == SetupPath.LOCAL
+
+ def test_without_anyone_to_answer_the_cleanup_never_runs(self, config_dir: Path, mocker: MockerFixture) -> None:
+ """Nobody answers when `pipelex doctor --fix` calls init, and a cleanup nobody agreed to is not init's to run.
+
+ The doctor asks its own question about the cleanup; whatever was answered there, init's unattended run leaves
+ every file a former release left exactly as it found it.
+ """
+ _install_a_former_release(config_dir)
+ before = _snapshot(directory=config_dir)
+ confirm = mocker.patch("pipelex.cli.commands.init.command.Confirm.ask")
+
+ choose_initialization(console=Console(quiet=True), inspection=_inspect(focus=InitFocus.CONFIG), skip_confirmation=True)
+
+ confirm.assert_not_called()
+ assert _snapshot(directory=config_dir) == before
+ assert not detect_former_release(config_dir=config_dir).is_clean
+
+ @pytest.mark.parametrize("project_has_its_own_bases", [False, True])
+ def test_a_no_says_pipelex_will_not_start_only_when_the_boot_would_refuse(
+ self, config_dir: Path, config_manager: MagicMock, tmp_path: Path, mocker: MockerFixture, project_has_its_own_bases: bool
+ ) -> None:
+ _install_a_former_release(config_dir)
+ if project_has_its_own_bases:
+ project_inference = tmp_path / "project" / ".pipelex" / "inference"
+ shutil.copytree(Path(str(get_kit_configs_dir())) / "inference", project_inference)
+ config_manager.backends_file_paths.return_value = [project_inference / "backends.toml"]
+ config_manager.routing_profiles_file_paths.return_value = [project_inference / "routing_profiles.toml"]
+
+ def decline_the_cleanup_only(prompt: str, **_kwargs: object) -> bool:
+ return "Clean it up" not in prompt
+
+ mocker.patch("pipelex.cli.commands.init.command.Confirm.ask", side_effect=decline_the_cleanup_only)
+ mocker.patch("pipelex.cli.commands.init.command.prompt_setup_path", return_value=SetupPath.LOCAL)
+ console = Console(record=True, width=400, color_system=None)
+
+ choose_initialization(console=console, inspection=_inspect(focus=InitFocus.ALL), skip_confirmation=False)
+
+ assert ("Pipelex will not start until then." in console.export_text()) is not project_has_its_own_bases
diff --git a/tests/unit/pipelex/cli/commands/init/test_initialization_panel.py b/tests/unit/pipelex/cli/commands/init/test_initialization_panel.py
new file mode 100644
index 000000000..745f82c97
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_initialization_panel.py
@@ -0,0 +1,45 @@
+"""The confirmation panel says what a run will do, a reset of the inference backends to the kit's defaults included."""
+
+from __future__ import annotations
+
+import pytest
+from rich.console import Console
+
+from pipelex.cli.commands.init.ui.general_ui import build_initialization_panel
+
+
+def _panel_text(*, asks_setup_path: bool, reset: bool, needs_inference: bool = True) -> str:
+ console = Console(width=300, record=True, color_system=None)
+ console.print(
+ build_initialization_panel(
+ needs_config=True,
+ needs_inference=needs_inference,
+ needs_routing=False,
+ needs_telemetry=True,
+ reset=reset,
+ check_credentials=True,
+ asks_setup_path=asks_setup_path,
+ )
+ )
+ return console.export_text()
+
+
+class TestInitializationPanel:
+ @pytest.mark.parametrize("asks_setup_path", [True, False])
+ def test_a_reset_that_touches_inference_says_the_backends_go_back_to_the_kit_defaults(self, asks_setup_path: bool) -> None:
+ text = _panel_text(asks_setup_path=asks_setup_path, reset=True)
+
+ assert "inference backends" in text
+ assert "kit's defaults" in text
+
+ def test_the_question_names_both_paths(self) -> None:
+ text = _panel_text(asks_setup_path=True, reset=True)
+
+ assert "on the hosted Pipelex API" in text
+ assert "on this machine" in text
+ assert "pipelex init inference" in text
+
+ def test_a_run_that_does_not_touch_inference_says_nothing_about_the_backends(self) -> None:
+ text = _panel_text(asks_setup_path=False, reset=True, needs_inference=False)
+
+ assert "kit's defaults" not in text
diff --git a/tests/unit/pipelex/cli/commands/init/test_project_execution_shadow.py b/tests/unit/pipelex/cli/commands/init/test_project_execution_shadow.py
new file mode 100644
index 000000000..b1a29cda8
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_project_execution_shadow.py
@@ -0,0 +1,73 @@
+"""A global choice of where runs execute is overridden in a project whose `.pipelex/pipelex.toml` sets another: init says so."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.cli.commands.init.command import InitChoices, execute_initialization, inspect_initialization
+from pipelex.cli.commands.init.setup_path import SetupPath, find_project_execution_shadow, read_run_execution, write_run_execution
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.hosted.run_config import RunExecution
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+ from tests.helpers.recorded_console import RecordedConsole
+
+
+def project_config_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, pipelex_home: Path) -> Path:
+ """A project (a `.git` marker) whose `.pipelex/pipelex.toml` runs on this machine, as the working directory.
+
+ It depends on `pipelex_home` so that its change of directory comes after that fixture's.
+ """
+ assert not pipelex_home.exists()
+ project_root = tmp_path / "project"
+ (project_root / ".git").mkdir(parents=True)
+ config_dir = project_root / ".pipelex"
+ write_run_execution(pipelex_toml_path=config_dir / "pipelex.toml", execution=RunExecution.LOCAL)
+ monkeypatch.chdir(project_root)
+ return config_dir
+
+
+class TestProjectExecutionShadow:
+ def test_a_project_setting_another_execution_is_found(self, project_config_dir: Path, pipelex_home: Path) -> None:
+ shadow = find_project_execution_shadow(target_config_dir=pipelex_home, execution=RunExecution.HOSTED, project_config_dir=project_config_dir)
+
+ assert shadow is not None
+ assert shadow.pipelex_toml_path == project_config_dir / "pipelex.toml"
+ assert shadow.execution == RunExecution.LOCAL
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_the_same_execution_or_the_project_itself_is_no_shadow(self, project_config_dir: Path, pipelex_home: Path) -> None:
+ same = find_project_execution_shadow(target_config_dir=pipelex_home, execution=RunExecution.LOCAL, project_config_dir=project_config_dir)
+ itself = find_project_execution_shadow(
+ target_config_dir=project_config_dir, execution=RunExecution.HOSTED, project_config_dir=project_config_dir
+ )
+ none = find_project_execution_shadow(target_config_dir=pipelex_home, execution=RunExecution.HOSTED, project_config_dir=None)
+
+ assert same is None
+ assert itself is None
+ assert none is None
+
+ def test_a_global_hosted_init_names_the_project_that_overrides_it(
+ self, mocker: MockerFixture, project_config_dir: Path, pipelex_home: Path, recorded_console: RecordedConsole
+ ) -> None:
+ mocker.patch("pipelex.cli.commands.init.command.ensure_pipelex_api_key")
+ inspection = inspect_initialization(focus=InitFocus.ALL, local=False)
+ assert inspection.target_config_dir == pipelex_home
+
+ execute_initialization(
+ console=recorded_console.console, inspection=inspection, choices=InitChoices(setup_path=SetupPath.HOSTED, interactive=False)
+ )
+
+ assert read_run_execution(pipelex_toml_path=pipelex_home / "pipelex.toml") == RunExecution.HOSTED
+ # Rich wraps long lines at spaces; joining the words back reads the message as written
+ printed = " ".join(recorded_console.text().split())
+ assert str(project_config_dir / "pipelex.toml") in printed
+ assert 'execution = "local"' in printed
+ assert "--hosted" in printed
diff --git a/tests/unit/pipelex/cli/commands/init/test_run_execution_setting.py b/tests/unit/pipelex/cli/commands/init/test_run_execution_setting.py
new file mode 100644
index 000000000..e2f7ac046
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_run_execution_setting.py
@@ -0,0 +1,54 @@
+"""`[run] execution` is read from and written to a `pipelex.toml`, every other line kept."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.cli.commands.init.setup_path import read_run_execution, write_run_execution
+from pipelex.cli.exceptions import PipelexCLIError
+from pipelex.hosted.run_config import RunExecution
+from pipelex.kit.paths import get_kit_configs_dir
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+class TestRunExecutionSetting:
+ def test_it_rewrites_the_kit_setting_keeping_the_comments(self, tmp_path: Path) -> None:
+ pipelex_toml_path = tmp_path / "pipelex.toml"
+ kit_text = (get_kit_configs_dir() / "pipelex.toml").read_text(encoding="utf-8")
+ pipelex_toml_path.write_text(kit_text, encoding="utf-8")
+ assert read_run_execution(pipelex_toml_path=pipelex_toml_path) == RunExecution.LOCAL
+
+ write_run_execution(pipelex_toml_path=pipelex_toml_path, execution=RunExecution.HOSTED)
+
+ assert read_run_execution(pipelex_toml_path=pipelex_toml_path) == RunExecution.HOSTED
+ rewritten = pipelex_toml_path.read_text(encoding="utf-8")
+ assert rewritten.replace('execution = "hosted"', 'execution = "local"') == kit_text
+
+ def test_it_creates_the_file_or_the_table_when_missing(self, tmp_path: Path) -> None:
+ missing = tmp_path / "missing" / "pipelex.toml"
+ write_run_execution(pipelex_toml_path=missing, execution=RunExecution.HOSTED)
+ assert read_run_execution(pipelex_toml_path=missing) == RunExecution.HOSTED
+
+ without_run = tmp_path / "without_run.toml"
+ without_run.write_text("# mine\n[log]\nlevel = 'info'\n", encoding="utf-8")
+ write_run_execution(pipelex_toml_path=without_run, execution=RunExecution.LOCAL)
+ assert read_run_execution(pipelex_toml_path=without_run) == RunExecution.LOCAL
+ assert without_run.read_text(encoding="utf-8").startswith("# mine\n[log]\nlevel = 'info'\n")
+
+ def test_a_run_that_is_not_a_table_is_refused(self, tmp_path: Path) -> None:
+ pipelex_toml_path = tmp_path / "pipelex.toml"
+ pipelex_toml_path.write_text('run = "hosted"\n', encoding="utf-8")
+
+ with pytest.raises(PipelexCLIError):
+ write_run_execution(pipelex_toml_path=pipelex_toml_path, execution=RunExecution.HOSTED)
+
+ @pytest.mark.parametrize("content", [None, "", "[run]\n", '[run]\nexecution = "elsewhere"\n', "[run\nbroken", 'run = "hosted"\n'])
+ def test_reading_finds_none_when_nothing_valid_is_set(self, tmp_path: Path, content: str | None) -> None:
+ pipelex_toml_path = tmp_path / "pipelex.toml"
+ if content is not None:
+ pipelex_toml_path.write_text(content, encoding="utf-8")
+ assert read_run_execution(pipelex_toml_path=pipelex_toml_path) is None
diff --git a/tests/unit/pipelex/cli/commands/init/test_setup_path_prompt.py b/tests/unit/pipelex/cli/commands/init/test_setup_path_prompt.py
new file mode 100644
index 000000000..d72da24ca
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/init/test_setup_path_prompt.py
@@ -0,0 +1,52 @@
+"""`pipelex init` asks where runs execute, the hosted Pipelex API listed first and taken by Enter."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.cli.commands.init.setup_path import DEFAULT_SETUP_PATH, SetupPath
+from pipelex.cli.commands.init.ui.setup_path_ui import SETUP_PATH_QUESTION, prompt_setup_path
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+ from tests.helpers.recorded_console import RecordedConsole
+
+
+class TestSetupPathPrompt:
+ def test_hosted_is_the_default(self) -> None:
+ assert DEFAULT_SETUP_PATH == SetupPath.HOSTED
+
+ def test_enter_takes_the_hosted_api_listed_first(self, mocker: MockerFixture, recorded_console: RecordedConsole) -> None:
+ def press_enter(*_args: object, default: str, **_kwargs: object) -> str:
+ return default
+
+ prompt = mocker.patch("pipelex.cli.commands.init.ui.setup_path_ui.Prompt.ask", side_effect=press_enter)
+
+ assert prompt_setup_path(console=recorded_console.console) == SetupPath.HOSTED
+
+ assert prompt.call_args.kwargs["default"] == "1"
+ shown = recorded_console.text()
+ assert SETUP_PATH_QUESTION in shown
+ assert shown.index("On the hosted Pipelex API, with a Pipelex API key") < shown.index("On this machine, with your own provider keys")
+
+ @pytest.mark.parametrize(
+ ("answer", "expected"),
+ [("", SetupPath.HOSTED), ("1", SetupPath.HOSTED), ("2", SetupPath.LOCAL), ("local", SetupPath.LOCAL), (" Hosted ", SetupPath.HOSTED)],
+ )
+ def test_a_number_or_a_name_picks_the_path(
+ self, mocker: MockerFixture, recorded_console: RecordedConsole, answer: str, expected: SetupPath
+ ) -> None:
+ mocker.patch("pipelex.cli.commands.init.ui.setup_path_ui.Prompt.ask", return_value=answer)
+
+ assert prompt_setup_path(console=recorded_console.console) == expected
+
+ def test_anything_else_asks_again(self, mocker: MockerFixture, recorded_console: RecordedConsole) -> None:
+ prompt = mocker.patch("pipelex.cli.commands.init.ui.setup_path_ui.Prompt.ask", side_effect=["3", "[bold]x", "2"])
+
+ assert prompt_setup_path(console=recorded_console.console) == SetupPath.LOCAL
+
+ assert prompt.call_count == 3
+ assert "Invalid choice" in recorded_console.text()
diff --git a/tests/unit/pipelex/cli/commands/login/conftest.py b/tests/unit/pipelex/cli/commands/login/conftest.py
new file mode 100644
index 000000000..8504c007d
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/login/conftest.py
@@ -0,0 +1,42 @@
+"""A home configuration directory, a working directory and a console of each test's own, for the login tests."""
+
+from __future__ import annotations
+
+from io import StringIO
+from typing import TYPE_CHECKING
+
+import pytest
+from rich.console import Console
+
+from pipelex.cli.commands.login.command import PIPELEX_APP_URL_ENV_KEY
+from pipelex.system.environment import PIPELEX_HOME_ENV_KEY
+from tests.helpers.pipelex_api_key_env import isolate_pipelex_api_key
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+
+def pipelex_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ """A home configuration directory of the test's own, an empty working directory, and no key in the environment.
+
+ The working directory matters: the runtime loads its `.env` after the home one, so a key set there would be reported.
+ """
+ home = tmp_path / "pipelex_home"
+ working_dir = tmp_path / "work"
+ working_dir.mkdir()
+ monkeypatch.chdir(working_dir)
+ monkeypatch.setenv(PIPELEX_HOME_ENV_KEY, str(home))
+ monkeypatch.delenv(PIPELEX_APP_URL_ENV_KEY, raising=False)
+ isolate_pipelex_api_key(monkeypatch)
+ return home
+
+
+def output(mocker: MockerFixture) -> StringIO:
+ """What `pipelex login` prints."""
+ buffer = StringIO()
+ mocker.patch("pipelex.cli.commands.login.command.get_console", return_value=Console(file=buffer, width=200))
+ return buffer
diff --git a/tests/unit/pipelex/cli/commands/login/test_api_key_store.py b/tests/unit/pipelex/cli/commands/login/test_api_key_store.py
new file mode 100644
index 000000000..d0bf0dd8b
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/login/test_api_key_store.py
@@ -0,0 +1,150 @@
+"""The key is saved to the home `.env` as `PIPELEX_API_KEY`, readable by its owner only, every other line kept."""
+
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+
+import pytest
+
+from pipelex.cli.commands.init.credentials import set_env_file_entry
+from pipelex.cli.commands.login.api_key_store import (
+ find_pipelex_api_key,
+ find_shadowing_env_file,
+ read_saved_pipelex_api_key,
+ save_pipelex_api_key,
+)
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from tests.helpers.login_browser import TEST_KEY
+
+
+def _mode(path: Path) -> int:
+ return stat.S_IMODE(path.stat().st_mode)
+
+
+class TestApiKeyStore:
+ def test_it_saves_to_the_env_of_the_home_named_by_pipelex_home(self, pipelex_home: Path) -> None:
+ env_path = save_pipelex_api_key(api_key=TEST_KEY)
+
+ assert env_path == pipelex_home / ".env"
+ assert f"{PIPELEX_API_KEY_ENV_KEY}={TEST_KEY}" in env_path.read_text(encoding="utf-8").splitlines()
+ assert _mode(env_path) == 0o600
+ assert os.environ[PIPELEX_API_KEY_ENV_KEY] == TEST_KEY
+
+ def test_it_keeps_every_other_line_and_tightens_the_mode(self, pipelex_home: Path) -> None:
+ pipelex_home.mkdir()
+ env_path = pipelex_home / ".env"
+ env_path.write_text("# my providers\nOPENAI_API_KEY=sk-openai\n\nexport MISTRAL_API_KEY=mistral\n", encoding="utf-8")
+ env_path.chmod(0o644)
+
+ save_pipelex_api_key(api_key=TEST_KEY)
+
+ lines = env_path.read_text(encoding="utf-8").splitlines()
+ assert lines[:4] == ["# my providers", "OPENAI_API_KEY=sk-openai", "", "export MISTRAL_API_KEY=mistral"]
+ assert lines[4] == f"{PIPELEX_API_KEY_ENV_KEY}={TEST_KEY}"
+ assert _mode(env_path) == 0o600
+
+ def test_it_replaces_every_earlier_assignment_of_the_key(self, pipelex_home: Path) -> None:
+ """A later duplicate would shadow the new key when the file is loaded, so none is left."""
+ pipelex_home.mkdir()
+ env_path = pipelex_home / ".env"
+ env_path.write_text(
+ f"{PIPELEX_API_KEY_ENV_KEY}=plx_sk_old\nOTHER=1\nexport {PIPELEX_API_KEY_ENV_KEY}='plx_sk_older'\n",
+ encoding="utf-8",
+ )
+
+ save_pipelex_api_key(api_key=TEST_KEY)
+
+ content = env_path.read_text(encoding="utf-8")
+ assert "plx_sk_old" not in content
+ assert content.count(PIPELEX_API_KEY_ENV_KEY) == 2
+ assert read_saved_pipelex_api_key() == TEST_KEY
+ assert "OTHER=1" in content.splitlines()
+
+ def test_none_when_neither_the_environment_nor_the_file_holds_one(self, pipelex_home: Path) -> None:
+ assert pipelex_home.exists() is False
+ assert find_pipelex_api_key() is None
+
+ def test_the_environment_wins(self, pipelex_home: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="plx_sk_saved")
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "plx_sk_exported")
+
+ assert find_pipelex_api_key() == "plx_sk_exported"
+
+ def test_the_saved_key_is_found_when_the_environment_has_none(self, pipelex_home: Path) -> None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="plx_sk_saved")
+
+ assert find_pipelex_api_key() == "plx_sk_saved"
+
+ def test_a_symlinked_env_stays_a_symlink_and_its_target_holds_the_key(self, pipelex_home: Path, tmp_path: Path) -> None:
+ """A dotfiles setup links `~/.pipelex/.env` elsewhere: the save writes through the link instead of replacing it."""
+ target = tmp_path / "dotfiles" / "pipelex.env"
+ target.parent.mkdir()
+ target.write_text("OPENAI_API_KEY=sk-openai\n", encoding="utf-8")
+ pipelex_home.mkdir()
+ link = pipelex_home / ".env"
+ link.symlink_to(target)
+
+ save_pipelex_api_key(api_key=TEST_KEY)
+
+ assert link.is_symlink()
+ assert link.resolve() == target.resolve()
+ assert target.read_text(encoding="utf-8").splitlines() == ["OPENAI_API_KEY=sk-openai", f"{PIPELEX_API_KEY_ENV_KEY}={TEST_KEY}"]
+
+ def test_an_empty_key_in_the_environment_is_no_key_even_with_one_saved(self, pipelex_home: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """An empty `PIPELEX_API_KEY=` in the working directory's `.env` is what the next process sends, so it is no key."""
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="plx_sk_saved")
+ Path(".env").write_text(f"{PIPELEX_API_KEY_ENV_KEY}=\n", encoding="utf-8")
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "")
+
+ assert find_pipelex_api_key() is None
+
+ def test_no_working_directory_env_shadows_nothing(self, pipelex_home: Path) -> None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="plx_sk_saved")
+
+ assert find_shadowing_env_file() is None
+
+ @pytest.mark.parametrize(
+ ("project_line", "saved", "sets_empty_value"),
+ [
+ (f"{PIPELEX_API_KEY_ENV_KEY}=", "plx_sk_saved", True),
+ (f"{PIPELEX_API_KEY_ENV_KEY}=", None, True),
+ (f"{PIPELEX_API_KEY_ENV_KEY}=plx_sk_other", "plx_sk_saved", False),
+ ],
+ )
+ def test_a_working_directory_env_setting_another_value_shadows_the_saved_key(
+ self, pipelex_home: Path, project_line: str, saved: str | None, sets_empty_value: bool
+ ) -> None:
+ if saved is not None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value=saved)
+ Path(".env").write_text(f"OTHER=1\n{project_line}\n", encoding="utf-8")
+
+ shadow = find_shadowing_env_file()
+
+ assert shadow is not None
+ assert shadow.path == Path(".env").resolve()
+ assert shadow.sets_empty_value is sets_empty_value
+
+ @pytest.mark.parametrize(
+ ("project_content", "saved"),
+ [
+ ("OTHER=1\n", "plx_sk_saved"),
+ (f"{PIPELEX_API_KEY_ENV_KEY}=plx_sk_saved\n", "plx_sk_saved"),
+ (f"{PIPELEX_API_KEY_ENV_KEY}=plx_sk_project_only\n", None),
+ ],
+ )
+ def test_a_working_directory_env_agreeing_with_the_saved_key_shadows_nothing(
+ self, pipelex_home: Path, project_content: str, saved: str | None
+ ) -> None:
+ if saved is not None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value=saved)
+ Path(".env").write_text(project_content, encoding="utf-8")
+
+ assert find_shadowing_env_file() is None
+
+ def test_the_home_env_is_not_its_own_shadow(self, pipelex_home: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ set_env_file_entry(env_path=pipelex_home / ".env", key=PIPELEX_API_KEY_ENV_KEY, value="")
+ monkeypatch.chdir(pipelex_home)
+
+ assert find_shadowing_env_file() is None
diff --git a/tests/unit/pipelex/cli/commands/login/test_app_origin.py b/tests/unit/pipelex/cli/commands/login/test_app_origin.py
new file mode 100644
index 000000000..76e352137
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/login/test_app_origin.py
@@ -0,0 +1,33 @@
+"""`pipelex login` opens the Pipelex app, or the one `PIPELEX_APP_URL` names when it is an origin."""
+
+from __future__ import annotations
+
+import pytest
+
+from pipelex.cli.commands.login.command import PIPELEX_APP_URL_ENV_KEY, build_cli_auth_url, resolve_app_origin
+from pipelex.cli.exceptions import PipelexCLIError
+
+
+class TestAppOrigin:
+ def test_the_default_is_the_pipelex_app(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv(PIPELEX_APP_URL_ENV_KEY, raising=False)
+ assert resolve_app_origin() == "https://app.pipelex.com"
+
+ @pytest.mark.parametrize("value", ["https://app-dev.pipelex.com", "https://app-dev.pipelex.com/", "http://localhost:3000"])
+ def test_pipelex_app_url_names_another_app(self, monkeypatch: pytest.MonkeyPatch, value: str) -> None:
+ monkeypatch.setenv(PIPELEX_APP_URL_ENV_KEY, value)
+ assert resolve_app_origin() == value.rstrip("/")
+
+ @pytest.mark.parametrize(
+ "value", ["", "app.pipelex.com", "ftp://app.test", "https://app.test/auth", "https://user:[email protected]", "https://app.test?x=1"]
+ )
+ def test_a_value_that_is_not_an_origin_is_refused_without_quoting_credentials(self, monkeypatch: pytest.MonkeyPatch, value: str) -> None:
+ monkeypatch.setenv(PIPELEX_APP_URL_ENV_KEY, value)
+ with pytest.raises(PipelexCLIError) as exc_info:
+ resolve_app_origin()
+ assert PIPELEX_APP_URL_ENV_KEY in exc_info.value.message
+ assert "secret" not in exc_info.value.message
+
+ def test_the_auth_url_names_the_port_and_the_state(self) -> None:
+ url = build_cli_auth_url(app_origin="https://app-dev.pipelex.com", callback_port=51234, state="abc")
+ assert url == "https://app-dev.pipelex.com/auth/cli?callback_port=51234&state=abc"
diff --git a/tests/unit/pipelex/cli/commands/login/test_login_with_browser.py b/tests/unit/pipelex/cli/commands/login/test_login_with_browser.py
new file mode 100644
index 000000000..7b4599e55
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/login/test_login_with_browser.py
@@ -0,0 +1,162 @@
+"""`pipelex login` opens the app with a fresh `state`, waits for the key, checks it and saves it, never printing it."""
+
+from __future__ import annotations
+
+import stat
+from io import StringIO
+from typing import TYPE_CHECKING
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+import typer
+from rich.console import Console
+
+from pipelex.cli.commands.login.command import LOGIN_PASTE_COMMAND, PIPELEX_APP_URL_ENV_KEY, LoginOutcome, login_cmd, login_with_browser
+from pipelex.hosted.api_key_check import ApiKeyCheck, ApiKeyVerdict
+from tests.helpers.login_browser import ACCEPTED, API_URL, TEST_KEY, BrowserStandIn, patch_browser, patch_check, saved_key
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+
+class TestLoginWithBrowser:
+ def test_a_checked_key_is_saved_and_never_printed(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ browser = BrowserStandIn(api_key=TEST_KEY)
+ patch_browser(mocker, browser=browser)
+ check = patch_check(mocker, check=ACCEPTED)
+
+ login_cmd()
+ browser.join()
+
+ check.assert_called_once_with(api_key=TEST_KEY)
+ assert saved_key(home=pipelex_home) == TEST_KEY
+ assert stat.S_IMODE((pipelex_home / ".env").stat().st_mode) == 0o600
+ printed = output.getvalue()
+ assert TEST_KEY not in printed
+ assert "[email protected]" in printed
+ assert len(browser.opened_urls) == 1
+ assert browser.opened_urls[0] in printed
+
+ @pytest.mark.usefixtures("output")
+ def test_it_opens_the_app_pipelex_app_url_names(
+ self,
+ mocker: MockerFixture,
+ pipelex_home: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ monkeypatch.setenv(PIPELEX_APP_URL_ENV_KEY, "https://app-dev.pipelex.com")
+ browser = BrowserStandIn(api_key=TEST_KEY)
+ patch_browser(mocker, browser=browser)
+ patch_check(mocker, check=ACCEPTED)
+
+ login_cmd()
+ browser.join()
+
+ parts = urlsplit(browser.opened_urls[0])
+ assert f"{parts.scheme}://{parts.netloc}{parts.path}" == "https://app-dev.pipelex.com/auth/cli"
+ assert set(parse_qs(parts.query)) == {"callback_port", "state"}
+ assert saved_key(home=pipelex_home) == TEST_KEY
+
+ def test_a_key_the_hosted_api_refuses_is_not_saved(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ browser = BrowserStandIn(api_key=TEST_KEY)
+ patch_browser(mocker, browser=browser)
+ patch_check(mocker, check=ApiKeyCheck(verdict=ApiKeyVerdict.REFUSED, base_url=API_URL, http_status=401))
+
+ with pytest.raises(typer.Exit) as exc_info:
+ login_cmd()
+ browser.join()
+
+ assert exc_info.value.exit_code == 1
+ assert saved_key(home=pipelex_home) is None
+ assert "HTTP 401" in output.getvalue()
+ assert TEST_KEY not in output.getvalue()
+
+ def test_a_key_that_could_not_be_checked_is_saved_with_a_warning(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ browser = BrowserStandIn(api_key=TEST_KEY)
+ patch_browser(mocker, browser=browser)
+ patch_check(
+ mocker,
+ check=ApiKeyCheck(verdict=ApiKeyVerdict.UNCHECKED, base_url=API_URL, reason="the hosted API at https://api.test could not be reached"),
+ )
+
+ login_cmd()
+ browser.join()
+
+ assert saved_key(home=pipelex_home) == TEST_KEY
+ assert "Could not check the key" in output.getvalue()
+ assert "could not be reached" in output.getvalue()
+
+ def test_a_value_that_is_not_a_pipelex_api_key_is_refused_unchecked(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ browser = BrowserStandIn(api_key="sk-not-a-pipelex-key")
+ patch_browser(mocker, browser=browser)
+ check = patch_check(mocker, check=ACCEPTED)
+
+ with pytest.raises(typer.Exit):
+ login_cmd()
+ browser.join()
+
+ check.assert_not_called()
+ assert saved_key(home=pipelex_home) is None
+ assert "plx_sk_" in output.getvalue()
+
+ def test_a_callback_with_a_forged_state_is_refused_and_the_login_times_out(self, mocker: MockerFixture, pipelex_home: Path) -> None:
+ buffer = StringIO()
+ console = Console(file=buffer, width=200)
+ browser = BrowserStandIn(api_key=TEST_KEY, echo_state=False)
+ patch_browser(mocker, browser=browser)
+ check = patch_check(mocker, check=ACCEPTED)
+
+ outcome = login_with_browser(console=console, timeout_seconds=1)
+ browser.join()
+
+ assert outcome == LoginOutcome.NO_KEY
+ check.assert_not_called()
+ assert saved_key(home=pipelex_home) is None
+ assert "did not come from the page this login opened" in buffer.getvalue()
+
+ def test_the_timeout_exits_1_and_names_paste(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ patch_browser(mocker, browser=BrowserStandIn(api_key=None))
+ mocker.patch("pipelex.cli.commands.login.command.LOGIN_TIMEOUT_SECONDS", 0.2)
+
+ with pytest.raises(typer.Exit) as exc_info:
+ login_cmd()
+
+ assert exc_info.value.exit_code == 1
+ assert LOGIN_PASTE_COMMAND in output.getvalue()
+ assert saved_key(home=pipelex_home) is None
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_a_browser_that_does_not_open_says_so_at_once_and_names_paste(self, mocker: MockerFixture) -> None:
+ buffer = StringIO()
+ console = Console(file=buffer, width=200)
+ browser = BrowserStandIn(api_key=None, opens=False)
+ patch_browser(mocker, browser=browser)
+
+ outcome = login_with_browser(console=console, timeout_seconds=0.3)
+
+ assert outcome == LoginOutcome.NO_KEY
+ printed = buffer.getvalue()
+ assert "No browser could be opened" in printed
+ timeout_at = printed.index("No key arrived")
+ assert printed.index("No browser could be opened") < timeout_at
+ assert printed.index(LOGIN_PASTE_COMMAND) < timeout_at
+ assert printed.index(browser.opened_urls[0]) < timeout_at
+
+ @pytest.mark.usefixtures("pipelex_home")
+ def test_an_invalid_pipelex_app_url_opens_nothing(
+ self,
+ mocker: MockerFixture,
+ output: StringIO,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ monkeypatch.setenv(PIPELEX_APP_URL_ENV_KEY, "https://app.test/auth/cli")
+ browser = BrowserStandIn(api_key=TEST_KEY)
+ patch_browser(mocker, browser=browser)
+
+ with pytest.raises(typer.Exit):
+ login_cmd()
+
+ assert browser.opened_urls == []
+ assert PIPELEX_APP_URL_ENV_KEY in output.getvalue()
diff --git a/tests/unit/pipelex/cli/commands/login/test_login_with_paste.py b/tests/unit/pipelex/cli/commands/login/test_login_with_paste.py
new file mode 100644
index 000000000..44d74a2d6
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/login/test_login_with_paste.py
@@ -0,0 +1,112 @@
+"""`pipelex login --paste` takes a key from a hidden prompt through the same check and save as the browser flow."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+import typer
+
+from pipelex.cli.commands.login.command import login_cmd
+from pipelex.hosted.api_key_check import ApiKeyCheck, ApiKeyVerdict
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from tests.helpers.login_browser import ACCEPTED, API_URL, TEST_KEY, patch_check, saved_key
+
+if TYPE_CHECKING:
+ from io import StringIO
+
+ from pytest_mock import MockerFixture
+
+
+class TestLoginWithPaste:
+ def test_a_pasted_key_takes_the_same_check_and_save(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ prompt = mocker.patch("pipelex.cli.commands.login.command.Prompt.ask", return_value=f" {TEST_KEY}\n")
+ check = patch_check(mocker, check=ACCEPTED)
+ browser_open = mocker.patch("pipelex.cli.commands.login.command.webbrowser.open")
+
+ login_cmd(paste=True)
+
+ assert prompt.call_args.kwargs["password"] is True
+ check.assert_called_once_with(api_key=TEST_KEY)
+ browser_open.assert_not_called()
+ assert saved_key(home=pipelex_home) == TEST_KEY
+ assert TEST_KEY not in output.getvalue()
+
+ def test_a_pasted_key_the_hosted_api_refuses_is_not_saved(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO) -> None:
+ mocker.patch("pipelex.cli.commands.login.command.Prompt.ask", return_value=TEST_KEY)
+ patch_check(mocker, check=ApiKeyCheck(verdict=ApiKeyVerdict.REFUSED, base_url=API_URL, http_status=403))
+
+ with pytest.raises(typer.Exit):
+ login_cmd(paste=True)
+
+ assert saved_key(home=pipelex_home) is None
+ assert "HTTP 403" in output.getvalue()
+
+ def test_a_working_directory_env_that_would_shadow_the_saved_key_is_named(
+ self, mocker: MockerFixture, pipelex_home: Path, output: StringIO
+ ) -> None:
+ Path(".env").write_text(f"{PIPELEX_API_KEY_ENV_KEY}=\n", encoding="utf-8")
+ mocker.patch("pipelex.cli.commands.login.command.Prompt.ask", return_value=TEST_KEY)
+ patch_check(mocker, check=ACCEPTED)
+
+ login_cmd(paste=True)
+
+ assert saved_key(home=pipelex_home) == TEST_KEY
+ printed = output.getvalue()
+ assert str(Path(".env").resolve()) in printed.replace("\n", "")
+ assert "remove" in printed.lower()
+ assert TEST_KEY not in printed
+
+ @pytest.mark.parametrize("unwritable", ["set_key_raises", "read_only_home"])
+ def test_a_key_that_cannot_be_saved_exits_1_and_says_it_went_unused(
+ self, mocker: MockerFixture, pipelex_home: Path, output: StringIO, unwritable: str
+ ) -> None:
+ mocker.patch("pipelex.cli.commands.login.command.Prompt.ask", return_value=TEST_KEY)
+ patch_check(mocker, check=ACCEPTED)
+ env_path = pipelex_home / ".env"
+ if unwritable == "set_key_raises":
+ mocker.patch(
+ "pipelex.cli.commands.init.credentials.set_key",
+ side_effect=PermissionError(13, "Permission denied", str(env_path)),
+ )
+ else:
+ pipelex_home.mkdir()
+ pipelex_home.chmod(0o500)
+
+ try:
+ with pytest.raises(typer.Exit) as exc_info:
+ login_cmd(paste=True)
+ finally:
+ if pipelex_home.exists():
+ pipelex_home.chmod(0o700)
+
+ assert exc_info.value.exit_code == 1
+ assert saved_key(home=pipelex_home) is None
+ # Rich wraps long lines at spaces; joining the words back reads the message as written
+ printed = " ".join(output.getvalue().split())
+ assert str(env_path) in printed
+ assert "Permission denied" in printed
+ assert "revoke" in printed.lower()
+ assert TEST_KEY not in printed
+
+ @pytest.mark.parametrize("answer", ["", " "])
+ def test_nothing_pasted_saves_nothing(self, mocker: MockerFixture, pipelex_home: Path, output: StringIO, answer: str) -> None:
+ mocker.patch("pipelex.cli.commands.login.command.Prompt.ask", return_value=answer)
+ check = patch_check(mocker, check=ACCEPTED)
+
+ with pytest.raises(typer.Exit):
+ login_cmd(paste=True)
+
+ check.assert_not_called()
+ assert saved_key(home=pipelex_home) is None
+ assert "No key was entered" in output.getvalue()
+
+ @pytest.mark.usefixtures("output")
+ def test_no_stdin_saves_nothing(self, mocker: MockerFixture, pipelex_home: Path) -> None:
+ mocker.patch("pipelex.cli.commands.login.command.Prompt.ask", side_effect=EOFError)
+
+ with pytest.raises(typer.Exit):
+ login_cmd(paste=True)
+
+ assert saved_key(home=pipelex_home) is None
diff --git a/tests/unit/pipelex/cli/commands/login/test_loopback.py b/tests/unit/pipelex/cli/commands/login/test_loopback.py
new file mode 100644
index 000000000..abcbcc64a
--- /dev/null
+++ b/tests/unit/pipelex/cli/commands/login/test_loopback.py
@@ -0,0 +1,132 @@
+"""The loopback listener keeps a key only from a callback carrying the `state` its own login generated."""
+
+from __future__ import annotations
+
+import http.client
+import threading
+from urllib.parse import urlencode
+
+import pytest
+
+from pipelex.cli.commands.login.loopback import CALLBACK_PATH, LOOPBACK_HOST, CallbackRefusal, LoopbackListener
+
+TEST_KEY = "plx_sk_test_not_a_secret"
+
+
+def _get(*, port: int, path: str) -> int:
+ """Send one GET to the listener and return the status it answered."""
+ connection = http.client.HTTPConnection(LOOPBACK_HOST, port, timeout=5)
+ try:
+ connection.request("GET", path)
+ return connection.getresponse().status
+ finally:
+ connection.close()
+
+
+def _callback_path(*, api_key: str | None, state: str | None) -> str:
+ params = {name: value for name, value in (("api_key", api_key), ("state", state)) if value is not None}
+ return f"{CALLBACK_PATH}?{urlencode(params)}"
+
+
+class _BrowserStandIn:
+ """Sends requests to the listener from another thread, as the browser does while the listener waits."""
+
+ def __init__(self, *, port: int, paths: list[str]) -> None:
+ self.statuses: list[int] = []
+ self._port = port
+ self._paths = paths
+ self._thread = threading.Thread(target=self._send_all, daemon=True)
+
+ def _send_all(self) -> None:
+ for path in self._paths:
+ self.statuses.append(_get(port=self._port, path=path))
+
+ def start(self) -> None:
+ self._thread.start()
+
+ def join(self) -> None:
+ self._thread.join(timeout=10)
+
+
+class TestLoopbackListener:
+ def test_it_binds_the_loopback_address_and_generates_a_fresh_state(self) -> None:
+ with LoopbackListener() as first, LoopbackListener() as second:
+ assert first.port > 0
+ assert first.state != second.state
+ assert len(first.state) >= 32
+
+ def test_a_callback_with_the_matching_state_hands_over_the_key(self) -> None:
+ with LoopbackListener() as listener:
+ browser = _BrowserStandIn(port=listener.port, paths=[_callback_path(api_key=TEST_KEY, state=listener.state)])
+ browser.start()
+ api_key = listener.wait_for_api_key(timeout_seconds=10)
+ browser.join()
+
+ assert api_key == TEST_KEY
+ assert browser.statuses == [200]
+
+ @pytest.mark.parametrize("state", ["not-the-state", None, ""])
+ def test_a_callback_with_a_wrong_or_missing_state_is_refused_and_the_wait_goes_on(self, state: str | None) -> None:
+ refusals: list[CallbackRefusal] = []
+ with LoopbackListener() as listener:
+ browser = _BrowserStandIn(
+ port=listener.port,
+ paths=[
+ _callback_path(api_key="plx_sk_from_another_page", state=state),
+ _callback_path(api_key=TEST_KEY, state=listener.state),
+ ],
+ )
+ browser.start()
+ api_key = listener.wait_for_api_key(timeout_seconds=10, on_refusal=refusals.append)
+ browser.join()
+
+ assert api_key == TEST_KEY
+ assert browser.statuses == [400, 200]
+ assert refusals == [CallbackRefusal.STATE_MISMATCH]
+
+ def test_a_callback_carrying_the_state_twice_is_refused(self) -> None:
+ """Two `state` values cannot both be this login's, whichever one matches."""
+ with LoopbackListener() as listener:
+ path = f"{CALLBACK_PATH}?{urlencode([('api_key', TEST_KEY), ('state', listener.state), ('state', 'other')])}"
+ browser = _BrowserStandIn(port=listener.port, paths=[path])
+ browser.start()
+ api_key = listener.wait_for_api_key(timeout_seconds=2)
+ browser.join()
+
+ assert api_key is None
+ assert browser.statuses == [400]
+
+ def test_a_callback_without_a_key_is_refused_and_the_wait_goes_on(self) -> None:
+ refusals: list[CallbackRefusal] = []
+ with LoopbackListener() as listener:
+ browser = _BrowserStandIn(
+ port=listener.port,
+ paths=[_callback_path(api_key=None, state=listener.state), _callback_path(api_key=TEST_KEY, state=listener.state)],
+ )
+ browser.start()
+ api_key = listener.wait_for_api_key(timeout_seconds=10, on_refusal=refusals.append)
+ browser.join()
+
+ assert api_key == TEST_KEY
+ assert browser.statuses == [400, 200]
+ assert refusals == [CallbackRefusal.NO_API_KEY]
+
+ def test_other_paths_are_ignored(self) -> None:
+ with LoopbackListener() as listener:
+ browser = _BrowserStandIn(
+ port=listener.port,
+ paths=[
+ f"/other?{urlencode({'api_key': TEST_KEY, 'state': listener.state})}",
+ _callback_path(api_key=TEST_KEY, state=listener.state),
+ ],
+ )
+ browser.start()
+ api_key = listener.wait_for_api_key(timeout_seconds=10)
+ browser.join()
+
+ assert api_key == TEST_KEY
+ assert browser.statuses == [404, 200]
+
+ def test_no_callback_in_time_returns_none(self) -> None:
+ with LoopbackListener() as listener:
+ assert listener.wait_for_api_key(timeout_seconds=0.2) is None
diff --git a/tests/unit/pipelex/cli/test_agent_doctor_cmd.py b/tests/unit/pipelex/cli/test_agent_doctor_cmd.py
index 1a167fd3c..c4cef0dbe 100644
--- a/tests/unit/pipelex/cli/test_agent_doctor_cmd.py
+++ b/tests/unit/pipelex/cli/test_agent_doctor_cmd.py
@@ -363,6 +363,35 @@ class TestAgentDoctorCmd:
assert any(action.startswith(f"Run '{MIGRATE_COMMAND}' ") for action in actions)
assert any(f"'{MIGRATE_COMMAND} --dry-run'" in action for action in actions)
+ def test_what_a_former_release_left_rides_the_envelope_with_the_command_that_cleans_it(
+ self,
+ mocker: MockerFixture,
+ capsys: pytest.CaptureFixture[str],
+ ) -> None:
+ self._stub_every_other_check_healthy(mocker)
+ mocker.patch(
+ "pipelex.cli.agent_cli.commands.doctor_cmd.check_pending_migrations",
+ return_value=PendingMigrationsCheck(
+ finding=PendingMigrationsFinding.PENDING,
+ message="1 file(s) carry what a former release left",
+ former_release_files=["/home/user/.pipelex/inference/backends.toml"],
+ former_release_blocks_boot=True,
+ ),
+ )
+
+ agent_doctor_cmd(output_format=CliOutputFormat.JSON)
+
+ parsed = json.loads(capsys.readouterr().out)
+ row = parsed["checks"]["pending_migrations"]
+ assert row["former_release_files"] == ["/home/user/.pipelex/inference/backends.toml"]
+ assert row["former_release_blocks_boot"] is True
+ actions: list[str] = parsed["recommended_actions"]
+ assert any(
+ action.startswith(f"Run '{MIGRATE_COMMAND}' to clean up what a former release left")
+ and "/home/user/.pipelex/inference/backends.toml" in action
+ for action in actions
+ )
+
def test_an_unreadable_row_is_reported_as_unchecked_rather_than_healthy(
self,
mocker: MockerFixture,
diff --git a/tests/unit/pipelex/cli/test_agent_doctor_hosted_execution.py b/tests/unit/pipelex/cli/test_agent_doctor_hosted_execution.py
new file mode 100644
index 000000000..541dd4b5d
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_agent_doctor_hosted_execution.py
@@ -0,0 +1,125 @@
+"""`pipelex-agent doctor` reports where runs execute and, for a hosted setup, judges it by its Pipelex API key."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat
+from pipelex.cli.agent_cli.commands.doctor_cmd import agent_doctor_cmd
+from pipelex.cli.commands.doctor_cmd import (
+ DoctorRuntimeSetup,
+ LogSinkCheck,
+ PendingMigrationsCheck,
+ PendingMigrationsFinding,
+ PluginsCheck,
+ SecretsProviderCheck,
+ TelemetryConfigCheck,
+ TelemetryConfigFinding,
+)
+from pipelex.cogt.model_backends.backend_credentials import BackendCredentialsReport
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from pipelex.hosted.run_config import RunExecution
+from pipelex.system.environment import PIPELEX_HOME_ENV_KEY
+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
+from tests.helpers.pipelex_api_key_env import isolate_pipelex_api_key
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture, MockType
+
+TEST_KEY = "plx_sk_test_not_a_secret"
+HEALTHY_RUNTIME_SETUP = DoctorRuntimeSetup(
+ plugins=PluginsCheck(is_healthy=True, message="Plugins discovered and registered"),
+ secrets_provider=SecretsProviderCheck(is_healthy=True, message="Secrets provider 'env' built"),
+ log_sink=LogSinkCheck(is_healthy=True, message="Log sink 'console' installed"),
+ built_secrets_provider=EnvSecretsProvider(),
+)
+MISSING_OPENAI = {
+ "openai": BackendCredentialsReport(
+ backend_name="openai",
+ required_vars=["OPENAI_API_KEY"],
+ missing_vars=["OPENAI_API_KEY"],
+ placeholder_vars=[],
+ all_credentials_valid=False,
+ ),
+}
+
+
+def _doctor_json(capsys: pytest.CaptureFixture[str]) -> dict[str, Any]:
+ agent_doctor_cmd(output_format=CliOutputFormat.JSON)
+ parsed: dict[str, Any] = json.loads(capsys.readouterr().out)
+ return parsed
+
+
+class TestAgentDoctorHostedExecution:
+ @pytest.fixture
+ def execution(self, mocker: MockerFixture, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> MockType:
+ """Every row healthy but the two that need provider keys; the returned mock says where runs execute."""
+ monkeypatch.setenv(PIPELEX_HOME_ENV_KEY, str(tmp_path / "pipelex_home"))
+ working_dir = tmp_path / "work"
+ working_dir.mkdir()
+ monkeypatch.chdir(working_dir)
+ isolate_pipelex_api_key(monkeypatch)
+ module = "pipelex.cli.agent_cli.commands.doctor_cmd"
+ mocker.patch(f"{module}.setup_doctor_runtime", return_value=HEALTHY_RUNTIME_SETUP)
+ mocker.patch(f"{module}.apply_agent_cli_output_discipline")
+ mocker.patch(f"{module}.silence_logging_for_agent_cli")
+ mocker.patch(
+ f"{module}.check_pending_migrations",
+ return_value=PendingMigrationsCheck(finding=PendingMigrationsFinding.UP_TO_DATE, message="up to date"),
+ )
+ mocker.patch(f"{module}.check_config_files", return_value=(True, 0, "OK"))
+ mocker.patch(
+ f"{module}.check_telemetry_config",
+ return_value=TelemetryConfigCheck(finding=TelemetryConfigFinding.HEALTHY, message="OK"),
+ )
+ mocker.patch(f"{module}.check_backend_credentials", return_value=(False, MISSING_OPENAI, "1 backend(s) have missing credentials"))
+ mocker.patch(f"{module}.check_models", return_value=(False, "Error checking models: Could not get variable 'OPENAI_API_KEY'", {}))
+ return mocker.patch(f"{module}.resolve_doctor_run_execution", return_value=RunExecution.HOSTED)
+
+ @pytest.mark.usefixtures("execution")
+ def test_a_hosted_setup_with_a_key_is_healthy_and_says_why(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, TEST_KEY)
+
+ parsed = _doctor_json(capsys)
+
+ assert parsed["all_healthy"] is True
+ assert parsed["execution"] == "hosted"
+ assert parsed["checks"]["pipelex_api_key"]["healthy"] is True
+ assert parsed["checks"]["pipelex_api_key"]["finding"] == "set"
+ assert parsed["checks"]["backend_credentials"]["informational"] is True
+ assert parsed["checks"]["models"]["informational"] is True
+ assert "recommended_actions" not in parsed
+ assert TEST_KEY not in json.dumps(parsed)
+
+ @pytest.mark.usefixtures("execution")
+ def test_a_hosted_setup_without_a_key_names_pipelex_login(self, capsys: pytest.CaptureFixture[str]) -> None:
+ parsed = _doctor_json(capsys)
+
+ assert parsed["all_healthy"] is False
+ assert parsed["checks"]["pipelex_api_key"] == {
+ "healthy": False,
+ "finding": "missing",
+ "message": parsed["checks"]["pipelex_api_key"]["message"],
+ }
+ actions = parsed["recommended_actions"]
+ assert any("pipelex login" in action for action in actions)
+ assert not any("OPENAI_API_KEY" in action for action in actions)
+
+ def test_a_local_setup_reports_its_execution_and_no_key_row(
+ self, execution: MockType, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+ ) -> None:
+ execution.return_value = RunExecution.LOCAL
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, TEST_KEY)
+
+ parsed = _doctor_json(capsys)
+
+ assert parsed["all_healthy"] is False
+ assert parsed["execution"] == "local"
+ assert "pipelex_api_key" not in parsed["checks"]
+ assert "informational" not in parsed["checks"]["backend_credentials"]
+ assert any("OPENAI_API_KEY" in action for action in parsed["recommended_actions"])
diff --git a/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py b/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py
index ab5006ce4..9270946b9 100644
--- a/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py
+++ b/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py
@@ -59,6 +59,7 @@ class TestCliEntrypointStarts:
["run", "--help"],
["validate", "--help"],
["init", "--help"],
+ ["login", "--help"],
],
)
def test_subcommand_starts_and_exits_zero(self, args: list[str]) -> None:
diff --git a/tests/unit/pipelex/cli/test_doctor_display_report.py b/tests/unit/pipelex/cli/test_doctor_display_report.py
index 519686841..c49e42b69 100644
--- a/tests/unit/pipelex/cli/test_doctor_display_report.py
+++ b/tests/unit/pipelex/cli/test_doctor_display_report.py
@@ -325,6 +325,38 @@ class TestDoctorDisplayReport:
assert f"Run {MIGRATE_COMMAND} to bring 1 configuration file(s) up to date" in output
assert f"Run {MIGRATE_COMMAND} --dry-run" in output, "a file the command will not repair is still owed a look"
+ def test_the_files_a_former_release_left_are_named_with_the_cleanup(self, console: Console) -> None:
+ """The machine a release on the Pipelex Gateway set up: the row says the command cleans it, and that it stops the boot."""
+ kwargs = _healthy_report_kwargs()
+ kwargs["pending_migrations_check"] = PendingMigrationsCheck(
+ finding=PendingMigrationsFinding.PENDING,
+ message="2 file(s) carry what a former release left",
+ former_release_files=["/home/user/.pipelex/inference/backends.toml", "/home/user/.pipelex/pipelex_service.toml"],
+ former_release_blocks_boot=True,
+ )
+
+ display_health_report(**kwargs)
+
+ output = console.export_text()
+ assert "/home/user/.pipelex/inference/backends.toml — left by a former release" in output
+ assert "/home/user/.pipelex/pipelex_service.toml — left by a former release" in output
+ assert f"Run {MIGRATE_COMMAND} to clean up 2 file(s) a former release left" in output
+
+ def test_a_boot_still_stopped_with_nothing_to_write_names_the_dry_run(self, console: Console) -> None:
+ """An override's `active` naming a profile no file defines: no file to list, nothing for `--fix`, and still a remedy."""
+ kwargs = _healthy_report_kwargs()
+ kwargs["pending_migrations_check"] = PendingMigrationsCheck(
+ finding=PendingMigrationsFinding.NEEDS_ATTENTION,
+ message=f"Pipelex would still not start once '{MIGRATE_COMMAND}' has run — '{MIGRATE_COMMAND} --dry-run' says why",
+ boot_still_blocked=True,
+ )
+
+ display_health_report(**kwargs)
+
+ solutions = console.export_text().split("Possible Solutions", 1)[-1]
+ assert f"Run {MIGRATE_COMMAND} --dry-run to see what still stops Pipelex from starting" in solutions
+ assert "pipelex doctor --fix" not in solutions, "the fix has nothing to write here"
+
def test_a_row_that_could_not_be_checked_is_not_reported_as_healthy(self, console: Console) -> None:
"""Not knowing is not the same as being up to date, and it names a way to find out."""
kwargs = _healthy_report_kwargs()
diff --git a/tests/unit/pipelex/cli/test_doctor_fix_mode.py b/tests/unit/pipelex/cli/test_doctor_fix_mode.py
index 6ccfba4bc..740d73af7 100644
--- a/tests/unit/pipelex/cli/test_doctor_fix_mode.py
+++ b/tests/unit/pipelex/cli/test_doctor_fix_mode.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from pathlib import Path as PurePath
from typing import TYPE_CHECKING, Any
+from unittest import mock
import pytest
from rich.console import Console
@@ -20,6 +21,7 @@ from pipelex.cli.commands.init.ui.types import InitFocus
from pipelex.cogt.model_backends.backend_credentials import BackendCredentialsReport
from pipelex.cogt.models.deck_manifest import DeckFileStatus, DeckSyncReport
from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.migration.former_release_cleanup import FormerReleaseCleanup, FormerReleaseFileAction, FormerReleaseFileCleanup
from pipelex.migration.plan import MigrationPlan, MigrationReport
if TYPE_CHECKING:
@@ -40,6 +42,23 @@ ONE_MIGRATION_PENDING = PendingMigrationsCheck(
migratable_files=[STALE_TELEMETRY_FILE],
)
ONE_FILE_MIGRATED = MigrationReport(plans=[MigrationPlan(surface_id="telemetry-config", file_path=PurePath(STALE_TELEMETRY_FILE), was_written=True)])
+TWO_FILES_CLEANED_UP = FormerReleaseCleanup(
+ dry_run=False,
+ files=[
+ FormerReleaseFileCleanup(
+ file_path=PurePath("/home/user/.pipelex/inference/backends.toml"),
+ action=FormerReleaseFileAction.REWRITE,
+ changes=["removed the 'pipelex_gateway' backend"],
+ was_applied=True,
+ ),
+ FormerReleaseFileCleanup(
+ file_path=PurePath("/home/user/.pipelex/pipelex_service.toml"),
+ action=FormerReleaseFileAction.REMOVE,
+ changes=["removed the record of the Pipelex Gateway's terms acceptance"],
+ was_applied=True,
+ ),
+ ],
+)
class TestDoctorFixMode:
@@ -54,6 +73,7 @@ class TestDoctorFixMode:
# machine's own configuration directories, and no unit test should depend on those.
"migrations": mocker.patch("pipelex.cli.commands.doctor_cmd.check_pending_migrations", return_value=NO_PENDING_MIGRATIONS),
"migrate": mocker.patch("pipelex.cli.commands.doctor_cmd.apply_pending_migrations"),
+ "clean_up": mocker.patch("pipelex.cli.commands.doctor_cmd.apply_former_release_cleanup"),
"backends": mocker.patch("pipelex.cli.commands.doctor_cmd.check_backend_credentials", return_value=(True, {}, "OK")),
"models": mocker.patch("pipelex.cli.commands.doctor_cmd.check_models", return_value=(True, "OK", {})),
"deck": mocker.patch("pipelex.cli.commands.doctor_cmd.check_deck_sync", return_value=(True, CLEAN_DECK, "OK")),
@@ -150,6 +170,43 @@ class TestDoctorFixMode:
assert "Migrated 1 configuration file(s)" in output
assert "Re-run" in output, "the rows below were measured before the migration ran"
+ def test_fix_cleans_up_what_a_former_release_left_before_it_migrates(self, doctor_mocks: dict[str, Any]) -> None:
+ """The cleanup is `pipelex migrate`'s first step, so the fix that runs the command's write pass runs it first."""
+ doctor_mocks["migrations"].return_value = PendingMigrationsCheck(
+ finding=PendingMigrationsFinding.PENDING,
+ message="1 configuration file(s) can be brought up to date; 2 file(s) carry what a former release left",
+ migratable_files=[STALE_TELEMETRY_FILE],
+ former_release_files=["/home/user/.pipelex/inference/backends.toml", "/home/user/.pipelex/pipelex_service.toml"],
+ former_release_blocks_boot=True,
+ )
+ doctor_mocks["clean_up"].return_value = TWO_FILES_CLEANED_UP
+ doctor_mocks["migrate"].return_value = ONE_FILE_MIGRATED
+ calls = mock.Mock()
+ calls.attach_mock(doctor_mocks["clean_up"], "clean_up")
+ calls.attach_mock(doctor_mocks["migrate"], "migrate")
+
+ self._run_doctor_expecting_exit_one()
+
+ assert [name for name, _, _ in calls.mock_calls] == ["clean_up", "migrate"]
+ prompts = [call.args[0] for call in doctor_mocks["confirm"].call_args_list]
+ assert any("Clean up 2 file(s) a former release left and migrate 1 configuration file(s)" in prompt for prompt in prompts)
+ output = doctor_mocks["console"].export_text()
+ assert "Cleaned up 2 file(s) a former release left" in output
+ assert "Migrated 1 configuration file(s)" in output
+
+ def test_fix_with_only_a_former_release_to_clean_runs_no_migration(self, doctor_mocks: dict[str, Any]) -> None:
+ doctor_mocks["migrations"].return_value = PendingMigrationsCheck(
+ finding=PendingMigrationsFinding.PENDING,
+ message="2 file(s) carry what a former release left",
+ former_release_files=["/home/user/.pipelex/inference/backends.toml", "/home/user/.pipelex/pipelex_service.toml"],
+ )
+ doctor_mocks["clean_up"].return_value = TWO_FILES_CLEANED_UP
+
+ self._run_doctor_expecting_exit_one()
+
+ doctor_mocks["clean_up"].assert_called_once()
+ doctor_mocks["migrate"].assert_not_called()
+
def test_declining_the_migration_writes_nothing(self, doctor_mocks: dict[str, Any]) -> None:
"""The prompt is a real question — a no leaves the machine exactly as it was."""
doctor_mocks["migrations"].return_value = ONE_MIGRATION_PENDING
diff --git a/tests/unit/pipelex/cli/test_doctor_former_release.py b/tests/unit/pipelex/cli/test_doctor_former_release.py
new file mode 100644
index 000000000..003a51882
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_doctor_former_release.py
@@ -0,0 +1,198 @@
+"""The doctor on a machine a former release set up: the migrations row names the cleanup, and the models row defers to it.
+
+The migrations row is `pipelex migrate`'s own dry run, and that command's first step is the cleanup of what a release
+that ran on the Pipelex Gateway left, so the row reports it with the files it is about. The models
+row would otherwise meet the refusal a boot meets on such a machine, about one backend or one profile; it names the
+cleanup instead.
+
+The home is the v0.72 kit, copied from that release's wheel. Whether what it left stops the boot is read off the files
+the boot itself merges, across the home and the project, so a project that boots on a base of its own is not told it
+cannot start, and a profile one directory activates from the other is not missed.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.cli.agent_cli.commands.doctor_cmd import _pending_migrations_actions # pyright: ignore[reportPrivateUsage]
+from pipelex.cli.commands import doctor_cmd as doctor_cmd_module
+from pipelex.cli.commands.doctor_cmd import PendingMigrationsFinding, check_backend_credentials, check_models, check_pending_migrations
+from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.kit.paths import RETIRED_SERVICE_FILE_NAME, get_kit_configs_dir
+from pipelex.migration.former_release import RETIRED_BACKEND_NAMES
+from pipelex.system.configuration.config_loader import BACKENDS_DIR_NAME, BACKENDS_FILE_NAME, INFERENCE_DIR_NAME, ROUTING_PROFILES_FILE_NAME
+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+pytestmark = pytest.mark.usefixtures("no_pipelex_home")
+
+V0_72_CONFIG_DIR = Path("tests/data/migration/former_release/v0_72")
+
+# The files the cleanup touches in the v0.72 directory, in the order it reports them.
+V0_72_CLEANUP_FILES = [
+ f"{INFERENCE_DIR_NAME}/{BACKENDS_FILE_NAME}",
+ f"{INFERENCE_DIR_NAME}/{BACKENDS_DIR_NAME}/pipelex_gateway.toml",
+ f"{INFERENCE_DIR_NAME}/{BACKENDS_DIR_NAME}/pipelex_gateway_models.md",
+ f"{INFERENCE_DIR_NAME}/{BACKENDS_DIR_NAME}/pipelex_gateway_models_plain.md",
+ RETIRED_SERVICE_FILE_NAME,
+ f"{INFERENCE_DIR_NAME}/{ROUTING_PROFILES_FILE_NAME}",
+]
+
+# A model table whose key the inference-backend ledger renames.
+MIGRATABLE_MODEL_TABLE = '\n["gpt-4o"]\nprompting_target = "openai"\n'
+
+
+def machine(tmp_path: Path, mocker: MockerFixture) -> tuple[Path, Path]:
+ """A fake home and a project, both configuration directories still to be written: the home's and the project's."""
+ fake_home = tmp_path / "home"
+ fake_home.mkdir()
+ project_root = tmp_path / "project"
+ (project_root / ".git").mkdir(parents=True)
+ mocker.patch.object(Path, "home", return_value=fake_home)
+ mocker.patch.object(Path, "cwd", return_value=project_root)
+ return fake_home / ".pipelex", project_root / ".pipelex"
+
+
+def former_release_home(machine: tuple[Path, Path]) -> Path:
+ """A fake home holding the v0.72 kit, and a project without a `.pipelex/`, so the walk reads this test's files."""
+ config_dir, _ = machine
+ shutil.copytree(V0_72_CONFIG_DIR, config_dir)
+ return config_dir
+
+
+def _snapshot(*, directory: Path) -> dict[str, bytes]:
+ return {path.relative_to(directory).as_posix(): path.read_bytes() for path in sorted(directory.rglob("*")) if path.is_file()}
+
+
+class TestTheDoctorOnAFormerRelease:
+ def test_the_migrations_row_names_what_a_former_release_left_and_that_it_stops_the_boot(self, former_release_home: Path) -> None:
+ check = check_pending_migrations()
+
+ assert check.finding == PendingMigrationsFinding.PENDING
+ assert check.finding.is_repaired_by_migrating
+ assert check.migratable_files == []
+ assert check.former_release_files == [str(former_release_home / relative_path) for relative_path in V0_72_CLEANUP_FILES]
+ assert check.former_release_blocks_boot
+ assert "former release" in check.message
+ assert f"'{MIGRATE_COMMAND}'" in check.message
+ assert "cannot start" in check.message
+
+ def test_the_row_is_a_dry_run_and_writes_nothing(self, former_release_home: Path) -> None:
+ before = _snapshot(directory=former_release_home)
+
+ check_pending_migrations()
+
+ assert _snapshot(directory=former_release_home) == before
+
+ def test_the_models_row_names_the_cleanup_rather_than_a_refusal_about_one_backend(self, former_release_home: Path) -> None:
+ healthy, message, backend_file_reports = check_models(secrets_provider=EnvSecretsProvider(), config_dir=former_release_home)
+
+ assert not healthy
+ assert backend_file_reports == {}
+ assert f"'{MIGRATE_COMMAND}'" in message
+ assert "former release" in message
+
+ def test_the_credentials_row_never_asks_for_a_retired_backends_key(self, former_release_home: Path) -> None:
+ """Setting the Gateway's key is no remedy: the backend is gone, and the migrations row names the cleanup."""
+ _, backend_reports, _ = check_backend_credentials(config_dir=former_release_home)
+
+ assert backend_reports, "the other enabled backends are still checked"
+ assert not set(backend_reports) & RETIRED_BACKEND_NAMES
+
+ @pytest.mark.usefixtures("former_release_home")
+ def test_a_project_booting_on_a_base_of_its_own_is_not_told_it_cannot_start(self, machine: tuple[Path, Path]) -> None:
+ """The common case: a v0.72 home, and a project set up since, whose own base files are the ones the boot reads."""
+ _, project_config_dir = machine
+ shutil.copytree(Path(str(get_kit_configs_dir())), project_config_dir)
+
+ check = check_pending_migrations()
+
+ assert check.former_release_files, "the home still carries what the release left, and the row lists it"
+ assert not check.former_release_blocks_boot
+ assert "cannot start" not in check.message
+
+ def test_a_retired_profile_activated_from_the_other_directory_stops_the_boot(self, machine: tuple[Path, Path]) -> None:
+ """Each directory alone boots; merged as the boot merges them, the home's override activates the project's retired profile."""
+ home_config_dir, project_config_dir = machine
+ kit_inference = Path(str(get_kit_configs_dir())) / INFERENCE_DIR_NAME
+ for config_dir in (home_config_dir, project_config_dir):
+ shutil.copytree(kit_inference, config_dir / INFERENCE_DIR_NAME)
+ (home_config_dir / INFERENCE_DIR_NAME / "routing_profiles_override.toml").write_text('active = "my_gw"\n', encoding="utf-8")
+ with (project_config_dir / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME).open("a", encoding="utf-8") as stream:
+ stream.write('\n[profiles.my_gw]\ndescription = "mine"\ndefault = "pipelex_gateway"\n')
+
+ check = check_pending_migrations()
+
+ assert check.former_release_blocks_boot
+ assert "cannot start" in check.message
+
+ @pytest.mark.usefixtures("former_release_home")
+ def test_a_failure_reading_what_the_boot_reads_leaves_the_row_unchecked(self, mocker: MockerFixture) -> None:
+ mocker.patch.object(doctor_cmd_module, "former_release_boot_blockers", side_effect=PermissionError(13, "Permission denied"))
+
+ check = check_pending_migrations()
+
+ assert check.finding == PendingMigrationsFinding.UNAVAILABLE
+
+ def test_a_file_the_cleanup_removes_is_not_offered_for_migration(self, former_release_home: Path) -> None:
+ gateway_file = former_release_home / INFERENCE_DIR_NAME / BACKENDS_DIR_NAME / "pipelex_gateway.toml"
+ with gateway_file.open("a", encoding="utf-8") as stream:
+ stream.write(MIGRATABLE_MODEL_TABLE)
+
+ check = check_pending_migrations()
+
+ assert str(gateway_file) in check.former_release_files
+ assert str(gateway_file) not in check.migratable_files
+
+ def test_a_catch_all_route_to_the_gateway_is_offered_for_cleanup_rather_than_crashing_the_row(self, machine: tuple[Path, Path]) -> None:
+ home, _ = machine
+ shutil.copytree(Path(str(get_kit_configs_dir())), home)
+ routing_path = home / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME
+ routing_path.write_text(
+ 'active = "mine"\n\n[profiles.mine]\ndescription = "everything through the gateway except claude"\ndefault = "openai"\n\n'
+ '[profiles.mine.routes]\n"*" = "pipelex_gateway"\n"claude-*" = "anthropic"\n',
+ encoding="utf-8",
+ )
+
+ check = check_pending_migrations()
+
+ assert check.finding is PendingMigrationsFinding.PENDING
+ assert check.former_release_files == [str(routing_path)]
+ assert check.former_release_blocks_boot
+
+ def test_a_boot_still_stopped_with_nothing_left_to_clean_needs_attention(self, machine: tuple[Path, Path]) -> None:
+ """What a cleanup run from another project leaves: no file a former release left, and an `active` naming a profile it removed."""
+ home, project = machine
+ shutil.copytree(Path(str(get_kit_configs_dir())), home)
+ project_override = project / INFERENCE_DIR_NAME / "routing_profiles_override.toml"
+ project_override.parent.mkdir(parents=True)
+ project_override.write_text('active = "team_gateway"\n', encoding="utf-8")
+
+ check = check_pending_migrations()
+
+ assert check.finding is PendingMigrationsFinding.NEEDS_ATTENTION
+ assert check.former_release_files == []
+ assert f"Pipelex would still not start once '{MIGRATE_COMMAND}' has run" in check.message
+
+ def test_a_home_override_activating_a_profile_no_file_defines_names_the_dry_run_as_its_remedy(self, machine: tuple[Path, Path]) -> None:
+ """Nothing is left to clean and `--fix` has nothing to write, so the dry run, which says what stops the boot, is the one move left."""
+ home, _ = machine
+ shutil.copytree(Path(str(get_kit_configs_dir())), home)
+ (home / INFERENCE_DIR_NAME / "routing_profiles_override.toml").write_text('active = "team_gateway"\n', encoding="utf-8")
+
+ check = check_pending_migrations()
+
+ assert check.finding is PendingMigrationsFinding.NEEDS_ATTENTION
+ assert not check.finding.is_repaired_by_migrating
+ assert (check.former_release_files, check.migratable_files, check.attention_files) == ([], [], [])
+ actions = _pending_migrations_actions(check=check)
+ assert any(action.startswith(f"Run '{MIGRATE_COMMAND} --dry-run' to see what still stops Pipelex from starting") for action in actions)
+ assert check.boot_still_blocked
diff --git a/tests/unit/pipelex/cli/test_doctor_hosted_execution.py b/tests/unit/pipelex/cli/test_doctor_hosted_execution.py
new file mode 100644
index 000000000..bd0eeb34e
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_doctor_hosted_execution.py
@@ -0,0 +1,153 @@
+"""`pipelex doctor` judges a hosted setup by its Pipelex API key, and reports provider credentials as for `--local` runs only."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from rich.console import Console
+
+from pipelex.base_exceptions import PipelexConfigError
+from pipelex.cli.commands.doctor_cmd import (
+ DoctorRuntimeSetup,
+ LogSinkCheck,
+ PendingMigrationsCheck,
+ PendingMigrationsFinding,
+ PluginsCheck,
+ SecretsProviderCheck,
+ TelemetryConfigCheck,
+ TelemetryConfigFinding,
+ do_doctor_cmd,
+ resolve_doctor_run_execution,
+)
+from pipelex.cogt.model_backends.backend_credentials import BackendCredentialsReport
+from pipelex.cogt.models.deck_manifest import DeckSyncReport
+from pipelex.hosted.client_factory import PIPELEX_API_KEY_ENV_KEY
+from pipelex.hosted.run_config import RunExecution
+from pipelex.system.environment import PIPELEX_HOME_ENV_KEY
+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
+from tests.helpers.pipelex_api_key_env import isolate_pipelex_api_key
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from pytest_mock import MockerFixture
+
+TEST_KEY = "plx_sk_test_not_a_secret"
+HEALTHY_RUNTIME_SETUP = DoctorRuntimeSetup(
+ plugins=PluginsCheck(is_healthy=True, message="Plugins discovered and registered"),
+ secrets_provider=SecretsProviderCheck(is_healthy=True, message="Secrets provider 'env' built"),
+ log_sink=LogSinkCheck(is_healthy=True, message="Log sink 'console' installed"),
+ built_secrets_provider=EnvSecretsProvider(),
+)
+CLEAN_DECK = DeckSyncReport(kit_version="1.2.0", installed_kit_version="1.2.0", manifest_present=True, files={})
+MISSING_OPENAI = {
+ "openai": BackendCredentialsReport(
+ backend_name="openai",
+ required_vars=["OPENAI_API_KEY"],
+ missing_vars=["OPENAI_API_KEY"],
+ placeholder_vars=[],
+ all_credentials_valid=False,
+ ),
+}
+MODELS_NEED_A_PROVIDER_KEY = "Error checking models: Could not get variable 'OPENAI_API_KEY'"
+
+
+def _run_doctor(*, fix: bool = False) -> int | str | None:
+ with pytest.raises(SystemExit) as exc_info:
+ do_doctor_cmd(fix=fix)
+ return exc_info.value.code
+
+
+class TestDoctorHostedExecution:
+ @pytest.fixture
+ def doctor_mocks(self, mocker: MockerFixture, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
+ """Every row healthy but the two that need provider keys, which a hosted setup does not have."""
+ monkeypatch.setenv(PIPELEX_HOME_ENV_KEY, str(tmp_path / "pipelex_home"))
+ working_dir = tmp_path / "work"
+ working_dir.mkdir()
+ monkeypatch.chdir(working_dir)
+ isolate_pipelex_api_key(monkeypatch)
+ mocks: dict[str, Any] = {
+ "setup": mocker.patch("pipelex.cli.commands.doctor_cmd.setup_doctor_runtime", return_value=HEALTHY_RUNTIME_SETUP),
+ "config": mocker.patch("pipelex.cli.commands.doctor_cmd.check_config_files", return_value=(True, 0, "OK")),
+ "telemetry": mocker.patch(
+ "pipelex.cli.commands.doctor_cmd.check_telemetry_config",
+ return_value=TelemetryConfigCheck(finding=TelemetryConfigFinding.HEALTHY, message="OK"),
+ ),
+ "migrations": mocker.patch(
+ "pipelex.cli.commands.doctor_cmd.check_pending_migrations",
+ return_value=PendingMigrationsCheck(finding=PendingMigrationsFinding.UP_TO_DATE, message="up to date"),
+ ),
+ "backends": mocker.patch(
+ "pipelex.cli.commands.doctor_cmd.check_backend_credentials",
+ return_value=(False, MISSING_OPENAI, "1 backend(s) have missing or invalid credentials"),
+ ),
+ "models": mocker.patch("pipelex.cli.commands.doctor_cmd.check_models", return_value=(False, MODELS_NEED_A_PROVIDER_KEY, {})),
+ "deck": mocker.patch("pipelex.cli.commands.doctor_cmd.check_deck_sync", return_value=(True, CLEAN_DECK, "OK")),
+ "internal_backend": mocker.patch("pipelex.cli.commands.doctor_cmd.check_internal_backend_sync", return_value=(True, CLEAN_DECK, "OK")),
+ "execution": mocker.patch("pipelex.cli.commands.doctor_cmd.resolve_doctor_run_execution", return_value=RunExecution.HOSTED),
+ "init_cmd": mocker.patch("pipelex.cli.commands.doctor_cmd.init_cmd"),
+ "confirm": mocker.patch("pipelex.cli.commands.doctor_cmd.Confirm.ask", return_value=True),
+ }
+ recorded_console = Console(width=200, record=True, color_system=None)
+ mocker.patch("pipelex.cli.commands.doctor_cmd.get_console", return_value=recorded_console)
+ mocks["console"] = recorded_console
+ return mocks
+
+ def test_a_hosted_setup_with_a_key_is_healthy_without_provider_keys(self, doctor_mocks: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, TEST_KEY)
+
+ assert _run_doctor() == 0
+
+ output = doctor_mocks["console"].export_text()
+ assert "All systems healthy" in output
+ assert "Manual Fixes Required" not in output
+ assert "Set the following environment variables" not in output
+ assert "--local" in output
+ assert "Pipelex API Key" in output
+ assert TEST_KEY not in output
+
+ def test_a_hosted_setup_without_a_key_fails_naming_pipelex_login(self, doctor_mocks: dict[str, Any]) -> None:
+ assert _run_doctor() == 1
+
+ output = doctor_mocks["console"].export_text()
+ assert "pipelex login" in output
+ assert "Set the following environment variables" not in output
+ assert "OPENAI_API_KEY=" not in output
+
+ def test_a_hosted_setup_with_a_value_that_is_not_a_pipelex_key_fails(self, doctor_mocks: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "sk-not-a-pipelex-key")
+
+ assert _run_doctor() == 1
+
+ output = doctor_mocks["console"].export_text()
+ assert "pipelex login" in output
+ assert "sk-not-a-pipelex-key" not in output
+
+ def test_fix_mode_on_a_hosted_setup_with_a_key_has_nothing_to_fix(self, doctor_mocks: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, TEST_KEY)
+
+ assert _run_doctor(fix=True) == 0
+
+ doctor_mocks["init_cmd"].assert_not_called()
+
+ def test_a_local_setup_still_needs_its_provider_keys(self, doctor_mocks: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
+ doctor_mocks["execution"].return_value = RunExecution.LOCAL
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, TEST_KEY)
+
+ assert _run_doctor() == 1
+
+ output = doctor_mocks["console"].export_text()
+ assert "Set the following environment variables" in output
+ assert "Pipelex API Key" not in output
+
+ def test_a_configuration_that_does_not_load_is_judged_as_local(self, mocker: MockerFixture) -> None:
+ mocker.patch("pipelex.cli.commands.doctor_cmd.configured_run_execution", side_effect=PipelexConfigError("broken"))
+
+ assert resolve_doctor_run_execution() == RunExecution.LOCAL
+
+ def test_the_configured_execution_is_the_one_judged(self, mocker: MockerFixture) -> None:
+ mocker.patch("pipelex.cli.commands.doctor_cmd.configured_run_execution", return_value=RunExecution.HOSTED)
+
+ assert resolve_doctor_run_execution() == RunExecution.HOSTED
diff --git a/tests/unit/pipelex/cli/test_error_handlers_more.py b/tests/unit/pipelex/cli/test_error_handlers_more.py
index c8d4be84d..4f091f83f 100644
--- a/tests/unit/pipelex/cli/test_error_handlers_more.py
+++ b/tests/unit/pipelex/cli/test_error_handlers_more.py
@@ -18,6 +18,7 @@ from pipelex.base_exceptions import MigrationErrorBlock
from pipelex.cli.error_handlers import (
ErrorContext,
display_error_panel,
+ handle_former_release_config_error,
handle_model_deck_preset_error,
handle_telemetry_config_validation_error,
handle_validate_bundle_error,
@@ -27,6 +28,7 @@ from pipelex.cogt.inference.error_classification import UserAction, UserActionKi
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.core.exceptions import DryRunFailureErrorData, PipelexBundleBlueprintValidationErrorData, PipesAndConceptValidationErrorData
from pipelex.core.validation import MIGRATE_COMMAND
+from pipelex.migration.exceptions import FormerReleaseConfigError
from pipelex.pipeline.exceptions import ValidateBundleError
from pipelex.system.telemetry.exceptions import TelemetryConfigValidationError
from pipelex.validation_error_types import PipeValidationErrorType
@@ -273,3 +275,20 @@ class TestErrorHandlersExtended:
output = console.export_text()
assert "payload has [red]markup[/red] inside" in output
+
+ def test_handle_former_release_config_error_shows_the_message_and_no_traceback(self, console: Console) -> None:
+ """The boot's refusal on what a former release left: its message names the files and both remedies, and is the whole answer."""
+ exc = FormerReleaseConfigError(
+ "This configuration was set up by a former Pipelex release.\n- 'backends.toml' enables the 'pipelex_gateway' backend\n\n"
+ f"Run `{MIGRATE_COMMAND}` to clean it up."
+ )
+
+ with pytest.raises(typer.Exit) as exc_info:
+ handle_former_release_config_error(exc=exc)
+
+ assert exc_info.value.exit_code == 1
+ output = console.export_text()
+ assert "Configuration left by a former release" in output
+ assert "'backends.toml' enables the 'pipelex_gateway' backend" in output
+ assert f"Run `{MIGRATE_COMMAND}` to clean it up." in output
+ assert "Traceback" not in output
diff --git a/tests/unit/pipelex/cli/test_former_release_hosted_manifold.py b/tests/unit/pipelex/cli/test_former_release_hosted_manifold.py
new file mode 100644
index 000000000..c865cb2d7
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_former_release_hosted_manifold.py
@@ -0,0 +1,132 @@
+"""The hosted plane's configuration is not a former release's: Pipelex Manifold is live, and nothing refuses or cleans it.
+
+The Pipelex Gateway is retired; Pipelex Manifold is not. The hosted plane runs every model through it, with a plugin of
+its own, and each of its members that boots Pipelex — the runner, the worker and the Temporal suite — ships a
+`.pipelex/inference/` declaring an enabled `[pipelex_manifold]` backend, an `all_pipelex_manifold` profile made active,
+and the model registry's runtime slice as `backends/pipelex_manifold.toml`. `tests/data/migration/former_release/hosted_manifold/`
+holds those trees as the hosted plane ships them, the slice cut to two models. Each stands as a project's `.pipelex/`
+beside a home holding the current kit, which is what a boot fills a home configuration directory with: no boot of it is
+refused, and no command that finds or cleans what a former release left finds anything here or writes anything.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from rich.console import Console
+
+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat
+from pipelex.cli.agent_cli.commands.migrate_cmd import agent_migrate_cmd
+from pipelex.cli.commands import migrate_cmd as migrate_cmd_module
+from pipelex.cli.commands.doctor_cmd import PendingMigrationsFinding, check_backend_credentials, check_pending_migrations
+from pipelex.cli.commands.init.command import inspect_initialization
+from pipelex.cli.commands.init.ui.types import InitFocus
+from pipelex.cli.commands.migrate_cmd import migrate_cmd
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.migration.former_release import detect_former_release_across, former_release_boot_blockers
+from pipelex.migration.former_release_cleanup import clean_former_release, what_stops_the_boot
+from pipelex.system.configuration.config_loader import BACKENDS_FILE_NAME, INFERENCE_DIR_NAME, ROUTING_PROFILES_FILE_NAME
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+pytestmark = pytest.mark.usefixtures("no_pipelex_home")
+
+HOSTED_MANIFOLD_DIR = Path("tests/data/migration/former_release/hosted_manifold")
+
+#: The members of the hosted plane that boot Pipelex, each with the `.pipelex/inference/` it ships.
+HOSTED_MEMBERS = ["api-hosted", "worker", "temporal"]
+
+
[email protected](params=HOSTED_MEMBERS)
+def hosted_machine(request: pytest.FixtureRequest, tmp_path: Path, mocker: MockerFixture) -> tuple[Path, Path]:
+ """A home holding the current kit, and a project whose `.pipelex/` is one hosted member's: the home's and the project's."""
+ fake_home = tmp_path / "home"
+ home_config_dir = fake_home / ".pipelex"
+ shutil.copytree(Path(str(get_kit_configs_dir())), home_config_dir)
+ project_root = tmp_path / "project"
+ (project_root / ".git").mkdir(parents=True)
+ project_config_dir = project_root / ".pipelex"
+ shutil.copytree(HOSTED_MANIFOLD_DIR / request.param, project_config_dir)
+ mocker.patch.object(Path, "home", return_value=fake_home)
+ mocker.patch.object(Path, "cwd", return_value=project_root)
+ return home_config_dir, project_config_dir
+
+
+def _snapshot(*, directories: tuple[Path, ...]) -> dict[str, bytes]:
+ return {str(path): path.read_bytes() for directory in directories for path in sorted(directory.rglob("*")) if path.is_file()}
+
+
+class TestTheHostedPlanesManifoldConfiguration:
+ def test_no_boot_of_it_is_refused(self, hosted_machine: tuple[Path, Path]) -> None:
+ _, project_config_dir = hosted_machine
+ inference_dir = project_config_dir / INFERENCE_DIR_NAME
+
+ blockers = former_release_boot_blockers(
+ backends_library_paths=[inference_dir / BACKENDS_FILE_NAME],
+ routing_profile_library_paths=[inference_dir / ROUTING_PROFILES_FILE_NAME],
+ )
+
+ assert blockers == []
+ assert what_stops_the_boot(config_dirs=list(hosted_machine)) == []
+
+ def test_nothing_in_it_is_what_a_former_release_left(self, hosted_machine: tuple[Path, Path]) -> None:
+ assert all(findings.is_clean for findings in detect_former_release_across(config_dirs=list(hosted_machine)))
+
+ def test_the_cleanup_finds_nothing_and_writes_nothing(self, hosted_machine: tuple[Path, Path]) -> None:
+ before = _snapshot(directories=hosted_machine)
+
+ cleanup = clean_former_release(config_dirs=list(hosted_machine), dry_run=False)
+
+ assert cleanup.is_clean
+ assert cleanup.files == []
+ assert _snapshot(directories=hosted_machine) == before
+
+ def test_pipelex_migrate_dry_run_has_nothing_to_do(self, hosted_machine: tuple[Path, Path], mocker: MockerFixture) -> None:
+ console = Console(width=400, record=True, color_system=None)
+ mocker.patch.object(migrate_cmd_module, "get_console", return_value=console)
+ before = _snapshot(directories=hosted_machine)
+
+ migrate_cmd(dry_run=True)
+
+ output = console.export_text()
+ assert "Every configuration file on this machine is at the current schema." in output
+ assert "former release" not in output
+ assert _snapshot(directories=hosted_machine) == before
+
+ @pytest.mark.usefixtures("hosted_machine")
+ def test_pipelex_agent_migrate_reports_a_clean_machine(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ mocker.patch("pipelex.cli.agent_cli.commands.migrate_cmd.silence_logging_for_agent_cli")
+
+ agent_migrate_cmd(output_format=CliOutputFormat.JSON)
+
+ result = json.loads(capsys.readouterr().out)
+ assert result["is_clean"] is True
+ assert result["needs_attention"] is False
+ assert result["former_release"] == {"is_clean": True, "needs_attention": False, "files": [], "still_blocking": []}
+
+ @pytest.mark.usefixtures("hosted_machine")
+ def test_the_doctors_migrations_row_is_up_to_date(self) -> None:
+ check = check_pending_migrations()
+
+ assert check.finding is PendingMigrationsFinding.UP_TO_DATE
+ assert check.former_release_files == []
+
+ def test_the_doctor_checks_manifolds_credentials(self, hosted_machine: tuple[Path, Path]) -> None:
+ """An enabled `pipelex_manifold` backend is one like any other: its endpoint and key are checked, never skipped as retired."""
+ _, project_config_dir = hosted_machine
+
+ _, backend_reports, _ = check_backend_credentials(config_dir=project_config_dir)
+
+ assert "pipelex_manifold" in backend_reports
+
+ @pytest.mark.usefixtures("hosted_machine")
+ def test_init_finds_nothing_to_offer_to_clean_up(self) -> None:
+ inspection = inspect_initialization(focus=InitFocus.ALL, local=True)
+
+ assert inspection.former_release_findings == []
+ assert inspection.former_release_boot_blockers == []
diff --git a/tests/unit/pipelex/cli/test_migrate_cmd_former_release.py b/tests/unit/pipelex/cli/test_migrate_cmd_former_release.py
new file mode 100644
index 000000000..7c25fce05
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_migrate_cmd_former_release.py
@@ -0,0 +1,174 @@
+"""`pipelex migrate` on a machine a former release set up: the cleanup is its first step, rehearsed, asked about and reported.
+
+The machine is the v0.72 kit, copied from that release's wheel, as one configuration directory. Its ledger-claimed
+files are already at the current schema, so everything the command has to do here is the cleanup, which is what lets
+each test say exactly what the command printed and wrote.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from rich.console import Console
+
+from pipelex.cli.commands import migrate_cmd as migrate_cmd_module
+from pipelex.cli.commands.migrate_cmd import migrate_cmd
+from pipelex.kit.paths import RETIRED_SERVICE_FILE_NAME, get_kit_configs_dir
+from pipelex.migration.backup import existing_backups_of
+from pipelex.migration.former_release import detect_former_release, kit_default_routing_profile_name
+from pipelex.system.configuration.config_loader import BACKENDS_DIR_NAME, BACKENDS_FILE_NAME, INFERENCE_DIR_NAME, ROUTING_PROFILES_FILE_NAME
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+V0_72_CONFIG_DIR = Path("tests/data/migration/former_release/v0_72")
+V0_72_CLEANED_DIR = Path("tests/data/migration/former_release/v0_72_cleaned")
+
+# The files the cleanup touches in the v0.72 directory, rewritten or removed.
+V0_72_FILE_COUNT = len(detect_former_release(config_dir=V0_72_CONFIG_DIR).file_paths)
+
+# A model table whose key the inference-backend ledger renames, and a key no schema knows.
+MIGRATABLE_MODEL_TABLE = '\n["gpt-4o"]\nprompting_target = "openai"\n'
+UNKNOWN_MODEL_TABLE = '\n["claude-x"]\nsome_unknown_key = 1\n'
+
+
+def console(mocker: MockerFixture) -> Console:
+ recorded = Console(width=400, record=True, color_system=None)
+ mocker.patch.object(migrate_cmd_module, "get_console", return_value=recorded)
+ return recorded
+
+
+def former_release_machine(tmp_path: Path, mocker: MockerFixture) -> Path:
+ config_dir = tmp_path / ".pipelex"
+ shutil.copytree(V0_72_CONFIG_DIR, config_dir)
+ mocker.patch.object(migrate_cmd_module, "config_directories_to_migrate", return_value=[config_dir])
+ return config_dir
+
+
+def _snapshot(*, directory: Path) -> dict[str, bytes]:
+ return {path.relative_to(directory).as_posix(): path.read_bytes() for path in sorted(directory.rglob("*")) if path.is_file()}
+
+
+class TestTheMigrateCommandOnAFormerRelease:
+ def test_yes_cleans_up_every_file_and_says_so(self, former_release_machine: Path, console: Console) -> None:
+ migrate_cmd(yes=True)
+
+ output = console.export_text()
+ inference_dir = former_release_machine / INFERENCE_DIR_NAME
+ assert str(inference_dir / BACKENDS_FILE_NAME) in output
+ assert "✓ removed the 'pipelex_gateway' backend" in output
+ assert "✓ moved the active routing profile from 'all_pipelex_gateway' to 'all_enabled_backends'" in output
+ assert f"backup: {inference_dir / BACKENDS_FILE_NAME}.bak." in output
+ assert f"Cleaned up {V0_72_FILE_COUNT} file(s) a former release left; a copy of each original is beside it." in output
+ for name in (BACKENDS_FILE_NAME, ROUTING_PROFILES_FILE_NAME):
+ assert (inference_dir / name).read_bytes() == (V0_72_CLEANED_DIR / INFERENCE_DIR_NAME / name).read_bytes()
+ assert detect_former_release(config_dir=former_release_machine).is_clean
+
+ def test_a_dry_run_rehearses_the_cleanup_and_writes_nothing(self, former_release_machine: Path, console: Console) -> None:
+ before = _snapshot(directory=former_release_machine)
+
+ migrate_cmd(dry_run=True)
+
+ output = console.export_text()
+ assert "→ removed the 'pipelex_gateway' backend" in output
+ assert "→ removed the record of the Pipelex Gateway's terms acceptance" in output
+ assert "Dry run — nothing was written." in output
+ assert _snapshot(directory=former_release_machine) == before
+
+ def test_the_question_counts_the_cleanup_and_a_no_writes_nothing(
+ self, former_release_machine: Path, console: Console, mocker: MockerFixture
+ ) -> None:
+ before = _snapshot(directory=former_release_machine)
+ asked = mocker.patch.object(migrate_cmd_module.Confirm, "ask", return_value=False)
+
+ migrate_cmd()
+
+ assert f"Clean up {V0_72_FILE_COUNT} file(s)?" in asked.call_args.args[0]
+ assert "nothing was written" in console.export_text()
+ assert _snapshot(directory=former_release_machine) == before
+
+ def test_a_file_the_cleanup_could_not_write_leaves_a_non_zero_exit(
+ self, former_release_machine: Path, console: Console, mocker: MockerFixture
+ ) -> None:
+ mocker.patch("pipelex.migration.runner.commit_file_updates", side_effect=PermissionError(13, "Permission denied"))
+
+ with pytest.raises(SystemExit) as leaving:
+ migrate_cmd(yes=True)
+
+ assert leaving.value.code == 1
+ output = console.export_text()
+ assert "✗ unwritable" in output
+ routing_path = former_release_machine / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME
+ assert routing_path.read_bytes() == (V0_72_CONFIG_DIR / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME).read_bytes()
+ assert existing_backups_of(path=routing_path) == []
+ assert not (former_release_machine / RETIRED_SERVICE_FILE_NAME).exists(), "a removal does not go through the commit that failed"
+
+ def test_a_file_the_cleanup_removes_is_not_counted_among_the_files_to_migrate(
+ self, former_release_machine: Path, console: Console, mocker: MockerFixture
+ ) -> None:
+ """A key the ledger would carry forward, in a retired backend's file: the file goes, so nothing of it is migrated."""
+ gateway_file = former_release_machine / INFERENCE_DIR_NAME / BACKENDS_DIR_NAME / "pipelex_gateway.toml"
+ with gateway_file.open("a", encoding="utf-8") as stream:
+ stream.write(MIGRATABLE_MODEL_TABLE)
+ asked = mocker.patch.object(migrate_cmd_module.Confirm, "ask", return_value=False)
+
+ migrate_cmd()
+
+ assert asked.call_args.args[0] == f"[bold]Clean up {V0_72_FILE_COUNT} file(s)?[/bold]"
+ assert f"{gateway_file} (inference-backend)" not in console.export_text()
+
+ def test_an_unknown_key_in_a_file_the_cleanup_removes_needs_no_attention(self, former_release_machine: Path, console: Console) -> None:
+ gateway_file = former_release_machine / INFERENCE_DIR_NAME / BACKENDS_DIR_NAME / "pipelex_gateway.toml"
+ with gateway_file.open("a", encoding="utf-8") as stream:
+ stream.write(UNKNOWN_MODEL_TABLE)
+
+ migrate_cmd(dry_run=True)
+
+ assert "some_unknown_key" not in console.export_text()
+
+ def test_a_machine_that_still_cannot_start_after_the_cleanup_is_reported_and_exits_non_zero(
+ self, former_release_machine: Path, console: Console
+ ) -> None:
+ """The check after the write: an `active` naming a profile no file defines still stops the boot, and success is not claimed."""
+ routing_path = former_release_machine / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME
+ text = routing_path.read_text(encoding="utf-8")
+ assert text.count('active = "all_pipelex_gateway"') == 1
+ routing_path.write_text(text.replace('active = "all_pipelex_gateway"', 'active = "ghost"'), encoding="utf-8")
+
+ with pytest.raises(SystemExit) as leaving:
+ migrate_cmd(yes=True)
+
+ assert leaving.value.code == 1
+ output = console.export_text()
+ assert "After the cleanup, Pipelex still cannot start:" in output
+ assert "'ghost'" in output
+ assert "a copy of each original is beside it" not in output
+
+ @pytest.mark.parametrize("dry_run", [True, False])
+ def test_a_machine_with_nothing_left_to_clean_that_cannot_start_is_never_up_to_date(
+ self, tmp_path: Path, console: Console, mocker: MockerFixture, dry_run: bool
+ ) -> None:
+ """A cleanup left half done — run from another project, or stopped by a file it could not write — leaves no file a
+ former release left and a boot still stopped: the retry says what stops it, and exits non-zero.
+ """
+ config_dir = tmp_path / ".pipelex"
+ shutil.copytree(Path(str(get_kit_configs_dir())), config_dir)
+ routing_path = config_dir / INFERENCE_DIR_NAME / ROUTING_PROFILES_FILE_NAME
+ text = routing_path.read_text(encoding="utf-8")
+ routing_path.write_text(text.replace(f'active = "{kit_default_routing_profile_name()}"', 'active = "team_gateway"'), encoding="utf-8")
+ mocker.patch.object(migrate_cmd_module, "config_directories_to_migrate", return_value=[config_dir])
+
+ with pytest.raises(SystemExit) as leaving:
+ migrate_cmd(dry_run=dry_run, yes=not dry_run)
+
+ assert leaving.value.code == 1
+ output = console.export_text()
+ assert "Pipelex cannot start:" in output
+ assert "'team_gateway'" in output
+ assert "at the current schema" not in output
+ assert "file(s)?" not in output
diff --git a/tests/unit/pipelex/cogt/models/test_model_manager_former_release.py b/tests/unit/pipelex/cogt/models/test_model_manager_former_release.py
new file mode 100644
index 000000000..ef56b342b
--- /dev/null
+++ b/tests/unit/pipelex/cogt/models/test_model_manager_former_release.py
@@ -0,0 +1,117 @@
+"""A home a former release set up is refused at boot with one error naming the cleanup, before any low-level refusal.
+
+The model manager is where the boot reads the inference files, so it is where the check runs, ahead of the backend
+library and the routing profile library. Without it a v0.72 home meets whichever refusal comes first: the Gateway's
+missing key on a boot that needs inference, its backend file declaring no model on a keyless one, or
+`RoutingProfileDisabledBackendError` once the Gateway is turned off and `active` still names its profile. None of them
+says that the release behind the files is gone, or what to run.
+
+The home is faked as the other model-manager tests fake it: the kit's inference tree, with the v0.72 release's own
+files laid over it, copied from that release's wheel.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.base_exceptions import PipelexSetupError
+from pipelex.cogt.models.model_manager import ModelManager
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.migration.exceptions import FormerReleaseConfigError
+from pipelex.migration.former_release import kit_default_routing_profile_name
+from pipelex.migration.former_release_cleanup import clean_former_release
+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations
+from pipelex.system.configuration.config_loader import BACKENDS_FILE_NAME, INFERENCE_DIR_NAME, ROUTING_PROFILES_OVERRIDE_FILE_NAME
+from pipelex.system.environment import CONFIG_DIR_NAME
+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+
+pytestmark = pytest.mark.usefixtures("no_pipelex_home")
+
+V0_72_CONFIG_DIR = Path("tests/data/migration/former_release/v0_72")
+
+# The line of the v0.72 kit's `backends.toml` that enables the Pipelex Gateway.
+GATEWAY_ENABLED_LINE = "enabled = true # Enable after accepting terms via `pipelex init config`"
+
+
+def _setup(*, needs_inference: bool) -> ModelManager:
+ models_manager = ModelManager()
+ models_manager.setup(
+ secrets_provider=EnvSecretsProvider(),
+ plugin_model_declarations=PluginModelDeclarations.make_empty(),
+ needs_inference=needs_inference,
+ )
+ return models_manager
+
+
+class TestAFormerReleaseHomeAtBoot:
+ @pytest.fixture
+ def home_config_dir(self, tmp_path: Path, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch) -> Path:
+ """A faked home holding the kit's inference tree with the v0.72 release's files over it, and no project."""
+ fake_home = tmp_path / "home"
+ config_dir = fake_home / CONFIG_DIR_NAME
+ shutil.copytree(Path(str(get_kit_configs_dir())) / INFERENCE_DIR_NAME, config_dir / INFERENCE_DIR_NAME)
+ shutil.copytree(V0_72_CONFIG_DIR, config_dir, dirs_exist_ok=True)
+ project_root = tmp_path / "project"
+ (project_root / ".git").mkdir(parents=True)
+ mocker.patch.object(Path, "home", return_value=fake_home)
+ mocker.patch.object(Path, "cwd", return_value=project_root)
+ # The Gateway's own key is unset, so a boot that resolved credentials first would stop on it instead.
+ monkeypatch.delenv("PIPELEX_GATEWAY_API_KEY", raising=False)
+ return config_dir
+
+ @pytest.mark.parametrize("needs_inference", [False, True])
+ def test_a_v0_72_home_is_refused_with_the_one_error_naming_both_remedies(self, home_config_dir: Path, needs_inference: bool) -> None:
+ with pytest.raises(FormerReleaseConfigError) as refused:
+ _setup(needs_inference=needs_inference)
+
+ message = str(refused.value)
+ assert isinstance(refused.value, PipelexSetupError)
+ assert "`pipelex migrate`" in message
+ assert "`pipelex init`" in message
+ assert str(home_config_dir / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME) in message
+ assert "'pipelex_gateway'" in message
+ assert "'all_pipelex_gateway'" in message
+ assert "PIPELEX_GATEWAY_API_KEY" not in message, "the refusal is the former release, not the key it once needed"
+
+ def test_a_disabled_gateway_is_refused_by_the_same_error_rather_than_the_routing_refusal(self, home_config_dir: Path) -> None:
+ backends_path = home_config_dir / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME
+ text = backends_path.read_text(encoding="utf-8")
+ assert text.count(GATEWAY_ENABLED_LINE) == 1
+ backends_path.write_text(text.replace(GATEWAY_ENABLED_LINE, "enabled = false"), encoding="utf-8")
+
+ with pytest.raises(FormerReleaseConfigError) as refused:
+ _setup(needs_inference=False)
+
+ message = str(refused.value)
+ assert "'all_pipelex_gateway'" in message
+ assert "enables the 'pipelex_gateway' backend" not in message
+
+ def test_a_home_whose_overrides_moved_it_off_the_gateway_boots(self, home_config_dir: Path) -> None:
+ """Only what stops the boot is refused: what is inert is left for `pipelex migrate` and the doctor to report."""
+ inference_dir = home_config_dir / INFERENCE_DIR_NAME
+ (inference_dir / "backends_override.toml").write_text("[pipelex_gateway]\nenabled = false\n", encoding="utf-8")
+ (inference_dir / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('active = "all_openai"\n', encoding="utf-8")
+
+ models_manager = _setup(needs_inference=False)
+
+ assert models_manager.routing_profile.name == "all_openai"
+ assert "pipelex_gateway" not in models_manager.inference_backend_library.root
+
+ @pytest.mark.parametrize("needs_inference", [False, True])
+ def test_a_v0_72_home_boots_once_cleaned_on_the_kit_default_profile(self, home_config_dir: Path, needs_inference: bool) -> None:
+ cleanup = clean_former_release(config_dirs=[home_config_dir], dry_run=False)
+ assert not cleanup.needs_attention
+
+ models_manager = _setup(needs_inference=needs_inference)
+
+ assert models_manager.routing_profile.name == kit_default_routing_profile_name()
+ assert "pipelex_gateway" not in models_manager.inference_backend_library.root
+ assert "pipelex_manifold" not in models_manager.inference_backend_library.root, "the table the release left for Manifold stays, disabled"
diff --git a/tests/unit/pipelex/hosted/test_api_key_check.py b/tests/unit/pipelex/hosted/test_api_key_check.py
new file mode 100644
index 000000000..8c3c471b5
--- /dev/null
+++ b/tests/unit/pipelex/hosted/test_api_key_check.py
@@ -0,0 +1,160 @@
+"""A key is checked with `GET /v1/me`: accepted, refused by a 401 or a 403, or left unchecked by anything else."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import httpx
+import pytest
+from pipelex_sdk.client import PipelexAPIClient
+from typing_extensions import override
+
+from pipelex.hosted.api_key_check import ApiKeyVerdict, check_pipelex_api_key, is_well_formed_pipelex_api_key
+from pipelex.hosted.client_factory import PIPELEX_BASE_URL_ENV_KEY
+from tests.helpers.pipelex_api_key_env import isolate_pipelex_api_key
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from pytest_mock import MockerFixture
+
+API_URL = "https://api.test"
+TEST_KEY = "plx_sk_test_not_a_secret"
+PROFILE = {"email": "[email protected]", "user_id": "u_1", "full_name": "Ada"}
+
+
+def _html_page(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(200, content=b"<html>not an API</html>", request=request)
+
+
+def _unexpected_json(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(200, json={"unexpected": True}, request=request)
+
+
+class _MockTransportClient(PipelexAPIClient):
+ """The real client, its HTTP answered by a handler instead of the network."""
+
+ def __init__(self, *, api_key: str | None, handler: Callable[[httpx.Request], httpx.Response]) -> None:
+ super().__init__(api_key=api_key, base_url=API_URL)
+ self._handler = handler
+
+ @override
+ def start_client(self) -> PipelexAPIClient:
+ self.client = httpx.AsyncClient(
+ headers={"Authorization": f"Bearer {self.api_key}"},
+ transport=httpx.MockTransport(self._handler),
+ )
+ return self
+
+
+class _Requests:
+ """The requests the hosted API stand-in received."""
+
+ def __init__(self) -> None:
+ self.received: list[httpx.Request] = []
+
+
+def _patch_client(mocker: MockerFixture, *, handler: Callable[[httpx.Request], httpx.Response]) -> _Requests:
+ requests = _Requests()
+
+ def _recording_handler(request: httpx.Request) -> httpx.Response:
+ requests.received.append(request)
+ return handler(request)
+
+ def _make(*, base_url: str | None = None, api_key: str | None = None) -> PipelexAPIClient:
+ assert base_url is None
+ return _MockTransportClient(api_key=api_key, handler=_recording_handler)
+
+ mocker.patch("pipelex.hosted.api_key_check.make_hosted_client", side_effect=_make)
+ return requests
+
+
+def _problem(*, status: int) -> Callable[[httpx.Request], httpx.Response]:
+ body = json.dumps({"type": "about:blank", "title": "Refused", "status": status, "detail": "Invalid API key"})
+
+ def _handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(status, content=body.encode(), headers={"content-type": "application/problem+json"}, request=request)
+
+ return _handler
+
+
+class TestApiKeyCheck:
+ @pytest.fixture(autouse=True)
+ def no_hosted_env(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ isolate_pipelex_api_key(monkeypatch, value="plx_sk_the_environment_key")
+ monkeypatch.delenv(PIPELEX_BASE_URL_ENV_KEY, raising=False)
+
+ @pytest.mark.parametrize("api_key", [TEST_KEY, "plx_sk_0123abcdef", "plx_sk_a.b~c-d_e"])
+ def test_a_pipelex_api_key_is_well_formed(self, api_key: str) -> None:
+ assert is_well_formed_pipelex_api_key(api_key=api_key)
+
+ @pytest.mark.parametrize("api_key", ["", "plx_sk_", "sk-openai", "plx_pk_abc", "plx_sk_abc def", "plx_sk_abc'", "plx_sk_abc#x", " plx_sk_abc"])
+ def test_anything_else_is_not(self, api_key: str) -> None:
+ assert not is_well_formed_pipelex_api_key(api_key=api_key)
+
+ def test_an_accepted_key_names_its_account_and_is_the_one_sent(self, mocker: MockerFixture) -> None:
+ requests = _patch_client(mocker, handler=lambda request: httpx.Response(200, json=PROFILE, request=request))
+
+ check = check_pipelex_api_key(api_key=TEST_KEY)
+
+ assert check.verdict == ApiKeyVerdict.ACCEPTED
+ assert check.account_email == "[email protected]"
+ assert check.base_url == API_URL
+ assert [request.url.path for request in requests.received] == ["/v1/me"]
+ assert requests.received[0].headers["Authorization"] == f"Bearer {TEST_KEY}"
+
+ @pytest.mark.parametrize("status", [401, 403])
+ def test_a_401_or_a_403_refuses_the_key(self, mocker: MockerFixture, status: int) -> None:
+ _patch_client(mocker, handler=_problem(status=status))
+
+ check = check_pipelex_api_key(api_key=TEST_KEY)
+
+ assert check.verdict == ApiKeyVerdict.REFUSED
+ assert check.http_status == status
+
+ @pytest.mark.parametrize("status", [404, 429, 500, 503])
+ def test_another_status_leaves_the_key_unchecked(self, mocker: MockerFixture, status: int) -> None:
+ _patch_client(mocker, handler=_problem(status=status))
+
+ check = check_pipelex_api_key(api_key=TEST_KEY)
+
+ assert check.verdict == ApiKeyVerdict.UNCHECKED
+ assert check.http_status == status
+ assert check.reason is not None
+ assert str(status) in check.reason
+
+ def test_an_unreachable_api_leaves_the_key_unchecked(self, mocker: MockerFixture) -> None:
+ def _unreachable(request: httpx.Request) -> httpx.Response:
+ msg = "connection refused"
+ raise httpx.ConnectError(msg, request=request)
+
+ _patch_client(mocker, handler=_unreachable)
+
+ check = check_pipelex_api_key(api_key=TEST_KEY)
+
+ assert check.verdict == ApiKeyVerdict.UNCHECKED
+ assert check.reason is not None
+ assert "could not be reached" in check.reason
+ assert TEST_KEY not in check.reason
+
+ @pytest.mark.parametrize("response_factory", [_html_page, _unexpected_json])
+ def test_an_answer_that_is_not_a_profile_leaves_the_key_unchecked(
+ self,
+ mocker: MockerFixture,
+ response_factory: Callable[[httpx.Request], httpx.Response],
+ ) -> None:
+ _patch_client(mocker, handler=response_factory)
+
+ check = check_pipelex_api_key(api_key=TEST_KEY)
+
+ assert check.verdict == ApiKeyVerdict.UNCHECKED
+
+ def test_a_base_url_that_is_not_an_origin_leaves_the_key_unchecked(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(PIPELEX_BASE_URL_ENV_KEY, "https://api.test/v1")
+
+ check = check_pipelex_api_key(api_key=TEST_KEY)
+
+ assert check.verdict == ApiKeyVerdict.UNCHECKED
+ assert check.reason is not None
+ assert PIPELEX_BASE_URL_ENV_KEY in check.reason
diff --git a/tests/unit/pipelex/hosted/test_hosted_client_factory.py b/tests/unit/pipelex/hosted/test_hosted_client_factory.py
index e813c73f0..76543992a 100644
--- a/tests/unit/pipelex/hosted/test_hosted_client_factory.py
+++ b/tests/unit/pipelex/hosted/test_hosted_client_factory.py
@@ -49,6 +49,11 @@ class TestHostedClientFactory:
monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "plx_sk_test_not_a_secret")
assert make_hosted_client().api_key == "plx_sk_test_not_a_secret"
+ def test_a_key_given_wins_over_the_environment(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """`pipelex login` checks a key it has not saved yet, whatever `PIPELEX_API_KEY` holds."""
+ monkeypatch.setenv(PIPELEX_API_KEY_ENV_KEY, "plx_sk_the_environment_key")
+ assert make_hosted_client(api_key="plx_sk_the_key_to_check").api_key == "plx_sk_the_key_to_check"
+
def test_no_key_is_anonymous(self) -> None:
"""No key is not refused here: a self-hosted runner answers its protocol routes without one."""
assert make_hosted_client().api_key == ""
diff --git a/tests/unit/pipelex/migration/test_former_release_across_directories.py b/tests/unit/pipelex/migration/test_former_release_across_directories.py
new file mode 100644
index 000000000..b61fd42f2
--- /dev/null
+++ b/tests/unit/pipelex/migration/test_former_release_across_directories.py
@@ -0,0 +1,322 @@
+"""The cleanup of a former release over the home and a project directory, read as the boot reads them.
+
+The boot merges one base `routing_profiles.toml`, the project's when it has one and the home's otherwise, with the
+home's override and then the project's. A profile one directory defines can therefore be made active by a file in the
+other, and a cleanup that read each directory alone would delete the profile and leave the `active` naming it, so the
+boot would fail on a profile it cannot find after a cleanup that reported success. The cleanup plans over the
+sequences the boot reads, and checks after writing that nothing it can see still stops the boot.
+"""
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from pipelex.cogt.model_routing.routing_profile import RoutingProfile
+from pipelex.cogt.model_routing.routing_profile_loader import load_active_routing_profile
+from pipelex.migration.former_release import (
+ detect_former_release_across,
+ former_release_boot_blockers,
+ inference_merge_sequences,
+ kit_default_routing_profile_name,
+ kit_routing_profile_library_path,
+)
+from pipelex.migration.former_release_cleanup import clean_former_release
+from pipelex.migration.plan import FileBlockedReason
+from pipelex.system.configuration.config_loader import (
+ BACKENDS_FILE_NAME,
+ BACKENDS_OVERRIDE_FILE_NAME,
+ INFERENCE_DIR_NAME,
+ ROUTING_PROFILES_FILE_NAME,
+ ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+ ConfigLoader,
+)
+from pipelex.system.environment import PIPELEX_HOME_ENV_KEY
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+
+V0_72_CONFIG_DIR = Path("tests/data/migration/former_release/v0_72")
+ENABLED_BACKENDS = ["openai", "anthropic"]
+BACKENDS_TEXT = '[openai]\nenabled = true\napi_key = "${OPENAI_API_KEY}"\n\n[anthropic]\nenabled = true\napi_key = "${ANTHROPIC_API_KEY}"\n'
+RETIRED_PROFILE_TEXT = '\n[profiles.{name}]\ndescription = "mine"\ndefault = "pipelex_gateway"\n'
+
+
+def _write(*, path: Path, text: str) -> Path:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="utf-8")
+ return path
+
+
+def _kit_routing_text() -> str:
+ return kit_routing_profile_library_path().read_text(encoding="utf-8")
+
+
+def _inference(*, config_dir: Path) -> Path:
+ return config_dir / INFERENCE_DIR_NAME
+
+
+def _boot_routing_sequence(*, home: Path, project: Path) -> list[Path]:
+ """The routing sequence the boot reads, spelled out here rather than derived: the base, then each override."""
+ project_base = _inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME
+ base = project_base if project_base.is_file() else _inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME
+ return [
+ base,
+ _inference(config_dir=home) / ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+ _inference(config_dir=project) / ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+ ]
+
+
+def _boot_backends_sequence(*, home: Path, project: Path) -> list[Path]:
+ project_base = _inference(config_dir=project) / BACKENDS_FILE_NAME
+ base = project_base if project_base.is_file() else _inference(config_dir=home) / BACKENDS_FILE_NAME
+ return [base, _inference(config_dir=home) / BACKENDS_OVERRIDE_FILE_NAME, _inference(config_dir=project) / BACKENDS_OVERRIDE_FILE_NAME]
+
+
+def _active_profile_the_boot_loads(*, home: Path, project: Path) -> str:
+ """The name of the profile the boot activates: it raises when the library names a profile it does not define."""
+ routing_paths = _boot_routing_sequence(home=home, project=project)
+ assert not former_release_boot_blockers(
+ backends_library_paths=_boot_backends_sequence(home=home, project=project), routing_profile_library_paths=routing_paths
+ )
+ return load_active_routing_profile(routing_profile_library_paths=routing_paths, enabled_backends=ENABLED_BACKENDS).name
+
+
+def _profile_the_home_boot_loads(*, home: Path) -> RoutingProfile:
+ """The profile a boot started outside any project activates: the home's base and override alone."""
+ routing_paths = [_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME, _inference(config_dir=home) / ROUTING_PROFILES_OVERRIDE_FILE_NAME]
+ assert not former_release_boot_blockers(
+ backends_library_paths=[_inference(config_dir=home) / BACKENDS_FILE_NAME, _inference(config_dir=home) / BACKENDS_OVERRIDE_FILE_NAME],
+ routing_profile_library_paths=routing_paths,
+ )
+ return load_active_routing_profile(routing_profile_library_paths=routing_paths, enabled_backends=ENABLED_BACKENDS)
+
+
+def home(tmp_path: Path) -> Path:
+ return tmp_path / "home" / ".pipelex"
+
+
+def project(tmp_path: Path) -> Path:
+ return tmp_path / "project" / ".pipelex"
+
+
+class TestFormerReleaseAcrossDirectories:
+ def test_a_home_override_activating_a_retired_profile_the_project_defines_stops_naming_it(self, home: Path, project: Path) -> None:
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME, text=_kit_routing_text())
+ home_override = _write(path=_inference(config_dir=home) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='active = "my_gw"\n')
+ _write(path=_inference(config_dir=project) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(path=_inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME, text=_kit_routing_text() + RETIRED_PROFILE_TEXT.format(name="my_gw"))
+
+ rehearsal = clean_former_release(config_dirs=[home, project], dry_run=True)
+ assert home_override in [file.file_path for file in rehearsal.files], "the override in the other directory is part of the plan"
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert not cleanup.needs_attention
+ assert "active" not in load_toml_from_path(home_override)
+ assert _active_profile_the_boot_loads(home=home, project=project) == kit_default_routing_profile_name()
+ assert all(findings.is_clean for findings in detect_former_release_across(config_dirs=[home, project]))
+
+ def test_a_project_override_activating_a_retired_profile_the_home_defines_stops_naming_it(self, home: Path, project: Path) -> None:
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME, text=_kit_routing_text() + RETIRED_PROFILE_TEXT.format(name="custom"))
+ project_override = _write(path=_inference(config_dir=project) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='active = "custom"\n')
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert not cleanup.needs_attention
+ assert "active" not in load_toml_from_path(project_override)
+ assert _active_profile_the_boot_loads(home=home, project=project) == kit_default_routing_profile_name()
+
+ def test_a_project_with_a_base_of_its_own_keeps_its_choice_when_the_home_is_cleaned(self, home: Path, project: Path) -> None:
+ """The home's base is not what this project boots on, and cleaning it moves nothing the project chose."""
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(
+ path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME,
+ text=_kit_routing_text().replace(f'active = "{kit_default_routing_profile_name()}"', 'active = "custom"')
+ + RETIRED_PROFILE_TEXT.format(name="custom"),
+ )
+ _write(path=_inference(config_dir=project) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ project_routing = _write(
+ path=_inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME,
+ text=_kit_routing_text() + '\n[profiles.custom]\ndescription = "the project\'s own"\ndefault = "openai"\n',
+ )
+ _write(path=_inference(config_dir=project) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='active = "custom"\n')
+ project_routing_before = project_routing.read_bytes()
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert not cleanup.needs_attention
+ assert project_routing.read_bytes() == project_routing_before
+ assert _active_profile_the_boot_loads(home=home, project=project) == "custom"
+ assert load_toml_from_path(_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME)["active"] == kit_default_routing_profile_name()
+
+ def test_a_write_that_leaves_the_boot_stopped_needs_attention_and_says_why(self, home: Path, project: Path) -> None:
+ """The safety net: after writing, what still stops the boot is reported, never a success."""
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT + "\n[pipelex_gateway]\nenabled = false\n")
+ _write(
+ path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME,
+ text=_kit_routing_text().replace(f'active = "{kit_default_routing_profile_name()}"', 'active = "ghost"'),
+ )
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert [file.was_applied for file in cleanup.files] == [True]
+ assert cleanup.needs_attention
+ assert len(cleanup.still_blocking) == 1
+ assert "'ghost'" in cleanup.still_blocking[0]
+
+ def test_a_dry_run_checks_the_files_as_it_would_leave_them(self, home: Path, project: Path) -> None:
+ """A dry run reports what its writes would leave stopping the boot, and writes nothing to find out."""
+ backends = _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT + "\n[pipelex_gateway]\nenabled = false\n")
+ _write(
+ path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME,
+ text=_kit_routing_text().replace(f'active = "{kit_default_routing_profile_name()}"', 'active = "ghost"'),
+ )
+ backends_before = backends.read_bytes()
+
+ rehearsal = clean_former_release(config_dirs=[home, project], dry_run=True)
+
+ assert len(rehearsal.still_blocking) == 1
+ assert "'ghost'" in rehearsal.still_blocking[0]
+ assert rehearsal.needs_attention
+ assert backends.read_bytes() == backends_before
+
+ def test_a_dry_run_does_not_report_what_its_own_writes_would_fix(self, home: Path, project: Path) -> None:
+ shutil.copytree(V0_72_CONFIG_DIR, home)
+
+ rehearsal = clean_former_release(config_dirs=[home, project], dry_run=True)
+
+ assert rehearsal.files
+ assert rehearsal.still_blocking == []
+ assert not rehearsal.needs_attention
+
+ @pytest.mark.parametrize("dry_run", [True, False])
+ def test_a_run_with_nothing_left_to_clean_still_says_what_stops_the_boot(self, home: Path, project: Path, dry_run: bool) -> None:
+ """A cleanup half done — from another project, or a file it could not write — leaves nothing a former release
+ left, and a boot still stopped: the retry is not clean, and says why.
+ """
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME, text=_kit_routing_text())
+ _write(path=_inference(config_dir=project) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='active = "team_gateway"\n')
+
+ retry = clean_former_release(config_dirs=[home, project], dry_run=dry_run)
+
+ assert retry.files == []
+ assert not retry.is_clean
+ assert retry.needs_attention
+ assert any("'team_gateway'" in problem for problem in retry.still_blocking)
+
+ @pytest.mark.parametrize("project_active", ["custom", "all_openai"])
+ def test_a_profile_one_boot_still_extends_keeps_its_description_in_the_other(self, home: Path, project: Path, project_active: str) -> None:
+ """The home's override adds routes to `custom`, which the home defines and the project's own base defines with a
+ Gateway default: the project's definition loses its default and keeps its description, so both boots start.
+ """
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(
+ path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME,
+ text='active = "custom"\n\n[profiles.custom]\ndescription = "home custom"\ndefault = "openai"\n',
+ )
+ _write(path=_inference(config_dir=home) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='[profiles.custom.routes]\n"claude-*" = "anthropic"\n')
+ _write(
+ path=_inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME,
+ text=f'active = "{project_active}"\n\n[profiles.all_openai]\ndescription = "o"\ndefault = "openai"\n\n'
+ '[profiles.custom]\ndescription = "project custom"\ndefault = "pipelex_gateway"\n',
+ )
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert not cleanup.needs_attention
+ assert _profile_the_home_boot_loads(home=home).name == "custom"
+ assert _active_profile_the_boot_loads(home=home, project=project) == project_active
+ project_library = load_toml_from_path(_inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME)
+ assert project_library["profiles"]["custom"] == {"description": "project custom"}
+
+ def test_a_home_profile_a_project_override_retargets_stays_for_the_project_while_the_home_moves_off_it(self, home: Path, project: Path) -> None:
+ """v0.72's home activates `all_pipelex_gateway`; the project's override retargets it to OpenAI. The project's boot
+ keeps starting on it, through an `active` of its own, and the boot outside the project moves to the kit's profile
+ instead of being left on a profile with nothing to route by.
+ """
+ shutil.copytree(V0_72_CONFIG_DIR, home)
+ project_override = _write(
+ path=_inference(config_dir=project) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='[profiles.all_pipelex_gateway]\ndefault = "openai"\n'
+ )
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert not cleanup.needs_attention
+ assert load_toml_from_path(project_override)["active"] == "all_pipelex_gateway"
+ home_boot = _profile_the_home_boot_loads(home=home)
+ assert home_boot.name == kit_default_routing_profile_name()
+ assert home_boot.fallback_order
+ assert _active_profile_the_boot_loads(home=home, project=project) == "all_pipelex_gateway"
+ project_boot = load_active_routing_profile(
+ routing_profile_library_paths=_boot_routing_sequence(home=home, project=project), enabled_backends=ENABLED_BACKENDS
+ )
+ assert project_boot.default == "openai"
+
+ @pytest.mark.parametrize("project_has_an_override", [False, True])
+ def test_a_shared_override_naming_a_profile_only_one_boot_has_is_never_moved_under_the_other(
+ self, home: Path, project: Path, project_has_an_override: bool
+ ) -> None:
+ """The home's override activates the user's own `all_pipelex_gateway`, routed to OpenAI and defined in the home's
+ base; the project's base has no such profile. Deleting that `active` would switch the boot outside the project to
+ another profile: it stays. The project is given its own `active` when it has an override to hold it, and the
+ shared file is otherwise left for a hand edit naming the conflict.
+ """
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(
+ path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME,
+ text='active = "all_openai"\n\n[profiles.all_openai]\ndescription = "o"\ndefault = "openai"\n\n'
+ '[profiles.all_pipelex_gateway]\ndescription = "mine"\ndefault = "openai"\n',
+ )
+ home_override = _write(path=_inference(config_dir=home) / ROUTING_PROFILES_OVERRIDE_FILE_NAME, text='active = "all_pipelex_gateway"\n')
+ _write(
+ path=_inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME,
+ text='active = "all_openai"\n\n[profiles.all_openai]\ndescription = "o"\ndefault = "openai"\n',
+ )
+ project_override = _inference(config_dir=project) / ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ if project_has_an_override:
+ _write(path=project_override, text="# my project's own settings\n")
+ home_override_before = home_override.read_bytes()
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ assert home_override.read_bytes() == home_override_before
+ assert _profile_the_home_boot_loads(home=home).name == "all_pipelex_gateway"
+ if project_has_an_override:
+ assert not cleanup.needs_attention
+ assert load_toml_from_path(project_override)["active"] == "all_openai"
+ assert _active_profile_the_boot_loads(home=home, project=project) == "all_openai"
+ else:
+ assert [(file.file_path, file.blocked_reason) for file in cleanup.files] == [(home_override, FileBlockedReason.NEEDS_A_HAND_EDIT)]
+ assert str(project_override) in (cleanup.files[0].blocked_detail or "")
+ assert cleanup.still_blocking
+ assert not project_override.exists()
+
+ def test_the_sequences_are_the_ones_the_config_loader_reads(self, home: Path, project: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """One derivation of what a boot reads: the cleanup's sequences, on a real home and project, are the loader's."""
+ _write(path=_inference(config_dir=home) / BACKENDS_FILE_NAME, text=BACKENDS_TEXT)
+ _write(path=_inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME, text=_kit_routing_text())
+ _write(path=project.parent / "pyproject.toml", text="[project]\nname = 'x'\n")
+ _write(path=_inference(config_dir=project) / ROUTING_PROFILES_FILE_NAME, text=_kit_routing_text())
+ monkeypatch.setenv(PIPELEX_HOME_ENV_KEY, str(home))
+ monkeypatch.chdir(project.parent)
+ loader = ConfigLoader()
+ config_dirs = loader.existing_config_dirs
+ assert config_dirs == [home, project]
+
+ routing_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=ROUTING_PROFILES_FILE_NAME, override_file_name=ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ )
+ backends_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=BACKENDS_FILE_NAME, override_file_name=BACKENDS_OVERRIDE_FILE_NAME
+ )
+
+ assert routing_sequences[-1] == loader.routing_profiles_file_paths()
+ assert backends_sequences[-1] == loader.backends_file_paths()
+ assert routing_sequences[0] == [
+ _inference(config_dir=home) / ROUTING_PROFILES_FILE_NAME,
+ _inference(config_dir=home) / ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+ ]
diff --git a/tests/unit/pipelex/migration/test_former_release_cleanup.py b/tests/unit/pipelex/migration/test_former_release_cleanup.py
new file mode 100644
index 000000000..c14b5711b
--- /dev/null
+++ b/tests/unit/pipelex/migration/test_former_release_cleanup.py
@@ -0,0 +1,458 @@
+"""The cleanup of what a former release left: goldens, backups, dry runs, overrides, and a write that cannot be made.
+
+The v0.72 directory under `tests/data/migration/former_release/v0_72/` is that release's kit, copied from its wheel;
+`v0_72_cleaned/` holds what its two rewritten documents must be, byte for byte, once cleaned. Everything else the
+release left is a file of its own, which the cleanup removes and keeps a copy of.
+"""
+
+import shutil
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from pipelex.cogt.model_routing.routing_profile_loader import load_active_routing_profile
+from pipelex.fix_ops.file_transaction import PendingFileUpdate, commit_file_updates
+from pipelex.kit.paths import RETIRED_SERVICE_FILE_NAME, get_kit_configs_dir
+from pipelex.migration import former_release_cleanup as former_release_cleanup_module
+from pipelex.migration.backup import backup_path_for, existing_backups_of
+from pipelex.migration.former_release import (
+ MODEL_SPECS_SECTION_KEY,
+ FormerReleaseFinding,
+ detect_former_release,
+ kit_default_routing_profile_name,
+)
+from pipelex.migration.former_release_cleanup import CleanedText, FormerReleaseFileAction, clean_former_release
+from pipelex.migration.plan import FileBlockedReason
+from pipelex.suggested_fix import DeleteKeyOp
+from pipelex.system.configuration.config_loader import (
+ BACKENDS_DIR_NAME,
+ BACKENDS_FILE_NAME,
+ BACKENDS_OVERRIDE_FILE_NAME,
+ INFERENCE_DIR_NAME,
+ ROUTING_PROFILES_FILE_NAME,
+ ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+)
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+
+if TYPE_CHECKING:
+ from pytest_mock import MockerFixture
+
+V0_72_CONFIG_DIR = Path("tests/data/migration/former_release/v0_72")
+V0_72_CLEANED_DIR = Path("tests/data/migration/former_release/v0_72_cleaned")
+PREVIOUS_RELEASE_KIT_DIR = Path("tests/data/inference/previous_release_kit")
+
+MOMENT = datetime(2026, 10, 7, 12, 0, 0, tzinfo=UTC)
+
+BACKENDS = f"{INFERENCE_DIR_NAME}/{BACKENDS_FILE_NAME}"
+ROUTING = f"{INFERENCE_DIR_NAME}/{ROUTING_PROFILES_FILE_NAME}"
+BACKEND_FILES = f"{INFERENCE_DIR_NAME}/{BACKENDS_DIR_NAME}"
+
+# What the cleanup does to the v0.72 directory, file by file, in the order it reports them.
+V0_72_ACTIONS = [
+ (BACKENDS, FormerReleaseFileAction.REWRITE),
+ (f"{BACKEND_FILES}/pipelex_gateway.toml", FormerReleaseFileAction.REMOVE),
+ (f"{BACKEND_FILES}/pipelex_gateway_models.md", FormerReleaseFileAction.REMOVE),
+ (f"{BACKEND_FILES}/pipelex_gateway_models_plain.md", FormerReleaseFileAction.REMOVE),
+ (RETIRED_SERVICE_FILE_NAME, FormerReleaseFileAction.REMOVE),
+ (ROUTING, FormerReleaseFileAction.REWRITE),
+]
+
+
+def _copy_v0_72(*, tmp_path: Path) -> Path:
+ config_dir = tmp_path / ".pipelex"
+ shutil.copytree(V0_72_CONFIG_DIR, config_dir)
+ return config_dir
+
+
+def _snapshot(*, directory: Path) -> dict[str, bytes]:
+ return {path.relative_to(directory).as_posix(): path.read_bytes() for path in sorted(directory.rglob("*")) if path.is_file()}
+
+
+class TestCleanFormerRelease:
+ def test_a_v0_72_directory_is_cleaned_to_its_goldens_with_one_backup_per_file(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ before = _snapshot(directory=config_dir)
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert [(file.file_path.relative_to(config_dir).as_posix(), file.action) for file in cleanup.files] == V0_72_ACTIONS
+ assert not cleanup.needs_attention
+ assert all(file.was_applied and file.changes for file in cleanup.files)
+ for relative_path in (BACKENDS, ROUTING):
+ assert (config_dir / relative_path).read_bytes() == (V0_72_CLEANED_DIR / relative_path).read_bytes(), relative_path
+ for relative_path, action in V0_72_ACTIONS:
+ original = config_dir / relative_path
+ backup = backup_path_for(path=original, moment=MOMENT)
+ assert backup.read_bytes() == before[relative_path], f"the backup of {relative_path} is not the original"
+ assert existing_backups_of(path=original) == [backup]
+ assert original.exists() == (action == FormerReleaseFileAction.REWRITE)
+ assert detect_former_release(config_dir=config_dir).is_clean
+
+ def test_what_the_release_left_for_pipelex_manifold_stays_as_it_was(self, tmp_path: Path) -> None:
+ """Manifold is not retired: its disabled table, its file and its profile are inert, and the hosted plane declares its own."""
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ manifold_file = config_dir / BACKEND_FILES / "pipelex_manifold.toml"
+ manifold_file_before = manifold_file.read_bytes()
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert manifold_file.read_bytes() == manifold_file_before
+ assert not any("manifold" in change for file in cleanup.files for change in file.changes)
+ backends = load_toml_from_path(config_dir / BACKENDS)
+ assert backends["pipelex_manifold"] == load_toml_from_path(V0_72_CONFIG_DIR / BACKENDS)["pipelex_manifold"]
+ routing = load_toml_from_path(config_dir / ROUTING)
+ assert routing["profiles"]["all_pipelex_manifold"] == load_toml_from_path(V0_72_CONFIG_DIR / ROUTING)["profiles"]["all_pipelex_manifold"]
+
+ def test_the_cleaned_routing_profiles_make_the_kit_default_active(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+
+ clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ routing = load_toml_from_path(config_dir / ROUTING)
+ kit_routing = load_toml_from_path(Path(str(get_kit_configs_dir())) / ROUTING)
+ kit_default = kit_default_routing_profile_name()
+ assert routing["active"] == kit_default
+ assert routing["profiles"][kit_default] == kit_routing["profiles"][kit_default]
+ assert next(iter(routing["profiles"])) == kit_default, "the profile the file makes active comes first, as in the kit"
+
+ def test_a_dry_run_reports_the_same_cleanup_and_writes_nothing(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ before = _snapshot(directory=config_dir)
+
+ rehearsal = clean_former_release(config_dirs=[config_dir], dry_run=True, moment=MOMENT)
+
+ assert _snapshot(directory=config_dir) == before
+ assert rehearsal.dry_run
+ assert [(file.file_path.relative_to(config_dir).as_posix(), file.action) for file in rehearsal.files] == V0_72_ACTIONS
+ assert not any(file.was_applied or file.backup_path for file in rehearsal.files)
+
+ applied = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+ assert [file.changes for file in rehearsal.files] == [file.changes for file in applied.files]
+
+ def test_a_second_run_finds_nothing_to_do(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+ after_first = _snapshot(directory=config_dir)
+
+ second = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=datetime(2026, 10, 8, tzinfo=UTC))
+
+ assert second.is_clean
+ assert _snapshot(directory=config_dir) == after_first
+
+ def test_the_changes_name_what_was_removed_and_where_the_active_profile_went(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=True)
+
+ changes = {file.file_path.relative_to(config_dir).as_posix(): file.changes for file in cleanup.files}
+ assert changes[BACKENDS] == ["removed the 'pipelex_gateway' backend"]
+ kit_default = kit_default_routing_profile_name()
+ assert changes[ROUTING] == [
+ f"added the routing profile '{kit_default}', as this release ships it",
+ f"moved the active routing profile from 'all_pipelex_gateway' to '{kit_default}'",
+ "removed the routing profile 'all_pipelex_gateway'",
+ "removed the routing profile 'example_routing_using_patterns'",
+ "removed the route of 'gpt-5.4-nano' to 'pipelex_gateway' from the routing profile 'example_routing_using_specific_models'",
+ "removed the route of 'claude-4-sonnet' to 'pipelex_gateway' from the routing profile 'example_routing_using_specific_models'",
+ "removed the route of 'gemini-2.5-flash-lite' to 'pipelex_gateway' from the routing profile 'example_routing_using_specific_models'",
+ "removed the route of 'grok-3' to 'pipelex_gateway' from the routing profile 'example_routing_using_specific_models'",
+ "removed a comment about the Pipelex Gateway",
+ ]
+ assert changes[RETIRED_SERVICE_FILE_NAME] == ["removed the record of the Pipelex Gateway's terms acceptance"]
+
+ def test_an_override_loses_its_retired_active_profile_and_its_base_moves(self, tmp_path: Path) -> None:
+ """An override that named the Gateway's profile stops naming any, so its base, moved to the kit default, applies."""
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ inference_dir = config_dir / INFERENCE_DIR_NAME
+ (inference_dir / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('# my choice\nactive = "all_pipelex_gateway"\n', encoding="utf-8")
+ (inference_dir / BACKENDS_OVERRIDE_FILE_NAME).write_text("[pipelex_gateway]\nenabled = true\n\n[openai]\nenabled = true\n", encoding="utf-8")
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert not cleanup.needs_attention
+ assert (inference_dir / ROUTING_PROFILES_OVERRIDE_FILE_NAME).read_text(encoding="utf-8") == ""
+ assert (inference_dir / BACKENDS_OVERRIDE_FILE_NAME).read_text(encoding="utf-8") == "[openai]\nenabled = true\n"
+ assert load_toml_from_path(inference_dir / ROUTING_PROFILES_FILE_NAME)["active"] == kit_default_routing_profile_name()
+ assert detect_former_release(config_dir=config_dir).is_clean
+
+ def test_a_base_already_holding_the_kit_default_profile_is_given_no_second_one(self, tmp_path: Path) -> None:
+ """The current kit with `active` left on the Gateway's profile: moving it back is the whole cleanup."""
+ kit_routing = Path(str(get_kit_configs_dir())) / ROUTING
+ kit_text = kit_routing.read_text(encoding="utf-8")
+ kit_default = kit_default_routing_profile_name()
+ config_dir = tmp_path / ".pipelex"
+ (config_dir / INFERENCE_DIR_NAME).mkdir(parents=True)
+ active_line = f'active = "{kit_default}"'
+ assert kit_text.count(active_line) == 1
+ (config_dir / ROUTING).write_text(kit_text.replace(active_line, 'active = "all_pipelex_gateway"'), encoding="utf-8")
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert [file.changes for file in cleanup.files] == [[f"moved the active routing profile from 'all_pipelex_gateway' to '{kit_default}'"]]
+ assert (config_dir / ROUTING).read_text(encoding="utf-8") == kit_text
+
+ @pytest.mark.parametrize(
+ "routing_text",
+ [
+ pytest.param('active = "all_pipelex_gateway"\n', id="no_profiles"),
+ pytest.param(
+ 'active = "all_pipelex_gateway"\n\n[profiles.all_openai]\ndescription = "o"\ndefault = "openai"\n\n[notes]\nseen = true\n\n'
+ '[profiles.all_anthropic]\ndescription = "a"\ndefault = "anthropic"\n',
+ id="profiles_spread_between_tables",
+ ),
+ ],
+ )
+ def test_a_hand_shaped_base_is_given_the_kit_default_profile(self, tmp_path: Path, routing_text: str) -> None:
+ config_dir = tmp_path / ".pipelex"
+ (config_dir / INFERENCE_DIR_NAME).mkdir(parents=True)
+ (config_dir / ROUTING).write_text(routing_text, encoding="utf-8")
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ # The shape is the cleanup's to handle; a table the routing library does not know, like the `notes` one, is
+ # the boot's to refuse, and the check after the write reports it rather than calling the machine fixed.
+ assert not cleanup.blocked_files
+ kit_default = kit_default_routing_profile_name()
+ routing = load_toml_from_path(config_dir / ROUTING)
+ kit_routing = load_toml_from_path(Path(str(get_kit_configs_dir())) / ROUTING)
+ assert routing["active"] == kit_default
+ assert routing["profiles"][kit_default] == kit_routing["profiles"][kit_default]
+
+ def test_a_model_specs_section_key_is_removed_and_its_backend_kept(self, tmp_path: Path) -> None:
+ config_dir = tmp_path / ".pipelex"
+ shutil.copytree(PREVIOUS_RELEASE_KIT_DIR, config_dir / INFERENCE_DIR_NAME)
+
+ clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ backends = load_toml_from_path(config_dir / BACKENDS)
+ assert "retired_preview" in backends
+ assert not any(MODEL_SPECS_SECTION_KEY in table for table in backends.values() if isinstance(table, dict))
+ assert detect_former_release(config_dir=config_dir).is_clean
+
+ def test_a_file_that_cannot_be_written_is_left_as_it_was_and_its_siblings_are_cleaned(self, tmp_path: Path, mocker: "MockerFixture") -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ routing_path = config_dir / ROUTING
+ original_routing = routing_path.read_bytes()
+
+ def refuse_the_routing_file(updates: list[PendingFileUpdate]) -> None:
+ if updates[0].snapshot.path == routing_path.resolve():
+ raise PermissionError(13, "Permission denied")
+ commit_file_updates(updates)
+
+ mocker.patch("pipelex.migration.runner.commit_file_updates", side_effect=refuse_the_routing_file)
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert cleanup.needs_attention
+ assert [(file.file_path.name, file.blocked_reason) for file in cleanup.blocked_files] == [
+ (ROUTING_PROFILES_FILE_NAME, FileBlockedReason.UNWRITABLE)
+ ]
+ assert routing_path.read_bytes() == original_routing
+ assert existing_backups_of(path=routing_path) == [], "the copy taken for a write that did not happen is taken back"
+ assert (config_dir / BACKENDS).read_bytes() == (V0_72_CLEANED_DIR / BACKENDS).read_bytes()
+ assert not (config_dir / RETIRED_SERVICE_FILE_NAME).exists()
+
+ def test_a_symlinked_file_of_its_own_is_removed_as_a_link_and_what_it_names_is_kept(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ dotfiles_copy = tmp_path / "dotfiles" / RETIRED_SERVICE_FILE_NAME
+ dotfiles_copy.parent.mkdir()
+ (config_dir / RETIRED_SERVICE_FILE_NAME).rename(dotfiles_copy)
+ (config_dir / RETIRED_SERVICE_FILE_NAME).symlink_to(dotfiles_copy)
+
+ clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert not (config_dir / RETIRED_SERVICE_FILE_NAME).is_symlink()
+ assert not (config_dir / RETIRED_SERVICE_FILE_NAME).exists()
+ assert dotfiles_copy.is_file()
+
+ def test_each_directory_is_cleaned_once_and_a_clean_one_reports_nothing(self, tmp_path: Path) -> None:
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ clean_dir = tmp_path / "current"
+ shutil.copytree(Path(str(get_kit_configs_dir())), clean_dir)
+
+ cleanup = clean_former_release(config_dirs=[config_dir, clean_dir, config_dir], dry_run=True)
+
+ assert len(cleanup.files) == len(V0_72_ACTIONS)
+ assert clean_former_release(config_dirs=[clean_dir], dry_run=False).is_clean
+
+ @pytest.mark.parametrize("dry_run", [False, True])
+ def test_a_directory_with_nothing_to_clean_gains_no_gitignore(self, tmp_path: Path, dry_run: bool) -> None:
+ clean_dir = tmp_path / ".pipelex"
+ (clean_dir / INFERENCE_DIR_NAME).mkdir(parents=True)
+
+ clean_former_release(config_dirs=[clean_dir], dry_run=dry_run)
+
+ assert not (clean_dir / ".gitignore").exists()
+
+ @pytest.mark.parametrize("dry_run", [True, False])
+ def test_inline_profiles_are_left_for_a_hand_edit_rather_than_activating_a_profile_they_lack(self, tmp_path: Path, dry_run: bool) -> None:
+ """The kit's profile cannot be added to an inline `profiles` table, so `active` is not moved to it either."""
+ config_dir = tmp_path / ".pipelex"
+ routing_path = config_dir / ROUTING
+ routing_path.parent.mkdir(parents=True)
+ routing_path.write_text(
+ 'active = "all_pipelex_gateway"\n'
+ 'profiles = { all_pipelex_gateway = { description = "g", default = "pipelex_gateway" }, '
+ 'mine = { description = "m", default = "openai" } }\n',
+ encoding="utf-8",
+ )
+ before = routing_path.read_bytes()
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=dry_run, moment=MOMENT)
+
+ assert [(file.file_path, file.blocked_reason) for file in cleanup.files] == [(routing_path, FileBlockedReason.NEEDS_A_HAND_EDIT)]
+ assert kit_default_routing_profile_name() in (cleanup.files[0].blocked_detail or "")
+ assert cleanup.needs_attention
+ assert routing_path.read_bytes() == before
+
+ def test_an_override_retargeting_a_profile_to_a_retired_backend_loses_that_default_and_nothing_else(self, tmp_path: Path) -> None:
+ """The profile is the user's, defined in the base with routes of its own: only the override's choice of backend goes."""
+ config_dir = tmp_path / ".pipelex"
+ base_path = config_dir / ROUTING
+ override_path = config_dir / INFERENCE_DIR_NAME / ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ base_path.parent.mkdir(parents=True)
+ base_path.write_text(
+ 'active = "custom"\n\n[profiles.custom]\ndescription = "my tuned routes"\ndefault = "openai"\n\n'
+ '[profiles.custom.routes]\n"claude-*" = "anthropic"\n"gpt-5*" = "openai"\n',
+ encoding="utf-8",
+ )
+ override_path.write_text('[profiles.custom]\ndefault = "pipelex_gateway"\n', encoding="utf-8")
+ base_before = base_path.read_bytes()
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert not cleanup.needs_attention
+ assert [(file.file_path, file.changes) for file in cleanup.files] == [
+ (override_path, ["removed 'default', which named 'pipelex_gateway', from the routing profile 'custom'"])
+ ]
+ assert base_path.read_bytes() == base_before
+ assert override_path.read_text(encoding="utf-8") == "", "a table left with nothing to override goes with its key"
+ profile = load_active_routing_profile(routing_profile_library_paths=[base_path, override_path], enabled_backends=["openai", "anthropic"])
+ assert (profile.name, profile.default, profile.routes) == ("custom", "openai", {"claude-*": "anthropic", "gpt-5*": "openai"})
+ assert detect_former_release(config_dir=config_dir).is_clean
+
+ def test_a_note_of_the_users_about_a_gateway_of_their_own_stays(self, tmp_path: Path) -> None:
+ """Only the former release's own wording marks a head comment as its own; a user's note about another gateway is theirs."""
+ config_dir = tmp_path / ".pipelex"
+ backends_path = config_dir / BACKENDS
+ backends_path.parent.mkdir(parents=True)
+ user_note = (
+ "# Team note: all OpenAI traffic goes through our corporate API gateway (see wiki).\n# Do not change the endpoint without asking infra.\n"
+ )
+ backends_path.write_text(
+ f'{user_note}\n[openai]\nenabled = true\napi_key = "${{OPENAI_API_KEY}}"\n\n[pipelex_gateway]\nenabled = false\n', encoding="utf-8"
+ )
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert [file.changes for file in cleanup.files] == [["removed the 'pipelex_gateway' backend"]]
+ text = backends_path.read_text(encoding="utf-8")
+ assert text.startswith(user_note)
+ assert "pipelex_gateway" not in text
+
+ def test_a_catch_all_route_to_the_gateway_is_deleted_as_the_literal_key_it_is(self, tmp_path: Path) -> None:
+ """`"*"` is a route a user writes to send every other model somewhere; to the cleanup it is a key like any other."""
+ config_dir = tmp_path / ".pipelex"
+ routing_path = config_dir / ROUTING
+ routing_path.parent.mkdir(parents=True)
+ routing_path.write_text(
+ 'active = "mine"\n\n[profiles.mine]\ndescription = "everything through the gateway except claude"\ndefault = "openai"\n\n'
+ '[profiles.mine.routes]\n"*" = "pipelex_gateway"\n"claude-*" = "anthropic"\n',
+ encoding="utf-8",
+ )
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert not cleanup.needs_attention
+ assert [file.changes for file in cleanup.files] == [["removed the route of '*' to 'pipelex_gateway' from the routing profile 'mine'"]]
+ profile = load_active_routing_profile(routing_profile_library_paths=[routing_path], enabled_backends=["openai", "anthropic"])
+ assert (profile.name, profile.default, profile.routes) == ("mine", "openai", {"claude-*": "anthropic"})
+
+ def test_a_profile_named_like_the_wildcard_is_the_only_one_removed(self, tmp_path: Path) -> None:
+ """A profile spelled `"*"` names itself, never every profile of the file."""
+ config_dir = tmp_path / ".pipelex"
+ routing_path = config_dir / ROUTING
+ routing_path.parent.mkdir(parents=True)
+ routing_path.write_text(
+ 'active = "mine"\n\n[profiles."*"]\ndescription = "odd"\ndefault = "pipelex_gateway"\n\n'
+ '[profiles.mine]\ndescription = "mine"\ndefault = "openai"\n',
+ encoding="utf-8",
+ )
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert not cleanup.needs_attention
+ assert list(load_toml_from_path(routing_path)["profiles"]) == ["mine"]
+
+ def test_a_change_the_cleanup_cannot_express_blocks_its_file_and_never_the_run(self, tmp_path: Path, mocker: "MockerFixture") -> None:
+ """One odd file is reported for a hand edit, and every other file is still cleaned: nothing crosses the per-file boundary."""
+ config_dir = _copy_v0_72(tmp_path=tmp_path)
+ routing_path = config_dir / ROUTING
+ routing_before = routing_path.read_bytes()
+ real_cleaned_text = former_release_cleanup_module.cleaned_text
+
+ def _refused_for_the_routing_file(*, text: str, findings: list[FormerReleaseFinding], kit_default: str, is_override: bool) -> CleanedText:
+ if findings[0].file_path == routing_path:
+ DeleteKeyOp(table_path=[], key="*")
+ return real_cleaned_text(text=text, findings=findings, kit_default=kit_default, is_override=is_override)
+
+ mocker.patch.object(former_release_cleanup_module, "cleaned_text", side_effect=_refused_for_the_routing_file)
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert [file.file_path for file in cleanup.blocked_files] == [routing_path]
+ assert cleanup.blocked_files[0].blocked_reason is FileBlockedReason.NEEDS_A_HAND_EDIT
+ assert routing_path.read_bytes() == routing_before
+ assert (config_dir / BACKENDS).read_bytes() == (V0_72_CLEANED_DIR / BACKENDS).read_bytes()
+
+ def test_a_users_notes_mentioning_the_gateway_at_the_head_stay(self, tmp_path: Path) -> None:
+ """Only the paragraphs the former release itself shipped go from the head of a document; a user's note naming the Gateway stays."""
+ config_dir = tmp_path / ".pipelex"
+ backends_path = config_dir / BACKENDS
+ routing_path = config_dir / ROUTING
+ backends_path.parent.mkdir(parents=True)
+ backends_note = "# My backends. Note to self: I stopped using pipelex_gateway in March, keys are in 1Password.\n"
+ routing_note = "# Kept for history: the old pipelex_gateway profile was my default until v0.72.\n"
+ backends_path.write_text(
+ f'{backends_note}[openai]\nenabled = true\napi_key = "${{OPENAI_API_KEY}}"\nmodel_specs_section = "openai"\n\n'
+ '[anthropic]\nenabled = true\napi_key = "${ANTHROPIC_API_KEY}"\n',
+ encoding="utf-8",
+ )
+ routing_path.write_text(
+ f'active = "mine"\n\n{routing_note}[profiles.mine]\ndescription = "mine"\ndefault = "openai"\n\n'
+ '[profiles.old]\ndescription = "old"\ndefault = "pipelex_gateway"\n',
+ encoding="utf-8",
+ )
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=False, moment=MOMENT)
+
+ assert not cleanup.needs_attention
+ assert len(cleanup.applied_files) == 2
+ assert backends_path.read_text(encoding="utf-8").startswith(backends_note)
+ assert routing_note in routing_path.read_text(encoding="utf-8")
+ assert all("removed a comment" not in change for file in cleanup.files for change in file.changes)
+
+ @pytest.mark.parametrize("dry_run", [True, False])
+ def test_a_rewrite_that_would_stop_a_boot_that_starts_is_never_written(self, tmp_path: Path, mocker: "MockerFixture", dry_run: bool) -> None:
+ """The last check before writing reads the new texts as the boot would: a rewrite the plan did not foresee leaving a
+ boot that starts today without its profile is left for a hand edit, whatever produced it.
+ """
+ config_dir = tmp_path / ".pipelex"
+ routing_path = config_dir / ROUTING
+ routing_path.parent.mkdir(parents=True)
+ kit_routing = (Path(str(get_kit_configs_dir())) / ROUTING).read_text(encoding="utf-8")
+ routing_path.write_text(kit_routing + '\n[profiles.old]\ndescription = "old"\ndefault = "pipelex_gateway"\n', encoding="utf-8")
+ routing_before = routing_path.read_bytes()
+
+ def _a_cleanup_bug(**kwargs: object) -> CleanedText:
+ return CleanedText(text=f'active = "{kwargs["kit_default"]}"\n', changes=["removed every routing profile"])
+
+ mocker.patch.object(former_release_cleanup_module, "cleaned_text", side_effect=_a_cleanup_bug)
+
+ cleanup = clean_former_release(config_dirs=[config_dir], dry_run=dry_run, moment=MOMENT)
+
+ assert [(file.file_path, file.blocked_reason) for file in cleanup.blocked_files] == [(routing_path, FileBlockedReason.NEEDS_A_HAND_EDIT)]
+ assert "would stop Pipelex from starting as it does now" in (cleanup.blocked_files[0].blocked_detail or "")
+ assert routing_path.read_bytes() == routing_before
diff --git a/tests/unit/pipelex/migration/test_former_release_detection.py b/tests/unit/pipelex/migration/test_former_release_detection.py
new file mode 100644
index 000000000..b547bc8be
--- /dev/null
+++ b/tests/unit/pipelex/migration/test_former_release_detection.py
@@ -0,0 +1,239 @@
+"""What a former release left in a configuration directory, found by a pure read.
+
+The machine this is about is every former user's: a release that ran models through the Pipelex Gateway, or offered
+Pipelex Manifold, wrote tables, routing profiles and files that the current runtime refuses or ignores. The v0.72
+directory under `tests/data/migration/former_release/v0_72/` is that release's kit, copied from its wheel, and
+`tests/data/inference/previous_release_kit/` is the same era under a neutral name for the Manifold table, which is
+what makes it the specimen for a `model_specs_section` key on a backend whose name means nothing to the detector.
+"""
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from pipelex.kit.paths import get_kit_configs_dir
+from pipelex.migration.former_release import (
+ FormerReleaseFinding,
+ FormerReleaseFindingKind,
+ detect_former_release,
+ former_release_boot_blockers,
+)
+from pipelex.system.configuration.config_loader import (
+ BACKENDS_DIR_NAME,
+ BACKENDS_FILE_NAME,
+ BACKENDS_OVERRIDE_FILE_NAME,
+ INFERENCE_DIR_NAME,
+ ROUTING_PROFILES_FILE_NAME,
+ ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+)
+
+V0_72_CONFIG_DIR = Path("tests/data/migration/former_release/v0_72")
+PREVIOUS_RELEASE_KIT_DIR = Path("tests/data/inference/previous_release_kit")
+
+# The line of the v0.72 kit's `backends.toml` that enables the Pipelex Gateway.
+GATEWAY_ENABLED_LINE = "enabled = true # Enable after accepting terms via `pipelex init config`"
+
+
+def _copy_tree(*, source: Path, destination: Path) -> Path:
+ shutil.copytree(source, destination)
+ return destination
+
+
+def _summary(*, findings: list[FormerReleaseFinding], config_dir: Path) -> set[tuple[str, str, str | None, str | None, bool]]:
+ """Each finding as (kind, file relative to the directory, subject, route pattern, blocks boot)."""
+ return {
+ (finding.kind, finding.file_path.relative_to(config_dir).as_posix(), finding.subject, finding.route_pattern, finding.blocks_boot)
+ for finding in findings
+ }
+
+
+def _snapshot(*, directory: Path) -> dict[str, bytes]:
+ return {path.relative_to(directory).as_posix(): path.read_bytes() for path in sorted(directory.rglob("*")) if path.is_file()}
+
+
+class TestDetectFormerRelease:
+ def test_a_v0_72_directory_is_found_in_full(self, tmp_path: Path) -> None:
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ backends = f"{INFERENCE_DIR_NAME}/{BACKENDS_FILE_NAME}"
+ routing = f"{INFERENCE_DIR_NAME}/{ROUTING_PROFILES_FILE_NAME}"
+ backend_files = f"{INFERENCE_DIR_NAME}/{BACKENDS_DIR_NAME}"
+ assert findings.config_dir == config_dir
+ assert _summary(findings=findings.findings, config_dir=config_dir) == {
+ (FormerReleaseFindingKind.RETIRED_BACKEND_TABLE, backends, "pipelex_gateway", None, True),
+ (FormerReleaseFindingKind.RETIRED_BACKEND_FILE, f"{backend_files}/pipelex_gateway.toml", "pipelex_gateway", None, False),
+ (FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE, f"{backend_files}/pipelex_gateway_models.md", None, None, False),
+ (FormerReleaseFindingKind.GATEWAY_MODELS_REFERENCE, f"{backend_files}/pipelex_gateway_models_plain.md", None, None, False),
+ (FormerReleaseFindingKind.SERVICE_FILE, "pipelex_service.toml", None, None, False),
+ (FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE, routing, "all_pipelex_gateway", None, True),
+ (FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE, routing, "all_pipelex_gateway", None, False),
+ (FormerReleaseFindingKind.RETIRED_ROUTING_PROFILE, routing, "example_routing_using_patterns", None, False),
+ (FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND, routing, "example_routing_using_specific_models", "gpt-5.4-nano", False),
+ (FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND, routing, "example_routing_using_specific_models", "claude-4-sonnet", False),
+ (FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND, routing, "example_routing_using_specific_models", "gemini-2.5-flash-lite", False),
+ (FormerReleaseFindingKind.ROUTE_TO_RETIRED_BACKEND, routing, "example_routing_using_specific_models", "grok-3", False),
+ }
+ assert not findings.is_clean
+ assert findings.blocks_boot
+
+ def test_pipelex_manifold_is_never_found_even_enabled_and_active(self, tmp_path: Path) -> None:
+ """Manifold is live: the v0.72 table enabled, its `model_specs_section` kept, and its profile active, is no finding.
+
+ A boot on it is the backend library's to refuse, for the key, with a message naming the change.
+ """
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ inference_dir = config_dir / INFERENCE_DIR_NAME
+ (inference_dir / BACKENDS_OVERRIDE_FILE_NAME).write_text(
+ "[pipelex_gateway]\nenabled = false\n\n[pipelex_manifold]\nenabled = true\n", encoding="utf-8"
+ )
+ (inference_dir / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('active = "all_pipelex_manifold"\n', encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ assert not any("manifold" in str(finding.subject) or finding.file_path.name == "pipelex_manifold.toml" for finding in findings.findings)
+ assert not findings.blocks_boot
+
+ def test_a_model_specs_section_key_is_found_on_a_backend_of_any_name(self, tmp_path: Path) -> None:
+ """The previous kit under its neutral name: the key is what is found, whatever the backend is called."""
+ config_dir = tmp_path / ".pipelex"
+ _copy_tree(source=PREVIOUS_RELEASE_KIT_DIR, destination=config_dir / INFERENCE_DIR_NAME)
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ model_specs_findings = [finding for finding in findings.findings if finding.kind == FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY]
+ assert [(finding.subject, finding.blocks_boot) for finding in model_specs_findings] == [("retired_preview", False)]
+ assert {finding.subject for finding in findings.findings if finding.blocks_boot} == {"pipelex_gateway", "all_pipelex_gateway"}
+
+ def test_an_enabled_backend_still_naming_model_specs_section_stops_the_boot(self, tmp_path: Path) -> None:
+ config_dir = tmp_path / ".pipelex"
+ inference_dir = config_dir / INFERENCE_DIR_NAME
+ inference_dir.mkdir(parents=True)
+ (inference_dir / BACKENDS_FILE_NAME).write_text('[acme]\nenabled = true\nmodel_specs_section = "acme_specs"\n', encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ assert [(finding.kind, finding.subject, finding.blocks_boot) for finding in findings.findings] == [
+ (FormerReleaseFindingKind.MODEL_SPECS_SECTION_KEY, "acme", True)
+ ]
+
+ def test_a_current_kit_and_a_missing_directory_are_clean(self, tmp_path: Path) -> None:
+ current = _copy_tree(source=Path(str(get_kit_configs_dir())), destination=tmp_path / "current")
+
+ assert detect_former_release(config_dir=current).is_clean
+ assert detect_former_release(config_dir=tmp_path / "missing").is_clean
+ assert not detect_former_release(config_dir=current).blocks_boot
+
+ def test_a_disabled_gateway_still_stops_the_boot_through_the_active_profile(self, tmp_path: Path) -> None:
+ """The machine the developer met: the gateway turned off, `active` left on its profile."""
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ backends_path = config_dir / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME
+ text = backends_path.read_text(encoding="utf-8")
+ assert text.count(GATEWAY_ENABLED_LINE) == 1, "the v0.72 kit no longer enables the gateway on the line this test turns off"
+ backends_path.write_text(text.replace(GATEWAY_ENABLED_LINE, "enabled = false"), encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ blocking = [(finding.kind, finding.subject) for finding in findings.findings if finding.blocks_boot]
+ assert blocking == [(FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE, "all_pipelex_gateway")]
+
+ def test_an_override_in_the_same_directory_is_read_with_its_base(self, tmp_path: Path) -> None:
+ """A personal override that turned the gateway off and picked another profile: nothing stops the boot."""
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ inference_dir = config_dir / INFERENCE_DIR_NAME
+ (inference_dir / BACKENDS_OVERRIDE_FILE_NAME).write_text("[pipelex_gateway]\nenabled = false\n", encoding="utf-8")
+ (inference_dir / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('active = "all_openai"\n', encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ assert not findings.blocks_boot
+ override_findings = [finding for finding in findings.findings if finding.file_path.name == BACKENDS_OVERRIDE_FILE_NAME]
+ assert [(finding.kind, finding.subject) for finding in override_findings] == [
+ (FormerReleaseFindingKind.RETIRED_BACKEND_TABLE, "pipelex_gateway")
+ ]
+
+ def test_an_override_naming_a_retired_profile_its_base_does_not_hold_is_found(self, tmp_path: Path) -> None:
+ config_dir = _copy_tree(source=Path(str(get_kit_configs_dir())), destination=tmp_path / ".pipelex")
+ (config_dir / INFERENCE_DIR_NAME / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('active = "all_pipelex_gateway"\n', encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ assert [(finding.kind, finding.file_path.name, finding.subject, finding.blocks_boot) for finding in findings.findings] == [
+ (FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE, ROUTING_PROFILES_OVERRIDE_FILE_NAME, "all_pipelex_gateway", True)
+ ]
+
+ def test_every_file_naming_a_retired_active_profile_is_found_and_only_the_one_the_boot_reads_blocks(self, tmp_path: Path) -> None:
+ """A base left on the Gateway's profile under an override that picked another: inert today, refused the day the override goes."""
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ (config_dir / INFERENCE_DIR_NAME / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('active = "all_openai"\n', encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ active_findings = [finding for finding in findings.findings if finding.kind == FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE]
+ assert [(finding.file_path.name, finding.subject, finding.blocks_boot) for finding in active_findings] == [
+ (ROUTING_PROFILES_FILE_NAME, "all_pipelex_gateway", False)
+ ]
+
+ def test_an_active_profile_routing_a_model_to_a_retired_backend_stops_the_boot_on_that_route(self, tmp_path: Path) -> None:
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ override_path = config_dir / INFERENCE_DIR_NAME / ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ override_path.write_text('active = "example_routing_using_specific_models"\n', encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ active_findings = [finding for finding in findings.findings if finding.kind == FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE]
+ assert [
+ (finding.file_path.name, finding.subject, finding.route_pattern, finding.retired_backend, finding.blocks_boot)
+ for finding in active_findings
+ ] == [
+ (ROUTING_PROFILES_FILE_NAME, "all_pipelex_gateway", None, "pipelex_gateway", False),
+ (ROUTING_PROFILES_OVERRIDE_FILE_NAME, "example_routing_using_specific_models", "gpt-5.4-nano", "pipelex_gateway", True),
+ ]
+ assert active_findings[1].description == (
+ f"'{override_path}' makes 'example_routing_using_specific_models' the active routing profile, "
+ "and it routes 'gpt-5.4-nano' to 'pipelex_gateway'"
+ )
+
+ def test_an_unreadable_document_is_skipped_rather_than_raised(self, tmp_path: Path) -> None:
+ """Whether the file parses is the boot's and the doctor's to say; the detector judges what it can read."""
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ (config_dir / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME).write_text("[pipelex_gateway\nbroken", encoding="utf-8")
+
+ findings = detect_former_release(config_dir=config_dir)
+
+ assert FormerReleaseFindingKind.RETIRED_BACKEND_TABLE not in {finding.kind for finding in findings.findings}
+ assert FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE in {finding.kind for finding in findings.findings}
+
+ def test_detecting_writes_nothing(self, tmp_path: Path) -> None:
+ config_dir = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / ".pipelex")
+ before = _snapshot(directory=config_dir)
+
+ detect_former_release(config_dir=config_dir)
+
+ assert _snapshot(directory=config_dir) == before
+
+ @pytest.mark.parametrize("use_override", [False, True])
+ def test_the_boot_blockers_are_read_from_the_files_the_boot_merges(self, tmp_path: Path, use_override: bool) -> None:
+ """Across directories, as the boot reads them: a project's override can lift what the home's base would stop."""
+ home_inference = _copy_tree(source=V0_72_CONFIG_DIR, destination=tmp_path / "home") / INFERENCE_DIR_NAME
+ project_inference = tmp_path / "project" / INFERENCE_DIR_NAME
+ project_inference.mkdir(parents=True)
+ if use_override:
+ (project_inference / BACKENDS_OVERRIDE_FILE_NAME).write_text("[pipelex_gateway]\nenabled = false\n", encoding="utf-8")
+ (project_inference / ROUTING_PROFILES_OVERRIDE_FILE_NAME).write_text('active = "all_openai"\n', encoding="utf-8")
+
+ blockers = former_release_boot_blockers(
+ backends_library_paths=[home_inference / BACKENDS_FILE_NAME, project_inference / BACKENDS_OVERRIDE_FILE_NAME],
+ routing_profile_library_paths=[home_inference / ROUTING_PROFILES_FILE_NAME, project_inference / ROUTING_PROFILES_OVERRIDE_FILE_NAME],
+ )
+
+ if use_override:
+ assert blockers == []
+ else:
+ assert [(finding.kind, finding.subject) for finding in blockers] == [
+ (FormerReleaseFindingKind.RETIRED_BACKEND_TABLE, "pipelex_gateway"),
+ (FormerReleaseFindingKind.ACTIVE_ROUTING_PROFILE, "all_pipelex_gateway"),
+ ]
+ assert all(finding.blocks_boot for finding in blockers)
diff --git a/tests/unit/pipelex/migration/test_former_release_invariants.py b/tests/unit/pipelex/migration/test_former_release_invariants.py
new file mode 100644
index 000000000..81b57884c
--- /dev/null
+++ b/tests/unit/pipelex/migration/test_former_release_invariants.py
@@ -0,0 +1,155 @@
+"""The former-release cleanup over random layouts of a home and a project, held to what it promises every boot.
+
+Each layout is a home `routing_profiles.toml`, and maybe a home override, a project base and a project override, each
+drawn from a small set of profile names, `default` backends and route shapes, with a former release's Gateway among
+them. The cleanup runs over the home and the project together, then each boot — the one outside the project, reading
+the home's files alone, and the project's — is judged as `ModelManager.setup` judges it: the former release's refusal
+first, then the routing profile library the boot loads.
+
+What must hold for every layout:
+
+- the cleanup never raises;
+- a boot that started before the cleanup still starts after it, on the same active profile, and if that profile sent
+ models somewhere, it still does;
+- a boot that only the former release stopped — one that would start with the Gateway still enabled — starts
+ after the cleanup, with an active profile that sends models somewhere; or the cleanup left a file both boots read
+ for a hand edit, because no change to it suits both.
+
+The seed is fixed, so a failure names a layout that can be replayed. The profile the kit makes active is left out of
+the names drawn: a user's own profile under that name, sent to the Gateway, is a known gap of the cleanup tracked on
+its own.
+"""
+
+import random
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from pipelex.cogt.exceptions import RoutingProfileDisabledBackendError, RoutingProfileLibraryError, RoutingProfileLibraryNotFoundError
+from pipelex.cogt.model_routing.routing_profile_loader import load_active_routing_profile
+from pipelex.migration.former_release import former_release_boot_blockers, inference_merge_sequences
+from pipelex.migration.former_release_cleanup import clean_former_release
+from pipelex.migration.plan import FileBlockedReason
+from pipelex.system.configuration.config_loader import (
+ BACKENDS_FILE_NAME,
+ BACKENDS_OVERRIDE_FILE_NAME,
+ INFERENCE_DIR_NAME,
+ ROUTING_PROFILES_FILE_NAME,
+ ROUTING_PROFILES_OVERRIDE_FILE_NAME,
+)
+
+SEED = 20261008
+LAYOUT_COUNT = 400
+
+PROFILE_NAMES = ("all_pipelex_gateway", "custom", "mine", "team")
+DEFAULTS = ("openai", "pipelex_gateway", None)
+ROUTE_BACKENDS = ("openai", "pipelex_gateway")
+ROUTE_PATTERNS = ("gpt-*", "*")
+ENABLED_BACKENDS = ["openai", "anthropic"]
+WITH_THE_GATEWAY = [*ENABLED_BACKENDS, "pipelex_gateway"]
+BACKENDS_TEXT = '[openai]\nenabled = true\napi_key = "${OPENAI_API_KEY}"\n\n[anthropic]\nenabled = true\napi_key = "${ANTHROPIC_API_KEY}"\n'
+
+# How each file of a layout is drawn: whether it is a base, and how likely it is to exist.
+FILES = (
+ ("home", ROUTING_PROFILES_FILE_NAME, True, 1.0),
+ ("home", ROUTING_PROFILES_OVERRIDE_FILE_NAME, False, 0.5),
+ ("project", ROUTING_PROFILES_FILE_NAME, True, 0.4),
+ ("project", ROUTING_PROFILES_OVERRIDE_FILE_NAME, False, 0.6),
+)
+
+
+def _routing_text(*, rng: random.Random, is_base: bool) -> str:
+ """One routing profiles document: an `active`, always in a base and sometimes in an override, and a few profiles."""
+ lines: list[str] = []
+ active = rng.choice(PROFILE_NAMES) if is_base else rng.choice([None, None, *PROFILE_NAMES])
+ if active is not None:
+ lines.append(f'active = "{active}"')
+ for name in PROFILE_NAMES:
+ if rng.random() < (0.45 if is_base else 0.7):
+ continue
+ shape = "full" if is_base else rng.choice(["full", "default_only", "routes_only"])
+ profile: dict[str, Any] = {}
+ if shape == "full":
+ profile["description"] = f"{name} description"
+ if (default := rng.choice(DEFAULTS)) is not None:
+ profile["default"] = default
+ if rng.random() < 0.4:
+ profile["routes"] = {rng.choice(ROUTE_PATTERNS): rng.choice(ROUTE_BACKENDS)}
+ elif shape == "default_only":
+ profile["default"] = rng.choice(ROUTE_BACKENDS)
+ else:
+ profile["routes"] = {rng.choice(ROUTE_PATTERNS): rng.choice(ROUTE_BACKENDS)}
+ lines += ["", f"[profiles.{name}]"]
+ lines += [f'{key} = "{profile[key]}"' for key in ("description", "default") if key in profile]
+ if "routes" in profile:
+ lines += ["", f"[profiles.{name}.routes]"]
+ lines += [f'"{pattern}" = "{backend}"' for pattern, backend in profile["routes"].items()]
+ return "\n".join(lines) + "\n"
+
+
+def _write_layout(*, rng: random.Random, home: Path, project: Path) -> dict[str, str]:
+ (home / INFERENCE_DIR_NAME).mkdir(parents=True)
+ (project / INFERENCE_DIR_NAME).mkdir(parents=True)
+ (home / INFERENCE_DIR_NAME / BACKENDS_FILE_NAME).write_text(BACKENDS_TEXT, encoding="utf-8")
+ layout: dict[str, str] = {}
+ for directory, file_name, is_base, likelihood in FILES:
+ if rng.random() >= likelihood:
+ continue
+ text = _routing_text(rng=rng, is_base=is_base)
+ ((home if directory == "home" else project) / INFERENCE_DIR_NAME / file_name).write_text(text, encoding="utf-8")
+ layout[f"{directory}/{file_name}"] = text
+ return layout
+
+
+def _boots(*, home: Path, project: Path, retired_backends_enabled: bool) -> list[tuple[str, str | None, bool]]:
+ """Each boot of the machine as `(verdict, active profile, routes something)`: `starts`, `refused` or `fails`."""
+ config_dirs = [home, project]
+ routing_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=ROUTING_PROFILES_FILE_NAME, override_file_name=ROUTING_PROFILES_OVERRIDE_FILE_NAME
+ )
+ backends_sequences = inference_merge_sequences(
+ config_dirs=config_dirs, file_name=BACKENDS_FILE_NAME, override_file_name=BACKENDS_OVERRIDE_FILE_NAME
+ )
+ boots: list[tuple[str, str | None, bool]] = []
+ for routing_paths, backends_paths in zip(routing_sequences, backends_sequences, strict=True):
+ if not retired_backends_enabled and former_release_boot_blockers(
+ backends_library_paths=backends_paths, routing_profile_library_paths=routing_paths
+ ):
+ boots.append(("refused", None, False))
+ continue
+ try:
+ profile = load_active_routing_profile(
+ routing_profile_library_paths=routing_paths,
+ enabled_backends=WITH_THE_GATEWAY if retired_backends_enabled else ENABLED_BACKENDS,
+ )
+ except (RoutingProfileLibraryError, RoutingProfileLibraryNotFoundError, RoutingProfileDisabledBackendError):
+ boots.append(("fails", None, False))
+ continue
+ boots.append(("starts", profile.name, bool(profile.default or profile.routes or profile.fallback_order)))
+ return boots
+
+
+class TestFormerReleaseCleanupInvariants:
+ @pytest.mark.parametrize("case", range(LAYOUT_COUNT))
+ def test_every_boot_starts_as_it_did_or_better(self, tmp_path: Path, case: int) -> None:
+ rng = random.Random(SEED * 10_000 + case)
+ home = tmp_path / "home"
+ project = tmp_path / "project" / ".pipelex"
+ layout = _write_layout(rng=rng, home=home, project=project)
+ with_the_gateway = _boots(home=home, project=project, retired_backends_enabled=True)
+ before = _boots(home=home, project=project, retired_backends_enabled=False)
+
+ cleanup = clean_former_release(config_dirs=[home, project], dry_run=False)
+
+ after = _boots(home=home, project=project, retired_backends_enabled=False)
+ left_for_a_hand_edit = [file for file in cleanup.files if file.blocked_reason is FileBlockedReason.NEEDS_A_HAND_EDIT]
+ for boot_name, gateway_boot, boot_before, boot_after in zip(("home", "project"), with_the_gateway, before, after, strict=True):
+ context = f"case {case}, {boot_name} boot: before {boot_before}, after {boot_after}, layout {layout}"
+ if boot_before[0] == "starts":
+ assert boot_after[0] == "starts", f"a boot that started no longer does — {context}"
+ assert boot_after[1] == boot_before[1], f"a boot that started switched profile — {context}"
+ assert boot_after[2] or not boot_before[2], f"a boot's profile no longer routes anything — {context}"
+ elif gateway_boot[0] == "starts" and gateway_boot[2]:
+ whole = boot_after[0] == "starts" and boot_after[2]
+ assert whole or left_for_a_hand_edit, f"a boot the former release stopped is left without a profile to run on, unreported — {context}"
diff --git a/tests/unit/pipelex/migration/test_runner.py b/tests/unit/pipelex/migration/test_runner.py
index 23d465255..0484e5629 100644
--- a/tests/unit/pipelex/migration/test_runner.py
+++ b/tests/unit/pipelex/migration/test_runner.py
@@ -14,10 +14,10 @@ from pathlib import Path
from pytest_mock import MockerFixture
from pipelex.fix_ops.exceptions import FixTransactionError, FixWriteConflictError
-from pipelex.fix_ops.file_transaction import FileSnapshot, PendingFileUpdate, commit_file_updates
+from pipelex.fix_ops.file_transaction import FileSnapshot, PendingFileUpdate, assert_snapshot_unchanged, commit_file_updates, read_file_snapshot
from pipelex.migration.backup import WrittenBackup, existing_backups_of, write_backup
from pipelex.migration.plan import FileBlockedReason
-from pipelex.migration.runner import migrate_directories, migrate_file
+from pipelex.migration.runner import FileWriteOutcome, migrate_directories, migrate_file, remove_file_with_backup
from pipelex.migration.surfaces import SurfaceRegistry
from pipelex.suggested_fix import RenameTableKeyOp
from tests.unit.pipelex.migration.conftest import EXAMPLE_SURFACE_ID, EntryBuilder, LedgerBuilder, SurfaceBuilder
@@ -759,6 +759,58 @@ class TestMigrationRunner:
assert not stale_backup.exists()
assert rescued.read_text(encoding="utf-8") == "the original, from a run that could not vouch for its write\n"
+ def test_two_removals_of_one_file_in_the_same_second_keep_the_only_copy(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """The removal's twin of the same-second collision: the run that loses keeps the copy the winner adopted.
+
+ Run A backs the file up; run B, in the same second, finds that backup's name taken, adopts it, and removes the
+ file. Run A then finds the file gone and gives up. Its copy is the one run B's report names, and the only copy
+ of the file anywhere: deleting it as an unneeded backup of a removal that did not happen would lose the file.
+ """
+ target = tmp_path / "pipelex_service.toml"
+ target.write_text("[terms]\naccepted = true\n", encoding="utf-8")
+ snapshot_of_run_a = read_file_snapshot(target)
+ snapshot_of_run_b = read_file_snapshot(target)
+ real_check = assert_snapshot_unchanged
+ checks: list[FileSnapshot] = []
+ outcomes_of_run_b: list[FileWriteOutcome] = []
+
+ def _run_b_runs_whole_first(snapshot: FileSnapshot) -> None:
+ checks.append(snapshot)
+ if len(checks) == 1:
+ outcomes_of_run_b.append(remove_file_with_backup(snapshot=snapshot_of_run_b, moment=MOMENT))
+ real_check(snapshot)
+
+ mocker.patch("pipelex.migration.runner.assert_snapshot_unchanged", side_effect=_run_b_runs_whole_first)
+
+ outcome_of_run_a = remove_file_with_backup(snapshot=snapshot_of_run_a, moment=MOMENT)
+
+ assert outcomes_of_run_b[0].was_written
+ assert outcome_of_run_a.blocked_reason is FileBlockedReason.CHANGED_DURING_RUN
+ assert not target.exists()
+ backups = existing_backups_of(path=target)
+ assert [backup.read_text(encoding="utf-8") for backup in backups] == ["[terms]\naccepted = true\n"]
+ assert outcome_of_run_a.backup_path == backups[0]
+
+ def test_a_removal_refused_over_a_users_edit_takes_its_copy_back(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ """A file still there with other content is the user's edit: nothing was removed, and no copy is left beside it."""
+ target = tmp_path / "pipelex_service.toml"
+ target.write_text("[terms]\naccepted = true\n", encoding="utf-8")
+ snapshot = read_file_snapshot(target)
+ real_check = assert_snapshot_unchanged
+
+ def _the_user_edits_it_first(checked: FileSnapshot) -> None:
+ target.write_text("[terms]\naccepted = false\n", encoding="utf-8")
+ real_check(checked)
+
+ mocker.patch("pipelex.migration.runner.assert_snapshot_unchanged", side_effect=_the_user_edits_it_first)
+
+ outcome = remove_file_with_backup(snapshot=snapshot, moment=MOMENT)
+
+ assert outcome.blocked_reason is FileBlockedReason.CHANGED_DURING_RUN
+ assert outcome.backup_path is None
+ assert target.read_text(encoding="utf-8") == "[terms]\naccepted = false\n"
+ assert not existing_backups_of(path=target)
+
def test_a_missing_configuration_directory_is_skipped_rather_than_refused(
self,
tmp_path: Path,