Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Pipelex-usage-stats · L-261011-9237fa (#1418)
Louis Choquele7cee8b93e12
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.tomlindex f71ff9d56..8b6ea5482 100644--- a/.drift/acks/cli-docs.toml+++ b/.drift/acks/cli-docs.toml@@ -1,29 +1,29 @@contract = "cli-docs"-digest = "sha256:28d7d45c73daa1f5e9f217d43878037d98508effdb35ba278d1cbd66d2f1c5b0"-reviewed_by = "Claude (Opus 5.5)"-reviewed_at = "2026-10-10T11:26:22Z"-rationale = "clean-pass: the triggers change log lines only. The agent run core's cost-summary warning and graph render's refused-spec error are reworded to one sentence, same level and fields, and the IDE-extension DEBUG line of both init modules carries the EOFError as error fields rather than in its text; no option, output, prompt or exit code moved. Read docs/tools/cli/ and pipelex/cli/agent_cli/CLAUDE.md for the old and new messages, the cost summary, a refused graph spec and the IDE extension suggestion: no page quotes any of these log lines. The agent CLAUDE.md claim that cost_report is absent when cost-summary aggregation fails, caught and skipped, still matches run_pipeline_core, which logs the warning and leaves the report out. Nothing was stale."+digest = "sha256:0d0482f20caaffbade6fb43ccf4f5c8aa3684c40f99b961699cea94bc3a8f30c"+reviewed_by = "Louis Choquel"+reviewed_at = "2026-10-11T19:29:59Z"+rationale = "clean-pass: pipelex init now shows the usage statistics notice on stderr whenever they are on, and docs/tools/cli/init.md says so in this same change; pipelex-agent is untouched, so the agent CLI guide stands."[trigger_files]"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"-"pipelex/cli/_cli.py" = "blob:2f95b05d84078bf40a59dce030dd9fecf8e500bc"+"pipelex/cli/_cli.py" = "blob:9eca8e06a004933e5cfa2ba871e4fb1ff6c6bf0c""pipelex/cli/agent_cli/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"-"pipelex/cli/agent_cli/_agent_cli.py" = "blob:690be5b3b3f0fbe69ed3e7baddd24538f590de01"+"pipelex/cli/agent_cli/_agent_cli.py" = "blob:5ab17e334cdfd4b5f8e4afdc932f3fca3747ed59""pipelex/cli/agent_cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/agent_cli_factory.py" = "blob:050ca1344abc4c1ca22d431923e1577681552fe6"-"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:53812c4c3c6664255f9df52c69bf5948258330c8"+"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:10588465f86eb7ce7d0bed3985304612b154d013""pipelex/cli/agent_cli/commands/bundle_path_resolver.py" = "blob:a1904517c731afb72af7d49adf6d217dc50b2660""pipelex/cli/agent_cli/commands/check_model_cmd.py" = "blob:3a9c93daecadf266b3eba466904b3d005d448d40""pipelex/cli/agent_cli/commands/codegen/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/codegen/app.py" = "blob:56130b89fe9efec5ad3d014ed6bc16bef1bd1645""pipelex/cli/agent_cli/commands/codegen/check_cmd.py" = "blob:9e8beb084a9f04f39e8acca9e6ae2554d40cb667"-"pipelex/cli/agent_cli/commands/codegen/types_cmd.py" = "blob:5da4131cb459383ab4007c2cdb15f3cfde74510f"+"pipelex/cli/agent_cli/commands/codegen/types_cmd.py" = "blob:5c78bd70803a8f32616ccba9c57538ed49ab2d60""pipelex/cli/agent_cli/commands/doctor_cmd.py" = "blob:948be63cf4fab165846c83b4bd2517bd4a46be5e""pipelex/cli/agent_cli/commands/fix/__init__.py" = "blob:65662c86f17f610ad55ecfc3e27a022b9aa26e8b""pipelex/cli/agent_cli/commands/fix/app.py" = "blob:e6190d9bde6a05755c04bd21806f699461038ac0""pipelex/cli/agent_cli/commands/fix/bundle_cmd.py" = "blob:cfd9e645986636c44c02a6a04a0fc67ed60d444b""pipelex/cli/agent_cli/commands/fmt_cmd.py" = "blob:97d6fb6c328e0b718b1293bd6b73cba03edcb7dc"-"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:2151fa938858604e7cd9dba9462e030e95c137c0"+"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:513b1160d8e3815b34b905bbabccd59d6c379842""pipelex/cli/agent_cli/commands/inputs/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/inputs/_inputs_core.py" = "blob:4161029643f3a302de54fa637d2676521e10d873""pipelex/cli/agent_cli/commands/inputs/app.py" = "blob:7e19284687b334c684fb03381a8d9d20af1aa152"@@ -33,10 +33,10 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/agent_cli/commands/lint_cmd.py" = "blob:91ad9cb4d618ba4c0b7da366cde2f74f38d350df""pipelex/cli/agent_cli/commands/migrate_cmd.py" = "blob:dbe1f689607ccea0b1f5f1dadca20f19f65ce400""pipelex/cli/agent_cli/commands/models_cmd.py" = "blob:f48a921fa89d8275c585b3e15f135f5585ac01aa"-"pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:a1c530d110482485ce462f199b9e79fac29f8896"+"pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:289adc65f49a185ca9f81bb73f87b4c68facb62b""pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/run/_output_helpers.py" = "blob:40ed1700e0d5e86d7696eadd7f9887baeb8059f6"-"pipelex/cli/agent_cli/commands/run/_run_core.py" = "blob:4f11b537de3be6c4cbab7e53309bdda44b01d390"+"pipelex/cli/agent_cli/commands/run/_run_core.py" = "blob:6049690867e53e8efcf92d0e44bd4311dfe9f7f8""pipelex/cli/agent_cli/commands/run/_run_hosted.py" = "blob:cea011e61feb082a7b9d82d7693b4fd3246737e5""pipelex/cli/agent_cli/commands/run/app.py" = "blob:7546d9b31ea2dc22b1ddae538edfbc8e7f7d08df""pipelex/cli/agent_cli/commands/run/bundle_cmd.py" = "blob:86988b3506ebc057df6a55d8862105ba6ceb693b"@@ -44,7 +44,7 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/agent_cli/commands/run/pipe_cmd.py" = "blob:f421c75ee696c18f85a7f63805455ef23637302f""pipelex/cli/agent_cli/commands/run/stdin_resolver.py" = "blob:c7c7d7495508f1c4e5a46eba336a4ab1694ebcf7""pipelex/cli/agent_cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"-"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:2a04aeff94db4afc56ac0c033a9d4add54d85c88"+"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:a0adc62061077bd47725147becc34914c4a9c7d4""pipelex/cli/agent_cli/commands/validate/app.py" = "blob:d2d0ed19288c2d1c25be1650dd962189fcdb3b86""pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:6b02bb77dadf0ecd1814176580920962fcda2a42""pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:0f4cf76d99b39422610c5daf6b5a0500de234818"@@ -66,13 +66,13 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/commands/build/output/bundle_cmd.py" = "blob:7cafc55438704ecc6e86d1182069e4b1f73f9794""pipelex/cli/commands/build/output/method_cmd.py" = "blob:242434d518aff0b9706b910e7f937b53a601fde1""pipelex/cli/commands/build/output/pipe_cmd.py" = "blob:eb0db8279c93fcff9d5fe4546f593d753282acb5"-"pipelex/cli/commands/build/structures_cmd.py" = "blob:d13ba3b6fc1433f787e1aa97c6c04a67483ec7ed"+"pipelex/cli/commands/build/structures_cmd.py" = "blob:177c190e1679e26f7ab20d9ff3f6199aaf2cfe19""pipelex/cli/commands/bundle_path_resolver.py" = "blob:c98794b23efde0f279ac3c113fd5f5175f2db479""pipelex/cli/commands/codegen/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/codegen/app.py" = "blob:9158084a89da23654e122e9df3a0a9c726401642""pipelex/cli/commands/codegen/check_cmd.py" = "blob:329f410bca90eb022fad2abc729e8b5203138c02"-"pipelex/cli/commands/codegen/inputs_cmd.py" = "blob:28b22cfbca717d3f5ee75162b68e132439655deb"-"pipelex/cli/commands/codegen/types_cmd.py" = "blob:27d7a06d571ad832b449d21b6f9eb66c017fb197"+"pipelex/cli/commands/codegen/inputs_cmd.py" = "blob:7e1db7c76857e77c1af2c68d0c8a14d52746fc73"+"pipelex/cli/commands/codegen/types_cmd.py" = "blob:23ad05431006dfe74ad7840b916eab74d769cebe""pipelex/cli/commands/crate_loading.py" = "blob:3d7d651931ecf1a15a3e2d72411b3212b55d74ef""pipelex/cli/commands/doctor_cmd.py" = "blob:a991d210787c0df68d9cc6bac29361df89e8e9c7""pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"@@ -83,10 +83,11 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/commands/graph_cmd.py" = "blob:d215ad2b66c4421364c90f2d8f875b6312dadfe4""pipelex/cli/commands/init/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/init/backends.py" = "blob:0cc3543c78f832149ebbbf3cc8beafee619e4f63"-"pipelex/cli/commands/init/command.py" = "blob:7282c6937f2e5a29a468685e3193df475b09ef0e"+"pipelex/cli/commands/init/command.py" = "blob:4747c1b9dff00163ca5cd48595ea8545a3b02b4b""pipelex/cli/commands/init/config_files.py" = "blob:8345ae46e7f77a6f49f526ed20c8dd8263f118f3""pipelex/cli/commands/init/credentials.py" = "blob:1817fd9b1988e4fea528a3977259b57f7274b69b""pipelex/cli/commands/init/ide_extension.py" = "blob:9f0ec1494c07062246ecc906bc55b7f2cf95c715"+"pipelex/cli/commands/init/init_usage_stats.py" = "blob:fca415ba8fef02967cef142f0011a016990274e2""pipelex/cli/commands/init/routing.py" = "blob:c73927deb99804556cb545cdaa6eabf8d1fc3450""pipelex/cli/commands/init/setup_path.py" = "blob:6485fae3ee6dd1b112badffdc956f1b734dc1b10""pipelex/cli/commands/init/telemetry.py" = "blob:07a33b6540e1fc4146d0777463c93abd4ea70231"@@ -108,7 +109,7 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/commands/run/_inputs_file_loader.py" = "blob:957e6eb00531d168b3b1e179a517717d069cb7fc""pipelex/cli/commands/run/_inputs_path_resolver.py" = "blob:c8ec89d98718e18af7aaaded44c9f545be60adab""pipelex/cli/commands/run/_main_stuff_file.py" = "blob:ca806e55085aa2e5748492751521296ce1f42960"-"pipelex/cli/commands/run/_run_core.py" = "blob:54d326f5bc5fa62dfef11d96b36b9f8d2b352c22"+"pipelex/cli/commands/run/_run_core.py" = "blob:f812f4b766a2ce3d348aa031783ebcd37892fd0e""pipelex/cli/commands/run/_run_hosted.py" = "blob:dd5801877a636f71187e9b23c2fab18d4d25ba32""pipelex/cli/commands/run/app.py" = "blob:e312b159625e0a72bc9a18d3bfd2c73e7ac59f24""pipelex/cli/commands/run/bundle_cmd.py" = "blob:aba274dc8d6d0ac08baa3f21afbc06b2fdbec356"@@ -117,9 +118,10 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/commands/run/pipe_cmd.py" = "blob:f58885186d59142146eed340451b21832526af47""pipelex/cli/commands/show_cmd.py" = "blob:4c87c70058ea3b15827d03a370f3345569a2b705""pipelex/cli/commands/update_cmd.py" = "blob:b2db6170ddd9c67f17d88096ba1f1796a897557d"+"pipelex/cli/commands/usage_stats_cmd.py" = "blob:b3723c64f911d1b69e1e1aa126610a132a16e3da""pipelex/cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"-"pipelex/cli/commands/validate/_validate_core.py" = "blob:27c343323180c73280f4ad5f29336811d06cf635"-"pipelex/cli/commands/validate/app.py" = "blob:309061423af13f2f6c496a024306ad696af567ab"+"pipelex/cli/commands/validate/_validate_core.py" = "blob:fbdc9ab24e434b57d57959a8fdfad1f0ea39a20d"+"pipelex/cli/commands/validate/app.py" = "blob:33b6c2d374e456b2a47a27861f99c581945b21af""pipelex/cli/commands/validate/bundle_cmd.py" = "blob:1fbf3e0cf695db39101c83160a724df6468988e9""pipelex/cli/commands/validate/method_cmd.py" = "blob:f12ee233fc985d04a4512f1fef6b9ea02a1b9644""pipelex/cli/commands/validate/pipe_cmd.py" = "blob:791b9a693f4ec52f727b42dab4b102af46a4ee22"@@ -128,6 +130,7 @@ rationale = "clean-pass: the triggers change log lines only. The agent run core'"pipelex/cli/error_handlers.py" = "blob:b4ddd1a0a40d75f948779e86a52739d793e4cff6""pipelex/cli/exceptions.py" = "blob:1d51c3a53ddd3f6cf8de566fa2ade3fa75ab7a3a""pipelex/cli/installed_methods.py" = "blob:f7818b7857a8350389cf196e32630e7b3c51818c"-"pipelex/cli/method_resolver.py" = "blob:afaf77b2d8b6a3467d6ab1f355b1ee78a71eafa7"+"pipelex/cli/method_resolver.py" = "blob:34612bb21e9aae60cd5244abfa111d554443533a""pipelex/cli/readiness.py" = "blob:6d185324ed0784663e1dc78d1194a28312c54798"+"pipelex/cli/usage_stats_cli.py" = "blob:d04e2191e6f6a18027558a3e063f945be7b77679""pipelex/cli/version_report.py" = "blob:f4a2311bb629bd77196ea2f6b031c166db0d362a"diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.tomlindex 395da7216..21314f84d 100644--- a/.drift/acks/config-docs.toml+++ b/.drift/acks/config-docs.toml@@ -1,8 +1,8 @@contract = "config-docs"-digest = "sha256:51d1fd8be2b31fb9319489713a54973482d907d85bcdb7a0fbb4d978134a499b"-reviewed_by = "Claude (Opus 5.5)"-reviewed_at = "2026-10-10T16:14:48Z"-rationale = "real-catch: RichLogConfig gained package_prefix (PackagePrefix libraries|all|none, strict=False) with its default libraries and a prose comment in pipelex/pipelex.toml. Grepping docs/configuration for rich_log found logging-config.md the only page naming the console settings, both in its Display Options excerpt and its full example; both now carry package_prefix, with the three values and the root-logger exception documented under Display Options, and the section intro's claim that the fields and layouts have no setting still holds. configs.py and config_cogt.py are untouched since RichLogConfig lives in log_config.py. The key is additive, so the defaults layer supplies it and no ledger entry is owed: make umig regenerated the pipelex-config@8 golden and check-migration-schemas passes on all three surfaces."+digest = "sha256:66bf51a2e90466528e0059c0722fccebd1accb16236e9b81af538eb24a1b00ef"+reviewed_by = "Louis Choquel"+reviewed_at = "2026-10-11T20:09:33Z"+rationale = "clean-pass: ConfigLoader.find_project_root now stats each marker instead of asking Path.exists(), which from Python 3.14 swallows every OSError; the markers, the walk and the home stop are unchanged, and a directory that cannot be traversed raises as it already did on 3.13. No config model, default or documented behaviour changed, so docs/configuration and the migration ledgers stand."[trigger_files]"pipelex/cogt/config_cogt.py" = "blob:293ef22ae123266e6a1886d4ffb07f455e5be9e3"@@ -10,14 +10,14 @@ rationale = "real-catch: RichLogConfig gained package_prefix (PackagePrefix libr"pipelex/graph/graph_config.py" = "blob:dc0939a6d2730c0a10b88aa1a6e0b2dd583462a6""pipelex/language/mthds_config.py" = "blob:353009ad8c59dc5266bdd9dfa42b1ad18b67dca7""pipelex/methods/methods_config.py" = "blob:76ca9a0e64f8c10ef6a355aafd0d79d36b09f06a"-"pipelex/pipelex.toml" = "blob:0de0caa573498250eb7a7331fa4969cafcbb7946"+"pipelex/pipelex.toml" = "blob:95eed7d7186c0a6b25349893c002c16fc82b4d4d""pipelex/system/configuration/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc""pipelex/system/configuration/config_check.py" = "blob:2d5de29aba3d2ce205d3d82b4623f37b53108332"-"pipelex/system/configuration/config_loader.py" = "blob:2850f6b39df6ad2afafefc4d431672cb6ee91ced"+"pipelex/system/configuration/config_loader.py" = "blob:8703fc3e5a82d7c6a3d148ea195c25044c2f0fe0""pipelex/system/configuration/config_model.py" = "blob:b7d141d559c0302f0d8c56859ae0a8edeb9bfdf0""pipelex/system/configuration/config_root.py" = "blob:c7d43851270912d17413cfc6a511247d3c7cc160""pipelex/system/configuration/config_surface.py" = "blob:49424a8103a5611f5b2885135dd699273bde57c8"-"pipelex/system/configuration/configs.py" = "blob:ff56c3d33aa358c6f9d8036d47a83924d473904f"+"pipelex/system/configuration/configs.py" = "blob:4a95dd4fe7d17ad5b32c1b61063dbba43b18e40b""pipelex/system/configuration/pipe_func_config.py" = "blob:dc8890e914fbcdb611147e42eaa0cd87b32288a5""pipelex/system/telemetry/telemetry_config.py" = "blob:068a785d97a2d52ee43e82e723cd944b8b6601bc""pipelex/tools/aws/aws_config.py" = "blob:0a7c9dbf276e61355b1fa79b9db8caf252450a39"diff --git a/.github/workflows/check-test-count-badge.yml b/.github/workflows/check-test-count-badge.ymlindex 8657f9cc6..96c76ed24 100644--- a/.github/workflows/check-test-count-badge.yml+++ b/.github/workflows/check-test-count-badge.yml@@ -5,6 +5,10 @@ on:branches:- main+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:check:runs-on: ubuntu-latestdiff --git a/.github/workflows/dependency-canary.yml b/.github/workflows/dependency-canary.ymlindex 868ad372e..03ff961e0 100644--- a/.github/workflows/dependency-canary.yml+++ b/.github/workflows/dependency-canary.yml@@ -15,6 +15,10 @@ concurrency:group: ${{ github.workflow }}cancel-in-progress: true+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:# 1. Resolve every dependency to the latest version still allowed by our# declared ranges, then run the suite AND real CLI commands. Catchesdiff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.ymlindex d85eb7b20..dad1cc587 100644--- a/.github/workflows/dependency-review.yml+++ b/.github/workflows/dependency-review.yml@@ -12,6 +12,10 @@ permissions:contents: readpull-requests: write+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:dependency-review:name: Review new dependenciesdiff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.ymlindex 260a0ff56..7e76d78fd 100644--- a/.github/workflows/deploy-docs.yml+++ b/.github/workflows/deploy-docs.yml@@ -10,6 +10,10 @@ permissions:contents: writepages: write+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:deploy:runs-on: ubuntu-latestdiff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.ymlindex c73711812..ede891327 100644--- a/.github/workflows/lint-check.yml+++ b/.github/workflows/lint-check.yml@@ -7,6 +7,10 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref }}cancel-in-progress: true+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:# --------------------------------------------------------------------------# 1. Ruff + plxt — single job (target-version = min supported, in pyproject.toml)diff --git a/.github/workflows/lint-fresh-check.yml b/.github/workflows/lint-fresh-check.ymlindex 24f5e2567..e8e40140d 100644--- a/.github/workflows/lint-fresh-check.yml+++ b/.github/workflows/lint-fresh-check.yml@@ -21,6 +21,10 @@ concurrency:group: ${{ github.workflow }}-${{ github.ref }}cancel-in-progress: true+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:lint-fresh:name: Lint fresh (py${{ matrix.python-version }})diff --git a/.github/workflows/manual-trigger-tests-check.yml b/.github/workflows/manual-trigger-tests-check.ymlindex bbcea7ce4..596b0b82a 100644--- a/.github/workflows/manual-trigger-tests-check.yml+++ b/.github/workflows/manual-trigger-tests-check.yml@@ -9,6 +9,10 @@ on:# - "release/v[0-9]+.[0-9]+.[0-9]+"# workflow_dispatch:+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:manual-trigger-tests:runs-on: ubuntu-latestdiff --git a/.github/workflows/mthds-standard-check.yml b/.github/workflows/mthds-standard-check.ymlindex 986a7ced7..f1f155410 100644--- a/.github/workflows/mthds-standard-check.yml+++ b/.github/workflows/mthds-standard-check.yml@@ -22,6 +22,10 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}cancel-in-progress: true+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:natives-vs-standard:name: "Pinned natives vs the standard's page (a red with no pinned-set change in this PR means the standard moved)"diff --git a/.github/workflows/publish-docker-hub.yml b/.github/workflows/publish-docker-hub.ymlindex 8cd5225be..1b94183e5 100644--- a/.github/workflows/publish-docker-hub.yml+++ b/.github/workflows/publish-docker-hub.yml@@ -37,6 +37,10 @@ on:type: stringrequired: true+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:publish:name: Build, smoke-test, push and confirm the imagediff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.ymlindex e0dd4428f..6f262f430 100644--- a/.github/workflows/publish-pypi.yml+++ b/.github/workflows/publish-pypi.yml@@ -31,6 +31,10 @@ on:# below recovers the bytes this version published rather than rebuilding them.workflow_dispatch:+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:build:name: Build distribution 📦diff --git a/.github/workflows/tests-check.yml b/.github/workflows/tests-check.ymlindex 563080ba2..8ee4061fa 100644--- a/.github/workflows/tests-check.yml+++ b/.github/workflows/tests-check.yml@@ -7,6 +7,10 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref }}cancel-in-progress: true+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:# --------------------------------------------------------------------------# 1. Tests — floor Python version, sharded across free GitHub-hosted runners.diff --git a/.github/workflows/tests-full-check.yml b/.github/workflows/tests-full-check.ymlindex be3d45be4..bc9b61a19 100644--- a/.github/workflows/tests-full-check.yml+++ b/.github/workflows/tests-full-check.yml@@ -24,6 +24,10 @@ concurrency:group: ${{ github.workflow }}-${{ github.ref }}cancel-in-progress: false+# No pipelex process a job starts reports anonymous usage statistics (docs/setup/usage-statistics.md).+env:+ DO_NOT_TRACK: "1"+jobs:matrix-test-full:name: Tests full (py${{ matrix.python-version }}, shard ${{ matrix.group }}/8)diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 809c40c3b..684880d9a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,17 @@## [Unreleased]+### Added++- **Anonymous usage statistics**: pipelex now reports anonymous counts of its use to Pipelex, on by default: a `command_completed` per command of the `pipelex` and `pipelex-agent` CLIs, a `run_completed` per run however it ended, with a `model_used` per model when the run's usage events were read, a `method_validated` per validation and a `code_generated` per code generation, whatever started them. Every property is a count, a boolean or a value from a closed list: no method, pipe code, concept name, prompt, input, output, file name, error message or argument is ever sent, and a model outside the backend tables pipelex ships is sent as `other`. The first command with statistics on prints a notice on stderr, once per machine, except where stderr is a machine contract (`pipelex-agent` with JSON errors), and `pipelex init` shows the same text on stderr whenever statistics are on, once per run. Statistics stay off under `DO_NOT_TRACK`, `PIPELEX_USAGE_STATS=0`, `pipelex usage-stats disable`, the `mthds` CLI's former `DISABLE_TELEMETRY` opt-out, `[usage_stats] enabled = false` in a project's `.pipelex/pipelex.toml` or in the packaged configuration, a `~/.mthds/usage-stats/` pipelex cannot look into, and the `pytest` integration mode; the [usage statistics page](https://docs.pipelex.com/latest/setup/usage-statistics/) lists what is collected. A dedicated client sends them to PostHog's EU region, from a detached sender after a CLI command and from a daemon thread in a Python program or the API server; `telemetry.toml` and the telemetry you configure are untouched.+- **`pipelex usage-stats`**: `status` (with `--format json`), `disable`, `enable` and `reset-id` show and change the statistics of every Pipelex tool on the machine, through the state they share in `~/.mthds/usage-stats/`. `PIPELEX_USAGE_STATS_DEBUG` prints each event to stderr as a JSON line instead of sending it, and `PIPELEX_USAGE_STATS_ENDPOINT` sends to a loopback origin for tests. Each queued file names the endpoint its events were queued for, and a sender posts the files naming its own origin and those naming none, which earlier builds wrote, so events queued under the variable never reach PostHog.+- **`[usage_stats] enabled`**: a new `pipelex.toml` section, `true` in the packaged configuration. Set to `false` in a project's `.pipelex/pipelex.toml` it turns statistics off for everyone working in the project; it is read from the project's own file, never from `~/.pipelex/`, and a file that cannot be read turns them off too.+- **Token usage names the backend that served the call**: `LLMTokensUsage`, `ImgGenTokensUsage`, `ExtractTokensUsage`, `SearchTokensUsage` and `JudgmentTokensUsage` gain `backend_name`, set by the job from the backend whose worker ran it, so a run's usage events credit each call to its own backend even when two backends call the model by the same id, as after a fallback. A synthetic report no backend made, a dry run's, leaves it `None`.++### Changed++- **`ConceptLibraryAbstract.list_own_concepts` (Breaking)**: the concept library declares the concepts a loaded method declares itself, without the natives and the concepts of its dependency packages, so a custom implementation of `ConceptLibraryAbstract` implements it.+### Fixed- **A `{% filter %}` block prints its result as a `{{ }}` tag does**: a filter block whose filter returns something other than text, such as `{% filter markdown %}…{% endfilter %}` in a document template or `{% filter length %}`, now prints its result as a `{{ }}` tag prints a value, through the environment's `finalize` and then as text, where it used to fail the render with a type error naming a Python type. A block whose filter returns text prints as before, and the printed result is charged to the render's budget like any printed value.diff --git a/README.md b/README.mdindex 9682d5a8d..2f99f3db9 100644--- a/README.md+++ b/README.md@@ -234,6 +234,10 @@ make test # the unit testsThe [contributing guide](https://github.com/Pipelex/pipelex/blob/main/CONTRIBUTING.md) says how to configure `.env`, name a branch and open a pull request.+## Usage statistics++Pipelex collects anonymous usage statistics: counts of commands, runs, pipe types, models and tokens, and the country your IP address maps to; the address itself is never stored. It never collects your methods, prompts, inputs, outputs, file names or pipe names. To turn it off, run `pipelex usage-stats disable` or set `DO_NOT_TRACK=1`. The [usage statistics page](https://docs.pipelex.com/latest/setup/usage-statistics/) lists what is collected and every way to turn it off.+## CommunityAsk questions and share what you build on [Discord](https://go.pipelex.com/discord), report a bug in [GitHub Issues](https://github.com/Pipelex/pipelex/issues), propose an idea in [Discussions](https://github.com/Pipelex/pipelex/discussions), and watch demos on [YouTube](https://www.youtube.com/@PipelexAI).diff --git a/api/pipelex_api/routes/pipelex/codegen.py b/api/pipelex_api/routes/pipelex/codegen.pyindex 129f19962..7a54e0e84 100644--- a/api/pipelex_api/routes/pipelex/codegen.py+++ b/api/pipelex_api/routes/pipelex/codegen.py@@ -8,6 +8,8 @@ from pipelex.codegen.emitters.target import CodegenKind, CodegenTargetfrom pipelex.codegen.emitters.types_emitter import emit_typesfrom pipelex.codegen.lock import CODEGEN_LOCK_FILENAMEfrom pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.usage_stats_reporting import CodeGenerationReport, usage_stats_codegen_target+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifactfrom pipelex.tools.misc.package_utils import get_package_versionfrom pipelex.tools.typing.pydantic_utils import empty_list_factory_offrom pydantic import BaseModel, Field, model_validator@@ -136,12 +138,22 @@ async def codegen_mthds(request_data: CodegenRequest) -> JSONResponse:`method_ref` is a 501 until server-side method registry resolution exists; auth is 401/403;server fault is 5xx."""+ with CodeGenerationReport(+ artifact=UsageStatsCodegenArtifact.TYPES, target=usage_stats_codegen_target(target=request_data.target)+ ) as generation_report:+ return _generate_codegen_response(request_data=request_data, generation_report=generation_report)+++def _generate_codegen_response(*, request_data: CodegenRequest, generation_report: CodeGenerationReport) -> JSONResponse:+ """The body of `codegen_mthds`, timed by `generation_report`, which an invalid verdict declares failed."""try:# `types` is concept-set-wide: no pipe is selected, so the manifest's `main_pipe` beside the crate is unused.# A future per-pipe kind takes it from the `ResolvedClosure` and hands it to `resolve_requested_pipe`.crate = resolve_requested_crate(request_data).crateexcept ValidateBundleError as validate_error:+ generation_report.declare_failed()return invalid_crate_report_response(validate_error.to_error_report())+ generation_report.describe(crate=crate)try:emitted = emit_types(crate, target=request_data.target)projection = build_stamped_projection(diff --git a/api/tests/conftest.py b/api/tests/conftest.pyindex cd63503d9..56bf86fad 100644--- a/api/tests/conftest.py+++ b/api/tests/conftest.py@@ -1,4 +1,8 @@import os+from collections.abc import Generator++import pytest+from pipelex.test_extras.usage_stats_isolation import turn_usage_stats_off_for_the_session, usage_stats_guarded# Set a placeholder for COMPLETION_CALLBACK_SECRET so tests that exercise /start# never depend on the developer's shell env. CAVEAT: `make` test targets export@@ -11,3 +15,15 @@ os.environ.setdefault("COMPLETION_CALLBACK_SECRET", "test-placeholder-completionpytest_plugins = ["pipelex.test_extras.shared_pytest_plugins",]++# Anonymous usage statistics: no test reports to Pipelex, and none touches this machine's shared state in `~/.mthds/`.+# The server boots in the `fastapi` integration mode, which reports, so its tests turn statistics off in the environment+# when this conftest loads, and `isolate_usage_stats` guards the process.+USAGE_STATS_TEST_HOME = turn_usage_stats_off_for_the_session()++[email protected](scope="session", autouse=True)+def isolate_usage_stats() -> Generator[None, None, None]:+ """Keep the shared statistics state in a temporary home, and fail any attempt to send to a host that is not loopback."""+ with usage_stats_guarded(home=USAGE_STATS_TEST_HOME):+ yielddiff --git a/docs/configuration/config-practical/telemetry-config.md b/docs/configuration/config-practical/telemetry-config.mdindex 7a5b0cf35..eee0b1fb9 100644--- a/docs/configuration/config-practical/telemetry-config.md+++ b/docs/configuration/config-practical/telemetry-config.md@@ -9,7 +9,7 @@ description: "Set up custom telemetry destinations for observability data — seTelemetry configuration is stored in `.pipelex/telemetry.toml`. This file controls **your custom telemetry** destinations—where you want to send observability data for your own analytics and monitoring.!!! info "One Telemetry Stream, Yours"- The runtime has a single telemetry stream, and it is the one you configure in this file and send to your own backends. Nothing reports to Pipelex.+ The runtime has a single telemetry stream, and it is the one you configure in this file and send to your own backends. Nothing in this file reports to Pipelex. The anonymous usage statistics Pipelex collects are separate: this file neither configures nor reaches them, and [Usage Statistics](../../setup/usage-statistics.md) says how to turn them off.For an overview, see [Telemetry Setup](../../setup/telemetry.md).diff --git a/docs/configuration/index.md b/docs/configuration/index.mdindex e57eacd92..c3c1b52f4 100644--- a/docs/configuration/index.md+++ b/docs/configuration/index.md@@ -66,13 +66,14 @@ In addition to the base `pipelex.toml`, Pipelex applies override files from **in## Configuration Structure-The configuration is organized into five main sections. The first three mirror the layers of the runtime:+The configuration is organized into these main sections. The first three mirror the layers of the runtime:1. `[runtime]` - process-scoped infrastructure: storage, secrets, logging, cloud credentials, outbound network posture, reporting, tracing, observation, and the plugin denylist2. `[inference]` - the model-calling seam: the model deck, LLM, image generation, extraction, the default templating style, and dry-run mocks3. `[interpreter]` - library-scoped method machinery: MTHDS parsing, pipe runs, pipe functions, pipeline execution, source scanning, and the builder4. `[run]` - where `pipelex run` and `pipelex-agent run` execute a method by default: on this machine or on the hosted Pipelex API, see [Run Configuration](config-practical/run-config.md)5. `[kit]` - settings for the `pipelex-dev` kit tooling+6. `[usage_stats]` - `enabled = false` turns off Pipelex's anonymous usage statistics for everyone working in the project; it is read from the project's own `.pipelex/pipelex.toml` and from the configuration Pipelex is packaged with, never from `~/.pipelex/`, and `true` has no effect, see [Usage Statistics](../setup/usage-statistics.md)Each section contains multiple subsections for specific features and functionalities. A setting's address tells you which layer owns it: `[runtime.*]` applies to any process, whatever it loads; `[interpreter.*]` only means something once a method is loaded.diff --git a/docs/features/telemetry.md b/docs/features/telemetry.mdindex d809fd92f..a22073d70 100644--- a/docs/features/telemetry.md+++ b/docs/features/telemetry.md@@ -9,7 +9,7 @@ Production-ready monitoring for your AI methods.## Overview-Pipelex provides comprehensive telemetry and observability capabilities to monitor your AI methods in production. Track costs, latency, errors, and execution patterns across multiple destinations. Every destination is yours, opt-in and configurable per integration mode (CLI, pytest, API), and nothing reports to Pipelex.+Pipelex provides comprehensive telemetry and observability capabilities to monitor your AI methods in production. Track costs, latency, errors, and execution patterns across multiple destinations. Every destination is yours, opt-in and configurable per integration mode (CLI, pytest, API), and none of it reports to Pipelex. The anonymous [usage statistics](../setup/usage-statistics.md) Pipelex collects are separate, with switches of their own.## Langfuse Integrationdiff --git a/docs/get-started/configure-ai-providers.md b/docs/get-started/configure-ai-providers.mdindex a559428fb..93ba674c8 100644--- a/docs/get-started/configure-ai-providers.md+++ b/docs/get-started/configure-ai-providers.md@@ -6,7 +6,7 @@ description: "Set up API keys for OpenAI, Anthropic, Mistral, OpenRouter and oth## Configure API Access-To run pipelines on your own machine, Pipelex needs access to AI models. **You have two options**: your own provider keys, or models running locally. Nothing reports to Pipelex either way. This is the setup `pipelex init` runs when you answer **On this machine, with your own provider keys** to its question "Where should your runs execute?". If you would rather hold no provider keys at all, take its default answer, the hosted Pipelex API, which signs you in for a Pipelex API key instead (see [Run It Yourself](./run-it-yourself.md) and [`pipelex login`](../tools/cli/login.md)).+To run pipelines on your own machine, Pipelex needs access to AI models. **You have two options**: your own provider keys, or models running locally. Your methods, prompts, inputs and outputs never reach Pipelex either way; only anonymous [usage statistics](../setup/usage-statistics.md) do, which you can turn off. This is the setup `pipelex init` runs when you answer **On this machine, with your own provider keys** to its question "Where should your runs execute?". If you would rather hold no provider keys at all, take its default answer, the hosted Pipelex API, which signs you in for a Pipelex API key instead (see [Run It Yourself](./run-it-yourself.md) and [`pipelex login`](../tools/cli/login.md)).### Option 1: Bring Your Own API Keysdiff --git a/docs/setup/telemetry.md b/docs/setup/telemetry.mdindex 51eeaba4d..1fc5430d1 100644--- a/docs/setup/telemetry.md+++ b/docs/setup/telemetry.md@@ -1,10 +1,13 @@----description: "Understand Pipelex telemetry — nothing reports to Pipelex, and the streams you configure yourself are off until you turn them on."+description: "Understand Pipelex telemetry — telemetry.toml never reports to Pipelex, and the streams you configure yourself are off until you turn them on."---# Telemetry-**Pipelex sends no telemetry to Pipelex.** The only telemetry the runtime produces is yours: you configure it in `telemetry.toml`, it is off until you turn it on, and it goes to analytics and monitoring systems you operate.+**The telemetry you configure in `telemetry.toml` never reports to Pipelex.** It is yours: it is off until you turn it on, and it goes to analytics and monitoring systems you operate.++!!! note "Usage statistics are separate"+ Pipelex does collect anonymous usage statistics, counts of commands, runs, pipe types, models and tokens, through a client of their own that `telemetry.toml` neither configures nor reaches. They never carry your methods, prompts, inputs, outputs, file names or pipe names, and you turn them off with `pipelex usage-stats disable` or `DO_NOT_TRACK=1`. See [Usage Statistics](usage-statistics.md).## Your Telemetry Streams@@ -120,7 +123,7 @@ These are labels for grouping, not content: they land in the reserved identity fWe take your privacy seriously:-- Nothing reports to Pipelex: every stream is one you configure, sent to a system you operate+- Nothing in `telemetry.toml` reports to Pipelex: every stream is one you configure, sent to a system you operate; the anonymous [usage statistics](usage-statistics.md) are separate and have switches of their own- Your `telemetry.toml` gives you full control over what data is captured- PostHog tracing includes privacy controls to redact sensitive content- All telemetry can be completely disableddiff --git a/docs/setup/usage-statistics.md b/docs/setup/usage-statistics.mdnew file mode 100644index 000000000..929b875b8--- /dev/null+++ b/docs/setup/usage-statistics.md@@ -0,0 +1,116 @@+---+description: "What anonymous usage statistics Pipelex collects, what it never collects, and every way to turn them off."+---++# Usage Statistics++Pipelex collects **anonymous usage statistics**: counts of commands, runs, pipe types, models and tokens, and the country your IP address maps to. The address itself is never stored. Pipelex never collects your methods, prompts, inputs, outputs, file names or pipe names. The statistics tell us which features, pipe types and models people use, so we know where to put our work.++They are on by default, and you can turn them off at any time, for one command, one project or the whole machine, as [Turning them off](#turning-them-off) explains.++The first time a Pipelex tool runs with statistics on, it prints this notice on stderr, once per machine. A command whose stderr a program reads rather than a person, `pipelex-agent` with JSON errors (`--error-format json`, or `--format json` with no `--error-format`), neither prints nor records it, and still sends its statistics; the next command whose stderr a person reads prints it:++> Pipelex collects anonymous usage statistics: counts of commands, runs, pipe types, models and tokens, and the country your IP address maps to; the address itself is never stored. It never collects your methods, prompts, inputs, outputs, file names or pipe names. To turn it off, run `pipelex usage-stats disable` or set DO_NOT_TRACK=1. Details: https://docs.pipelex.com/latest/setup/usage-statistics/++`pipelex init` carries the same text: whenever statistics are on, it shows this notice on stderr ahead of its own output, even after the first run printed it. A first run that is itself `pipelex init` shows it once, not twice. Showing it there records nothing, and with statistics off it shows nothing.++!!! note "Not your telemetry"+ The usage statistics are separate from the telemetry you configure in `telemetry.toml`, which goes only to analytics and monitoring systems you operate and never to Pipelex. See [Telemetry](telemetry.md).++## What is collected++Every event is a count, a yes-or-no, or a value from a closed list. No property can hold text you wrote. Each event carries:++- the tool and its version (`pipelex`, and the release), and the surface it ran from: the `pipelex` CLI, the `pipelex-agent` CLI, the API server or a Python program;+- the operating system and processor family, the Python minor version, and whether the process ran in CI or in a container;+- an anonymous **installation identifier**, a random UUID drawn once per machine and shared by every Pipelex tool on it, never derived from anything about the machine or you.++The events themselves are:++| Event | Sent | What it counts |+| --- | --- | --- |+| `command_completed` | once per command of the `pipelex` and `pipelex-agent` CLIs | the command's path in the command tree (`run bundle`, `validate pipe`, `init` and so on, a shorthand such as `pipelex validate my_pipe` counting as the command it runs; never its arguments or option values), its outcome, its duration, who could fix a failure (`input`, `config`, `runtime` or `other`), and whether it ran on the hosted API; for `init`, also where it set runs to execute and, when it set up backends on this machine, which backend kinds it enabled |+| `run_completed` | once per run, whatever started it | the outcome, live or dry, the kind of source the method came from (a local bundle, an installed method, a published address), the duration, the bundle's size and nesting depth, which language features it used (batching, parallel branches, conditions, structured outputs, image and document inputs), how many pipes of each type ran, and its inference totals: calls, failures, tokens, images, pages and cost |+| `model_used` | once per model a run used | the backend kind (`openai`, `anthropic`, `mistral`, and so on, `custom` for a backend Pipelex does not ship), the model's name in the backend tables Pipelex ships (`other` for any other), and that run's calls, failures by category, tokens, images, pages and cost for it |+| `method_validated` | once per validation of a method | the verdict, the duration, the number of errors and warnings, and the bundle's size and depth |+| `code_generated` | once per code generation | what was generated (types, input templates or structure classes), the target language, the outcome, the duration and the bundle's size |++A run's token totals come from the run's own usage events: when you turn tracing or cost reporting off, the run reports that its usage is unknown and no `model_used` is sent. The statistics never turn tracing on.++### What is never collected++No event carries a method or its MTHDS source; a pipe code, concept name, domain name, class name or any other identifier you wrote; a prompt, input, output or generated content; a file name, path, URL or repository address; a method name, catalog id or run id; an error message, stack trace or exception; a command argument or option value; an environment variable's value; a hostname, user name or e-mail address; an IP address or any location finer than the country; the order in which a run's pipes executed.++A test in Pipelex runs, validates and generates code from a method whose every name, prompt, input and error message holds a unique marker, and fails if the marker appears anywhere in what would be sent.++## Turning them off++Statistics are on unless any of these says off, and nothing can turn them back on over another switch that says off:++| Switch | How to set it | Scope |+| --- | --- | --- |+| `DO_NOT_TRACK` | set to anything but empty, `0` or `false`, the [Console Do Not Track](https://consoledonottrack.com/) convention | the process, and every process it starts |+| `PIPELEX_USAGE_STATS` | set to `0`, `false`, `no` or `off`; an unrecognised value also turns statistics off | the process, without silencing the rest of your tools the way `DO_NOT_TRACK` does |+| The machine's choice | `pipelex usage-stats disable` | every Pipelex tool on the machine: pipelex, `plxt`, the VS Code extension and the `mthds` CLI |+| The `mthds` CLI's former opt-out | `DISABLE_TELEMETRY=1` in `~/.mthds/config`, which `mthds telemetry disable` wrote before these statistics existed | every Pipelex tool on the machine: the first tool that runs turns it into the machine's choice |+| The project setting | `[usage_stats] enabled = false` in the project's `.pipelex/pipelex.toml` | everyone working in the project, once the file is committed |+| The packaged default | `[usage_stats] enabled = false` in the `pipelex.toml` Pipelex is packaged with, for a distribution or a corporate image | every user of that installation |+| The `pytest` integration mode | Pipelex booted with `IntegrationMode.PYTEST`, as test suites do | the test process: no test suite reports |++The project setting can only turn statistics off: `enabled = true` has no effect. It fails closed: a `.pipelex/pipelex.toml` that cannot be read, is not valid TOML, or holds an `enabled` that is not a boolean turns them off. Pipelex reads it from the project's own file, found from the working directory the way Pipelex finds its project configuration, and never from your home configuration in `~/.pipelex/`.++```toml+# .pipelex/pipelex.toml+[usage_stats]+enabled = false+```++Turning statistics off creates nothing and sends nothing: a process with statistics off writes nothing to the shared state, except when it turns the `mthds` CLI's former opt-out into the machine's choice.++## The `usage-stats` commands++`pipelex usage-stats` shows and changes the statistics of every Pipelex tool on the machine, `pipelex-agent` included, which has no group of its own. These commands never send anything, never print the notice, and exit with `0` unless they could not do what they say.++| Command | What it does |+| --- | --- |+| `pipelex usage-stats status` | Prints whether statistics are on, every switch that turns them off, the installation identifier and the notice version seen. It changes nothing |+| `pipelex usage-stats status --format json` | The same as one JSON object on stdout, the form other tools read |+| `pipelex usage-stats disable` | Turns statistics off for every Pipelex tool on the machine and deletes every event still waiting to be sent |+| `pipelex usage-stats enable` | Turns the machine's choice back on, and names any other switch that still keeps statistics off |+| `pipelex usage-stats reset-id` | Deletes every event waiting to be sent and the installation identifier, draws a new one and prints it |++```json+{+ "tool": "pipelex",+ "enabled": false,+ "off_by": ["machine_choice"],+ "installation_id": "0f8fad5b-d9cb-469f-a165-70867728950e",+ "notice_version": 1,+ "internal": false,+ "ci": false,+ "container": false,+ "state_dir": "/home/alice/.mthds/usage-stats"+}+```++`off_by` names the switches that say off, in the order of the table above: `do_not_track`, `tool_variable`, `machine_choice`, `legacy_mthds_choice`, `project_setting`, `packaged_default`, `integration_mode`, and `invalid_endpoint_override` for the test-only endpoint variable below.++## Where it is kept and where it goes++Every Pipelex tool on a machine shares one directory, `~/.mthds/usage-stats/` (`%USERPROFILE%\.mthds\usage-stats\` on Windows), created with owner-only permissions. It holds the installation identifier, the machine's choice, a marker per notice version shown, and each tool's queue of events waiting to be sent, under `spool/`. A file there is created or deleted whole, so two tools running at once never undo each other's choice. When Pipelex cannot look into that directory, statistics stay off rather than risk ignoring a choice stored there, and your command runs as usual.++Events go straight to PostHog's EU region, to a project that holds nothing but these statistics. PostHog discards the IP address after deriving the country from it, and keeps no finer location. No person profile is ever created. PostHog keeps the events for its retention period, and we delete the events of an installation on request, as our [Privacy Policy](https://go.pipelex.com/privacy-policy) explains: send us the identifier `pipelex usage-stats status` prints, after turning statistics off or running `pipelex usage-stats reset-id`.++How an event leaves depends on the process:++- **A CLI command** writes its events to its queue when it ends and starts a small detached sender process, then exits at once; the sender posts the queue within ten seconds. The sender is a single file that needs nothing but Python itself, so it delivers whichever way Pipelex was installed and never loads Pipelex. An event that cannot be sent, because you are offline for instance, stays queued for the next command's sender. In CI or in a container, where a detached process would be killed, the command posts its events itself within one second before exiting and, once PostHog has accepted them, what earlier commands left queued, within that same second.+- **A Python program, or a long-running process** such as the API server, posts from a background thread within seconds of each event, and flushes what is left when Pipelex is torn down or the program exits, within a short time budget. Each time its events get through, what earlier processes left queued goes too, within the same budget.++A send never delays you by more than these budgets, and a failure to send never changes what your command or your run does.++## For developers++- **`PIPELEX_USAGE_STATS_DEBUG`**, set to anything but empty, `0` or `false`, prints each event to stderr as one JSON line, `{"usage_stats_debug": {...}}`, instead of sending it. The switches still decide, so nothing is printed while statistics are off.+- **`PIPELEX_USAGE_STATS_ENDPOINT`**, for tests, sends the events to a loopback `http` origin such as `http://127.0.0.1:8123` instead of PostHog. Any other value turns statistics off. Each queued file names the endpoint its events were queued for, and a sender posts only the files naming the origin it posts to, plus any file naming none, as earlier builds wrote them. So an event queued while the variable is set is only ever sent to that origin, and one queued for PostHog never goes there.+- **`PIPELEX_USAGE_STATS_HOST`** names the tool that embeds Pipelex, from a closed list (`cocode`, `mthds-agent`, `other`), so its runs can be told apart.+- Pipelex's own continuous integration sets `DO_NOT_TRACK=1`, and its test suites turn statistics off and keep the shared state in a temporary directory.diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.mdindex 78ff2e1d6..f9ba2cd6f 100644--- a/docs/tools/cli/agent-cli.md+++ b/docs/tools/cli/agent-cli.md@@ -218,6 +218,10 @@ For the structured commands, exactly one such object is written to stderr per fa`fmt` and `lint` are raw passthroughs to the `plxt` binary and bypass the JSON output contract, producing native `plxt` output instead.+## Usage Statistics++`pipelex-agent` sends Pipelex's [anonymous usage statistics](../../setup/usage-statistics.md) as `pipelex` does: one `command_completed` per command, plus the events of the runs, validations and code generations it performs, all off under `DO_NOT_TRACK=1`, `PIPELEX_USAGE_STATS=0` or `pipelex usage-stats disable`. When the invocation's errors are JSON, with `--error-format json` or with `--format json` and no `--error-format`, stderr is a machine contract, so the one-time notice is neither printed nor recorded there; the next command whose stderr a person reads prints it. The `plxt` that `fmt` and `lint` start reports nothing of its own. There is no `usage-stats` group on this CLI: [`pipelex usage-stats`](usage-stats.md) shows and changes the statistics for both.+## Graph VisualizationGraph visualization is available through the `validate bundle` subcommand with the `--graph` flag, and through `run` subcommands where it is enabled by default. Use `--no-graph` on `run` to disable it.diff --git a/docs/tools/cli/index.md b/docs/tools/cli/index.mdindex 9acbdd103..184c34a3b 100644--- a/docs/tools/cli/index.md+++ b/docs/tools/cli/index.md@@ -34,6 +34,7 @@ The Pipelex CLI is organized into several command groups:| [**run**](run.md) | Execute pipelines, on this machine or on the hosted Pipelex API || [**run method by address**](run-by-address.md) | Fetch and run a method straight from a public GitHub repository || [**build**](build/index.md) | Generate concept structures and example inputs and outputs for a pipe |+| [**usage-stats**](usage-stats.md) | See and change the anonymous usage statistics of every Pipelex tool on the machine |## Global flagsdiff --git a/docs/tools/cli/init.md b/docs/tools/cli/init.mdindex 90ecf86b9..dd3ab3fa8 100644--- a/docs/tools/cli/init.md+++ b/docs/tools/cli/init.md@@ -15,6 +15,8 @@ pipelex init --local [FOCUS]By default, `pipelex init` writes to the home configuration directory (`~/.pipelex/`, or `PIPELEX_HOME`; see [Configuration](../../configuration/index.md#the-home-configuration-directory-pipelex_home)). Use `--local` to create a project-level `.pipelex/` directory at the detected project root. Credentials always remain in the home configuration directory regardless of `--local`.+Whenever [usage statistics](../../setup/usage-statistics.md) are on, `pipelex init` shows their notice on stderr ahead of its own output, once per run; with them off, it shows nothing about them.+!!! note "Config updates not yet supported"The `pipelex init` command always performs a full reset of the configuration. Incremental config updates will be supported in a future release.diff --git a/docs/tools/cli/usage-stats.md b/docs/tools/cli/usage-stats.mdnew file mode 100644index 000000000..311975944--- /dev/null+++ b/docs/tools/cli/usage-stats.md@@ -0,0 +1,61 @@+---+title: "CLI Usage Stats"+description: "See and change Pipelex's anonymous usage statistics for every Pipelex tool on the machine with `pipelex usage-stats`."+---++# Usage Stats++`pipelex usage-stats` shows and changes Pipelex's anonymous usage statistics for every Pipelex tool on the machine: pipelex and `pipelex-agent`, `plxt`, the VS Code extension and the `mthds` CLI, which all read the same state in `~/.mthds/usage-stats/`. [Usage Statistics](../../setup/usage-statistics.md) says what is collected and every other way to turn the statistics off.++These commands never send anything and never print the notice. They exit with `0` unless they could not do what they say, such as a state directory that cannot be written, and then print why on stderr and exit with `1`.++## status++```bash+pipelex usage-stats status+pipelex usage-stats status --format json+```++Prints whether statistics are on, every switch that turns them off, the installation identifier, the notice version seen on the machine, whether it is marked as one of the Pipelex team's, whether the process is in CI or in a container, and the shared state directory. It changes nothing: it creates no identifier and turns no former `mthds` opt-out into the machine's choice.++With `--format json`, it prints one JSON object on stdout, the form other tools read:++```json+{+ "tool": "pipelex",+ "enabled": false,+ "off_by": ["machine_choice"],+ "installation_id": "0f8fad5b-d9cb-469f-a165-70867728950e",+ "notice_version": 1,+ "internal": false,+ "ci": false,+ "container": false,+ "state_dir": "/home/alice/.mthds/usage-stats"+}+```++`off_by` lists the switches that say off, in the order [Turning them off](../../setup/usage-statistics.md#turning-them-off) gives, and is empty when statistics are on. `installation_id` is `null` until a tool has drawn one.++## disable++```bash+pipelex usage-stats disable+```++Turns statistics off on this machine for every Pipelex tool, and deletes every event any tool still had waiting to be sent.++## enable++```bash+pipelex usage-stats enable+```++Turns the machine's choice back on, including after a former `mthds telemetry disable`, and names every other switch that still keeps statistics off, such as `DO_NOT_TRACK` in your environment or a project's `[usage_stats] enabled = false`.++## reset-id++```bash+pipelex usage-stats reset-id+```++Deletes every event waiting to be sent and the installation identifier, draws a new one and prints it on stdout. Events made under the old identifier are never sent afterwards.diff --git a/docs/under-the-hood/init-cli-flows.md b/docs/under-the-hood/init-cli-flows.mdindex 1e7c999d5..05a61ba2c 100644--- a/docs/under-the-hood/init-cli-flows.md+++ b/docs/under-the-hood/init-cli-flows.md@@ -33,6 +33,8 @@ These two categories are managed by separate steps. `init_config()` copies only| `pipelex init telemetry` | `telemetry` | Telemetry config template copy || `pipelex init credentials` | `credentials` | Credential setup for enabled backends |+Every one of them first shows the [usage statistics](../setup/usage-statistics.md) notice on stderr when statistics are on, through `show_usage_stats_notice()`, which the `init` Typer command in `pipelex/cli/_cli.py` calls before `init_cmd()`, so `pipelex doctor --fix`, which calls `init_cmd()` itself, does not show it. A first run that has just printed the notice does not print it again, and showing it records no marker.+All commands except `credentials` perform a **full reset** (overwrite existing files) — every setting in the file is replaced by the template's. That is why init is not the answer to a configuration file that has fallen behind the current schema: [`pipelex migrate`](../tools/cli/migrate.md) rewrites such a file in place and keeps what is in it.### Inputsdiff --git a/mkdocs.yml b/mkdocs.ymlindex dbb2b8d2f..e53f77809 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -300,6 +300,7 @@ plugins:- tools/cli/run.md: "CLI Run"- tools/cli/run-by-address.md: "CLI Run by Address"- tools/cli/show.md: "CLI Show"+ - tools/cli/usage-stats.md: "CLI Usage Stats"- tools/cli/pkg.md: "CLI Pkg"- tools/cli/build/index.md: "CLI Build Overview"- tools/cli/build/structures.md: "CLI Build Structures"@@ -323,6 +324,7 @@ plugins:- tools/logging.md: "Logging"- analytics/data-extraction.md: "Observer Data Extraction"- setup/telemetry.md: "Telemetry"+ - setup/usage-statistics.md: "Usage Statistics"Advanced:- advanced/index.md: "Advanced Customizations Overview"- advanced/secrets-provider-injection.md: "Secrets Provider"@@ -544,6 +546,7 @@ nav:- Run: tools/cli/run.md- Run by Address: tools/cli/run-by-address.md- Show: tools/cli/show.md+ - Usage Stats: tools/cli/usage-stats.md- Pkg: tools/cli/pkg.md- Build:- Overview: tools/cli/build/index.md@@ -558,6 +561,7 @@ nav:- Analytics:- Observer Data Extraction: analytics/data-extraction.md- Telemetry: setup/telemetry.md+ - Usage Statistics: setup/usage-statistics.md- Error Reference:- Overview: errors/index.md- Authoring & language: errors/authoring-and-language.mddiff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 2f95b05d8..9eca8e06a 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -1,4 +1,6 @@-from typing import Annotated+import sys+from collections.abc import Sequence+from typing import Annotated, Anyimport typerfrom click import Command, Context@@ -19,13 +21,17 @@ from pipelex.cli.commands.resolve_cmd import resolve_cmdfrom pipelex.cli.commands.run.app import run_appfrom pipelex.cli.commands.show_cmd import show_appfrom pipelex.cli.commands.update_cmd import update_cmd+from pipelex.cli.commands.usage_stats_cmd import usage_stats_appfrom pipelex.cli.commands.validate.app import validate_appfrom pipelex.cli.commands.which_cmd import which_cmdfrom pipelex.cli.deck_notice import warn_if_deck_stalefrom pipelex.cli.error_handlers import set_traceback_requestedfrom pipelex.cli.readiness import check_readiness+from pipelex.cli.usage_stats_cli import STATISTICS_GROUP_NAME, run_with_usage_statsfrom pipelex.cli.version_report import version_report_linesfrom pipelex.runtime_hub import get_console+from pipelex.system.usage_stats.usage_stats_client import show_usage_stats_notice+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsSurfacefrom pipelex.tools.misc.package_utils import get_package_version# Core commands in display order (natural ordering doesn't work between Typer groups and commands).@@ -45,12 +51,44 @@ _CORE_COMMAND_ORDER: list[str] = ["show","which","plugins",+ "usage-stats",]+#: The global flags `PipelexCLI.make_context` removes wherever they appear on the command line.+_GLOBAL_FLAGS: frozenset[str] = frozenset({"--no-logo", "--traceback"})+#: The commands that run before the readiness checks, which they diagnose, repair or do not need.+_COMMANDS_WITHOUT_READINESS_CHECKS: frozenset[str] = frozenset({"init", "login", "doctor", "update", "migrate", "which", STATISTICS_GROUP_NAME})+class PipelexCLI(TyperGroup):"""Custom CLI group that handles global options like --no-logo."""+ @override+ def main(+ self,+ args: Sequence[str] | None = None,+ prog_name: str | None = None,+ complete_var: str | None = None,+ standalone_mode: bool = True,+ windows_expand_args: bool = True,+ **extra: Any,+ ) -> Any:+ """Run the command between the start and the end of its usage statistics (`pipelex.cli.usage_stats_cli`)."""+ return run_with_usage_stats(+ root=self,+ args=list(args) if args is not None else sys.argv[1:],+ surface=UsageStatsSurface.PIPELEX,+ strip_flags=_GLOBAL_FLAGS,+ invoke=lambda: super(PipelexCLI, self).main(+ args=args,+ prog_name=prog_name,+ complete_var=complete_var,+ standalone_mode=standalone_mode,+ windows_expand_args=windows_expand_args,+ **extra,+ ),+ )+@overridedef list_commands(self, ctx: Context) -> list[str]:return list(_CORE_COMMAND_ORDER)@@ -126,6 +164,10 @@ def app_callback(),] = False,) -> None:+ # The statistics group prints what it was asked for and nothing else: its JSON status is read by other programs.+ if ctx.invoked_subcommand == STATISTICS_GROUP_NAME:+ return+console = get_console()package_version = get_package_version()@@ -153,7 +195,7 @@ def app_callback(""")# Skip checks if no command is being run (e.g., just --help) or if running setup/diagnostic commands- if ctx.invoked_subcommand is None or ctx.invoked_subcommand in {"init", "login", "doctor", "update", "migrate", "which"}:+ if ctx.invoked_subcommand is None or ctx.invoked_subcommand in _COMMANDS_WITHOUT_READINESS_CHECKS:return# Check system readiness (dependencies and venv for dev installs)@@ -207,6 +249,8 @@ def init_command(telemetry Reset telemetry preferences"""+ # `pipelex init` carries the usage statistics notice's text whenever they are on, ahead of its own output.+ show_usage_stats_notice()init_cmd(focus=focus, local=local)@@ -288,3 +332,8 @@ app.add_typer(graph_app, name="graph", help="Generate and render execution graphapp.add_typer(show_app, name="show", help="Show configuration, pipes, and list AI models")app.command(name="which", help="Locate where a pipe is defined, similar to 'which' for executables")(which_cmd)app.add_typer(plugins_app, name="plugins", help="Inspect the discovered plugins (inference backends, orchestrators) and their contributions")+app.add_typer(+ usage_stats_app,+ name=STATISTICS_GROUP_NAME,+ help="See and change the anonymous usage statistics of every Pipelex tool on this machine (status, disable, enable, reset-id)",+)diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.mdindex c2af16d8f..ddae2998b 100644--- a/pipelex/cli/agent_cli/CLAUDE.md+++ b/pipelex/cli/agent_cli/CLAUDE.md@@ -108,4 +108,5 @@ commands/- **Hosted runs**: every `run` subcommand takes `--runner local|hosted`, `--hosted/--local` and `--base-url`; `resolve_agent_run_execution()` (`run/_run_hosted.py`) settles them through `pipelex.hosted.execution.resolve_run_execution` (the flag, then `[run] execution`, then local), refusing a `--runner` that contradicts the flag pair and a `--base-url` on a local run; a `PipelexConfigError` there carries `AGENT_INIT_FAILURE_HINT`, the hint a failed boot gives (`agent_cli_factory.py`). A hosted run boots nothing: it builds a `HostedRunRequest` from what `pipelex/cli/commands/run/_hosted_sources.py` resolves (the bundle and its library's `.mthds` files, a pipe's library, or a method's files; an address or a catalog id as `method_ref` / `method_id`, never fetched) and `run_hosted_for_agent()` runs it through `pipelex.hosted.hosted_run.run_hosted`, which reads the signature and uploads local input files (only when some input string names an existing local file, so a method with an address-based dependency runs with any other inputs), then drives the lifecycle `start_and_wait` drives step by step (the `/v1/version` handshake, `start`, `wait_for_result`, and the blocking `execute` on a server with no run store), so a failure while following an acknowledged run is a `HostedRunPollingError` naming the run, and a transport failure after the run request was sent is a `HostedRunOutcomeUnknownError` rather than an `ApiUnreachableError`. `run_hosted`'s `on_started` callback records the acknowledged run's id, so Ctrl-C (`KeyboardInterrupt`, or the `CancelledError` of a cancelled task) leaves as a `HostedRunInterruptedError` envelope carrying `pipeline_run_id`, with exit code 130. `run bundle` on a file with no `-L` sends `PIPELEXPATH`'s files (`hosted_library_dirs`), and an installed method named like a catalog id is that method. An exception no hosted arm names leaves through the command-boundary catch-all as the envelope. Its success envelope is `build_run_output()`'s, with the main output's content as `json`, an empty `markdown`, the working memory dump and `pipeline_run_id`; nothing is written to disk. `--dry-run` / `--mock-inputs` are refused on it, while `--graph` and `--costs` are accepted and ignored. There is no global `--runner` any more, and no `ctx.obj`.- **Hosted refusals**: `agent_error_hosted()` routes pipelex-sdk's `ApiResponseError` (every `PipelexAPIClient` route raises it on a non-2xx answer) to `agent_error_api_response()`. `api_response_error_payload()` builds the envelope from the problem document alone: `error_type` is the runner's class (`ValidateBundleError` at load, as a local run reports it, or the root fault of a failed run, which a local run reports as `cause_type` under `PipelineExecutionError`; this path has no `cause_type`), `message` is the error's own `API POST /v1/start failed (<status>): <reason>` without the next step it appends, `hint` is `user_action.detail`, and `error_domain` / `error_category` / `retryable` / `validation_errors` / `pipe_code` / `pipe_stack` / `request_id` are the runner's, plus `error_code` (the problem's `code`), `retry_after_seconds` (the `Retry-After` header) and `http_status`. It deliberately bypasses `agent_error()`'s lookups: the dict hints keyed on a class name local commands (`doctor`, `check-model`) that cannot fix a remote refusal, and the domain is only the runner's to say, so one with no `error_domain` carries none. Both `error_domain` and `retryable` come from `pipelex.hosted.error_rendering.sent_verdict()`, the members the problem document carried, never from the verdict pipelex-sdk decides for every `ApiResponseError`, which fills an unsent member from its own status table and reads every 5xx, a 503 included, as retryable. A refusal with no `user_action` — the hosted plane's own answers in front of the runner: the gateway's 403 for a bad key, the rate limiter's 429, the unreachable runner's 503 — gets its status's next step from `pipelex.hosted.error_rendering` (`HOSTED_REFUSAL_NEXT_STEPS_BY_STATUS`, the table the human CLI reads too; 401 and 403 point at `PIPELEX_API_KEY` and `pipelex login`, 404 at the method and the origin), else `HOSTED_REFUSAL_CALLER_NEXT_STEP` for any other 4xx and `HOSTED_SERVER_FAULT_NEXT_STEP` for the rest. Only a 429 that did not say otherwise is inferred `retryable`: the platform can answer a 503 after the run completed, when it failed to record the result, so its hint warns that a retry can repeat a paid run. Every other failure, the SDK's and pipelex's own `HostedRunError` subclasses alike, goes through `agent_error()` with what `describe_hosted_error()` reads off it, the view the human CLI prints too: the class, message, next step, domain and `retryable`, the `pipeline_run_id` of any failure met once the run was acknowledged (`RunFailedError`, `RunTimeoutError`, `MissingMainStuffError`, `HostedRunPollingError`, `HostedRunInterruptedError`), a `retryable` passed whenever the hosted API gave a verdict, `false` included, so `RETRYABLE_ERROR_TYPES` keyed on the runner's class never overrides it, and the `validation_errors` of a failed run or of a `HostedMethodInvalidError` (pipe-io's verdict on a method that does not load, whose items name the faulty file in `source`); a `RunFailedError` adds `run_status` and its report's category, model and provider. The markdown renderer keeps a runner item this pipelex's `ValidationErrorItem` cannot type (a newer `error_type`, an unknown field) as JSON under Details instead of failing.- **Method-reference failures**: the `run`/`validate`/`inputs` method commands call `resolve_method_target(..., raise_ref_errors=True)` and shape the typed `MethodRefError` subclasses into `agent_error(...)` themselves (validate uses `exit_code=2`, its no-verdict convention) — the human CLI's red-text rendering stays in `resolve_method_target`'s default arm.+- **Usage statistics**: the root callback runs every command through `run_with_usage_stats` (`pipelex/cli/usage_stats_cli.py`) with surface `pipelex-agent`, which reads the switches before the command does anything and sends one `command_completed` when it ends. Stderr is a machine contract when the invocation's errors are JSON (`--error-format json`, or `--format json` with no `--error-format`), so the anonymous-statistics notice is then neither printed nor recorded, though the events are still sent. `_assemble_error_payload` and `api_response_error_payload` note the envelope's `error_domain` as the command's, never the error itself. `fmt` and `lint` start `plxt` with `PIPELEX_USAGE_STATS=0`, since the command's statistics are this CLI's own. There is no `usage-stats` group here: `pipelex usage-stats` serves both CLIs. The contract is `conformance/specs/usage-statistics.md`; the user docs are `docs/setup/usage-statistics.md`.- **TOML handling**: Uses `tomlkit` (not `tomllib`) to preserve formatting and inline tables.diff --git a/pipelex/cli/agent_cli/_agent_cli.py b/pipelex/cli/agent_cli/_agent_cli.pyindex 690be5b3b..5ab17e334 100644--- a/pipelex/cli/agent_cli/_agent_cli.py+++ b/pipelex/cli/agent_cli/_agent_cli.py@@ -1,6 +1,8 @@"""Main entry point for the agent CLI."""-from typing import Annotated+import sys+from collections.abc import Sequence+from typing import Annotated, Anyimport typerfrom click import Command, Context@@ -21,12 +23,40 @@ from pipelex.cli.agent_cli.commands.migrate_cmd import agent_migrate_cmdfrom pipelex.cli.agent_cli.commands.models_cmd import agent_models_cmdfrom pipelex.cli.agent_cli.commands.run.app import run_appfrom pipelex.cli.agent_cli.commands.validate.app import validate_app+from pipelex.cli.usage_stats_cli import run_with_usage_statsfrom pipelex.cli.version_report import version_report_lines+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsSurfaceclass PipelexAgentCLI(TyperGroup):"""Custom Typer group for pipelex-agent CLI."""+ @override+ def main(+ self,+ args: Sequence[str] | None = None,+ prog_name: str | None = None,+ complete_var: str | None = None,+ standalone_mode: bool = True,+ windows_expand_args: bool = True,+ **extra: Any,+ ) -> Any:+ """Run the command between the start and the end of its usage statistics (`pipelex.cli.usage_stats_cli`)."""+ return run_with_usage_stats(+ root=self,+ args=list(args) if args is not None else sys.argv[1:],+ surface=UsageStatsSurface.PIPELEX_AGENT,+ strip_flags=frozenset(),+ invoke=lambda: super(PipelexAgentCLI, self).main(+ args=args,+ prog_name=prog_name,+ complete_var=complete_var,+ standalone_mode=standalone_mode,+ windows_expand_args=windows_expand_args,+ **extra,+ ),+ )+@overridedef list_commands(self, ctx: Context) -> list[str]:"""List commands in proper order."""diff --git a/pipelex/cli/agent_cli/commands/agent_output.py b/pipelex/cli/agent_cli/commands/agent_output.pyindex 53812c4c3..10588465f 100644--- a/pipelex/cli/agent_cli/commands/agent_output.py+++ b/pipelex/cli/agent_cli/commands/agent_output.py@@ -34,6 +34,8 @@ from pipelex.hosted.error_rendering import hosted_refusal_message, hosted_refusafrom pipelex.pipe_run.located_failure import find_root_faultfrom pipelex.pipeline.exceptions import PipelineExecutionError, ValidateBundleErrorfrom pipelex.pipeline.validation_render import build_fix_command, count_applicable_fixes, format_validation_error_items_markdown+from pipelex.system.usage_stats.usage_stats_client import note_command_error_domain+from pipelex.system.usage_stats.usage_stats_runs import usage_stats_error_domain_offrom pipelex.tools.misc.json_utils import clean_json_dumps@@ -277,6 +279,8 @@ def _assemble_error_payload(message: str, *, error_type: str, cause: BaseExceptidomain = report_domain or AGENT_ERROR_DOMAINS.get(error_type)if domain:error_json["error_domain"] = domain+ # The command's usage statistics carry the same classification, never the error itself.+ note_command_error_domain(error_domain=usage_stats_error_domain_of(domain=domain))# error_category: from report when availableif report_category:@@ -526,6 +530,7 @@ def api_response_error_payload(*, error: ApiResponseError) -> dict[str, Any]:payload["retryable"] = Trueif sent.error_domain:payload["error_domain"] = sent.error_domain+ note_command_error_domain(error_domain=usage_stats_error_domain_of(domain=sent.error_domain))if error_category := _non_empty_string(value=problem.get("error_category")):payload["error_category"] = error_categoryif error.validation_errors:diff --git a/pipelex/cli/agent_cli/commands/codegen/types_cmd.py b/pipelex/cli/agent_cli/commands/codegen/types_cmd.pyindex 5da4131cb..5c78bd708 100644--- a/pipelex/cli/agent_cli/commands/codegen/types_cmd.py+++ b/pipelex/cli/agent_cli/commands/codegen/types_cmd.py@@ -28,6 +28,8 @@ from pipelex.codegen.emitters.types_emitter import emit_typesfrom pipelex.libraries.exceptions import LibraryLoadingErrorfrom pipelex.libraries.pipe.exceptions import PipeLibraryErrorfrom pipelex.pipelex import Pipelex+from pipelex.pipeline.usage_stats_reporting import CodeGenerationReport, usage_stats_codegen_target+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifactfrom pipelex.tools.misc.package_utils import get_package_versionif TYPE_CHECKING:@@ -77,25 +79,27 @@ def agent_codegen_types_cmd(try:combined_dirs: list[Path] = [*(paths or []), *(Path(lib_dir) for lib_dir in library_dir or [])]- crate: LibraryCrate- try:- crate = load_normalized_crate(library_dirs=combined_dirs or None)- except (LibraryLoadingError, PipeLibraryError) as exc:- # Negative verdict: the library is structurally invalid, so no crate can be produced.- agent_error(f"Cannot resolve — the library is invalid: {exc}", error_type=type(exc).__name__, cause=exc, exit_code=1)- except FileNotFoundError as exc:- # No verdict: the closure could not even be assembled.- agent_error(f"Cannot resolve — {exc}", error_type="FileNotFoundError", cause=exc, exit_code=2)-- emitted = emit_types(crate, target=target)- report = write_stamped_projection(- emitted,- output_dir=output_root,- crate_fingerprint=crate.fingerprint,- engine_version=get_package_version(),- kind=CodegenKind.TYPES,- target=target,- )+ with CodeGenerationReport(artifact=UsageStatsCodegenArtifact.TYPES, target=usage_stats_codegen_target(target=target)) as generation_report:+ crate: LibraryCrate+ try:+ crate = load_normalized_crate(library_dirs=combined_dirs or None)+ except (LibraryLoadingError, PipeLibraryError) as exc:+ # Negative verdict: the library is structurally invalid, so no crate can be produced.+ agent_error(f"Cannot resolve — the library is invalid: {exc}", error_type=type(exc).__name__, cause=exc, exit_code=1)+ except FileNotFoundError as exc:+ # No verdict: the closure could not even be assembled.+ agent_error(f"Cannot resolve — {exc}", error_type="FileNotFoundError", cause=exc, exit_code=2)+ generation_report.describe(crate=crate)++ emitted = emit_types(crate, target=target)+ report = write_stamped_projection(+ emitted,+ output_dir=output_root,+ crate_fingerprint=crate.fingerprint,+ engine_version=get_package_version(),+ kind=CodegenKind.TYPES,+ target=target,+ )result: dict[str, Any] = {"success": True,"kind": str(CodegenKind.TYPES),diff --git a/pipelex/cli/agent_cli/commands/init_cmd.py b/pipelex/cli/agent_cli/commands/init_cmd.pyindex 2151fa938..513b1160d 100644--- a/pipelex/cli/agent_cli/commands/init_cmd.py+++ b/pipelex/cli/agent_cli/commands/init_cmd.py@@ -16,6 +16,7 @@ from pipelex.cli.agent_cli.commands.agent_output import ()from pipelex.cli.commands.init.backends import get_selected_backend_keys, update_backends_in_tomlfrom pipelex.cli.commands.init.config_files import init_config+from pipelex.cli.commands.init.init_usage_stats import note_init_backends, note_init_executionfrom pipelex.cli.commands.init.setup_path import describe_project_execution_shadow, find_project_execution_shadow, write_run_executionfrom pipelex.cli.commands.init.ui.backends_ui import get_backend_options_from_tomlfrom pipelex.cli.commands.login.api_key_store import find_pipelex_api_key@@ -434,6 +435,7 @@ def agent_init_cmd(# Step 1.7: Write where runs executewrite_run_execution(pipelex_toml_path=target_dir / "pipelex.toml", execution=execution)+ note_init_execution(execution=execution)result_payload: dict[str, Any] = {"success": True,@@ -453,6 +455,7 @@ def agent_init_cmd(backends_enabled = _configure_backends(config=parsed_config, backends_toml_path=backends_toml_path, template_backends_path=template_backends_path)+ note_init_backends(backend_keys=backends_enabled)# Step 3: Configure routingrouting_profile = _configure_routing(backends_enabled, config=parsed_config, target_dir=target_dir)diff --git a/pipelex/cli/agent_cli/commands/plxt_passthrough.py b/pipelex/cli/agent_cli/commands/plxt_passthrough.pyindex a1c530d11..289adc65f 100644--- a/pipelex/cli/agent_cli/commands/plxt_passthrough.py+++ b/pipelex/cli/agent_cli/commands/plxt_passthrough.py@@ -1,11 +1,13 @@"""Passthrough helper for delegating to the plxt CLI binary."""+import osimport shutilimport subprocess # ruff: ignore[suspicious-subprocess-import]import typerfrom pipelex.cli.agent_cli.commands.agent_output import agent_error+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsEnvVardef run_plxt(subcommand: str, *, file_path: str) -> None:@@ -25,8 +27,10 @@ def run_plxt(subcommand: str, *, file_path: str) -> None:error_type="BinaryNotFoundError",)+ # The command's usage statistics are pipelex-agent's own, so the plxt it delegates to reports none of its own.result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true][plxt_path, subcommand, file_path],check=False,+ env={**os.environ, UsageStatsEnvVar.TOOL_SWITCH: "0"},)raise typer.Exit(result.returncode)diff --git a/pipelex/cli/agent_cli/commands/run/_run_core.py b/pipelex/cli/agent_cli/commands/run/_run_core.pyindex 4f11b537d..604969086 100644--- a/pipelex/cli/agent_cli/commands/run/_run_core.py+++ b/pipelex/cli/agent_cli/commands/run/_run_core.py@@ -17,6 +17,8 @@ from pipelex.core.memory.absence_render import build_absence_html, build_absencefrom pipelex.graph.graph_factory import generate_graph_outputs, save_graph_outputs_to_dirfrom pipelex.pipeline.runner import PipelexMTHDSProtocolfrom pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsMethodSource+from pipelex.system.usage_stats.usage_stats_runs import method_source_scopefrom pipelex.tools.log.error_fields import error_fieldsfrom pipelex.tools.misc.json_utils import clean_json_dumps@@ -102,11 +104,13 @@ async def run_pipeline_core(# The directories a CLI loads are the caller's own: a refusal there is their invalid bundle.library_dirs_are_callers=True,)- response = await runner.execute(- pipe_code=pipe_code,- mthds_contents=mthds_contents,- inputs=inputs,- )+ # The CLI reads every bundle it runs from the caller's files; a method it resolved declared its own source first.+ with method_source_scope(source=UsageStatsMethodSource.LOCAL_BUNDLE):+ response = await runner.execute(+ pipe_code=pipe_code,+ mthds_contents=mthds_contents,+ inputs=inputs,+ )pipe_output = response.pipe_output# A completed run always resolves its declared output: a value or a recorded absence. Andiff --git a/pipelex/cli/agent_cli/commands/validate/_validate_core.py b/pipelex/cli/agent_cli/commands/validate/_validate_core.pyindex 2a04aeff9..a0adc6206 100644--- a/pipelex/cli/agent_cli/commands/validate/_validate_core.py+++ b/pipelex/cli/agent_cli/commands/validate/_validate_core.py@@ -8,6 +8,7 @@ from pipelex.interpreter_hub import (clear_current_library,get_current_library_id_or_none,get_library_manager,+ get_own_pipes,get_pipe_library,get_required_entry_pipe,resolve_library_dirs,@@ -17,6 +18,7 @@ from pipelex.pipeline.advisory_warnings import collect_advisory_warnings, collecfrom pipelex.pipeline.blueprint_selection import collect_entry_pipe_refsfrom pipelex.pipeline.bundle_validator import BundleValidatorfrom pipelex.pipeline.execution_seams import acquire_library+from pipelex.pipeline.usage_stats_reporting import MethodValidationReportfrom pipelex.pipeline.validate_bundle import build_pending_signatures, build_validated_pipes, validate_bundlefrom pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_error@@ -42,54 +44,57 @@ async def validate_all_core(*, library_dirs: list[Path] | None = None, allow_sig# teardown) inline rather than calling it, because `validate all` is strict-by-default on# signatures and must read the LIBRARY-WIDE pending set BEFORE teardown — acquire_and_validate# returns only the per-pipe status map and tears the library down before we could compute it.- prev_library_id = get_current_library_id_or_none()- # A refusal of the libraries while loading them is the invalid verdict, through the shared- # bundle-loading cascade, as on `validate bundle`. The directories are the caller's own, so- # acquire_library translates their load itself, while the library is still current: the translation- # reads the library's pipe sources to locate an item, and a translation around the call would run- # only after acquire_library had torn the failed library down, leaving the item without its file.- # The sweep below stays untranslated on purpose: a failing dry run is answered as the DryRunError it- # is, as on `validate pipe`.- acquired_id, _ = acquire_library(- library_id="",- library_dirs=[str(library_dir) for library_dir in library_dirs] if library_dirs else None,- library_dirs_are_callers=True,- )- try:- # acquire_library left the freshly-acquired library current, so the inner sweep targets it- # (it filters signatures in strict mode itself). The returned map is keyed by namespaced pipe_ref.- # A pipe whose dry run fails is the invalid verdict, one located dry_run item per failing pipe,- # through the same cascade and item builder as `validate bundle`.- with translate_to_validate_bundle_error():- dry_run_results = await BundleValidator().validate_current_library(allow_signatures=allow_signatures)+ with MethodValidationReport() as validation_report:+ prev_library_id = get_current_library_id_or_none()+ # A refusal of the libraries while loading them is the invalid verdict, through the shared+ # bundle-loading cascade, as on `validate bundle`. The directories are the caller's own, so+ # acquire_library translates their load itself, while the library is still current: the translation+ # reads the library's pipe sources to locate an item, and a translation around the call would run+ # only after acquire_library had torn the failed library down, leaving the item without its file.+ # The sweep below stays untranslated on purpose: a failing dry run is answered as the DryRunError it+ # is, as on `validate pipe`.+ acquired_id, _ = acquire_library(+ library_id="",+ library_dirs=[str(library_dir) for library_dir in library_dirs] if library_dirs else None,+ library_dirs_are_callers=True,+ )+ try:+ validation_report.describe(pipes=get_own_pipes())+ # acquire_library left the freshly-acquired library current, so the inner sweep targets it+ # (it filters signatures in strict mode itself). The returned map is keyed by namespaced pipe_ref.+ # A pipe whose dry run fails is the invalid verdict, one located dry_run item per failing pipe,+ # through the same cascade and item builder as `validate bundle`.+ with translate_to_validate_bundle_error():+ dry_run_results = await BundleValidator().validate_current_library(allow_signatures=allow_signatures)- # pending_signatures is the library-wide set of still-unimplemented forward declarations;- # is_runnable = not pending. `validate all` now makes a strict runnability claim (the consumer- # gates the exit code on it unless --allow-signatures), so both keys ride the envelope.- pending_signatures = build_pending_signatures(get_pipe_library().get_pipes_dict())- # warnings: same advisory lint channel as the bundle surfaces — `validate all` has the- # whole library loaded, which is all the flow context the cross-flow aggregation needs.- # Its entry pipes come off the blueprints the library manager accumulated on the way in;- # there is no `ValidateBundleResult` on this path to read a `main_pipe` from.- library_pipes = list(get_pipe_library().get_pipes_dict().values())- warnings = collect_advisory_warnings(pipes=library_pipes, entry_pipe_refs=collect_current_library_entry_pipe_refs())- return {- "success": True,- "is_valid": True,- "validated_pipes": build_validated_pipes(dry_run_results),- "total_pipes": len(dry_run_results),- "pending_signatures": pending_signatures,- "is_runnable": not pending_signatures,- "warnings": [warning.model_dump(exclude_none=True) for warning in warnings],- }- finally:- # Restore the caller's outer current-library FIRST (so the guarantee survives a teardown- # raise), then tear the acquired library down — mirroring acquire_and_validate / validate_pipe_core.- if prev_library_id is not None:- set_current_library(library_id=prev_library_id)- else:- clear_current_library()- get_library_manager().teardown(library_id=acquired_id)+ # pending_signatures is the library-wide set of still-unimplemented forward declarations;+ # is_runnable = not pending. `validate all` now makes a strict runnability claim (the consumer+ # gates the exit code on it unless --allow-signatures), so both keys ride the envelope.+ pending_signatures = build_pending_signatures(get_pipe_library().get_pipes_dict())+ # warnings: same advisory lint channel as the bundle surfaces — `validate all` has the+ # whole library loaded, which is all the flow context the cross-flow aggregation needs.+ # Its entry pipes come off the blueprints the library manager accumulated on the way in;+ # there is no `ValidateBundleResult` on this path to read a `main_pipe` from.+ library_pipes = list(get_pipe_library().get_pipes_dict().values())+ warnings = collect_advisory_warnings(pipes=library_pipes, entry_pipe_refs=collect_current_library_entry_pipe_refs())+ validation_report.count_warnings(count=len(warnings))+ return {+ "success": True,+ "is_valid": True,+ "validated_pipes": build_validated_pipes(dry_run_results),+ "total_pipes": len(dry_run_results),+ "pending_signatures": pending_signatures,+ "is_runnable": not pending_signatures,+ "warnings": [warning.model_dump(exclude_none=True) for warning in warnings],+ }+ finally:+ # Restore the caller's outer current-library FIRST (so the guarantee survives a teardown+ # raise), then tear the acquired library down — mirroring acquire_and_validate / validate_pipe_core.+ if prev_library_id is not None:+ set_current_library(library_id=prev_library_id)+ else:+ clear_current_library()+ get_library_manager().teardown(library_id=acquired_id)async def validate_bundle_core(@@ -112,11 +117,15 @@ async def validate_bundle_core(Raises:ValidateBundleError: If validation fails."""- result = await validate_bundle(- mthds_file_path=bundle_path,- library_dirs=library_dirs,- allow_signatures=allow_signatures,- )+ with MethodValidationReport() as validation_report:+ result = await validate_bundle(+ mthds_file_path=bundle_path,+ library_dirs=library_dirs,+ allow_signatures=allow_signatures,+ )+ warnings = collect_advisory_warnings(pipes=result.pipes, entry_pipe_refs=collect_entry_pipe_refs(result.blueprints))+ validation_report.describe(pipes=result.pipes)+ validation_report.count_warnings(count=len(warnings))# Consume the real per-pipe status from the sweep — a fixed all-"SUCCESS" list would hide allowed# failures and SKIPPED cross-package pipes the dry-run actually recorded (C-8). pending_signatures@@ -131,10 +140,7 @@ async def validate_bundle_core("total_pipes": len(result.dry_run_result),"pending_signatures": result.pending_signatures,"is_runnable": not result.pending_signatures,- "warnings": [- warning.model_dump(exclude_none=True)- for warning in collect_advisory_warnings(pipes=result.pipes, entry_pipe_refs=collect_entry_pipe_refs(result.blueprints))- ],+ "warnings": [warning.model_dump(exclude_none=True) for warning in warnings],}@@ -158,44 +164,46 @@ async def validate_pipe_core(Raises:ValidateBundleError: If validation fails."""- library_manager = get_library_manager()- # Capture the caller's outer current-library so it can be restored after this temporary validation- # library is torn down.- prev_library_id = get_current_library_id_or_none()- library_id, _ = library_manager.open_library()- try:- set_current_library(library_id=library_id)- effective_dirs, _ = resolve_library_dirs(library_dirs)+ with MethodValidationReport() as validation_report:+ library_manager = get_library_manager()+ # Capture the caller's outer current-library so it can be restored after this temporary validation+ # library is torn down.+ prev_library_id = get_current_library_id_or_none()+ library_id, _ = library_manager.open_library()+ try:+ set_current_library(library_id=library_id)+ effective_dirs, _ = resolve_library_dirs(library_dirs)- if effective_dirs:- # A refusal of the libraries while loading them is the invalid verdict, through the shared- # bundle-loading cascade, as on `validate bundle`.- with translate_to_validate_bundle_error():- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)+ if effective_dirs:+ # A refusal of the libraries while loading them is the invalid verdict, through the shared+ # bundle-loading cascade, as on `validate bundle`.+ with translate_to_validate_bundle_error():+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)- # The entry-pipe lookup stays outside the cascade: a code naming no pipe is no verdict. A pipe whose- # dry run fails is the invalid verdict, one located dry_run item per failing pipe, as on `validate bundle`.- the_pipe = get_required_entry_pipe(pipe_code=pipe_code)- with translate_to_validate_bundle_error():- dry_run_results = await BundleValidator().validate_pipes(pipes=[the_pipe], library_id=library_id, allow_signatures=allow_signatures)+ # The entry-pipe lookup stays outside the cascade: a code naming no pipe is no verdict. A pipe whose+ # dry run fails is the invalid verdict, one located dry_run item per failing pipe, as on `validate bundle`.+ the_pipe = get_required_entry_pipe(pipe_code=pipe_code)+ validation_report.describe(pipes=get_own_pipes())+ with translate_to_validate_bundle_error():+ dry_run_results = await BundleValidator().validate_pipes(pipes=[the_pipe], library_id=library_id, allow_signatures=allow_signatures)- return {- "success": True,- "is_valid": True,- "validated_pipes": build_validated_pipes(dry_run_results),- "total_pipes": len(dry_run_results),- }- finally:- # validate_pipes is the D6 inner sweep and never tears the library down, and validate_pipe_core- # does not need the library after returning its results dict — so this caller owns the full- # lifecycle on BOTH paths. Restore the outer current-library FIRST (so the guarantee survives a- # teardown raise), then tear the temporary library down. set_current_library cannot take None,- # so route the "no outer was set" case through clear_current_library.- if prev_library_id is not None:- set_current_library(library_id=prev_library_id)- else:- clear_current_library()- library_manager.teardown(library_id=library_id)+ return {+ "success": True,+ "is_valid": True,+ "validated_pipes": build_validated_pipes(dry_run_results),+ "total_pipes": len(dry_run_results),+ }+ finally:+ # validate_pipes is the D6 inner sweep and never tears the library down, and validate_pipe_core+ # does not need the library after returning its results dict — so this caller owns the full+ # lifecycle on BOTH paths. Restore the outer current-library FIRST (so the guarantee survives a+ # teardown raise), then tear the temporary library down. set_current_library cannot take None,+ # so route the "no outer was set" case through clear_current_library.+ if prev_library_id is not None:+ set_current_library(library_id=prev_library_id)+ else:+ clear_current_library()+ library_manager.teardown(library_id=library_id)async def validate_pipe_in_bundle_core(@@ -232,12 +240,16 @@ async def validate_pipe_in_bundle_core(selector makes it INPUT-domained; it is a `PipeNotFoundError` subclass, so an existing`except PipeNotFoundError` still catches it."""- result = await validate_bundle(- mthds_file_path=bundle_path,- library_dirs=library_dirs,- allow_signatures=allow_signatures,- dry_run_pipe_codes=[pipe_code],- )+ with MethodValidationReport() as validation_report:+ result = await validate_bundle(+ mthds_file_path=bundle_path,+ library_dirs=library_dirs,+ allow_signatures=allow_signatures,+ dry_run_pipe_codes=[pipe_code],+ )+ warnings = collect_advisory_warnings(pipes=result.pipes, entry_pipe_refs=collect_entry_pipe_refs(result.blueprints))+ validation_report.describe(pipes=result.pipes)+ validation_report.count_warnings(count=len(warnings))# Consume the real per-pipe status from the sliced sweep — a fixed "SUCCESS" would hide an allowed# failure or a cross-package SKIPPED the dry-run actually recorded (C-8). pending_signatures is the@@ -251,8 +263,5 @@ async def validate_pipe_in_bundle_core("total_pipes": len(result.dry_run_result),"pending_signatures": result.pending_signatures,"is_runnable": not result.pending_signatures,- "warnings": [- warning.model_dump(exclude_none=True)- for warning in collect_advisory_warnings(pipes=result.pipes, entry_pipe_refs=collect_entry_pipe_refs(result.blueprints))- ],+ "warnings": [warning.model_dump(exclude_none=True) for warning in warnings],}diff --git a/pipelex/cli/commands/build/structures_cmd.py b/pipelex/cli/commands/build/structures_cmd.pyindex d13ba3b6f..177c190e1 100644--- a/pipelex/cli/commands/build/structures_cmd.py+++ b/pipelex/cli/commands/build/structures_cmd.py@@ -11,8 +11,10 @@ from typing import Annotatedimport typer-from pipelex.cli.commands.codegen.types_cmd import codegen_types_cmd+from pipelex.cli.commands.codegen.types_cmd import generate_typesfrom pipelex.codegen.emitters.target import CodegenTarget+from pipelex.pipeline.usage_stats_reporting import CodeGenerationReport+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifactSUB_COMMAND_STRUCTURES = "structures"@@ -52,9 +54,10 @@ def build_structures_command(raise typer.Exit(2)closure_dir = target_path.parent if target_path.is_file() else target_path- codegen_types_cmd(+ generate_types(target=CodegenTarget.PYTHON_STRUCTURES,paths=[closure_dir],output_dir=output_dir or str(closure_dir / "structures"),library_dir=[Path(lib_dir) for lib_dir in library_dir] if library_dir else None,+ generation_report=CodeGenerationReport(artifact=UsageStatsCodegenArtifact.STRUCTURES),)diff --git a/pipelex/cli/commands/codegen/inputs_cmd.py b/pipelex/cli/commands/codegen/inputs_cmd.pyindex 28b22cfbc..7e1db7c76 100644--- a/pipelex/cli/commands/codegen/inputs_cmd.py+++ b/pipelex/cli/commands/codegen/inputs_cmd.py@@ -24,9 +24,11 @@ from pipelex.pipe_machinery.pipe_abstract import PipeAbstractfrom pipelex.pipe_machinery.rendering.input_renderer import InputsTemplateFormat, NoInputsRequiredError, render_inputs, render_inputs_tomlfrom pipelex.pipelex import Pipelexfrom pipelex.pipeline.default_file_names import DEFAULT_INPUTS_FILE_NAME, DEFAULT_INPUTS_TOML_FILE_NAME+from pipelex.pipeline.usage_stats_reporting import CodeGenerationReportfrom pipelex.runtime_hub import get_telemetry_managerfrom pipelex.system.runtime import IntegrationModefrom pipelex.system.telemetry.events import EventProperty+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifactfrom pipelex.tools.misc.file_utils import ensure_directory_for_file_path, save_text_to_pathfrom pipelex.tools.misc.package_utils import get_package_version@@ -64,13 +66,14 @@ def codegen_inputs_cmd(make_pipelex_for_cli(context=ErrorContext.VALIDATION, needs_inference=False)try:- with get_telemetry_manager().telemetry_context():+ with get_telemetry_manager().telemetry_context(), CodeGenerationReport(artifact=UsageStatsCodegenArtifact.INPUTS) as generation_report:tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND}")combined_dirs: list[Path] = [*(paths or []), *(library_dir or [])]crate = load_normalized_crate_or_exit(library_dirs=combined_dirs or None)+ generation_report.describe(crate=crate)pipe_ref = pipe or _default_main_pipe_ref(crate=crate)try:@@ -83,6 +86,8 @@ def codegen_inputs_cmd(content, default_file_name = _render(the_pipe=the_pipe, template_format=template_format, explicit=explicit)except NoInputsRequiredError as exc:typer.secho(str(exc), fg=typer.colors.YELLOW)+ # A pipe with no inputs has none to template: nothing failed.+ generation_report.declare_succeeded()raise typer.Exit(0) from excdestination = Path(output).expanduser() if output else Path("results") / default_file_namediff --git a/pipelex/cli/commands/codegen/types_cmd.py b/pipelex/cli/commands/codegen/types_cmd.pyindex 27d7a06d5..23ad05431 100644--- a/pipelex/cli/commands/codegen/types_cmd.py+++ b/pipelex/cli/commands/codegen/types_cmd.py@@ -21,9 +21,11 @@ from pipelex.codegen.emission import write_stamped_projectionfrom pipelex.codegen.emitters.target import CodegenKind, CodegenTargetfrom pipelex.codegen.emitters.types_emitter import emit_typesfrom pipelex.pipelex import Pipelex+from pipelex.pipeline.usage_stats_reporting import CodeGenerationReport, usage_stats_codegen_targetfrom pipelex.runtime_hub import get_telemetry_managerfrom pipelex.system.runtime import IntegrationModefrom pipelex.system.telemetry.events import EventProperty+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifactfrom pipelex.tools.misc.package_utils import get_package_versionCOMMAND = "codegen"@@ -49,17 +51,39 @@ def codegen_types_cmd(] = None,) -> None:"""Project the crate's concept set into typed artifacts for the chosen target."""+ generate_types(+ target=target,+ paths=paths,+ output_dir=output_dir,+ library_dir=library_dir,+ generation_report=CodeGenerationReport(artifact=UsageStatsCodegenArtifact.TYPES, target=usage_stats_codegen_target(target=target)),+ )+++def generate_types(+ *,+ target: CodegenTarget,+ paths: list[Path] | None,+ output_dir: str,+ library_dir: list[Path] | None,+ generation_report: CodeGenerationReport,+) -> None:+ """Load the closure, project its concept set for `target` and write the stamped files, timed by `generation_report`.++ The body of `codegen types` and of its alias `build structures`, which report the generation as different artifacts.+ """make_pipelex_for_cli(context=ErrorContext.VALIDATION, needs_inference=False)output_root = Path(output_dir).expanduser()try:- with get_telemetry_manager().telemetry_context():+ with get_telemetry_manager().telemetry_context(), generation_report:tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND}")combined_dirs: list[Path] = [*(paths or []), *(library_dir or [])]crate = load_normalized_crate_or_exit(library_dirs=combined_dirs or None)+ generation_report.describe(crate=crate)emitted = emit_types(crate, target=target)report = write_stamped_projection(diff --git a/pipelex/cli/commands/init/command.py b/pipelex/cli/commands/init/command.pyindex 7282c6937..4747c1b9d 100644--- a/pipelex/cli/commands/init/command.py+++ b/pipelex/cli/commands/init/command.py@@ -32,6 +32,7 @@ from pipelex.cli.commands.init.backends import (from pipelex.cli.commands.init.config_files import init_configfrom pipelex.cli.commands.init.credentials import prompt_credentialsfrom pipelex.cli.commands.init.ide_extension import suggest_extension_install_if_needed+from pipelex.cli.commands.init.init_usage_stats import note_init_backends, note_init_executionfrom pipelex.cli.commands.init.routing import customize_routing_profilefrom pipelex.cli.commands.init.setup_path import (DEFAULT_SETUP_PATH,@@ -528,10 +529,12 @@ def execute_initialization(*, console: Console, inspection: InitInspection, choipipelex_toml_path=inspection.pipelex_toml_path,project_config_dir=config_manager.project_config_dir,)+ note_init_execution(execution=setup_path.run_execution)if choices.interactive:_suggest_extension(console=console)case SetupPath.LOCAL | None:customize_backends_config(is_first_time_setup=first_time_setup, target_config_dir=target_config_dir)+ note_init_backends(backend_keys=get_selected_backend_keys(backends_toml_path))# Automatically set up routing after backends (unless routing is the specific focus)if not inspection.check_routing:@@ -547,6 +550,7 @@ def execute_initialization(*, console: Console, inspection: InitInspection, choipipelex_toml_path=inspection.pipelex_toml_path,project_config_dir=config_manager.project_config_dir,)+ note_init_execution(execution=setup_path.run_execution)# Step 2.5: Prompt for missing credentials, which only runs on this machine useruns_hosted = (setup_path is not None and setup_path.is_hosted) or inspection.keeps_hosted_runsdiff --git a/pipelex/cli/commands/init/init_usage_stats.py b/pipelex/cli/commands/init/init_usage_stats.pynew file mode 100644index 000000000..fca415ba8--- /dev/null+++ b/pipelex/cli/commands/init/init_usage_stats.py@@ -0,0 +1,30 @@+"""What an `init` command set, noted for its `command_completed`: where runs execute and which backends it enabled.++Both `pipelex init` and `pipelex-agent init` note each choice once the file holding it is written, so a setup that+stops partway reports only what it set. Backend keys are reported by the provider names `model_used` uses, a backend+pipelex does not ship counting as `custom`.+"""++from collections.abc import Iterable++from pipelex.hosted.run_config import RunExecution+from pipelex.system.usage_stats.usage_stats_client import note_command_init_choices+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsInitExecution, UsageStatsProvider+from pipelex.system.usage_stats.usage_stats_events import InitChoices+++def note_init_execution(*, execution: RunExecution) -> None:+ """Note where this `init` set runs to execute."""+ init_execution: UsageStatsInitExecution+ match execution:+ case RunExecution.LOCAL:+ init_execution = UsageStatsInitExecution.LOCAL+ case RunExecution.HOSTED:+ init_execution = UsageStatsInitExecution.HOSTED+ note_command_init_choices(init_choices=InitChoices(execution=init_execution))+++def note_init_backends(*, backend_keys: Iterable[str]) -> None:+ """Note the backends this `init` left enabled, every other one counting as not enabled."""+ providers = frozenset(UsageStatsProvider.for_backend(backend_name=backend_key) for backend_key in backend_keys)+ note_command_init_choices(init_choices=InitChoices(enabled_providers=providers))diff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.pyindex 54d326f5b..f812f4b76 100644--- a/pipelex/cli/commands/run/_run_core.py+++ b/pipelex/cli/commands/run/_run_core.py@@ -46,6 +46,8 @@ from pipelex.runtime_hub import get_console, get_telemetry_managerfrom pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.runtime import IntegrationModefrom pipelex.system.telemetry.events import EventProperty+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsMethodSource+from pipelex.system.usage_stats.usage_stats_runs import method_source_scopefrom pipelex.tools.misc.exceptions import JsonTypeError, TomlErrorfrom pipelex.tools.misc.file_utils import get_incremental_directory_pathfrom pipelex.tools.misc.json_utils import save_as_json_to_path@@ -266,12 +268,14 @@ async def _execute_run(# The directories a CLI loads are the caller's own: a refusal there is their invalid bundle.library_dirs_are_callers=True,)- response = await runner.execute(- pipe_code=pipe_code,- mthds_contents=[mthds_content] if mthds_content else None,- inputs=pipeline_inputs,- dynamic_output_concept_ref=dynamic_output_concept_ref,- )+ # The CLI reads every bundle it runs from the caller's files; a method it resolved declared its own source first.+ with method_source_scope(source=UsageStatsMethodSource.LOCAL_BUNDLE):+ response = await runner.execute(+ pipe_code=pipe_code,+ mthds_contents=[mthds_content] if mthds_content else None,+ inputs=pipeline_inputs,+ dynamic_output_concept_ref=dynamic_output_concept_ref,+ )pipe_output = response.pipe_outputexcept ValidateBundleError as exc:# The run refused the bundle while loading it, before any pipe ran.diff --git a/pipelex/cli/commands/usage_stats_cmd.py b/pipelex/cli/commands/usage_stats_cmd.pynew file mode 100644index 000000000..b3723c64f--- /dev/null+++ b/pipelex/cli/commands/usage_stats_cmd.py@@ -0,0 +1,115 @@+"""`pipelex usage-stats`: see and change the anonymous usage statistics of every Pipelex tool on this machine.++The group's commands act on the shared state every reporting tool reads (`~/.mthds/usage-stats/`), so `disable` here+turns statistics off for `plxt`, the VS Code extension and the `mthds` CLI too. They never send, queue or print the+notice, and exit 0 unless they could not do what they say. The group also serves `pipelex-agent`, which has none of its+own. The contract is `conformance/specs/usage-statistics.md`, "Commands".+"""++import os+from enum import StrEnum+from pathlib import Path+from typing import Annotated, NoReturn++import typer++from pipelex.system.usage_stats.usage_stats_client import is_pytest_process+from pipelex.system.usage_stats.usage_stats_contract import NOTICE_DETAILS_URL+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState+from pipelex.system.usage_stats.usage_stats_status import describe_off_by, read_usage_stats_status+from pipelex.system.usage_stats.usage_stats_switches import read_switches++usage_stats_app = typer.Typer(no_args_is_help=True)+++class UsageStatsStatusFormat(StrEnum):+ TEXT = "text"+ JSON = "json"++ @property+ def is_json(self) -> bool:+ match self:+ case UsageStatsStatusFormat.JSON:+ return True+ case UsageStatsStatusFormat.TEXT:+ return False+++def _current_dir(*, state: UsageStatsState) -> Path:+ try:+ return Path.cwd()+ except OSError:+ # A working directory deleted under the process: no project around it, so no project setting.+ return state.home+++def _fail(*, message: str) -> NoReturn:+ typer.echo(message, err=True)+ raise typer.Exit(1)+++@usage_stats_app.command(name="status", help="Show whether usage statistics are on, what turns them off, and the installation identifier")+def status_command(+ output_format: Annotated[+ UsageStatsStatusFormat,+ typer.Option("--format", help="Output format: text (default) or json, the status object other tools read"),+ ] = UsageStatsStatusFormat.TEXT,+) -> None:+ """Show the statistics status. Read-only: it creates nothing, migrates nothing and prints no notice."""+ state = UsageStatsState.make_for_current_home()+ status = read_usage_stats_status(state=state, env=os.environ, cwd=_current_dir(state=state), is_pytest=is_pytest_process())+ if output_format.is_json:+ typer.echo(status.model_dump_json())+ return+ if status.enabled:+ typer.echo("Usage statistics are on.")+ else:+ typer.echo("Usage statistics are off, turned off by:")+ for token in status.off_by:+ typer.echo(f" - {describe_off_by(token=token)} ({token})")+ typer.echo(f"Installation identifier: {status.installation_id or 'none yet'}")+ typer.echo(f"Notice version seen on this machine: {status.notice_version}")+ if status.internal:+ typer.echo("This machine is marked as one of the Pipelex team's.")+ typer.echo(f"In CI: {'yes' if status.ci else 'no'}; in a container: {'yes' if status.container else 'no'}")+ typer.echo(f"Shared state: {status.state_dir}")+ typer.echo(f"Details: {NOTICE_DETAILS_URL}")+++@usage_stats_app.command(name="disable", help="Turn usage statistics off on this machine, for every Pipelex tool")+def disable_command() -> None:+ """Create the shared `disabled` file and delete every event any tool has queued."""+ state = UsageStatsState.make_for_current_home()+ try:+ state.disable()+ except OSError as os_error:+ _fail(message=f"Usage statistics could not be turned off: {state.disabled_file} could not be written ({os_error.strerror}).")+ typer.echo("Usage statistics are off on this machine, for every Pipelex tool, and every queued event was deleted.")+++@usage_stats_app.command(name="enable", help="Turn usage statistics back on on this machine, and name any other switch that keeps them off")+def enable_command() -> None:+ """Delete the shared `disabled` file, set `legacy-migrated`, and name every switch that still says off."""+ state = UsageStatsState.make_for_current_home()+ try:+ state.enable()+ except OSError as os_error:+ _fail(message=f"Usage statistics could not be turned on: the shared state in {state.state_dir} could not be written ({os_error.strerror}).")+ still_off = read_switches(state=state, env=os.environ, cwd=_current_dir(state=state), is_pytest=is_pytest_process())+ if not still_off:+ typer.echo("Usage statistics are on on this machine.")+ return+ typer.echo("This machine's choice is now on, but usage statistics stay off, turned off by:")+ for token in still_off:+ typer.echo(f" - {describe_off_by(token=token)} ({token})")+++@usage_stats_app.command(name="reset-id", help="Replace this machine's anonymous installation identifier with a new one")+def reset_id_command() -> None:+ """Delete every queued event and the identifier, create a new identifier and print it."""+ state = UsageStatsState.make_for_current_home()+ try:+ new_id = state.replace_installation_id()+ except OSError as os_error:+ _fail(message=f"The installation identifier could not be replaced in {state.state_dir} ({os_error.strerror}).")+ typer.echo(new_id)diff --git a/pipelex/cli/commands/validate/_validate_core.py b/pipelex/cli/commands/validate/_validate_core.pyindex 27c343323..fbdc9ab24 100644--- a/pipelex/cli/commands/validate/_validate_core.py+++ b/pipelex/cli/commands/validate/_validate_core.py@@ -15,6 +15,7 @@ from pipelex.cli.error_handlers import ()from pipelex.interpreter_hub import (get_library_manager,+ get_own_pipes,get_pipe_library,get_pipes,get_required_entry_pipe,@@ -30,6 +31,7 @@ from pipelex.pipeline.blueprint_selection import collect_entry_pipe_refsfrom pipelex.pipeline.bundle_validator import BundleValidatorfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.execution_seams import load_libraries_and_activate+from pipelex.pipeline.usage_stats_reporting import MethodValidationReportfrom pipelex.pipeline.validate_bundle import build_pending_signatures, validate_bundlefrom pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_errorfrom pipelex.runtime_hub import get_console, get_telemetry_manager@@ -46,15 +48,17 @@ if TYPE_CHECKING:COMMAND = "validate"-def _echo_advisory_warnings(*, pipes: list[PipeAbstract], entry_pipe_refs: list[str]) -> None:- """Render every advisory lint family in yellow, in the composition point's fixed order.+def _echo_advisory_warnings(*, pipes: list[PipeAbstract], entry_pipe_refs: list[str]) -> int:+ """Render every advisory lint family in yellow, in the composition point's fixed order, and return how many there were.Advisory only — a warning never changes the validation verdict or the exit code. Requires thevalidation library to still be loaded: the taint walk resolves child pipes via the hub, and thedescriptors and the hint lint read the library's accumulated crate."""- for warning in collect_advisory_warnings(pipes=pipes, entry_pipe_refs=entry_pipe_refs):+ warnings = collect_advisory_warnings(pipes=pipes, entry_pipe_refs=entry_pipe_refs)+ for warning in warnings:typer.secho(f"Warning: {warning.message}", fg=typer.colors.YELLOW)+ return len(warnings)def _format_signatures_summary_suffix(*, signature_count: int) -> str:@@ -70,153 +74,165 @@ def _format_signatures_summary_suffix(*, signature_count: int) -> str:def do_validate_all_libraries_and_dry_run(*, library_dirs: list[Path] | None = None, allow_signatures: bool = False) -> None:- try:- with get_telemetry_manager().telemetry_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")-- # Single public composer for the open/set/load ceremony — leaves the library loaded and- # current for the sweep below, owning the standard 3-tier dir resolution and load-failure- # teardown. No teardown on success here: the caller (validate_pipe_cmd) owns Pipelex teardown.- # The shared bundle-loading cascade turns a refusal of the libraries into the invalid- # verdict (exit 1) rendered below, exactly as `validate bundle` does.- with translate_to_validate_bundle_error():- load_libraries_and_activate(library_dirs)+ with MethodValidationReport() as validation_report:+ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=get_package_version())+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} all")- # The pipe list is needed only to render the "Validating N" line; validate_current_library- # re-derives its own sweep candidates from the current library (validate_pipes excludes- # signature pipes in strict mode). The count line mirrors that strict-mode exclusion.- all_pipes = get_pipes()- pipes = all_pipes if allow_signatures else [pipe for pipe in all_pipes if not pipe.is_signature]- if library_dirs:- dirs_str = ", ".join(f'"{lib_dir}"' for lib_dir in library_dirs)- typer.echo(f"Validating {count_with_noun(count=len(pipes), singular='pipe')} from: {dirs_str}")+ # Single public composer for the open/set/load ceremony — leaves the library loaded and+ # current for the sweep below, owning the standard 3-tier dir resolution and load-failure+ # teardown. No teardown on success here: the caller (validate_pipe_cmd) owns Pipelex teardown.+ # The shared bundle-loading cascade turns a refusal of the libraries into the invalid+ # verdict (exit 1) rendered below, exactly as `validate bundle` does.+ with translate_to_validate_bundle_error():+ load_libraries_and_activate(library_dirs)- # validate_current_library owns the static wiring pass and the single PIPE_DRY_RUN telemetry- # event — sweeping the library we just loaded, without teardown. A pipe whose dry run fails- # raises DryRunError, which the cascade turns into the same invalid verdict.- with translate_to_validate_bundle_error():- asyncio.run(BundleValidator().validate_current_library(allow_signatures=allow_signatures))+ # The pipe list is needed only to render the "Validating N" line; validate_current_library+ # re-derives its own sweep candidates from the current library (validate_pipes excludes+ # signature pipes in strict mode). The count line mirrors that strict-mode exclusion.+ all_pipes = get_pipes()+ validation_report.describe(pipes=get_own_pipes())+ pipes = all_pipes if allow_signatures else [pipe for pipe in all_pipes if not pipe.is_signature]+ if library_dirs:+ dirs_str = ", ".join(f'"{lib_dir}"' for lib_dir in library_dirs)+ typer.echo(f"Validating {count_with_noun(count=len(pipes), singular='pipe')} from: {dirs_str}")- # Advisory lints over the whole loaded library (the optionality lint's cross-flow- # aggregation needs every flow) — printed even when the signature gate below exits- # non-zero. The entry pipes come off the blueprints the library manager accumulated.- _echo_advisory_warnings(pipes=all_pipes, entry_pipe_refs=collect_current_library_entry_pipe_refs())+ # validate_current_library owns the static wiring pass and the single PIPE_DRY_RUN telemetry+ # event — sweeping the library we just loaded, without teardown. A pipe whose dry run fails+ # raises DryRunError, which the cascade turns into the same invalid verdict.+ with translate_to_validate_bundle_error():+ asyncio.run(BundleValidator().validate_current_library(allow_signatures=allow_signatures))- # Gate-from-report (D-B consumer-decides): signatures are never a validation error, but a- # library with unsatisfied PipeSignature placeholders is valid yet NOT runnable. `validate- # --all` is strict by default — it exits non-zero on pending signatures unless- # --allow-signatures tolerates them. The library is still loaded here (no teardown), so the- # library-wide pending set is read directly off it.- pending_signatures = build_pending_signatures(get_pipe_library().get_pipes_dict())- if pending_signatures and not allow_signatures:- pending = ", ".join(pending_signatures)- typer.secho(- f"Libraries are valid but NOT yet runnable — unimplemented PipeSignature placeholder(s): {pending}\n"- "Implement them, or re-run with --allow-signatures to accept placeholders.",- fg=typer.colors.RED,- err=True,+ # Advisory lints over the whole loaded library (the optionality lint's cross-flow+ # aggregation needs every flow) — printed even when the signature gate below exits+ # non-zero. The entry pipes come off the blueprints the library manager accumulated.+ validation_report.count_warnings(+ count=_echo_advisory_warnings(pipes=all_pipes, entry_pipe_refs=collect_current_library_entry_pipe_refs()))- raise typer.Exit(1)- signature_count = sum(1 for pipe in all_pipes if pipe.is_signature)- typer.echo(- f"Setup sequence passed OK, config and pipelines are validated.{_format_signatures_summary_suffix(signature_count=signature_count)}"- )- except ValidateBundleError as bundle_error:- # A produced negative verdict (exit 1): a refusal of the libraries while loading them, an unknown- # model among them, or a pipe whose dry run fails — rendered like any other invalid bundle.- handle_validate_bundle_error(bundle_error, library_dirs=library_dirs, allow_signatures=allow_signatures)- except PipeOperatorModelAvailabilityError as exc:- handle_model_availability_error(exc, context=ErrorContext.VALIDATION, exit_code=2)+ # Gate-from-report (D-B consumer-decides): signatures are never a validation error, but a+ # library with unsatisfied PipeSignature placeholders is valid yet NOT runnable. `validate+ # --all` is strict by default — it exits non-zero on pending signatures unless+ # --allow-signatures tolerates them. The library is still loaded here (no teardown), so the+ # library-wide pending set is read directly off it.+ pending_signatures = build_pending_signatures(get_pipe_library().get_pipes_dict())+ # Valid whatever the gate below decides: a library left unrunnable by its placeholders is still valid.+ validation_report.declare_valid()+ if pending_signatures and not allow_signatures:+ pending = ", ".join(pending_signatures)+ typer.secho(+ f"Libraries are valid but NOT yet runnable — unimplemented PipeSignature placeholder(s): {pending}\n"+ "Implement them, or re-run with --allow-signatures to accept placeholders.",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)++ signature_count = sum(1 for pipe in all_pipes if pipe.is_signature)+ summary_suffix = _format_signatures_summary_suffix(signature_count=signature_count)+ typer.echo(f"Setup sequence passed OK, config and pipelines are validated.{summary_suffix}")+ except ValidateBundleError as bundle_error:+ # A produced negative verdict (exit 1): a refusal of the libraries while loading them, an unknown+ # model among them, or a pipe whose dry run fails — rendered like any other invalid bundle.+ handle_validate_bundle_error(bundle_error, library_dirs=library_dirs, allow_signatures=allow_signatures)+ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.VALIDATION, exit_code=2)async def _validate_pipe_or_bundle(*, pipe_code: str | None = None, bundle_path: Path | None = None, library_dirs: list[Path] | None = None, allow_signatures: bool = False) -> None:"""Core async validation logic shared between method and pipe subcommands."""- if bundle_path:- try:- bundle_result = await validate_bundle(- mthds_file_path=bundle_path,- library_dirs=library_dirs,- allow_signatures=allow_signatures,- )- # Advisory lints, printed BEFORE the signature gate below exits non-zero — same ordering- # as `validate --all`. A method with an unimplemented placeholder is the state an author- # is most often in while building, and it is exactly then that the advisories help.- # validate_bundle leaves its validation library open on success, so the taint walk behind- # the lints can still resolve the bundle's pipes.- _echo_advisory_warnings(pipes=bundle_result.pipes, entry_pipe_refs=collect_entry_pipe_refs(bundle_result.blueprints))+ with MethodValidationReport() as validation_report:+ if bundle_path:+ try:+ bundle_result = await validate_bundle(+ mthds_file_path=bundle_path,+ library_dirs=library_dirs,+ allow_signatures=allow_signatures,+ )+ # Advisory lints, printed BEFORE the signature gate below exits non-zero — same ordering+ # as `validate --all`. A method with an unimplemented placeholder is the state an author+ # is most often in while building, and it is exactly then that the advisories help.+ # validate_bundle leaves its validation library open on success, so the taint walk behind+ # the lints can still resolve the bundle's pipes.+ validation_report.describe(pipes=bundle_result.pipes)+ validation_report.count_warnings(+ count=_echo_advisory_warnings(pipes=bundle_result.pipes, entry_pipe_refs=collect_entry_pipe_refs(bundle_result.blueprints))+ )+ # Valid whatever the gate below decides: a bundle left unrunnable by its placeholders is still valid.+ validation_report.declare_valid()- # Gate-from-report (D-B consumer-decides): signatures are never a validation error, but a- # bundle with unsatisfied PipeSignature placeholders is valid yet NOT runnable. The CLI- # exits non-zero on `not is_runnable` unless --allow-signatures tolerates the placeholders.- if bundle_result.pending_signatures and not allow_signatures:- pending = ", ".join(bundle_result.pending_signatures)+ # Gate-from-report (D-B consumer-decides): signatures are never a validation error, but a+ # bundle with unsatisfied PipeSignature placeholders is valid yet NOT runnable. The CLI+ # exits non-zero on `not is_runnable` unless --allow-signatures tolerates the placeholders.+ if bundle_result.pending_signatures and not allow_signatures:+ pending = ", ".join(bundle_result.pending_signatures)+ typer.secho(+ f"Bundle '{bundle_path}' is valid but NOT yet runnable — unimplemented PipeSignature placeholder(s): {pending}\n"+ "Implement them, or re-run with --allow-signatures to accept placeholders.",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)+ signature_count = sum(1 for pipe in bundle_result.pipes if pipe.is_signature)+ typer.secho(+ f"Successfully validated bundle '{bundle_path}'{_format_signatures_summary_suffix(signature_count=signature_count)}",+ fg=typer.colors.GREEN,+ )+ except FileNotFoundError as exc:+ get_console().print(Traceback())typer.secho(- f"Bundle '{bundle_path}' is valid but NOT yet runnable — unimplemented PipeSignature placeholder(s): {pending}\n"- "Implement them, or re-run with --allow-signatures to accept placeholders.",+ f"Failed to load bundle '{bundle_path}':",fg=typer.colors.RED,err=True,)- raise typer.Exit(1)- signature_count = sum(1 for pipe in bundle_result.pipes if pipe.is_signature)+ raise typer.Exit(2) from exc+ except ValidateBundleError as bundle_error:+ handle_validate_bundle_error(bundle_error, bundle_path=bundle_path, library_dirs=library_dirs, allow_signatures=allow_signatures)+ elif pipe_code:+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ set_current_library(library_id=library_id)+ effective_dirs, _ = resolve_library_dirs(library_dirs)++ try:+ # The load and the dry run go through the shared bundle-loading cascade, so a refusal of the+ # libraries and a pipe whose dry run fails are invalid verdicts (exit 1) rendered like any+ # other invalid bundle. The entry-pipe lookup between them stays outside it: a code that names+ # no pipe, or several, is an unresolvable target — no verdict, handled by execute_validate.+ if effective_dirs:+ with translate_to_validate_bundle_error():+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)++ pipe = get_required_entry_pipe(pipe_code=pipe_code)+ validation_report.describe(pipes=get_own_pipes())+ typer.echo(f"Validating pipe '{pipe_code}'...")+ # Signatures are never an error (D-B): a single-pipe validation reaching a PipeSignature+ # dry-runs trivially (the placeholder mints a mock). validate pipe makes no library-wide+ # runnability claim — pending_signatures is a bundle-surface fact — so there is no gate here.+ with translate_to_validate_bundle_error():+ await BundleValidator().validate_pipes(+ pipes=[pipe],+ library_id=library_id,+ allow_signatures=allow_signatures,+ )+ except ValidateBundleError as bundle_error:+ handle_validate_bundle_error(bundle_error, library_dirs=library_dirs, allow_signatures=allow_signatures)+ signature_count = len(collect_signature_refs(pipe=pipe))typer.secho(- f"Successfully validated bundle '{bundle_path}'{_format_signatures_summary_suffix(signature_count=signature_count)}",+ f"Successfully validated pipe '{pipe_code}'{_format_signatures_summary_suffix(signature_count=signature_count)}",fg=typer.colors.GREEN,)- except FileNotFoundError as exc:- get_console().print(Traceback())+ else:typer.secho(- f"Failed to load bundle '{bundle_path}':",+ "Failed to validate: no pipe code or bundle specified",fg=typer.colors.RED,err=True,)- raise typer.Exit(2) from exc- except ValidateBundleError as bundle_error:- handle_validate_bundle_error(bundle_error, bundle_path=bundle_path, library_dirs=library_dirs, allow_signatures=allow_signatures)- elif pipe_code:- library_manager = get_library_manager()- library_id, _ = library_manager.open_library()- set_current_library(library_id=library_id)- effective_dirs, _ = resolve_library_dirs(library_dirs)-- try:- # The load and the dry run go through the shared bundle-loading cascade, so a refusal of the- # libraries and a pipe whose dry run fails are invalid verdicts (exit 1) rendered like any- # other invalid bundle. The entry-pipe lookup between them stays outside it: a code that names- # no pipe, or several, is an unresolvable target — no verdict, handled by execute_validate.- if effective_dirs:- with translate_to_validate_bundle_error():- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)-- pipe = get_required_entry_pipe(pipe_code=pipe_code)- typer.echo(f"Validating pipe '{pipe_code}'...")- # Signatures are never an error (D-B): a single-pipe validation reaching a PipeSignature- # dry-runs trivially (the placeholder mints a mock). validate pipe makes no library-wide- # runnability claim — pending_signatures is a bundle-surface fact — so there is no gate here.- with translate_to_validate_bundle_error():- await BundleValidator().validate_pipes(- pipes=[pipe],- library_id=library_id,- allow_signatures=allow_signatures,- )- except ValidateBundleError as bundle_error:- handle_validate_bundle_error(bundle_error, library_dirs=library_dirs, allow_signatures=allow_signatures)- signature_count = len(collect_signature_refs(pipe=pipe))- typer.secho(- f"Successfully validated pipe '{pipe_code}'{_format_signatures_summary_suffix(signature_count=signature_count)}",- fg=typer.colors.GREEN,- )- else:- typer.secho(- "Failed to validate: no pipe code or bundle specified",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(2)+ raise typer.Exit(2)def execute_validate(diff --git a/pipelex/cli/commands/validate/app.py b/pipelex/cli/commands/validate/app.pyindex 309061423..33b6c2d37 100644--- a/pipelex/cli/commands/validate/app.py+++ b/pipelex/cli/commands/validate/app.py@@ -1,3 +1,5 @@+from collections.abc import Sequence+import clickimport typerimport typer.core@@ -15,29 +17,31 @@ class _ValidateGroup(typer.core.TyperGroup):or a pipe code), prepends 'pipe' so that:pipelex validate --all → pipelex validate pipe --allpipelex validate my_pipe → pipelex validate pipe my_pipe- """- @override- def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:- if not args:- return super().parse_args(ctx, args)+ The usage statistics read the same routing (`default_command_for`), so such an invocation counts as `validate pipe`.+ """- # Don't intercept --help- if "--help" in args or "-h" in args:- return super().parse_args(ctx, args)+ default_command_name = "pipe"- # Find the first non-option argument+ def default_command_for(self, *, args: Sequence[str]) -> str | None:+ """The subcommand `args` are routed to when they name none: 'pipe', or `None` when they name one or ask for help."""+ if not args or "--help" in args or "-h" in args:+ return None+ # The first non-option argument names the subcommand, when it is one.first_positional: str | None = Nonefor arg in args:if not arg.startswith("-"):first_positional = argbreak-- # If no positional arg (only flags like --all), or the first positional- # is not a known subcommand, default to 'pipe'if first_positional is None or first_positional not in self.commands:- args = ["pipe", *args]+ return self.default_command_name+ return None+ @override+ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:+ default_command = self.default_command_for(args=args)+ if default_command is not None:+ args = [default_command, *args]return super().parse_args(ctx, args)diff --git a/pipelex/cli/method_resolver.py b/pipelex/cli/method_resolver.pyindex afaf77b2d..34612bb21 100644--- a/pipelex/cli/method_resolver.py+++ b/pipelex/cli/method_resolver.py@@ -24,6 +24,8 @@ from pipelex.methods.structures_check import (describe_structured_content_violations,scan_structured_content_classes,)+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsMethodSource+from pipelex.system.usage_stats.usage_stats_runs import declare_method_sourcedef _get_all_exported_pipes(method: InstalledMethod) -> set[str]:@@ -198,6 +200,7 @@ def resolve_method_target("""method: InstalledMethodif looks_like_method_ref(method_name):+ declare_method_source(source=UsageStatsMethodSource.PUBLISHED_ADDRESS)if raise_ref_errors:method = resolve_method_from_ref(method_name)else:@@ -207,8 +210,10 @@ def resolve_method_target(typer.secho(str(exc), fg=typer.colors.RED, err=True)raise typer.Exit(1) from excelif is_local_path(method_name):+ declare_method_source(source=UsageStatsMethodSource.LOCAL_BUNDLE)method = resolve_method_from_path(method_name)else:+ declare_method_source(source=UsageStatsMethodSource.INSTALLED_METHOD)try:method = find_method_by_name(method_name, library_dirs=library_dirs)except MethodNotFoundError as exc:diff --git a/pipelex/cli/usage_stats_cli.py b/pipelex/cli/usage_stats_cli.pynew file mode 100644index 000000000..d04e2191e--- /dev/null+++ b/pipelex/cli/usage_stats_cli.py@@ -0,0 +1,315 @@+"""The usage statistics of a `pipelex` or `pipelex-agent` command: read before it runs, reported once it ends.++Both CLIs' root groups run every invocation through `run_with_usage_stats`, which, before the command does any work,+finds the command's path in the click tree (never its arguments), reads every switch and prints the notice; and once+the command ends, whatever its outcome, records one `command_completed` and delivers what the command recorded. The+contract is `conformance/specs/usage-statistics.md`, "`command_completed`" and "Notice".++No event is sent for `--help`, `--version`, an invocation with no command, or the `usage-stats` group itself, whose+commands never send, queue or print the notice.+"""++import time+from collections.abc import Callable, Sequence+from typing import Any, NamedTuple, Protocol, TypeVar, runtime_checkable++import click++from pipelex import log+from pipelex.base_exceptions import PipelexError+from pipelex.system.usage_stats.usage_stats_client import UsageStatsClient, configure_usage_stats_for_cli, get_usage_stats_client+from pipelex.system.usage_stats.usage_stats_contract import (+ INIT_COMMAND,+ OTHER_COMMAND,+ PIPELEX_AGENT_COMMANDS,+ PIPELEX_COMMANDS,+ UsageStatsEventName,+ UsageStatsOutcome,+ UsageStatsSurface,+)+from pipelex.system.usage_stats.usage_stats_events import CommandCompletedProperties, UsageStatsEvent, bounded_duration_ms+from pipelex.system.usage_stats.usage_stats_runs import usage_stats_error_domain+from pipelex.tools.log.error_fields import error_fields++#: The statistics command group, `pipelex usage-stats`.+STATISTICS_GROUP_NAME = "usage-stats"+#: The exit code of a command interrupted by Ctrl-C, as Typer and the shells report it.+INTERRUPTED_EXIT_CODE = 130++_FORMAT_OPTION = "--format"+_ERROR_FORMAT_OPTION = "--error-format"+_JSON = "json"++ResultT = TypeVar("ResultT")+++@runtime_checkable+class RoutesToDefaultCommand(Protocol):+ """A command group that runs one of its commands when an invocation names none, as `pipelex validate --all` runs `validate pipe`."""++ def default_command_for(self, *, args: Sequence[str]) -> str | None:+ """The command `args`, the arguments after the group's name, are routed to when they name none; `None` otherwise."""+ ...+++class CliInvocation(NamedTuple):+ """What an invocation is, as far as statistics go: the command it runs, if one is counted, and who reads its stderr."""++ command: str | None+ is_statistics_group: bool+ is_machine_read: bool+++def _find_param(*, command: click.Command, token: str) -> click.Parameter | None:+ for param in command.params:+ if token in param.opts or token in param.secondary_opts:+ return param+ return None+++def _takes_value(*, param: click.Parameter) -> bool:+ if isinstance(param, click.Option):+ return not param.is_flag and not param.count+ return True+++def _option_value(*, args: Sequence[str], option: str) -> str | None:+ """The last value given to `option` among a leaf command's arguments, as `--option value` or `--option=value`."""+ value: str | None = None+ index_arg = 0+ while index_arg < len(args):+ token = args[index_arg]+ if token == "--":+ break+ if token == option and index_arg + 1 < len(args):+ value = args[index_arg + 1]+ index_arg += 2+ continue+ if token.startswith(f"{option}="):+ value = token.split("=", 1)[1]+ index_arg += 1+ return value+++def _default_of(*, command: click.Command, option: str) -> str | None:+ param = _find_param(command=command, token=option)+ if param is None or param.default is None or callable(param.default):+ return None+ return str(param.default)+++def _errors_are_json(*, command: click.Command, args: Sequence[str]) -> bool:+ """Whether a `pipelex-agent` command's errors are JSON: `--error-format`, else `--format`, else its defaults.++ A command with neither option is one of the JSON-only commands, whose errors are always the JSON envelope.+ """+ has_error_format = _find_param(command=command, token=_ERROR_FORMAT_OPTION) is not None+ has_format = _find_param(command=command, token=_FORMAT_OPTION) is not None+ if not has_error_format and not has_format:+ return True+ chosen = _option_value(args=args, option=_ERROR_FORMAT_OPTION) if has_error_format else None+ if chosen is None and has_format:+ chosen = _option_value(args=args, option=_FORMAT_OPTION)+ if chosen is None and has_error_format:+ chosen = _default_of(command=command, option=_ERROR_FORMAT_OPTION)+ if chosen is None and has_format:+ chosen = _default_of(command=command, option=_FORMAT_OPTION)+ return chosen is not None and chosen.strip().lower() == _JSON+++def _allowed_commands(*, surface: UsageStatsSurface) -> frozenset[str]:+ match surface:+ case UsageStatsSurface.PIPELEX:+ return PIPELEX_COMMANDS+ case UsageStatsSurface.PIPELEX_AGENT:+ return PIPELEX_AGENT_COMMANDS+ case UsageStatsSurface.API | UsageStatsSurface.PYTHON:+ return frozenset({OTHER_COMMAND})+++def _is_machine_read_surface(*, surface: UsageStatsSurface) -> bool:+ match surface:+ case UsageStatsSurface.PIPELEX_AGENT:+ return True+ case UsageStatsSurface.PIPELEX | UsageStatsSurface.API | UsageStatsSurface.PYTHON:+ return False+++def resolve_invocation(*, root: click.Group, args: Sequence[str], surface: UsageStatsSurface) -> CliInvocation:+ """Find the command an invocation runs by walking its arguments down the click tree, reading no argument's value.++ The command is its path of names in the tree, aliases and a group's default routing resolved, `other` when the schema+ does not list it (or no command has that name), and `None` when nothing is counted: help, the version, no command at+ all, or the statistics group.+ """+ not_counted = CliInvocation(command=None, is_statistics_group=False, is_machine_read=False)+ help_names = {"--help"}+ current: click.Command = root+ path: list[str] = []+ index_arg = 0+ while isinstance(current, click.Group):+ if isinstance(current, RoutesToDefaultCommand):+ default_name = current.default_command_for(args=args[index_arg:])+ default_command = current.commands.get(default_name) if default_name is not None else None+ if default_name is not None and default_command is not None:+ # The group routes the arguments to its default command, whose path is what ran: no argument is consumed.+ path.append(default_name)+ current = default_command+ continue+ if index_arg >= len(args):+ # A group with no command under it: its help, or a usage error, but no command was run.+ return not_counted+ token = args[index_arg]+ if token in help_names or token == "--":+ return not_counted+ if token.startswith("-"):+ param = _find_param(command=current, token=token.split("=", 1)[0])+ if param is not None and isinstance(param, click.Option) and param.is_eager:+ # `--version` and `-V`: they print and exit before any command runs.+ return not_counted+ consumed = 2 if param is not None and "=" not in token and _takes_value(param=param) else 1+ index_arg += consumed+ continue+ if not path and token == STATISTICS_GROUP_NAME:+ return CliInvocation(command=None, is_statistics_group=True, is_machine_read=False)+ subcommand = current.commands.get(token)+ if subcommand is None:+ return CliInvocation(command=OTHER_COMMAND, is_statistics_group=False, is_machine_read=_is_machine_read_surface(surface=surface))+ path.append(token)+ current = subcommand+ index_arg += 1++ leaf_args = list(args[index_arg:])+ before_separator = leaf_args[: leaf_args.index("--")] if "--" in leaf_args else leaf_args+ if any(token in help_names for token in before_separator):+ return not_counted+ command_path = " ".join(path)+ command = command_path if command_path in _allowed_commands(surface=surface) else OTHER_COMMAND+ is_machine_read = _is_machine_read_surface(surface=surface) and _errors_are_json(command=current, args=leaf_args)+ return CliInvocation(command=command, is_statistics_group=False, is_machine_read=is_machine_read)+++def outcome_for_exit_code(*, code: object) -> UsageStatsOutcome:+ """A command's outcome by its exit code: 0 succeeded, 130 cancelled by Ctrl-C, anything else failed."""+ if code is None or code == 0:+ return UsageStatsOutcome.SUCCEEDED+ if code == INTERRUPTED_EXIT_CODE:+ return UsageStatsOutcome.CANCELLED+ return UsageStatsOutcome.FAILED+++def is_stopped_by_the_person(*, abort: click.exceptions.Abort) -> bool:+ """Whether Click's `Abort` means the person stopped the command: Ctrl-C at a prompt, or a confirmation declined.++ Click raises it from a prompt interrupted by Ctrl-C and from a `confirm(abort=True)` answered no; it also raises it+ when a prompt finds its input ended, which is a failure, not a choice.+ """+ reason = abort.__cause__ or abort.__context__+ return not isinstance(reason, EOFError)+++def outcome_for_exit(*, exit_signal: SystemExit) -> UsageStatsOutcome:+ """A command's outcome by how it exited.++ Typer turns Click's `Abort` into exit code 1, the code of a failure, from within its handler of the `Abort`, so the+ exit's context still names it: a command the person stopped that way is `cancelled`, as one stopped by Ctrl-C+ anywhere else, exiting 130, is.+ """+ reason = exit_signal.__context__+ if isinstance(reason, click.exceptions.Abort) and is_stopped_by_the_person(abort=reason):+ return UsageStatsOutcome.CANCELLED+ return outcome_for_exit_code(code=exit_signal.code)+++def run_with_usage_stats(+ *,+ root: click.Group,+ args: Sequence[str],+ surface: UsageStatsSurface,+ strip_flags: frozenset[str],+ invoke: Callable[[], ResultT],+) -> ResultT:+ """Run a CLI invocation between its statistics' start and end.++ Args:+ root: The CLI's root group, whose tree the command is found in.+ args: The invocation's arguments, without the program name.+ surface: Which CLI this is.+ strip_flags: Global flags the root group removes wherever they appear, so they are no command's argument.+ invoke: Runs the invocation, as click's `main` does, standalone mode included.+ """+ invocation = resolve_invocation(root=root, args=[arg for arg in args if arg not in strip_flags], surface=surface)+ if invocation.command is None:+ return invoke()++ client = configure_usage_stats_for_cli(surface=surface, is_machine_read=invocation.is_machine_read)+ if client is None:+ return invoke()+ started = time.monotonic()+ outcome = UsageStatsOutcome.FAILED+ failure: PipelexError | None = None+ try:+ result = invoke()+ outcome = UsageStatsOutcome.SUCCEEDED+ return result+ except SystemExit as exit_signal:+ outcome = outcome_for_exit(exit_signal=exit_signal)+ raise+ except KeyboardInterrupt:+ outcome = UsageStatsOutcome.CANCELLED+ raise+ except click.exceptions.Abort as abort:+ # Outside standalone mode Click's `Abort` reaches the caller as it is.+ if is_stopped_by_the_person(abort=abort):+ outcome = UsageStatsOutcome.CANCELLED+ raise+ except PipelexError as pipelex_error:+ failure = pipelex_error+ raise+ finally:+ if client.is_enabled:+ _end_command(client=client, command=invocation.command, outcome=outcome, failure=failure, duration_seconds=time.monotonic() - started)+++def _end_command(+ *, client: UsageStatsClient, command: str, outcome: UsageStatsOutcome, failure: PipelexError | None, duration_seconds: float+) -> None:+ """Record the command's `command_completed` and deliver what every client of the process holds; nothing here raises.++ It runs while the command's own exit or error is on its way out, which statistics must never replace, and each step+ runs whatever the one before it did.+ """+ try:+ error_domain = None+ if outcome.is_failed:+ error_domain = usage_stats_error_domain(error=failure) if failure is not None else client.command_error_domain+ init_properties: dict[str, Any] = {}+ if client.command_init_choices is not None and command == INIT_COMMAND:+ init_properties = client.command_init_choices.as_properties()+ client.record(+ event=UsageStatsEvent(+ name=UsageStatsEventName.COMMAND_COMPLETED,+ properties=CommandCompletedProperties(+ command=command,+ outcome=outcome,+ error_domain=error_domain,+ duration_ms=bounded_duration_ms(seconds=duration_seconds),+ hosted=True if client.is_command_hosted else None,+ **init_properties,+ ),+ )+ )+ except Exception as exc: # ruff: ignore[blind-except]+ # (1) The root of every command of both CLIs: the command's own exit or error is on its way out, and a+ # statistic that cannot be built must never replace it.+ log.debug("The command's usage statistics could not be recorded", fields=error_fields(exc=exc))+ # The process's client may have been replaced while the command ran, never its events: finish both.+ current_client = get_usage_stats_client()+ finishing = [client] if current_client is None or current_client is client else [client, current_client]+ for each_client in finishing:+ try:+ each_client.finish_command()+ except Exception as exc: # ruff: ignore[blind-except]+ # (1) The root of every command of both CLIs, as above: delivery never replaces the command's exit, and+ # one client's failure never keeps the other's events from leaving.+ log.debug("The command's usage statistics could not be delivered", fields=error_fields(exc=exc))diff --git a/pipelex/cogt/extract/extract_job.py b/pipelex/cogt/extract/extract_job.pyindex b1f719bcd..17dd7b049 100644--- a/pipelex/cogt/extract/extract_job.py+++ b/pipelex/cogt/extract/extract_job.py@@ -30,6 +30,7 @@ class ExtractJob(InferenceJobAbstract):inference_model_name=inference_model.name,unit_costs=inference_model.costs,inference_model_id=inference_model.model_id,+ backend_name=inference_model.backend_name,nb_tokens_by_category={},)diff --git a/pipelex/cogt/extract/extract_report.py b/pipelex/cogt/extract/extract_report.pyindex a71c5d769..db0f178dc 100644--- a/pipelex/cogt/extract/extract_report.py+++ b/pipelex/cogt/extract/extract_report.py@@ -69,6 +69,9 @@ class ExtractTokensUsage(BaseModel):inference_model_name: strunit_costs: CostsByCategoryDictinference_model_id: str+ # The backend that served the call, which a fallback can make another than the first one tried, so the usage is+ # credited to it even when two backends name the model alike; `None` for a synthetic report no backend made.+ backend_name: str | None = Nonenb_tokens_by_category: NbTokensByCategoryDict# What the counts are: tokens, or requests or pages recorded as a million tokens each so the rates price one.pricing_unit: PricingUnit = PricingUnit.TOKENdiff --git a/pipelex/cogt/extract/extract_worker_abstract.py b/pipelex/cogt/extract/extract_worker_abstract.pyindex 926e73e54..f62153c82 100644--- a/pipelex/cogt/extract/extract_worker_abstract.py+++ b/pipelex/cogt/extract/extract_worker_abstract.py@@ -124,7 +124,7 @@ class ExtractWorkerAbstract(InferenceWorkerAbstract):extract_job: ExtractJob,) -> ExtractOutput:# The call ends with its summary event whichever way it ends, a refusal by the checks below included- with self._call_summary(extract_job=extract_job):+ with self._call_summary(extract_job=extract_job) as call_summary:# Verify that the job is validextract_job.validate_before_execution()@@ -145,6 +145,8 @@ class ExtractWorkerAbstract(InferenceWorkerAbstract):exc.fill_model_and_provider(model_handle=self.inference_model.name, backend_name=self.inference_model.backend_name)raise+ call_summary.count_pages_extracted(count=len(result.pages))+# Price the call by its pages when the provider reported no usageif (extract_tokens_usage := extract_job.job_report.extract_tokens_usage) and not extract_tokens_usage.nb_tokens_by_category:record_unit_priced_usage(tokens_usage=extract_tokens_usage, pricing_unit=PricingUnit.PAGE, nb_units=len(result.pages))diff --git a/pipelex/cogt/img_gen/img_gen_job.py b/pipelex/cogt/img_gen/img_gen_job.pyindex bc01a867d..9c3abb033 100644--- a/pipelex/cogt/img_gen/img_gen_job.py+++ b/pipelex/cogt/img_gen/img_gen_job.py@@ -30,6 +30,7 @@ class ImgGenJob(InferenceJobAbstract):inference_model_name=inference_model.name,unit_costs=inference_model.costs,inference_model_id=inference_model.model_id,+ backend_name=inference_model.backend_name,nb_tokens_by_category={},)diff --git a/pipelex/cogt/img_gen/img_gen_report.py b/pipelex/cogt/img_gen/img_gen_report.pyindex e6dae170f..bff341d9f 100644--- a/pipelex/cogt/img_gen/img_gen_report.py+++ b/pipelex/cogt/img_gen/img_gen_report.py@@ -69,6 +69,9 @@ class ImgGenTokensUsage(BaseModel):inference_model_name: strunit_costs: CostsByCategoryDictinference_model_id: str+ # The backend that served the call, which a fallback can make another than the first one tried, so the usage is+ # credited to it even when two backends name the model alike; `None` for a synthetic report no backend made.+ backend_name: str | None = Nonenb_tokens_by_category: NbTokensByCategoryDict# What the counts are: tokens, or requests or pages recorded as a million tokens each so the rates price one.pricing_unit: PricingUnit = PricingUnit.TOKENdiff --git a/pipelex/cogt/img_gen/img_gen_worker_abstract.py b/pipelex/cogt/img_gen/img_gen_worker_abstract.pyindex 13057db41..2925f3dd1 100644--- a/pipelex/cogt/img_gen/img_gen_worker_abstract.py+++ b/pipelex/cogt/img_gen/img_gen_worker_abstract.py@@ -63,7 +63,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):img_gen_job: ImgGenJob,) -> GeneratedImageRawDetails:# The call ends with its summary event whichever way it ends, a refusal by the checks below included- with self._call_summary(img_gen_job=img_gen_job):+ with self._call_summary(img_gen_job=img_gen_job) as call_summary:# Verify that the job is validimg_gen_job.validate_before_execution()@@ -82,6 +82,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):except CogtError as exc:exc.fill_model_and_provider(model_handle=self.inference_model.name, backend_name=self.inference_model.backend_name)raise+ call_summary.count_images_generated(count=1)# Report jobimg_gen_job.img_gen_job_after_complete()@@ -104,7 +105,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):nb_images: int,) -> list[GeneratedImageRawDetails]:# The call ends with its summary event whichever way it ends, a refusal by the checks below included- with self._call_summary(img_gen_job=img_gen_job):+ with self._call_summary(img_gen_job=img_gen_job) as call_summary:# Verify that the job is validimg_gen_job.validate_before_execution()@@ -123,6 +124,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):except CogtError as exc:exc.fill_model_and_provider(model_handle=self.inference_model.name, backend_name=self.inference_model.backend_name)raise+ call_summary.count_images_generated(count=len(result))# Report jobimg_gen_job.img_gen_job_after_complete()diff --git a/pipelex/cogt/inference/inference_call_summary.py b/pipelex/cogt/inference/inference_call_summary.pyindex c4506638a..3948977cb 100644--- a/pipelex/cogt/inference/inference_call_summary.py+++ b/pipelex/cogt/inference/inference_call_summary.py@@ -38,9 +38,11 @@ from typing import TYPE_CHECKING, Anyfrom typing_extensions import overridefrom pipelex import log+from pipelex.cogt.exceptions import find_inference_error_category_in_chainfrom pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.cogt.usage.usage_cost import compute_tokens_usage_costfrom pipelex.system.telemetry.otel_constants import GenAISpanAttr+from pipelex.system.usage_stats.usage_stats_runs import current_run_tallyfrom pipelex.tools.log.console_layouts import LogLayoutfrom pipelex.tools.log.summary_event import SummaryEventfrom pipelex.tools.log.summary_fields import COST_USD_FIELD@@ -117,6 +119,45 @@ class InferenceCallSummary(SummaryEvent):self._read_tokens_usage = read_tokens_usageself._read_request_model = read_request_modelself._read_response_model = read_response_model+ self._images_generated = 0+ self._pages_extracted = 0++ def count_images_generated(self, *, count: int) -> None:+ """Count the images the call generated, for the usage statistics of the run it belongs to."""+ self._images_generated += count++ def count_pages_extracted(self, *, count: int) -> None:+ """Count the document pages the call extracted, for the usage statistics of the run it belongs to."""+ self._pages_extracted += count++ @override+ def _end(self, *, error: BaseException | None) -> None:+ """Log the event, then note the call in the usage-statistics tally of the run it belongs to, when one is tallying."""+ super()._end(error=error)+ self._note_in_run_tally(error=error)++ def _note_in_run_tally(self, *, error: BaseException | None) -> None:+ """Note how the call ended in its run's tally: its model and backend, and, for a failure, the runtime's category.++ A call stopped from outside is a call, not a failure. A call whose handle resolved to no model counts in the run's+ totals only, since no backend served it.+ """+ run_tally = current_run_tally()+ if run_tally is None:+ return+ inference_model = self._read_inference_model()+ failed = isinstance(error, Exception)+ failure_category: str | None = None+ if isinstance(error, Exception):+ failure_category = find_inference_error_category_in_chain(error)+ run_tally.note_call(+ model_handle=inference_model.name if inference_model is not None else self._model_handle,+ backend_name=inference_model.backend_name if inference_model is not None else None,+ failed=failed,+ failure_category=failure_category,+ images_generated=self._images_generated,+ pages_extracted=self._pages_extracted,+ )@overridedef _work_fields(self) -> dict[str, Any]:diff --git a/pipelex/cogt/judgment/judgment_job.py b/pipelex/cogt/judgment/judgment_job.pyindex 60087938d..0bcde57e6 100644--- a/pipelex/cogt/judgment/judgment_job.py+++ b/pipelex/cogt/judgment/judgment_job.py@@ -52,6 +52,7 @@ class JudgmentJob(InferenceJobAbstract):inference_model_name=inference_model.name,unit_costs=inference_model.costs,inference_model_id=inference_model.model_id,+ backend_name=inference_model.backend_name,nb_tokens_by_category={},)diff --git a/pipelex/cogt/judgment/judgment_report.py b/pipelex/cogt/judgment/judgment_report.pyindex a4a3292ce..8a5478480 100644--- a/pipelex/cogt/judgment/judgment_report.py+++ b/pipelex/cogt/judgment/judgment_report.py@@ -77,6 +77,9 @@ class JudgmentTokensUsage(BaseModel):inference_model_name: strunit_costs: CostsByCategoryDictinference_model_id: str+ # The backend that served the call, which a fallback can make another than the first one tried, so the usage is+ # credited to it even when two backends name the model alike; `None` for a synthetic report no backend made.+ backend_name: str | None = Nonenb_tokens_by_category: NbTokensByCategoryDict# What the counts are: tokens, or requests or pages recorded as a million tokens each so the rates price one.pricing_unit: PricingUnit = PricingUnit.TOKENdiff --git a/pipelex/cogt/llm/llm_job.py b/pipelex/cogt/llm/llm_job.pyindex 129a85383..349e77b79 100644--- a/pipelex/cogt/llm/llm_job.py+++ b/pipelex/cogt/llm/llm_job.py@@ -38,6 +38,7 @@ class LLMJob(InferenceJobAbstract):inference_model_name=inference_model.name,unit_costs=inference_model.costs,inference_model_id=inference_model.model_id,+ backend_name=inference_model.backend_name,nb_tokens_by_category={},)diff --git a/pipelex/cogt/llm/llm_report.py b/pipelex/cogt/llm/llm_report.pyindex 6c5872579..f350fbe89 100644--- a/pipelex/cogt/llm/llm_report.py+++ b/pipelex/cogt/llm/llm_report.py@@ -69,6 +69,9 @@ class LLMTokensUsage(BaseModel):inference_model_name: strunit_costs: CostsByCategoryDictinference_model_id: str+ # The backend that served the call, which a fallback can make another than the first one tried, so the usage is+ # credited to it even when two backends name the model alike; `None` for a synthetic report no backend made.+ backend_name: str | None = Nonenb_tokens_by_category: NbTokensByCategoryDict# What the counts are: tokens, or requests or pages recorded as a million tokens each so the rates price one.pricing_unit: PricingUnit = PricingUnit.TOKENdiff --git a/pipelex/cogt/search/search_job.py b/pipelex/cogt/search/search_job.pyindex 678c7bd38..77979cf48 100644--- a/pipelex/cogt/search/search_job.py+++ b/pipelex/cogt/search/search_job.py@@ -39,6 +39,7 @@ class SearchJob(InferenceJobAbstract):inference_model_name=inference_model.name,unit_costs=inference_model.costs,inference_model_id=inference_model.model_id,+ backend_name=inference_model.backend_name,nb_tokens_by_category={},)diff --git a/pipelex/cogt/search/search_report.py b/pipelex/cogt/search/search_report.pyindex b18eefacb..7041f0446 100644--- a/pipelex/cogt/search/search_report.py+++ b/pipelex/cogt/search/search_report.py@@ -69,6 +69,9 @@ class SearchTokensUsage(BaseModel):inference_model_name: strunit_costs: CostsByCategoryDictinference_model_id: str+ # The backend that served the call, which a fallback can make another than the first one tried, so the usage is+ # credited to it even when two backends name the model alike; `None` for a synthetic report no backend made.+ backend_name: str | None = Nonenb_tokens_by_category: NbTokensByCategoryDict# What the counts are: tokens, or requests or pages recorded as a million tokens each so the rates price one.pricing_unit: PricingUnit = PricingUnit.TOKENdiff --git a/pipelex/hosted/hosted_run.py b/pipelex/hosted/hosted_run.pyindex 6334e92fc..93d47f6a3 100644--- a/pipelex/hosted/hosted_run.py+++ b/pipelex/hosted/hosted_run.py@@ -28,6 +28,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validafrom pipelex.hosted.exceptions import HostedRunOutcomeUnknownError, HostedRunPollingErrorfrom pipelex.hosted.hosted_inputs import prepare_hosted_inputs+from pipelex.system.usage_stats.usage_stats_client import note_command_hosted#: The `implementation` a bare runner names in its `GET /v1/version` handshake: an open-source pipelex-api, which#: keeps no runs to poll. The same value pipelex-sdk's `start_and_wait` tests to choose the blocking route.@@ -209,6 +210,8 @@ async def run_hosted(*, client: PipelexAPIClient, request: HostedRunRequest, on_ApiUnreachableError: If the hosted API cannot be reached before the run request is sent.InputPreparationError: If a local file cannot be read or uploaded."""+ # A command running this sends its work to the hosted API; its `command_completed` says so and no more.+ note_command_hosted()prepared = await prepare_hosted_inputs(client=client,mthds_files=request.mthds_files,diff --git a/pipelex/libraries/concept/concept_library.py b/pipelex/libraries/concept/concept_library.pyindex 5f0eece4e..44c1a6176 100644--- a/pipelex/libraries/concept/concept_library.py+++ b/pipelex/libraries/concept/concept_library.py@@ -89,6 +89,15 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptLibraryAbstract):def list_concepts(self) -> list[Concept]:return list(self.root.values())+ @override+ def list_own_concepts(self) -> list[Concept]:+ # A dependency's concept is keyed `alias->domain.Code` (see `add_dependency_concept`); the natives are the library's own seed.+ return [+ concept+ for key, concept in self.root.items()+ if not QualifiedRef.has_cross_package_prefix(key) and not SpecialDomain.is_native(domain_code=concept.domain_code)+ ]+@overridedef list_concepts_by_domain(self, domain_code: str) -> list[Concept]:return [concept for key, concept in self.root.items() if key.startswith(f"{domain_code}.")]diff --git a/pipelex/libraries/concept/concept_library_abstract.py b/pipelex/libraries/concept/concept_library_abstract.pyindex 56a0ea7be..f3e55ad1d 100644--- a/pipelex/libraries/concept/concept_library_abstract.py+++ b/pipelex/libraries/concept/concept_library_abstract.py@@ -50,6 +50,10 @@ class ConceptLibraryAbstract(ConceptProviderAbstract):def list_concepts(self) -> list[Concept]:pass+ @abstractmethod+ def list_own_concepts(self) -> list[Concept]:+ """The concepts the loaded method declares itself: neither a native nor one a dependency package contributed."""+@abstractmethoddef setup(self) -> None:passdiff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index 0de0caa57..95eed7d71 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -499,3 +499,15 @@ execution = "local"# - "cursor" installs .cursor/rules/*.mdc files (cannot be combined with others)# - "agents" / "claude" install AGENTS.md / CLAUDE.md respectively — both can be listed togetherpreferred_agent_targets = ["claude", "agents"]++####################################################################################################+# Usage statistics — the anonymous counts pipelex sends to the Pipelex project+####################################################################################################++[usage_stats]+# Whether pipelex may send anonymous usage statistics: counts of commands, runs, pipe types, models and tokens,+# never a method, a prompt, an input, an output, a file name or a pipe name (https://docs.pipelex.com/latest/setup/usage-statistics/).+# Only `false` has an effect, and only in two places: this packaged file, for a packager shipping pipelex with statistics+# off, and a project's own .pipelex/pipelex.toml, for a team turning them off for everyone working in the project.+# For yourself, run `pipelex usage-stats disable` or set DO_NOT_TRACK=1.+enabled = truediff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index 307a95ea1..a89e69188 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -1510,6 +1510,17 @@"required": true,"default": ".pipelex/traces","open_node": false+ },+ "usage_stats": {+ "value_type": "table",+ "required": true,+ "open_node": false+ },+ "usage_stats.enabled": {+ "value_type": "bool",+ "required": true,+ "default": true,+ "open_node": false}}}diff --git a/pipelex/pipe_run/dry_run_in_process.py b/pipelex/pipe_run/dry_run_in_process.pyindex 3aff07748..70adbbb54 100644--- a/pipelex/pipe_run/dry_run_in_process.py+++ b/pipelex/pipe_run/dry_run_in_process.py@@ -144,7 +144,7 @@ async def dry_run_pipe_in_process(pipe: PipeAbstract, *, library_id: str, caller# try/finally that guarantees its close — an entry in the process-wide GraphTracerManager# must never outlive this call.pipe_router = PipeRouter(observer=ObserverNoOp())- pipe_run = PipeRun(pipe_router=pipe_router)+ pipe_run = PipeRun(pipe_router=pipe_router, reports_usage_stats=False)content_generator = ContentGenerator.make_inline()graph_tracer_manager = GraphTracerManager.get_or_create_instance()diff --git a/pipelex/pipe_run/pipe_run.py b/pipelex/pipe_run/pipe_run.pyindex 7df9a3875..67977ac0a 100644--- a/pipelex/pipe_run/pipe_run.py+++ b/pipelex/pipe_run/pipe_run.py@@ -1,5 +1,6 @@from __future__ import annotations+import timefrom typing import TYPE_CHECKINGfrom typing_extensions import override@@ -7,17 +8,22 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.base_exceptions import ErrorReport, PipelexError, PipelexUnexpectedErrorfrom pipelex.graph.graph_tracer_manager import GraphTracerManager-from pipelex.interpreter_hub import get_own_pipes+from pipelex.interpreter_hub import get_concept_library, get_own_pipesfrom pipelex.pipe_run.delivery_assignment import DeliveryAssignment, DeliveryStatusfrom pipelex.pipe_run.delivery_executor import DeliveryExecutorfrom pipelex.pipe_run.exceptions import DeliveryErrorfrom pipelex.pipe_run.pipe_run_protocol import PipeRunProtocol-from pipelex.pipe_run.tracing_assembly import assemble_tracing_on_output+from pipelex.pipe_run.tracing_assembly import RunTraceEvents, assemble_tracing_on_output, read_run_trace_eventsfrom pipelex.pipeline.build_pipe_io_artifacts import build_pipe_io_artifacts+from pipelex.pipeline.usage_stats_reporting import RunUsageFacts, input_kinds_of, report_run_usage+from pipelex.system.usage_stats.usage_stats_client import usage_stats_are_enabled+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsOutcome+from pipelex.system.usage_stats.usage_stats_runs import RunInferenceTally, tallying_inference_callsfrom pipelex.tools.log.error_fields import error_fieldsif TYPE_CHECKING:from pipelex.core.pipes.pipe_output import PipeOutput+ from pipelex.graph.graphspec import GraphSpecfrom pipelex.pipe_run.pipe_job import PipeJobfrom pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol@@ -25,9 +31,18 @@ if TYPE_CHECKING:class PipeRun(PipeRunProtocol):"""Direct-mode PipeRun: executes the pipe inline, then delivers results."""- def __init__(self, pipe_router: PipeRouterProtocol) -> None:+ def __init__(self, pipe_router: PipeRouterProtocol, *, reports_usage_stats: bool = True) -> None:+ """Execute pipes through `pipe_router`.++ Args:+ pipe_router: The router the run's pipes are dispatched through.+ reports_usage_stats: Whether a run reports its anonymous usage statistics, its `run_completed` and+ `model_used` events, when statistics are on. Off for the runs a validation makes, which its own+ `method_validated` describes.+ """self._pipe_router = pipe_routerself._delivery_executor = DeliveryExecutor()+ self._reports_usage_stats = reports_usage_stats@overrideasync def run(@@ -52,9 +67,15 @@ class PipeRun(PipeRunProtocol):pipe_output: PipeOutput | None = Noneexecution_error: Exception | None = Noneerror_report: ErrorReport | None = None+ closed_graph: GraphSpec | None = None+ # The run's statistics: its inputs are read before the run adds its own outputs to its working memory.+ usage_tally = RunInferenceTally() if self._reports_usage_stats and usage_stats_are_enabled() else None+ input_kinds = input_kinds_of(working_memory=pipe_job.working_memory) if usage_tally is not None else (False, False)+ started_at = time.monotonic()try:- pipe_output = await self._pipe_router.run(pipe_job)+ with tallying_inference_calls(tally=usage_tally):+ pipe_output = await self._pipe_router.run(pipe_job)# Stamp the job onto the run's OWN output, here and nowhere else.# Only the top-level output crosses a transport boundary; a sub-pipe's# output stays in-process, so stamping at every `PipeOutput(...)` site@@ -83,7 +104,7 @@ class PipeRun(PipeRunProtocol):tracer_manager = GraphTracerManager.get_instance()if tracer_manager is not None:try:- tracer_manager.close_tracer(pipeline_run_id)+ closed_graph = tracer_manager.close_tracer(pipeline_run_id)except OSError as close_error:if execution_error is None:raise@@ -98,8 +119,16 @@ class PipeRun(PipeRunProtocol):# concerns are gated independently: graph events feed the GraphSpec (so a costs-only run does# not set an empty GraphSpec, preserving the --no-graph contract), usage events feed the# tokens_usages the submitter renders the cost report from.+ # The run's statistics read the same events, its usage events, whether the run completed or not.trace_context = pipe_job.job_metadata.trace_context- if pipe_output is not None and trace_context is not None and (trace_context.emit_graph_events or trace_context.emit_usage_events):+ assembles_output = (+ pipe_output is not None and trace_context is not None and (trace_context.emit_graph_events or trace_context.emit_usage_events)+ )+ counts_usage = usage_tally is not None and trace_context is not None and trace_context.emit_usage_events+ trace_events: RunTraceEvents | None = None+ if assembles_output or counts_usage:+ trace_events = read_run_trace_events(pipeline_run_id=pipeline_run_id)+ if pipe_output is not None and trace_context is not None and assembles_output:assemble_tracing_on_output(pipe_output=pipe_output,pipeline_run_id=pipeline_run_id,@@ -108,10 +137,31 @@ class PipeRun(PipeRunProtocol):domain_code=pipe_job.pipe.domain_code,main_pipe_code=pipe_job.pipe.code,run_mode=pipe_job.pipe_run_params.run_mode,+ trace_events=trace_events,)if trace_context.describe_pipe_io and pipe_output.graph_spec is not None:self._build_pipe_io_artifacts_on_output(pipe_output=pipe_output, pipeline_run_id=pipeline_run_id)+ if usage_tally is not None:+ outcome: UsageStatsOutcome+ if execution_error is not None:+ outcome = UsageStatsOutcome.FAILED+ elif pipe_output is None:+ # Neither a result nor an exception: the run was cancelled or interrupted, which no `except Exception` catches.+ outcome = UsageStatsOutcome.CANCELLED+ else:+ outcome = UsageStatsOutcome.SUCCEEDED+ self._report_usage_stats(+ pipe_job=pipe_job,+ outcome=outcome,+ execution_error=execution_error,+ duration_seconds=time.monotonic() - started_at,+ input_kinds=input_kinds,+ trace_events=trace_events if counts_usage else None,+ closed_graph=closed_graph,+ usage_tally=usage_tally,+ )+if delivery_assignment is not None:log.debug("Executing the delivery", fields={"delivery_status": status})try:@@ -139,6 +189,47 @@ class PipeRun(PipeRunProtocol):assert pipe_output is not Nonereturn pipe_output+ @classmethod+ def _report_usage_stats(+ cls,+ *,+ pipe_job: PipeJob,+ outcome: UsageStatsOutcome,+ execution_error: Exception | None,+ duration_seconds: float,+ input_kinds: tuple[bool, bool],+ trace_events: RunTraceEvents | None,+ closed_graph: GraphSpec | None,+ usage_tally: RunInferenceTally,+ ) -> None:+ """Record the run's `run_completed`, and a `model_used` per model when its usage events were read.++ Runs inside the run's library window, which the bundle's description reads. It runs in the `finally` ahead of+ the delivery, and an anonymous statistic must never replace a run's outcome or skip its delivery, so a failure+ to build it is logged at debug level and the event is not sent.+ """+ has_image_input, has_document_input = input_kinds+ try:+ report_run_usage(+ facts=RunUsageFacts(+ outcome=outcome,+ error=execution_error,+ is_dry_run=pipe_job.pipe_run_params.run_mode.is_dry,+ duration_seconds=duration_seconds,+ main_pipe=pipe_job.pipe,+ own_pipes=get_own_pipes(),+ own_concepts=get_concept_library().list_own_concepts(),+ has_image_input=has_image_input,+ has_document_input=has_document_input,+ events=trace_events.events if trace_events is not None else None,+ closed_graph=closed_graph,+ tally=usage_tally,+ )+ )+ except (PipelexError, ValueError, OSError) as report_error:+ # A library that cannot describe the bundle, an event that will not validate, a state that cannot be read.+ log.debug("The usage statistics of a run could not be built", fields=error_fields(exc=report_error))+@classmethoddef _build_pipe_io_artifacts_on_output(cls, *, pipe_output: PipeOutput, pipeline_run_id: str) -> None:"""Describe the graph's data: build the three I/O artifacts over the run library's own pipes and set them on the output.diff --git a/pipelex/pipe_run/tracing_assembly.py b/pipelex/pipe_run/tracing_assembly.pyindex f0ba26ce5..db332d300 100644--- a/pipelex/pipe_run/tracing_assembly.py+++ b/pipelex/pipe_run/tracing_assembly.py@@ -11,6 +11,7 @@ that crosses the Temporal activity boundary (returned by ``act_assemble_tracing`"""import json+from typing import NamedTuplefrom pydantic import BaseModel, ConfigDict, PrivateAttr, ValidationError@@ -28,6 +29,7 @@ from pipelex.tools.log.error_fields import error_fieldsfrom pipelex.tracing.event_log_factory import make_event_logfrom pipelex.tracing.exceptions import EventLogErrorfrom pipelex.tracing.graphspec_assembler import GraphSpecAssembler+from pipelex.tracing.trace_events import TraceEventfrom pipelex.tracing.usage_aggregator import UsageAggregator@@ -74,6 +76,60 @@ class TracingAssembly(BaseModel):self._run_metadata = run_metadata+class RunTraceEvents(NamedTuple):+ """The one read of a run's trace events: what it returned, or why it returned nothing.++ ``events`` is ``None`` when the events were not read (tracing is off) or could not be; ``read_error`` says+ which, carrying the note the assembly records when the read failed.+ """++ events: list[TraceEvent] | None+ read_error: str | None++ @property+ def was_read(self) -> bool:+ return self.events is not None+++def read_run_trace_events(*, pipeline_run_id: str) -> RunTraceEvents:+ """Read the trace events of ``pipeline_run_id`` once, from the run's scoped event log or the configured backend.++ Best-effort, for the reasons :func:`assemble_tracing` gives: a backend failure, malformed event data or broken+ tracing infrastructure returns no events and the failure's note, never raises. With tracing off and no scoped+ event log, nothing is read.+ """+ tracing_config = get_config().runtime.tracing+ # A scoped override (see hub.scoped_event_log) is the run's transport and implies+ # tracing-enabled (D1) — it must not be skipped by the is_enabled early-return.+ event_log_override = get_event_log_override()+ if event_log_override is None and not tracing_config.is_enabled:+ return RunTraceEvents(events=None, read_error=None)++ # Best-effort: a backend failure degrades to an *_assembly_error note rather than failing the run.+ # The DynamoDB backend converts its botocore failures (ClientError / BotoCoreError) into our+ # EventLogError family — EventLogSetupError on client construction (make_event_log), EventLogReadError+ # inside ``DynamoDBEventLog.read_events`` — so the assembly layer catches the EventLogError base and+ # never imports boto3 itself; NDJSON read failures surface as OSError. Both backends refuse a log holding an+ # event the current models do not accept with EventLogSchemaMismatchError, which drops the graph and the usage.+ try:+ if event_log_override is not None:+ # The scope owner keeps the instance's lifecycle: read without close().+ events = event_log_override.read_events(pipeline_run_id)+ else:+ event_log = make_event_log(tracing_config)+ try:+ events = event_log.read_events(pipeline_run_id)+ finally:+ event_log.close()+ except (OSError, json.JSONDecodeError, ValidationError, PipelexConfigError, MissingDependencyError, EventLogError) as read_error:+ log.warning("The trace events of a run could not be read for its tracing assembly", fields=error_fields(exc=read_error))+ return RunTraceEvents(+ events=None,+ read_error=f"Tracing assembly failed to read events for pipeline_run_id={pipeline_run_id}: {read_error}",+ )+ return RunTraceEvents(events=events, read_error=None)++def assemble_tracing(pipeline_run_id: str,*,@@ -83,6 +139,7 @@ def assemble_tracing(main_pipe_code: str | None = None,run_mode: PipeRunMode = PipeRunMode.LIVE,run_metadata: RunMetadata | None = None,+ trace_events: RunTraceEvents | None = None,) -> TracingAssembly:"""Read the trace events for ``pipeline_run_id`` once and assemble the requested artifacts.@@ -110,6 +167,8 @@ def assemble_tracing(run_metadata: The run this assembly belongs to, stamped onto the result so atransport that offloads it keys it inside the run's namespace. None for arun with no host-supplied scope.+ trace_events: The run's events, already read by the caller for another reader of the same events (the run's+ usage statistics), so the run's events are still read once; None to read them here.Returns:A TracingAssembly carrying whichever artifacts were requested and succeeded.@@ -128,6 +187,7 @@ def assemble_tracing(domain_code=domain_code,main_pipe_code=main_pipe_code,run_mode=run_mode,+ trace_events=trace_events,)@@ -140,42 +200,21 @@ def _assemble_tracing_events(domain_code: str | None,main_pipe_code: str | None,run_mode: PipeRunMode,+ trace_events: RunTraceEvents | None,) -> TracingAssembly:"""The body of ``assemble_tracing``, run inside the log context it binds: read the events once, assemble onto ``result``."""- tracing_config = get_config().runtime.tracing- # A scoped override (see hub.scoped_event_log) is the run's transport and implies- # tracing-enabled (D1) — it must not be skipped by the is_enabled early-return.- event_log_override = get_event_log_override()- if event_log_override is None and not tracing_config.is_enabled:- return resultif not (assemble_graph or assemble_usage):return result- # Best-effort: a backend failure degrades to an *_assembly_error note rather than failing the run.- # The DynamoDB backend converts its botocore failures (ClientError / BotoCoreError) into our- # EventLogError family — EventLogSetupError on client construction (make_event_log), EventLogReadError- # inside ``DynamoDBEventLog.read_events`` — so the assembly layer catches the EventLogError base and- # never imports boto3 itself; NDJSON read failures surface as OSError. Both backends refuse a log holding an- # event the current models do not accept with EventLogSchemaMismatchError, which drops the graph and the usage.- try:- if event_log_override is not None:- # The scope owner keeps the instance's lifecycle: read without close().- events = event_log_override.read_events(pipeline_run_id)- else:- event_log = make_event_log(tracing_config)- try:- events = event_log.read_events(pipeline_run_id)- finally:- event_log.close()- except (OSError, json.JSONDecodeError, ValidationError, PipelexConfigError, MissingDependencyError, EventLogError) as read_error:- message = f"Tracing assembly failed to read events for pipeline_run_id={pipeline_run_id}: {read_error}"- log.warning("The trace events of a run could not be read for its tracing assembly", fields=error_fields(exc=read_error))+ if trace_events is None:+ trace_events = read_run_trace_events(pipeline_run_id=pipeline_run_id)+ if trace_events.read_error is not None:if assemble_graph:- result.graph_assembly_error = message+ result.graph_assembly_error = trace_events.read_errorif assemble_usage:- result.usage_assembly_error = message+ result.usage_assembly_error = trace_events.read_errorreturn result-+ events = trace_events.eventsif not events:return result@@ -210,6 +249,7 @@ def assemble_tracing_on_output(domain_code: str | None = None,main_pipe_code: str | None = None,run_mode: PipeRunMode = PipeRunMode.LIVE,+ trace_events: RunTraceEvents | None = None,) -> None:"""Assemble graph and/or usage from trace events and set them on pipe_output (DIRECT mode).@@ -225,6 +265,7 @@ def assemble_tracing_on_output(domain_code: Domain code for the graph's pipeline ref.main_pipe_code: Main pipe code for the graph's pipeline ref.run_mode: Final pipe run mode used to stamp GraphSpec provenance.+ trace_events: The run's events when the caller already read them; None to read them here."""result = assemble_tracing(pipeline_run_id=pipeline_run_id,@@ -235,6 +276,7 @@ def assemble_tracing_on_output(run_mode=run_mode,# Inherited from the output this assembly decorates, so both key together.run_metadata=pipe_output.job_metadata.run_metadata if pipe_output.job_metadata else None,+ trace_events=trace_events,)if result.graph_spec is not None:pipe_output.graph_spec = result.graph_specdiff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 25b3757ee..e2d3d5c1f 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -53,6 +53,7 @@ from pipelex.system.telemetry.telemetry_config import (from pipelex.system.telemetry.telemetry_manager_abstract import (TelemetryManagerAbstract,)+from pipelex.system.usage_stats.usage_stats_client import configure_usage_stats_for_runtime, flush_usage_statsfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract@@ -215,6 +216,9 @@ class Pipelex(RuntimeBoot):# process re-bootable, so skipping it would wedge the process for good — see its docstring. The# order is preserved by keeping both statements in the ``try``. No ``except``: the failure still# propagates.+ # The usage statistics this process holds leave first, within their budget: a long-running process shutting+ # pipelex down sends what it recorded rather than leaving it to the exit flush. It never raises.+ flush_usage_stats()try:self._teardown_plugin_callbacks()self.pipeline_manager.teardown()@@ -310,6 +314,10 @@ class Pipelex(RuntimeBoot):# bare runtime boot blocks this one too.cls.raise_if_a_boot_already_holds_the_process_globals()+ # Before any work: a process that boots pipelex reads every usage-statistics switch now, unless a CLI command+ # already did, and the `pytest` integration mode turns them off for good.+ configure_usage_stats_for_runtime(integration_mode=integration_mode)+pipelex_instance = cls(config_dir=config_dir, config_overrides=config_overrides)try:pipelex_instance.setup(diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 0de0caa57..95eed7d71 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -499,3 +499,15 @@ execution = "local"# - "cursor" installs .cursor/rules/*.mdc files (cannot be combined with others)# - "agents" / "claude" install AGENTS.md / CLAUDE.md respectively — both can be listed togetherpreferred_agent_targets = ["claude", "agents"]++####################################################################################################+# Usage statistics — the anonymous counts pipelex sends to the Pipelex project+####################################################################################################++[usage_stats]+# Whether pipelex may send anonymous usage statistics: counts of commands, runs, pipe types, models and tokens,+# never a method, a prompt, an input, an output, a file name or a pipe name (https://docs.pipelex.com/latest/setup/usage-statistics/).+# Only `false` has an effect, and only in two places: this packaged file, for a packager shipping pipelex with statistics+# off, and a project's own .pipelex/pipelex.toml, for a team turning them off for everyone working in the project.+# For yourself, run `pipelex usage-stats disable` or set DO_NOT_TRACK=1.+enabled = truediff --git a/pipelex/pipeline/bundle_validator.py b/pipelex/pipeline/bundle_validator.pyindex 9cb46827e..fd0b38615 100644--- a/pipelex/pipeline/bundle_validator.py+++ b/pipelex/pipeline/bundle_validator.py@@ -207,7 +207,7 @@ class BundleValidator:# Mirrors runtime_bridge.direct_orchestrator.DirectOrchestrator.execute. Being the sweep's own# router, it also notes the innermost pipe each failure left (see `_DryRunSweepRouter`).self._pipe_router = _DryRunSweepRouter(observer=ObserverNoOp())- self._pipe_run: PipeRunProtocol = PipeRun(pipe_router=self._pipe_router)+ self._pipe_run: PipeRunProtocol = PipeRun(pipe_router=self._pipe_router, reports_usage_stats=False)async def acquire_and_validate(self,diff --git a/pipelex/pipeline/runner.py b/pipelex/pipeline/runner.pyindex 0ecffd6ef..e6db6c9ff 100644--- a/pipelex/pipeline/runner.py+++ b/pipelex/pipeline/runner.py@@ -35,6 +35,8 @@ from pipelex.system.caller_identity import CallerIdentityfrom pipelex.system.job_metadata import validate_request_idfrom pipelex.system.storage_scope import LOCAL_STORAGE_SCOPE, LOCAL_USER_IDfrom pipelex.system.telemetry.events import EventName, EventProperty, Outcome+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsMethodSource+from pipelex.system.usage_stats.usage_stats_runs import method_source_scopefrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorif TYPE_CHECKING:@@ -226,6 +228,17 @@ class PipelexMTHDSProtocol(MTHDSProtocol["PipeOutput"]):"""return CallerIdentity.make_from_host(user_id=self.user_id, extras=self.extras)+ def _method_source(self, *, mthds_contents: list[str] | None) -> UsageStatsMethodSource:+ """Where the method this runner runs came from, as far as the runner can tell, for the run's usage statistics.++ A caller that knows better, a CLI resolving an installed method or an address, declares it first and wins.+ """+ if mthds_contents:+ return UsageStatsMethodSource.INLINE_SOURCE+ if self.bundle_uris or self.library_dirs:+ return UsageStatsMethodSource.LOCAL_BUNDLE+ return UsageStatsMethodSource.OTHER+@overrideasync def execute(self,@@ -340,7 +353,8 @@ class PipelexMTHDSProtocol(MTHDSProtocol["PipeOutput"]):request_id=request_id,)effective_pipe_run = self._pipe_run or get_pipe_run()- pipe_output = await effective_pipe_run.run(pipe_job, delivery_assignment=delivery_assignment)+ with method_source_scope(source=self._method_source(mthds_contents=mthds_contents)):+ pipe_output = await effective_pipe_run.run(pipe_job, delivery_assignment=delivery_assignment)except PipelexError as exc:# If pipe_job is None, the error occurred during pipeline_run_setup, before the job existed:# it is not a run failure, and it propagates untouched.diff --git a/pipelex/pipeline/usage_stats_reporting.py b/pipelex/pipeline/usage_stats_reporting.pynew file mode 100644index 000000000..94cd45176--- /dev/null+++ b/pipelex/pipeline/usage_stats_reporting.py@@ -0,0 +1,636 @@+"""The usage-statistics events of the method machinery: a run's `run_completed` and `model_used`, `method_validated`+and `code_generated`, built from what the runtime already knows and nothing a user wrote.++Every property is a count, a boolean or a value from a closed list: a bundle is described by how many pipes and+concepts it declares and which language features it uses, never by a name, and a model by the handle a shipped backend+table gives it, or `other`. The contract is `conformance/specs/usage-statistics.md`, "Events".++Building an event never changes the work it describes: nothing is built while statistics are off, and the events go+through the process's client, which owns their delivery.+"""++import re+import time+from collections.abc import Callable, Iterator, Sequence+from functools import cache+from types import TracebackType+from typing import NamedTuple, Self, cast++import tomli++from pipelex import log+from pipelex.base_exceptions import iter_cause_chain+from pipelex.cogt.usage.token_category import TokenCategory+from pipelex.cogt.usage.usage_cost import compute_tokens_usage_cost+from pipelex.core.concepts.concept import Concept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.graph.graphspec import GraphSpec+from pipelex.interpreter_hub import get_concept_library+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.libraries.exceptions import LibraryError+from pipelex.libraries.library_crate import LibraryCrate+from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel+from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequence+from pipelex.pipe_machinery.pipe_abstract import PipeAbstract+from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.system.usage_stats.usage_stats_client import note_command_error_domain, record_usage_stats_event, usage_stats_are_enabled+from pipelex.system.usage_stats.usage_stats_contract import (+ MODEL_HANDLE_PATTERN,+ OTHER_MODEL,+ UsageStatsCodegenArtifact,+ UsageStatsCodegenOutcome,+ UsageStatsCodegenTarget,+ UsageStatsEventName,+ UsageStatsOutcome,+ UsageStatsProvider,+ UsageStatsValidationOutcome,+)+from pipelex.system.usage_stats.usage_stats_events import (+ MAX_BUNDLE_DEPTH,+ MAX_BUNDLE_SIZE,+ MAX_CALLS,+ MAX_IMAGES,+ MAX_MODELS,+ MAX_PAGES,+ MAX_PIPE_COUNT,+ MAX_TOKENS,+ MAX_VALIDATION_ITEMS,+ BundleShape,+ CodeGeneratedProperties,+ MethodValidatedProperties,+ ModelUsedProperties,+ RunCompletedProperties,+ UsageStatsEvent,+ bounded,+ bounded_cost,+ bounded_duration_ms,+)+from pipelex.system.usage_stats.usage_stats_runs import (+ FAILURE_CATEGORIES,+ RunInferenceTally,+ current_method_source,+ usage_stats_error_domain,+)+from pipelex.tools.log.error_fields import error_fields+from pipelex.tracing.trace_events import PipeStartEvent, TraceEvent, UsageReportEvent++_MODEL_HANDLE = re.compile(MODEL_HANDLE_PATTERN)++#: The `pipes_*` property each pipe type pipelex ships is counted under; any other type is counted under `pipes_other`.+PIPE_TYPE_COUNT_FIELDS: dict[str, str] = {+ "PipeLLM": "pipes_llm",+ "PipeExtract": "pipes_extract",+ "PipeImgGen": "pipes_img_gen",+ "PipeSearch": "pipes_search",+ "PipeCompose": "pipes_compose",+ "PipeStructure": "pipes_structure",+ "PipeDocGen": "pipes_doc_gen",+ "PipeJudge": "pipes_judge",+ "PipeFunc": "pipes_func",+ "PipeSequence": "pipes_sequence",+ "PipeParallel": "pipes_parallel",+ "PipeCondition": "pipes_condition",+ "PipeBatch": "pipes_batch",+}+PIPES_OTHER_FIELD = "pipes_other"+_BATCH_PIPE_TYPE = "PipeBatch"+_PARALLEL_PIPE_TYPE = "PipeParallel"+_CONDITION_PIPE_TYPE = "PipeCondition"+_DEFAULTS_TABLE = "defaults"+_TOML_SUFFIX = ".toml"+++# -------------------------------------------------------------------------------------------------+# What a bundle declares+# -------------------------------------------------------------------------------------------------+++class BundleFeatureFlags(NamedTuple):+ batch: bool+ parallel: bool+ condition: bool+ structured_output: bool+++def _resolver_for(*, pipes: Sequence[PipeAbstract]) -> Callable[..., PipeAbstract | None]:+ """Find a pipe a controller names among the bundle's own pipes: by its domain-qualified ref, or bare in its domain."""+ by_ref: dict[str, PipeAbstract] = {pipe.pipe_ref: pipe for pipe in pipes}++ def resolve(*, pipe_code: str, domain_code: str) -> PipeAbstract | None:+ return by_ref.get(pipe_code) or by_ref.get(f"{domain_code}.{pipe_code}")++ return resolve+++def controller_depth(*, pipe: PipeAbstract, pipes: Sequence[PipeAbstract]) -> int:+ """How deeply controllers nest under `pipe`: 0 for an operator, 1 for a controller of operators, and so on.++ A pipe of a dependency package, or one the bundle does not hold, ends the walk, and so does a cycle.+ """+ resolve = _resolver_for(pipes=pipes)+ memo: dict[str, int] = {}++ def depth_of(*, current: PipeAbstract, visiting: frozenset[str]) -> int:+ visit_key = current.visit_key+ if not current.is_controller or visit_key in visiting:+ return 0+ if visit_key in memo:+ return memo[visit_key]+ deepest_child = 0+ for dependency_code in current.pipe_dependencies():+ child = resolve(pipe_code=dependency_code, domain_code=current.domain_code)+ if child is not None:+ deepest_child = max(deepest_child, depth_of(current=child, visiting=visiting | {visit_key}))+ memo[visit_key] = 1 + deepest_child+ return memo[visit_key]++ return depth_of(current=pipe, visiting=frozenset())+++def bundle_shape_of(*, pipes: Sequence[PipeAbstract], concepts: Sequence[Concept], main_pipe: PipeAbstract | None) -> BundleShape:+ """The bundle's size and depth: under `main_pipe` for a run, the deepest of all its pipes otherwise."""+ roots = [main_pipe] if main_pipe is not None else list(pipes)+ depth = max((controller_depth(pipe=root, pipes=pipes) for root in roots), default=0)+ return BundleShape(+ bundle_pipes=bounded(value=len(pipes), maximum=MAX_BUNDLE_SIZE),+ bundle_concepts=bounded(value=len(concepts), maximum=MAX_BUNDLE_SIZE),+ bundle_depth=bounded(value=depth, maximum=MAX_BUNDLE_DEPTH),+ )+++def is_structured_concept(*, concept: Concept) -> bool:+ """Whether a concept is structured: it declares fields, or its structure is a class of its own rather than a native's.++ A concept that refines another counts as not structured unless it declares fields itself: the walk up its parents+ is the library's to make, and a refinement of a native, the common case, is not structured.+ """+ if concept.declared_structure is not None:+ return True+ if Concept.is_native_concept(concept=concept) or concept.is_described_only:+ return False+ return concept.refines is None+++def _batches(*, pipe: PipeAbstract) -> bool:+ if pipe.pipe_type == _BATCH_PIPE_TYPE:+ return True+ if isinstance(pipe, PipeSequence):+ return any(step.batch_params is not None for step in pipe.pipe_steps)+ if isinstance(pipe, PipeParallel):+ return any(sub_pipe.batch_params is not None for sub_pipe in pipe.parallel_sub_pipes)+ return False+++def bundle_features_of(*, pipes: Sequence[PipeAbstract]) -> BundleFeatureFlags:+ """Which language features the bundle's pipes use."""+ return BundleFeatureFlags(+ batch=any(_batches(pipe=pipe) for pipe in pipes),+ parallel=any(pipe.pipe_type == _PARALLEL_PIPE_TYPE for pipe in pipes),+ condition=any(pipe.pipe_type == _CONDITION_PIPE_TYPE for pipe in pipes),+ structured_output=any(is_structured_concept(concept=pipe.output.concept) for pipe in pipes),+ )+++def _contents_of(*, content: StuffContent) -> Iterator[StuffContent]:+ yield content+ if isinstance(content, ListContent):+ list_content = cast("ListContent[StuffContent]", content)+ for item in list_content.items:+ yield from _contents_of(content=item)+++def input_kinds_of(*, working_memory: WorkingMemory | None) -> tuple[bool, bool]:+ """Whether the run's inputs hold an image, and whether they hold a document."""+ if working_memory is None:+ return False, False+ has_image = False+ has_document = False+ for stuff in working_memory.root.values():+ for content in _contents_of(content=stuff.content):+ has_image = has_image or isinstance(content, ImageContent)+ has_document = has_document or isinstance(content, DocumentContent)+ return has_image, has_document+++# -------------------------------------------------------------------------------------------------+# Providers and models+# -------------------------------------------------------------------------------------------------+++@cache+def shipped_backend_models() -> dict[str, frozenset[str]]:+ """The model names each backend table of this release's kit declares, by backend name."""+ backends_dir = get_kit_configs_dir().joinpath("inference", "backends")+ shipped: dict[str, frozenset[str]] = {}+ for entry in backends_dir.iterdir():+ if not entry.name.endswith(_TOML_SUFFIX):+ continue+ try:+ document = tomli.loads(entry.read_text(encoding="utf-8"))+ except (OSError, UnicodeDecodeError, tomli.TOMLDecodeError):+ continue+ shipped[entry.name.removesuffix(_TOML_SUFFIX)] = frozenset(name for name in document if name != _DEFAULTS_TABLE)+ return shipped+++def reported_model(*, provider: UsageStatsProvider, model_handle: str) -> str:+ """The model a `model_used` names: its handle when the release ships it for that provider, `other` otherwise.++ A custom backend's model is always `other`, since its name is the user's.+ """+ if provider.is_custom or _MODEL_HANDLE.fullmatch(model_handle) is None:+ return OTHER_MODEL+ shipped = shipped_backend_models()+ known: bool+ if provider.serves_every_shipped_table:+ known = any(model_handle in models for models in shipped.values())+ else:+ known = model_handle in shipped.get(provider, frozenset())+ return model_handle if known else OTHER_MODEL+++# -------------------------------------------------------------------------------------------------+# A run: `run_completed` and `model_used`+# -------------------------------------------------------------------------------------------------+++class _ModelUsageRow:+ """What a run did with one reported model, gathered from the tally of its calls and from its usage events."""++ def __init__(self) -> None:+ self.tallied_calls = 0+ self.usage_reports = 0+ self.failures = 0+ self.failures_by_category: dict[str, int] = dict.fromkeys(FAILURE_CATEGORIES, 0)+ self.input_tokens = 0+ self.output_tokens = 0+ self.cached_tokens = 0+ self.images_generated = 0+ self.pages_extracted = 0+ self.cost_usd = 0.0+ self.is_cost_known = True++ @property+ def calls(self) -> int:+ return max(self.tallied_calls, self.usage_reports)++ @property+ def known_cost(self) -> float | None:+ return self.cost_usd if self.is_cost_known else None+++def _model_usage_rows(*, tally: RunInferenceTally | None, events: Sequence[TraceEvent]) -> dict[tuple[UsageStatsProvider, str], _ModelUsageRow]:+ """The run's models as `model_used` reports them, merged by provider and reported model."""+ rows: dict[tuple[UsageStatsProvider, str], _ModelUsageRow] = {}++ def row_for(*, model_handle: str, backend_name: str | None) -> _ModelUsageRow:+ provider = UsageStatsProvider.for_backend(backend_name=backend_name)+ key = (provider, reported_model(provider=provider, model_handle=model_handle))+ if key not in rows:+ rows[key] = _ModelUsageRow()+ return rows[key]++ if tally is not None:+ for (tallied_handle, tallied_backend), model_tally in tally.models.items():+ row = row_for(model_handle=tallied_handle, backend_name=tallied_backend)+ row.tallied_calls += model_tally.calls+ row.failures += model_tally.failures+ for category, count in model_tally.failures_by_category.items():+ row.failures_by_category[category] += count+ row.images_generated += model_tally.images_generated+ row.pages_extracted += model_tally.pages_extracted++ for event in events:+ if not isinstance(event, UsageReportEvent):+ continue+ tokens_usage = event.tokens_usage+ # The report names the backend that served the call, which the tally's row for that call names too.+ row = row_for(model_handle=tokens_usage.inference_model_name, backend_name=tokens_usage.backend_name)+ row.usage_reports += 1+ if tokens_usage.pricing_unit.counts_tokens:+ nb_tokens_by_category = tokens_usage.nb_tokens_by_category+ row.input_tokens += nb_tokens_by_category.get(TokenCategory.INPUT, 0)+ row.output_tokens += nb_tokens_by_category.get(TokenCategory.OUTPUT, 0)+ row.cached_tokens += nb_tokens_by_category.get(TokenCategory.INPUT_CACHED, 0)+ cost = compute_tokens_usage_cost(tokens_usage)+ if cost is None:+ row.is_cost_known = False+ else:+ row.cost_usd += cost+ return rows+++def count_pipe_types(*, pipe_types: Sequence[str]) -> dict[str, int]:+ """Every `pipes_*` count, zero included, of the pipe types a run executed."""+ counts: dict[str, int] = dict.fromkeys([*PIPE_TYPE_COUNT_FIELDS.values(), PIPES_OTHER_FIELD], 0)+ for pipe_type in pipe_types:+ counts[PIPE_TYPE_COUNT_FIELDS.get(pipe_type, PIPES_OTHER_FIELD)] += 1+ return {field: bounded(value=count, maximum=MAX_PIPE_COUNT) for field, count in counts.items()}+++def executed_pipe_types(*, events: Sequence[TraceEvent], closed_graph: GraphSpec | None) -> list[str]:+ """The types of the pipes a run executed: from its pipe-start events, or from its tracer's graph when it wrote none.++ A run reporting its costs without its graph writes no pipe-start event, while its tracer still drew the graph in memory.+ """+ started = [event.pipe_type for event in events if isinstance(event, PipeStartEvent)]+ if started or closed_graph is None:+ return started+ return [node.pipe_type for node in closed_graph.nodes if node.pipe_type]+++class RunUsageFacts(NamedTuple):+ """What a run's end knows that its `run_completed` describes."""++ outcome: UsageStatsOutcome+ error: BaseException | None+ is_dry_run: bool+ duration_seconds: float+ main_pipe: PipeAbstract+ own_pipes: Sequence[PipeAbstract]+ own_concepts: Sequence[Concept]+ has_image_input: bool+ has_document_input: bool+ events: Sequence[TraceEvent] | None+ closed_graph: GraphSpec | None+ tally: RunInferenceTally | None+++def build_run_usage_events(*, facts: RunUsageFacts) -> list[UsageStatsEvent]:+ """The run's `run_completed`, then one `model_used` per model when its events were read."""+ shape = bundle_shape_of(pipes=facts.own_pipes, concepts=facts.own_concepts, main_pipe=facts.main_pipe)+ features = bundle_features_of(pipes=facts.own_pipes)+ is_usage_known = facts.events is not None+ run_properties = RunCompletedProperties(+ outcome=facts.outcome,+ error_domain=usage_stats_error_domain(error=facts.error) if facts.outcome.is_failed else None,+ dry_run=facts.is_dry_run,+ method_source=current_method_source(),+ duration_ms=bounded_duration_ms(seconds=facts.duration_seconds),+ bundle_pipes=shape.bundle_pipes,+ bundle_concepts=shape.bundle_concepts,+ bundle_depth=shape.bundle_depth,+ feature_batch=features.batch,+ feature_parallel=features.parallel,+ feature_condition=features.condition,+ feature_structured_output=features.structured_output,+ feature_image_input=facts.has_image_input,+ feature_document_input=facts.has_document_input,+ usage_known=is_usage_known,+ )+ if facts.events is None:+ return [UsageStatsEvent(name=UsageStatsEventName.RUN_COMPLETED, properties=run_properties)]++ rows = _model_usage_rows(tally=facts.tally, events=facts.events)+ model_costs = [row.known_cost for row in rows.values()]+ run_cost: float | None = None if any(cost is None for cost in model_costs) else sum(cost for cost in model_costs if cost is not None)+ tallied_calls = facts.tally.calls if facts.tally is not None else 0+ tallied_failures = facts.tally.failures if facts.tally is not None else 0+ pipe_counts = count_pipe_types(pipe_types=executed_pipe_types(events=facts.events, closed_graph=facts.closed_graph))+ run_properties = run_properties.model_copy(+ update={+ **pipe_counts,+ "models_used": bounded(value=len(rows), maximum=MAX_MODELS),+ "inference_calls": bounded(value=max(tallied_calls, sum(row.calls for row in rows.values())), maximum=MAX_CALLS),+ "inference_failures": bounded(value=tallied_failures, maximum=MAX_CALLS),+ "input_tokens": bounded(value=sum(row.input_tokens for row in rows.values()), maximum=MAX_TOKENS),+ "output_tokens": bounded(value=sum(row.output_tokens for row in rows.values()), maximum=MAX_TOKENS),+ "cached_tokens": bounded(value=sum(row.cached_tokens for row in rows.values()), maximum=MAX_TOKENS),+ "images_generated": bounded(value=sum(row.images_generated for row in rows.values()), maximum=MAX_IMAGES),+ "pages_extracted": bounded(value=sum(row.pages_extracted for row in rows.values()), maximum=MAX_PAGES),+ "cost_usd": bounded_cost(cost_usd=run_cost),+ }+ )+ # Validated again: `model_copy` does not validate the counts it sets.+ usage_events = [+ UsageStatsEvent(name=UsageStatsEventName.RUN_COMPLETED, properties=RunCompletedProperties.model_validate(run_properties.model_dump()))+ ]+ for (provider, model), row in rows.items():+ usage_events.append(+ UsageStatsEvent(+ name=UsageStatsEventName.MODEL_USED,+ properties=ModelUsedProperties(+ provider=provider,+ model=model,+ calls=bounded(value=row.calls, maximum=MAX_CALLS),+ failures=bounded(value=row.failures, maximum=MAX_CALLS),+ failures_transient=bounded(value=row.failures_by_category["transient"], maximum=MAX_CALLS),+ failures_configuration=bounded(value=row.failures_by_category["configuration"], maximum=MAX_CALLS),+ failures_content=bounded(value=row.failures_by_category["content"], maximum=MAX_CALLS),+ failures_capacity=bounded(value=row.failures_by_category["capacity"], maximum=MAX_CALLS),+ failures_ambiguous=bounded(value=row.failures_by_category["ambiguous"], maximum=MAX_CALLS),+ failures_unknown=bounded(value=row.failures_by_category["unknown"], maximum=MAX_CALLS),+ input_tokens=bounded(value=row.input_tokens, maximum=MAX_TOKENS),+ output_tokens=bounded(value=row.output_tokens, maximum=MAX_TOKENS),+ cached_tokens=bounded(value=row.cached_tokens, maximum=MAX_TOKENS),+ images_generated=bounded(value=row.images_generated, maximum=MAX_IMAGES),+ pages_extracted=bounded(value=row.pages_extracted, maximum=MAX_PAGES),+ cost_usd=bounded_cost(cost_usd=row.known_cost),+ ),+ )+ )+ return usage_events+++def report_run_usage(*, facts: RunUsageFacts) -> None:+ """Record a run's `run_completed` and `model_used` events, and its error domain for the command that ran it."""+ usage_events = build_run_usage_events(facts=facts)+ if facts.outcome.is_failed:+ note_command_error_domain(error_domain=usage_stats_error_domain(error=facts.error))+ for usage_event in usage_events:+ record_usage_stats_event(event=usage_event)+++# -------------------------------------------------------------------------------------------------+# A validation: `method_validated`+# -------------------------------------------------------------------------------------------------+++def _find_validate_bundle_error(*, error: BaseException | None) -> ValidateBundleError | None:+ """The invalid verdict an error carries, raised itself or handled on the way to it (a CLI's exit after rendering it)."""+ current = error+ seen: set[int] = set()+ while current is not None and id(current) not in seen:+ seen.add(id(current))+ for node in iter_cause_chain(current):+ if isinstance(node, ValidateBundleError):+ return node+ current = current.__context__+ return None+++class MethodValidationReport:+ """Times one validation of a method and records its `method_validated` when it ends, however it ends.++ A context manager around the validation. Its verdict is read off the way the block ends: an invalid verdict when a+ `ValidateBundleError` raised in it, or handled on the way to the exception leaving it; `failed` for any other+ exception, unless the block declared the verdict valid first (a valid library refused for its pending signatures+ is valid); `valid` otherwise. The block describes the bundle as soon as it knows it.+ """++ def __init__(self) -> None:+ self._started_at = 0.0+ self._pipes: Sequence[PipeAbstract] = ()+ self._concepts: Sequence[Concept] = ()+ self._warnings = 0+ self._is_declared_valid = False+ self._could_not_describe = False++ def __enter__(self) -> Self:+ self._started_at = time.monotonic()+ return self++ def __exit__(+ self,+ exc_type: type[BaseException] | None,+ exc_value: BaseException | None,+ exc_traceback: TracebackType | None,+ ) -> None:+ if not usage_stats_are_enabled():+ return+ try:+ self._record(exc_value=exc_value)+ except (ValueError, OSError) as report_error:+ # An event that will not validate, a state that cannot be read: a statistic that cannot be built is lost,+ # and it never replaces the verdict it describes.+ log.debug("A usage statistics event could not be built", fields=error_fields(exc=report_error))++ def _record(self, *, exc_value: BaseException | None) -> None:+ if self._could_not_describe:+ # A bundle the report could not describe would be counted wrong: the event is lost instead.+ return+ outcome: UsageStatsValidationOutcome+ errors = 0+ invalid_verdict = _find_validate_bundle_error(error=exc_value)+ if invalid_verdict is not None:+ outcome = UsageStatsValidationOutcome.INVALID+ errors = max(1, len(invalid_verdict.validation_error_items()))+ elif exc_value is None or self._is_declared_valid:+ outcome = UsageStatsValidationOutcome.VALID+ else:+ outcome = UsageStatsValidationOutcome.FAILED+ shape = bundle_shape_of(pipes=self._pipes, concepts=self._concepts, main_pipe=None)+ record_usage_stats_event(+ event=UsageStatsEvent(+ name=UsageStatsEventName.METHOD_VALIDATED,+ properties=MethodValidatedProperties(+ outcome=outcome,+ duration_ms=bounded_duration_ms(seconds=time.monotonic() - self._started_at),+ errors=bounded(value=errors, maximum=MAX_VALIDATION_ITEMS),+ warnings=bounded(value=self._warnings, maximum=MAX_VALIDATION_ITEMS),+ bundle_pipes=shape.bundle_pipes,+ bundle_concepts=shape.bundle_concepts,+ bundle_depth=shape.bundle_depth,+ ),+ )+ )++ def describe(self, *, pipes: Sequence[PipeAbstract]) -> None:+ """Describe the validated bundle by its pipes and by the concepts of the library it loaded, which must be current."""+ if not usage_stats_are_enabled():+ return+ try:+ self._pipes = list(pipes)+ self._concepts = get_concept_library().list_own_concepts()+ except (RuntimeError, LibraryError) as describe_error:+ # No interpreter hub or no library manager set up (`RuntimeError`), or no current library (`LibraryError`):+ # describing the bundle for a statistic must never break the validation itself.+ self._could_not_describe = True+ log.debug("A validated bundle could not be described for usage statistics", fields=error_fields(exc=describe_error))++ def count_warnings(self, *, count: int) -> None:+ self._warnings += count++ def declare_valid(self) -> None:+ """Declare the verdict valid, so a refusal the block raises next for another reason does not read as a failure."""+ self._is_declared_valid = True+++# -------------------------------------------------------------------------------------------------+# A code generation: `code_generated`+# -------------------------------------------------------------------------------------------------+++def usage_stats_codegen_target(*, target: str) -> UsageStatsCodegenTarget:+ """The target flavour a `code_generated` names: the codegen target itself, which the contract lists, or `other`."""+ try:+ return UsageStatsCodegenTarget(target)+ except ValueError:+ return UsageStatsCodegenTarget.OTHER+++class CodeGenerationReport:+ """Times one code generation and records its `code_generated` when it ends.++ A context manager around the generation, the load of its library included. It succeeded unless an exception leaves+ the block, or unless the block declared the outcome first: a surface that answers a failure without raising, an+ API's invalid verdict, declares it failed, and one that leaves by an exception after a generation with nothing to+ generate declares it succeeded.+ """++ def __init__(self, *, artifact: UsageStatsCodegenArtifact, target: UsageStatsCodegenTarget | None = None) -> None:+ self._artifact = artifact+ self._target = target+ self._started_at = 0.0+ self._pipes = 0+ self._concepts = 0+ self._declared_outcome: UsageStatsCodegenOutcome | None = None++ @property+ def artifact(self) -> UsageStatsCodegenArtifact:+ """What the generation produces, as its `code_generated` reports it."""+ return self._artifact++ def __enter__(self) -> Self:+ self._started_at = time.monotonic()+ return self++ def __exit__(+ self,+ exc_type: type[BaseException] | None,+ exc_value: BaseException | None,+ exc_traceback: TracebackType | None,+ ) -> None:+ if not usage_stats_are_enabled():+ return+ try:+ self._record(exc_value=exc_value)+ except (ValueError, OSError) as report_error:+ # A statistic that cannot be built is lost; it never replaces the outcome of the work it describes.+ log.debug("A usage statistics event could not be built", fields=error_fields(exc=report_error))++ def _record(self, *, exc_value: BaseException | None) -> None:+ target = self._target+ if self._artifact.has_target and target is None:+ target = UsageStatsCodegenTarget.OTHER+ outcome = self._declared_outcome+ if outcome is None:+ outcome = UsageStatsCodegenOutcome.SUCCEEDED if exc_value is None else UsageStatsCodegenOutcome.FAILED+ record_usage_stats_event(+ event=UsageStatsEvent(+ name=UsageStatsEventName.CODE_GENERATED,+ properties=CodeGeneratedProperties(+ outcome=outcome,+ artifact=self._artifact,+ target=target if self._artifact.has_target else None,+ duration_ms=bounded_duration_ms(seconds=time.monotonic() - self._started_at),+ bundle_pipes=bounded(value=self._pipes, maximum=MAX_BUNDLE_SIZE),+ bundle_concepts=bounded(value=self._concepts, maximum=MAX_BUNDLE_SIZE),+ ),+ )+ )++ def describe(self, *, crate: LibraryCrate) -> None:+ """Describe the library generated from by the size of its crate, which the event counts."""+ self._pipes = len(crate.pipes)+ self._concepts = len(crate.concepts)++ def declare_failed(self) -> None:+ self._declared_outcome = UsageStatsCodegenOutcome.FAILED++ def declare_succeeded(self) -> None:+ self._declared_outcome = UsageStatsCodegenOutcome.SUCCEEDEDdiff --git a/pipelex/pipeline/validate_in_process.py b/pipelex/pipeline/validate_in_process.pyindex 5bf251052..4a0896917 100644--- a/pipelex/pipeline/validate_in_process.py+++ b/pipelex/pipeline/validate_in_process.py@@ -33,6 +33,7 @@ from pipelex.pipeline.build_pipe_io_artifacts import build_pipe_io_artifactsfrom pipelex.pipeline.controller_taint import collect_controller_taint_analysesfrom pipelex.pipeline.input_form import qualify_current_library_cratefrom pipelex.pipeline.liftable_pipes import LiftablePipeEntry, build_liftable_pipes+from pipelex.pipeline.usage_stats_reporting import MethodValidationReportfrom pipelex.pipeline.validate_bundle import validate_bundlefrom pipelex.pipeline.validation_report import PipelexValidationReport, build_validation_reportfrom pipelex.system.caller_identity import CallerIdentity, scoped_caller_identity@@ -90,7 +91,7 @@ async def validate_bundles_in_process(Raises:PipelexError: When the bundle is invalid (parse, static validation, or dry-run failure)."""- with scoped_caller_identity(caller_identity=caller_identity):+ with scoped_caller_identity(caller_identity=caller_identity), MethodValidationReport() as validation_report:return await _validate_bundles_in_scope(mthds_contents=mthds_contents,mthds_sources=mthds_sources,@@ -99,6 +100,7 @@ async def validate_bundles_in_process(graph_pipe_code=graph_pipe_code,log_context=log_context,library_dirs_are_callers=library_dirs_are_callers,+ validation_report=validation_report,)@@ -111,8 +113,9 @@ async def _validate_bundles_in_scope(graph_pipe_code: str | None,log_context: str,library_dirs_are_callers: bool,+ validation_report: MethodValidationReport,) -> PipelexValidationReport:- """The body of :func:`validate_bundles_in_process`, run inside the caller's scope."""+ """The body of :func:`validate_bundles_in_process`, run inside the caller's scope and timed by `validation_report`."""# `validate_bundle` deliberately leaves its validation library OPEN and current on# success (the CLI surfaces consume it before process exit). This entry point is# long-lived, so restore the caller's current-library and tear the validation library@@ -141,6 +144,7 @@ async def _validate_bundles_in_scope(# current — the graph arm and the finally must target the SAME library even if# something inside the window later moves the contextvar.validation_library_id = get_current_library_id_or_none()+ validation_report.describe(pipes=result.pipes)# One crate qualification per validate pass — the descriptors and the hint lint read the same one.qualified_crate = qualify_current_library_crate()# The three I/O artifacts come from the one builder a run uses too, so a validate report and@@ -155,6 +159,7 @@ async def _validate_bundles_in_scope(entry_pipe_refs=collect_entry_pipe_refs(result.blueprints),qualified_crate=qualified_crate,)+ validation_report.count_warnings(count=len(warnings))graph_target_ref = graph_pipe_code if graph_pipe_code is not None else select_primary_blueprint(result.blueprints).main_pipe_refgraph_spec: GraphSpec | None = await best_effort_graph_spec(pipe_ref=graph_target_ref,diff --git a/pipelex/system/configuration/config_loader.py b/pipelex/system/configuration/config_loader.pyindex 2850f6b39..8703fc3e5 100644--- a/pipelex/system/configuration/config_loader.py+++ b/pipelex/system/configuration/config_loader.py@@ -95,6 +95,12 @@ class ConfigLoader:real project root.Returns the directory containing the marker, or None if not found.++ Raises:+ OSError: When whether a marker exists cannot be told, as when a directory on the way up may not be+ traversed. Each marker is stat'ed rather than asked `Path.exists()`, which from Python 3.14 swallows+ every `OSError` and answers `False`.+ RuntimeError: When no home directory can be found."""current = start_dir.resolve()home_dir = Path.home().resolve()@@ -102,8 +108,11 @@ class ConfigLoader:if current == home_dir:return Nonefor marker in PROJECT_ROOT_MARKERS:- if (current / marker).exists():- return current+ try:+ (current / marker).stat()+ except (FileNotFoundError, NotADirectoryError):+ continue+ return currentparent = current.parentif parent == current:return Nonediff --git a/pipelex/system/configuration/configs.py b/pipelex/system/configuration/configs.pyindex ff56c3d33..4a95dd4fe 100644--- a/pipelex/system/configuration/configs.py+++ b/pipelex/system/configuration/configs.py@@ -205,9 +205,22 @@ class InterpreterConfig(ConfigModel):builder: BuilderConfig+class UsageStatsConfig(ConfigModel):+ """`[usage_stats]`: whether pipelex may send its anonymous usage statistics.++ The switch is read from two files alone, never from this merged model: pipelex's packaged configuration, where a+ packager may ship `false`, and the project's own `.pipelex/pipelex.toml`, where a team commits `false`; both can only+ turn statistics off (`pipelex.system.usage_stats.usage_stats_switches`). The model declares the table so that a file+ carrying it loads, and validates it like every other setting.+ """++ enabled: bool++class PipelexConfig(ConfigRoot):runtime: RuntimeConfiginference: InferenceConfiginterpreter: InterpreterConfigrun: RunConfigkit: KitConfig+ usage_stats: UsageStatsConfigdiff --git a/pipelex/system/usage_stats/__init__.py b/pipelex/system/usage_stats/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/system/usage_stats/usage_stats_client.py b/pipelex/system/usage_stats/usage_stats_client.pynew file mode 100644index 000000000..2704b802c--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_client.py@@ -0,0 +1,652 @@+"""The process's usage-statistics client: the switches it read, the events it holds, and how it delivers them.++A dedicated client, separate from the runtime's own telemetry (`TelemetryManager` and `telemetry.toml`), the user's+PostHog client and OpenTelemetry, all of which it never touches. One client per process:++- **A CLI command** (`pipelex`, `pipelex-agent`) configures it before the command does any work, which reads every+ switch and prints the notice; the command's events are held in memory and, when it ends, written to the spool for a+ detached sender (`usage_stats_core`, run by its file path), or posted in-process within one second in CI, in a+ container, or when the sender cannot start, followed within that second by what earlier commands queued.+- **A Python program, or a long-running process** such as the self-hosted API, configures it when it boots pipelex; a+ daemon thread posts events within ten seconds of their creation, and at exit (or at shutdown) what is left is flushed+ within the budget and spooled when it cannot be sent. Each time its events get through, what earlier processes+ queued goes too, within the same budget. Such a process never starts a child process.++With `PIPELEX_USAGE_STATS_DEBUG`, each event is printed on stderr as one JSON line instead, and nothing is sent or+spooled. The contract is `conformance/specs/usage-statistics.md`.+"""++import atexit+import json+import os+import sys+import threading+import time+from collections.abc import Mapping+from enum import StrEnum+from pathlib import Path+from typing import Any, NamedTuple++from pipelex import log+from pipelex.system.runtime import IntegrationMode+from pipelex.system.usage_stats.usage_stats_contract import (+ NOTICE_VERSION,+ TOOL_NAME,+ UsageStatsErrorDomain,+ UsageStatsHost,+ UsageStatsIntegration,+ UsageStatsOffBy,+ UsageStatsSurface,+ render_notice,+)+from pipelex.system.usage_stats.usage_stats_core import (+ BUDGET_BACKGROUND_POST_DELAY_MS,+ BUDGET_CLI_IN_PROCESS_MS,+ BUDGET_PYTHON_EXIT_FLUSH_MS,+ BUDGET_SHUTDOWN_FLUSH_MS,+ InstallationId,+ PostVerdict,+ SpoolContent,+ UsageStatsState,+ drain_spool_until,+ post_in_process,+ write_spool_file,+)+from pipelex.system.usage_stats.usage_stats_delivery import start_detached_sender+from pipelex.system.usage_stats.usage_stats_environment import (+ current_arch,+ current_language_runtime,+ current_os,+ declared_host,+ is_ci,+ is_container,+ is_debug,+ release_version,+ resolve_endpoint,+)+from pipelex.system.usage_stats.usage_stats_events import CommonProperties, InitChoices, UsageStatsEvent, make_envelope+from pipelex.system.usage_stats.usage_stats_switches import read_switches+from pipelex.tools.log.error_fields import error_fields+from pipelex.tools.misc.package_utils import get_package_version++#: How long the background thread waits after an event's creation for others to join its batch.+BACKGROUND_COALESCE_SECONDS = 1.0+#: The key each event printed under the debug variable is wrapped in.+DEBUG_KEY = "usage_stats_debug"+++class UsageStatsDelivery(StrEnum):+ """How a process delivers its events."""++ CLI_COMMAND = "cli_command"+ PYTHON_PROGRAM = "python_program"+ LONG_RUNNING = "long_running"++ @property+ def flush_budget_ms(self) -> int:+ """The budget of the flush at exit or at shutdown."""+ match self:+ case UsageStatsDelivery.CLI_COMMAND:+ return BUDGET_CLI_IN_PROCESS_MS+ case UsageStatsDelivery.PYTHON_PROGRAM:+ return BUDGET_PYTHON_EXIT_FLUSH_MS+ case UsageStatsDelivery.LONG_RUNNING:+ return BUDGET_SHUTDOWN_FLUSH_MS+++class UsageStatsSettings(NamedTuple):+ """What a process decided about its statistics before doing any work."""++ off_by: tuple[UsageStatsOffBy, ...]+ origin: str | None+ surface: UsageStatsSurface+ integration: UsageStatsIntegration+ host: UsageStatsHost | None+ delivery: UsageStatsDelivery+ is_debug: bool+ is_ci: bool+ is_container: bool+ state: UsageStatsState+++def write_stderr_line(*, line: str) -> None:+ """Write one line to stderr; a stderr that is closed or broken loses the line and nothing else."""+ try:+ sys.stderr.write(f"{line}\n")+ sys.stderr.flush()+ except (OSError, ValueError):+ # A closed stream raises ValueError, a broken pipe OSError: the line is lost, the command is not.+ return+++class _BackgroundSender:+ """The daemon thread of a Python program or a long-running process, posting events within ten seconds of their creation."""++ def __init__(self, *, settings: UsageStatsSettings, own_id: str) -> None:+ self._settings = settings+ self._own_id = own_id+ self._condition = threading.Condition()+ self._pending: list[tuple[float, dict[str, Any]]] = []+ self._in_flight: list[dict[str, Any]] = []+ self._thread: threading.Thread | None = None+ self._closed = False++ def enqueue(self, *, envelope: dict[str, Any]) -> None:+ with self._condition:+ if self._closed:+ self._spool(events=[envelope])+ return+ self._pending.append((time.monotonic(), envelope))+ if self._thread is None or not self._thread.is_alive():+ # The first event starts the thread, and a later one starts it again should it ever have ended.+ self._thread = threading.Thread(target=self._run, name="pipelex-usage-stats", daemon=True)+ self._thread.start()+ self._condition.notify_all()++ def _run(self) -> None:+ while True:+ with self._condition:+ while not self._pending and not self._closed:+ self._condition.wait()+ if self._closed:+ return+ oldest_created = self._pending[0][0]+ still_to_wait = oldest_created + BACKGROUND_COALESCE_SECONDS - time.monotonic()+ if still_to_wait > 0:+ self._condition.wait(timeout=still_to_wait)+ continue+ batch = [envelope for _, envelope in self._pending]+ self._pending.clear()+ self._in_flight = batch+ deadline = oldest_created + BUDGET_BACKGROUND_POST_DELAY_MS / 1000+ try:+ self._post(events=batch, deadline=deadline)+ except (OSError, ValueError) as exc:+ # A filesystem failure no reader of the state absorbed (`OSError`), or a path or value the standard+ # library rejected (`ValueError`): the batch is lost, the thread lives on. Any other error is a bug,+ # which ends the thread with its traceback; `enqueue` starts a new one for the next event.+ log.debug("A usage statistics batch could not be posted", fields=error_fields(exc=exc))+ finally:+ with self._condition:+ self._in_flight = []+ self._condition.notify_all()++ def _post(self, *, events: list[dict[str, Any]], deadline: float) -> None:+ origin = self._settings.origin+ if origin is None:+ return+ state = self._settings.state+ verdict, posted = post_in_process(state=state, events=events, origin=origin, deadline=deadline, own_id=self._own_id)+ match verdict:+ case PostVerdict.RETRY:+ self._spool(events=posted)+ case PostVerdict.ACCEPTED:+ if state.read_installation_id() == self._own_id:+ # PostHog accepted the batch, and this process sends under the machine's identifier: what earlier+ # processes queued under it goes too, within what is left of the batch's budget.+ drain_spool_until(state=state, tool=TOOL_NAME, origin=origin, deadline=deadline)+ case PostVerdict.REFUSED | None:+ # Refused for good, or nothing posted at all: nothing says PostHog would take what is queued.+ return++ def _spool(self, *, events: list[dict[str, Any]]) -> None:+ origin = self._settings.origin+ if not events or origin is None or self._settings.state.read_installation_id() != self._own_id:+ # A process-local identifier is this process's alone: its events are never persisted.+ return+ try:+ write_spool_file(spool_dir=self._settings.state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=origin, events=events))+ except OSError:+ return++ def flush(self, *, budget_ms: int, close: bool) -> None:+ """Post what is pending within `budget_ms`, wait for a post in flight, and spool what is left."""+ deadline = time.monotonic() + budget_ms / 1000+ with self._condition:+ batch = [envelope for _, envelope in self._pending]+ self._pending.clear()+ if close:+ self._closed = True+ self._condition.notify_all()+ if batch:+ self._post(events=batch, deadline=deadline)+ with self._condition:+ self._condition.wait_for(lambda: not self._in_flight, timeout=max(0.0, deadline - time.monotonic()))+ unfinished = list(self._in_flight)+ if unfinished:+ # Spooled while its post may still land: a resend carries the same uuid, and PostHog keeps a single copy.+ self._spool(events=unfinished)+++class UsageStatsClient:+ """The statistics of one process: whether they are on, the identifier its events carry, and where they go."""++ def __init__(self, *, settings: UsageStatsSettings) -> None:+ self.settings = settings+ self._lock = threading.Lock()+ self._installation_id: InstallationId | None = None+ self._common: CommonProperties | None = None+ self._held: list[dict[str, Any]] = []+ self._background: _BackgroundSender | None = None+ #: Who can fix the failure the command reported, when something said so; read for its `command_completed`.+ self.command_error_domain: UsageStatsErrorDomain | None = None+ #: Whether the command sent its work to the hosted API, which holds the rest of what it did.+ self.is_command_hosted = False+ #: What an `init` command set, for its `command_completed`.+ self.command_init_choices: InitChoices | None = None+ #: Whether this command printed the notice, which it then never prints a second time.+ self.is_notice_printed = False++ @property+ def is_enabled(self) -> bool:+ return not self.settings.off_by++ def note_command_error_domain(self, *, error_domain: UsageStatsErrorDomain) -> None:+ """Record who can fix the failure the command is reporting, as its error output classifies it; the last one wins."""+ self.command_error_domain = error_domain++ def note_command_hosted(self) -> None:+ """Record that the command sent its work to the hosted API, for its `command_completed`."""+ self.is_command_hosted = True++ def note_command_init_choices(self, *, init_choices: InitChoices) -> None:+ """Record what an `init` command set, for its `command_completed`; a later note completes an earlier one."""+ current = self.command_init_choices+ if current is None:+ self.command_init_choices = init_choices+ return+ self.command_init_choices = InitChoices(+ execution=init_choices.execution or current.execution,+ enabled_providers=init_choices.enabled_providers if init_choices.enabled_providers is not None else current.enabled_providers,+ )++ def reset_after_fork(self) -> None:+ """In a forked child: drop the parent's lock, its held events and its thread, which the child does not have."""+ self._lock = threading.Lock()+ self._held = []+ self._background = None++ def record(self, *, event: UsageStatsEvent) -> None:+ """Take one event: print it under the debug variable, hold it for the command's end, or hand it to the thread."""+ if not self.is_enabled:+ return+ try:+ envelope = self._make_envelope(event=event)+ except (OSError, ValueError):+ # A measurement that cannot be built is lost; the work it measures is not affected.+ return+ if self.settings.is_debug:+ write_stderr_line(line=json.dumps({DEBUG_KEY: envelope}))+ return+ match self.settings.delivery:+ case UsageStatsDelivery.CLI_COMMAND:+ with self._lock:+ self._held.append(envelope)+ case UsageStatsDelivery.PYTHON_PROGRAM | UsageStatsDelivery.LONG_RUNNING:+ self._background_sender().enqueue(envelope=envelope)++ def _make_envelope(self, *, event: UsageStatsEvent) -> dict[str, Any]:+ with self._lock:+ installation_id = self._installation_id+ if installation_id is None:+ installation_id = self.settings.state.obtain_installation_id()+ self._installation_id = installation_id+ common = self._common+ if common is None:+ common = self._make_common_properties(installation_id=installation_id)+ self._common = common+ return make_envelope(event=event, distinct_id=installation_id.value, common=common)++ def _make_common_properties(self, *, installation_id: InstallationId) -> CommonProperties:+ settings = self.settings+ return CommonProperties(+ tool_version=release_version(version=get_package_version()),+ surface=settings.surface,+ integration=settings.integration,+ host=settings.host,+ id_new=installation_id.is_new,+ os=current_os(),+ arch=current_arch(),+ language_runtime=current_language_runtime(),+ ci=settings.is_ci,+ container=settings.is_container,+ internal=settings.state.is_internal(),+ notice_version=settings.state.seen_notice_version(up_to=NOTICE_VERSION),+ )++ def _background_sender(self) -> _BackgroundSender:+ with self._lock:+ if self._background is None:+ own_id = self._installation_id.value if self._installation_id is not None else ""+ self._background = _BackgroundSender(settings=self.settings, own_id=own_id)+ atexit.register(self._flush_at_exit)+ return self._background++ def _flush_at_exit(self) -> None:+ background = self._background+ if background is not None:+ background.flush(budget_ms=self.settings.delivery.flush_budget_ms, close=True)++ def flush(self) -> None:+ """Flush what the thread holds within the delivery's budget, as a long-running process does at shutdown."""+ background = self._background+ if background is not None:+ background.flush(budget_ms=self.settings.delivery.flush_budget_ms, close=False)++ def finish_command(self) -> None:+ """At a CLI command's end: spool its events and start the detached sender, or post them in-process."""+ with self._lock:+ events = list(self._held)+ self._held.clear()+ installation_id = self._installation_id+ origin = self.settings.origin+ if not events or origin is None or installation_id is None:+ return+ state = self.settings.state+ is_process_local = state.read_installation_id() != installation_id.value+ if self.settings.is_ci or self.settings.is_container or is_process_local:+ self._post_in_process(events=events, origin=origin, own_id=installation_id.value, may_spool=not is_process_local)+ return+ try:+ state.ensure_state_dir()+ spool_file = write_spool_file(spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=origin, events=events))+ except OSError:+ self._post_in_process(events=events, origin=origin, own_id=installation_id.value, may_spool=False)+ return+ if start_detached_sender(state=state, origin=origin):+ return+ drain_spool_until(state=state, tool=TOOL_NAME, origin=origin, deadline=time.monotonic() + BUDGET_CLI_IN_PROCESS_MS / 1000, first=spool_file)++ def _post_in_process(self, *, events: list[dict[str, Any]], origin: str, own_id: str, may_spool: bool) -> None:+ """Post the command's events within its budget and, once PostHog accepted them, what earlier commands queued.++ Args:+ events: The command's events.+ origin: Where they go.+ own_id: The identifier they carry.+ may_spool: Whether this process sends under the machine's identifier and can write the spool: only then are+ its unsent events queued, and the events earlier processes queued sent.+ """+ state = self.settings.state+ deadline = time.monotonic() + BUDGET_CLI_IN_PROCESS_MS / 1000+ verdict, posted = post_in_process(state=state, events=events, origin=origin, deadline=deadline, own_id=own_id)+ if not may_spool:+ return+ match verdict:+ case PostVerdict.RETRY:+ try:+ write_spool_file(spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=origin, events=posted))+ except OSError:+ return+ case PostVerdict.ACCEPTED:+ drain_spool_until(state=state, tool=TOOL_NAME, origin=origin, deadline=deadline)+ case PostVerdict.REFUSED | None:+ # Refused for good, or nothing posted at all: nothing says PostHog would take what is queued.+ return+++# -------------------------------------------------------------------------------------------------+# The process's client+# -------------------------------------------------------------------------------------------------+++class _ProcessUsageStats:+ """The process-wide state behind the module functions below."""++ def __init__(self) -> None:+ self.lock = threading.Lock()+ self.client: UsageStatsClient | None = None+ self.is_pytest = False+ self.embedder_host: UsageStatsHost | None = None+ self.runtime_notice_written = False+ #: Whether a CLI command decides this process's statistics, even when it could not configure them: booting+ #: pipelex inside the command then builds no client of its own.+ self.is_cli_command = False+++_PROCESS = _ProcessUsageStats()+++def _reset_after_fork() -> None:+ _PROCESS.lock = threading.Lock()+ client = _PROCESS.client+ if client is not None:+ client.reset_after_fork()+++if hasattr(os, "register_at_fork"):+ os.register_at_fork(after_in_child=_reset_after_fork)+++def _integration_for(*, integration_mode: IntegrationMode) -> UsageStatsIntegration | None:+ """The `integration` property of a runtime integration mode; `None` for `pytest`, which turns statistics off."""+ match integration_mode:+ case IntegrationMode.CLI:+ return UsageStatsIntegration.CLI+ case IntegrationMode.PYTHON:+ return UsageStatsIntegration.PYTHON+ case IntegrationMode.FASTAPI:+ return UsageStatsIntegration.FASTAPI+ case IntegrationMode.DOCKER:+ return UsageStatsIntegration.DOCKER+ case IntegrationMode.MCP:+ return UsageStatsIntegration.MCP+ case IntegrationMode.N8N:+ return UsageStatsIntegration.N8N+ case IntegrationMode.CI:+ return UsageStatsIntegration.CI+ case IntegrationMode.PYTEST:+ return None+++def _surface_and_delivery_for(*, integration: UsageStatsIntegration) -> tuple[UsageStatsSurface, UsageStatsDelivery]:+ """The surface and delivery of a process that booted pipelex itself rather than through a CLI command."""+ match integration:+ case UsageStatsIntegration.FASTAPI | UsageStatsIntegration.DOCKER:+ return UsageStatsSurface.API, UsageStatsDelivery.LONG_RUNNING+ case UsageStatsIntegration.MCP | UsageStatsIntegration.N8N:+ return UsageStatsSurface.PYTHON, UsageStatsDelivery.LONG_RUNNING+ case UsageStatsIntegration.CLI | UsageStatsIntegration.PYTHON | UsageStatsIntegration.CI:+ return UsageStatsSurface.PYTHON, UsageStatsDelivery.PYTHON_PROGRAM+++def _current_dir() -> Path | None:+ try:+ return Path.cwd()+ except OSError:+ return None+++def build_settings(+ *,+ state: UsageStatsState,+ env: Mapping[str, str],+ cwd: Path | None,+ is_pytest: bool,+ surface: UsageStatsSurface,+ integration: UsageStatsIntegration,+ delivery: UsageStatsDelivery,+ embedder_host: UsageStatsHost | None,+) -> UsageStatsSettings:+ """Run the legacy migration, then read every switch and the facts the process's events carry."""+ off_by: list[UsageStatsOffBy]+ if is_pytest:+ # No test suite reports, and none touches the machine's shared state to find that out.+ off_by = [UsageStatsOffBy.INTEGRATION_MODE]+ else:+ state.migrate_legacy_opt_out()+ off_by = read_switches(state=state, env=env, cwd=cwd or state.home, is_pytest=False)+ return UsageStatsSettings(+ off_by=tuple(off_by),+ origin=resolve_endpoint(env=env),+ surface=surface,+ integration=integration,+ host=embedder_host or declared_host(env=env),+ delivery=delivery,+ is_debug=is_debug(env=env),+ is_ci=is_ci(env=env),+ is_container=is_container(env=env),+ state=state,+ )+++def configure_usage_stats_for_cli(*, surface: UsageStatsSurface, is_machine_read: bool) -> UsageStatsClient | None:+ """Read every switch for a CLI command about to run, and print the notice unless its stderr is read by a program.++ The notice goes to stderr before the command does any work, then `notice-v<N>` records it for every tool on the+ machine; a marker that cannot be written leaves the notice to be printed again next time. Where stderr is a machine+ contract, the notice is neither printed nor recorded, and the command's statistics are still sent.++ Returns:+ The command's client, or `None` when the switches could not be read at all, as when no home directory can be+ found: the command then runs without statistics, and pipelex booting inside it does not configure them again.+ """+ with _PROCESS.lock:+ _PROCESS.is_cli_command = True+ try:+ state = UsageStatsState.make_for_current_home()+ with _PROCESS.lock:+ settings = build_settings(+ state=state,+ env=os.environ,+ cwd=_current_dir(),+ is_pytest=_PROCESS.is_pytest,+ surface=surface,+ integration=UsageStatsIntegration.CLI,+ delivery=UsageStatsDelivery.CLI_COMMAND,+ embedder_host=_PROCESS.embedder_host,+ )+ client = UsageStatsClient(settings=settings)+ _PROCESS.client = client+ except (OSError, RuntimeError):+ # No home directory (`Path.home()` raises `RuntimeError`), or a filesystem failure no reader absorbed: the+ # command runs without statistics, which never stand in its way.+ return None+ if client.is_enabled and not is_machine_read and not state.notice_was_recorded(version=NOTICE_VERSION):+ write_stderr_line(line=render_notice())+ client.is_notice_printed = True+ state.record_notice(version=NOTICE_VERSION)+ return client+++def show_usage_stats_notice() -> None:+ """Show the notice's text as part of a command's own output, as `pipelex init` does whenever statistics are on.++ The text goes to stderr, as the first-run notice does, and only when this command has not printed it already: a first+ run's notice stands for it. With statistics off, or switches that could not be read, nothing is shown. Showing the+ text records no `notice-v<N>` marker, and a marker already there does not stop it.+ """+ client = _PROCESS.client+ if client is None or not client.is_enabled or client.is_notice_printed:+ return+ write_stderr_line(line=render_notice())+ client.is_notice_printed = True+++def configure_usage_stats_for_runtime(*, integration_mode: IntegrationMode) -> None:+ """Read every switch when a process boots pipelex, unless a CLI command decides them; `pytest` mode turns statistics off.++ A Python program or a long-running process writes the notice to stderr once per process, at startup, and records no+ marker, since nothing says a person reads that stream. Switches that cannot be read at all leave the process without+ statistics; booting pipelex never fails for them.+ """+ integration = _integration_for(integration_mode=integration_mode)+ with _PROCESS.lock:+ if integration is None:+ _PROCESS.is_pytest = True+ existing = _PROCESS.client+ if existing is not None and UsageStatsOffBy.INTEGRATION_MODE not in existing.settings.off_by:+ _PROCESS.client = UsageStatsClient(+ settings=existing.settings._replace(off_by=(*existing.settings.off_by, UsageStatsOffBy.INTEGRATION_MODE))+ )+ if _PROCESS.client is not None or _PROCESS.is_cli_command:+ # A CLI command's client stands, and a command that could not configure one runs without statistics: a+ # program's client built now would print the runtime notice on a stderr the command may keep for JSON.+ return+ surface, delivery = _surface_and_delivery_for(integration=integration or UsageStatsIntegration.PYTHON)+ try:+ settings = build_settings(+ state=UsageStatsState.make_for_current_home(),+ env=os.environ,+ cwd=_current_dir(),+ is_pytest=_PROCESS.is_pytest,+ surface=surface,+ integration=integration or UsageStatsIntegration.PYTHON,+ delivery=delivery,+ embedder_host=_PROCESS.embedder_host,+ )+ except (OSError, RuntimeError):+ # No home directory, or a filesystem failure no reader absorbed: the process runs without statistics.+ return+ client = UsageStatsClient(settings=settings)+ _PROCESS.client = client+ write_notice = client.is_enabled and not _PROCESS.runtime_notice_written+ _PROCESS.runtime_notice_written = _PROCESS.runtime_notice_written or write_notice+ if write_notice:+ write_stderr_line(line=render_notice())+++def declare_usage_stats_host(*, host: UsageStatsHost) -> None:+ """Declare the program of ours that embeds pipelex, such as `cocode`, before it boots pipelex."""+ with _PROCESS.lock:+ _PROCESS.embedder_host = host+++def is_pytest_process() -> bool:+ """Whether this process booted pipelex in the `pytest` integration mode, which turns statistics off for good."""+ return _PROCESS.is_pytest+++def get_usage_stats_client() -> UsageStatsClient | None:+ return _PROCESS.client+++def usage_stats_are_enabled() -> bool:+ """Whether this process records events, so a caller can skip building one that would be dropped."""+ client = _PROCESS.client+ return client is not None and client.is_enabled+++def record_usage_stats_event(*, event: UsageStatsEvent) -> None:+ """Record one event with the process's client; nothing happens when there is none or statistics are off."""+ client = _PROCESS.client+ if client is not None:+ client.record(event=event)+++def note_command_error_domain(*, error_domain: UsageStatsErrorDomain) -> None:+ """Record, for the running command's `command_completed`, who can fix the failure it is reporting."""+ client = _PROCESS.client+ if client is not None:+ client.note_command_error_domain(error_domain=error_domain)+++def note_command_hosted() -> None:+ """Record, for the running command's `command_completed`, that it sent its work to the hosted API."""+ client = _PROCESS.client+ if client is not None:+ client.note_command_hosted()+++def note_command_init_choices(*, init_choices: InitChoices) -> None:+ """Record, for the running `init` command's `command_completed`, where it set runs to execute and which backends it enabled."""+ client = _PROCESS.client+ if client is not None:+ client.note_command_init_choices(init_choices=init_choices)+++def flush_usage_stats() -> None:+ """Flush what the process's thread holds, within its delivery's budget, as a process shutting pipelex down does."""+ client = _PROCESS.client+ if client is not None:+ client.flush()+++def reset_usage_stats_for_tests() -> None:+ """Forget the process's client and every decision it made, so a test configures its own."""+ with _PROCESS.lock:+ _PROCESS.client = None+ _PROCESS.is_pytest = False+ _PROCESS.embedder_host = None+ _PROCESS.runtime_notice_written = False+ _PROCESS.is_cli_command = Falsediff --git a/pipelex/system/usage_stats/usage_stats_contract.py b/pipelex/system/usage_stats/usage_stats_contract.pynew file mode 100644index 000000000..f71df4db4--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_contract.py@@ -0,0 +1,399 @@+"""The facts of the usage-statistics contract that pipelex sends by, mirrored from its schema.++The contract is `conformance/specs/usage-statistics.md`, whose machine-readable source is+`usage-statistics.json`. pipelex vendors that file byte for byte at+`tests/data/usage_stats/usage-statistics.json`, and the tests under `tests/unit/pipelex/system/usage_stats/`+check every constant and enum here against it, so a change to the contract that these do not follow fails+the suite rather than reaching PostHog's gate, which drops what it does not know.++The facts the detached sender needs, the destination, the shared state's names, the spool's limits and the budgets,+are in `usage_stats_core`, which the sender runs alone; the tests check them against the schema the same way.+"""++from enum import StrEnum+from typing import Final++SCHEMA_VERSION = 1++#: The reporting tool's name, the `tool` property and the name of its spool directory.+TOOL_NAME: Final = "pipelex"++PERSON_PROFILE_PROPERTY = "$process_person_profile"++NOTICE_VERSION = 1+NOTICE_DETAILS_URL = "https://docs.pipelex.com/latest/setup/usage-statistics/"+NOTICE_DISABLE_COMMAND = "pipelex usage-stats disable"+NOTICE_TEXT_TEMPLATE = (+ "Pipelex collects anonymous usage statistics: counts of commands, runs, pipe types, models and tokens, and the country your IP "+ "address maps to; the address itself is never stored. It never collects your methods, prompts, inputs, outputs, file names or "+ "pipe names. To turn it off, run `{disable_command}` or set DO_NOT_TRACK=1. Details: {details_url}"+)+++def render_notice() -> str:+ """The notice as pipelex prints it."""+ return NOTICE_TEXT_TEMPLATE.format(disable_command=NOTICE_DISABLE_COMMAND, details_url=NOTICE_DETAILS_URL)+++class UsageStatsEnvVar(StrEnum):+ """The environment variables the contract names."""++ DO_NOT_TRACK = "DO_NOT_TRACK"+ TOOL_SWITCH = "PIPELEX_USAGE_STATS"+ DEBUG = "PIPELEX_USAGE_STATS_DEBUG"+ ENDPOINT_OVERRIDE = "PIPELEX_USAGE_STATS_ENDPOINT"+ HOST = "PIPELEX_USAGE_STATS_HOST"+++#: Values of `DO_NOT_TRACK` and `PIPELEX_USAGE_STATS_DEBUG` that have no effect, trimmed and lowercased; unset has none either.+FALSY_VALUES: frozenset[str] = frozenset({"", "0", "false"})+#: Values of `PIPELEX_USAGE_STATS` that turn statistics off. Any value in neither set turns them off too, failing closed.+TOOL_SWITCH_OFF_VALUES: frozenset[str] = frozenset({"0", "false", "no", "off"})+#: Values of `PIPELEX_USAGE_STATS` that have no effect, since a variable cannot turn statistics on over another switch.+TOOL_SWITCH_NO_EFFECT_VALUES: frozenset[str] = frozenset({"", "1", "true", "yes", "on"})+CI_VARIABLES: tuple[str, ...] = (+ "CI",+ "CONTINUOUS_INTEGRATION",+ "BUILD_NUMBER",+ "GITHUB_ACTIONS",+ "GITLAB_CI",+ "CIRCLECI",+ "TRAVIS",+ "BUILDKITE",+ "TF_BUILD",+ "JENKINS_URL",+ "TEAMCITY_VERSION",+ "BITBUCKET_BUILD_NUMBER",+ "CODEBUILD_BUILD_ID",+ "DRONE",+ "APPVEYOR",+ "SEMAPHORE",+ "WOODPECKER",+)+CONTAINER_MARKER_FILES: tuple[str, ...] = ("/.dockerenv", "/run/.containerenv")+CONTAINER_MARKER_VARIABLES: tuple[str, ...] = ("container", "KUBERNETES_SERVICE_HOST")+CONTAINER_CGROUP_FILE = "/proc/1/cgroup"+CONTAINER_CGROUP_WORDS: tuple[str, ...] = ("docker", "kubepods", "containerd", "lxc")++#: The project setting's table and key in a project's `.pipelex/pipelex.toml`, and in pipelex's packaged configuration.+SETTING_TABLE = "usage_stats"+SETTING_KEY = "enabled"+++class UsageStatsOffBy(StrEnum):+ """The switches pipelex honours, by their `off_by` token, in the contract's order."""++ DO_NOT_TRACK = "do_not_track"+ TOOL_VARIABLE = "tool_variable"+ MACHINE_CHOICE = "machine_choice"+ LEGACY_MTHDS_CHOICE = "legacy_mthds_choice"+ PROJECT_SETTING = "project_setting"+ PACKAGED_DEFAULT = "packaged_default"+ INTEGRATION_MODE = "integration_mode"+ INVALID_ENDPOINT_OVERRIDE = "invalid_endpoint_override"+++class UsageStatsEventName(StrEnum):+ """The events pipelex sends."""++ RUN_COMPLETED = "run_completed"+ MODEL_USED = "model_used"+ COMMAND_COMPLETED = "command_completed"+ METHOD_VALIDATED = "method_validated"+ CODE_GENERATED = "code_generated"+++class UsageStatsSurface(StrEnum):+ """Which entry point of pipelex sent an event."""++ PIPELEX = "pipelex"+ PIPELEX_AGENT = "pipelex-agent"+ API = "api"+ PYTHON = "python"+++class UsageStatsIntegration(StrEnum):+ """The pipelex runtime's integration mode, as the `integration` property names it."""++ CLI = "cli"+ PYTHON = "python"+ FASTAPI = "fastapi"+ DOCKER = "docker"+ MCP = "mcp"+ N8N = "n8n"+ CI = "ci"+++class UsageStatsHost(StrEnum):+ """The program of ours that embeds or spawns pipelex, when one declares itself."""++ COCODE = "cocode"+ MTHDS_AGENT = "mthds-agent"+ OTHER = "other"+++class UsageStatsOs(StrEnum):+ LINUX = "linux"+ DARWIN = "darwin"+ WINDOWS = "windows"+ OTHER = "other"+++class UsageStatsArch(StrEnum):+ X86_64 = "x86_64"+ ARM64 = "arm64"+ OTHER = "other"+++class UsageStatsOutcome(StrEnum):+ """How a command or a run ended."""++ SUCCEEDED = "succeeded"+ FAILED = "failed"+ CANCELLED = "cancelled"++ @property+ def is_failed(self) -> bool:+ """Whether the event carries an error domain, which only a failure does."""+ match self:+ case UsageStatsOutcome.FAILED:+ return True+ case UsageStatsOutcome.SUCCEEDED | UsageStatsOutcome.CANCELLED:+ return False+++class UsageStatsErrorDomain(StrEnum):+ """Who can fix a failure: the runtime's own error domains, `other` when it cannot say."""++ INPUT = "input"+ CONFIG = "config"+ RUNTIME = "runtime"+ OTHER = "other"+++class UsageStatsMethodSource(StrEnum):+ """Where a run's method came from, by kind only."""++ LOCAL_BUNDLE = "local_bundle"+ INLINE_SOURCE = "inline_source"+ INSTALLED_METHOD = "installed_method"+ PUBLISHED_ADDRESS = "published_address"+ CATALOG_ID = "catalog_id"+ OTHER = "other"+++class UsageStatsValidationOutcome(StrEnum):+ VALID = "valid"+ INVALID = "invalid"+ FAILED = "failed"+++class UsageStatsCodegenOutcome(StrEnum):+ SUCCEEDED = "succeeded"+ FAILED = "failed"+++class UsageStatsCodegenArtifact(StrEnum):+ TYPES = "types"+ INPUTS = "inputs"+ STRUCTURES = "structures"+ OTHER = "other"++ @property+ def has_target(self) -> bool:+ """Whether a `code_generated` of this artifact names a target: only typed artifacts are generated for one."""+ match self:+ case UsageStatsCodegenArtifact.TYPES:+ return True+ case UsageStatsCodegenArtifact.INPUTS | UsageStatsCodegenArtifact.STRUCTURES | UsageStatsCodegenArtifact.OTHER:+ return False+++class UsageStatsCodegenTarget(StrEnum):+ PYTHON_STRUCTURES = "python-structures"+ PYTHON_PYDANTIC = "python-pydantic"+ TS_ZOD = "ts-zod"+ OTHER = "other"+++class UsageStatsInitExecution(StrEnum):+ """Where `init` set runs to execute."""++ LOCAL = "local"+ HOSTED = "hosted"+++class UsageStatsProvider(StrEnum):+ """The backend a model was called through: one pipelex ships, one our own plugins add, or `custom`."""++ ANTHROPIC = "anthropic"+ AZURE_OPENAI = "azure_openai"+ BEDROCK = "bedrock"+ BLACKBOXAI = "blackboxai"+ FAL = "fal"+ GOOGLE = "google"+ GROQ = "groq"+ HUGGINGFACE = "huggingface"+ INTERNAL = "internal"+ LINKUP = "linkup"+ MINIMAX = "minimax"+ MISTRAL = "mistral"+ OLLAMA = "ollama"+ OPENAI = "openai"+ OPENROUTER = "openrouter"+ PORTKEY = "portkey"+ SCALEWAY = "scaleway"+ TYPESAFE = "typesafe"+ VERTEXAI = "vertexai"+ XAI = "xai"+ PIPELEX_MANIFOLD = "pipelex_manifold"+ CUSTOM = "custom"++ @property+ def is_custom(self) -> bool:+ match self:+ case UsageStatsProvider.CUSTOM:+ return True+ case (+ UsageStatsProvider.ANTHROPIC+ | UsageStatsProvider.AZURE_OPENAI+ | UsageStatsProvider.BEDROCK+ | UsageStatsProvider.BLACKBOXAI+ | UsageStatsProvider.FAL+ | UsageStatsProvider.GOOGLE+ | UsageStatsProvider.GROQ+ | UsageStatsProvider.HUGGINGFACE+ | UsageStatsProvider.INTERNAL+ | UsageStatsProvider.LINKUP+ | UsageStatsProvider.MINIMAX+ | UsageStatsProvider.MISTRAL+ | UsageStatsProvider.OLLAMA+ | UsageStatsProvider.OPENAI+ | UsageStatsProvider.OPENROUTER+ | UsageStatsProvider.PORTKEY+ | UsageStatsProvider.SCALEWAY+ | UsageStatsProvider.TYPESAFE+ | UsageStatsProvider.VERTEXAI+ | UsageStatsProvider.XAI+ | UsageStatsProvider.PIPELEX_MANIFOLD+ ):+ return False++ @classmethod+ def for_backend(cls, *, backend_name: str | None) -> "UsageStatsProvider":+ """The provider a backend is reported as: its own name when pipelex or our plugins ship it, `custom` otherwise."""+ if not backend_name:+ return UsageStatsProvider.CUSTOM+ try:+ return UsageStatsProvider(backend_name)+ except ValueError:+ return UsageStatsProvider.CUSTOM++ @property+ def serves_every_shipped_table(self) -> bool:+ """Whether the backend serves the models of every backend table pipelex ships, as Manifold, the hosted gateway, does."""+ match self:+ case UsageStatsProvider.PIPELEX_MANIFOLD:+ return True+ case (+ UsageStatsProvider.CUSTOM+ | UsageStatsProvider.ANTHROPIC+ | UsageStatsProvider.AZURE_OPENAI+ | UsageStatsProvider.BEDROCK+ | UsageStatsProvider.BLACKBOXAI+ | UsageStatsProvider.FAL+ | UsageStatsProvider.GOOGLE+ | UsageStatsProvider.GROQ+ | UsageStatsProvider.HUGGINGFACE+ | UsageStatsProvider.INTERNAL+ | UsageStatsProvider.LINKUP+ | UsageStatsProvider.MINIMAX+ | UsageStatsProvider.MISTRAL+ | UsageStatsProvider.OLLAMA+ | UsageStatsProvider.OPENAI+ | UsageStatsProvider.OPENROUTER+ | UsageStatsProvider.PORTKEY+ | UsageStatsProvider.SCALEWAY+ | UsageStatsProvider.TYPESAFE+ | UsageStatsProvider.VERTEXAI+ | UsageStatsProvider.XAI+ ):+ return False+++#: The model name a custom backend, or a model the release does not ship, is sent as.+OTHER_MODEL = "other"++#: The `command` value of `init`, on either CLI, the only command whose `command_completed` carries the `init_*` properties.+INIT_COMMAND = "init"++#: The `command` values of the `pipelex` CLI, by their path in its command tree.+PIPELEX_COMMANDS: frozenset[str] = frozenset(+ {+ INIT_COMMAND,+ "login",+ "doctor",+ "update",+ "migrate",+ "build inputs bundle",+ "build inputs method",+ "build inputs pipe",+ "build output bundle",+ "build output method",+ "build output pipe",+ "build structures",+ "validate",+ "validate bundle",+ "validate method",+ "validate pipe",+ "fix bundle",+ "resolve",+ "codegen types",+ "codegen inputs",+ "codegen check",+ "run bundle",+ "run method",+ "run pipe",+ "graph render",+ "show config",+ "show pipe",+ "show models",+ "show backends",+ "which",+ "plugins list",+ "other",+ }+)+#: The `command` values of the `pipelex-agent` CLI.+PIPELEX_AGENT_COMMANDS: frozenset[str] = frozenset(+ {+ INIT_COMMAND,+ "run bundle",+ "run method",+ "run pipe",+ "validate",+ "validate bundle",+ "validate method",+ "validate pipe",+ "fix bundle",+ "fmt",+ "lint",+ "inputs bundle",+ "inputs method",+ "inputs pipe",+ "codegen types",+ "codegen inputs",+ "codegen check",+ "models",+ "check-model",+ "migrate",+ "doctor",+ "other",+ }+)+#: The `command` a command the schema does not list yet sends.+OTHER_COMMAND = "other"++#: The patterns of the contract the emitter's own values must match, in Python's `re.fullmatch`.+RELEASE_VERSION_PATTERN = r"^[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}([.-]?(a|b|rc|alpha|beta|dev|post|insider)[.]?[0-9]{0,4})?$"+MODEL_HANDLE_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,95}$"diff --git a/pipelex/system/usage_stats/usage_stats_core.py b/pipelex/system/usage_stats/usage_stats_core.pynew file mode 100644index 000000000..c5ef90a57--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_core.py@@ -0,0 +1,804 @@+"""The self-contained core of pipelex's usage statistics: the shared state, the spool, the post, and the detached sender.++Everything a sender needs lives here, and nothing else does, because a CLI command starts this very file as its+detached sender: `python -I <path of this file> --home <home> --origin <origin> --tool pipelex`. Isolated mode keeps the+interpreter away from the environment's `PYTHON*` variables, the user's site-packages and the current directory, and a+file started by path imports none of the modules beside it, so this module imports the standard library and nothing+else, and never the `pipelex` package: the sender then delivers whether pipelex is installed for the user, through+`PYTHONPATH`, or in a virtual environment the sender's interpreter does not see, and it starts without loading pipelex.+The rest of the package imports what it needs from here, so the in-process paths and the sender run one code.++- **The shared state**, `~/.mthds/usage-stats/`: one small file per fact, each created or deleted whole and never+ rewritten, so two tools running at once can never undo each other's choice. `id` holds the installation identifier,+ published with a hard link so that racing first runs converge on one; `disabled` turns statistics off for every+ tool; `legacy-migrated` records when the legacy `mthds` opt-out was last carried over; `notice-v<N>` records that+ notice N was shown; `internal` marks a team machine; and `spool/<tool>/` queues each tool's unsent events. A+ filesystem failure is never raised from a reader: statistics are a measurement, never part of what the person asked+ for, so an unreadable state reads as absent, except `disabled`, which it may hold: statistics are then off.+- **The spool**: a file is named `<created-ms>-<random>.json` and holds+ `{"spool_version": 1, "endpoint": <origin>, "events": [...]}`, `endpoint` being the contract's member for the origin+ the events were queued for: a sender posts a file naming the origin it posts to, and a file naming none, as earlier+ builds wrote them, wherever it posts. It is written under a name starting with a dot and renamed into place, so a+ reader never sees it partial, and readers ignore names starting with a dot. A sender claims a file by renaming it+ to `<name>.sending-<random>`, so two senders never post one file, and after its attempt deletes it or renames it back.+- **The post**: `POST <origin>/batch/` with `{"api_key": <project token>, "batch": [...]}` and nothing else, so no SDK+ library property, no `sent_at`. Redirects are never followed, so a batch can never be carried anywhere but the+ origin it was addressed to, and a loopback origin is reached without a proxy. A post returns within the time left in+ its budget whatever the network does, name resolution included.+- **The checks before each post**: the legacy migration runs, nothing is sent when `disabled` exists or when a legacy+ opt-out the migration could not carry over is still pending, and every event whose `distinct_id` is not the current+ identifier is dropped, since it was made under one `reset-id` has replaced.++The contract is `conformance/specs/usage-statistics.md`, "Shared state" and "Delivery"; the constants here are checked+against its schema by the tests, beside the ones of `usage_stats_contract`.+"""++import argparse+import http.client+import json+import os+import re+import secrets+import sys+import threading+import time+import urllib.request+import uuid+from enum import StrEnum+from pathlib import Path+from typing import Any, NamedTuple, cast++# -------------------------------------------------------------------------------------------------+# The contract's facts a sender needs+# -------------------------------------------------------------------------------------------------++CAPTURE_HOST = "https://eu.i.posthog.com"+BATCH_PATH = "/batch/"+#: The project's ingestion token. It is public by design: it can only write events, and the gate decides what is kept.+PROJECT_TOKEN = "phc_uQjSVbzNPJxA8bHxfevn7NMzB6PbC6aJQRB6ka6MxT7a"+#: Responses after which a sender deletes what it posted, since a resend would be refused again.+PERMANENT_REFUSAL_STATUSES: frozenset[int] = frozenset({400, 401, 403, 413})++#: The shared state directory, under the home directory.+STATE_DIR_PARTS: tuple[str, ...] = (".mthds", "usage-stats")+ID_FILE_NAME = "id"+DISABLED_FILE_NAME = "disabled"+LEGACY_MIGRATED_FILE_NAME = "legacy-migrated"+NOTICE_FILE_PREFIX = "notice-v"+INTERNAL_FILE_NAME = "internal"+SPOOL_DIR_NAME = "spool"+#: Today's `mthds` CLI configuration, under the home directory, the key its opt-out is stored under, and the values meaning off.+LEGACY_CONFIG_PARTS: tuple[str, ...] = (".mthds", "config")+LEGACY_KEY = "DISABLE_TELEMETRY"+LEGACY_DISABLE_ON_VALUES: frozenset[str] = frozenset({"1", "true", "yes", "on"})+#: Every tool's spool directory, which `disable`, `reset-id` and the legacy migration empty.+SPOOL_TOOLS: tuple[str, ...] = ("pipelex", "plxt", "vscode-pipelex", "mthds")++SPOOL_VERSION = 1+SPOOL_MAX_FILES_PER_TOOL = 100+SPOOL_MAX_AGE_DAYS = 30+SPOOL_STALE_CLAIM_MINUTES = 10+SPOOL_STALE_TEMP_MINUTES = 60+#: A temporary identifier file older than this is a crashed writer's leftover, which any tool may delete.+ID_STALE_TEMP_MINUTES = 60++BUDGET_CLI_IN_PROCESS_MS = 1000+BUDGET_PYTHON_EXIT_FLUSH_MS = 250+BUDGET_SHUTDOWN_FLUSH_MS = 2000+BUDGET_DETACHED_SENDER_MS = 10000+BUDGET_BACKGROUND_POST_DELAY_MS = 10000++INSTALLATION_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"++#: Below this many seconds left in a budget, a sender does not start a request it could not finish.+MINIMUM_REQUEST_SECONDS = 0.05++#: A loopback `http` origin with a port and no path: the only endpoint override the contract accepts.+_LOOPBACK_ORIGIN = re.compile(r"^http://(127\.0\.0\.1|localhost|\[::1\]):([0-9]{1,5})$")+++def is_loopback_origin(*, origin: str) -> bool:+ """Whether `origin` is `http://127.0.0.1:<port>`, `http://localhost:<port>` or `http://[::1]:<port>`, with no path."""+ match = _LOOPBACK_ORIGIN.fullmatch(origin)+ if match is None:+ return False+ port = int(match.group(2))+ return 0 < port < 65536+++def is_allowed_origin(*, origin: str) -> bool:+ """Whether events may be posted to `origin`: the project's capture host, or the machine itself."""+ return origin == CAPTURE_HOST or is_loopback_origin(origin=origin)+++# -------------------------------------------------------------------------------------------------+# The shared state+# -------------------------------------------------------------------------------------------------++_INSTALLATION_ID = re.compile(INSTALLATION_ID_PATTERN)+_ID_TEMP_PREFIX = f"{ID_FILE_NAME}.tmp-"+_STATE_DIR_MODE = 0o700+++class InstallationId(NamedTuple):+ """The identifier a process sends under, and whether this process created it or drew it for itself alone."""++ value: str+ is_new: bool+++def path_exists(*, path: Path) -> bool | None:+ """Whether `path` exists, and `None` when that cannot be told, as when a directory above it may not be traversed.++ It stats the path itself rather than asking `Path.exists()`, which from Python 3.14 swallows every `OSError` and+ answers `False`, so a `disabled` file in a state that cannot be looked into would read as absent.+ """+ try:+ path.stat()+ except (FileNotFoundError, NotADirectoryError):+ return False+ except OSError:+ return None+ return True+++def delete_file(*, path: Path) -> None:+ """Delete `path` if it exists; a file that cannot be deleted stays, and is met again by whoever looks next."""+ try:+ path.unlink(missing_ok=True)+ except OSError:+ return+++def new_installation_id() -> str:+ """A random UUID version 4 in its canonical lowercase form, derived from nothing about the machine or the person."""+ return str(uuid.uuid4())+++def is_installation_id(*, value: str) -> bool:+ return _INSTALLATION_ID.fullmatch(value) is not None+++def parse_legacy_config(*, content: str) -> dict[str, str]:+ """Parse `~/.mthds/config` in the dialect of `conformance/specs/mthds-config-file.md`.++ Lines split on newline characters only, each trimmed; blank lines and whole-line `#` comments are skipped, as is a+ line with no `=`; the key is left of the first `=` and the value right of it, both trimmed; a later duplicate key wins.+ """+ entries: dict[str, str] = {}+ for raw_line in content.split("\n"):+ line = raw_line.strip()+ if not line or line.startswith("#") or "=" not in line:+ continue+ key, value = line.split("=", 1)+ entries[key.strip()] = value.strip()+ return entries+++class UsageStatsState:+ """The shared state directory under one home directory, and every operation the contract makes on it."""++ def __init__(self, *, home: Path) -> None:+ self.home = home+ self.state_dir = home.joinpath(*STATE_DIR_PARTS)+ self.id_file = self.state_dir / ID_FILE_NAME+ self.disabled_file = self.state_dir / DISABLED_FILE_NAME+ self.legacy_marker = self.state_dir / LEGACY_MIGRATED_FILE_NAME+ self.internal_file = self.state_dir / INTERNAL_FILE_NAME+ self.spool_root = self.state_dir / SPOOL_DIR_NAME+ self.legacy_config = home.joinpath(*LEGACY_CONFIG_PARTS)++ @classmethod+ def make_for_current_home(cls) -> "UsageStatsState":+ """The state under this user's home directory: `$HOME` on POSIX, `%USERPROFILE%` on Windows."""+ return cls(home=Path.home())++ def spool_dir(self, *, tool: str) -> Path:+ return self.spool_root / tool++ def notice_marker(self, *, version: int) -> Path:+ return self.state_dir / f"{NOTICE_FILE_PREFIX}{version}"++ def ensure_state_dir(self) -> bool:+ """Create the directory, mode 0700, when a tool first needs it; whether it exists now."""+ try:+ self.state_dir.mkdir(mode=_STATE_DIR_MODE, parents=True, exist_ok=True)+ return self.state_dir.is_dir()+ except OSError:+ return False++ # ---------------------------------------------------------------------------------------------+ # The installation identifier+ # ---------------------------------------------------------------------------------------------++ def read_installation_id(self) -> str | None:+ """The identifier `id` holds, surrounding whitespace removed, or `None` when it is absent, unreadable or malformed."""+ try:+ content = self.id_file.read_text(encoding="utf-8")+ except (OSError, UnicodeDecodeError):+ return None+ candidate = content.strip()+ return candidate if is_installation_id(value=candidate) else None++ def obtain_installation_id(self) -> InstallationId:+ """The identifier this process sends under, creating `id` when it does not exist.++ A fresh identifier is written, with no newline, to `id.tmp-<random>`, flushed to disk and published with a hard+ link to `id`, which fails when `id` exists; the temporary file is deleted either way. When the link fails because+ `id` exists, the identifier it holds is used, so racing first runs converge on one and exactly one of them reports+ it as new. When no link can be made at all, the directory is not writable, or `id` exists but is malformed, the+ process uses its fresh identifier for itself alone and persists nothing; a malformed `id` is never rewritten.+ """+ fresh_id = new_installation_id()+ process_local = InstallationId(value=fresh_id, is_new=True)+ try:+ existing = self.id_file.read_text(encoding="utf-8")+ except FileNotFoundError:+ existing = None+ except (OSError, UnicodeDecodeError):+ return process_local+ if existing is not None:+ candidate = existing.strip()+ return InstallationId(value=candidate, is_new=False) if is_installation_id(value=candidate) else process_local++ if not self.ensure_state_dir():+ return process_local+ self._delete_stale_id_temps()+ temporary = self.state_dir / f"{_ID_TEMP_PREFIX}{secrets.token_hex(6)}"+ try:+ descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)+ try:+ os.write(descriptor, fresh_id.encode("utf-8"))+ os.fsync(descriptor)+ finally:+ os.close(descriptor)+ except OSError:+ delete_file(path=temporary)+ return process_local++ published: bool+ try:+ os.link(temporary, self.id_file)+ published = True+ except FileExistsError:+ published = False+ except OSError:+ # A filesystem without hard links, or one that refuses them here: nothing is persisted.+ delete_file(path=temporary)+ return process_local+ delete_file(path=temporary)++ if published:+ return InstallationId(value=fresh_id, is_new=True)+ winner = self.read_installation_id()+ return InstallationId(value=winner, is_new=False) if winner is not None else process_local++ def replace_installation_id(self) -> str:+ """`reset-id`: delete every tool's spool and `id`, then create a new identifier and return it.++ Raises:+ OSError: when the new identifier could not be persisted, so `reset-id` can say it did not do what it says.+ """+ self.delete_every_spool()+ self.id_file.unlink(missing_ok=True)+ installation_id = self.obtain_installation_id()+ if self.read_installation_id() != installation_id.value:+ msg = f"The new installation identifier could not be written to {self.id_file}"+ raise OSError(msg)+ return installation_id.value++ def _delete_stale_id_temps(self) -> None:+ """Delete `id.tmp-*` files older than an hour, the leftovers of a writer that crashed."""+ threshold = time.time() - ID_STALE_TEMP_MINUTES * 60+ try:+ candidates = [entry for entry in self.state_dir.iterdir() if entry.name.startswith(_ID_TEMP_PREFIX)]+ except OSError:+ return+ for candidate in candidates:+ try:+ if candidate.stat().st_mtime < threshold:+ candidate.unlink(missing_ok=True)+ except OSError:+ continue++ # ---------------------------------------------------------------------------------------------+ # The facts+ # ---------------------------------------------------------------------------------------------++ def is_disabled(self) -> bool:+ """Whether `disabled` exists; a state that cannot be looked into fails closed, since it may hold that choice."""+ return path_exists(path=self.disabled_file) is not False++ def is_internal(self) -> bool:+ return path_exists(path=self.internal_file) is True++ def seen_notice_version(self, *, up_to: int) -> int:+ """The highest notice version N, no higher than `up_to`, for which `notice-v<N>` exists, and 0 when none does."""+ for version in range(up_to, 0, -1):+ if path_exists(path=self.notice_marker(version=version)) is True:+ return version+ return 0++ def notice_was_recorded(self, *, version: int) -> bool:+ """Whether `notice-v<version>` exists; a marker that cannot be looked for is absent, so the notice is printed."""+ return path_exists(path=self.notice_marker(version=version)) is True++ def record_notice(self, *, version: int) -> bool:+ """Create `notice-v<version>`; whether it exists now. A notice that cannot be recorded is shown again next time."""+ if not self.ensure_state_dir():+ return False+ try:+ self.notice_marker(version=version).touch(exist_ok=True)+ except OSError:+ return False+ return True++ def disable(self) -> None:+ """`disable`: create `disabled` and delete every spooled file of every tool.++ Raises:+ OSError: when `disabled` could not be created.+ """+ self.state_dir.mkdir(mode=_STATE_DIR_MODE, parents=True, exist_ok=True)+ self.disabled_file.touch(exist_ok=True)+ self.delete_every_spool()++ def enable(self) -> None:+ """`enable`: delete `disabled`, and set `legacy-migrated` so the legacy opt-out already carried over is not migrated again.++ Raises:+ OSError: when `disabled` could not be deleted or the marker could not be set.+ """+ self.disabled_file.unlink(missing_ok=True)+ self.state_dir.mkdir(mode=_STATE_DIR_MODE, parents=True, exist_ok=True)+ self._set_legacy_marker()++ def delete_every_spool(self) -> None:+ """Delete every spooled file of every tool, claimed and temporary ones included; what cannot be deleted stays."""+ for tool in SPOOL_TOOLS:+ spool_dir = self.spool_dir(tool=tool)+ try:+ entries = list(spool_dir.iterdir())+ except OSError:+ continue+ for entry in entries:+ try:+ if entry.is_file():+ entry.unlink(missing_ok=True)+ except OSError:+ continue++ # ---------------------------------------------------------------------------------------------+ # The legacy `mthds` opt-out+ # ---------------------------------------------------------------------------------------------++ def legacy_opt_out_is_on(self) -> bool:+ """Whether `DISABLE_TELEMETRY` in `~/.mthds/config` says off, read in that file's dialect."""+ try:+ content = self.legacy_config.read_text(encoding="utf-8")+ except (OSError, UnicodeDecodeError):+ return False+ value = parse_legacy_config(content=content).get(LEGACY_KEY)+ return value is not None and value.lower() in LEGACY_DISABLE_ON_VALUES++ def legacy_opt_out_is_pending(self) -> bool:+ """Whether the legacy opt-out says off and was written after it was last migrated (or was never migrated)."""+ if not self.legacy_opt_out_is_on():+ return False+ try:+ config_mtime_ns = self.legacy_config.stat().st_mtime_ns+ except OSError:+ return False+ try:+ marker_mtime_ns = self.legacy_marker.stat().st_mtime_ns+ except OSError:+ return True+ return config_mtime_ns > marker_mtime_ns++ def migrate_legacy_opt_out(self) -> None:+ """Carry a pending legacy opt-out into the shared state: create `disabled`, delete every spool, set the marker.++ Run by every reporting process before it reads any other switch, and by every sender before each post. A legacy+ value that does not mean off migrates nothing and writes nothing. A state that cannot be written leaves the legacy+ opt-out pending, so it still turns statistics off as the `legacy_mthds_choice` switch.+ """+ if not self.legacy_opt_out_is_pending():+ return+ try:+ self.disable()+ self._set_legacy_marker()+ except OSError:+ return++ def _set_legacy_marker(self) -> None:+ """Create `legacy-migrated` and set its modification time to the later of now and the legacy config's."""+ try:+ config_mtime_ns = self.legacy_config.stat().st_mtime_ns+ except OSError:+ config_mtime_ns = 0+ marker_ns = max(time.time_ns(), config_mtime_ns)+ self.legacy_marker.touch(exist_ok=True)+ os.utime(self.legacy_marker, ns=(marker_ns, marker_ns))+++# -------------------------------------------------------------------------------------------------+# The spool+# -------------------------------------------------------------------------------------------------++SPOOL_FILE_SUFFIX = ".json"+#: The spool file's member naming the endpoint its events were queued for, as the contract's shape has it.+SPOOL_ENDPOINT_KEY = "endpoint"+CLAIM_MARKER = ".sending-"+_TEMP_PREFIX = ".tmp-"+++def _created_ms(*, path: Path) -> int | None:+ """The creation time a spool file's name starts with, or `None` for a name that is not a spool file's."""+ prefix = path.name.split("-", 1)[0]+ return int(prefix) if prefix.isdigit() else None+++def is_pending(*, path: Path) -> bool:+ """Whether `path` is a spool file waiting for a sender: not temporary, not claimed."""+ name = path.name+ return not name.startswith(".") and CLAIM_MARKER not in name and name.endswith(SPOOL_FILE_SUFFIX) and _created_ms(path=path) is not None+++def pending_files(*, spool_dir: Path) -> list[Path]:+ """The files waiting in `spool_dir`, oldest first."""+ try:+ entries = [entry for entry in spool_dir.iterdir() if is_pending(path=entry)]+ except OSError:+ return []+ return sorted(entries, key=lambda entry: (_created_ms(path=entry) or 0, entry.name))+++class SpoolContent(NamedTuple):+ """What a spool file holds: the endpoint its events were queued for, when it names one, and the events."""++ endpoint: str | None+ events: list[dict[str, Any]]++ def is_queued_for(self, *, origin: str) -> bool:+ """Whether a sender posting to `origin` may post these events: the file names that endpoint, or names none."""+ return self.endpoint is None or self.endpoint == origin+++def _spool_payload(*, content: SpoolContent) -> str:+ payload: dict[str, Any] = {"spool_version": SPOOL_VERSION}+ if content.endpoint is not None:+ payload[SPOOL_ENDPOINT_KEY] = content.endpoint+ payload["events"] = content.events+ return json.dumps(payload)+++def write_spool_file(*, spool_dir: Path, content: SpoolContent) -> Path:+ """Queue events in a new file, then keep at most the contract's number of files, deleting the oldest.++ Raises:+ OSError: when the file could not be written.+ """+ spool_dir.mkdir(parents=True, exist_ok=True)+ final_path = spool_dir / f"{int(time.time() * 1000)}-{secrets.token_hex(6)}{SPOOL_FILE_SUFFIX}"+ temporary = spool_dir / f"{_TEMP_PREFIX}{secrets.token_hex(6)}"+ try:+ temporary.write_text(_spool_payload(content=content), encoding="utf-8")+ temporary.rename(final_path)+ finally:+ temporary.unlink(missing_ok=True)+ _enforce_file_limit(spool_dir=spool_dir)+ return final_path+++def _enforce_file_limit(*, spool_dir: Path) -> None:+ waiting = pending_files(spool_dir=spool_dir)+ for oldest in waiting[: max(0, len(waiting) - SPOOL_MAX_FILES_PER_TOOL)]:+ delete_file(path=oldest)+++def read_spool_file(*, path: Path) -> SpoolContent | None:+ """What a spool file holds, or `None` for a file that is not one: unreadable, not JSON, or of another shape."""+ try:+ payload: object = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):+ return None+ if not isinstance(payload, dict):+ return None+ payload_dict = cast("dict[str, Any]", payload)+ events: object = payload_dict.get("events")+ endpoint: object = payload_dict.get(SPOOL_ENDPOINT_KEY)+ if payload_dict.get("spool_version") != SPOOL_VERSION or not isinstance(events, list):+ return None+ if endpoint is not None and not isinstance(endpoint, str):+ return None+ event_list = cast("list[object]", events)+ if not all(isinstance(event, dict) for event in event_list):+ return None+ return SpoolContent(endpoint=endpoint, events=cast("list[dict[str, Any]]", event_list))+++def rewrite_spool_file(*, path: Path, content: SpoolContent) -> None:+ """Replace what a file this sender has claimed holds, as a whole.++ Raises:+ OSError: when the file could not be replaced.+ """+ temporary = path.parent / f"{_TEMP_PREFIX}{secrets.token_hex(6)}"+ try:+ temporary.write_text(_spool_payload(content=content), encoding="utf-8")+ Path(temporary).replace(path)+ finally:+ temporary.unlink(missing_ok=True)+++def claim(*, path: Path) -> Path | None:+ """Claim a waiting file for this sender, or `None` when another sender took it first.++ The claim's modification time is set to now, so a claim's age is how long its sender has held it.+ """+ claimed = path.with_name(f"{path.name}{CLAIM_MARKER}{secrets.token_hex(6)}")+ try:+ path.rename(claimed)+ except OSError:+ return None+ try:+ os.utime(claimed)+ except OSError:+ # The claim stands without its fresh time: it can only look stale sooner, and be resent, never be lost.+ return claimed+ return claimed+++def release(*, claimed: Path) -> None:+ """Give a claimed file back for the next sender, under its name before the claim."""+ original = claimed.with_name(claimed.name.split(CLAIM_MARKER, 1)[0])+ try:+ claimed.rename(original)+ except OSError:+ return+++def prune(*, spool_dir: Path) -> None:+ """Tidy a spool before draining it: give back stale claims, delete stale temporary files and files past their age."""+ now = time.time()+ try:+ entries = list(spool_dir.iterdir())+ except OSError:+ return+ for entry in entries:+ try:+ age_seconds = now - entry.stat().st_mtime+ except OSError:+ continue+ name = entry.name+ if name.startswith("."):+ if age_seconds > SPOOL_STALE_TEMP_MINUTES * 60:+ delete_file(path=entry)+ elif CLAIM_MARKER in name:+ if age_seconds > SPOOL_STALE_CLAIM_MINUTES * 60:+ release(claimed=entry)+ elif is_pending(path=entry):+ created_ms = _created_ms(path=entry) or 0+ if now * 1000 - created_ms > SPOOL_MAX_AGE_DAYS * 24 * 3600 * 1000:+ delete_file(path=entry)+++# -------------------------------------------------------------------------------------------------+# The post+# -------------------------------------------------------------------------------------------------+++class PostVerdict(StrEnum):+ """What a sender does with what it posted, by PostHog's answer."""++ ACCEPTED = "accepted"+ REFUSED = "refused"+ RETRY = "retry"++ @property+ def is_final(self) -> bool:+ """Whether the events are done with: accepted, or refused for good so a resend would be refused again."""+ match self:+ case PostVerdict.ACCEPTED | PostVerdict.REFUSED:+ return True+ case PostVerdict.RETRY:+ return False+++def verdict_for_status(*, status: int) -> PostVerdict:+ if 200 <= status < 300:+ return PostVerdict.ACCEPTED+ if status in PERMANENT_REFUSAL_STATUSES:+ return PostVerdict.REFUSED+ return PostVerdict.RETRY+++def _make_opener(*, origin: str) -> urllib.request.OpenerDirector:+ """An opener that follows no redirect and turns no status into an exception: the status is read as it came.++ A loopback origin is reached without a proxy; any other goes through the proxies the environment names.+ """+ opener = urllib.request.OpenerDirector()+ proxy_handler = urllib.request.ProxyHandler({}) if is_loopback_origin(origin=origin) else urllib.request.ProxyHandler()+ for handler in (proxy_handler, urllib.request.HTTPHandler(), urllib.request.HTTPSHandler()):+ opener.add_handler(handler)+ return opener+++def _request_verdict(*, origin: str, body: bytes, timeout_seconds: float, answers: list[PostVerdict]) -> None:+ """Make the request and append PostHog's answer to `answers`; any failure to get an answer is `RETRY`.++ It runs on its own thread and does network I/O only: the body is encoded before the thread starts.+ """+ request = urllib.request.Request( # ruff: ignore[suspicious-url-open-usage] — the origin is PostHog's capture host or a loopback override, both checked+ url=f"{origin.rstrip('/')}{BATCH_PATH}",+ data=body,+ headers={"Content-Type": "application/json"},+ method="POST",+ )+ try:+ response = _make_opener(origin=origin).open(request, timeout=timeout_seconds)+ with response:+ status: int = response.status+ except (OSError, http.client.HTTPException, ValueError):+ # No answer at all: a network failure, a timeout, a connection closed mid-answer or an origin the opener refused.+ answers.append(PostVerdict.RETRY)+ return+ answers.append(verdict_for_status(status=status))+++def post_batch(*, origin: str, events: list[dict[str, Any]], timeout_seconds: float) -> PostVerdict:+ """Post `events` in one request and return within `timeout_seconds`, whatever the network does.++ A socket's timeout bounds each of its operations, never name resolution nor the whole exchange, so a slow resolver+ or an answer trickling in byte by byte would hold the caller past its budget. The request runs on a daemon thread+ instead, which the caller waits for no longer than its budget: a request still running then is `RETRY`, and is left+ to finish or fail on its own. A resend carries the same `uuid`s, and PostHog keeps a single copy of each event.++ The body is encoded here, before the thread starts: events JSON cannot encode never will be, so they are `REFUSED`+ and dropped, where an error on the thread would read as `RETRY` and queue them again forever.+ """+ if timeout_seconds < MINIMUM_REQUEST_SECONDS:+ return PostVerdict.RETRY+ try:+ body = json.dumps({"api_key": PROJECT_TOKEN, "batch": events}).encode("utf-8")+ except (TypeError, ValueError):+ # A value JSON has no form for (`TypeError`), or a circular reference (`ValueError`): refused for good.+ return PostVerdict.REFUSED+ answers: list[PostVerdict] = []+ request_thread = threading.Thread(+ target=_request_verdict,+ kwargs={"origin": origin, "body": body, "timeout_seconds": timeout_seconds, "answers": answers},+ name="pipelex-usage-stats-post",+ daemon=True,+ )+ try:+ request_thread.start()+ except RuntimeError:+ # The interpreter can start no thread: the events stay to be sent another time.+ return PostVerdict.RETRY+ request_thread.join(timeout=timeout_seconds)+ return answers[0] if answers else PostVerdict.RETRY+++# -------------------------------------------------------------------------------------------------+# The checks before each post, and the drain+# -------------------------------------------------------------------------------------------------+++def screen_events(*, state: UsageStatsState, events: list[dict[str, Any]], own_id: str | None) -> list[dict[str, Any]] | None:+ """The events still to send, checked immediately before a post; `None` when statistics were turned off since.++ Args:+ state: The shared state.+ events: The events about to be posted.+ own_id: The identifier this process sends under when `id` holds none it can read: a process-local identifier,+ whose events are this process's own and still to be sent. A detached sender has none.+ """+ state.migrate_legacy_opt_out()+ if state.is_disabled() or state.legacy_opt_out_is_pending():+ # A legacy opt-out the migration could not carry over still says off.+ return None+ current_id = state.read_installation_id() or own_id+ return [event for event in events if event.get("distinct_id") == current_id]+++def send_claimed_file(*, state: UsageStatsState, claimed: Path, origin: str, deadline: float, own_id: str | None) -> PostVerdict | None:+ """Post a file this sender claimed, then delete it or give it back by PostHog's answer; `None` when nothing was posted."""+ content = read_spool_file(path=claimed)+ if content is None:+ delete_file(path=claimed)+ return None+ if not content.is_queued_for(origin=origin):+ release(claimed=claimed)+ return None+ events = content.events+ kept = screen_events(state=state, events=events, own_id=own_id)+ if not kept:+ delete_file(path=claimed)+ return None+ if len(kept) != len(events):+ try:+ rewrite_spool_file(path=claimed, content=content._replace(events=kept))+ except OSError:+ # The dropped events must never be sent: a file that cannot be rewritten without them is deleted with them.+ delete_file(path=claimed)+ return None+ verdict = post_batch(origin=origin, events=kept, timeout_seconds=deadline - time.monotonic())+ if verdict.is_final:+ delete_file(path=claimed)+ else:+ release(claimed=claimed)+ return verdict+++def drain_spool(*, state: UsageStatsState, tool: str, origin: str, budget_ms: int) -> None:+ """Post the files of `tool`'s spool queued for `origin`, oldest first, within `budget_ms`, as the detached sender does."""+ drain_spool_until(state=state, tool=tool, origin=origin, deadline=time.monotonic() + budget_ms / 1000)+++def drain_spool_until(*, state: UsageStatsState, tool: str, origin: str, deadline: float, first: Path | None = None) -> None:+ """Post the files of `tool`'s spool queued for `origin` until `deadline`: `first` when given, then the rest oldest first.++ A file queued for another endpoint is left for a sender posting there, so events queued under a test's loopback+ override never reach PostHog, and events queued for PostHog never reach a loopback endpoint. The events of each+ file are checked against the identifier `id` holds, so a process whose identifier is its own alone never drains.+ """+ spool_dir = state.spool_dir(tool=tool)+ prune(spool_dir=spool_dir)+ candidates = pending_files(spool_dir=spool_dir)+ if first is not None:+ candidates = [first, *(candidate for candidate in candidates if candidate != first)]+ for candidate in candidates:+ if time.monotonic() >= deadline:+ return+ content = read_spool_file(path=candidate)+ if content is not None and not content.is_queued_for(origin=origin):+ continue+ claimed = claim(path=candidate)+ if claimed is None:+ continue+ send_claimed_file(state=state, claimed=claimed, origin=origin, deadline=deadline, own_id=None)+++def post_in_process(+ *, state: UsageStatsState, events: list[dict[str, Any]], origin: str, deadline: float, own_id: str | None+) -> tuple[PostVerdict | None, list[dict[str, Any]]]:+ """Post events held in memory after the checks; the verdict (`None` when nothing was posted) and the events posted."""+ kept = screen_events(state=state, events=events, own_id=own_id)+ if not kept:+ return None, []+ return post_batch(origin=origin, events=kept, timeout_seconds=deadline - time.monotonic()), kept+++# -------------------------------------------------------------------------------------------------+# The detached sender+# -------------------------------------------------------------------------------------------------+++def sender_command(*, executable: str, home: Path, origin: str, tool: str) -> list[str]:+ """The command line a CLI starts its detached sender with: this file, by path, in isolated mode."""+ return [executable, "-I", str(Path(__file__).resolve()), "--home", str(home), "--origin", origin, "--tool", tool]+++def main(*, argv: list[str] | None = None) -> int:+ """Post every file of the tool's spool within the detached sender's budget, and exit.++ The origin was checked by the command that started this process; it is checked again here, so this file can never+ be made to post anywhere but the project or the machine itself.+ """+ parser = argparse.ArgumentParser(prog="pipelex-usage-stats-sender", add_help=False)+ parser.add_argument("--home", required=True)+ parser.add_argument("--origin", required=True)+ parser.add_argument("--tool", required=True)+ arguments = parser.parse_args(argv)+ origin: str = arguments.origin+ tool: str = arguments.tool+ if not is_allowed_origin(origin=origin) or tool not in SPOOL_TOOLS:+ return 2+ drain_spool(state=UsageStatsState(home=Path(arguments.home)), tool=tool, origin=origin, budget_ms=BUDGET_DETACHED_SENDER_MS)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/pipelex/system/usage_stats/usage_stats_delivery.py b/pipelex/system/usage_stats/usage_stats_delivery.pynew file mode 100644index 000000000..585f094e6--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_delivery.py@@ -0,0 +1,51 @@+"""The start of the detached sender a CLI command leaves behind to drain pipelex's spool after it exits.++The sender is `usage_stats_core` itself, started by its file path in isolated mode, so it imports the standard library+alone and never the `pipelex` package: it delivers from any interpreter that can run this file, whether or not that+interpreter sees pipelex, and it starts in a fraction of the time a pipelex import takes. The contract is+`conformance/specs/usage-statistics.md`, "Senders".+"""++import subprocess # ruff: ignore[suspicious-subprocess-import] — starts the detached sender, a fixed file of this package+import sys+from pathlib import Path++from pipelex.system.usage_stats import usage_stats_core+from pipelex.system.usage_stats.usage_stats_contract import TOOL_NAME+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState, sender_command+++def start_detached_sender(*, state: UsageStatsState, origin: str) -> bool:+ """Start the process that drains pipelex's spool after the command exits: no terminal, no output, outliving it.++ Returns:+ Whether the process started; when it did not, the command posts in-process instead.+ """+ if not sys.executable or not Path(usage_stats_core.__file__).is_file():+ return False+ command = sender_command(executable=sys.executable, home=state.home, origin=origin, tool=TOOL_NAME)+ try:+ if sys.platform == "win32":+ creation_flags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW+ subprocess.Popen( # ruff: ignore[subprocess-without-shell-equals-true] — a fixed argument list, no shell+ command,+ stdin=subprocess.DEVNULL,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ close_fds=True,+ cwd=state.state_dir,+ creationflags=creation_flags,+ )+ else:+ subprocess.Popen( # ruff: ignore[subprocess-without-shell-equals-true] — a fixed argument list, no shell+ command,+ stdin=subprocess.DEVNULL,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ close_fds=True,+ cwd=state.state_dir,+ start_new_session=True,+ )+ except (OSError, ValueError):+ return False+ return Truediff --git a/pipelex/system/usage_stats/usage_stats_environment.py b/pipelex/system/usage_stats/usage_stats_environment.pynew file mode 100644index 000000000..017ae7c9e--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_environment.py@@ -0,0 +1,132 @@+"""What a process can tell about where it runs, read the way the usage-statistics contract reads it.++Every reader takes the environment as a mapping rather than reading `os.environ` itself, so a test hands it+the environment it means, and so the detached sender can read the same facts from the environment it was given.+Standard library only, for the reason `usage_stats_contract` gives.+"""++import platform+import re+import sys+from collections.abc import Mapping+from pathlib import Path++from pipelex.system.usage_stats.usage_stats_contract import (+ CI_VARIABLES,+ CONTAINER_CGROUP_FILE,+ CONTAINER_CGROUP_WORDS,+ CONTAINER_MARKER_FILES,+ CONTAINER_MARKER_VARIABLES,+ FALSY_VALUES,+ RELEASE_VERSION_PATTERN,+ TOOL_SWITCH_NO_EFFECT_VALUES,+ UsageStatsArch,+ UsageStatsEnvVar,+ UsageStatsHost,+ UsageStatsOs,+)+from pipelex.system.usage_stats.usage_stats_core import CAPTURE_HOST, is_loopback_origin, path_exists++_LEADING_RELEASE = re.compile(r"^[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}")+++def read_switch_value(*, env: Mapping[str, str], name: str) -> str | None:+ """A switch variable's value, trimmed and lowercased as the contract reads every one, or `None` when unset."""+ raw_value = env.get(name)+ if raw_value is None:+ return None+ return raw_value.strip().lower()+++def is_truthy_flag(*, env: Mapping[str, str], name: str) -> bool:+ """Whether a variable read as `DO_NOT_TRACK` is set to anything but the values with no effect."""+ value = read_switch_value(env=env, name=name)+ return value is not None and value not in FALSY_VALUES+++def tool_switch_says_off(*, env: Mapping[str, str]) -> bool:+ """Whether `PIPELEX_USAGE_STATS` turns statistics off: an off value, or any value nobody can read, failing closed."""+ value = read_switch_value(env=env, name=UsageStatsEnvVar.TOOL_SWITCH)+ return value is not None and value not in TOOL_SWITCH_NO_EFFECT_VALUES+++def is_debug(*, env: Mapping[str, str]) -> bool:+ """Whether `PIPELEX_USAGE_STATS_DEBUG` asks for every event on stderr instead of sent."""+ return is_truthy_flag(env=env, name=UsageStatsEnvVar.DEBUG)+++def resolve_endpoint(*, env: Mapping[str, str]) -> str | None:+ """The origin events are posted to: PostHog's capture host, or a loopback override; `None` for an override that is neither.++ `None` turns statistics off for the process (`invalid_endpoint_override`), so the variable can never send events+ anywhere but the project or the machine itself.+ """+ override = env.get(UsageStatsEnvVar.ENDPOINT_OVERRIDE)+ if override is None:+ return CAPTURE_HOST+ return override if is_loopback_origin(origin=override) else None+++def is_ci(*, env: Mapping[str, str]) -> bool:+ """Whether any CI variable is set to anything but the empty string, `0` or `false`."""+ return any(is_truthy_flag(env=env, name=variable) for variable in CI_VARIABLES)+++def is_container(*, env: Mapping[str, str]) -> bool:+ """Whether the process runs in a container: a marker file, a marker variable that is not empty, or PID 1's cgroup."""+ if any(path_exists(path=Path(marker)) is True for marker in CONTAINER_MARKER_FILES):+ return True+ if any(env.get(variable, "").strip() for variable in CONTAINER_MARKER_VARIABLES):+ return True+ try:+ cgroup_text = Path(CONTAINER_CGROUP_FILE).read_text(encoding="utf-8", errors="replace")+ except OSError:+ # No such file outside Linux, or one this process may not read: no evidence of a container either way.+ return False+ return any(word in cgroup_text for word in CONTAINER_CGROUP_WORDS)+++def declared_host(*, env: Mapping[str, str]) -> UsageStatsHost | None:+ """The host `PIPELEX_USAGE_STATS_HOST` declares: one of the contract's values, `other` for any other, `None` when unset or empty."""+ value = read_switch_value(env=env, name=UsageStatsEnvVar.HOST)+ if not value:+ return None+ try:+ return UsageStatsHost(value)+ except ValueError:+ return UsageStatsHost.OTHER+++def current_os() -> UsageStatsOs:+ if sys.platform.startswith("linux"):+ return UsageStatsOs.LINUX+ if sys.platform == "darwin":+ return UsageStatsOs.DARWIN+ if sys.platform in {"win32", "cygwin"}:+ return UsageStatsOs.WINDOWS+ return UsageStatsOs.OTHER+++def current_arch() -> UsageStatsArch:+ machine = platform.machine().lower()+ if machine in {"x86_64", "amd64", "x64"}:+ return UsageStatsArch.X86_64+ if machine in {"arm64", "aarch64"}:+ return UsageStatsArch.ARM64+ return UsageStatsArch.OTHER+++def current_language_runtime() -> str:+ return f"python-{sys.version_info.major}.{sys.version_info.minor}"+++def release_version(*, version: str) -> str:+ """The version as the `release_version` pattern admits it: as it is when it matches, else its leading three numbers.++ A build from a checkout can carry a local segment (`0.74.0+g1a2b3c`), which the pattern refuses so that no path or+ hash can ride the version; the release it was built from is what the property means.+ """+ if re.fullmatch(RELEASE_VERSION_PATTERN, version):+ return version+ leading = _LEADING_RELEASE.match(version)+ return leading.group(0) if leading else "0.0.0"diff --git a/pipelex/system/usage_stats/usage_stats_events.py b/pipelex/system/usage_stats/usage_stats_events.pynew file mode 100644index 000000000..089e01618--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_events.py@@ -0,0 +1,275 @@+"""The events pipelex sends, typed: their properties, the properties every event carries, and the envelope they ride in.++Each model is one event's declared properties and nothing else, `extra="forbid"`, so no property the contract does not+declare can reach a payload. Optional properties are `None` until set and are left out of the payload rather than sent+as `null`. The tests check every event these build against the vendored schema, the same checks PostHog's gate makes.+"""++import math+from datetime import UTC, datetime+from typing import Any, Literal+from uuid import uuid4++from pydantic import BaseModel, ConfigDict, Field++from pipelex.system.usage_stats.usage_stats_contract import (+ PERSON_PROFILE_PROPERTY,+ SCHEMA_VERSION,+ TOOL_NAME,+ UsageStatsArch,+ UsageStatsCodegenArtifact,+ UsageStatsCodegenOutcome,+ UsageStatsCodegenTarget,+ UsageStatsErrorDomain,+ UsageStatsEventName,+ UsageStatsHost,+ UsageStatsInitExecution,+ UsageStatsIntegration,+ UsageStatsMethodSource,+ UsageStatsOs,+ UsageStatsOutcome,+ UsageStatsProvider,+ UsageStatsSurface,+ UsageStatsValidationOutcome,+)++MAX_DURATION_MS = 2_592_000_000+MAX_BUNDLE_SIZE = 100_000+MAX_BUNDLE_DEPTH = 1_000+MAX_PIPE_COUNT = 1_000_000+MAX_MODELS = 10_000+MAX_CALLS = 10_000_000+MAX_TOKENS = 1_000_000_000_000+MAX_IMAGES = 1_000_000+MAX_PAGES = 10_000_000+MAX_COST_USD = 1_000_000.0+MAX_VALIDATION_ITEMS = 100_000+#: The prefix of the `command_completed` property saying whether `init` enabled a provider's backend.+INIT_BACKEND_PREFIX = "init_backend_"+++def bounded(*, value: int, maximum: int) -> int:+ """A count held within the contract's bounds, which the gate drops an event for leaving."""+ return max(0, min(value, maximum))+++def bounded_duration_ms(*, seconds: float) -> int:+ return bounded(value=round(seconds * 1000), maximum=MAX_DURATION_MS)+++def bounded_cost(*, cost_usd: float | None) -> float | None:+ if cost_usd is None or math.isnan(cost_usd):+ return None+ return max(0.0, min(cost_usd, MAX_COST_USD))+++class UsageStatsProperties(BaseModel):+ """The base of every event's properties: frozen, and closed to anything the contract does not declare."""++ model_config = ConfigDict(frozen=True, extra="forbid")+++class CommonProperties(UsageStatsProperties):+ """The properties every event pipelex sends carries, computed once per process."""++ schema_version: int = Field(default=SCHEMA_VERSION, serialization_alias="schema")+ tool: Literal["pipelex"] = TOOL_NAME+ tool_version: str+ surface: UsageStatsSurface+ integration: UsageStatsIntegration+ host: UsageStatsHost | None = None+ id_new: bool+ os: UsageStatsOs+ arch: UsageStatsArch+ language_runtime: str+ ci: bool+ container: bool+ internal: bool+ notice_version: int+++class InitChoices(UsageStatsProperties):+ """What an `init` command set: where runs execute, and which backends it enabled, by the providers they report as.++ Either is `None` when this `init` did not set it: a hosted setup enables no backend, and a step it skipped sets nothing.+ """++ execution: UsageStatsInitExecution | None = None+ enabled_providers: frozenset[UsageStatsProvider] | None = None++ def as_properties(self) -> dict[str, Any]:+ """The `init_*` properties of the command's `command_completed`: one boolean per provider once backends were set.++ A provider the event has no field for is left out rather than refused, so a provider added to the contract's list+ before the event's fields can never break the event; the contract tests keep the two lists equal.+ """+ properties: dict[str, Any] = {}+ if self.execution is not None:+ properties["init_execution"] = self.execution+ if self.enabled_providers is not None:+ for provider in UsageStatsProvider:+ name = f"{INIT_BACKEND_PREFIX}{provider}"+ if name in INIT_BACKEND_FIELDS:+ properties[name] = provider in self.enabled_providers+ return properties+++class CommandCompletedProperties(UsageStatsProperties):+ """A command's end; the `init_*` properties only on `init`, from its `InitChoices`."""++ command: str+ outcome: UsageStatsOutcome+ error_domain: UsageStatsErrorDomain | None = None+ duration_ms: int+ hosted: bool | None = None+ init_execution: UsageStatsInitExecution | None = None+ init_backend_anthropic: bool | None = None+ init_backend_azure_openai: bool | None = None+ init_backend_bedrock: bool | None = None+ init_backend_blackboxai: bool | None = None+ init_backend_fal: bool | None = None+ init_backend_google: bool | None = None+ init_backend_groq: bool | None = None+ init_backend_huggingface: bool | None = None+ init_backend_internal: bool | None = None+ init_backend_linkup: bool | None = None+ init_backend_minimax: bool | None = None+ init_backend_mistral: bool | None = None+ init_backend_ollama: bool | None = None+ init_backend_openai: bool | None = None+ init_backend_openrouter: bool | None = None+ init_backend_portkey: bool | None = None+ init_backend_scaleway: bool | None = None+ init_backend_typesafe: bool | None = None+ init_backend_vertexai: bool | None = None+ init_backend_xai: bool | None = None+ init_backend_pipelex_manifold: bool | None = None+ init_backend_custom: bool | None = None+++#: The `init_backend_*` fields `command_completed` declares, one per provider of `UsageStatsProvider`.+INIT_BACKEND_FIELDS: frozenset[str] = frozenset(name for name in CommandCompletedProperties.model_fields if name.startswith(INIT_BACKEND_PREFIX))+++class BundleShape(UsageStatsProperties):+ """What a bundle declares, described statically: its size and the deepest nesting of its controllers."""++ bundle_pipes: int+ bundle_concepts: int+ bundle_depth: int+++class BundleFeatures(UsageStatsProperties):+ feature_batch: bool+ feature_parallel: bool+ feature_condition: bool+ feature_structured_output: bool+++class RunCompletedProperties(UsageStatsProperties):+ outcome: UsageStatsOutcome+ error_domain: UsageStatsErrorDomain | None = None+ dry_run: bool+ method_source: UsageStatsMethodSource+ duration_ms: int+ bundle_pipes: int+ bundle_concepts: int+ bundle_depth: int+ feature_batch: bool+ feature_parallel: bool+ feature_condition: bool+ feature_structured_output: bool+ feature_image_input: bool+ feature_document_input: bool+ usage_known: bool+ pipes_llm: int | None = None+ pipes_extract: int | None = None+ pipes_img_gen: int | None = None+ pipes_search: int | None = None+ pipes_compose: int | None = None+ pipes_structure: int | None = None+ pipes_doc_gen: int | None = None+ pipes_judge: int | None = None+ pipes_func: int | None = None+ pipes_sequence: int | None = None+ pipes_parallel: int | None = None+ pipes_condition: int | None = None+ pipes_batch: int | None = None+ pipes_other: int | None = None+ models_used: int | None = None+ inference_calls: int | None = None+ inference_failures: int | None = None+ input_tokens: int | None = None+ output_tokens: int | None = None+ cached_tokens: int | None = None+ images_generated: int | None = None+ pages_extracted: int | None = None+ cost_usd: float | None = None+++class ModelUsedProperties(UsageStatsProperties):+ provider: UsageStatsProvider+ model: str+ calls: int+ failures: int+ failures_transient: int+ failures_configuration: int+ failures_content: int+ failures_capacity: int+ failures_ambiguous: int+ failures_unknown: int+ input_tokens: int+ output_tokens: int+ cached_tokens: int+ images_generated: int+ pages_extracted: int+ cost_usd: float | None = None+++class MethodValidatedProperties(UsageStatsProperties):+ outcome: UsageStatsValidationOutcome+ duration_ms: int+ errors: int+ warnings: int+ bundle_pipes: int+ bundle_concepts: int+ bundle_depth: int+++class CodeGeneratedProperties(UsageStatsProperties):+ outcome: UsageStatsCodegenOutcome+ artifact: UsageStatsCodegenArtifact+ target: UsageStatsCodegenTarget | None = None+ duration_ms: int+ bundle_pipes: int+ bundle_concepts: int+++class UsageStatsEvent(BaseModel):+ """An event as it is created: its name and its own properties. The envelope adds who sent it, and when."""++ model_config = ConfigDict(frozen=True)++ name: UsageStatsEventName+ properties: UsageStatsProperties+++def event_timestamp(*, moment: datetime) -> str:+ """RFC 3339 in UTC with milliseconds, such as `2026-10-11T09:30:00.123Z`."""+ return moment.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")+++def make_envelope(*, event: UsageStatsEvent, distinct_id: str, common: CommonProperties) -> dict[str, Any]:+ """The event as it is posted, its `uuid` and `timestamp` fixed now and kept by every resend."""+ properties: dict[str, Any] = {+ **common.model_dump(mode="json", by_alias=True, exclude_none=True),+ **event.properties.model_dump(mode="json", by_alias=True, exclude_none=True),+ PERSON_PROFILE_PROPERTY: False,+ }+ return {+ "event": event.name,+ "distinct_id": distinct_id,+ "uuid": str(uuid4()),+ "timestamp": event_timestamp(moment=datetime.now(UTC)),+ "properties": properties,+ }diff --git a/pipelex/system/usage_stats/usage_stats_runs.py b/pipelex/system/usage_stats/usage_stats_runs.pynew file mode 100644index 000000000..8894d4304--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_runs.py@@ -0,0 +1,157 @@+"""What a run's inference calls did, tallied as they end, for the run's `run_completed` and `model_used` events.++The run's events carry its pipes and its token usage, but not what a call that failed was refused for, which backend+served a model, or how many images and pages a call produced. The inference call's own summary event knows all of it+when the call ends, so it notes it here, in the tally of the run the call belongs to: a context variable `PipeRun`+sets for the duration of a run, which the run's tasks inherit. With no run tallying (statistics off, or a call made+outside any run), noting costs one context-variable read.++The tally is in-process: a run whose calls execute in other processes, a Temporal worker's, is tallied there or not at+all. The hosted plane sets `DO_NOT_TRACK=1`, so no such run reports.+"""++import threading+from collections.abc import Generator+from contextlib import contextmanager+from contextvars import ContextVar++from pipelex.base_exceptions import ErrorDomain, PipelexError+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsErrorDomain, UsageStatsMethodSource++#: The failure categories a `model_used` event counts, the runtime's own `InferenceErrorCategory` values.+FAILURE_CATEGORIES: tuple[str, ...] = ("transient", "configuration", "content", "capacity", "ambiguous", "unknown")+#: The category a failure is counted under when the runtime could not classify it.+UNCLASSIFIED_FAILURE_CATEGORY = "unknown"+++class ModelCallTally:+ """The calls a run made to one model through one backend, as their summary events saw them end."""++ def __init__(self, *, backend_name: str) -> None:+ self.backend_name = backend_name+ self.calls = 0+ self.failures = 0+ self.failures_by_category: dict[str, int] = dict.fromkeys(FAILURE_CATEGORIES, 0)+ self.images_generated = 0+ self.pages_extracted = 0+++class RunInferenceTally:+ """The inference calls of one run, by model handle and backend; calls whose model never resolved count only in the totals.++ A handle can be served by more than one backend in one run, as when a call falls back to another, so each pair is+ tallied on its own and each call is attributed to the backend that made it. The run's usage reports name that backend+ themselves, since two backends can call a model by the same identifier.+ """++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self.calls = 0+ self.failures = 0+ self.models: dict[tuple[str, str], ModelCallTally] = {}++ def note_call(+ self,+ *,+ model_handle: str,+ backend_name: str | None,+ failed: bool,+ failure_category: str | None,+ images_generated: int,+ pages_extracted: int,+ ) -> None:+ with self._lock:+ self.calls += 1+ if failed:+ self.failures += 1+ if backend_name is None:+ return+ model_tally = self.models.get((model_handle, backend_name))+ if model_tally is None:+ model_tally = ModelCallTally(backend_name=backend_name)+ self.models[model_handle, backend_name] = model_tally+ model_tally.calls += 1+ model_tally.images_generated += images_generated+ model_tally.pages_extracted += pages_extracted+ if failed:+ model_tally.failures += 1+ category = failure_category if failure_category in model_tally.failures_by_category else UNCLASSIFIED_FAILURE_CATEGORY+ model_tally.failures_by_category[category] += 1+++_RUN_TALLY: ContextVar[RunInferenceTally | None] = ContextVar("pipelex_usage_stats_run_tally", default=None)+++def current_run_tally() -> RunInferenceTally | None:+ """The tally of the run this code runs in, or `None` when no run is tallying."""+ return _RUN_TALLY.get()+++@contextmanager+def tallying_inference_calls(*, tally: RunInferenceTally | None) -> Generator[None, None, None]:+ """Tally the inference calls made inside the block in `tally`, the run's tasks included, which inherit the context.++ With no tally, the run reports no statistics, and the block runs as it would without this.+ """+ if tally is None:+ yield+ return+ token = _RUN_TALLY.set(tally)+ try:+ yield+ finally:+ _RUN_TALLY.reset(token)+++def usage_stats_error_domain(*, error: BaseException | None) -> UsageStatsErrorDomain:+ """Who can fix a failure, by the runtime's own error report: `input`, `config`, `runtime`, or `other` when it cannot say."""+ if not isinstance(error, PipelexError):+ return UsageStatsErrorDomain.OTHER+ return usage_stats_error_domain_of(domain=error.to_error_report().error_domain)+++def usage_stats_error_domain_of(*, domain: ErrorDomain | str | None) -> UsageStatsErrorDomain:+ """The statistics' error domain of a runtime domain, given as the enum or as its value; `other` for none or another."""+ if domain is None:+ return UsageStatsErrorDomain.OTHER+ try:+ runtime_domain = ErrorDomain(domain)+ except ValueError:+ return UsageStatsErrorDomain.OTHER+ match runtime_domain:+ case ErrorDomain.INPUT:+ return UsageStatsErrorDomain.INPUT+ case ErrorDomain.CONFIG:+ return UsageStatsErrorDomain.CONFIG+ case ErrorDomain.RUNTIME:+ return UsageStatsErrorDomain.RUNTIME+++_METHOD_SOURCE: ContextVar[UsageStatsMethodSource | None] = ContextVar("pipelex_usage_stats_method_source", default=None)+++def declare_method_source(*, source: UsageStatsMethodSource) -> None:+ """Declare where the method about to run came from, by kind only, for its `run_completed`.++ Called where the kind is known for sure, a CLI resolving a method by name, address or path, before the runner+ sees only the files it was handed; a declaration stands for the rest of the context it was made in.+ """+ _METHOD_SOURCE.set(source)+++@contextmanager+def method_source_scope(*, source: UsageStatsMethodSource) -> Generator[None, None, None]:+ """Declare the method's source for the block, unless the caller already declared a more precise one."""+ if _METHOD_SOURCE.get() is not None:+ yield+ return+ token = _METHOD_SOURCE.set(source)+ try:+ yield+ finally:+ _METHOD_SOURCE.reset(token)+++def current_method_source() -> UsageStatsMethodSource:+ """Where the method running now came from, `other` when nothing declared it."""+ return _METHOD_SOURCE.get() or UsageStatsMethodSource.OTHERdiff --git a/pipelex/system/usage_stats/usage_stats_status.py b/pipelex/system/usage_stats/usage_stats_status.pynew file mode 100644index 000000000..2a686860a--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_status.py@@ -0,0 +1,68 @@+"""What `pipelex usage-stats status` reports, read without changing anything: it creates, migrates and prints nothing else.++The JSON form is the contract's status object, field for field, which is how other tools and the conformance suite+read pipelex's statistics surface. The contract is `conformance/specs/usage-statistics.md`, "Commands".+"""++from collections.abc import Mapping+from pathlib import Path++from pydantic import BaseModel, ConfigDict++from pipelex.system.usage_stats.usage_stats_contract import NOTICE_VERSION, TOOL_NAME, UsageStatsOffBy+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState+from pipelex.system.usage_stats.usage_stats_environment import is_ci, is_container+from pipelex.system.usage_stats.usage_stats_switches import read_switches+++class UsageStatsStatus(BaseModel):+ """The status object of the contract: whether statistics are on, why not, and the shared facts they would carry."""++ model_config = ConfigDict(frozen=True, extra="forbid")++ tool: str+ enabled: bool+ off_by: list[UsageStatsOffBy]+ installation_id: str | None+ notice_version: int+ internal: bool+ ci: bool+ container: bool+ state_dir: str+++def read_usage_stats_status(*, state: UsageStatsState, env: Mapping[str, str], cwd: Path, is_pytest: bool) -> UsageStatsStatus:+ """Read every switch and the shared state as they are now, writing nothing."""+ off_by = read_switches(state=state, env=env, cwd=cwd, is_pytest=is_pytest)+ return UsageStatsStatus(+ tool=TOOL_NAME,+ enabled=not off_by,+ off_by=off_by,+ installation_id=state.read_installation_id(),+ notice_version=state.seen_notice_version(up_to=NOTICE_VERSION),+ internal=state.is_internal(),+ ci=is_ci(env=env),+ container=is_container(env=env),+ state_dir=str(state.state_dir),+ )+++def describe_off_by(*, token: UsageStatsOffBy) -> str:+ """What a switch is, in words, for the text form of `status` and for `enable`."""+ match token:+ case UsageStatsOffBy.DO_NOT_TRACK:+ return "the DO_NOT_TRACK environment variable"+ case UsageStatsOffBy.TOOL_VARIABLE:+ return "the PIPELEX_USAGE_STATS environment variable"+ case UsageStatsOffBy.MACHINE_CHOICE:+ return "this machine's choice (`pipelex usage-stats disable`, or another tool's)"+ case UsageStatsOffBy.LEGACY_MTHDS_CHOICE:+ return "the telemetry setting of the mthds CLI in ~/.mthds/config"+ case UsageStatsOffBy.PROJECT_SETTING:+ return "this project's .pipelex/pipelex.toml ([usage_stats] enabled = false, or a file that cannot be read)"+ case UsageStatsOffBy.PACKAGED_DEFAULT:+ return "the configuration pipelex was packaged with"+ case UsageStatsOffBy.INTEGRATION_MODE:+ return "the pytest integration mode"+ case UsageStatsOffBy.INVALID_ENDPOINT_OVERRIDE:+ return "the PIPELEX_USAGE_STATS_ENDPOINT environment variable, which is not a loopback http origin"diff --git a/pipelex/system/usage_stats/usage_stats_switches.py b/pipelex/system/usage_stats/usage_stats_switches.pynew file mode 100644index 000000000..c5b6a47e5--- /dev/null+++ b/pipelex/system/usage_stats/usage_stats_switches.py@@ -0,0 +1,117 @@+"""The switches that turn usage statistics off, read in the contract's order before a process does any work.++Statistics are on unless a switch says off, and no switch can turn them back on over another that says off. Every+switch is read, so `status` names each one that says off, in this order: `DO_NOT_TRACK`, `PIPELEX_USAGE_STATS`, the+shared `disabled` file, the legacy `mthds` opt-out newer than its migration, the project's `.pipelex/pipelex.toml`+(failing closed), pipelex's packaged configuration, the `pytest` integration mode, and an endpoint override that is not a+loopback origin. The contract is `conformance/specs/usage-statistics.md`, "Switches".+"""++from collections.abc import Mapping+from pathlib import Path+from typing import Any, cast++import tomli++from pipelex.system.configuration.config_loader import CONFIG_NAME, ConfigLoader+from pipelex.system.environment import CONFIG_DIR_NAME+from pipelex.system.usage_stats.usage_stats_contract import SETTING_KEY, SETTING_TABLE, UsageStatsEnvVar, UsageStatsOffBy+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState, path_exists+from pipelex.system.usage_stats.usage_stats_environment import is_truthy_flag, resolve_endpoint, tool_switch_says_off++#: pipelex's packaged configuration, which carries `[usage_stats] enabled = true`; a packager may ship `false`.+PACKAGED_CONFIG_PATH = Path(__file__).resolve().parents[2] / CONFIG_NAME+++def setting_says_off(*, config_path: Path) -> bool:+ """Whether a configuration file's `[usage_stats] enabled` turns statistics off, failing closed.++ `false` says off, and so does a file that cannot be read, is not valid TOML, has a `usage_stats` that is not a+ table, or an `enabled` that is not a boolean. `true`, or no `enabled` at all, has no effect.+ """+ try:+ document = tomli.loads(config_path.read_text(encoding="utf-8"))+ except (OSError, UnicodeDecodeError, tomli.TOMLDecodeError):+ return True+ table: object = document.get(SETTING_TABLE)+ if table is None:+ return False+ if not isinstance(table, dict):+ return True+ enabled: object = cast("dict[str, Any]", table).get(SETTING_KEY)+ if enabled is None:+ return False+ if not isinstance(enabled, bool):+ return True+ return not enabled+++def project_setting_says_off(*, cwd: Path) -> bool:+ """Whether the project's own `.pipelex/pipelex.toml` turns statistics off, read alone, never the merged configuration.++ The project is found the way pipelex finds its project configuration: from `cwd`, walking up to the first directory+ holding a project marker, stopping at the home directory, which is never a project, and at the filesystem root. So+ `~/.pipelex/pipelex.toml`, pipelex's home layer, is not a switch. No project, or no file there, means no setting;+ a project or a file that cannot be looked for is a file that cannot be read, which turns statistics off.+ """+ try:+ project_root = ConfigLoader.find_project_root(cwd)+ except (OSError, RuntimeError):+ # A directory on the way up that cannot be looked into, or no home directory to stop at: the project's file+ # cannot be read, which fails closed.+ return True+ if project_root is None:+ return False+ config_path = project_root / CONFIG_DIR_NAME / CONFIG_NAME+ config_exists = path_exists(path=config_path)+ if config_exists is None:+ # A file whose very existence cannot be told cannot be read either, which fails closed.+ return True+ if not config_exists:+ return False+ return setting_says_off(config_path=config_path)+++def packaged_default_says_off(*, packaged_config_path: Path = PACKAGED_CONFIG_PATH) -> bool:+ """Whether pipelex's packaged configuration ships statistics off; a package that does not say `true` fails closed."""+ try:+ document = tomli.loads(packaged_config_path.read_text(encoding="utf-8"))+ except (OSError, UnicodeDecodeError, tomli.TOMLDecodeError):+ return True+ table: object = document.get(SETTING_TABLE)+ if not isinstance(table, dict):+ return True+ return cast("dict[str, Any]", table).get(SETTING_KEY) is not True+++def read_switches(+ *,+ state: UsageStatsState,+ env: Mapping[str, str],+ cwd: Path,+ is_pytest: bool,+ packaged_config_path: Path = PACKAGED_CONFIG_PATH,+) -> list[UsageStatsOffBy]:+ """Every switch that says off, by its token, in the contract's order; empty when statistics are on.++ Read-only: the legacy migration, which every reporting process runs first, is the caller's to run, since `status`+ reads the switches without migrating anything.+ """+ off_by: list[UsageStatsOffBy] = []+ if is_truthy_flag(env=env, name=UsageStatsEnvVar.DO_NOT_TRACK):+ off_by.append(UsageStatsOffBy.DO_NOT_TRACK)+ if tool_switch_says_off(env=env):+ off_by.append(UsageStatsOffBy.TOOL_VARIABLE)+ if state.is_disabled():+ off_by.append(UsageStatsOffBy.MACHINE_CHOICE)+ if state.legacy_opt_out_is_pending():+ off_by.append(UsageStatsOffBy.LEGACY_MTHDS_CHOICE)+ if project_setting_says_off(cwd=cwd):+ off_by.append(UsageStatsOffBy.PROJECT_SETTING)+ if packaged_default_says_off(packaged_config_path=packaged_config_path):+ off_by.append(UsageStatsOffBy.PACKAGED_DEFAULT)+ if is_pytest:+ off_by.append(UsageStatsOffBy.INTEGRATION_MODE)+ if resolve_endpoint(env=env) is None:+ off_by.append(UsageStatsOffBy.INVALID_ENDPOINT_OVERRIDE)+ return off_bydiff --git a/pipelex/test_extras/usage_stats_isolation.py b/pipelex/test_extras/usage_stats_isolation.pynew file mode 100644index 000000000..7c4046f41--- /dev/null+++ b/pipelex/test_extras/usage_stats_isolation.py@@ -0,0 +1,80 @@+"""Keep a test session's anonymous usage statistics off, away from the machine's shared state, and away from any host+that is not loopback.++pipelex's own suites and the API server's call these from their root `conftest.py`: no test reports to Pipelex, and no+test touches `~/.mthds/`. A test that exercises the statistics turns them on itself, with a capture server of its own on+loopback.+"""++import atexit+import os+import shutil+import tempfile+from collections.abc import Generator+from contextlib import contextmanager+from pathlib import Path+from typing import Any++import pytest++from pipelex.system.usage_stats import usage_stats_client, usage_stats_core+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsEnvVar+from pipelex.system.usage_stats.usage_stats_core import PostVerdict, UsageStatsState, is_loopback_origin++#: A loopback origin nothing listens on: a test that turns statistics on without a capture server of its own posts nowhere.+USAGE_STATS_DEAD_LOOPBACK_ORIGIN = "http://127.0.0.1:9"+++def usage_stats_off_environment() -> dict[str, str]:+ """The variables that keep a process's statistics off and its endpoint on loopback, for an environment built from scratch."""+ return {+ UsageStatsEnvVar.TOOL_SWITCH: "0",+ UsageStatsEnvVar.ENDPOINT_OVERRIDE: USAGE_STATS_DEAD_LOOPBACK_ORIGIN,+ }+++def turn_usage_stats_off_for_the_session() -> Path:+ """Turn statistics off in this process's environment, which every process it starts inherits, and make a home for its shared state.++ Called when a root `conftest.py` loads, before any test runs and before xdist starts its workers. The home is+ deleted when the process exits.+ """+ os.environ.update(usage_stats_off_environment())+ home = Path(tempfile.mkdtemp(prefix="pipelex-tests-usage-stats-home-"))+ atexit.register(shutil.rmtree, home, ignore_errors=True)+ return home+++@contextmanager+def usage_stats_guarded(*, home: Path) -> Generator[None, None, None]:+ """Point the shared state at `home`, and refuse any post, or detached sender, for an origin that is not loopback.++ A refused attempt fails the test that made it, and fails the session when the block ends, since an attempt made on a+ background thread cannot fail a test.+ """+ refused_origins: list[str] = []+ original_post_batch = usage_stats_core.post_batch+ original_start_detached_sender = usage_stats_client.start_detached_sender++ def guarded_post_batch(*, origin: str, events: list[dict[str, Any]], timeout_seconds: float) -> PostVerdict:+ if not is_loopback_origin(origin=origin):+ refused_origins.append(origin)+ pytest.fail(f"A test tried to post usage statistics to {origin}, which is not loopback")+ return original_post_batch(origin=origin, events=events, timeout_seconds=timeout_seconds)++ def guarded_start_detached_sender(*, state: UsageStatsState, origin: str) -> bool:+ if not is_loopback_origin(origin=origin):+ refused_origins.append(origin)+ pytest.fail(f"A test tried to start a usage statistics sender for {origin}, which is not loopback")+ return original_start_detached_sender(state=state, origin=origin)++ def make_test_state() -> UsageStatsState:+ return UsageStatsState(home=home)++ with pytest.MonkeyPatch.context() as monkeypatch:+ monkeypatch.setattr(UsageStatsState, "make_for_current_home", staticmethod(make_test_state))+ monkeypatch.setattr(usage_stats_core, "post_batch", guarded_post_batch)+ monkeypatch.setattr(usage_stats_client, "start_detached_sender", guarded_start_detached_sender)+ yield+ if refused_origins:+ pytest.fail(f"Tests tried to send usage statistics to hosts that are not loopback: {sorted(set(refused_origins))}")diff --git a/tests/conftest.py b/tests/conftest.pyindex 129d942dc..d8bf7812d 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -13,6 +13,7 @@ from pipelex.system.environment import PIPELEX_HOME_ENV_KEYfrom pipelex.system.job_metadata import JobMetadata, RunMetadatafrom pipelex.system.runtime import IntegrationMode, runtime_managerfrom pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract+from pipelex.test_extras.usage_stats_isolation import turn_usage_stats_off_for_the_session, usage_stats_guardedfrom pipelex.tools.log.error_fields import error_fieldsif TYPE_CHECKING:@@ -24,6 +25,11 @@ pytest_plugins = [TEST_OUTPUTS_DIR = "temp/test_outputs"+# Anonymous usage statistics: no test reports to Pipelex, and none touches this machine's shared state in `~/.mthds/`.+# Turned off in the environment when this conftest loads, before any test runs and before xdist starts its workers, so+# every process a test starts inherits it; `isolate_usage_stats` below guards this process and its workers.+USAGE_STATS_TEST_HOME = turn_usage_stats_off_for_the_session()+def _fast_telemetry_teardown(self: "TelemetryManager") -> None:"""Skip expensive OTel/PostHog shutdown during tests (~0.49s per call).@@ -58,6 +64,13 @@ def cache_configs_for_session(session_mocker: MockerFixture):session_mocker.patch.object(TelemetryManager, "teardown", _fast_telemetry_teardown)[email protected](scope="session", autouse=True)+def isolate_usage_stats() -> Generator[None, None, None]:+ """Keep the shared statistics state in a temporary home, and fail any attempt to send to a host that is not loopback."""+ with usage_stats_guarded(home=USAGE_STATS_TEST_HOME):+ yield++def _get_test_integration_mode() -> IntegrationMode:"""Return the appropriate integration mode for tests.diff --git a/tests/data/usage_stats/usage-statistics.json b/tests/data/usage_stats/usage-statistics.jsonnew file mode 100644index 000000000..95a929cb5--- /dev/null+++ b/tests/data/usage_stats/usage-statistics.json@@ -0,0 +1,1634 @@+{+ "description": "The usage-statistics contract in machine-readable form: the source of truth for specs/usage-statistics.md's generated tables, the PostHog gate generated from it, and every emitter's typed events. Emitters vendor it byte-identical; conformance/scripts/check-usage-statistics-schema.py gates the copies.",+ "spec": "conformance/specs/usage-statistics.md",+ "schema_version": 1,+ "destination": {+ "project": "Open-source usage",+ "region": "eu",+ "capture_host": "https://eu.i.posthog.com",+ "batch_path": "/batch/",+ "project_token": "phc_uQjSVbzNPJxA8bHxfevn7NMzB6PbC6aJQRB6ka6MxT7a"+ },+ "notice": {+ "version": 1,+ "details_url": "https://docs.pipelex.com/latest/setup/usage-statistics/",+ "text": "Pipelex collects anonymous usage statistics: counts of commands, runs, pipe types, models and tokens, and the country your IP address maps to; the address itself is never stored. It never collects your methods, prompts, inputs, outputs, file names or pipe names. To turn it off, run `{disable_command}` or set DO_NOT_TRACK=1. Details: {details_url}",+ "disable_commands": {+ "pipelex": "pipelex usage-stats disable",+ "plxt": "plxt usage-stats disable",+ "vscode-pipelex": "Pipelex: Disable Usage Statistics",+ "mthds": "mthds telemetry disable"+ }+ },+ "patterns": {+ "release_version": {+ "regex": "^[0-9]{1,4}\\.[0-9]{1,4}\\.[0-9]{1,4}([.-]?(a|b|rc|alpha|beta|dev|post|insider)[.]?[0-9]{0,4})?$",+ "max_length": 32,+ "description": "A released version: three numeric components and at most one pre-release, development or post-release tag from a closed set."+ },+ "language_runtime": {+ "regex": "^(python-[0-9]{1,2}\\.[0-9]{1,3}|node-[0-9]{1,3})$",+ "max_length": 16,+ "description": "A language runtime and its version: major and minor for Python, such as `python-3.13`, major only for Node, such as `node-22`."+ },+ "model_handle": {+ "regex": "^[A-Za-z0-9][A-Za-z0-9._:/-]{0,95}$",+ "max_length": 96,+ "description": "A model's name as a shipped backend table names it, or `other`."+ },+ "installation_id": {+ "regex": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",+ "max_length": 36,+ "description": "A random UUID version 4 in its canonical lowercase form."+ },+ "ip_address": {+ "regex": "^[0-9A-Fa-f.:]{2,45}$",+ "max_length": 45,+ "description": "An IPv4 or IPv6 address as PostHog's capture writes it."+ },+ "country_code": {+ "regex": "^[A-Z]{2}$",+ "max_length": 2,+ "description": "An ISO 3166-1 alpha-2 country code."+ },+ "country_name": {+ "regex": "^[^\\x00-\\x1f]{1,64}$",+ "max_length": 64,+ "description": "A country's English name as PostHog's GeoIP database gives it."+ },+ "event_uuid": {+ "regex": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$",+ "max_length": 36,+ "description": "An event's UUID: the version 4 the sender drew, or the one PostHog assigns to an event that came without one."+ },+ "event_timestamp": {+ "regex": "^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}([.][0-9]{1,9})?(Z|[+-][0-9]{2}:?[0-9]{2})?$",+ "max_length": 40,+ "description": "An RFC 3339 date and time, such as `2026-10-11T09:30:00.123Z`, as the sender wrote it or PostHog normalised it."+ }+ },+ "envelope": {+ "fields": {+ "distinct_id": {+ "pattern": "installation_id",+ "description": "The installation identifier."+ },+ "uuid": {+ "pattern": "event_uuid",+ "description": "Drawn when the event is created and kept by every resend."+ },+ "timestamp": {+ "pattern": "event_timestamp",+ "description": "When the event was created, in UTC with milliseconds."+ }+ },+ "empty_fields": {+ "elements_chain": "PostHog derives it from `$elements_chain`, which no event declares; no sender sets it.",+ "url": "PostHog derives it from `$current_url`, which no event declares; no sender sets it."+ }+ },+ "common_properties": {+ "schema": {+ "type": "integer",+ "required": true,+ "min": 1,+ "max": 1,+ "description": "The version of this event schema the event follows."+ },+ "tool": {+ "type": "enum",+ "required": true,+ "values": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ],+ "description": "The reporting tool."+ },+ "tool_version": {+ "type": "pattern",+ "required": true,+ "pattern": "release_version",+ "description": "The reporting tool's released version."+ },+ "surface": {+ "type": "enum",+ "required": true,+ "values_by": {+ "property": "tool",+ "values": {+ "pipelex": [+ "pipelex",+ "pipelex-agent",+ "api",+ "python"+ ],+ "plxt": [+ "plxt"+ ],+ "vscode-pipelex": [+ "extension"+ ],+ "mthds": [+ "mthds",+ "mthds-agent"+ ]+ }+ },+ "description": "Which entry point of the tool sent the event; the values depend on `tool`."+ },+ "integration": {+ "type": "enum",+ "required": true,+ "values": [+ "cli",+ "python",+ "fastapi",+ "docker",+ "mcp",+ "n8n",+ "ci"+ ],+ "description": "The pipelex runtime's integration mode. Sent by pipelex only.",+ "when": {+ "property": "tool",+ "equals": "pipelex"+ }+ },+ "host": {+ "type": "enum",+ "required": false,+ "values": [+ "cocode",+ "mthds-agent",+ "other"+ ],+ "description": "The program of ours that embeds or spawns the tool, when one declares itself; absent otherwise."+ },+ "id_new": {+ "type": "boolean",+ "required": true,+ "description": "Whether this process created the installation identifier."+ },+ "os": {+ "type": "enum",+ "required": true,+ "values": [+ "linux",+ "darwin",+ "windows",+ "other"+ ],+ "description": "The operating system family."+ },+ "arch": {+ "type": "enum",+ "required": true,+ "values": [+ "x86_64",+ "arm64",+ "other"+ ],+ "description": "The processor architecture."+ },+ "language_runtime": {+ "type": "pattern",+ "required": false,+ "pattern": "language_runtime",+ "description": "The language runtime the tool runs on, for the tools that run on one."+ },+ "ci": {+ "type": "boolean",+ "required": true,+ "description": "Whether the process runs in continuous integration, by the rule of the CI section."+ },+ "container": {+ "type": "boolean",+ "required": true,+ "description": "Whether the process runs in a container, by the rule of the container section."+ },+ "internal": {+ "type": "boolean",+ "required": true,+ "description": "Whether the shared state marks the machine as one of the team's."+ },+ "notice_version": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1,+ "description": "The highest notice version recorded as shown on this machine, 0 when none."+ }+ },+ "events": {+ "run_completed": {+ "description": "One per run, whatever started it and however it ended.",+ "tools": [+ "pipelex"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "outcome": {+ "type": "enum",+ "required": true,+ "values": [+ "succeeded",+ "failed",+ "cancelled"+ ],+ "description": "How the run ended."+ },+ "error_domain": {+ "type": "enum",+ "required": true,+ "values": [+ "input",+ "config",+ "runtime",+ "other"+ ],+ "description": "For a failed run, who can fix the error, `other` when the runtime cannot say.",+ "when": {+ "property": "outcome",+ "equals": "failed"+ }+ },+ "dry_run": {+ "type": "boolean",+ "required": true,+ "description": "Whether the run was a dry run."+ },+ "method_source": {+ "type": "enum",+ "required": true,+ "values": [+ "local_bundle",+ "inline_source",+ "installed_method",+ "published_address",+ "catalog_id",+ "other"+ ],+ "description": "Where the method came from, by kind only."+ },+ "duration_ms": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 2592000000,+ "description": "The run's wall-clock duration in milliseconds."+ },+ "bundle_pipes": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of pipes the run's bundle declares."+ },+ "bundle_concepts": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of concepts the run's bundle declares."+ },+ "bundle_depth": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000,+ "description": "The deepest nesting of controllers under the run's main pipe."+ },+ "feature_batch": {+ "type": "boolean",+ "required": true,+ "description": "Whether the bundle uses batching."+ },+ "feature_parallel": {+ "type": "boolean",+ "required": true,+ "description": "Whether the bundle uses parallel branches."+ },+ "feature_condition": {+ "type": "boolean",+ "required": true,+ "description": "Whether the bundle uses a condition."+ },+ "feature_structured_output": {+ "type": "boolean",+ "required": true,+ "description": "Whether a pipe of the bundle produces a structured concept."+ },+ "feature_image_input": {+ "type": "boolean",+ "required": true,+ "description": "Whether the run received an image input."+ },+ "feature_document_input": {+ "type": "boolean",+ "required": true,+ "description": "Whether the run received a document input."+ },+ "usage_known": {+ "type": "boolean",+ "required": true,+ "description": "Whether the run's events were read for its counts; false when tracing or usage events are off."+ },+ "pipes_llm": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeLLM pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_extract": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeExtract pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_img_gen": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeImgGen pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_search": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeSearch pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_compose": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeCompose pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_structure": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeStructure pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_doc_gen": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeDocGen pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_judge": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeJudge pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_func": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeFunc pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_sequence": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeSequence pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_parallel": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeParallel pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_condition": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeCondition pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_batch": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of PipeBatch pipes the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pipes_other": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The number of pipes of a type pipelex does not ship that the run executed.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "models_used": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000,+ "description": "The number of distinct models the run called, one `model_used` each.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "inference_calls": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The run's inference calls.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "inference_failures": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The run's failed inference calls.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "input_tokens": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000000000,+ "description": "The run's input tokens.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "output_tokens": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000000000,+ "description": "The run's output tokens.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "cached_tokens": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000000000,+ "description": "The run's cached input tokens.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "images_generated": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The images the run generated.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "pages_extracted": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The document pages the run extracted.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ },+ "cost_usd": {+ "type": "number",+ "required": false,+ "min": 0,+ "max": 1000000,+ "description": "The run's inference cost in US dollars from the release's own rate tables, absent when they do not know every model used.",+ "when": {+ "property": "usage_known",+ "equals": true+ }+ }+ }+ },+ "model_used": {+ "description": "One per distinct provider and model a run called, sent with the run's `run_completed` when `usage_known` is true.",+ "tools": [+ "pipelex"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "provider": {+ "type": "enum",+ "required": true,+ "values": [+ "anthropic",+ "azure_openai",+ "bedrock",+ "blackboxai",+ "fal",+ "google",+ "groq",+ "huggingface",+ "internal",+ "linkup",+ "minimax",+ "mistral",+ "ollama",+ "openai",+ "openrouter",+ "portkey",+ "scaleway",+ "typesafe",+ "vertexai",+ "xai",+ "pipelex_manifold",+ "custom"+ ],+ "description": "The backend the model was called through: a backend pipelex ships or one our plugins add, `custom` for any other."+ },+ "model": {+ "type": "pattern",+ "required": true,+ "pattern": "model_handle",+ "description": "The model's name in the shipped backend's table, `other` for a custom backend or a model the release does not ship.",+ "fixed_when": {+ "property": "provider",+ "equals": "custom",+ "value": "other"+ }+ },+ "calls": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The run's calls to this model."+ },+ "failures": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The run's failed calls to this model."+ },+ "failures_transient": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The failed calls the runtime classified as `transient`."+ },+ "failures_configuration": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The failed calls the runtime classified as `configuration`."+ },+ "failures_content": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The failed calls the runtime classified as `content`."+ },+ "failures_capacity": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The failed calls the runtime classified as `capacity`."+ },+ "failures_ambiguous": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The failed calls the runtime classified as `ambiguous`."+ },+ "failures_unknown": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "The failed calls the runtime classified as `unknown`."+ },+ "input_tokens": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000000000,+ "description": "Input tokens sent to this model."+ },+ "output_tokens": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000000000,+ "description": "Output tokens this model produced."+ },+ "cached_tokens": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000000000,+ "description": "Cached input tokens for this model."+ },+ "images_generated": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "Images this model generated."+ },+ "pages_extracted": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 10000000,+ "description": "Pages this model extracted."+ },+ "cost_usd": {+ "type": "number",+ "required": false,+ "min": 0,+ "max": 1000000,+ "description": "The cost of these calls in US dollars, absent when the release has no rate for the model."+ }+ }+ },+ "command_completed": {+ "description": "One per CLI command, whatever its outcome, except `--help`, `--version`, the statistics command group, `plxt lsp` and `plxt completions`.",+ "tools": [+ "pipelex",+ "plxt",+ "mthds"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "command": {+ "type": "enum",+ "required": true,+ "values_by": {+ "property": "surface",+ "values": {+ "pipelex": [+ "init",+ "login",+ "doctor",+ "update",+ "migrate",+ "build inputs bundle",+ "build inputs method",+ "build inputs pipe",+ "build output bundle",+ "build output method",+ "build output pipe",+ "build structures",+ "validate",+ "validate bundle",+ "validate method",+ "validate pipe",+ "fix bundle",+ "resolve",+ "codegen types",+ "codegen inputs",+ "codegen check",+ "run bundle",+ "run method",+ "run pipe",+ "graph render",+ "show config",+ "show pipe",+ "show models",+ "show backends",+ "which",+ "plugins list",+ "other"+ ],+ "pipelex-agent": [+ "init",+ "run bundle",+ "run method",+ "run pipe",+ "validate",+ "validate bundle",+ "validate method",+ "validate pipe",+ "fix bundle",+ "fmt",+ "lint",+ "inputs bundle",+ "inputs method",+ "inputs pipe",+ "codegen types",+ "codegen inputs",+ "codegen check",+ "models",+ "check-model",+ "migrate",+ "doctor",+ "other"+ ],+ "plxt": [+ "format",+ "lint",+ "get",+ "config",+ "other"+ ],+ "mthds": [+ "login",+ "run bundle",+ "run method",+ "run pipe",+ "build inputs method",+ "build inputs pipe",+ "validate bundle",+ "validate method",+ "validate pipe",+ "install",+ "publish",+ "share",+ "config get",+ "config list",+ "config set",+ "runner setup",+ "runner set-default",+ "runner status",+ "package init",+ "package list",+ "package validate",+ "other"+ ],+ "mthds-agent": [+ "bootstrap",+ "install",+ "publish",+ "share",+ "config get",+ "config list",+ "config set",+ "package init",+ "package list",+ "package validate",+ "runner setup api",+ "runner setup pipelex",+ "plxt fmt",+ "plxt lint",+ "doctor",+ "init",+ "validate bundle",+ "validate method",+ "validate pipe",+ "run bundle",+ "run method",+ "run pipe",+ "models",+ "check-model",+ "inputs bundle",+ "inputs method",+ "inputs pipe",+ "codegen check",+ "codegen types",+ "update-check",+ "upgrade",+ "codex apply-config",+ "codex hook",+ "other"+ ]+ }+ },+ "description": "The command's path in the CLI's own command tree, aliases resolved to the canonical name, never its arguments; `other` for a command this schema does not list yet."+ },+ "outcome": {+ "type": "enum",+ "required": true,+ "values": [+ "succeeded",+ "failed",+ "cancelled"+ ],+ "description": "How the command ended; `cancelled` for an interrupt."+ },+ "error_domain": {+ "type": "enum",+ "required": false,+ "values": [+ "input",+ "config",+ "runtime",+ "other"+ ],+ "description": "For a failed command, who can fix the error, when the tool can say.",+ "when": {+ "property": "outcome",+ "equals": "failed"+ }+ },+ "duration_ms": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 2592000000,+ "description": "The command's wall-clock duration in milliseconds."+ },+ "hosted": {+ "type": "boolean",+ "required": false,+ "description": "Whether the command was sent to the hosted Pipelex API (`--hosted` or the configured execution)."+ },+ "init_execution": {+ "type": "enum",+ "required": false,+ "values": [+ "local",+ "hosted"+ ],+ "description": "Where `init` set runs to execute.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_anthropic": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `anthropic` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_azure_openai": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `azure_openai` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_bedrock": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `bedrock` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_blackboxai": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `blackboxai` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_fal": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `fal` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_google": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `google` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_groq": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `groq` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_huggingface": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `huggingface` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_internal": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `internal` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_linkup": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `linkup` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_minimax": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `minimax` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_mistral": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `mistral` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_ollama": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `ollama` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_openai": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `openai` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_openrouter": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `openrouter` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_portkey": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `portkey` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_scaleway": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `scaleway` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_typesafe": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `typesafe` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_vertexai": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `vertexai` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_xai": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `xai` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_pipelex_manifold": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled the `pipelex_manifold` backend.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "init_backend_custom": {+ "type": "boolean",+ "required": false,+ "description": "Whether `init` enabled a backend pipelex does not ship.",+ "when": {+ "property": "command",+ "equals": "init"+ }+ },+ "files_bucket": {+ "type": "enum",+ "required": true,+ "values": [+ "0",+ "1",+ "2-10",+ "11-100",+ "101-1000",+ "1001+"+ ],+ "description": "How many files `plxt` processed, in buckets.",+ "when": {+ "property": "tool",+ "equals": "plxt"+ }+ },+ "invocation": {+ "type": "enum",+ "required": true,+ "values": [+ "terminal",+ "pre_commit",+ "ci",+ "other"+ ],+ "description": "What started `plxt`.",+ "when": {+ "property": "tool",+ "equals": "plxt"+ }+ },+ "diagnostics_syntax": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The `syntax` diagnostics `plxt` reported.",+ "when": {+ "property": "tool",+ "equals": "plxt"+ }+ },+ "diagnostics_semantic": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The `semantic` diagnostics `plxt` reported.",+ "when": {+ "property": "tool",+ "equals": "plxt"+ }+ },+ "diagnostics_schema": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000000,+ "description": "The `schema` diagnostics `plxt` reported.",+ "when": {+ "property": "tool",+ "equals": "plxt"+ }+ }+ }+ },+ "method_validated": {+ "description": "One per validation of a method, whatever started it.",+ "tools": [+ "pipelex"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "outcome": {+ "type": "enum",+ "required": true,+ "values": [+ "valid",+ "invalid",+ "failed"+ ],+ "description": "The verdict: `valid`, `invalid`, or `failed` when no verdict could be reached."+ },+ "duration_ms": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 2592000000,+ "description": "The validation's duration in milliseconds."+ },+ "errors": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of error items the verdict carries."+ },+ "warnings": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of warning items the verdict carries."+ },+ "bundle_pipes": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of pipes the bundle declares."+ },+ "bundle_concepts": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of concepts the bundle declares."+ },+ "bundle_depth": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 1000,+ "description": "The deepest nesting of controllers in the bundle."+ }+ }+ },+ "code_generated": {+ "description": "One per code generation, whatever started it.",+ "tools": [+ "pipelex"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "outcome": {+ "type": "enum",+ "required": true,+ "values": [+ "succeeded",+ "failed"+ ],+ "description": "How the generation ended."+ },+ "artifact": {+ "type": "enum",+ "required": true,+ "values": [+ "types",+ "inputs",+ "structures",+ "other"+ ],+ "description": "What was generated."+ },+ "target": {+ "type": "enum",+ "required": true,+ "values": [+ "python-structures",+ "python-pydantic",+ "ts-zod",+ "other"+ ],+ "description": "The target flavour of a `types` generation.",+ "when": {+ "property": "artifact",+ "equals": "types"+ }+ },+ "duration_ms": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 2592000000,+ "description": "The generation's duration in milliseconds."+ },+ "bundle_pipes": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of pipes the bundle declares."+ },+ "bundle_concepts": {+ "type": "integer",+ "required": true,+ "min": 0,+ "max": 100000,+ "description": "The number of concepts the bundle declares."+ }+ }+ },+ "extension_activated": {+ "description": "At most one per installation and UTC day, when the VS Code extension activates with statistics on.",+ "tools": [+ "vscode-pipelex"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "editor_version": {+ "type": "pattern",+ "required": true,+ "pattern": "release_version",+ "description": "The editor's version, as `vscode.version` reports it."+ },+ "editor": {+ "type": "enum",+ "required": true,+ "values": [+ "vscode",+ "vscode-insiders",+ "cursor",+ "windsurf",+ "vscodium",+ "other"+ ],+ "description": "Which editor hosts the extension, from `vscode.env.appName`."+ }+ }+ },+ "feature_used": {+ "description": "The VS Code extension's uses of one feature, counted in memory and sent in small batches.",+ "tools": [+ "vscode-pipelex"+ ],+ "common": true,+ "person_profile": "never",+ "properties": {+ "feature": {+ "type": "enum",+ "required": true,+ "values": [+ "graph_preview",+ "validation",+ "run",+ "format",+ "other"+ ],+ "description": "The feature used."+ },+ "count": {+ "type": "integer",+ "required": true,+ "min": 1,+ "max": 100000,+ "description": "How many times it was used since the previous `feature_used` for that feature."+ },+ "editor_version": {+ "type": "pattern",+ "required": true,+ "pattern": "release_version",+ "description": "The editor's version, as `vscode.version` reports it."+ },+ "editor": {+ "type": "enum",+ "required": true,+ "values": [+ "vscode",+ "vscode-insiders",+ "cursor",+ "windsurf",+ "vscodium",+ "other"+ ],+ "description": "Which editor hosts the extension, from `vscode.env.appName`."+ }+ }+ },+ "deletion_requested": {+ "description": "Sent only by the deletion procedure, never by a tool. The gate passes it only with the secret deletion key, which it strips.",+ "tools": [],+ "common": false,+ "person_profile": "deletion",+ "properties": {+ "schema": {+ "type": "integer",+ "required": true,+ "min": 1,+ "max": 1,+ "description": "The version of this event schema the event follows."+ },+ "deletion_key": {+ "type": "secret",+ "required": true,+ "description": "The secret the gate compares with its `deletion_key` input; stripped before the event is stored."+ }+ }+ }+ },+ "posthog_added_properties": {+ "$ip": {+ "type": "pattern",+ "required": false,+ "pattern": "ip_address",+ "description": "The client address, which PostHog's capture adds and drops after the transformations, since the project discards client IP data."+ },+ "$geoip_country_code": {+ "type": "pattern",+ "required": false,+ "pattern": "country_code",+ "description": "Added by PostHog's GeoIP transformation."+ },+ "$geoip_country_name": {+ "type": "pattern",+ "required": false,+ "pattern": "country_name",+ "description": "Added by PostHog's GeoIP transformation."+ }+ },+ "person_profile_property": "$process_person_profile",+ "switch_values": {+ "falsy": [+ "",+ "0",+ "false"+ ],+ "tool_switch_off": [+ "0",+ "false",+ "no",+ "off"+ ],+ "tool_switch_no_effect": [+ "",+ "1",+ "true",+ "yes",+ "on"+ ],+ "mthds_disable_on": [+ "1",+ "true",+ "yes",+ "on"+ ]+ },+ "switches": [+ {+ "token": "do_not_track",+ "source": "`DO_NOT_TRACK` set to anything but empty, `0` or `false`",+ "tools": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ]+ },+ {+ "token": "tool_variable",+ "source": "`PIPELEX_USAGE_STATS` saying off; for the `mthds` CLI also its own `DISABLE_TELEMETRY` environment variable",+ "tools": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ]+ },+ {+ "token": "machine_choice",+ "source": "the shared state's `disabled` file",+ "tools": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ]+ },+ {+ "token": "legacy_mthds_choice",+ "source": "`DISABLE_TELEMETRY` on in `~/.mthds/config`, newer than the `legacy-migrated` marker",+ "tools": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ]+ },+ {+ "token": "project_setting",+ "source": "`[usage_stats] enabled = false` in the project's `.pipelex/pipelex.toml`, or that file unreadable",+ "tools": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ]+ },+ {+ "token": "packaged_default",+ "source": "`[usage_stats] enabled = false` in pipelex's packaged configuration",+ "tools": [+ "pipelex"+ ]+ },+ {+ "token": "editor_setting",+ "source": "VS Code's telemetry setting, its enterprise policy included",+ "tools": [+ "vscode-pipelex"+ ]+ },+ {+ "token": "integration_mode",+ "source": "pipelex's `pytest` integration mode",+ "tools": [+ "pipelex"+ ]+ },+ {+ "token": "invalid_endpoint_override",+ "source": "`PIPELEX_USAGE_STATS_ENDPOINT` set to anything but a loopback `http` origin",+ "tools": [+ "pipelex",+ "plxt",+ "vscode-pipelex",+ "mthds"+ ]+ }+ ],+ "environment": {+ "do_not_track": "DO_NOT_TRACK",+ "tool_switch": "PIPELEX_USAGE_STATS",+ "debug": "PIPELEX_USAGE_STATS_DEBUG",+ "endpoint_override": "PIPELEX_USAGE_STATS_ENDPOINT",+ "host": "PIPELEX_USAGE_STATS_HOST",+ "mthds_tool_switch": "DISABLE_TELEMETRY"+ },+ "state": {+ "directory": "~/.mthds/usage-stats",+ "id": "id",+ "disabled": "disabled",+ "legacy_migrated": "legacy-migrated",+ "notice_prefix": "notice-v",+ "internal": "internal",+ "spool": "spool",+ "legacy_config": "~/.mthds/config",+ "legacy_key": "DISABLE_TELEMETRY"+ },+ "spool": {+ "max_files_per_tool": 100,+ "max_age_days": 30,+ "stale_claim_minutes": 10,+ "stale_temp_minutes": 60+ },+ "budgets_ms": {+ "cli_in_process": 1000,+ "python_exit_flush": 250,+ "shutdown_flush": 2000,+ "detached_sender": 10000,+ "background_post_delay": 10000+ },+ "ci_variables": [+ "CI",+ "CONTINUOUS_INTEGRATION",+ "BUILD_NUMBER",+ "GITHUB_ACTIONS",+ "GITLAB_CI",+ "CIRCLECI",+ "TRAVIS",+ "BUILDKITE",+ "TF_BUILD",+ "JENKINS_URL",+ "TEAMCITY_VERSION",+ "BITBUCKET_BUILD_NUMBER",+ "CODEBUILD_BUILD_ID",+ "DRONE",+ "APPVEYOR",+ "SEMAPHORE",+ "WOODPECKER"+ ],+ "container_markers": {+ "files": [+ "/.dockerenv",+ "/run/.containerenv"+ ],+ "environment": [+ "container",+ "KUBERNETES_SERVICE_HOST"+ ],+ "cgroup_words": [+ "docker",+ "kubepods",+ "containerd",+ "lxc"+ ]+ },+ "project_markers": [+ ".pipelex",+ ".git",+ "pyproject.toml",+ "setup.py",+ "setup.cfg",+ "package.json",+ ".hg"+ ]+}diff --git a/tests/e2e/agent_cli/conftest.py b/tests/e2e/agent_cli/conftest.pyindex 06829c4a4..f98667085 100644--- a/tests/e2e/agent_cli/conftest.py+++ b/tests/e2e/agent_cli/conftest.py@@ -17,6 +17,7 @@ from pathlib import Pathimport pytestfrom pipelex.kit.paths import get_kit_configs_dir+from pipelex.test_extras.usage_stats_isolation import usage_stats_off_environmentREPO_ROOT = Path(__file__).resolve().parents[3]PIPELEX_AGENT_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex-agent"@@ -90,6 +91,8 @@ def offline_subprocess_env(hermetic_home: Path) -> dict[str, str]:env: dict[str, str] = {"HOME": str(hermetic_home),"PATH": "/usr/bin:/bin:/usr/local/bin",+ # No subprocess a test starts reports anonymous usage statistics, and none could reach a host that is not loopback.+ **usage_stats_off_environment(),# Force CI test mode, so a boot that resolves credentials (the doctor's) skips vertexai,# whose token it would otherwise mint over the network."RUN_MODE": "ci_test",diff --git a/tests/e2e/agent_cli/test_usage_stats_canary.py b/tests/e2e/agent_cli/test_usage_stats_canary.pynew file mode 100644index 000000000..5c0b77bbd--- /dev/null+++ b/tests/e2e/agent_cli/test_usage_stats_canary.py@@ -0,0 +1,116 @@+import json+import subprocess # ruff: ignore[suspicious-subprocess-import] — invokes the real CLIs for end-to-end coverage+from pathlib import Path+from typing import Any++from pipelex.system.usage_stats.usage_stats_contract import UsageStatsEnvVar+from tests.e2e.agent_cli.conftest import PIPELEX_AGENT_BIN, REPO_ROOT, write_active_routing_profile+from tests.helpers.usage_stats_capture import capture_server+from tests.helpers.usage_stats_schema import assert_envelope_is_valid+from tests.unit.pipelex.system.usage_stats.test_data import UsageStatsCanary++PIPELEX_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex"+_CANARY = UsageStatsCanary.MARKER+_TITLE = _CANARY.capitalize()++_VALID_BUNDLE = f"""\+domain = "{_CANARY}_domain"+description = "Canary {_CANARY} description"+main_pipe = "{_CANARY}_main"++[concept]+{_TITLE}Report = "Canary {_CANARY} concept"++[pipe.{_CANARY}_main]+type = "PipeSequence"+description = "Canary {_CANARY} sequence"+inputs = {{ {_CANARY}_input = "Text" }}+output = "{_TITLE}Report"+steps = [{{ pipe = "{_CANARY}_step", result = "{_CANARY}_result" }}]++[pipe.{_CANARY}_step]+type = "PipeLLM"+description = "Canary {_CANARY} step"+inputs = {{ {_CANARY}_input = "Text" }}+output = "{_TITLE}Report"+model = "claude-4.5-haiku"+prompt = "Canary {_CANARY}: ${_CANARY}_input"+"""+_INVALID_BUNDLE = _VALID_BUNDLE.replace(+ f'steps = [{{ pipe = "{_CANARY}_step", result = "{_CANARY}_result" }}]',+ f'steps = [{{ pipe = "{_CANARY}_step", result = "{_CANARY}_result" }}, {{ pipe = "{_CANARY}_missing", result = "{_CANARY}_other" }}]',+)+++def _stage(*, home: Path) -> tuple[Path, Path, Path]:+ """The canary bundles, valid and invalid, each alone in a directory whose name carries the canary too."""+ valid_dir = home / f"{_CANARY}_valid_dir"+ invalid_dir = home / f"{_CANARY}_invalid_dir"+ valid_dir.mkdir()+ invalid_dir.mkdir()+ valid_bundle = valid_dir / f"{_CANARY}_bundle.mthds"+ valid_bundle.write_text(_VALID_BUNDLE, encoding="utf-8")+ invalid_bundle = invalid_dir / f"{_CANARY}_bundle.mthds"+ invalid_bundle.write_text(_INVALID_BUNDLE, encoding="utf-8")+ return valid_dir, valid_bundle, invalid_bundle+++def _commands(*, home: Path) -> list[list[str]]:+ valid_dir, valid_bundle, invalid_bundle = _stage(home=home)+ output_dir = home / f"{_CANARY}_generated"+ agent = str(PIPELEX_AGENT_BIN)+ return [+ [agent, "validate", "bundle", str(valid_bundle), "--format", "json"],+ [agent, "validate", "bundle", str(invalid_bundle), "--format", "json"],+ [agent, "run", "bundle", str(valid_bundle), "--dry-run", "--mock-inputs", "--no-graph", "--format", "json"],+ [agent, "codegen", "types", str(valid_dir), "--target", "ts-zod", "--output", str(output_dir)],+ [agent, "inputs", "bundle", str(valid_bundle)],+ [str(PIPELEX_BIN), "--no-logo", "validate", "bundle", str(invalid_bundle)],+ [str(PIPELEX_BIN), "--no-logo", "codegen", "types", str(valid_dir), "--target", "python-pydantic", "--output", str(output_dir)],+ ]+++def _run(*, command: list[str], env: dict[str, str], cwd: Path) -> subprocess.CompletedProcess[str]:+ return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] — a fixed argument list, no shell+ command, env=env, cwd=str(cwd), capture_output=True, text=True, check=False, timeout=180+ )+++def _state_text(*, home: Path) -> str:+ """Everything the shared state directory holds, names and contents."""+ state_dir = home / ".mthds" / "usage-stats"+ parts: list[str] = []+ for path in sorted(state_dir.rglob("*")):+ parts.append(path.name)+ if path.is_file():+ parts.append(path.read_text(encoding="utf-8", errors="replace"))+ return "\n".join(parts)+++class TestUsageStatsCanary:+ def test_canary_never_leaves_the_machine(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:+ """Runs, validations and code generations on a method named everywhere with a canary print, send and keep no trace of it."""+ write_active_routing_profile(hermetic_home / ".pipelex" / "inference" / "routing_profiles.toml", "all_anthropic")+ commands = _commands(home=hermetic_home)+ stats_on_env = {name: value for name, value in offline_subprocess_env.items() if name != UsageStatsEnvVar.TOOL_SWITCH}++ printed: list[dict[str, Any]] = []+ for command in commands:+ result = _run(command=command, env={**stats_on_env, UsageStatsEnvVar.DEBUG: "1"}, cwd=hermetic_home)+ for line in result.stderr.splitlines():+ if line.startswith('{"usage_stats_debug"'):+ printed.append(json.loads(line)["usage_stats_debug"])+ printed_names = {envelope["event"] for envelope in printed}+ assert {"command_completed", "method_validated", "run_completed", "code_generated"} <= printed_names, printed_names+ for envelope in printed:+ assert_envelope_is_valid(envelope)+ assert _CANARY not in json.dumps(printed).lower()++ with capture_server() as server:+ for command in commands:+ _run(command=command, env={**stats_on_env, UsageStatsEnvVar.ENDPOINT_OVERRIDE: server.origin, "CI": "true"}, cwd=hermetic_home)+ sent = server.events+ assert len(sent) >= len(commands)+ assert all(envelope["properties"]["ci"] is True for envelope in sent)+ assert _CANARY not in json.dumps([request.body for request in server.requests]).lower()+ assert _CANARY not in _state_text(home=hermetic_home).lower()diff --git a/tests/e2e/agent_cli/test_usage_stats_init.py b/tests/e2e/agent_cli/test_usage_stats_init.pynew file mode 100644index 000000000..63b5d0b46--- /dev/null+++ b/tests/e2e/agent_cli/test_usage_stats_init.py@@ -0,0 +1,58 @@+import json+import subprocess # ruff: ignore[suspicious-subprocess-import] — invokes the real agent CLI for end-to-end coverage+from pathlib import Path+from typing import Any++import pytest++from pipelex.system.usage_stats.usage_stats_contract import UsageStatsEnvVar+from tests.e2e.agent_cli.conftest import PIPELEX_AGENT_BIN+from tests.helpers.usage_stats_process import make_project_dir+from tests.helpers.usage_stats_schema import assert_envelope_is_valid+++def _command_completed(stderr: str) -> dict[str, Any]:+ printed: list[dict[str, Any]] = [json.loads(line)["usage_stats_debug"] for line in stderr.splitlines() if line.startswith('{"usage_stats_debug"')]+ completed = [envelope for envelope in printed if envelope["event"] == "command_completed"]+ assert len(completed) == 1, printed+ return completed[0]+++class TestUsageStatsInit:+ @pytest.mark.parametrize(+ ("config", "execution", "enabled"),+ [+ ({"backends": ["anthropic", "openai"], "primary_backend": "anthropic"}, "local", {"anthropic", "openai"}),+ ({"execution": "hosted"}, "hosted", None),+ ],+ )+ def test_agent_init_reports_what_it_set(+ self,+ hermetic_home: Path,+ offline_subprocess_env: dict[str, str],+ config: dict[str, Any],+ execution: str,+ enabled: set[str] | None,+ ) -> None:+ """`pipelex-agent init` reports where it set runs to execute and, on this machine, the backends it enabled."""+ env = {name: value for name, value in offline_subprocess_env.items() if name != UsageStatsEnvVar.TOOL_SWITCH}+ env[UsageStatsEnvVar.DEBUG] = "1"+ result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] — a fixed argument list, no shell+ [str(PIPELEX_AGENT_BIN), "init", "--config", json.dumps(config), "--format", "json"],+ env=env,+ cwd=str(make_project_dir(parent=hermetic_home)),+ capture_output=True,+ text=True,+ check=False,+ timeout=120,+ )+ assert result.returncode == 0, result.stderr+ envelope = _command_completed(result.stderr)+ assert_envelope_is_valid(envelope)+ properties = envelope["properties"]+ assert (properties["command"], properties["outcome"], properties["init_execution"]) == ("init", "succeeded", execution)+ backends = {name.removeprefix("init_backend_") for name, value in properties.items() if name.startswith("init_backend_") and value}+ if enabled is None:+ assert not [name for name in properties if name.startswith("init_backend_")]+ else:+ assert backends == enableddiff --git a/tests/e2e/pipelex/cli/conftest.py b/tests/e2e/pipelex/cli/conftest.pyindex 007a28e4d..d7b155382 100644--- a/tests/e2e/pipelex/cli/conftest.py+++ b/tests/e2e/pipelex/cli/conftest.py@@ -12,6 +12,7 @@ from typing import TYPE_CHECKINGimport pytest+from pipelex.test_extras.usage_stats_isolation import usage_stats_off_environmentfrom tests.e2e.agent_cli.conftest import ( # ruff: ignore[unused-import] - fixtures re-exported for this directoryhermetic_home,offline_subprocess_env,@@ -44,4 +45,6 @@ def fresh_home_env(tmp_path: Path) -> dict[str, str]:"PIPELEX_API_KEY": TEST_KEY,"PIPELEX_BASE_URL": "http://127.0.0.1:9","PIPELEX_APP_URL": "http://127.0.0.1:9",+ # No subprocess a test starts reports anonymous usage statistics, and none could reach a host that is not loopback.+ **usage_stats_off_environment(),}diff --git a/tests/e2e/pipelex/cli/test_hosted_run_cli.py b/tests/e2e/pipelex/cli/test_hosted_run_cli.pyindex bbc05fc34..aa33cf5b5 100644--- a/tests/e2e/pipelex/cli/test_hosted_run_cli.py+++ b/tests/e2e/pipelex/cli/test_hosted_run_cli.py@@ -29,6 +29,7 @@ from pathlib import Pathimport pytestfrom pipelex.hosted.client_factory import HOSTED_API_DEFAULT_BASE_URL, PIPELEX_API_KEY_ENV_KEY, PIPELEX_BASE_URL_ENV_KEY+from pipelex.test_extras.usage_stats_isolation import usage_stats_off_environmentREPO_ROOT = Path(__file__).resolve().parents[4]PIPELEX_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex"@@ -73,6 +74,8 @@ def _hosted_env(*, home: Path, base_url: str | None) -> dict[str, str]:# Wide enough that Rich never wraps the lines the tests read."COLUMNS": "250",PIPELEX_API_KEY_ENV_KEY: os.environ[PIPELEX_API_KEY_ENV_KEY],+ # No subprocess a test starts reports anonymous usage statistics, and none could reach a host that is not loopback.+ **usage_stats_off_environment(),}if base_url is not None:env[PIPELEX_BASE_URL_ENV_KEY] = base_urldiff --git a/tests/helpers/usage_stats_capture.py b/tests/helpers/usage_stats_capture.pynew file mode 100644index 000000000..2a73aa0fa--- /dev/null+++ b/tests/helpers/usage_stats_capture.py@@ -0,0 +1,129 @@+"""A capture server on loopback standing in for PostHog's `/batch/` endpoint, for the tests that send usage statistics.++It records every request's path and JSON body and answers each with the next status of its script, `200` once the+script is spent. Tests point `PIPELEX_USAGE_STATS_ENDPOINT` at its origin, the only override the client accepts.+"""++import json+import socket+import threading+import time+from collections.abc import Generator+from contextlib import contextmanager+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from typing import Any, NamedTuple, cast++from typing_extensions import override+++class CapturedRequest(NamedTuple):+ path: str+ content_type: str | None+ body: dict[str, Any]+++class CaptureServer:+ """What the server received, and the statuses it is left to answer with."""++ def __init__(self, *, origin: str, statuses: list[int]) -> None:+ self.origin = origin+ self.statuses = statuses+ self.requests: list[CapturedRequest] = []+ self.lock = threading.Lock()++ def next_status(self) -> int:+ with self.lock:+ return self.statuses.pop(0) if self.statuses else 200++ def record(self, *, request: CapturedRequest) -> None:+ with self.lock:+ self.requests.append(request)++ @property+ def events(self) -> list[dict[str, Any]]:+ """Every event every request carried, in the order received."""+ with self.lock:+ requests = list(self.requests)+ events: list[dict[str, Any]] = []+ for request in requests:+ events.extend(cast("list[dict[str, Any]]", request.body.get("batch", [])))+ return events+++@contextmanager+def capture_server(*, statuses: list[int] | None = None) -> Generator[CaptureServer, None, None]:+ """Serve on `127.0.0.1` on a free port for the duration of the block."""+ server_state: CaptureServer | None = None++ class _Handler(BaseHTTPRequestHandler):+ def do_POST(self) -> None:+ length = int(self.headers.get("Content-Length", "0"))+ raw_body = self.rfile.read(length)+ body = cast("dict[str, Any]", json.loads(raw_body.decode("utf-8")))+ assert server_state is not None+ server_state.record(request=CapturedRequest(path=self.path, content_type=self.headers.get("Content-Type"), body=body))+ status = server_state.next_status()+ self.send_response(status)+ self.send_header("Content-Length", "2")+ self.end_headers()+ self.wfile.write(b"{}")++ @override+ def log_message(self, format: str, *args: Any) -> None:+ return++ httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ server_state = CaptureServer(origin=f"http://127.0.0.1:{httpd.server_address[1]}", statuses=list(statuses or []))+ thread = threading.Thread(target=httpd.serve_forever, daemon=True)+ thread.start()+ try:+ yield server_state+ finally:+ httpd.shutdown()+ httpd.server_close()+ thread.join(timeout=5)+++#: The answer a trickling server sends, one byte at a time.+_TRICKLED_ANSWER = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"+++@contextmanager+def trickling_server(*, seconds_per_byte: float) -> Generator[str, None, None]:+ """Serve on `127.0.0.1` a server that reads a request and answers `200` one byte at a time; yield its origin.++ No socket operation of the client waits long enough to time out, so only a deadline on the whole exchange ends the+ request before the answer is complete.+ """+ listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)+ listener.bind(("127.0.0.1", 0))+ listener.listen()+ listener.settimeout(0.1)+ stopping = threading.Event()++ def serve() -> None:+ while not stopping.is_set():+ try:+ connection, _ = listener.accept()+ except OSError:+ continue+ with connection:+ try:+ connection.settimeout(5)+ connection.recv(65536)+ for byte in _TRICKLED_ANSWER:+ if stopping.is_set():+ break+ connection.sendall(bytes([byte]))+ time.sleep(seconds_per_byte)+ except OSError:+ continue++ thread = threading.Thread(target=serve, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{listener.getsockname()[1]}"+ finally:+ stopping.set()+ thread.join(timeout=5)+ listener.close()diff --git a/tests/helpers/usage_stats_process.py b/tests/helpers/usage_stats_process.pynew file mode 100644index 000000000..565e57a55--- /dev/null+++ b/tests/helpers/usage_stats_process.py@@ -0,0 +1,121 @@+"""Turn this process's usage statistics on for one test, against a home and a loopback origin of the test's own.++Every test session keeps the statistics off (`pipelex.test_extras.usage_stats_isolation`): the environment carries+`PIPELEX_USAGE_STATS=0` and a dead loopback endpoint, and the shared state lives in a temporary home. A test exercising+the statistics takes them back on here, for its own duration: it clears every variable that would say off or mark the+process as in CI or a container, points the shared state at `home`, the endpoint at `origin`, the working directory at+`cwd`, and forgets the process's client before and after, so the process's own boot decided nothing for it.+"""++import os+import sys+from collections.abc import Generator, Mapping+from contextlib import contextmanager+from pathlib import Path++import pytest++from pipelex.system.usage_stats import usage_stats_client+from pipelex.system.usage_stats.usage_stats_client import reset_usage_stats_for_tests+from pipelex.system.usage_stats.usage_stats_contract import CI_VARIABLES, CONTAINER_MARKER_VARIABLES, UsageStatsEnvVar+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState+from pipelex.test_extras.usage_stats_isolation import USAGE_STATS_DEAD_LOOPBACK_ORIGIN++#: The variables that decide a process's statistics, cleared before a test sets the ones it needs.+DECIDING_VARIABLES: tuple[str, ...] = (+ UsageStatsEnvVar.DO_NOT_TRACK,+ UsageStatsEnvVar.TOOL_SWITCH,+ UsageStatsEnvVar.DEBUG,+ UsageStatsEnvVar.ENDPOINT_OVERRIDE,+ UsageStatsEnvVar.HOST,+ *CI_VARIABLES,+ *CONTAINER_MARKER_VARIABLES,+)+++#: Whether a directory can be made untraversable for this process: not on Windows, and not for root, who traverses anything.+CAN_LOCK_DIRECTORIES = sys.platform != "win32" and os.geteuid() != 0+++def _exists_as_python_314_does(self: Path, *, follow_symlinks: bool = True) -> bool:+ return os.path.exists(self) if follow_symlinks else os.path.lexists(self)+++def _is_dir_as_python_314_does(self: Path, *, follow_symlinks: bool = True) -> bool:+ assert follow_symlinks, "the code under test never asks without following links"+ return os.path.isdir(self)+++def _is_file_as_python_314_does(self: Path, *, follow_symlinks: bool = True) -> bool:+ assert follow_symlinks, "the code under test never asks without following links"+ return os.path.isfile(self) # ruff: ignore[os-path-isfile] (`Path.is_file` is the method this stands in for)+++@contextmanager+def locked_directory(*, path: Path) -> Generator[Path, None, None]:+ """Make `path` a directory no one may list or traverse for the block, as a state directory above an unreadable home is.++ Within the block, `Path.exists()`, `Path.is_dir()` and `Path.is_file()` answer as Python 3.14's do, whatever the+ interpreter: they swallow every `OSError` and answer `False`, where earlier versions raise one they do not expect.+ Code that must tell "absent" from "cannot tell" is then held to the interpreter that hides the difference.+ """+ path.mkdir(parents=True, exist_ok=True)+ path.chmod(0)+ try:+ with pytest.MonkeyPatch.context() as patcher:+ patcher.setattr(Path, "exists", _exists_as_python_314_does)+ patcher.setattr(Path, "is_dir", _is_dir_as_python_314_does)+ patcher.setattr(Path, "is_file", _is_file_as_python_314_does)+ yield path+ finally:+ path.chmod(0o700)+++def make_project_dir(*, parent: Path) -> Path:+ """A project root of the test's own, marked by `.git`, so no project around the test's checkout is read."""+ project_dir = parent / "project"+ (project_dir / ".git").mkdir(parents=True, exist_ok=True)+ return project_dir+++def stats_on_environment(*, origin: str = USAGE_STATS_DEAD_LOOPBACK_ORIGIN) -> dict[str, str]:+ """An environment for a child process with statistics on, posting to `origin`, and nothing else deciding them."""+ environment = {name: value for name, value in os.environ.items() if name not in DECIDING_VARIABLES}+ environment[UsageStatsEnvVar.ENDPOINT_OVERRIDE] = origin+ return environment+++def _never_a_container(*, env: Mapping[str, str]) -> bool:+ del env+ return False+++@contextmanager+def usage_stats_switched_on(+ *,+ monkeypatch: pytest.MonkeyPatch,+ home: Path,+ cwd: Path,+ origin: str = USAGE_STATS_DEAD_LOOPBACK_ORIGIN,+ is_debug: bool = False,+) -> Generator[UsageStatsState, None, None]:+ """Statistics on for the block, with their shared state under `home`; yields that state."""+ for name in DECIDING_VARIABLES:+ monkeypatch.delenv(name, raising=False)+ monkeypatch.setenv(UsageStatsEnvVar.ENDPOINT_OVERRIDE, origin)+ if is_debug:+ monkeypatch.setenv(UsageStatsEnvVar.DEBUG, "1")+ state = UsageStatsState(home=home)++ def make_state() -> UsageStatsState:+ return state++ monkeypatch.setattr(UsageStatsState, "make_for_current_home", staticmethod(make_state))+ # A machine that is a container would deliver in-process: the tests decide where they deliver, not the machine.+ monkeypatch.setattr(usage_stats_client, "is_container", _never_a_container)+ monkeypatch.chdir(cwd)+ reset_usage_stats_for_tests()+ try:+ yield state+ finally:+ reset_usage_stats_for_tests()diff --git a/tests/helpers/usage_stats_run_events.py b/tests/helpers/usage_stats_run_events.pynew file mode 100644index 000000000..3cb9abd2b--- /dev/null+++ b/tests/helpers/usage_stats_run_events.py@@ -0,0 +1,86 @@+"""Trace events and envelopes for the tests of a run's usage statistics, built the way a run writes them."""++from datetime import UTC, datetime+from typing import Any++from pipelex.cogt.llm.llm_report import LLMTokensUsage+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.cogt.usage.token_category import TokenCategory+from pipelex.graph.graphspec import NodeKind+from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsArch, UsageStatsIntegration, UsageStatsOs, UsageStatsSurface+from pipelex.system.usage_stats.usage_stats_events import CommonProperties, UsageStatsEvent, make_envelope+from pipelex.tracing.trace_events import PipeStartEvent, UsageReportEvent++USAGE_STATS_RUN_ID = "run-usage-stats"+USAGE_STATS_INSTALLATION_ID = "0f8fad5b-d9cb-469f-a165-70867728950e"+++def llm_usage_event(+ *,+ model_name: str,+ backend_name: str,+ input_tokens: int,+ output_tokens: int,+ cached_tokens: int = 0,+ is_rated: bool = True,+ sequence: int = 0,+ model_id: str | None = None,+) -> UsageReportEvent:+ """One inference call's usage, as the backend that served it reports it; an unrated call carries no rate table, so no cost."""+ tokens_usage = LLMTokensUsage(+ job_metadata=JobMetadata(+ run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user_test", pipeline_run_id=USAGE_STATS_RUN_ID),+ pipe_code="secret_pipe_code",+ unit_job_id=UnitJobId.LLM_GEN_TEXT,+ job_category=JobCategory.LLM_JOB,+ ),+ inference_model_name=model_name,+ inference_model_id=model_id or f"{model_name}-provider-id",+ backend_name=backend_name,+ unit_costs={CostCategory.INPUT: 0.003, CostCategory.OUTPUT: 0.015} if is_rated else {},+ nb_tokens_by_category={+ TokenCategory.INPUT: input_tokens,+ TokenCategory.OUTPUT: output_tokens,+ TokenCategory.INPUT_CACHED: cached_tokens,+ },+ )+ return UsageReportEvent(+ pipeline_run_id=USAGE_STATS_RUN_ID,+ workflow_id="direct",+ timestamp=datetime.now(UTC),+ sequence=sequence,+ node_id=f"{USAGE_STATS_RUN_ID}:node_{sequence}",+ tokens_usage=tokens_usage,+ )+++def pipe_start_event(*, pipe_type: str, sequence: int) -> PipeStartEvent:+ return PipeStartEvent(+ pipeline_run_id=USAGE_STATS_RUN_ID,+ workflow_id="direct",+ timestamp=datetime.now(UTC),+ sequence=sequence,+ node_id=f"{USAGE_STATS_RUN_ID}:node_{sequence}",+ pipe_code=f"secret_pipe_{sequence}",+ pipe_type=pipe_type,+ node_kind=NodeKind.OPERATOR,+ )+++def envelope_of(*, event: UsageStatsEvent) -> dict[str, Any]:+ """The event as a Python program running pipelex would post it."""+ common = CommonProperties(+ tool_version="0.83.0",+ surface=UsageStatsSurface.PYTHON,+ integration=UsageStatsIntegration.PYTHON,+ id_new=False,+ os=UsageStatsOs.LINUX,+ arch=UsageStatsArch.X86_64,+ language_runtime="python-3.13",+ ci=False,+ container=False,+ internal=False,+ notice_version=1,+ )+ return make_envelope(event=event, distinct_id=USAGE_STATS_INSTALLATION_ID, common=common)diff --git a/tests/helpers/usage_stats_schema.py b/tests/helpers/usage_stats_schema.pynew file mode 100644index 000000000..31e93218a--- /dev/null+++ b/tests/helpers/usage_stats_schema.py@@ -0,0 +1,143 @@+"""Check an event pipelex builds against the vendored usage-statistics schema, as PostHog's gate would.++The schema is `tests/data/usage_stats/usage-statistics.json`, a byte-identical copy of the contract's+`conformance/specs/usage-statistics.json`. `envelope_problems` applies the gate's rules to one posted envelope: a+declared event the tool may send, the envelope fields matching their patterns, `$process_person_profile` false, every+property declared, present only when its condition holds, of its type and within its bounds or values, and every+required property present. An empty list means the gate keeps the event.+"""++import json+import re+from functools import cache+from pathlib import Path+from typing import Any, cast++USAGE_STATS_SCHEMA_PATH = Path(__file__).resolve().parents[1] / "data" / "usage_stats" / "usage-statistics.json"+++@cache+def load_usage_stats_schema() -> dict[str, Any]:+ return cast("dict[str, Any]", json.loads(USAGE_STATS_SCHEMA_PATH.read_text(encoding="utf-8")))+++def _json_type(value: object) -> str:+ """The type a JSON reader in JavaScript gives a value: a number with no fractional part is an integer."""+ if value is None:+ return "null"+ if isinstance(value, bool):+ return "boolean"+ if isinstance(value, int):+ return "integer"+ if isinstance(value, float):+ return "integer" if value.is_integer() else "float"+ if isinstance(value, str):+ return "string"+ return "other"+++def _condition_holds(*, condition: dict[str, Any], properties: dict[str, Any]) -> bool:+ actual = properties.get(condition["property"])+ expected = condition["equals"]+ return _json_type(actual) == _json_type(expected) and actual == expected+++def _pattern_matches(*, pattern_name: str, value: str) -> bool:+ pattern = load_usage_stats_schema()["patterns"][pattern_name]+ return len(value) <= pattern["max_length"] and re.fullmatch(pattern["regex"], value) is not None+++def _value_problem(*, name: str, spec: dict[str, Any], value: object, properties: dict[str, Any]) -> str | None:+ kind = spec["type"]+ is_valid: bool+ if kind in {"integer", "number"}:+ is_number = not isinstance(value, bool) and isinstance(value, int | float)+ is_valid = is_number and spec["min"] <= cast("float", value) <= spec["max"]+ if kind == "integer":+ is_valid = is_valid and _json_type(value) == "integer"+ elif kind == "boolean":+ is_valid = isinstance(value, bool)+ elif kind == "enum":+ allowed: list[str] | None = spec.get("values")+ if allowed is None:+ selector = properties.get(spec["values_by"]["property"])+ allowed = spec["values_by"]["values"].get(selector) if isinstance(selector, str) else None+ is_valid = allowed is not None and isinstance(value, str) and value in allowed+ elif kind == "pattern":+ is_valid = isinstance(value, str) and _pattern_matches(pattern_name=spec["pattern"], value=value)+ else:+ return f"{name} is a {kind}, which no tool sends"+ if not is_valid:+ return f"invalid value for {name}: {value!r}"+ fixed_when: dict[str, Any] | None = spec.get("fixed_when")+ if fixed_when is not None and _condition_holds(condition=fixed_when, properties=properties) and value != fixed_when["value"]:+ return f"{name} must be {fixed_when['value']!r} when {fixed_when['property']} is {fixed_when['equals']!r}"+ return None+++def declared_properties(*, event_name: str) -> dict[str, dict[str, Any]]:+ """Every property an event may carry: the common ones when it carries them, then its own."""+ schema = load_usage_stats_schema()+ event_spec = schema["events"][event_name]+ declared: dict[str, dict[str, Any]] = {}+ if event_spec["common"]:+ declared.update(schema["common_properties"])+ declared.update(event_spec["properties"])+ return declared+++def envelope_problems(*, envelope: dict[str, Any]) -> list[str]:+ """What PostHog's gate would refuse in one posted envelope; empty when it keeps the event."""+ schema = load_usage_stats_schema()+ event_name = envelope.get("event")+ if not isinstance(event_name, str) or event_name not in schema["events"]:+ return [f"undeclared event {event_name!r}"]+ problems: list[str] = []+ allowed_keys = {"event", "properties", *schema["envelope"]["fields"]}+ for key in envelope:+ if key not in allowed_keys:+ problems.append(f"the envelope carries {key}, which no sender sets")+ for field_name, field in schema["envelope"]["fields"].items():+ field_value = envelope.get(field_name)+ if not isinstance(field_value, str) or not _pattern_matches(pattern_name=field["pattern"], value=field_value):+ problems.append(f"envelope field {field_name} is not a {field['pattern']}: {field_value!r}")+ raw_properties = envelope.get("properties")+ if not isinstance(raw_properties, dict):+ return [*problems, "properties are not an object"]+ properties = cast("dict[str, Any]", raw_properties)+ person_profile_property: str = schema["person_profile_property"]+ if properties.get(person_profile_property) is not False:+ problems.append(f"{person_profile_property} must be false")+ event_spec = schema["events"][event_name]+ if properties.get("tool") not in event_spec["tools"]:+ problems.append(f"tool {properties.get('tool')!r} may not send {event_name}")+ declared = declared_properties(event_name=event_name)+ for name, value in properties.items():+ if name == person_profile_property:+ continue+ spec = declared.get(name)+ if spec is None:+ problems.append(f"undeclared property {name}")+ continue+ if value is None:+ problems.append(f"{name} is null; an optional property is left out")+ continue+ when: dict[str, Any] | None = spec.get("when")+ if when is not None and not _condition_holds(condition=when, properties=properties):+ problems.append(f"{name} is not allowed here")+ continue+ value_problem = _value_problem(name=name, spec=spec, value=value, properties=properties)+ if value_problem is not None:+ problems.append(value_problem)+ for name, spec in declared.items():+ if not spec["required"] or properties.get(name) is not None:+ continue+ when = spec.get("when")+ if when is None or _condition_holds(condition=when, properties=properties):+ problems.append(f"missing required {name}")+ return problems+++def assert_envelope_is_valid(envelope: dict[str, Any]) -> None:+ problems = envelope_problems(envelope=envelope)+ assert not problems, f"The gate would drop this {envelope.get('event')}: {problems}\n{json.dumps(envelope, indent=2)}"diff --git a/tests/unit/pipelex/cli/agent_cli/commands/test_plxt_passthrough.py b/tests/unit/pipelex/cli/agent_cli/commands/test_plxt_passthrough.pyindex 1a076bb3c..298ad6947 100644--- a/tests/unit/pipelex/cli/agent_cli/commands/test_plxt_passthrough.py+++ b/tests/unit/pipelex/cli/agent_cli/commands/test_plxt_passthrough.py@@ -12,6 +12,7 @@ import typerfrom pipelex.cli.agent_cli.commands.fmt_cmd import fmt_cmdfrom pipelex.cli.agent_cli.commands.lint_cmd import lint_cmdfrom pipelex.cli.agent_cli.commands.plxt_passthrough import run_plxt+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsEnvVarif TYPE_CHECKING:from pytest_mock import MockerFixture@@ -79,7 +80,10 @@ class TestPlxtPassthrough:self,mocker: MockerFixture,) -> None:- """run_plxt passes [plxt_path, subcommand, file_path] to subprocess.run."""+ """run_plxt passes [plxt_path, subcommand, file_path] to subprocess.run, with plxt's own usage statistics off.++ The child's environment is compared by the one variable it adds, never printed whole: it holds the caller's.+ """mocker.patch("shutil.which", return_value="/usr/bin/plxt")mock_run = mocker.patch("subprocess.run",@@ -89,10 +93,12 @@ class TestPlxtPassthrough:with pytest.raises(typer.Exit):run_plxt("fmt", file_path="/path/to/file.mthds")- mock_run.assert_called_once_with(- ["/usr/bin/plxt", "fmt", "/path/to/file.mthds"],- check=False,- )+ assert mock_run.call_count == 1+ call_args, call_kwargs = mock_run.call_args+ assert call_args == (["/usr/bin/plxt", "fmt", "/path/to/file.mthds"],)+ assert sorted(call_kwargs) == ["check", "env"]+ assert call_kwargs["check"] is False+ assert call_kwargs["env"].get(UsageStatsEnvVar.TOOL_SWITCH) == "0"def test_fmt_cmd_delegates_to_plxt_with_fmt_subcommand(self,diff --git a/tests/unit/pipelex/cli/commands/build/test_structures_alias.py b/tests/unit/pipelex/cli/commands/build/test_structures_alias.pyindex 5ad81daec..1e6b8cb4a 100644--- a/tests/unit/pipelex/cli/commands/build/test_structures_alias.py+++ b/tests/unit/pipelex/cli/commands/build/test_structures_alias.py@@ -14,22 +14,33 @@ import typerfrom pipelex.cli.commands.build.structures_cmd import build_structures_commandfrom pipelex.codegen.emitters.target import CodegenTarget+from pipelex.pipeline.usage_stats_reporting import CodeGenerationReport+from pipelex.system.usage_stats.usage_stats_contract import UsageStatsCodegenArtifactif TYPE_CHECKING:from pathlib import Path- from pytest_mock import MockerFixture+ from pytest_mock import MockerFixture, MockTypeMODULE = "pipelex.cli.commands.build.structures_cmd"+def _assert_reported_as_structures(*, delegate: MockType) -> None:+ """The alias reports its generation as the `structures` artifact, which `codegen types` does not."""+ generation_report = delegate.call_args.kwargs["generation_report"]+ assert isinstance(generation_report, CodeGenerationReport)+ assert generation_report.artifact == UsageStatsCodegenArtifact.STRUCTURES++class TestBuildStructuresAlias:def test_directory_target_delegates_with_default_output(self, mocker: MockerFixture, tmp_path: Path) -> None:- delegate = mocker.patch(f"{MODULE}.codegen_types_cmd")+ delegate = mocker.patch(f"{MODULE}.generate_types")build_structures_command(target=str(tmp_path), output_dir=None, library_dir=None)+ _assert_reported_as_structures(delegate=delegate)delegate.assert_called_once_with(+ generation_report=mocker.ANY,target=CodegenTarget.PYTHON_STRUCTURES,paths=[tmp_path],output_dir=str(tmp_path / "structures"),@@ -37,13 +48,15 @@ class TestBuildStructuresAlias:)def test_file_target_resolves_to_parent_directory(self, mocker: MockerFixture, tmp_path: Path) -> None:- delegate = mocker.patch(f"{MODULE}.codegen_types_cmd")+ delegate = mocker.patch(f"{MODULE}.generate_types")bundle_file = tmp_path / "demo.mthds"bundle_file.write_text('domain = "demo"\n', encoding="utf-8")build_structures_command(target=str(bundle_file), output_dir=None, library_dir=None)+ _assert_reported_as_structures(delegate=delegate)delegate.assert_called_once_with(+ generation_report=mocker.ANY,target=CodegenTarget.PYTHON_STRUCTURES,paths=[tmp_path],output_dir=str(tmp_path / "structures"),@@ -51,13 +64,15 @@ class TestBuildStructuresAlias:)def test_explicit_output_dir_and_library_dirs_pass_through(self, mocker: MockerFixture, tmp_path: Path) -> None:- delegate = mocker.patch(f"{MODULE}.codegen_types_cmd")+ delegate = mocker.patch(f"{MODULE}.generate_types")shared_dir = tmp_path / "shared"shared_dir.mkdir()build_structures_command(target=str(tmp_path), output_dir=str(tmp_path / "generated"), library_dir=[str(shared_dir)])+ _assert_reported_as_structures(delegate=delegate)delegate.assert_called_once_with(+ generation_report=mocker.ANY,target=CodegenTarget.PYTHON_STRUCTURES,paths=[tmp_path],output_dir=str(tmp_path / "generated"),@@ -65,7 +80,7 @@ class TestBuildStructuresAlias:)def test_missing_target_is_no_verdict_exit_2(self, mocker: MockerFixture, tmp_path: Path) -> None:- delegate = mocker.patch(f"{MODULE}.codegen_types_cmd")+ delegate = mocker.patch(f"{MODULE}.generate_types")with pytest.raises(typer.Exit) as exc_info:build_structures_command(target=str(tmp_path / "does_not_exist"), output_dir=None, library_dir=None)diff --git a/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py b/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.pyindex 9270946b9..feeb48d45 100644--- a/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py+++ b/tests/unit/pipelex/cli/test_cli_entrypoint_smoke.py@@ -21,6 +21,7 @@ Two things make this test catch the bug where the existing CLI tests did not:from __future__ import annotations+import osimport shutilimport subprocess # ruff: ignore[suspicious-subprocess-import] -- invokes the real pipelex binary on purposeimport sys@@ -62,7 +63,7 @@ class TestCliEntrypointStarts:["login", "--help"],],)- def test_subcommand_starts_and_exits_zero(self, args: list[str]) -> None:+ def test_subcommand_starts_and_exits_zero(self, args: list[str], tmp_path: Path) -> None:bin_path = PIPELEX_BINif bin_path is None:pytest.fail("`pipelex` console script not found next to the interpreter or on PATH; install the package before running this test.")@@ -72,6 +73,8 @@ class TestCliEntrypointStarts:text=True,timeout=120,check=False,+ # A home of its own: every CLI command reads the usage statistics' shared state there, which stays this test's.+ env={**os.environ, "HOME": str(tmp_path)},)combined = f"{result.stdout}\n{result.stderr}"assert result.returncode == 0, (diff --git a/tests/unit/pipelex/cli/test_usage_stats_cli.py b/tests/unit/pipelex/cli/test_usage_stats_cli.pynew file mode 100644index 000000000..272393ff1--- /dev/null+++ b/tests/unit/pipelex/cli/test_usage_stats_cli.py@@ -0,0 +1,384 @@+import json+from collections.abc import Callable+from pathlib import Path+from typing import Any++import click+import pytest+import typer+from pytest_mock import MockerFixture++from pipelex.cli._cli import app as pipelex_app+from pipelex.cli.agent_cli._agent_cli import app as agent_app+from pipelex.cli.commands.init.init_usage_stats import note_init_backends, note_init_execution+from pipelex.cli.usage_stats_cli import resolve_invocation, run_with_usage_stats+from pipelex.hosted.run_config import RunExecution+from pipelex.system.runtime import IntegrationMode+from pipelex.system.usage_stats.usage_stats_client import (+ UsageStatsClient,+ configure_usage_stats_for_cli,+ configure_usage_stats_for_runtime,+ note_command_error_domain,+ note_command_hosted,+ show_usage_stats_notice,+)+from pipelex.system.usage_stats.usage_stats_contract import (+ NOTICE_VERSION,+ UsageStatsEnvVar,+ UsageStatsErrorDomain,+ UsageStatsProvider,+ UsageStatsSurface,+ render_notice,+)+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState+from tests.helpers.usage_stats_process import make_project_dir, usage_stats_switched_on+from tests.helpers.usage_stats_schema import assert_envelope_is_valid++_ROOTS: dict[UsageStatsSurface, click.Command] = {+ UsageStatsSurface.PIPELEX: typer.main.get_command(pipelex_app),+ UsageStatsSurface.PIPELEX_AGENT: typer.main.get_command(agent_app),+}+++def _root_of(surface: UsageStatsSurface) -> click.Group:+ root = _ROOTS[surface]+ assert isinstance(root, click.Group)+ return root+++def _run(*, surface: UsageStatsSurface, args: list[str], invoke: Callable[[], Any]) -> None:+ run_with_usage_stats(root=_root_of(surface), args=args, surface=surface, strip_flags=frozenset({"--no-logo"}), invoke=invoke)+++def _debug_events(stderr: str) -> list[dict[str, Any]]:+ return [json.loads(line)["usage_stats_debug"] for line in stderr.splitlines() if line.startswith('{"usage_stats_debug"')]+++def _succeed() -> None:+ return None+++def _fail_with_input_error() -> None:+ note_command_error_domain(error_domain=UsageStatsErrorDomain.INPUT)+ raise SystemExit(1)+++def _interrupted() -> None:+ raise KeyboardInterrupt+++def _cancelled_exit() -> None:+ raise SystemExit(130)+++def _hosted_run() -> None:+ note_command_hosted()+++def _local_init() -> None:+ note_init_backends(backend_keys=["anthropic", "my_company_gateway"])+ note_init_execution(execution=RunExecution.LOCAL)+++def _hosted_init() -> None:+ note_init_execution(execution=RunExecution.HOSTED)+++def _ask() -> None:+ typer.prompt("Name")+++def _confirm() -> None:+ typer.confirm("Proceed?", abort=True)+++def _prompting_app() -> click.Command:+ prompting = typer.Typer()+ prompting.command(name="ask")(_ask)+ prompting.command(name="confirm")(_confirm)+ return typer.main.get_command(prompting)+++def _off_by_do_not_track(*, state: UsageStatsState, monkeypatch: pytest.MonkeyPatch) -> None: # ruff: ignore[unused-function-argument]+ monkeypatch.setenv(UsageStatsEnvVar.DO_NOT_TRACK, "1")+++def _off_by_tool_switch(*, state: UsageStatsState, monkeypatch: pytest.MonkeyPatch) -> None: # ruff: ignore[unused-function-argument]+ monkeypatch.setenv(UsageStatsEnvVar.TOOL_SWITCH, "off")+++def _off_by_disable(*, state: UsageStatsState, monkeypatch: pytest.MonkeyPatch) -> None: # ruff: ignore[unused-function-argument]+ state.disable()+++def _replaces_its_client_then_is_cancelled() -> None:+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTEST)+ raise SystemExit(130)+++class TestUsageStatsCli:+ @pytest.mark.parametrize(+ ("surface", "args", "command", "is_machine_read"),+ [+ (UsageStatsSurface.PIPELEX, ["validate", "bundle", "x.mthds"], "validate bundle", False),+ (UsageStatsSurface.PIPELEX, ["validate", "--all"], "validate pipe", False),+ (UsageStatsSurface.PIPELEX, ["validate", "my_pipe", "--library-dir", "x"], "validate pipe", False),+ (UsageStatsSurface.PIPELEX, ["codegen", "types", "x.mthds", "--target", "ts-zod"], "codegen types", False),+ (UsageStatsSurface.PIPELEX, ["run", "x.mthds"], "other", False),+ (UsageStatsSurface.PIPELEX, ["no-such-command"], "other", False),+ (UsageStatsSurface.PIPELEX, ["--help"], None, False),+ (UsageStatsSurface.PIPELEX, ["--version"], None, False),+ (UsageStatsSurface.PIPELEX, ["-V"], None, False),+ (UsageStatsSurface.PIPELEX, [], None, False),+ (UsageStatsSurface.PIPELEX, ["show"], None, False),+ (UsageStatsSurface.PIPELEX, ["run", "bundle", "x.mthds", "--help"], None, False),+ (UsageStatsSurface.PIPELEX, ["usage-stats", "status"], None, False),+ (UsageStatsSurface.PIPELEX_AGENT, ["validate", "bundle", "x.mthds"], "validate bundle", False),+ (UsageStatsSurface.PIPELEX_AGENT, ["validate", "bundle", "x.mthds", "--format", "json"], "validate bundle", True),+ (UsageStatsSurface.PIPELEX_AGENT, ["validate", "bundle", "x.mthds", "--format=json"], "validate bundle", True),+ (UsageStatsSurface.PIPELEX_AGENT, ["validate", "bundle", "x.mthds", "--error-format", "json"], "validate bundle", True),+ (+ UsageStatsSurface.PIPELEX_AGENT,+ ["validate", "bundle", "x.mthds", "--format", "json", "--error-format", "markdown"],+ "validate bundle",+ False,+ ),+ (UsageStatsSurface.PIPELEX_AGENT, ["inputs", "pipe", "my_pipe"], "inputs pipe", True),+ (UsageStatsSurface.PIPELEX_AGENT, ["models"], "models", False),+ ],+ )+ def test_invocation_is_resolved_without_reading_arguments(+ self, surface: UsageStatsSurface, args: list[str], command: str | None, is_machine_read: bool+ ) -> None:+ """The command is its path in the tree, never an argument; a JSON error stream is a machine contract."""+ invocation = resolve_invocation(root=_root_of(surface), args=args, surface=surface)+ assert invocation.command == command+ assert invocation.is_machine_read is is_machine_read+ assert invocation.is_statistics_group is (args[:1] == ["usage-stats"])++ def test_notice_once_then_command_completed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:+ """The notice goes to stderr before the command works, once per machine; the command's event follows."""+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True) as state:+ _run(surface=UsageStatsSurface.PIPELEX, args=["--no-logo", "validate", "bundle", "secret_name.mthds"], invoke=_succeed)+ first = capsys.readouterr()+ assert first.out == ""+ assert first.err.splitlines()[0] == render_notice()+ assert state.notice_marker(version=1).exists()+ events = _debug_events(first.err)+ assert len(events) == 1+ assert_envelope_is_valid(events[0])+ assert events[0]["properties"]["command"] == "validate bundle"+ assert events[0]["properties"]["outcome"] == "succeeded"+ assert events[0]["properties"]["notice_version"] == 1+ assert "secret_name" not in first.err+ _run(surface=UsageStatsSurface.PIPELEX, args=["validate", "bundle", "x.mthds"], invoke=_succeed)+ second = capsys.readouterr()+ assert render_notice() not in second.err+ assert len(_debug_events(second.err)) == 1++ def test_machine_read_stderr_gets_no_notice(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:+ """Under JSON errors the notice is neither printed nor recorded, and the command's statistics are still sent."""+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True) as state:+ with pytest.raises(SystemExit):+ _run(+ surface=UsageStatsSurface.PIPELEX_AGENT, args=["validate", "bundle", "x.mthds", "--format", "json"], invoke=_fail_with_input_error+ )+ captured = capsys.readouterr()+ assert render_notice() not in captured.err+ assert not state.notice_marker(version=1).exists()+ events = _debug_events(captured.err)+ assert len(events) == 1+ assert_envelope_is_valid(events[0])+ assert events[0]["properties"]["surface"] == "pipelex-agent"+ assert events[0]["properties"]["outcome"] == "failed"+ assert events[0]["properties"]["error_domain"] == "input"++ @pytest.mark.parametrize(+ ("invoke", "expected"),+ [+ (_succeed, {"outcome": "succeeded"}),+ (_fail_with_input_error, {"outcome": "failed", "error_domain": "input"}),+ (_interrupted, {"outcome": "cancelled"}),+ (_cancelled_exit, {"outcome": "cancelled"}),+ (_hosted_run, {"outcome": "succeeded", "hosted": True}),+ ],+ )+ def test_outcome_of_the_command(+ self,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ invoke: Callable[[], Any],+ expected: dict[str, Any],+ ) -> None:+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ try:+ _run(surface=UsageStatsSurface.PIPELEX, args=["run", "bundle", "x.mthds"], invoke=invoke)+ except (SystemExit, KeyboardInterrupt):+ pass+ events = _debug_events(capsys.readouterr().err)+ assert len(events) == 1+ assert_envelope_is_valid(events[0])+ properties = events[0]["properties"]+ for name, value in expected.items():+ assert properties[name] == value+ assert ("error_domain" in properties) is ("error_domain" in expected)+ assert ("hosted" in properties) is ("hosted" in expected)++ @pytest.mark.parametrize(+ ("command", "prompt_answer", "outcome"),+ [+ ("ask", KeyboardInterrupt(), "cancelled"),+ ("confirm", "n", "cancelled"),+ ("ask", EOFError(), "failed"),+ ],+ )+ def test_a_command_stopped_at_a_prompt_is_cancelled(+ self,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ mocker: MockerFixture,+ command: str,+ prompt_answer: BaseException | str,+ outcome: str,+ ) -> None:+ """Ctrl-C at a prompt, or a confirmation declined, reaches Typer as Click's `Abort` and exits 1, yet the person stopped it."""+ mocker.patch("click.termui.visible_prompt_func", side_effect=[prompt_answer])+ prompting = _prompting_app()+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ with pytest.raises(SystemExit) as exit_info:+ _run(surface=UsageStatsSurface.PIPELEX, args=["run", "bundle", "x.mthds"], invoke=lambda: prompting.main(args=[command]))+ events = _debug_events(capsys.readouterr().err)+ assert exit_info.value.code == 1+ assert len(events) == 1+ assert events[0]["properties"]["outcome"] == outcome++ @pytest.mark.parametrize(+ ("surface", "args", "invoke", "expected"),+ [+ (UsageStatsSurface.PIPELEX, ["init"], _local_init, {"init_execution": "local", "anthropic": True, "custom": True, "openai": False}),+ (UsageStatsSurface.PIPELEX_AGENT, ["init", "--config", "{}"], _hosted_init, {"init_execution": "hosted"}),+ (UsageStatsSurface.PIPELEX, ["init", "routing"], _succeed, {}),+ (UsageStatsSurface.PIPELEX, ["run", "bundle", "x.mthds"], _local_init, {}),+ ],+ )+ def test_init_reports_what_it_set(+ self,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ surface: UsageStatsSurface,+ args: list[str],+ invoke: Callable[[], Any],+ expected: dict[str, Any],+ ) -> None:+ """`init` says where it set runs to execute and, once it set backends, a boolean for every provider; no other command carries them."""+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ _run(surface=surface, args=args, invoke=invoke)+ events = _debug_events(capsys.readouterr().err)+ assert len(events) == 1+ assert_envelope_is_valid(events[0])+ properties = events[0]["properties"]+ init_properties = {name: value for name, value in properties.items() if name.startswith("init_")}+ assert init_properties.get("init_execution") == expected.get("init_execution")+ backends = {name.removeprefix("init_backend_"): value for name, value in init_properties.items() if name.startswith("init_backend_")}+ expected_backends = {name: value for name, value in expected.items() if name != "init_execution"}+ if expected_backends:+ assert set(backends) == {provider.value for provider in UsageStatsProvider}+ assert {name for name, value in backends.items() if value} == {name for name, value in expected_backends.items() if value}+ else:+ assert backends == {}++ @pytest.mark.parametrize(+ ("args", "is_notice_recorded_before", "switch_off"),+ [+ (["--no-logo", "init"], False, None),+ (["--no-logo", "init"], True, None),+ (["--no-logo", "init", "routing"], True, None),+ (["--no-logo", "init"], False, _off_by_do_not_track),+ (["--no-logo", "init"], True, _off_by_tool_switch),+ (["--no-logo", "init"], True, _off_by_disable),+ ],+ )+ def test_init_carries_the_notice_once(+ self,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ mocker: MockerFixture,+ args: list[str],+ is_notice_recorded_before: bool,+ switch_off: Callable[..., None] | None,+ ) -> None:+ """`pipelex init` shows the notice's text on stderr whenever statistics are on, once, the first run's notice standing for it."""+ init_cmd = mocker.patch("pipelex.cli._cli.init_cmd")+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True) as state:+ if is_notice_recorded_before:+ state.record_notice(version=NOTICE_VERSION)+ if switch_off is not None:+ switch_off(state=state, monkeypatch=monkeypatch)+ _root_of(UsageStatsSurface.PIPELEX).main(args=args, standalone_mode=False)+ captured = capsys.readouterr()+ init_cmd.assert_called_once()+ assert render_notice() not in captured.out+ assert captured.err.count(render_notice()) == (0 if switch_off else 1)+ assert state.notice_marker(version=NOTICE_VERSION).exists() is (is_notice_recorded_before or switch_off is None)++ def test_the_notice_init_shows_records_nothing(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:+ """Showing the text writes no marker, and a command asked twice shows it once."""+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path)) as state:+ assert configure_usage_stats_for_cli(surface=UsageStatsSurface.PIPELEX_AGENT, is_machine_read=True) is not None+ show_usage_stats_notice()+ show_usage_stats_notice()+ captured = capsys.readouterr()+ assert captured.out == ""+ assert captured.err.count(render_notice()) == 1+ assert not state.notice_marker(version=NOTICE_VERSION).exists()++ @pytest.mark.parametrize("args", [["--help"], ["usage-stats", "status"], ["show"]])+ def test_uncounted_invocations_touch_nothing(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], args: list[str]+ ) -> None:+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True) as state:+ _run(surface=UsageStatsSurface.PIPELEX, args=args, invoke=_succeed)+ captured = capsys.readouterr()+ assert captured.err == ""+ assert not state.state_dir.exists()++ def test_statistics_off_print_and_write_nothing(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]+ ) -> None:+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True) as state:+ monkeypatch.setenv(UsageStatsEnvVar.DO_NOT_TRACK, "1")+ _run(surface=UsageStatsSurface.PIPELEX, args=["validate", "bundle", "x.mthds"], invoke=_succeed)+ captured = capsys.readouterr()+ assert captured.err == ""+ assert not state.state_dir.exists()++ def test_statistics_never_replace_the_commands_exit(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], mocker: MockerFixture+ ) -> None:+ """A delivery that raises leaves the command's own exit as it was, and the other client of the process still delivers."""+ finish_command = mocker.patch.object(UsageStatsClient, "finish_command", autospec=True, side_effect=[OSError("unlink refused"), None])+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ with pytest.raises(SystemExit) as exit_info:+ _run(surface=UsageStatsSurface.PIPELEX, args=["run", "bundle", "x.mthds"], invoke=_replaces_its_client_then_is_cancelled)+ capsys.readouterr()+ assert exit_info.value.code == 130+ assert finish_command.call_count == 2++ def test_a_command_runs_without_statistics_when_they_cannot_be_read(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], mocker: MockerFixture+ ) -> None:+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ mocker.patch.object(UsageStatsState, "make_for_current_home", side_effect=RuntimeError("Could not determine home directory."))+ run_with_usage_stats(+ root=_root_of(UsageStatsSurface.PIPELEX),+ args=["validate", "bundle", "x.mthds"],+ surface=UsageStatsSurface.PIPELEX,+ strip_flags=frozenset(),+ invoke=_succeed,+ )+ captured = capsys.readouterr()+ assert captured.err == ""diff --git a/tests/unit/pipelex/cli/test_usage_stats_cmd.py b/tests/unit/pipelex/cli/test_usage_stats_cmd.pynew file mode 100644index 000000000..a6d72c3be--- /dev/null+++ b/tests/unit/pipelex/cli/test_usage_stats_cmd.py@@ -0,0 +1,74 @@+import json+from pathlib import Path++import pytest+from typer.testing import CliRunner++from pipelex.cli.commands.usage_stats_cmd import usage_stats_app+from pipelex.system.usage_stats.usage_stats_contract import TOOL_NAME, UsageStatsEnvVar+from pipelex.system.usage_stats.usage_stats_core import SpoolContent, is_installation_id, pending_files, write_spool_file+from tests.helpers.usage_stats_process import make_project_dir, usage_stats_switched_on++_STATUS_FIELDS = {"tool", "enabled", "off_by", "installation_id", "notice_version", "internal", "ci", "container", "state_dir"}+++class TestUsageStatsCmd:+ def test_status_is_read_only_and_prints_the_status_object(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path)) as state:+ state.legacy_config.parent.mkdir(parents=True)+ state.legacy_config.write_text("DISABLE_TELEMETRY=1\n", encoding="utf-8")+ result = CliRunner().invoke(usage_stats_app, ["status", "--format", "json"])+ assert result.exit_code == 0, result.output+ status = json.loads(result.stdout)+ assert set(status) == _STATUS_FIELDS+ assert status["tool"] == "pipelex"+ assert status["enabled"] is False+ assert status["off_by"] == ["legacy_mthds_choice"]+ assert status["installation_id"] is None+ assert status["notice_version"] == 0+ assert status["state_dir"] == str(state.state_dir)+ assert not state.state_dir.exists()+ text = CliRunner().invoke(usage_stats_app, ["status"])+ assert text.exit_code == 0+ assert "legacy_mthds_choice" in text.stdout+ assert not state.state_dir.exists()++ def test_disable_enable_and_reset_id(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """`disable` empties every spool, `enable` names the switches still off, `reset-id` prints a new identifier."""+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path)) as state:+ first_id = state.obtain_installation_id().value+ write_spool_file(spool_dir=state.spool_dir(tool="mthds"), content=SpoolContent(endpoint=None, events=[]))+ runner = CliRunner()++ assert runner.invoke(usage_stats_app, ["disable"]).exit_code == 0+ assert state.is_disabled()+ assert pending_files(spool_dir=state.spool_dir(tool="mthds")) == []+ status = json.loads(runner.invoke(usage_stats_app, ["status", "--format", "json"]).stdout)+ assert status["off_by"] == ["machine_choice"]+ assert status["installation_id"] == first_id++ monkeypatch.setenv(UsageStatsEnvVar.DO_NOT_TRACK, "1")+ enabled = runner.invoke(usage_stats_app, ["enable"])+ assert enabled.exit_code == 0+ assert not state.is_disabled()+ assert "do_not_track" in enabled.stdout+ monkeypatch.delenv(UsageStatsEnvVar.DO_NOT_TRACK)+ assert json.loads(runner.invoke(usage_stats_app, ["status", "--format", "json"]).stdout)["enabled"] is True++ write_spool_file(spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[]))+ reset = runner.invoke(usage_stats_app, ["reset-id"])+ assert reset.exit_code == 0+ new_id = reset.stdout.strip()+ assert is_installation_id(value=new_id)+ assert new_id != first_id+ assert state.read_installation_id() == new_id+ assert pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME)) == []++ def test_disable_reports_a_state_it_cannot_write(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ home = tmp_path / "home"+ home.mkdir()+ (home / ".mthds").write_text("not a directory", encoding="utf-8")+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=home, cwd=make_project_dir(parent=tmp_path)):+ result = CliRunner().invoke(usage_stats_app, ["disable"])+ assert result.exit_code == 1+ assert "could not be turned off" in result.stderrdiff --git a/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py b/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.pynew file mode 100644index 000000000..b1320d2cf--- /dev/null+++ b/tests/unit/pipelex/pipe_run/test_pipe_run_usage_stats.py@@ -0,0 +1,208 @@+import asyncio+import json+from pathlib import Path+from typing import TYPE_CHECKING, Any++import pytest+from pytest_mock import MockerFixture, MockType++from pipelex.base_exceptions import PipelexError+from pipelex.pipe_run import pipe_run as pipe_run_module+from pipelex.pipe_run.pipe_run import PipeRun+from pipelex.pipe_run.tracing_assembly import RunTraceEvents+from pipelex.system.data_inclusion_config import DataInclusionConfig+from pipelex.system.job_metadata import JobMetadata, RunMetadata+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.system.runtime import IntegrationMode+from pipelex.system.trace_context import TraceContext+from pipelex.system.usage_stats.usage_stats_client import configure_usage_stats_for_runtime+from pipelex.system.usage_stats.usage_stats_runs import current_run_tally, usage_stats_error_domain+from tests.helpers.usage_stats_process import make_project_dir, usage_stats_switched_on+from tests.helpers.usage_stats_run_events import USAGE_STATS_RUN_ID, llm_usage_event+from tests.helpers.usage_stats_schema import assert_envelope_is_valid++if TYPE_CHECKING:+ from pipelex.tracing.trace_events import TraceEvent++_MODEL = "claude-4-sonnet"+_FAILURE = PipelexError("secret failure after the first call")+++def _job(*, mocker: MockerFixture, is_traced: bool) -> Any:+ job = mocker.MagicMock()+ trace_context = TraceContext(+ graph_id=USAGE_STATS_RUN_ID,+ data_inclusion=DataInclusionConfig(pipe_and_concept_registry=False, stuff_json_content=False, error_stack_traces=False),+ emit_graph_events=False,+ emit_usage_events=True,+ )+ job.job_metadata = JobMetadata(+ run_metadata=RunMetadata(user_id="pytest", pipeline_run_id=USAGE_STATS_RUN_ID, storage_scope="test/scope", read_scope=None),+ trace_context=trace_context if is_traced else None,+ )+ job.pipe_run_params.run_mode = PipeRunMode.LIVE+ job.working_memory = None+ job.pipe.is_controller = False+ job.pipe.code = "secret_main_pipe"+ return job+++def _note_one_call() -> None:+ tally = current_run_tally()+ assert tally is not None+ tally.note_call(model_handle=_MODEL, backend_name="anthropic", failed=False, failure_category=None, images_generated=0, pages_extracted=0)+++def _call_then_fail(pipe_job: Any) -> Any:+ del pipe_job+ _note_one_call()+ raise _FAILURE+++class _RunOutput:+ graph_spec = None++ def set_job_metadata(self, *, job_metadata: JobMetadata) -> None:+ del job_metadata+++def _call_then_succeed(pipe_job: Any) -> Any:+ del pipe_job+ _note_one_call()+ return _RunOutput()+++def _succeed_untallied(pipe_job: Any) -> Any:+ del pipe_job+ assert current_run_tally() is None+ return _RunOutput()+++def _cancelled(pipe_job: Any) -> Any:+ del pipe_job+ _note_one_call()+ raise asyncio.CancelledError+++def _debug_events(stderr: str) -> list[dict[str, Any]]:+ return [json.loads(line)["usage_stats_debug"] for line in stderr.splitlines() if line.startswith('{"usage_stats_debug"')]+++def run_seams(mocker: MockerFixture) -> MockType:+ """The run's surroundings, which a unit test does not boot: no graph tracer, an empty library, the run's usage events.++ Returns the stand-in for reading the run's trace events.+ """+ events: list[TraceEvent] = [llm_usage_event(model_name=_MODEL, backend_name="anthropic", input_tokens=120, output_tokens=30)]+ mocker.patch.object(pipe_run_module.GraphTracerManager, "get_instance", return_value=None)+ read_events = mocker.patch.object(pipe_run_module, "read_run_trace_events", return_value=RunTraceEvents(events=events, read_error=None))+ mocker.patch.object(pipe_run_module, "assemble_tracing_on_output")+ mocker.patch.object(pipe_run_module, "get_own_pipes", return_value=[])+ concept_library = mocker.MagicMock()+ concept_library.list_own_concepts.return_value = []+ mocker.patch.object(pipe_run_module, "get_concept_library", return_value=concept_library)+ return read_events++[email protected](loop_scope="class")+class TestPipeRunUsageStats:+ @pytest.mark.parametrize(+ ("router_run", "outcome", "raised"),+ [+ (_call_then_fail, "failed", PipelexError),+ (_cancelled, "cancelled", asyncio.CancelledError),+ (_call_then_succeed, "succeeded", None),+ ],+ )+ @pytest.mark.usefixtures("run_seams")+ async def test_every_run_end_reports_with_its_tokens(+ self,+ tmp_path: Path,+ mocker: MockerFixture,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ router_run: Any,+ outcome: str,+ raised: type[BaseException] | None,+ ) -> None:+ """A run that raises after its first inference call still sends its `run_completed`, with the tokens it used."""+ router = mocker.AsyncMock()+ router.run = mocker.AsyncMock(side_effect=router_run)+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTHON)+ pipe_run = PipeRun(pipe_router=router)+ if raised is None:+ await pipe_run.run(pipe_job=_job(mocker=mocker, is_traced=True))+ else:+ with pytest.raises(raised):+ await pipe_run.run(pipe_job=_job(mocker=mocker, is_traced=True))+ stderr = capsys.readouterr().err+ run_event, model_event = _debug_events(stderr)+ for envelope in (run_event, model_event):+ assert_envelope_is_valid(envelope)+ assert "secret" not in stderr+ run = run_event["properties"]+ assert run_event["event"] == "run_completed"+ assert run["outcome"] == outcome+ assert run["usage_known"] is True+ assert (run["inference_calls"], run["input_tokens"], run["output_tokens"], run["models_used"]) == (1, 120, 30, 1)+ assert run["surface"] == "python"+ if outcome == "failed":+ assert run["error_domain"] == usage_stats_error_domain(error=_FAILURE)+ assert model_event["event"] == "model_used"+ assert (model_event["properties"]["provider"], model_event["properties"]["model"]) == ("anthropic", _MODEL)++ async def test_untraced_run_reports_usage_unknown(+ self, tmp_path: Path, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], run_seams: MockType+ ) -> None:+ """Statistics never turn tracing on: an untraced run says its usage is unknown and sends no `model_used`."""+ router = mocker.AsyncMock()+ router.run = mocker.AsyncMock(side_effect=_call_then_fail)+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTHON)+ with pytest.raises(PipelexError):+ await PipeRun(pipe_router=router).run(pipe_job=_job(mocker=mocker, is_traced=False))+ events = _debug_events(capsys.readouterr().err)+ assert [event["event"] for event in events] == ["run_completed"]+ assert_envelope_is_valid(events[0])+ assert events[0]["properties"]["usage_known"] is False+ run_seams.assert_not_called()++ @pytest.mark.usefixtures("run_seams")+ async def test_validation_sweeps_and_statistics_off_report_nothing(+ self, tmp_path: Path, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]+ ) -> None:+ router = mocker.AsyncMock()+ router.run = mocker.AsyncMock(side_effect=_succeed_untallied)+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTHON)+ await PipeRun(pipe_router=router, reports_usage_stats=False).run(pipe_job=_job(mocker=mocker, is_traced=True))+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTEST)+ await PipeRun(pipe_router=router).run(pipe_job=_job(mocker=mocker, is_traced=True))+ assert _debug_events(capsys.readouterr().err) == []++ @pytest.mark.parametrize(("router_run", "raised"), [(_call_then_succeed, None), (_call_then_fail, PipelexError)])+ @pytest.mark.usefixtures("run_seams")+ async def test_a_statistic_never_changes_the_runs_result(+ self,+ tmp_path: Path,+ mocker: MockerFixture,+ monkeypatch: pytest.MonkeyPatch,+ capsys: pytest.CaptureFixture[str],+ router_run: Any,+ raised: type[BaseException] | None,+ ) -> None:+ """A statistic that cannot be built, as when the state cannot be read, leaves the run to end as it would have."""+ mocker.patch.object(pipe_run_module, "report_run_usage", side_effect=OSError("a state that cannot be read"))+ router = mocker.AsyncMock()+ router.run = mocker.AsyncMock(side_effect=router_run)+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path), is_debug=True):+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTHON)+ pipe_run = PipeRun(pipe_router=router)+ if raised is None:+ await pipe_run.run(pipe_job=_job(mocker=mocker, is_traced=True))+ else:+ with pytest.raises(raised):+ await pipe_run.run(pipe_job=_job(mocker=mocker, is_traced=True))+ assert _debug_events(capsys.readouterr().err) == []diff --git a/tests/unit/pipelex/pipeline/test_usage_stats_reporting.py b/tests/unit/pipelex/pipeline/test_usage_stats_reporting.pynew file mode 100644index 000000000..a55e902ff--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_usage_stats_reporting.py@@ -0,0 +1,339 @@+from typing import Any, cast++import pytest+from pytest_mock import MockerFixture++from pipelex.base_exceptions import PipelexError+from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.usage_stats_reporting import (+ CodeGenerationReport,+ MethodValidationReport,+ RunUsageFacts,+ build_run_usage_events,+ reported_model,+ shipped_backend_models,+ usage_stats_codegen_target,+)+from pipelex.system.usage_stats.usage_stats_contract import (+ UsageStatsCodegenArtifact,+ UsageStatsCodegenTarget,+ UsageStatsEventName,+ UsageStatsOutcome,+ UsageStatsProvider,+)+from pipelex.system.usage_stats.usage_stats_events import UsageStatsEvent+from pipelex.system.usage_stats.usage_stats_runs import RunInferenceTally, usage_stats_error_domain+from pipelex.tracing.trace_events import TraceEvent+from tests.helpers.usage_stats_run_events import envelope_of, llm_usage_event, pipe_start_event+from tests.helpers.usage_stats_schema import assert_envelope_is_valid++_SHIPPED_ANTHROPIC_MODEL = "claude-4-sonnet"+_SECRET_MESSAGE = "secret failure text"+++def _facts(+ *,+ mocker: MockerFixture,+ outcome: UsageStatsOutcome = UsageStatsOutcome.SUCCEEDED,+ error: BaseException | None = None,+ events: list[TraceEvent] | None = None,+ tally: RunInferenceTally | None = None,+) -> RunUsageFacts:+ main_pipe = mocker.MagicMock()+ main_pipe.is_controller = False+ return RunUsageFacts(+ outcome=outcome,+ error=error,+ is_dry_run=False,+ duration_seconds=1.5,+ main_pipe=main_pipe,+ own_pipes=[],+ own_concepts=[],+ has_image_input=True,+ has_document_input=False,+ events=events,+ closed_graph=None,+ tally=tally,+ )+++def _properties(*, usage_event: UsageStatsEvent) -> dict[str, Any]:+ envelope = envelope_of(event=usage_event)+ assert_envelope_is_valid(envelope)+ return cast("dict[str, Any]", envelope["properties"])+++def _recorded_events(mocker: MockerFixture) -> list[UsageStatsEvent]:+ mocker.patch("pipelex.pipeline.usage_stats_reporting.usage_stats_are_enabled", return_value=True)+ concept_library = mocker.MagicMock()+ concept_library.list_own_concepts.return_value = []+ mocker.patch("pipelex.pipeline.usage_stats_reporting.get_concept_library", return_value=concept_library)+ recorded: list[UsageStatsEvent] = []++ def _record(*, event: UsageStatsEvent) -> None:+ recorded.append(event)++ mocker.patch("pipelex.pipeline.usage_stats_reporting.record_usage_stats_event", side_effect=_record)+ return recorded+++def _validate_raising(*, error: BaseException) -> None:+ with MethodValidationReport():+ raise error+++def _validate_declared_valid_then_exit() -> None:+ """A valid library refused for its pending signatures: the verdict stays valid."""+ with MethodValidationReport() as report:+ report.declare_valid()+ raise SystemExit(1)+++def _validate_rendering_the_verdict_then_exit() -> None:+ with MethodValidationReport():+ try:+ raise ValidateBundleError(message=_SECRET_MESSAGE)+ except ValidateBundleError:+ raise SystemExit(1) from None+++def _generate_raising(*, error: BaseException) -> None:+ with CodeGenerationReport(artifact=UsageStatsCodegenArtifact.TYPES):+ raise error+++class TestUsageStatsReporting:+ def test_run_with_usage_reports_one_model_used_per_model(self, mocker: MockerFixture) -> None:+ """Pipe counts and tokens come from the run's events, failures and backends from its tally, one `model_used` per model."""+ tally = RunInferenceTally()+ for _ in range(2):+ tally.note_call(+ model_handle=_SHIPPED_ANTHROPIC_MODEL,+ backend_name="anthropic",+ failed=False,+ failure_category=None,+ images_generated=0,+ pages_extracted=0,+ )+ tally.note_call(+ model_handle=_SHIPPED_ANTHROPIC_MODEL,+ backend_name="anthropic",+ failed=True,+ failure_category="capacity",+ images_generated=0,+ pages_extracted=0,+ )+ events: list[TraceEvent] = [+ pipe_start_event(pipe_type="PipeSequence", sequence=0),+ pipe_start_event(pipe_type="PipeLLM", sequence=1),+ pipe_start_event(pipe_type="PipeLLM", sequence=2),+ pipe_start_event(pipe_type="PipeFromAPlugin", sequence=3),+ llm_usage_event(+ model_name=_SHIPPED_ANTHROPIC_MODEL, backend_name="anthropic", input_tokens=100, output_tokens=50, cached_tokens=20, sequence=4+ ),+ llm_usage_event(model_name=_SHIPPED_ANTHROPIC_MODEL, backend_name="anthropic", input_tokens=200, output_tokens=10, sequence=5),+ ]+ run_event, model_event = build_run_usage_events(facts=_facts(mocker=mocker, events=events, tally=tally))+ run = _properties(usage_event=run_event)+ assert run_event.name == UsageStatsEventName.RUN_COMPLETED+ assert run["outcome"] == "succeeded"+ assert run["usage_known"] is True+ assert run["feature_image_input"] is True+ assert run["duration_ms"] == 1500+ assert (run["pipes_sequence"], run["pipes_llm"], run["pipes_other"], run["pipes_extract"]) == (1, 2, 1, 0)+ assert (run["models_used"], run["inference_calls"], run["inference_failures"]) == (1, 3, 1)+ assert (run["input_tokens"], run["output_tokens"], run["cached_tokens"]) == (300, 60, 20)+ assert run["cost_usd"] > 0+ model = _properties(usage_event=model_event)+ assert (model["provider"], model["model"], model["calls"], model["failures"], model["failures_capacity"]) == (+ "anthropic",+ _SHIPPED_ANTHROPIC_MODEL,+ 3,+ 1,+ 1,+ )+ assert (model["input_tokens"], model["output_tokens"], model["cached_tokens"]) == (300, 60, 20)+ assert model["cost_usd"] == run["cost_usd"]++ def test_a_call_that_fell_back_is_attributed_to_the_backend_that_made_it(self, mocker: MockerFixture) -> None:+ """One handle served by two backends in one run, which call the model by the same id: each backend's calls,+ failures and tokens are its own, by the backend each usage report names.+ """+ tally = RunInferenceTally()+ for backend_name, failed in (("anthropic", True), ("bedrock", False), ("anthropic", False)):+ tally.note_call(+ model_handle=_SHIPPED_ANTHROPIC_MODEL,+ backend_name=backend_name,+ failed=failed,+ failure_category="capacity" if failed else None,+ images_generated=0,+ pages_extracted=0,+ )+ events: list[TraceEvent] = [+ llm_usage_event(+ model_name=_SHIPPED_ANTHROPIC_MODEL, backend_name="bedrock", input_tokens=40, output_tokens=20, model_id="claude-sonnet-4", sequence=0+ ),+ llm_usage_event(+ model_name=_SHIPPED_ANTHROPIC_MODEL,+ backend_name="anthropic",+ input_tokens=100,+ output_tokens=50,+ model_id="claude-sonnet-4",+ sequence=1,+ ),+ ]+ _, *model_events = build_run_usage_events(facts=_facts(mocker=mocker, events=events, tally=tally))+ by_provider = {properties["provider"]: properties for properties in (_properties(usage_event=event) for event in model_events)}+ assert set(by_provider) == {"anthropic", "bedrock"}+ assert (by_provider["anthropic"]["calls"], by_provider["anthropic"]["failures"], by_provider["anthropic"]["input_tokens"]) == (2, 1, 100)+ assert (by_provider["bedrock"]["calls"], by_provider["bedrock"]["failures"], by_provider["bedrock"]["input_tokens"]) == (1, 0, 40)++ def test_unrated_model_leaves_the_run_cost_out(self, mocker: MockerFixture) -> None:+ """A model on a backend of the user's own is `custom` and `other`, and its missing rates leave the run's cost out."""+ tally = RunInferenceTally()+ tally.note_call(+ model_handle=_SHIPPED_ANTHROPIC_MODEL,+ backend_name="anthropic",+ failed=False,+ failure_category=None,+ images_generated=0,+ pages_extracted=0,+ )+ tally.note_call(+ model_handle="my-own-gpu-model",+ backend_name="my_gpu_box",+ failed=False,+ failure_category=None,+ images_generated=0,+ pages_extracted=0,+ )+ events: list[TraceEvent] = [+ llm_usage_event(model_name=_SHIPPED_ANTHROPIC_MODEL, backend_name="anthropic", input_tokens=10, output_tokens=5, sequence=0),+ llm_usage_event(model_name="my-own-gpu-model", backend_name="my_gpu_box", input_tokens=10, output_tokens=5, is_rated=False, sequence=1),+ ]+ run_event, *model_events = build_run_usage_events(facts=_facts(mocker=mocker, events=events, tally=tally))+ run = _properties(usage_event=run_event)+ assert "cost_usd" not in run+ assert run["models_used"] == 2+ models = {_properties(usage_event=event)["model"]: _properties(usage_event=event) for event in model_events}+ assert set(models) == {_SHIPPED_ANTHROPIC_MODEL, "other"}+ assert models["other"]["provider"] == "custom"+ assert "cost_usd" not in models["other"]++ def test_usage_unknown_sends_no_counts_and_no_model_used(self, mocker: MockerFixture) -> None:+ """With tracing or usage events off, the run says its usage is unknown and nothing else."""+ usage_events = build_run_usage_events(facts=_facts(mocker=mocker, events=None, tally=RunInferenceTally()))+ assert len(usage_events) == 1+ run = _properties(usage_event=usage_events[0])+ assert run["usage_known"] is False+ assert not [name for name in run if name.startswith("pipes_") or name.endswith("_tokens") or name == "cost_usd"]++ @pytest.mark.parametrize(+ ("outcome", "error"),+ [+ (UsageStatsOutcome.FAILED, PipelexError(_SECRET_MESSAGE)),+ (UsageStatsOutcome.FAILED, RuntimeError(_SECRET_MESSAGE)),+ (UsageStatsOutcome.CANCELLED, None),+ ],+ )+ def test_failed_and_cancelled_runs(self, mocker: MockerFixture, outcome: UsageStatsOutcome, error: BaseException | None) -> None:+ events: list[TraceEvent] = [llm_usage_event(model_name=_SHIPPED_ANTHROPIC_MODEL, backend_name="anthropic", input_tokens=7, output_tokens=3)]+ run_event = build_run_usage_events(facts=_facts(mocker=mocker, outcome=outcome, error=error, events=events))[0]+ envelope = envelope_of(event=run_event)+ assert_envelope_is_valid(envelope)+ run = envelope["properties"]+ assert run["outcome"] == outcome+ assert run["input_tokens"] == 7+ if outcome.is_failed:+ assert run["error_domain"] == usage_stats_error_domain(error=error)+ else:+ assert "error_domain" not in run+ assert "secret" not in str(envelope)++ @pytest.mark.parametrize(+ ("backend_name", "model_handle", "provider", "model"),+ [+ ("anthropic", _SHIPPED_ANTHROPIC_MODEL, "anthropic", _SHIPPED_ANTHROPIC_MODEL),+ ("openai", _SHIPPED_ANTHROPIC_MODEL, "openai", "other"),+ ("anthropic", "my-fine-tune-secret", "anthropic", "other"),+ ("pipelex_manifold", _SHIPPED_ANTHROPIC_MODEL, "pipelex_manifold", _SHIPPED_ANTHROPIC_MODEL),+ ("pipelex_manifold", "my-fine-tune-secret", "pipelex_manifold", "other"),+ ("my_company_gateway", _SHIPPED_ANTHROPIC_MODEL, "custom", "other"),+ (None, _SHIPPED_ANTHROPIC_MODEL, "custom", "other"),+ ("anthropic", "has spaces in it", "anthropic", "other"),+ ],+ )+ def test_provider_and_model_are_from_closed_lists(self, backend_name: str | None, model_handle: str, provider: str, model: str) -> None:+ """A model is named only when a backend table this release ships names it; Manifold serves every table."""+ assert _SHIPPED_ANTHROPIC_MODEL in shipped_backend_models()["anthropic"]+ reported_provider = UsageStatsProvider.for_backend(backend_name=backend_name)+ assert reported_provider == provider+ assert reported_model(provider=reported_provider, model_handle=model_handle) == model+ assert UsageStatsProvider(provider) == reported_provider++ def test_method_validated(self, mocker: MockerFixture) -> None:+ """Valid, invalid with its error count, and failed when no verdict was reached."""+ recorded = _recorded_events(mocker)+ with MethodValidationReport() as valid_report:+ valid_report.describe(pipes=[])+ valid_report.count_warnings(count=2)+ with pytest.raises(ValidateBundleError):+ _validate_raising(error=ValidateBundleError(message=_SECRET_MESSAGE))+ with pytest.raises(OSError, match=_SECRET_MESSAGE):+ _validate_raising(error=OSError(_SECRET_MESSAGE))+ with pytest.raises(SystemExit):+ _validate_declared_valid_then_exit()+ outcomes = [(_properties(usage_event=event)["outcome"], _properties(usage_event=event)["errors"]) for event in recorded]+ assert outcomes == [("valid", 0), ("invalid", 1), ("failed", 0), ("valid", 0)]+ assert _properties(usage_event=recorded[0])["warnings"] == 2++ def test_a_validation_statistic_never_changes_the_validation(self, mocker: MockerFixture) -> None:+ """A library that cannot be described, or a bundle whose event cannot be built, loses the event and nothing else."""+ recorded = _recorded_events(mocker)+ mocker.patch("pipelex.pipeline.usage_stats_reporting.get_concept_library", side_effect=RuntimeError("no library loaded"))+ steps: list[str] = []+ with MethodValidationReport() as report:+ report.describe(pipes=[])+ steps.append("validated")+ assert steps == ["validated"]+ mocker.patch("pipelex.pipeline.usage_stats_reporting.bundle_shape_of", side_effect=ValueError("an event that will not validate"))+ with pytest.raises(ValidateBundleError):+ _validate_raising(error=ValidateBundleError(message=_SECRET_MESSAGE))+ assert recorded == []++ def test_invalid_verdict_handled_on_the_way_out_is_read(self, mocker: MockerFixture) -> None:+ """A CLI renders the invalid verdict and exits: the verdict is found on the exception it handled."""+ recorded = _recorded_events(mocker)+ with pytest.raises(SystemExit):+ _validate_rendering_the_verdict_then_exit()+ assert _properties(usage_event=recorded[0])["outcome"] == "invalid"++ def test_code_generated(self, mocker: MockerFixture) -> None:+ recorded = _recorded_events(mocker)+ crate = mocker.MagicMock()+ crate.pipes = [object(), object()]+ crate.concepts = [object()]+ with CodeGenerationReport(artifact=UsageStatsCodegenArtifact.TYPES, target=usage_stats_codegen_target(target="ts-zod")) as types_report:+ types_report.describe(crate=crate)+ with pytest.raises(OSError, match=_SECRET_MESSAGE):+ _generate_raising(error=OSError(_SECRET_MESSAGE))+ with CodeGenerationReport(artifact=UsageStatsCodegenArtifact.INPUTS) as inputs_report:+ inputs_report.declare_failed()+ with CodeGenerationReport(artifact=UsageStatsCodegenArtifact.STRUCTURES):+ pass+ generated = [_properties(usage_event=event) for event in recorded]+ assert [(event["artifact"], event.get("target"), event["outcome"]) for event in generated] == [+ ("types", "ts-zod", "succeeded"),+ ("types", "other", "failed"),+ ("inputs", None, "failed"),+ ("structures", None, "succeeded"),+ ]+ assert (generated[0]["bundle_pipes"], generated[0]["bundle_concepts"]) == (2, 1)+ assert usage_stats_codegen_target(target="kotlin") == UsageStatsCodegenTarget.OTHER++ def test_nothing_is_built_while_statistics_are_off(self, mocker: MockerFixture) -> None:+ record = mocker.patch("pipelex.pipeline.usage_stats_reporting.record_usage_stats_event")+ mocker.patch("pipelex.pipeline.usage_stats_reporting.usage_stats_are_enabled", return_value=False)+ with MethodValidationReport(), CodeGenerationReport(artifact=UsageStatsCodegenArtifact.TYPES):+ pass+ record.assert_not_called()diff --git a/tests/unit/pipelex/system/usage_stats/test_data.py b/tests/unit/pipelex/system/usage_stats/test_data.pynew file mode 100644index 000000000..e065b2ee2--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_data.py@@ -0,0 +1,58 @@+from typing import ClassVar+++class UsageStatsSwitchCases:+ TOOL_SWITCH_READINGS: ClassVar[list[tuple[str, bool]]] = [+ ("0", True),+ ("false", True),+ (" No ", True),+ ("OFF", True),+ ("", False),+ ("1", False),+ ("true", False),+ (" Yes", False),+ ("on", False),+ ("disabled", True),+ ("2", True),+ ]+ FLAG_READINGS: ClassVar[list[tuple[str, bool]]] = [+ ("", False),+ ("0", False),+ (" FALSE ", False),+ ("1", True),+ ("true", True),+ ("no", True),+ ("anything", True),+ ]+ ENDPOINT_OVERRIDES: ClassVar[list[tuple[str, bool]]] = [+ ("http://127.0.0.1:8123", True),+ ("http://localhost:9", True),+ ("http://[::1]:65535", True),+ ("http://127.0.0.1:0", False),+ ("http://127.0.0.1:65536", False),+ ("http://127.0.0.1:8123/batch/", False),+ ("http://127.0.0.1", False),+ ("https://127.0.0.1:8123", False),+ ("http://10.0.0.1:8123", False),+ ("https://eu.i.posthog.com", False),+ ("http://localhost.example.com:80", False),+ ("", False),+ ]+ PROJECT_SETTINGS: ClassVar[list[tuple[str | bytes, bool]]] = [+ ("[usage_stats]\nenabled = false\n", True),+ ("[usage_stats]\nenabled = true\n", False),+ ("[usage_stats]\n", False),+ ("[runtime]\nfoo = 1\n", False),+ ("", False),+ ("usage_stats = false\n", True),+ ('[usage_stats]\nenabled = "false"\n', True),+ ("[usage_stats]\nenabled = 0\n", True),+ ("[usage_stats\nenabled = false\n", True),+ (b"\xff\xfe[usage_stats]", True),+ ]+++class UsageStatsCanary:+ """A string that must never appear in anything the statistics print, spool, send or store."""++ MARKER: ClassVar[str] = "zqcanary7x"diff --git a/tests/unit/pipelex/system/usage_stats/test_usage_stats_client.py b/tests/unit/pipelex/system/usage_stats/test_usage_stats_client.pynew file mode 100644index 000000000..e1a7321e8--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_usage_stats_client.py@@ -0,0 +1,363 @@+import json+import threading+import time+from pathlib import Path+from typing import Any++import pytest+from pytest_mock import MockerFixture++from pipelex.system.runtime import IntegrationMode+from pipelex.system.usage_stats import usage_stats_client+from pipelex.system.usage_stats.usage_stats_client import (+ UsageStatsClient,+ UsageStatsDelivery,+ UsageStatsSettings,+ build_settings,+ configure_usage_stats_for_cli,+ configure_usage_stats_for_runtime,+ get_usage_stats_client,+)+from pipelex.system.usage_stats.usage_stats_contract import (+ TOOL_NAME,+ UsageStatsEnvVar,+ UsageStatsEventName,+ UsageStatsIntegration,+ UsageStatsOffBy,+ UsageStatsOutcome,+ UsageStatsSurface,+)+from pipelex.system.usage_stats.usage_stats_core import (+ PostVerdict,+ SpoolContent,+ UsageStatsState,+ pending_files,+ read_spool_file,+ write_spool_file,+)+from pipelex.system.usage_stats.usage_stats_events import CommandCompletedProperties, UsageStatsEvent+from pipelex.test_extras.usage_stats_isolation import USAGE_STATS_DEAD_LOOPBACK_ORIGIN+from tests.helpers.usage_stats_capture import capture_server+from tests.helpers.usage_stats_process import CAN_LOCK_DIRECTORIES, locked_directory, make_project_dir, usage_stats_switched_on+from tests.helpers.usage_stats_schema import assert_envelope_is_valid++_COMMAND_EVENT = UsageStatsEvent(+ name=UsageStatsEventName.COMMAND_COMPLETED,+ properties=CommandCompletedProperties(command="validate", outcome=UsageStatsOutcome.SUCCEEDED, duration_ms=12),+)+++def _settings(+ *,+ tmp_path: Path,+ origin: str = USAGE_STATS_DEAD_LOOPBACK_ORIGIN,+ delivery: UsageStatsDelivery = UsageStatsDelivery.CLI_COMMAND,+ env: dict[str, str] | None = None,+ is_ci: bool = False,+) -> UsageStatsSettings:+ settings = build_settings(+ state=UsageStatsState(home=tmp_path / "home"),+ env={UsageStatsEnvVar.ENDPOINT_OVERRIDE: origin, **(env or {})},+ cwd=make_project_dir(parent=tmp_path),+ is_pytest=False,+ surface=UsageStatsSurface.PIPELEX,+ integration=UsageStatsIntegration.CLI,+ delivery=delivery,+ embedder_host=None,+ )+ return settings._replace(is_ci=is_ci, is_container=False)+++def _spooled_events(*, state: UsageStatsState) -> list[dict[str, Any]]:+ events: list[dict[str, Any]] = []+ for spool_file in pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME)):+ events.extend((read_spool_file(path=spool_file) or SpoolContent(endpoint=None, events=[])).events)+ return events+++def _seed_spool(*, settings: UsageStatsSettings) -> str:+ """Queue one event an earlier command left under the machine's identifier; its uuid."""+ installation_id = settings.state.obtain_installation_id().value+ queued_uuid = "00000000-0000-4000-8000-0000000000ee"+ event: dict[str, Any] = {"event": "command_completed", "distinct_id": installation_id, "uuid": queued_uuid, "properties": {}}+ write_spool_file(spool_dir=settings.state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[event]))+ return queued_uuid+++class TestUsageStatsClient:+ def test_statistics_off_create_nothing(self, tmp_path: Path) -> None:+ """With any switch off, nothing is recorded and nothing is written to the shared state."""+ settings = _settings(tmp_path=tmp_path, env={UsageStatsEnvVar.DO_NOT_TRACK: "1"})+ assert settings.off_by == (UsageStatsOffBy.DO_NOT_TRACK,)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ assert not settings.state.state_dir.exists()++ def test_debug_prints_the_envelope_instead_of_sending(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:+ """Each event is one JSON line on stderr, the envelope as it would be posted; nothing is sent or spooled."""+ settings = _settings(tmp_path=tmp_path, env={UsageStatsEnvVar.DEBUG: "1"})+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ captured = capsys.readouterr()+ assert captured.out == ""+ lines = captured.err.splitlines()+ assert len(lines) == 1+ envelope = json.loads(lines[0])["usage_stats_debug"]+ assert_envelope_is_valid(envelope)+ assert envelope["distinct_id"] == settings.state.read_installation_id()+ assert envelope["properties"]["id_new"] is True+ assert pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME)) == []++ def test_envelope_carries_the_common_properties(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:+ settings = _settings(tmp_path=tmp_path, env={UsageStatsEnvVar.DEBUG: "true", UsageStatsEnvVar.HOST: "cocode"}, is_ci=True)+ settings.state.state_dir.mkdir(parents=True)+ settings.state.internal_file.touch()+ settings.state.record_notice(version=1)+ UsageStatsClient(settings=settings).record(event=_COMMAND_EVENT)+ envelope = json.loads(capsys.readouterr().err.splitlines()[0])["usage_stats_debug"]+ assert_envelope_is_valid(envelope)+ properties = envelope["properties"]+ assert properties["$process_person_profile"] is False+ assert properties["schema"] == 1+ assert properties["tool"] == "pipelex"+ assert properties["surface"] == "pipelex"+ assert properties["integration"] == "cli"+ assert properties["host"] == "cocode"+ assert properties["internal"] is True+ assert properties["ci"] is True+ assert properties["container"] is False+ assert properties["notice_version"] == 1+ assert set(envelope) == {"event", "distinct_id", "uuid", "timestamp", "properties"}++ def test_command_spools_and_starts_the_detached_sender(self, tmp_path: Path, mocker: MockerFixture) -> None:+ start = mocker.patch.object(usage_stats_client, "start_detached_sender", return_value=True)+ settings = _settings(tmp_path=tmp_path, origin="http://127.0.0.1:8123")+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ start.assert_called_once_with(state=settings.state, origin="http://127.0.0.1:8123")+ spool_files = pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME))+ assert [(read_spool_file(path=spool_file) or SpoolContent(endpoint=None, events=[])).endpoint for spool_file in spool_files] == [+ "http://127.0.0.1:8123"+ ]+ spooled = _spooled_events(state=settings.state)+ assert [event["event"] for event in spooled] == ["command_completed"]+ assert_envelope_is_valid(spooled[0])++ def test_command_posts_in_process_when_the_sender_cannot_start(self, tmp_path: Path, mocker: MockerFixture) -> None:+ mocker.patch.object(usage_stats_client, "start_detached_sender", return_value=False)+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ assert [event["event"] for event in server.events] == ["command_completed"]+ assert pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME)) == []++ def test_ci_posts_in_process_before_the_command_ends(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """In CI the command has posted its events, marked `ci`, when it ends, and leaves nothing queued."""+ start = mocker.patch.object(usage_stats_client, "start_detached_sender", return_value=True)+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin, is_ci=True)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ assert [event["properties"]["ci"] for event in server.events] == [True]+ start.assert_not_called()+ assert pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME)) == []++ def test_ci_spools_what_it_could_not_send(self, tmp_path: Path) -> None:+ settings = _settings(tmp_path=tmp_path, is_ci=True)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ started = time.monotonic()+ client.finish_command()+ assert time.monotonic() - started < 2+ assert [event["event"] for event in _spooled_events(state=settings.state)] == ["command_completed"]++ def test_in_process_posts_also_send_what_earlier_commands_queued(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """In CI, and when the detached sender cannot start, the command sends its own events first, then the spool."""+ mocker.patch.object(usage_stats_client, "start_detached_sender", return_value=False)+ with capture_server() as server:+ for index_case, is_ci in enumerate((True, False)):+ settings = _settings(tmp_path=tmp_path / f"case-{index_case}", origin=server.origin, is_ci=is_ci)+ queued_uuid = _seed_spool(settings=settings)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ received = [event["uuid"] for event in server.events]+ assert received[-1] == queued_uuid+ assert len(received) == 2 * (index_case + 1)+ assert pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME)) == []++ @pytest.mark.parametrize("delivery", [UsageStatsDelivery.LONG_RUNNING, UsageStatsDelivery.CLI_COMMAND])+ @pytest.mark.parametrize(+ ("verdict", "is_drained"),+ [(None, False), (PostVerdict.RETRY, False), (PostVerdict.REFUSED, False), (PostVerdict.ACCEPTED, True)],+ )+ def test_the_spool_goes_only_after_an_accepted_post(+ self, tmp_path: Path, mocker: MockerFixture, delivery: UsageStatsDelivery, verdict: PostVerdict | None, is_drained: bool+ ) -> None:+ """What earlier processes queued is sent once PostHog accepted a post, never after a post of nothing, as when every+ event was screened out, from a program's thread as from a command posting in-process.+ """++ def answered(**kwargs: Any) -> tuple[PostVerdict | None, list[dict[str, Any]]]:+ return verdict, (list(kwargs["events"]) if verdict is not None else [])++ mocker.patch.object(usage_stats_client, "post_in_process", side_effect=answered)+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin, delivery=delivery, is_ci=delivery == UsageStatsDelivery.CLI_COMMAND)+ queued_uuid = _seed_spool(settings=settings)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ if delivery == UsageStatsDelivery.CLI_COMMAND:+ client.finish_command()+ else:+ client.flush()+ received = [event["uuid"] for event in server.events]+ assert (queued_uuid in received) is is_drained++ def test_process_local_identifier_is_never_persisted(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A malformed `id`: the command posts in-process under an identifier of its own and spools nothing on failure."""+ start = mocker.patch.object(usage_stats_client, "start_detached_sender", return_value=True)+ settings = _settings(tmp_path=tmp_path)+ settings.state.state_dir.mkdir(parents=True)+ settings.state.id_file.write_text("malformed", encoding="utf-8")+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.finish_command()+ start.assert_not_called()+ assert not settings.state.spool_root.exists()+ assert settings.state.id_file.read_text(encoding="utf-8") == "malformed"++ def test_python_program_posts_from_its_thread(self, tmp_path: Path) -> None:+ """A program importing pipelex never starts a process: its daemon thread posts, and a flush sends what is pending."""+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin, delivery=UsageStatsDelivery.PYTHON_PROGRAM)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.record(event=_COMMAND_EVENT)+ client.flush()+ deadline = time.monotonic() + 5+ while len(server.events) < 2 and time.monotonic() < deadline:+ time.sleep(0.05)+ assert len(server.events) == 2+ assert len({event["uuid"] for event in server.events}) == 2+ assert pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME)) == []++ def test_thread_also_sends_what_earlier_processes_queued(self, tmp_path: Path) -> None:+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin, delivery=UsageStatsDelivery.LONG_RUNNING)+ queued_uuid = _seed_spool(settings=settings)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ client.flush()+ received = [event["uuid"] for event in server.events]+ assert len(received) == 2+ assert received[-1] == queued_uuid+ assert pending_files(spool_dir=settings.state.spool_dir(tool=TOOL_NAME)) == []++ @pytest.mark.parametrize(+ ("error", "ends_the_thread"),+ [(OSError("a state that cannot be read"), False), (RuntimeError("a bug in the sender"), True)],+ )+ def test_thread_after_an_error_in_a_post(+ self, tmp_path: Path, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, error: Exception, ends_the_thread: bool+ ) -> None:+ """A recoverable error loses its batch alone; any other ends the thread with its traceback, and the next event starts a new one.++ Either way a flush does not wait for the lost batch, and later events are posted.+ """+ thread_errors: list[threading.ExceptHookArgs] = []+ monkeypatch.setattr(threading, "excepthook", thread_errors.append)+ original_post_in_process = usage_stats_client.post_in_process+ calls: list[int] = []++ def fails_once(**kwargs: Any) -> Any:+ calls.append(1)+ if len(calls) == 1:+ raise error+ return original_post_in_process(**kwargs)++ mocker.patch.object(usage_stats_client, "post_in_process", side_effect=fails_once)+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin, delivery=UsageStatsDelivery.LONG_RUNNING)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ deadline = time.monotonic() + 5+ while not calls and time.monotonic() < deadline:+ time.sleep(0.02)+ if ends_the_thread:+ while not thread_errors and time.monotonic() < deadline:+ time.sleep(0.02)+ ended_thread = thread_errors[0].thread+ assert ended_thread is not None+ ended_thread.join(timeout=5)+ assert not ended_thread.is_alive()+ else:+ time.sleep(0.1)+ started = time.monotonic()+ client.flush()+ assert time.monotonic() - started < 1+ client.record(event=_COMMAND_EVENT)+ deadline = time.monotonic() + 5+ while not server.events and time.monotonic() < deadline:+ time.sleep(0.05)+ assert len(server.events) == 1+ assert [type(thread_error.exc_value) for thread_error in thread_errors] == ([type(error)] if ends_the_thread else [])++ def test_thread_drops_events_once_disabled(self, tmp_path: Path) -> None:+ """A `disabled` file created after the event, before its post, keeps it from being sent."""+ with capture_server() as server:+ settings = _settings(tmp_path=tmp_path, origin=server.origin, delivery=UsageStatsDelivery.LONG_RUNNING)+ client = UsageStatsClient(settings=settings)+ client.record(event=_COMMAND_EVENT)+ settings.state.disable()+ client.flush()+ assert server.requests == []++ def test_fork_reset_drops_what_the_parent_held(self, tmp_path: Path, mocker: MockerFixture) -> None:+ start = mocker.patch.object(usage_stats_client, "start_detached_sender", return_value=True)+ client = UsageStatsClient(settings=_settings(tmp_path=tmp_path))+ client.record(event=_COMMAND_EVENT)+ client.reset_after_fork()+ client.finish_command()+ start.assert_not_called()++ def test_configuration_never_fails_for_want_of_a_home(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture) -> None:+ """No home directory to find: a command and a booting process both run without statistics."""+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path)):+ mocker.patch.object(UsageStatsState, "make_for_current_home", side_effect=RuntimeError("Could not determine home directory."))+ assert configure_usage_stats_for_cli(surface=UsageStatsSurface.PIPELEX, is_machine_read=False) is None+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTHON)+ assert get_usage_stats_client() is None++ def test_a_command_that_could_not_configure_statistics_keeps_them_off(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]+ ) -> None:+ """A transient failure leaves `pipelex-agent --format json` without statistics: pipelex booting inside it builds no+ program's client, which would print the runtime notice on the stderr the command keeps for JSON.+ """+ with usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "home", cwd=make_project_dir(parent=tmp_path)) as state:+ mocker.patch.object(UsageStatsState, "make_for_current_home", side_effect=[RuntimeError("Could not determine home directory."), state])+ assert configure_usage_stats_for_cli(surface=UsageStatsSurface.PIPELEX_AGENT, is_machine_read=True) is None+ configure_usage_stats_for_runtime(integration_mode=IntegrationMode.PYTHON)+ assert get_usage_stats_client() is None+ assert capsys.readouterr().err == ""++ @pytest.mark.skipif(not CAN_LOCK_DIRECTORIES, reason="needs a directory this process cannot traverse")+ def test_configuration_never_fails_for_a_state_that_cannot_be_looked_into(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]+ ) -> None:+ """A state no one may traverse may hold `disabled`: the command runs with statistics off, and no notice is printed."""+ with (+ usage_stats_switched_on(monkeypatch=monkeypatch, home=tmp_path / "locked" / "home", cwd=make_project_dir(parent=tmp_path)),+ locked_directory(path=tmp_path / "locked"),+ ):+ client = configure_usage_stats_for_cli(surface=UsageStatsSurface.PIPELEX, is_machine_read=False)+ assert client is not None+ assert UsageStatsOffBy.MACHINE_CHOICE in client.settings.off_by+ assert capsys.readouterr().err == ""diff --git a/tests/unit/pipelex/system/usage_stats/test_usage_stats_contract.py b/tests/unit/pipelex/system/usage_stats/test_usage_stats_contract.pynew file mode 100644index 000000000..f282342e7--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_usage_stats_contract.py@@ -0,0 +1,206 @@+from enum import StrEnum+from pathlib import Path+from typing import Any++from pytest_mock import MockerFixture++from pipelex.system.usage_stats import usage_stats_events+from pipelex.system.usage_stats.usage_stats_contract import (+ CI_VARIABLES,+ CONTAINER_CGROUP_WORDS,+ CONTAINER_MARKER_FILES,+ CONTAINER_MARKER_VARIABLES,+ FALSY_VALUES,+ MODEL_HANDLE_PATTERN,+ NOTICE_DETAILS_URL,+ NOTICE_DISABLE_COMMAND,+ NOTICE_VERSION,+ PERSON_PROFILE_PROPERTY,+ PIPELEX_AGENT_COMMANDS,+ PIPELEX_COMMANDS,+ RELEASE_VERSION_PATTERN,+ SCHEMA_VERSION,+ TOOL_NAME,+ TOOL_SWITCH_NO_EFFECT_VALUES,+ TOOL_SWITCH_OFF_VALUES,+ UsageStatsArch,+ UsageStatsCodegenArtifact,+ UsageStatsCodegenOutcome,+ UsageStatsCodegenTarget,+ UsageStatsEnvVar,+ UsageStatsErrorDomain,+ UsageStatsEventName,+ UsageStatsHost,+ UsageStatsIntegration,+ UsageStatsMethodSource,+ UsageStatsOffBy,+ UsageStatsOs,+ UsageStatsOutcome,+ UsageStatsProvider,+ UsageStatsSurface,+ UsageStatsValidationOutcome,+ render_notice,+)+from pipelex.system.usage_stats.usage_stats_core import (+ BATCH_PATH,+ BUDGET_BACKGROUND_POST_DELAY_MS,+ BUDGET_CLI_IN_PROCESS_MS,+ BUDGET_DETACHED_SENDER_MS,+ BUDGET_PYTHON_EXIT_FLUSH_MS,+ BUDGET_SHUTDOWN_FLUSH_MS,+ CAPTURE_HOST,+ DISABLED_FILE_NAME,+ ID_FILE_NAME,+ INSTALLATION_ID_PATTERN,+ INTERNAL_FILE_NAME,+ LEGACY_DISABLE_ON_VALUES,+ LEGACY_KEY,+ LEGACY_MIGRATED_FILE_NAME,+ NOTICE_FILE_PREFIX,+ PROJECT_TOKEN,+ SPOOL_DIR_NAME,+ SPOOL_MAX_AGE_DAYS,+ SPOOL_MAX_FILES_PER_TOOL,+ SPOOL_STALE_CLAIM_MINUTES,+ SPOOL_STALE_TEMP_MINUTES,+ SPOOL_TOOLS,+ STATE_DIR_PARTS,+)+from pipelex.system.usage_stats.usage_stats_events import INIT_BACKEND_FIELDS, INIT_BACKEND_PREFIX, CommandCompletedProperties, InitChoices+from pipelex.system.usage_stats.usage_stats_switches import PACKAGED_CONFIG_PATH, packaged_default_says_off+from tests.helpers.usage_stats_schema import declared_properties, load_usage_stats_schema+++def _values(*, event: str, name: str) -> list[str]:+ return list(declared_properties(event_name=event)[name]["values"])+++def _values_by(*, event: str, name: str, selector: str) -> list[str]:+ return list(declared_properties(event_name=event)[name]["values_by"]["values"][selector])+++class _ProvidersWithANewcomer(StrEnum):+ ANTHROPIC = "anthropic"+ NEWCOMER = "newcomer"+++class TestUsageStatsContract:+ def test_destination_is_the_schemas(self) -> None:+ destination = load_usage_stats_schema()["destination"]+ assert destination["capture_host"] == CAPTURE_HOST+ assert destination["batch_path"] == BATCH_PATH+ assert destination["project_token"] == PROJECT_TOKEN+ assert load_usage_stats_schema()["schema_version"] == SCHEMA_VERSION+ assert load_usage_stats_schema()["person_profile_property"] == PERSON_PROFILE_PROPERTY++ def test_notice_is_the_schemas_text_for_pipelex(self) -> None:+ notice = load_usage_stats_schema()["notice"]+ assert notice["version"] == NOTICE_VERSION+ assert notice["details_url"] == NOTICE_DETAILS_URL+ assert notice["disable_commands"][TOOL_NAME] == NOTICE_DISABLE_COMMAND+ expected = str(notice["text"]).replace("{disable_command}", NOTICE_DISABLE_COMMAND).replace("{details_url}", NOTICE_DETAILS_URL)+ assert render_notice() == expected++ def test_environment_variables_are_the_schemas(self) -> None:+ environment = load_usage_stats_schema()["environment"]+ assert environment["do_not_track"] == UsageStatsEnvVar.DO_NOT_TRACK+ assert environment["tool_switch"] == UsageStatsEnvVar.TOOL_SWITCH+ assert environment["debug"] == UsageStatsEnvVar.DEBUG+ assert environment["endpoint_override"] == UsageStatsEnvVar.ENDPOINT_OVERRIDE+ assert environment["host"] == UsageStatsEnvVar.HOST++ def test_switch_values_are_the_schemas(self) -> None:+ switch_values = load_usage_stats_schema()["switch_values"]+ assert set(switch_values["falsy"]) == FALSY_VALUES+ assert set(switch_values["tool_switch_off"]) == TOOL_SWITCH_OFF_VALUES+ assert set(switch_values["tool_switch_no_effect"]) == TOOL_SWITCH_NO_EFFECT_VALUES+ assert set(switch_values["mthds_disable_on"]) == LEGACY_DISABLE_ON_VALUES++ def test_off_by_tokens_are_pipelexs_switches_in_order(self) -> None:+ switches: list[dict[str, Any]] = load_usage_stats_schema()["switches"]+ pipelex_tokens = [switch["token"] for switch in switches if TOOL_NAME in switch["tools"]]+ assert [token.value for token in UsageStatsOffBy] == pipelex_tokens++ def test_state_names_are_the_schemas(self) -> None:+ state = load_usage_stats_schema()["state"]+ assert str(Path("~", *STATE_DIR_PARTS)) == state["directory"]+ assert state["id"] == ID_FILE_NAME+ assert state["disabled"] == DISABLED_FILE_NAME+ assert state["legacy_migrated"] == LEGACY_MIGRATED_FILE_NAME+ assert state["notice_prefix"] == NOTICE_FILE_PREFIX+ assert state["internal"] == INTERNAL_FILE_NAME+ assert state["spool"] == SPOOL_DIR_NAME+ assert state["legacy_key"] == LEGACY_KEY++ def test_spool_limits_and_budgets_are_the_schemas(self) -> None:+ schema = load_usage_stats_schema()+ spool = schema["spool"]+ assert spool["max_files_per_tool"] == SPOOL_MAX_FILES_PER_TOOL+ assert spool["max_age_days"] == SPOOL_MAX_AGE_DAYS+ assert spool["stale_claim_minutes"] == SPOOL_STALE_CLAIM_MINUTES+ assert spool["stale_temp_minutes"] == SPOOL_STALE_TEMP_MINUTES+ budgets = schema["budgets_ms"]+ assert budgets["cli_in_process"] == BUDGET_CLI_IN_PROCESS_MS+ assert budgets["python_exit_flush"] == BUDGET_PYTHON_EXIT_FLUSH_MS+ assert budgets["shutdown_flush"] == BUDGET_SHUTDOWN_FLUSH_MS+ assert budgets["detached_sender"] == BUDGET_DETACHED_SENDER_MS+ assert budgets["background_post_delay"] == BUDGET_BACKGROUND_POST_DELAY_MS+ assert list(SPOOL_TOOLS) == declared_properties(event_name="run_completed")["tool"]["values"]++ def test_ci_and_container_markers_are_the_schemas(self) -> None:+ schema = load_usage_stats_schema()+ assert list(CI_VARIABLES) == schema["ci_variables"]+ markers = schema["container_markers"]+ assert list(CONTAINER_MARKER_FILES) == markers["files"]+ assert list(CONTAINER_MARKER_VARIABLES) == markers["environment"]+ assert list(CONTAINER_CGROUP_WORDS) == markers["cgroup_words"]++ def test_patterns_are_the_schemas(self) -> None:+ patterns = load_usage_stats_schema()["patterns"]+ assert patterns["release_version"]["regex"] == RELEASE_VERSION_PATTERN+ assert patterns["installation_id"]["regex"] == INSTALLATION_ID_PATTERN+ assert patterns["model_handle"]["regex"] == MODEL_HANDLE_PATTERN++ def test_enums_are_the_schemas_values(self) -> None:+ assert {event.value for event in UsageStatsEventName} <= set(load_usage_stats_schema()["events"])+ assert [surface.value for surface in UsageStatsSurface] == _values_by(event="run_completed", name="surface", selector=TOOL_NAME)+ assert [integration.value for integration in UsageStatsIntegration] == _values(event="run_completed", name="integration")+ assert [host.value for host in UsageStatsHost] == _values(event="run_completed", name="host")+ assert [os_family.value for os_family in UsageStatsOs] == _values(event="run_completed", name="os")+ assert [arch.value for arch in UsageStatsArch] == _values(event="run_completed", name="arch")+ assert [outcome.value for outcome in UsageStatsOutcome] == _values(event="run_completed", name="outcome")+ assert [domain.value for domain in UsageStatsErrorDomain] == _values(event="run_completed", name="error_domain")+ assert [source.value for source in UsageStatsMethodSource] == _values(event="run_completed", name="method_source")+ assert [outcome.value for outcome in UsageStatsValidationOutcome] == _values(event="method_validated", name="outcome")+ assert [outcome.value for outcome in UsageStatsCodegenOutcome] == _values(event="code_generated", name="outcome")+ assert [artifact.value for artifact in UsageStatsCodegenArtifact] == _values(event="code_generated", name="artifact")+ assert [target.value for target in UsageStatsCodegenTarget] == _values(event="code_generated", name="target")+ assert [provider.value for provider in UsageStatsProvider] == _values(event="model_used", name="provider")++ def test_init_backend_fields_are_the_providers_and_the_schemas(self) -> None:+ """A provider added to the list, to the event's fields or to the schema, is added to all three, or this fails."""+ fields = {name for name in CommandCompletedProperties.model_fields if name.startswith(INIT_BACKEND_PREFIX)}+ assert fields == INIT_BACKEND_FIELDS+ assert fields == {f"{INIT_BACKEND_PREFIX}{provider}" for provider in UsageStatsProvider}+ assert fields == {name for name in declared_properties(event_name="command_completed") if name.startswith(INIT_BACKEND_PREFIX)}++ def test_a_provider_without_its_field_never_breaks_init_s_event(self, mocker: MockerFixture) -> None:+ """A provider added to the list before the event's fields is left out of `init`'s `command_completed`, never refused."""+ mocker.patch.object(usage_stats_events, "UsageStatsProvider", _ProvidersWithANewcomer)+ properties = InitChoices(enabled_providers=frozenset({UsageStatsProvider.ANTHROPIC})).as_properties()+ assert properties == {f"{INIT_BACKEND_PREFIX}anthropic": True}+ event = CommandCompletedProperties(command="init", outcome=UsageStatsOutcome.SUCCEEDED, duration_ms=1, **properties)+ assert event.init_backend_anthropic is True++ def test_command_lists_are_the_schemas(self) -> None:+ assert set(PIPELEX_COMMANDS) == set(_values_by(event="command_completed", name="command", selector=UsageStatsSurface.PIPELEX))+ assert set(PIPELEX_AGENT_COMMANDS) == set(_values_by(event="command_completed", name="command", selector=UsageStatsSurface.PIPELEX_AGENT))++ def test_manifold_is_reported_as_its_shipped_backend_name(self) -> None:+ assert UsageStatsProvider.PIPELEX_MANIFOLD == "pipelex_manifold"+ assert UsageStatsProvider.PIPELEX_MANIFOLD.serves_every_shipped_table+ assert not UsageStatsProvider.OPENAI.serves_every_shipped_table++ def test_packaged_configuration_ships_statistics_on(self) -> None:+ assert PACKAGED_CONFIG_PATH.is_file()+ assert not packaged_default_says_off()diff --git a/tests/unit/pipelex/system/usage_stats/test_usage_stats_delivery.py b/tests/unit/pipelex/system/usage_stats/test_usage_stats_delivery.pynew file mode 100644index 000000000..5da20cd1a--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_usage_stats_delivery.py@@ -0,0 +1,271 @@+import ast+import json+import subprocess # ruff: ignore[suspicious-subprocess-import] — runs the detached sender module the way a command does+import sys+import threading+import time+import venv+from pathlib import Path+from typing import Any++import pytest+from pytest_mock import MockerFixture++from pipelex.system.usage_stats import usage_stats_core+from pipelex.system.usage_stats.usage_stats_contract import TOOL_NAME+from pipelex.system.usage_stats.usage_stats_core import (+ BATCH_PATH,+ CAPTURE_HOST,+ PROJECT_TOKEN,+ PostVerdict,+ SpoolContent,+ UsageStatsState,+ drain_spool,+ pending_files,+ post_batch,+ post_in_process,+ sender_command,+ write_spool_file,+)+from pipelex.system.usage_stats.usage_stats_core import main as sender_main+from pipelex.system.usage_stats.usage_stats_delivery import start_detached_sender+from tests.helpers.usage_stats_capture import capture_server, trickling_server+from tests.helpers.usage_stats_process import stats_on_environment+++def _event(*, distinct_id: str, uuid: str) -> dict[str, Any]:+ return {"event": "command_completed", "distinct_id": distinct_id, "uuid": uuid, "timestamp": "2026-10-11T09:30:00.123Z", "properties": {}}+++def _state_with_id(*, home: Path) -> tuple[UsageStatsState, str]:+ state = UsageStatsState(home=home)+ return state, state.obtain_installation_id().value+++class TestUsageStatsDelivery:+ def test_batch_is_posted_with_the_project_token(self) -> None:+ events = [_event(distinct_id="x", uuid="1")]+ with capture_server() as server:+ verdict = post_batch(origin=server.origin, events=events, timeout_seconds=2)+ assert verdict == PostVerdict.ACCEPTED+ assert len(server.requests) == 1+ request = server.requests[0]+ assert request.path == BATCH_PATH+ assert request.content_type == "application/json"+ assert request.body == {"api_key": PROJECT_TOKEN, "batch": events}++ @pytest.mark.parametrize(+ ("status", "verdict", "is_kept"),+ [+ (200, PostVerdict.ACCEPTED, False),+ (400, PostVerdict.REFUSED, False),+ (413, PostVerdict.REFUSED, False),+ (429, PostVerdict.RETRY, True),+ (503, PostVerdict.RETRY, True),+ ],+ )+ def test_answer_decides_what_happens_to_the_file(self, tmp_path: Path, status: int, verdict: PostVerdict, is_kept: bool) -> None:+ """Accepted or refused for good deletes the file; anything else gives it back, resent later with its uuid."""+ state, installation_id = _state_with_id(home=tmp_path)+ spool_file = write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ with capture_server(statuses=[status]) as server:+ drain_spool(state=state, tool=TOOL_NAME, origin=server.origin, budget_ms=2000)+ assert pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME)) == ([spool_file] if is_kept else [])+ assert [event["uuid"] for event in server.requests[0].body["batch"]] == ["u-1"]+ assert verdict.is_final is not is_kept++ def test_unreachable_endpoint_keeps_the_file(self, tmp_path: Path) -> None:+ state, installation_id = _state_with_id(home=tmp_path)+ spool_file = write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ drain_spool(state=state, tool=TOOL_NAME, origin="http://127.0.0.1:9", budget_ms=1000)+ assert pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME)) == [spool_file]++ def test_events_of_a_replaced_identifier_are_dropped(self, tmp_path: Path) -> None:+ """Before each post the sender reads `id`: events made under an identifier `reset-id` replaced never leave."""+ state, installation_id = _state_with_id(home=tmp_path)+ events = [_event(distinct_id=installation_id, uuid="kept"), _event(distinct_id="0f8fad5b-d9cb-469f-a165-70867728950e", uuid="dropped")]+ write_spool_file(spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=events))+ with capture_server() as server:+ drain_spool(state=state, tool=TOOL_NAME, origin=server.origin, budget_ms=2000)+ assert [event["uuid"] for event in server.events] == ["kept"]++ def test_nothing_is_sent_once_disabled_or_migrated(self, tmp_path: Path) -> None:+ """A `disabled` file, or a legacy opt-out migrated before the post, sends nothing and deletes the queue."""+ state, installation_id = _state_with_id(home=tmp_path)+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ state.legacy_config.write_text("DISABLE_TELEMETRY=true\n", encoding="utf-8")+ with capture_server() as server:+ drain_spool(state=state, tool=TOOL_NAME, origin=server.origin, budget_ms=2000)+ verdict, posted = post_in_process(+ state=state,+ events=[_event(distinct_id=installation_id, uuid="u-2")],+ origin=server.origin,+ deadline=time.monotonic() + 1,+ own_id=None,+ )+ assert server.requests == []+ assert (verdict, posted) == (None, [])+ assert state.is_disabled()+ assert pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME)) == []++ def test_nothing_is_sent_while_a_legacy_opt_out_cannot_be_migrated(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A legacy opt-out the migration cannot carry over, its state being unwritable, still says off before each post."""+ state, installation_id = _state_with_id(home=tmp_path)+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ state.legacy_config.write_text("DISABLE_TELEMETRY=true\n", encoding="utf-8")+ mocker.patch.object(UsageStatsState, "disable", side_effect=OSError("read-only state"))+ with capture_server() as server:+ drain_spool(state=state, tool=TOOL_NAME, origin=server.origin, budget_ms=2000)+ verdict, posted = post_in_process(+ state=state,+ events=[_event(distinct_id=installation_id, uuid="u-2")],+ origin=server.origin,+ deadline=time.monotonic() + 1,+ own_id=None,+ )+ assert server.requests == []+ assert (verdict, posted) == (None, [])+ assert state.legacy_opt_out_is_pending()++ def test_sender_posts_a_file_only_to_the_endpoint_it_was_queued_for(self, tmp_path: Path) -> None:+ """Events queued for PostHog never reach a loopback endpoint, nor events queued for one loopback endpoint another."""+ state, installation_id = _state_with_id(home=tmp_path)+ spool_dir = state.spool_dir(tool=TOOL_NAME)+ spool_dir.mkdir(parents=True)+ queued_ms = int(time.time() * 1000) - 1000+ for index_file, endpoint in enumerate((CAPTURE_HOST, "http://127.0.0.1:9")):+ payload = {+ "spool_version": 1,+ "endpoint": endpoint,+ "events": [_event(distinct_id=installation_id, uuid=f"queued-elsewhere-{index_file}")],+ }+ (spool_dir / f"{queued_ms}-elsewhere{index_file}.json").write_text(json.dumps(payload), encoding="utf-8")+ with capture_server() as server:+ write_spool_file(+ spool_dir=spool_dir, content=SpoolContent(endpoint=server.origin, events=[_event(distinct_id=installation_id, uuid="mine")])+ )+ write_spool_file(spool_dir=spool_dir, content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="any")]))+ drain_spool(state=state, tool=TOOL_NAME, origin=server.origin, budget_ms=2000)+ assert sorted(event["uuid"] for event in server.events) == ["any", "mine"]+ assert sorted(path.name for path in pending_files(spool_dir=spool_dir)) == sorted(+ path.name for path in spool_dir.iterdir() if "-elsewhere" in path.name+ )+ assert len(pending_files(spool_dir=spool_dir)) == 2++ def test_sender_drains_only_its_own_tool(self, tmp_path: Path) -> None:+ state, installation_id = _state_with_id(home=tmp_path)+ write_spool_file(+ spool_dir=state.spool_dir(tool="plxt"), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="plxt-1")])+ )+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME),+ content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="pipelex-1")]),+ )+ with capture_server() as server:+ assert sender_main(argv=["--home", str(tmp_path), "--origin", server.origin, "--tool", TOOL_NAME]) == 0+ assert [event["uuid"] for event in server.events] == ["pipelex-1"]+ assert len(pending_files(spool_dir=state.spool_dir(tool="plxt"))) == 1++ @pytest.mark.parametrize("origin", ["https://example.com", "http://10.0.0.1:8123", "http://127.0.0.1:8123/batch/"])+ def test_sender_refuses_any_other_origin(self, tmp_path: Path, origin: str) -> None:+ state, installation_id = _state_with_id(home=tmp_path)+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ assert sender_main(argv=["--home", str(tmp_path), "--origin", origin, "--tool", TOOL_NAME]) == 2+ assert len(pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME))) == 1++ def test_events_that_cannot_be_encoded_are_refused_for_good(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A batch JSON cannot encode is refused, never posted and never queued again, and no thread dies of it."""+ thread_errors: list[threading.ExceptHookArgs] = []+ monkeypatch.setattr(threading, "excepthook", thread_errors.append)+ unencodable = {**_event(distinct_id="x", uuid="1"), "properties": {"value": object()}}+ with capture_server() as server:+ verdict = post_batch(origin=server.origin, events=[unencodable], timeout_seconds=2)+ assert verdict == PostVerdict.REFUSED+ assert verdict.is_final+ assert server.requests == []+ assert thread_errors == []++ def test_post_returns_within_its_budget_whatever_the_network_does(self) -> None:+ """An answer trickling in byte by byte never trips a socket timeout; the post still returns within its budget."""+ with trickling_server(seconds_per_byte=0.1) as origin:+ started = time.monotonic()+ verdict = post_batch(origin=origin, events=[_event(distinct_id="x", uuid="1")], timeout_seconds=0.5)+ elapsed = time.monotonic() - started+ assert verdict == PostVerdict.RETRY+ assert elapsed < 1.5++ def test_drain_returns_within_its_budget_and_keeps_the_file(self, tmp_path: Path) -> None:+ state, installation_id = _state_with_id(home=tmp_path)+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ with trickling_server(seconds_per_byte=0.1) as origin:+ started = time.monotonic()+ drain_spool(state=state, tool=TOOL_NAME, origin=origin, budget_ms=500)+ elapsed = time.monotonic() - started+ assert elapsed < 1.5+ assert len(pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME))) == 1++ def test_detached_sender_file_posts_the_queue(self, tmp_path: Path) -> None:+ """The sender's file run by path in isolated mode, as a command starts it, posts the queue and leaves it empty."""+ state, installation_id = _state_with_id(home=tmp_path)+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ with capture_server() as server:+ completed = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] — a fixed argument list, no shell+ sender_command(executable=sys.executable, home=tmp_path, origin=server.origin, tool=TOOL_NAME),+ env={**stats_on_environment(origin=server.origin), "HOME": str(tmp_path)},+ capture_output=True,+ check=False,+ timeout=60,+ )+ assert completed.returncode == 0, completed.stderr+ assert completed.stdout == b""+ assert [event["uuid"] for event in server.events] == ["u-1"]+ assert pending_files(spool_dir=state.spool_dir(tool=TOOL_NAME)) == []++ def test_sender_imports_the_standard_library_alone(self) -> None:+ """The sender is one file a bare interpreter can run: every module it imports is the standard library's."""+ tree = ast.parse(Path(usage_stats_core.__file__).read_text(encoding="utf-8"))+ imported: set[str] = set()+ for node in ast.walk(tree):+ if isinstance(node, ast.Import):+ imported.update(alias.name.split(".")[0] for alias in node.names)+ elif isinstance(node, ast.ImportFrom):+ assert node.level == 0, "the sender is run by path, so it can import nothing relative to itself"+ imported.add((node.module or "").split(".")[0])+ assert imported <= sys.stdlib_module_names, sorted(imported - sys.stdlib_module_names)++ def test_detached_sender_delivers_where_pipelex_is_not_importable(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A sender started by an interpreter that cannot import pipelex, as with a user or `PYTHONPATH` install, still delivers."""+ bare_environment = tmp_path / "bare-venv"+ venv.create(bare_environment, with_pip=False, symlinks=sys.platform != "win32")+ bare_python = bare_environment / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python")+ cannot_import = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] — a fixed argument list, no shell+ [str(bare_python), "-I", "-c", "import pipelex"], capture_output=True, check=False, timeout=60+ )+ assert cannot_import.returncode != 0+ home = tmp_path / "home"+ state, installation_id = _state_with_id(home=home)+ write_spool_file(+ spool_dir=state.spool_dir(tool=TOOL_NAME), content=SpoolContent(endpoint=None, events=[_event(distinct_id=installation_id, uuid="u-1")])+ )+ mocker.patch.object(sys, "executable", str(bare_python))+ with capture_server() as server:+ assert start_detached_sender(state=state, origin=server.origin)+ deadline = time.monotonic() + 30+ while not server.events and time.monotonic() < deadline:+ time.sleep(0.05)+ received = [event["uuid"] for event in server.events]+ assert received == ["u-1"]diff --git a/tests/unit/pipelex/system/usage_stats/test_usage_stats_spool.py b/tests/unit/pipelex/system/usage_stats/test_usage_stats_spool.pynew file mode 100644index 000000000..1a6051904--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_usage_stats_spool.py@@ -0,0 +1,106 @@+import json+import os+import time+from pathlib import Path++import pytest++from pipelex.system.usage_stats.usage_stats_core import (+ CLAIM_MARKER,+ SPOOL_MAX_FILES_PER_TOOL,+ SpoolContent,+ claim,+ pending_files,+ prune,+ read_spool_file,+ release,+ write_spool_file,+)+++def _age(*, path: Path, seconds: float) -> None:+ past = time.time() - seconds+ os.utime(path, (past, past))+++class TestUsageStatsSpool:+ def test_written_file_holds_its_events(self, tmp_path: Path) -> None:+ """A spool file is `<created-ms>-<random>.json` holding `spool_version` 1 and the events, and no temporary is left."""+ events = [{"event": "command_completed", "uuid": "a"}, {"event": "run_completed", "uuid": "b"}]+ spool_file = write_spool_file(spool_dir=tmp_path, content=SpoolContent(endpoint=None, events=events))+ created_ms, _, rest = spool_file.name.partition("-")+ assert created_ms.isdigit()+ assert rest.endswith(".json")+ assert json.loads(spool_file.read_text(encoding="utf-8")) == {"spool_version": 1, "events": events}+ assert read_spool_file(path=spool_file) == SpoolContent(endpoint=None, events=events)+ assert [entry.name for entry in tmp_path.iterdir()] == [spool_file.name]++ def test_written_file_records_the_endpoint_it_was_queued_for(self, tmp_path: Path) -> None:+ events = [{"event": "command_completed", "uuid": "a"}]+ spool_file = write_spool_file(spool_dir=tmp_path, content=SpoolContent(endpoint="http://127.0.0.1:8123", events=events))+ assert json.loads(spool_file.read_text(encoding="utf-8")) == {"spool_version": 1, "endpoint": "http://127.0.0.1:8123", "events": events}+ content = read_spool_file(path=spool_file)+ assert content is not None+ assert content == SpoolContent(endpoint="http://127.0.0.1:8123", events=events)+ assert content.is_queued_for(origin="http://127.0.0.1:8123")+ assert not content.is_queued_for(origin="https://eu.i.posthog.com")+ assert SpoolContent(endpoint=None, events=events).is_queued_for(origin="https://eu.i.posthog.com")++ @pytest.mark.parametrize(+ "content",+ [+ '{"spool_version": 2, "events": []}',+ '{"spool_version": 1, "events": {}}',+ '{"spool_version": 1, "events": [1]}',+ '{"spool_version": 1, "endpoint": 8123, "events": []}',+ "[]",+ "not json",+ ],+ )+ def test_file_of_another_shape_is_not_read(self, tmp_path: Path, content: str) -> None:+ path = tmp_path / "1-abc.json"+ path.write_text(content, encoding="utf-8")+ assert read_spool_file(path=path) is None++ def test_spool_keeps_at_most_its_limit_deleting_the_oldest(self, tmp_path: Path) -> None:+ for index_file in range(SPOOL_MAX_FILES_PER_TOOL + 3):+ (tmp_path / f"{1000 + index_file}-seed.json").write_text('{"spool_version": 1, "events": []}', encoding="utf-8")+ newest = write_spool_file(spool_dir=tmp_path, content=SpoolContent(endpoint=None, events=[]))+ waiting = pending_files(spool_dir=tmp_path)+ assert len(waiting) == SPOOL_MAX_FILES_PER_TOOL+ assert waiting[-1] == newest+ assert waiting[0].name == "1004-seed.json"++ def test_claim_and_release(self, tmp_path: Path) -> None:+ """A claimed file is no longer pending, a second claim fails, and a release gives it back under its name."""+ spool_file = write_spool_file(spool_dir=tmp_path, content=SpoolContent(endpoint=None, events=[]))+ claimed = claim(path=spool_file)+ assert claimed is not None+ assert CLAIM_MARKER in claimed.name+ assert pending_files(spool_dir=tmp_path) == []+ assert claim(path=spool_file) is None+ release(claimed=claimed)+ assert pending_files(spool_dir=tmp_path) == [spool_file]++ def test_prune(self, tmp_path: Path) -> None:+ """Stale claims come back, stale temporaries and month-old files go, fresh ones stay."""+ stale_claim = tmp_path / f"{int(time.time() * 1000)}-a.json{CLAIM_MARKER}crashed"+ stale_claim.write_text("{}", encoding="utf-8")+ _age(path=stale_claim, seconds=11 * 60)+ live_claim = tmp_path / f"{int(time.time() * 1000)}-b.json{CLAIM_MARKER}sending"+ live_claim.write_text("{}", encoding="utf-8")+ stale_temp = tmp_path / ".tmp-old"+ stale_temp.write_text("{}", encoding="utf-8")+ _age(path=stale_temp, seconds=61 * 60)+ fresh_temp = tmp_path / ".tmp-new"+ fresh_temp.write_text("{}", encoding="utf-8")+ month_old_ms = int((time.time() - 31 * 24 * 3600) * 1000)+ too_old = tmp_path / f"{month_old_ms}-c.json"+ too_old.write_text("{}", encoding="utf-8")+ prune(spool_dir=tmp_path)+ names = sorted(entry.name for entry in tmp_path.iterdir())+ assert stale_claim.name.split(CLAIM_MARKER)[0] in names+ assert live_claim.name in names+ assert fresh_temp.name in names+ assert stale_temp.name not in names+ assert too_old.name not in namesdiff --git a/tests/unit/pipelex/system/usage_stats/test_usage_stats_state.py b/tests/unit/pipelex/system/usage_stats/test_usage_stats_state.pynew file mode 100644index 000000000..74c7d4870--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_usage_stats_state.py@@ -0,0 +1,187 @@+import os+import stat+import sys+import threading+import time+from pathlib import Path++import pytest+from pytest_mock import MockerFixture++from pipelex.system.usage_stats.usage_stats_core import (+ SPOOL_TOOLS,+ InstallationId,+ SpoolContent,+ UsageStatsState,+ is_installation_id,+ parse_legacy_config,+ write_spool_file,+)+from tests.helpers.usage_stats_process import CAN_LOCK_DIRECTORIES, locked_directory+++def _seed_every_spool(*, state: UsageStatsState) -> None:+ for tool in SPOOL_TOOLS:+ write_spool_file(spool_dir=state.spool_dir(tool=tool), content=SpoolContent(endpoint=None, events=[{"event": "command_completed"}]))+++def _spooled_files(*, state: UsageStatsState) -> list[Path]:+ return [entry for tool in SPOOL_TOOLS if state.spool_dir(tool=tool).is_dir() for entry in state.spool_dir(tool=tool).iterdir()]+++class TestUsageStatsState:+ def test_first_identifier_is_created_once(self, tmp_path: Path) -> None:+ """The first process creates `id` with mode 0700 on its directory; the next reads it, not new."""+ state = UsageStatsState(home=tmp_path)+ first = state.obtain_installation_id()+ assert first.is_new+ assert is_installation_id(value=first.value)+ assert state.id_file.read_text(encoding="utf-8") == first.value+ if sys.platform != "win32":+ assert stat.S_IMODE(state.state_dir.stat().st_mode) == 0o700+ second = state.obtain_installation_id()+ assert second == InstallationId(value=first.value, is_new=False)+ assert [entry.name for entry in state.state_dir.iterdir()] == ["id"]++ def test_racing_first_runs_converge_on_one_identifier(self, tmp_path: Path) -> None:+ """Exactly one of the racing processes reports the identifier as new, and every one sends under it."""+ state = UsageStatsState(home=tmp_path)+ results: list[InstallationId] = []+ lock = threading.Lock()+ barrier = threading.Barrier(8)++ def obtain() -> None:+ barrier.wait()+ installation_id = UsageStatsState(home=tmp_path).obtain_installation_id()+ with lock:+ results.append(installation_id)++ threads = [threading.Thread(target=obtain) for _ in range(8)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join()+ assert len({result.value for result in results}) == 1+ assert sum(1 for result in results if result.is_new) == 1+ assert state.read_installation_id() == results[0].value+ assert not [entry for entry in state.state_dir.iterdir() if entry.name.startswith("id.tmp-")]++ @pytest.mark.parametrize("content", ["not-a-uuid", "0F8FAD5B-D9CB-469F-A165-70867728950E", "0f8fad5b-d9cb-169f-a165-70867728950e", ""])+ def test_malformed_identifier_is_never_rewritten(self, tmp_path: Path, content: str) -> None:+ """A malformed `id` gives the process an identifier of its own, persisted nowhere; only `reset-id` replaces it."""+ state = UsageStatsState(home=tmp_path)+ state.state_dir.mkdir(parents=True)+ state.id_file.write_text(content, encoding="utf-8")+ process_local = state.obtain_installation_id()+ assert process_local.is_new+ assert is_installation_id(value=process_local.value)+ assert state.id_file.read_text(encoding="utf-8") == content+ assert state.read_installation_id() is None+ replaced = state.replace_installation_id()+ assert state.read_installation_id() == replaced++ def test_identifier_with_whitespace_is_read(self, tmp_path: Path) -> None:+ state = UsageStatsState(home=tmp_path)+ state.state_dir.mkdir(parents=True)+ state.id_file.write_text(" 0f8fad5b-d9cb-469f-a165-70867728950e\n", encoding="utf-8")+ assert state.obtain_installation_id() == InstallationId(value="0f8fad5b-d9cb-469f-a165-70867728950e", is_new=False)++ def test_no_hard_links_means_nothing_persisted(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A filesystem without hard links: the process uses its fresh identifier for itself alone."""+ mocker.patch("pipelex.system.usage_stats.usage_stats_core.os.link", side_effect=OSError("no hard links here"))+ state = UsageStatsState(home=tmp_path)+ installation_id = state.obtain_installation_id()+ assert installation_id.is_new+ assert not state.id_file.exists()+ assert not [entry for entry in state.state_dir.iterdir() if entry.name.startswith("id.tmp-")]++ def test_stale_temporary_identifiers_are_deleted(self, tmp_path: Path) -> None:+ state = UsageStatsState(home=tmp_path)+ state.state_dir.mkdir(parents=True)+ stale = state.state_dir / "id.tmp-crashed"+ stale.write_text("0f8fad5b-d9cb-469f-a165-70867728950e", encoding="utf-8")+ two_hours_ago = time.time() - 7200+ os.utime(stale, (two_hours_ago, two_hours_ago))+ fresh = state.state_dir / "id.tmp-writing"+ fresh.write_text("", encoding="utf-8")+ state.obtain_installation_id()+ assert not stale.exists()+ assert fresh.exists()++ def test_reset_id_empties_every_spool(self, tmp_path: Path) -> None:+ state = UsageStatsState(home=tmp_path)+ first = state.obtain_installation_id()+ _seed_every_spool(state=state)+ replaced = state.replace_installation_id()+ assert replaced != first.value+ assert state.read_installation_id() == replaced+ assert _spooled_files(state=state) == []++ def test_disable_and_enable(self, tmp_path: Path) -> None:+ """`disable` creates `disabled` and empties every tool's spool; `enable` deletes it and sets the legacy marker."""+ state = UsageStatsState(home=tmp_path)+ _seed_every_spool(state=state)+ state.disable()+ assert state.is_disabled()+ assert _spooled_files(state=state) == []+ state.enable()+ assert not state.is_disabled()+ assert state.legacy_marker.exists()++ def test_legacy_opt_out_is_migrated_once(self, tmp_path: Path) -> None:+ """`DISABLE_TELEMETRY` on in `~/.mthds/config` becomes the machine's choice, and `enable` is not undone by it."""+ state = UsageStatsState(home=tmp_path)+ _seed_every_spool(state=state)+ state.legacy_config.write_text("# mthds\nDISABLE_TELEMETRY = Yes\n", encoding="utf-8")+ assert state.legacy_opt_out_is_pending()+ state.migrate_legacy_opt_out()+ assert state.is_disabled()+ assert _spooled_files(state=state) == []+ assert state.legacy_marker.stat().st_mtime_ns >= state.legacy_config.stat().st_mtime_ns+ assert not state.legacy_opt_out_is_pending()+ state.enable()+ state.migrate_legacy_opt_out()+ assert not state.is_disabled()+ later = time.time() + 5+ os.utime(state.legacy_config, (later, later))+ assert state.legacy_opt_out_is_pending()+ state.migrate_legacy_opt_out()+ assert state.is_disabled()++ @pytest.mark.parametrize("content", ["DISABLE_TELEMETRY=0\n", "DISABLE_TELEMETRY=false\n", "OTHER=1\n", "# DISABLE_TELEMETRY=1\n", ""])+ def test_legacy_value_that_does_not_mean_off_migrates_nothing(self, tmp_path: Path, content: str) -> None:+ state = UsageStatsState(home=tmp_path)+ state.legacy_config.parent.mkdir(parents=True)+ state.legacy_config.write_text(content, encoding="utf-8")+ state.migrate_legacy_opt_out()+ assert not state.state_dir.exists()++ def test_legacy_config_dialect(self) -> None:+ content = " # comment\nDISABLE_TELEMETRY = 0\nno equals here\nKEY=a=b\nDISABLE_TELEMETRY=1\r\n\n"+ assert parse_legacy_config(content=content) == {"DISABLE_TELEMETRY": "1", "KEY": "a=b"}++ def test_notice_markers_and_internal(self, tmp_path: Path) -> None:+ state = UsageStatsState(home=tmp_path)+ assert state.seen_notice_version(up_to=1) == 0+ assert state.record_notice(version=1)+ assert state.seen_notice_version(up_to=1) == 1+ assert not state.is_internal()+ state.internal_file.touch()+ assert state.is_internal()++ @pytest.mark.skipif(not CAN_LOCK_DIRECTORIES, reason="needs a directory this process cannot traverse")+ def test_no_reader_raises_for_a_state_that_cannot_be_looked_into(self, tmp_path: Path) -> None:+ """A directory above the state that may not be traversed: `disabled` fails closed, every other fact reads as absent."""+ state = UsageStatsState(home=tmp_path / "locked" / "home")+ with locked_directory(path=tmp_path / "locked"):+ assert state.is_disabled() is True+ assert state.is_internal() is False+ assert state.seen_notice_version(up_to=1) == 0+ assert state.notice_was_recorded(version=1) is False+ assert state.read_installation_id() is None+ assert state.obtain_installation_id().is_new+ assert state.ensure_state_dir() is False+ assert state.record_notice(version=1) is False+ assert state.legacy_opt_out_is_pending() is False+ state.migrate_legacy_opt_out()+ state.delete_every_spool()diff --git a/tests/unit/pipelex/system/usage_stats/test_usage_stats_switches.py b/tests/unit/pipelex/system/usage_stats/test_usage_stats_switches.pynew file mode 100644index 000000000..b14f3e310--- /dev/null+++ b/tests/unit/pipelex/system/usage_stats/test_usage_stats_switches.py@@ -0,0 +1,115 @@+from pathlib import Path++import pytest++from pipelex.system.usage_stats.usage_stats_contract import UsageStatsEnvVar, UsageStatsOffBy+from pipelex.system.usage_stats.usage_stats_core import UsageStatsState, is_loopback_origin+from pipelex.system.usage_stats.usage_stats_environment import declared_host, is_ci, is_debug, release_version, resolve_endpoint, tool_switch_says_off+from pipelex.system.usage_stats.usage_stats_switches import packaged_default_says_off, project_setting_says_off, read_switches, setting_says_off+from tests.helpers.usage_stats_process import CAN_LOCK_DIRECTORIES, locked_directory, make_project_dir+from tests.unit.pipelex.system.usage_stats.test_data import UsageStatsSwitchCases+++class TestUsageStatsSwitches:+ @pytest.mark.parametrize(("value", "says_off"), UsageStatsSwitchCases.TOOL_SWITCH_READINGS)+ def test_tool_switch_reading(self, value: str, says_off: bool) -> None:+ """`PIPELEX_USAGE_STATS` is trimmed and lowercased; an unknown value fails closed."""+ assert tool_switch_says_off(env={UsageStatsEnvVar.TOOL_SWITCH: value}) is says_off+ assert not tool_switch_says_off(env={})++ @pytest.mark.parametrize(("value", "is_on"), UsageStatsSwitchCases.FLAG_READINGS)+ def test_flag_reading(self, value: str, is_on: bool) -> None:+ """`DO_NOT_TRACK`, the debug variable and the CI variables are on unless empty, `0` or `false`."""+ assert is_debug(env={UsageStatsEnvVar.DEBUG: value}) is is_on+ assert is_ci(env={"GITHUB_ACTIONS": value}) is is_on+ off_by = read_switches(+ state=UsageStatsState(home=Path("/nonexistent-home")), env={UsageStatsEnvVar.DO_NOT_TRACK: value}, cwd=Path("/"), is_pytest=False+ )+ assert (UsageStatsOffBy.DO_NOT_TRACK in off_by) is is_on++ @pytest.mark.parametrize(("origin", "is_accepted"), UsageStatsSwitchCases.ENDPOINT_OVERRIDES)+ def test_endpoint_override_is_loopback_only(self, origin: str, is_accepted: bool) -> None:+ assert is_loopback_origin(origin=origin) is is_accepted+ assert resolve_endpoint(env={UsageStatsEnvVar.ENDPOINT_OVERRIDE: origin}) == (origin if is_accepted else None)+ assert resolve_endpoint(env={}) == "https://eu.i.posthog.com"++ def test_declared_host_and_release_version(self) -> None:+ assert declared_host(env={}) is None+ assert declared_host(env={UsageStatsEnvVar.HOST: " Cocode "}) == "cocode"+ assert declared_host(env={UsageStatsEnvVar.HOST: "my-own-tool"}) == "other"+ assert release_version(version="0.83.0") == "0.83.0"+ assert release_version(version="0.84.0rc1") == "0.84.0rc1"+ assert release_version(version="0.83.0+g1a2b3c.d20261011") == "0.83.0"+ assert release_version(version="not-a-version") == "0.0.0"++ @pytest.mark.parametrize(("content", "says_off"), UsageStatsSwitchCases.PROJECT_SETTINGS)+ def test_project_setting_fails_closed(self, tmp_path: Path, content: str | bytes, says_off: bool) -> None:+ """Only `false` turns statistics off, and so does a file nobody can read as the setting."""+ project_dir = make_project_dir(parent=tmp_path)+ config_path = project_dir / ".pipelex" / "pipelex.toml"+ config_path.parent.mkdir()+ if isinstance(content, bytes):+ config_path.write_bytes(content)+ else:+ config_path.write_text(content, encoding="utf-8")+ nested_dir = project_dir / "methods" / "nested"+ nested_dir.mkdir(parents=True)+ assert project_setting_says_off(cwd=nested_dir) is says_off+ assert setting_says_off(config_path=config_path) is says_off++ def test_project_setting_needs_a_project(self, tmp_path: Path) -> None:+ """No project around the directory, or no file in it, means no setting; a directory read as the file fails closed."""+ project_dir = make_project_dir(parent=tmp_path)+ assert not project_setting_says_off(cwd=project_dir)+ (project_dir / ".pipelex" / "pipelex.toml").mkdir(parents=True)+ assert project_setting_says_off(cwd=project_dir)++ @pytest.mark.skipif(not CAN_LOCK_DIRECTORIES, reason="needs a directory this process cannot traverse")+ def test_project_that_cannot_be_looked_into_fails_closed(self, tmp_path: Path) -> None:+ """A directory on the way up that may not be traversed hides the project's file, which counts as unreadable."""+ with locked_directory(path=tmp_path / "locked"):+ assert project_setting_says_off(cwd=tmp_path / "locked" / "project") is True+ state = UsageStatsState(home=tmp_path / "home")+ off_by = read_switches(state=state, env={}, cwd=tmp_path / "locked" / "project", is_pytest=False)+ assert UsageStatsOffBy.PROJECT_SETTING in off_by++ def test_home_configuration_is_not_a_switch(self, tmp_path: Path) -> None:+ """`~/.pipelex/pipelex.toml` is pipelex's home layer, never read for the setting."""+ home = tmp_path / "home"+ (home / ".pipelex").mkdir(parents=True)+ (home / ".pipelex" / "pipelex.toml").write_text("[usage_stats]\nenabled = false\n", encoding="utf-8")+ project_dir = make_project_dir(parent=tmp_path)+ off_by = read_switches(state=UsageStatsState(home=home), env={}, cwd=project_dir, is_pytest=False)+ assert off_by == []++ def test_packaged_default(self, tmp_path: Path) -> None:+ packaged = tmp_path / "pipelex.toml"+ packaged.write_text("[usage_stats]\nenabled = true\n", encoding="utf-8")+ assert not packaged_default_says_off(packaged_config_path=packaged)+ packaged.write_text("[usage_stats]\nenabled = false\n", encoding="utf-8")+ assert packaged_default_says_off(packaged_config_path=packaged)+ packaged.write_text("[runtime]\n", encoding="utf-8")+ assert packaged_default_says_off(packaged_config_path=packaged)+ assert packaged_default_says_off(packaged_config_path=tmp_path / "missing.toml")++ def test_every_switch_is_listed_in_the_contracts_order(self, tmp_path: Path) -> None:+ """No switch hides another: `status` names every one that says off, in order."""+ home = tmp_path / "home"+ state = UsageStatsState(home=home)+ state.disable()+ state.legacy_config.write_text("DISABLE_TELEMETRY=1\n", encoding="utf-8")+ project_dir = make_project_dir(parent=tmp_path)+ (project_dir / ".pipelex").mkdir()+ (project_dir / ".pipelex" / "pipelex.toml").write_text("[usage_stats]\nenabled = false\n", encoding="utf-8")+ packaged = tmp_path / "packaged.toml"+ packaged.write_text("[usage_stats]\nenabled = false\n", encoding="utf-8")+ env: dict[str, str] = {+ UsageStatsEnvVar.DO_NOT_TRACK: "1",+ UsageStatsEnvVar.TOOL_SWITCH: "off",+ UsageStatsEnvVar.ENDPOINT_OVERRIDE: "https://example.com",+ }+ off_by = read_switches(state=state, env=env, cwd=project_dir, is_pytest=True, packaged_config_path=packaged)+ assert off_by == list(UsageStatsOffBy)+ assert read_switches(state=UsageStatsState(home=tmp_path / "other-home"), env={}, cwd=project_dir, is_pytest=False) == [+ UsageStatsOffBy.PROJECT_SETTING+ ]