~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for the dynamic npm-update matrix in update-npm-dependencies.yml.
Coverage in two layers:
1. Structural ("guardian") tests parse the workflow YAML and pin the wiring
that makes the matrix dynamic: the ``discover`` job, its ``matrix``
output, the ``update`` job consuming
``fromJson(needs.discover.outputs.matrix)``, the ``set -euo pipefail``
and empty-matrix guard, the minimal ``contents: read`` permission,
``fail-fast: false`` on the per-directory legs, the read-only legs that
upload one ``npm-update-${{ matrix.npm_directory.slug }}`` artifact
instead of opening PRs, the single ``pr`` job that validates those
artifacts and opens ONE pull request per run, and the fail-loud
invalid-slug and slug-collision guards in discover (the slug is computed
once in discover's jq and names each leg's artifact). These prevent the
safety features from being silently removed — same philosophy as
``tests/ci/test_release_gate_integrity.py``.
2. Behavioral tests extract the discover step's real ``run:`` script from the
YAML and execute it against synthetic fixture trees via subprocess. This
proves the find|jq pipeline produces a correct matrix (repo root builds,
``tests/*`` dirs skip, excluded dirs ignored, a "slug" computed per entry)
and that the empty-matrix, unknown-path, invalid-slug, and slug-collision
guards all fail the job loudly — without duplicating the pipeline, so it
cannot drift from the workflow.
3. Hand-off tests run the legs' real pack script and the ``pr`` job's real
validate-and-apply script against fixture artifacts, including malicious
ones (``..`` members, extra files, links, unknown artifacts), to prove
only ``<dir>/package.json`` and ``<dir>/package-lock.json`` can ever
reach the combined PR.
"""
import json
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "update-npm-dependencies.yml"
_MAX_BRANCH_SLUG_LENGTH = 200
def _gnu_find_available() -> bool:
"""True if ``find`` supports the GNU ``-printf`` action.
Distinguishes GNU findutils (ubuntu-latest, the workflow's runner) from
BusyBox find (Alpine/containers), which lacks ``-printf`` — otherwise the
behavioral tests would fail unexpectedly on such hosts instead of skipping.
"""
try:
proc = subprocess.run(
["find", str(REPO_ROOT), "-maxdepth", "0", "-printf", "."],
capture_output=True,
text=True,
timeout=5,
)
return proc.returncode == 0 and proc.stdout == "."
except (OSError, subprocess.SubprocessError):
return False
# The discover pipeline relies on GNU find (``-printf``) and ``jq``, both
# present on the ubuntu-latest CI runner. Probe the platform, jq, AND the
# find variant so the tests skip cleanly on macOS/Windows AND on
# BusyBox-Linux (Alpine) rather than failing unexpectedly.
_REQUIRES_GNU = pytest.mark.skipif(
sys.platform != "linux"
or not shutil.which("jq")
or not _gnu_find_available(),
reason="discover pipeline needs GNU find (-printf) and jq (ubuntu-latest)",
)
@pytest.fixture(scope="module")
def workflow():
"""Load update-npm-dependencies.yml as parsed YAML."""
assert WORKFLOW.exists(), (
f"update-npm-dependencies.yml not found at {WORKFLOW}. "
"If you moved it, update the path — do NOT delete these tests."
)
return yaml.safe_load(WORKFLOW.read_text())
def _step_run(workflow: dict, job: str, step_id: str) -> str:
"""Return the ``run:`` script of ``job``'s step with ``id: step_id``."""
for step in workflow["jobs"][job]["steps"]:
if step.get("id") == step_id:
return step["run"]
raise AssertionError(f"{job} job has no step with id {step_id!r}")
def _discover_run(workflow: dict) -> str:
"""Return the discover job's ``list`` step ``run:`` script."""
for step in workflow["jobs"]["discover"]["steps"]:
if step.get("id") == "list":
return step["run"]
raise AssertionError(
"discover job has no step with id 'list' — cannot extract pipeline"
)
# ---------------------------------------------------------------------------
# Structural ("guardian") tests — pin the wiring, prevent silent removal
# ---------------------------------------------------------------------------
class TestWorkflowStructure:
"""Pin the dynamic-matrix wiring and safety features in the workflow."""
def test_discover_job_exists(self, workflow):
assert "discover" in workflow.get("jobs", {}), (
"The 'discover' job is missing — without it the matrix is no "
"longer generated dynamically and new lockfile dirs silently go "
"unswept (the regression PR #4997 fixed)."
)
def test_update_needs_discover(self, workflow):
needs = workflow["jobs"]["update"].get("needs", [])
assert "discover" in needs, (
"update.needs no longer includes 'discover' — the update job "
"would race ahead of (or skip) matrix generation."
)
def test_matrix_consumes_discover_output(self, workflow):
matrix = workflow["jobs"]["update"]["strategy"]["matrix"][
"npm_directory"
]
assert matrix == "${{ fromJson(needs.discover.outputs.matrix) }}", (
"The matrix no longer consumes the discover output dynamically; "
f"got: {matrix!r}"
)
def test_discover_exposes_matrix_output(self, workflow):
outputs = workflow["jobs"]["discover"].get("outputs", {})
assert outputs.get("matrix") == "${{ steps.list.outputs.matrix }}", (
"discover.outputs.matrix is missing/changed — update cannot read "
"the generated matrix."
)
def test_discover_uses_pipefail(self, workflow):
run = _discover_run(workflow)
assert "set -euo pipefail" in run, (
"discover no longer sets -euo pipefail — a mid-pipe failure in "
"the find|sed|jq chain could be silently swallowed."
)
def test_discover_has_empty_matrix_guard(self, workflow):
run = _discover_run(workflow)
# Pin the error message, not the comparison operator: the guard was
# hardened from a numeric "-eq 0"/"-ne 0" jq-length comparison (which
# `[` can error on for a non-integer substitution, silently skipping
# the guard) to a string comparison against "0", and both of
# discover's zero-length guards now contain that same `= "0"` shape
# — so pinning it here would stop distinguishing this guard from the
# invalid-slug guard below.
assert "No package-lock.json files discovered" in run and (
"exit 1" in run
), (
"The empty-matrix guard is gone — if find returns nothing, the "
"update job would silently no-op instead of failing loudly."
)
def test_discover_has_unknown_path_guard(self, workflow):
run = _discover_run(workflow)
assert "Unrecognized lockfile dir" in run, (
"The unknown-path guard is gone — a lockfile dir that is neither "
"the repo root nor under tests/ would silently enter the matrix "
"with no build/test, recreating the coverage gap this job exists "
"to prevent."
)
def test_discover_has_slug_collision_guard(self, workflow):
run = _discover_run(workflow)
assert "Colliding directory slugs" in run, (
"The slug-collision guard is gone from discover's list step — "
"two directories whose paths map to the same slug (e.g. "
"tests/a-b and tests/a/b, both -> tests-a-b) would upload the "
"same npm-update-<slug> artifact name, and the pr job could not "
"tell which directory an artifact belongs to."
)
def test_discover_has_invalid_slug_guard(self, workflow):
run = _discover_run(workflow)
assert "Invalid directory slug" in run, (
"The invalid-slug guard is gone from discover's list step — a "
"directory such as tests/a..b or 'tests/a b' would enter the "
"update matrix with a slug outside the conservative identifier "
"policy used for its artifact name."
)
def test_discover_keeps_paths_nul_delimited_until_json_encoding(
self, workflow
):
run = _discover_run(workflow)
assert "-printf '%h\\0'" in run, (
"discover no longer emits NUL-delimited paths — a newline in a "
"directory name could split one real path into fake matrix legs."
)
assert "read -r -d ''" in run, (
"discover no longer reads the NUL-delimited path stream safely."
)
def test_discover_permissions_are_read_only(self, workflow):
perms = workflow["jobs"]["discover"].get("permissions", {})
assert perms == {"contents": "read"}, (
"discover should hold the minimal contents:read permission; "
f"got {perms!r}. A discovery job must not be able to write."
)
def test_discover_checkout_is_pinned_and_credentialless(self, workflow):
for step in workflow["jobs"]["discover"]["steps"]:
uses = str(step.get("uses", ""))
if uses.startswith("actions/checkout@"):
assert step["with"].get("persist-credentials") is False, (
"discover checkout must use persist-credentials: false."
)
ref = uses.split("@", 1)[1]
assert len(ref) >= 40 and all(
c in "0123456789abcdef" for c in ref[:40]
), f"actions/checkout must be SHA-pinned, got @{ref}"
return
raise AssertionError("discover has no actions/checkout step")
def test_update_legs_are_read_only(self, workflow):
# The legs run third-party npm install/build code. They hand their
# result to the pr job as an artifact and must not be able to write
# the repo or open PRs themselves.
perms = workflow["jobs"]["update"].get("permissions", {})
assert perms == {"contents": "read"}, (
f"update legs must hold only contents:read; got {perms!r}."
)
def test_update_legs_do_not_fail_fast(self, workflow):
strategy = workflow["jobs"]["update"]["strategy"]
assert strategy.get("fail-fast") is False, (
"update.strategy.fail-fast must be false — the legs are "
"independent per-directory sweeps, and fail-fast lets one dir's "
"failure cancel all the others (run #36 lost 5 of 8 legs)."
)
def test_update_legs_do_not_create_pull_requests(self, workflow):
for step in workflow["jobs"]["update"]["steps"]:
assert not str(step.get("uses", "")).startswith(
"peter-evans/create-pull-request@"
), (
"an update leg creates a pull request again — that is one "
"PR per lockfile dir. Legs upload an artifact; the pr job "
"opens the single combined PR."
)
def test_update_legs_upload_one_artifact_named_by_slug(self, workflow):
steps = workflow["jobs"]["update"]["steps"]
uploads = [
s
for s in steps
if str(s.get("uses", "")).startswith("actions/upload-artifact@")
]
assert len(uploads) == 1, uploads
upload = uploads[0]
assert upload["uses"] == (
"actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9"
)
# Regex, not a substring: a brace typo such as
# ``${ matrix.npm_directory.slug }}`` still contains the substring
# but does not expand, so every leg would upload the same name.
assert re.fullmatch(
r"npm-update-\$\{\{\s*matrix\.npm_directory\.slug\s*\}\}",
upload["with"]["name"],
), upload["with"]["name"]
assert upload["with"]["if-no-files-found"] == "error"
assert int(upload["with"]["retention-days"]) <= 7
# The upload must sit directly after the pack step so it runs only
# when update, install, build and test all passed (no if: always()).
pack_index = next(
i for i, s in enumerate(steps) if s.get("id") == "pack"
)
assert steps[pack_index + 1] is upload
for step in steps[pack_index:]:
assert "always()" not in str(step.get("if", "")), step
def test_pack_step_only_packs_the_two_manifests(self, workflow):
run = _step_run(workflow, "update", "pack")
assert "for name in package.json package-lock.json; do" in run
assert "--no-recursion" in run, (
"pack must not recurse — a dir member would pull in arbitrary "
"files such as node_modules."
)
def test_pr_job_waits_for_legs_and_survives_leg_failures(self, workflow):
job = workflow["jobs"]["pr"]
assert set(job["needs"]) == {"discover", "update"}
condition = job["if"].replace(" ", "")
# A failed leg must not block the other dirs' updates, so the job
# needs an always()-style status function; without one a single
# failed leg skips the whole PR.
assert "!cancelled()" in condition or "always()" in condition, condition
assert "needs.discover.result=='success'" in condition, condition
def test_pr_job_alone_holds_write_permissions(self, workflow):
assert workflow["jobs"]["pr"]["permissions"] == {
"contents": "write",
"pull-requests": "write",
}
for name, job in workflow["jobs"].items():
if name == "pr":
continue
for scope, level in (job.get("permissions") or {}).items():
assert level != "write", (name, scope)
def test_pr_job_opens_one_pr_per_run(self, workflow):
steps = workflow["jobs"]["pr"]["steps"]
creates = [
s
for s in steps
if str(s.get("uses", "")).startswith(
"peter-evans/create-pull-request@"
)
]
assert len(creates) == 1, creates
create = creates[0]
branch = create["with"]["branch"]
# Two runs must not reuse one branch; nothing per-directory may be
# in the name any more.
assert branch == "update-npm-dependencies-${{ github.run_number }}", (
branch
)
assert create["if"] == "steps.apply.outputs.updated == 'true'"
add_paths = set(create["with"]["add-paths"].split())
assert add_paths == {
"package.json",
"package-lock.json",
"**/package.json",
"**/package-lock.json",
}, add_paths
assert create["with"]["sign-commits"] is True
assert create["with"]["base"] == "main"
# The body is built in a file from validated data, never by
# interpolating expressions into the PR body.
assert "body" not in create["with"]
assert "body-path" in create["with"]
def test_pr_job_downloads_and_validates_before_extracting(self, workflow):
steps = workflow["jobs"]["pr"]["steps"]
assert "harden-runner" in steps[0]["uses"]
download_index, download = next(
(i, s)
for i, s in enumerate(steps)
if str(s.get("uses", "")).startswith("actions/download-artifact@")
)
assert download["uses"] == (
"actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333"
)
assert download["with"]["pattern"] == "npm-update-*"
apply_index = next(
i for i, s in enumerate(steps) if s.get("id") == "apply"
)
assert download_index < apply_index
run = _step_run(workflow, "pr", "apply")
assert "disallowed member" in run and "non-regular member" in run
assert "--quoting-style=escape" in run
for step in steps:
assert "${{" not in (step.get("run") or ""), step.get("name")
# ---------------------------------------------------------------------------
# Behavioral tests — execute the REAL pipeline from the YAML (zero drift)
# ---------------------------------------------------------------------------
def _build_tree(root: Path, lockfiles):
"""Materialise a fixture repo containing the given package-lock.json paths."""
for rel in lockfiles:
p = root / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text("{}")
def _run_pipeline(run_script: str, root: Path, tmp_path: Path):
"""Run the extracted discover script with cwd=root, capturing GITHUB_OUTPUT."""
gh_output = tmp_path / "gh_output"
script_file = tmp_path / "discover.sh"
script_file.write_text(run_script)
return subprocess.run(
["bash", str(script_file)],
cwd=str(root),
env={**os.environ, "GITHUB_OUTPUT": str(gh_output)},
capture_output=True,
text=True,
)
def _matrix_from_output(tmp_path: Path) -> list[dict]:
gh_output = tmp_path / "gh_output"
for line in gh_output.read_text().splitlines():
if line.startswith("matrix="):
return json.loads(line[len("matrix=") :])
raise AssertionError(
"discover script wrote no matrix= line to GITHUB_OUTPUT"
)
def _assert_branch_safe_slug(slug: str, context) -> None:
"""Assert ``slug`` meets discover's conservative slug policy.
Slugs now name each leg's ``npm-update-<slug>`` artifact rather than a
PR branch, but the policy is still Git-ref shaped (see the comment
above discover's invalid-slug guard), so it is checked as a ref suffix.
Pure Python on purpose — it lets the test run without invoking
``git`` for every candidate slug. This encodes the subset of its
rules a slug can actually violate: the slug is a *suffix* of
``update-npm-dependencies-{run_number}-``, so rules about a ref
component's first character (a leading ``-`` or ``.``) cannot apply,
but ``..``, a trailing ``.``, and a ``.lock`` suffix all can — and all
three pass a bare ``[A-Za-z0-9._-]+`` charset check. A dir named
``tests/foo.lock`` or ``tests/a..b`` would slug cleanly yet make an
invalid ref. Git's syntax checker also permits a branch that
is too long for the runner filesystem to create with its temporary
``.lock`` suffix, so the policy caps slugs at 200 ASCII bytes.
Deliberately *stricter* than git in three ways. An empty slug and a
slug containing ``/`` both yield refs git would accept, but both mean
the transform is broken (an empty slug collapses every leg onto one
branch; a surviving ``/`` is the one character the transform exists to
remove). Beyond that the charset itself is conservative: git accepts
plenty of bytes it excludes (``+``, ``,``, ``@``, ``|``, non-ASCII
such as ``café``), so a lockfile dir using them would trip this
assertion despite producing a valid branch. That is the intended
trade-off — such a dir should be renamed rather than have the guard
loosened.
The production discover step enforces the same policy in jq. The
behavioral invalid-directory fixtures below execute that real script so
this Python mirror cannot be the only effective guard.
"""
# fullmatch, not match: "$" tolerates a trailing newline.
assert re.fullmatch(r"[A-Za-z0-9._-]+", slug), (context, slug)
assert ".." not in slug, (context, slug)
assert not slug.endswith("."), (context, slug)
assert not slug.endswith(".lock"), (context, slug)
assert len(slug) <= _MAX_BRANCH_SLUG_LENGTH, (context, slug)
class TestBranchSafeSlugHelper:
"""Unit-test the ref-validity rules themselves.
Without this, the helper is decorative: no live repo path or fixture
dir exercises the ``..`` / trailing-``.`` / ``.lock`` rules, so each
could be deleted with the rest of the suite still green. These cases
were cross-checked against the real ``git check-ref-format`` binary as
``refs/heads/update-npm-dependencies-42-<slug>`` — see the helper's
docstring for the three places the rules are stricter than git on
purpose.
"""
@pytest.mark.parametrize(
"slug",
[
"root",
"tests",
"tests-ui_tests-playwright",
"tests-api_tests_with_login",
"x.lockfile", # only an exact ".lock" suffix is invalid
"x.LOCK", # git's .lock rule is case-sensitive
"a.b", # a single dot is fine; only ".." is not
"a" * _MAX_BRANCH_SLUG_LENGTH, # exact policy boundary
],
)
def test_accepts_valid_slugs(self, slug):
_assert_branch_safe_slug(slug, "unit")
@pytest.mark.parametrize(
"slug",
[
"", # empty: every leg collapses onto one branch
"a/b", # "/" is what the transform exists to remove
"a..b", # git: ".." is never allowed in a ref
"a.", # git: a ref component may not end in "."
"a.lock", # git: a ref component may not end in ".lock"
"a b", # git: no spaces
"a~b", # git: no "~"
"a:b", # git: no ":"
"a" * (_MAX_BRANCH_SLUG_LENGTH + 1),
],
)
def test_rejects_invalid_slugs(self, slug):
with pytest.raises(AssertionError):
_assert_branch_safe_slug(slug, "unit")
@_REQUIRES_GNU
class TestPipelineBehavior:
"""Execute the real find|jq pipeline against fixture trees."""
def test_classifies_root_and_test_dirs(self, workflow, tmp_path):
root = tmp_path / "repo"
_build_tree(
root,
[
"package-lock.json", # repo root -> builds
"tests/alpha/package-lock.json", # test dir -> skip
"tests/beta/package-lock.json", # test dir -> skip
"node_modules/evil/package-lock.json", # excluded
".claude/evil/package-lock.json", # excluded (dot-dir)
".github/evil/package-lock.json", # excluded (dot-dir)
# excluded (dot-dir): a .venv can contain npm lockfiles
# generated at runtime by packages' own node bootstraps
# (e.g. readabilipy) — a repo-root .venv must not trip the
# unknown-dir guard on dev checkouts.
".venv/lib/pkg/js/package-lock.json",
],
)
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode == 0, proc.stderr
matrix = _matrix_from_output(tmp_path)
paths = [e["path"] for e in matrix]
assert paths == [".", "tests/alpha", "tests/beta"], paths
by_path = {e["path"]: e for e in matrix}
# Repo root is the only dir that builds and runs no test-skip echo.
assert by_path["."]["build_cmd"] == "npm run build"
assert by_path["."]["test_cmd"] == ""
# Every tests/* dir skips tests and does not build.
for d in ("tests/alpha", "tests/beta"):
assert by_path[d]["build_cmd"] == "", d
assert by_path[d]["test_cmd"].startswith("echo"), d
# Slugs: "." -> "root", "/" -> "-" for everything else.
assert by_path["."]["slug"] == "root"
assert by_path["tests/alpha"]["slug"] == "tests-alpha"
assert by_path["tests/beta"]["slug"] == "tests-beta"
# Excluded dirs (node_modules anywhere, dot-dirs at root) never
# appear in the matrix. The repo root "." is not a dot-dir.
assert not any(
"node_modules" in p or (p != "." and p.startswith("."))
for p in paths
), paths
# Every entry carries the keys the update job dereferences.
for e in matrix:
assert set(e) == {
"path",
"slug",
"build_cmd",
"test_cmd",
"cache_path",
}
assert e["cache_path"] == "package-lock.json"
def test_unknown_non_tests_path_fails_loudly(self, workflow, tmp_path):
# A lockfile dir that is neither the repo root nor under tests/ must
# FAIL discovery rather than silently entering the matrix with no
# build/test — that would recreate the coverage gap this job prevents.
root = tmp_path / "repo"
_build_tree(
root,
[
"package-lock.json", # root (valid)
"packages/app/package-lock.json", # unrecognized -> must fail
],
)
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode != 0, (
"an unrecognized lockfile dir must fail discovery, not silently "
"skip build/test"
)
# Pin the SPECIFIC failure mode so an unrelated shell error can't
# satisfy this test accidentally.
assert "Unrecognized lockfile dir" in proc.stdout, (
f"expected the unknown-path guard; got stdout={proc.stdout!r} "
f"stderr={proc.stderr!r}"
)
def test_colliding_slugs_fail_loudly(self, workflow, tmp_path):
# tests/a-b and tests/a/b both gsub("/", "-") to "tests-a-b" — two
# dirs would upload one artifact name and the pr job could not map
# it back to a single directory. Both are under tests/, so they pass the
# unknown-path guard cleanly; only the collision guard should fire.
# tests/a.b is also included: an interleaving path is what separated
# the duplicates under a locale-collating sort.
root = tmp_path / "repo"
_build_tree(
root,
[
"package-lock.json",
"tests/a-b/package-lock.json",
"tests/a.b/package-lock.json",
"tests/a/b/package-lock.json",
],
)
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode != 0, (
"colliding slugs must fail discovery, not silently enter the matrix"
)
# Pin the SPECIFIC failure mode so an unrelated shell error can't
# satisfy this test accidentally.
assert "Colliding directory slugs" in proc.stdout, (
f"expected the slug-collision guard; got stdout={proc.stdout!r} "
f"stderr={proc.stderr!r}"
)
@pytest.mark.parametrize(
"invalid_dir",
[
"tests/a..b", # git: ".." is never allowed in a ref
"tests/foo.", # git: a ref may not end in "."
"tests/foo.lock", # git: a ref may not end in ".lock"
"tests/a b", # git: no spaces
"tests/a+b", # valid to git; excluded by the ASCII policy
"tests/café", # valid to git; excluded by the ASCII policy
pytest.param(
"tests/a\ntests/b",
id="embedded-newline",
),
pytest.param(
"tests/" + "a" * (_MAX_BRANCH_SLUG_LENGTH - len("tests-") + 1),
id="slug-too-long",
),
],
)
def test_invalid_branch_slugs_fail_loudly(
self, workflow, tmp_path, invalid_dir
):
root = tmp_path / "repo"
_build_tree(
root,
[
"package-lock.json",
f"{invalid_dir}/package-lock.json",
],
)
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode != 0, (
"an invalid slug must fail discovery, not reach the update "
f"legs: dir={invalid_dir!r}"
)
assert "Invalid directory slug" in proc.stdout, (
f"expected the invalid-slug guard; got stdout={proc.stdout!r} "
f"stderr={proc.stderr!r}"
)
expected_path = json.dumps(invalid_dir, ensure_ascii=False)
expected_slug = json.dumps(
invalid_dir.replace("/", "-"), ensure_ascii=False
)
assert f"path={expected_path}" in proc.stdout, proc.stdout
assert f"slug={expected_slug}" in proc.stdout, proc.stdout
def test_empty_matrix_guard_fails_loudly(self, workflow, tmp_path):
# Only excluded dirs present => find yields nothing after filters,
# so the guard must fail the step rather than silently no-op.
root = tmp_path / "repo"
_build_tree(
root,
[
"node_modules/only/package-lock.json",
".github/only/package-lock.json",
],
)
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode != 0, (
"empty matrix must fail the discover step, not silently no-op"
)
assert "No package-lock.json files discovered" in proc.stdout, (
f"expected the empty-matrix guard; got stdout={proc.stdout!r} "
f"stderr={proc.stderr!r}"
)
def test_truly_empty_repo_fails(self, workflow, tmp_path):
root = tmp_path / "repo"
root.mkdir()
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode != 0, (
"a repo with no package-lock.json must fail the discover step"
)
assert "No package-lock.json files discovered" in proc.stdout, (
f"expected the empty-matrix guard; got stdout={proc.stdout!r} "
f"stderr={proc.stderr!r}"
)
# ---------------------------------------------------------------------------
# Real-repo invariant tests — guard against over-greedy exclusions without
# hardcoding a directory list (which would re-introduce the maintenance the
# PR removed and break spuriously whenever test dirs are added or removed).
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def real_repo_matrix(workflow, tmp_path_factory):
"""Run the pipeline against this repo ONCE and cache the parsed matrix.
Executing find over the whole tree (which may include node_modules) is
noticeably slower than the synthetic fixtures, so the three real-repo
invariant tests share this single result instead of spawning three runs.
"""
tmp = tmp_path_factory.mktemp("realrepo")
proc = _run_pipeline(_discover_run(workflow), REPO_ROOT, tmp)
assert proc.returncode == 0, proc.stderr
return _matrix_from_output(tmp)
@_REQUIRES_GNU
class TestPipelineAgainstRealRepo:
"""Run the real pipeline against this repository's actual tree.
Unlike TestPipelineBehavior (synthetic fixtures proving the logic in
isolation), these prove the *current* repo layout doesn't trip an edge
case — e.g. an exclusion pattern silently dropping a live lockfile dir,
or an unrecognized package dir slipping through unclassified.
Assertions are invariants, not an enumerated list, so they stay green
when test dirs are added or removed.
"""
def test_repo_root_is_discovered_and_marked_to_build(
self, real_repo_matrix
):
by_path = {e["path"]: e for e in real_repo_matrix}
assert "." in by_path, "repo root lockfile was not discovered"
root = by_path["."]
assert root["build_cmd"] == "npm run build", root
assert root["test_cmd"] == "", root
def test_every_non_root_dir_is_a_tests_dir_skip(self, real_repo_matrix):
# Convention: only "." builds; every OTHER lockfile dir must be a
# tests/ dir that skips (the discovery guard rejects anything else).
# Asserting the tests/ membership codifies the documented convention
# rather than the looser "anything non-root skips" behaviour.
for e in real_repo_matrix:
if e["path"] == ".":
continue
assert e["path"] == "tests" or e["path"].startswith("tests/"), e
assert e["build_cmd"] == "", e
assert e["test_cmd"].startswith("echo"), e
def test_discovers_more_than_just_the_root(self, real_repo_matrix):
# Sanity lower bound: a wipeout from an over-greedy exclusion would
# collapse the matrix to at most the root entry.
assert len(real_repo_matrix) >= 2, (
f"expected root + >=1 tests dir, got {len(real_repo_matrix)}: "
f"{[e['path'] for e in real_repo_matrix]}"
)
def test_slug_matches_expected_formula_and_is_unique(
self, real_repo_matrix
):
# Pure-Python check of the documented "." -> "root", "/" -> "-"
# formula against the REAL matrix output (not a re-implementation of
# the discover pipeline itself) — confirms the live repo's slugs
# match convention, fall in a branch-name-safe charset, and that no
# two live dirs collide onto one branch.
#
# The '/'->'-' mapping is not injective (tests/foo-bar and
# tests/foo/bar both slug to tests-foo-bar); no such pair exists
# today, and this invariant keeps it that way as lockfile dirs are
# added — the discover job itself also fails loudly on this (see
# TestWorkflowStructure.test_discover_has_slug_collision_guard and
# TestPipelineBehavior.test_colliding_slugs_fail_loudly).
slugs = []
for e in real_repo_matrix:
expected = (
"root" if e["path"] == "." else e["path"].replace("/", "-")
)
assert e["slug"] == expected, e
_assert_branch_safe_slug(e["slug"], e["path"])
slugs.append(e["slug"])
assert len(set(slugs)) == len(slugs), (
f"two live dirs slug to the same branch: {real_repo_matrix}"
)
@_REQUIRES_GNU
class TestSlugBehavior:
"""Derive branch slugs from the REAL executed discover pipeline.
The per-directory branch slug used to be produced by a separate sed
transform simulated in these tests; it is now computed once, in
discover's jq, as ``matrix.npm_directory.slug``. This test reads
slugs off the matrix output produced by actually running the find|jq
pipeline (see ``_run_pipeline``), so it cannot drift from the
workflow.
Scope, deliberately: this class asserts only *properties* every valid
slug must hold — ``.`` maps to ``root``, the charset is branch-safe,
and no two dirs collide. Those properties hold for any injective
transform, so this class alone would NOT catch the transform changing
(e.g. ``/`` -> ``_``). The intended transform is pinned separately, by
the hardcoded expectations in
``TestPipelineBehavior.test_classifies_root_and_test_dirs`` and the
formula check in ``TestPipelineAgainstRealRepo``'s
``test_slug_matches_expected_formula_and_is_unique``; a collision is
caught behaviorally by
``TestPipelineBehavior.test_colliding_slugs_fail_loudly``.
"""
def test_slug_maps_fixture_dirs_to_unique_valid_refs(
self, workflow, tmp_path
):
root = tmp_path / "repo"
max_length_dir = "tests/" + "a" * (
_MAX_BRANCH_SLUG_LENGTH - len("tests-")
)
_build_tree(
root,
[
"package-lock.json",
"tests/package-lock.json",
"tests/ui_tests/package-lock.json",
"tests/ui_tests/playwright/package-lock.json",
"tests/api_tests_with_login/package-lock.json",
"tests/x.lockfile/package-lock.json",
"tests/x.LOCK/package-lock.json",
"tests/a.b/package-lock.json",
f"{max_length_dir}/package-lock.json",
],
)
proc = _run_pipeline(_discover_run(workflow), root, tmp_path)
assert proc.returncode == 0, proc.stderr
matrix = _matrix_from_output(tmp_path)
slugs = {e["path"]: e["slug"] for e in matrix}
assert slugs["."] == "root", slugs
assert len(slugs[max_length_dir]) == _MAX_BRANCH_SLUG_LENGTH
assert len(set(slugs.values())) == len(slugs), (
f"slug collision — two dirs would share a branch: {slugs}"
)
for path, slug in slugs.items():
# "/" must be gone — it is the one character the transform
# exists to remove, and the charset check below would catch it
# anyway; asserted explicitly so a regression names the cause.
assert "/" not in slug, (path, slug)
_assert_branch_safe_slug(slug, path)
# ---------------------------------------------------------------------------
# Hand-off tests — the legs' real pack script and the pr job's real
# validate-and-apply script, run against fixture artifacts
# ---------------------------------------------------------------------------
def _gnu_tar_available() -> bool:
try:
proc = subprocess.run(
["tar", "--version"], capture_output=True, text=True, timeout=5
)
return proc.returncode == 0 and "GNU tar" in proc.stdout
except (OSError, subprocess.SubprocessError):
return False
_REQUIRES_GNU_TAR = pytest.mark.skipif(
sys.platform != "linux"
or not shutil.which("jq")
or not _gnu_tar_available(),
reason="pack/apply scripts need GNU tar and jq (ubuntu-latest)",
)
_HANDOFF_MATRIX = [
{"path": ".", "slug": "root"},
{"path": "tests/a", "slug": "tests-a"},
{"path": "tests/b", "slug": "tests-b"},
{"path": "tests/c", "slug": "tests-c"},
]
def _run_script(script: str, cwd: Path, tmp_path: Path, env: dict):
script_file = tmp_path / f"script-{len(list(tmp_path.iterdir()))}.sh"
script_file.write_text(script)
return subprocess.run(
["bash", str(script_file)],
cwd=str(cwd),
env={**os.environ, "GITHUB_OUTPUT": str(tmp_path / "gh_output"), **env},
capture_output=True,
text=True,
)
def _pack(workflow, checkout: Path, tmp_path: Path, npm_dir: str, slug: str):
"""Run the leg's pack step; return the artifact directory it filled."""
out_dir = tmp_path / "artifacts" / f"npm-update-{slug}"
proc = _run_script(
_step_run(workflow, "update", "pack"),
checkout,
tmp_path,
{"NPM_DIR": npm_dir, "CHANGED": "true", "OUT_DIR": str(out_dir)},
)
assert proc.returncode == 0, proc.stderr
return out_dir
def _checkout(tmp_path: Path) -> Path:
checkout = tmp_path / "checkout"
for d in ("", "tests/a", "tests/b", "tests/c"):
(checkout / d).mkdir(parents=True, exist_ok=True)
(checkout / d / "package.json").write_text('{"v": "old"}')
(checkout / d / "package-lock.json").write_text('{"v": "old"}')
return checkout
def _apply(workflow, checkout: Path, tmp_path: Path, artifacts: Path):
body = tmp_path / "body.md"
(tmp_path / "gh_output").write_text("")
proc = _run_script(
_step_run(workflow, "pr", "apply"),
checkout,
tmp_path,
{
"MATRIX": json.dumps(_HANDOFF_MATRIX),
"UPDATE_RESULT": "failure",
"ARTIFACTS_DIR": str(artifacts),
"BODY_FILE": str(body),
"RUN_URL": "https://example.invalid/run",
"GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"),
},
)
output = (tmp_path / "gh_output").read_text()
return proc, output, body.read_text() if body.exists() else ""
def _write_tar(target: Path, members):
"""Write a tar with ``members``: (name, kind, payload) tuples."""
import io
import tarfile
target.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(target, "w", format=tarfile.GNU_FORMAT) as tar:
for name, kind, payload in members:
info = tarfile.TarInfo(name)
if kind == "file":
data = payload.encode()
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
elif kind == "symlink":
info.type = tarfile.SYMTYPE
info.linkname = payload
tar.addfile(info)
elif kind == "hardlink":
info.type = tarfile.LNKTYPE
info.linkname = payload
tar.addfile(info)
elif kind == "dir":
info.type = tarfile.DIRTYPE
tar.addfile(info)
@_REQUIRES_GNU_TAR
class TestArtifactHandoff:
"""Pack in the legs, validate and apply in the pr job — for real."""
def test_round_trip_applies_updates_and_names_failed_dirs(
self, workflow, tmp_path
):
leg = tmp_path / "leg"
for d in ("", "tests/a"):
(leg / d).mkdir(parents=True, exist_ok=True)
(leg / d / "package.json").write_text(f'{{"v": "new {d}"}}')
(leg / d / "package-lock.json").write_text(f'{{"v": "new {d}"}}')
(leg / d / "node_modules").mkdir()
(leg / d / "stray.js").write_text("not packed")
_pack(workflow, leg, tmp_path, ".", "root")
_pack(workflow, leg, tmp_path, "tests/a", "tests-a")
unchanged = tmp_path / "artifacts" / "npm-update-tests-b"
unchanged.mkdir()
(unchanged / "unchanged").write_text("unchanged\n")
# tests/c uploaded nothing: its leg failed.
checkout = _checkout(tmp_path)
proc, output, body = _apply(
workflow, checkout, tmp_path, tmp_path / "artifacts"
)
assert proc.returncode == 0, proc.stdout + proc.stderr
assert "updated=true" in output
assert (checkout / "package.json").read_text() == '{"v": "new "}'
assert (checkout / "tests/a/package-lock.json").read_text() == (
'{"v": "new tests/a"}'
)
assert (checkout / "tests/b/package.json").read_text() == (
'{"v": "old"}'
)
assert not (checkout / "stray.js").exists()
assert not (checkout / "tests/a/stray.js").exists()
updated, _, rest = body.partition("**Not included")
assert "- `.`" in updated and "- `tests/a`" in updated
assert "tests/b" not in body
assert "- `tests/c`" in rest
def test_no_artifacts_skips_pr_cleanly(self, workflow, tmp_path):
checkout = _checkout(tmp_path)
proc, output, body = _apply(
workflow, checkout, tmp_path, tmp_path / "missing"
)
assert proc.returncode == 0, proc.stdout + proc.stderr
assert "updated=false" in output
for entry in _HANDOFF_MATRIX:
assert f"- `{entry['path']}`" in body
@pytest.mark.parametrize(
("members", "message"),
[
pytest.param(
[
("tests/a/package.json", "file", "{}"),
("../x", "file", "pwned"),
],
"disallowed member",
id="parent-traversal",
),
pytest.param(
[("tests/a/../../x", "file", "pwned")],
"disallowed member",
id="embedded-traversal",
),
pytest.param(
[("/tmp/x", "file", "pwned")],
"disallowed member",
id="absolute-path",
),
pytest.param(
[
("tests/a/package.json", "file", "{}"),
("tests/a/evil.js", "file", "pwned"),
],
"disallowed member",
id="extra-file",
),
pytest.param(
[("tests/b/package.json", "file", "{}")],
"disallowed member",
id="other-directory",
),
pytest.param(
[("package.json", "file", "{}")],
"disallowed member",
id="root-file-from-subdir-leg",
),
pytest.param(
[("tests/a/package.json", "symlink", "/etc/passwd")],
"non-regular member",
id="symlink",
),
pytest.param(
[
("tests/a/package-lock.json", "file", "{}"),
(
"tests/a/package.json",
"hardlink",
"tests/a/package-lock.json",
),
],
"non-regular member",
id="hardlink",
),
pytest.param(
[("tests/a/package.json", "dir", None)],
"non-regular member",
id="directory",
),
pytest.param(
[
(
"tests/a/package.json\ntests/a/package-lock.json",
"file",
"{}",
)
],
"disallowed member",
id="newline-in-name",
),
pytest.param(
[
("tests/a/package.json", "file", "{}"),
("tests/a/package.json", "file", "{}"),
],
"repeats a member",
id="duplicate-member",
),
pytest.param([], "empty or unreadable", id="empty-archive"),
],
)
def test_malicious_archive_is_rejected_before_extraction(
self, workflow, tmp_path, members, message
):
artifacts = tmp_path / "artifacts"
_write_tar(artifacts / "npm-update-tests-a" / "update.tar", members)
checkout = _checkout(tmp_path)
proc, output, _ = _apply(workflow, checkout, tmp_path, artifacts)
assert proc.returncode != 0, proc.stdout
assert message in proc.stdout, proc.stdout + proc.stderr
assert "updated=" not in output
assert not (tmp_path / "x").exists()
assert not (checkout / "tests/a/evil.js").exists()
assert (checkout / "tests/a/package.json").read_text() == (
'{"v": "old"}'
)
assert not (checkout / "tests/a/package.json").is_symlink()
@pytest.mark.parametrize(
("layout", "message"),
[
pytest.param(
{"npm-update-evil/unchanged": "x"},
"Unexpected artifact entry",
id="unknown-slug",
),
pytest.param(
{"stray/unchanged": "x"},
"Unexpected artifact entry",
id="not-an-npm-update-artifact",
),
pytest.param(
{
"npm-update-tests-a/unchanged": "x",
"npm-update-tests-a/evil.js": "x",
},
"exactly one regular file",
id="extra-file-beside-marker",
),
pytest.param(
{"npm-update-tests-a/.hidden": "x"},
"Unexpected file in artifact",
id="hidden-file",
),
pytest.param(
{"npm-update-tests-a/package.json": "{}"},
"Unexpected file in artifact",
id="loose-manifest",
),
],
)
def test_malformed_artifact_layout_is_rejected(
self, workflow, tmp_path, layout, message
):
artifacts = tmp_path / "artifacts"
for rel, content in layout.items():
(artifacts / rel).parent.mkdir(parents=True, exist_ok=True)
(artifacts / rel).write_text(content)
checkout = _checkout(tmp_path)
proc, output, _ = _apply(workflow, checkout, tmp_path, artifacts)
assert proc.returncode != 0, proc.stdout
assert message in proc.stdout, proc.stdout + proc.stderr
assert "updated=" not in output
def test_symlinked_artifact_file_is_rejected(self, workflow, tmp_path):
artifacts = tmp_path / "artifacts"
(artifacts / "npm-update-tests-a").mkdir(parents=True)
(artifacts / "npm-update-tests-a" / "update.tar").symlink_to(
"/etc/passwd"
)
checkout = _checkout(tmp_path)
proc, _, _ = _apply(workflow, checkout, tmp_path, artifacts)
assert proc.returncode != 0, proc.stdout
assert "exactly one regular file" in proc.stdout