~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Dependency PR reconciliation tests using local metadata fixtures."""
from copy import deepcopy
import json
import os
from pathlib import Path
import runpy
from types import SimpleNamespace
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
# Keep generated bytecode out of the action files scanned by CI policy tests.
ci = SimpleNamespace(
**runpy.run_path(
str(
ROOT / ".github/actions/request-dependency-ci/scripts/request_ci.py"
)
)
)
HEAD = "a" * 40
BRANCH = "update-npm-dependencies-52-tests-infrastructure_tests"
CORE_APPROVALS = [1, 2, 3, *range(20, 27)]
def workflow_run(number, filename, conclusion="action_required", **changes):
return {
"id": number,
"path": f".github/workflows/{filename}",
"head_sha": HEAD,
"head_branch": BRANCH,
"head_repository": {"full_name": "example/project"},
"pull_requests": [{"number": 7740}],
"event": "pull_request",
"status": "in_progress" if conclusion is None else "completed",
"conclusion": conclusion,
**changes,
}
def core_runs():
return [
workflow_run(1, "docker-tests.yml"),
workflow_run(2, "pre-commit.yml"),
workflow_run(3, "mypy-type-check.yml"),
*[
workflow_run(number, filename)
for number, filename in enumerate(
[
"codeql.yml",
"security-file-write-check.yml",
"check-env-vars.yml",
"gitleaks.yml",
"osv-scanner.yml",
"file-whitelist-check.yml",
"dependency-review.yml",
],
20,
)
],
]
class GitHubFixture:
def __init__(self, snapshots=None):
self.pr = {
"number": 7740,
"state": "open",
"base": {"ref": "main"},
"user": {"login": "github-actions[bot]"},
"head": {
"ref": BRANCH,
"sha": HEAD,
"repo": {"full_name": "example/project"},
},
}
self.files = [
{"filename": "tests/infrastructure_tests/package-lock.json"}
]
self.snapshots = snapshots if snapshots is not None else [core_runs()]
self.polls = 0
self.approved = []
self.labels = []
self.mutations = []
self.extra_runs = []
self.failure = None
self.no_pr = False
self.move_on_poll = None
self.queries = []
self.jobs = {}
def pages(self, route, key=None):
self.queries.append(route)
if "/pulls?" in route:
return [] if self.no_pr else [deepcopy(self.pr)]
if route.endswith("/files"):
return deepcopy(self.files)
if route.endswith("/jobs"):
return self.jobs.get(
int(route.split("/")[-2]), [{"conclusion": None}]
)
if "/actions/runs?" in route:
if self.polls == self.move_on_poll:
self.pr["head"]["sha"] = "b" * 40
index = min(self.polls, len(self.snapshots) - 1)
runs = deepcopy(self.snapshots[index] + self.extra_runs)
self.polls += 1
for run in runs:
if run["id"] in self.approved:
run["conclusion"] = None
return runs
raise AssertionError(route)
def request(self, method, route, payload=None):
if method != "GET":
self.mutations.append((method, route, payload))
if self.failure and self.failure[0] == method:
_, status = self.failure
self.failure = None
raise ci.APIError(method, status)
if method == "GET" and "/pulls/" in route:
return deepcopy(self.pr)
if method == "POST" and route.endswith("/approve"):
self.approved.append(int(route.split("/")[-2]))
return {}
if method == "DELETE":
raise ci.APIError(method, 404)
if method == "POST" and route.endswith("/labels"):
label = payload["labels"][0]
self.labels.append(label)
workflows = {
"test:puppeteer": "puppeteer-e2e-tests.yml",
"test:ui-full-shards": "ui-full-shards.yml",
"test:notes": "playwright-notes-tests.yml",
"test:webkit": "playwright-webkit-tests.yml",
}
self.extra_runs.append(
workflow_run(100 + len(self.labels), workflows[label], None)
)
return {}
raise AssertionError((method, route))
def reconcile(client, **changes):
options = {
"directory": "tests/infrastructure_tests",
"attempts": 8,
"sleep": lambda _: None,
}
options.update(changes)
return ci.reconcile(client, "example/project", BRANCH, "npm", **options)
class DependencyCITests(unittest.TestCase):
def test_standard_checks_are_approved(self):
client = GitHubFixture()
reconcile(client)
self.assertEqual(client.approved, CORE_APPROVALS)
self.assertEqual(client.labels, [])
self.assertGreaterEqual(client.polls, 3)
self.assertTrue(
all(
f"head_sha={HEAD}" in route and "event=pull_request" in route
for route in client.queries
if "/actions/runs?" in route
)
)
def test_delayed_workflows_are_discovered(self):
runs = core_runs()
client = GitHubFixture(
[
[],
runs[:1],
runs,
runs + [workflow_run(4, "check-workflow-status.yml")],
]
)
reconcile(client)
self.assertEqual(client.approved, CORE_APPROVALS + [4])
def test_missing_core_workflows_do_not_report_success(self):
for runs in ([], core_runs()[:1], core_runs()[:2], core_runs()[:-1]):
with self.subTest(runs=runs):
with self.assertRaisesRegex(
RuntimeError, "did not become ready"
):
reconcile(GitHubFixture([runs]), attempts=3)
def test_other_revisions_and_events_are_ignored(self):
runs = core_runs() + [
workflow_run(10, "other.yml", head_sha="b" * 40),
workflow_run(11, "other.yml", head_branch="other"),
workflow_run(12, "other.yml", event="push"),
workflow_run(
13, "other.yml", head_repository={"full_name": "example/other"}
),
workflow_run(14, "other.yml", pull_requests=[{"number": 42}]),
workflow_run(15, "other.yml", head_repository=None),
workflow_run(16, "other.yml", pull_requests=[]),
]
client = GitHubFixture([runs])
reconcile(client)
self.assertEqual(client.approved, CORE_APPROVALS)
def test_finished_checks_are_not_restarted(self):
runs = [dict(run, conclusion="failure") for run in core_runs()]
client = GitHubFixture([runs])
reconcile(client)
self.assertEqual(client.approved, [])
def test_other_pr_runs_cannot_satisfy_registration(self):
runs = [
dict(run, pull_requests=[{"number": 42}]) for run in core_runs()
]
client = GitHubFixture([runs])
with self.assertRaisesRegex(RuntimeError, "did not become ready"):
reconcile(client, attempts=3)
self.assertEqual(client.approved, [])
def test_late_baseline_workflow_is_still_reconciled(self):
runs = core_runs()
early = [run for run in runs if run["id"] != 20]
client = GitHubFixture([early, early, early, runs])
reconcile(client)
self.assertCountEqual(client.approved, CORE_APPROVALS)
self.assertGreaterEqual(client.polls, 6)
def test_unchanged_update_can_retry_failed_approval(self):
client = GitHubFixture()
client.failure = ("POST", 500)
with self.assertRaises(ci.APIError):
reconcile(client)
reconcile(client)
self.assertEqual(client.approved, CORE_APPROVALS)
def test_no_open_pr_is_a_successful_noop(self):
client = GitHubFixture()
client.no_pr = True
self.assertIn("No open", reconcile(client))
self.assertEqual(client.polls, 0)
def test_pr_metadata_must_match(self):
changes = [
{"state": "closed"},
{"base": {"ref": "other"}},
{"user": {"login": "contributor"}},
{"head": {"ref": BRANCH, "sha": HEAD, "repo": None}},
]
for change in changes:
with self.subTest(change=change):
client = GitHubFixture()
client.pr.update(change)
with self.assertRaisesRegex(RuntimeError, "metadata"):
reconcile(client)
self.assertEqual(client.approved, [])
def test_expected_manifest_files_are_required(self):
for files in (
[],
[{"filename": "src/module.py"}],
[
{
"filename": "tests/infrastructure_tests/package.json",
"previous_filename": "other.json",
}
],
):
with self.subTest(files=files):
client = GitHubFixture()
client.files = files
with self.assertRaisesRegex(RuntimeError, "PR files"):
reconcile(client)
def test_changed_revision_stops_reconciliation(self):
client = GitHubFixture()
client.move_on_poll = 0
with self.assertRaisesRegex(RuntimeError, "revision changed"):
reconcile(client)
def test_browser_projects_select_matching_suites(self):
projects = {
"tests": {"test:ui-full-shards"},
"tests/puppeteer": {"test:puppeteer"},
"tests/ui_tests": {"test:ui-full-shards"},
"tests/ui_tests/playwright": {"test:notes", "test:webkit"},
}
for directory, labels in projects.items():
with self.subTest(directory=directory):
client = GitHubFixture()
client.files = [{"filename": f"{directory}/package-lock.json"}]
reconcile(client, directory=directory)
self.assertEqual(set(client.labels), labels)
def test_existing_browser_run_is_reused(self):
runs = core_runs() + [workflow_run(4, "puppeteer-e2e-tests.yml", None)]
client = GitHubFixture([runs])
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
reconcile(client, directory="tests/puppeteer")
self.assertEqual(client.labels, [])
self.assertEqual(client.approved, CORE_APPROVALS)
def test_combined_npm_changes_select_matching_suites(self):
client = GitHubFixture()
client.files = [
{"filename": "package-lock.json"},
{"filename": "tests/puppeteer/package-lock.json"},
{"filename": "tests/ui_tests/playwright/package.json"},
]
reconcile(
client,
directories=[".", "tests/puppeteer", "tests/ui_tests/playwright"],
)
self.assertEqual(
set(client.labels), {"test:puppeteer", "test:notes", "test:webkit"}
)
def test_combined_npm_changes_must_match_discovered_directories(self):
client = GitHubFixture()
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
with self.assertRaisesRegex(RuntimeError, "PR files"):
reconcile(client, directories=["."])
self.assertEqual(client.approved, [])
def test_combined_npm_directory_list_is_validated(self):
for directories in ([], "tests", [{}], ["src"]):
with self.subTest(directories=directories):
with self.assertRaises(ValueError):
reconcile(GitHubFixture(), directories=directories)
def test_skipped_browser_runs_do_not_replace_test_selection(self):
runs = core_runs() + [
workflow_run(4, "puppeteer-e2e-tests.yml", "skipped")
]
client = GitHubFixture([runs])
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
reconcile(client, directory="tests/puppeteer")
self.assertEqual(client.labels, ["test:puppeteer"])
def test_queued_browser_jobs_are_already_selected(self):
runs = core_runs() + [
workflow_run(4, "puppeteer-e2e-tests.yml", None, status="queued")
]
client = GitHubFixture([runs])
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
reconcile(client, directory="tests/puppeteer")
self.assertEqual(client.labels, [])
def test_browser_workflow_with_only_skipped_jobs_needs_selection(self):
runs = core_runs() + [workflow_run(4, "puppeteer-e2e-tests.yml", None)]
client = GitHubFixture([runs])
client.jobs[4] = [{"conclusion": "skipped"}]
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
reconcile(client, directory="tests/puppeteer")
self.assertEqual(client.labels, ["test:puppeteer"])
def test_unapproved_browser_run_still_needs_test_selection(self):
runs = core_runs() + [workflow_run(4, "puppeteer-e2e-tests.yml")]
client = GitHubFixture([runs])
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
reconcile(client, directory="tests/puppeteer")
self.assertEqual(client.labels, ["test:puppeteer"])
self.assertEqual(client.approved, CORE_APPROVALS + [4])
def test_browser_selection_can_retry_label_failure(self):
client = GitHubFixture()
client.files = [{"filename": "tests/puppeteer/package-lock.json"}]
client.failure = ("DELETE", 503)
with self.assertRaises(ci.APIError):
reconcile(client, directory="tests/puppeteer")
reconcile(client, directory="tests/puppeteer")
self.assertEqual(client.labels, ["test:puppeteer"])
def test_grouped_dependabot_changes_select_all_matching_suites(self):
client = GitHubFixture()
client.pr["user"]["login"] = "dependabot[bot]"
client.files = [
{"filename": "tests/puppeteer/package-lock.json"},
{"filename": "tests/ui_tests/playwright/package.json"},
]
ci.reconcile(
client,
"example/project",
BRANCH,
"dependabot",
number=7740,
expected_head=HEAD,
attempts=8,
sleep=lambda _: None,
)
self.assertEqual(
set(client.labels), {"test:puppeteer", "test:notes", "test:webkit"}
)
self.assertEqual(client.approved, CORE_APPROVALS)
def test_dependabot_manifest_updates_are_approved(self):
for files in (
["pyproject.toml", "pdm.lock"],
[
"requirements/pre-commit-update.in",
"requirements/pre-commit-update.txt",
],
["requirements/bootstrap-runtime.txt"],
["package.json", "package-lock.json"],
[
"tests/accessibility_tests/package-lock.json",
"tests/api_tests_with_login/package.json",
"tests/infrastructure_tests/package-lock.json",
],
):
with self.subTest(files=files):
client = GitHubFixture()
client.pr["user"]["login"] = "dependabot[bot]"
client.files = [{"filename": filename} for filename in files]
result = ci.reconcile(
client,
"example/project",
BRANCH,
"dependabot",
attempts=8,
sleep=lambda _: None,
)
self.assertIn("registered and approved", result)
self.assertEqual(client.approved, CORE_APPROVALS)
self.assertEqual(client.labels, [])
def test_unlisted_dependabot_files_wait_for_manual_approval(self):
for filename in (
".github/workflows/pre-commit.yml",
".github/actions/example/action.yml",
"src/module.py",
"README.md",
"Dockerfile",
"requirements/README.md",
"requirements/new-tool.txt",
"requirements/pip-vendor-refresh.txt",
"other/package-lock.json",
):
for mixed in (False, True):
with self.subTest(filename=filename, mixed=mixed):
client = GitHubFixture()
client.pr["user"]["login"] = "dependabot[bot]"
client.files = [{"filename": filename}]
if mixed:
client.files.append(
{"filename": "tests/puppeteer/package-lock.json"}
)
result = ci.reconcile(
client,
"example/project",
BRANCH,
"dependabot",
attempts=8,
sleep=lambda _: None,
)
self.assertIn("manual CI approval", result)
self.assertEqual(client.polls, 0)
self.assertEqual(client.mutations, [])
def test_dependabot_renames_include_both_paths_in_the_policy(self):
client = GitHubFixture()
client.pr["user"]["login"] = "dependabot[bot]"
client.files = [
{
"filename": "tests/puppeteer/package-lock.json",
"previous_filename": "requirements/README.md",
}
]
result = ci.reconcile(
client,
"example/project",
BRANCH,
"dependabot",
attempts=8,
sleep=lambda _: None,
)
self.assertIn("manual CI approval", result)
self.assertEqual(client.polls, 0)
self.assertEqual(client.mutations, [])
def test_custom_updaters_only_approve_their_expected_manifests(self):
for ecosystem, manifest in (
("npm", "package-lock.json"),
("pdm", "pdm.lock"),
("pre-commit", ".pre-commit-config.yaml"),
):
with self.subTest(ecosystem=ecosystem):
client = GitHubFixture()
client.files = [
{"filename": manifest},
{"filename": ".github/workflows/pre-commit.yml"},
]
with self.assertRaisesRegex(RuntimeError, "PR files"):
ci.reconcile(client, "example/project", BRANCH, ecosystem)
self.assertEqual(client.polls, 0)
self.assertEqual(client.mutations, [])
def test_stale_dependabot_event_is_a_successful_noop(self):
client = GitHubFixture()
client.pr["user"]["login"] = "dependabot[bot]"
result = ci.reconcile(
client,
"example/project",
BRANCH,
"dependabot",
number=7740,
expected_head="b" * 40,
)
self.assertIn("newer PR revision", result)
self.assertEqual(client.polls, 0)
def test_npm_directory_is_validated(self):
for directory in ("/tmp", "../other", "src", "tests/../src"):
with self.subTest(directory=directory):
with self.assertRaises(ValueError):
ci.expected_files("npm", directory)
def test_other_updaters_have_their_expected_manifests(self):
self.assertEqual(ci.expected_files("pdm", "."), {"pdm.lock"})
self.assertEqual(
ci.expected_files("pre-commit", "."), {".pre-commit-config.yaml"}
)
def test_missing_credential_is_explicit(self):
with patch.dict(os.environ, {}, clear=True):
with self.assertRaisesRegex(RuntimeError, "credential is required"):
ci.main()
def test_api_payload_uses_structured_input(self):
with patch.object(ci.subprocess, "run") as run:
run.return_value.returncode = 0
run.return_value.stdout = "{}"
ci.GitHub().request("POST", "repos/example/project", {"labels": []})
self.assertEqual(
json.loads(run.call_args.kwargs["input"]), {"labels": []}
)
self.assertNotIn("shell", run.call_args.kwargs)
def test_workflow_request_jobs_are_independent_of_installer_jobs(self):
for name in (
"update-npm-dependencies.yml",
"update-dependencies.yml",
"update-precommit-hooks.yml",
):
with self.subTest(workflow=name):
workflow = (ROOT / ".github/workflows" / name).read_text()
before, request_job = workflow.split("\n request-ci:\n", 1)
update_job = before.split("\n update:\n", 1)[1]
self.assertNotIn("AUTO_MERGE_PAT_TOKEN", update_job)
self.assertIn("runs-on: ubuntu-latest", request_job)
self.assertIn("ref: ${{ github.workflow_sha }}", request_job)
self.assertNotIn("pull-request-operation", request_job)
for install in (
"npm ci",
"npm install",
"pip install",
"pdm install",
"pdm sync",
):
self.assertNotIn(install, request_job)
if __name__ == "__main__":
unittest.main()