Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

patch: reduce pytest and browser test overhead (#2044)

Ali Khokharcd7be2b22752
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 04b5616a..e5031ffb 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -137,7 +137,7 @@ jobs:
runner-label: ${{ matrix.runner }}
- name: Install Chromium
- run: uv run --locked playwright install ${{ matrix.os == 'Linux' && '--with-deps chromium' || 'chromium' }}
+ run: uv run --locked playwright install --only-shell ${{ matrix.os == 'Linux' && '--with-deps chromium' || 'chromium' }}
- name: Run Playwright
run: >-
diff --git a/e2e/artifacts.py b/e2e/artifacts.py
new file mode 100644
index 00000000..66399588
--- /dev/null
+++ b/e2e/artifacts.py
@@ -0,0 +1,30 @@
+"""Keep failure evidence without serializing artifacts already due for deletion."""
+
+import pytest
+
+
+def discard_passed_artifacts(recorder, request):
+ record_context = recorder.on_will_close_browser_context
+
+ def close_context(context):
+ report = getattr(request.node, "rep_call", None)
+ if (
+ report is not None
+ and report.passed
+ and request.config.getoption("--tracing") != "on"
+ and request.config.getoption("--screenshot") != "on"
+ ):
+ # Stop recording without building a ZIP or PNG that the plugin
+ # would immediately delete. Its normal video cleanup still runs.
+ context.tracing.stop()
+ else:
+ # A context closed during setup or the test may precede a failure.
+ record_context(context)
+
+ recorder.on_will_close_browser_context = close_context
+ return recorder
+
+
[email protected](name="_artifacts_recorder")
+def failure_artifacts_recorder(_artifacts_recorder, request):
+ return discard_passed_artifacts(_artifacts_recorder, request)
diff --git a/e2e/conftest.py b/e2e/conftest.py
index ea4cfa36..bf4a655d 100644
--- a/e2e/conftest.py
+++ b/e2e/conftest.py
@@ -12,6 +12,7 @@ import pytest
import uvicorn
from playwright.sync_api import Page
+from e2e.artifacts import failure_artifacts_recorder as failure_artifacts_recorder
from e2e.code_support import CodeControl
from e2e.server_shutdown import join_server
from free_claude_code.api.app import create_app
diff --git a/e2e/test_connected_accounts.py b/e2e/test_connected_accounts.py
index 3a4b8925..b08c8b53 100644
--- a/e2e/test_connected_accounts.py
+++ b/e2e/test_connected_accounts.py
@@ -413,8 +413,9 @@ def test_late_account_response_cannot_replace_discovered_model_count(
page.route("**/admin/api/status", status)
page.goto(f"{admin_base_url}/admin")
page.wait_for_function("state.startup?.startup?.providers?.openai === 'ready'")
+ pending = accounts.take_pending_status(page)
accounts.hold_status.clear()
- accounts.pending_status.pop().fulfill(json=account)
+ pending.fulfill(json=account)
card = page.locator('[data-provider="openai"]')
expect(card.get_by_role("button", name="Disconnect", exact=True)).to_be_enabled()
expect(card.locator(".provider-meta")).to_have_text("2 models available")
@@ -429,8 +430,9 @@ def test_auth_status_refresh_preserves_proxy_edit_and_selection(
proxy = dialog.locator("#field-OPENAI_PROXY")
proxy.fill("http://pending-proxy:8080")
proxy.evaluate("input => input.setSelectionRange(7, 14)")
+ pending = accounts.take_pending_status(page)
accounts.hold_status.clear()
- accounts.pending_status.pop().fulfill(json=_status("openai", connected=True))
+ pending.fulfill(json=_status("openai", connected=True))
expect(
page.locator('[data-provider="openai"]').get_by_role(
"button", name="Disconnect", exact=True
@@ -466,13 +468,14 @@ def test_slow_initial_account_check_does_not_erase_a_later_save_warning(
)
page.goto(f"{admin_base_url}/admin")
dialog = open_provider(page, "nvidia_nim")
- assert len(accounts.pending_status) == 1
+ pending = accounts.take_pending_status(page)
+ assert not accounts.pending_status
accounts.hold_status.clear()
dialog.locator("#field-NVIDIA_NIM_API_KEY").fill("new-key")
dialog.get_by_role("button", name="Save", exact=True).click()
expect(page.locator("#messageArea")).to_contain_text("Verification unavailable.")
with page.expect_response("**/admin/api/providers/github_copilot/auth") as response:
- accounts.pending_status.pop().fulfill(json=_status("github_copilot"))
+ pending.fulfill(json=_status("github_copilot"))
response.value.finished()
page.evaluate("() => new Promise(requestAnimationFrame)")
expect(page.locator("#messageArea")).to_contain_text("Verification unavailable.")
diff --git a/e2e/test_provider_modals.py b/e2e/test_provider_modals.py
index 74e2d8ce..6f6c88eb 100644
--- a/e2e/test_provider_modals.py
+++ b/e2e/test_provider_modals.py
@@ -11,18 +11,38 @@ def test_provider_titles_link_to_websites_and_every_logo_loads(
) -> None:
page.goto(f"{admin_base_url}/admin")
expect(page.locator("#messageArea")).to_have_text("")
- for card in page.locator(".provider-card").all():
- link = card.locator(".provider-title a")
- expect(link).to_have_count(1)
- href = link.get_attribute("href")
- assert href is not None and href.startswith("https://")
- expect(link).to_have_attribute("target", "_blank")
- expect(link).to_have_attribute("rel", "noopener noreferrer")
- logo = link.locator("img")
- expect(logo).to_have_count(1)
- expect(logo).to_have_attribute("alt", "")
- expect(logo).to_have_js_property("complete", True)
- assert logo.evaluate("image => image.naturalWidth > 0")
+ cards = page.locator(".provider-card")
+ expect(cards.first).to_be_visible()
+ page.wait_for_function("""() => Array.from(document.querySelectorAll(
+ '.provider-card .provider-title a img'
+ )).every(image => image.complete)""")
+ # Read the rendered DOM together instead of tracing a browser round trip
+ # for each attribute of every provider. Keep card identity in failures.
+ snapshots = cards.evaluate_all("""cards => cards.map(card => ({
+ provider: card.dataset.provider,
+ links: Array.from(card.querySelectorAll('.provider-title a'), link => ({
+ href: link.getAttribute('href'),
+ target: link.getAttribute('target'),
+ rel: link.getAttribute('rel'),
+ logos: Array.from(link.querySelectorAll('img'), image => ({
+ alt: image.getAttribute('alt'),
+ complete: image.complete,
+ naturalWidth: image.naturalWidth,
+ })),
+ })),
+ }))""")
+ assert snapshots
+ for card in snapshots:
+ assert len(card["links"]) == 1, card
+ link = card["links"][0]
+ assert link["href"] and link["href"].startswith("https://"), card
+ assert link["target"] == "_blank", card
+ assert link["rel"] == "noopener noreferrer", card
+ assert len(link["logos"]) == 1, card
+ logo = link["logos"][0]
+ assert logo["alt"] == "", card
+ assert logo["complete"] is True, card
+ assert logo["naturalWidth"] > 0, card
expect(page.locator('[data-provider="llamacpp"] .provider-title')).to_have_text(
"LLaMA.cpp"
)
diff --git a/pyproject.toml b/pyproject.toml
index 6ffc669e..370c8ea6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -138,6 +138,7 @@ skip-magic-trailing-comma = false
[tool.pytest.ini_options]
pythonpath = ["src"]
addopts = "-n auto --dist=worksteal"
+tmp_path_retention_policy = "failed"
filterwarnings = [
"error",
# Starlette's test client uses this deprecated alias, including in 1.6.0.
diff --git a/tests/messaging/test_limiter.py b/tests/messaging/test_limiter.py
index 3ed42f69..5cfdee58 100644
--- a/tests/messaging/test_limiter.py
+++ b/tests/messaging/test_limiter.py
@@ -73,33 +73,23 @@ class TestMessagingRateLimiter:
@pytest.mark.asyncio
async def test_compaction(self):
- """
- Verify multiple rapid requests with same dedup_key are compacted.
- Logic ported from verify_limiter.py
- """
+ """Queued edits with the same key deliver only the latest content."""
limiter = self.create_limiter(rate_limit=1, rate_window=1.0)
- call_counts = {}
+ edits = []
async def mock_edit(msg_id, content):
- call_counts[msg_id] = call_counts.get(msg_id, 0) + 1
+ edits.append((msg_id, content))
return f"done_{content}"
- # Spam 5 edits
+ # Submit the whole burst before yielding to the worker.
for i in range(5):
limiter.fire_and_forget(
lambda i=i: mock_edit("msg1", f"update_{i}"), dedup_key="edit:msg1"
)
- # Wait for processing
- # 1st might go through immediately, subsequent ones queue and compact
- await asyncio.sleep(2.5)
-
- # Expected: ~2 calls (first and last)
- assert call_counts["msg1"] <= 2, (
- f"Expected compaction to reduce calls, but got {call_counts.get('msg1', 0)}"
- )
- assert call_counts["msg1"] >= 1, "Expected at least one call"
+ await asyncio.wait_for(asyncio.gather(*limiter._background_tasks), timeout=2.0)
+ assert edits == [("msg1", "update_4")]
@pytest.mark.asyncio
async def test_compaction_and_futures_resolution(self):
@@ -114,7 +104,6 @@ class TestMessagingRateLimiter:
async def mock_edit(mid, content):
call_counts[mid] = call_counts.get(mid, 0) + 1
- await asyncio.sleep(0.05)
return f"result_{content}"
async def task(i):
@@ -122,20 +111,15 @@ class TestMessagingRateLimiter:
lambda i=i: mock_edit(msg_id, f"v{i}"), dedup_key=f"edit:{msg_id}"
)
- start_time = time.time()
-
# Enqueue 3 tasks concurrently
- results = await asyncio.gather(task(1), task(2), task(3))
-
- duration = time.time() - start_time
+ results = await asyncio.wait_for(
+ asyncio.gather(task(1), task(2), task(3)), timeout=2.0
+ )
# All results should be the LAST one executed
for res in results:
assert res == "result_v3", f"Expected result_v3, got {res}"
- # Should be reasonably fast
- assert duration < 2.0, "Execution took too long"
-
# Calls should be compacted
assert call_counts[msg_id] <= 2, f"Too many actual calls: {call_counts[msg_id]}"
@@ -279,9 +263,10 @@ class TestMessagingRateLimiter:
raise ValueError("fire_and_forget failed")
limiter.fire_and_forget(fail_task, dedup_key="fire_fail")
- await asyncio.sleep(1.5)
+ await asyncio.wait_for(asyncio.gather(*limiter._background_tasks), timeout=2.0)
joined = " ".join(str(r.message) for r in caplog.records)
+ assert "Final error in fire_and_forget for key fire_fail" in joined
assert "ValueError" in joined
assert "fire_and_forget failed" not in joined
diff --git a/tests/scripts/test_publish_release.py b/tests/scripts/test_publish_release.py
index 51c56a9f..8b2f9393 100644
--- a/tests/scripts/test_publish_release.py
+++ b/tests/scripts/test_publish_release.py
@@ -2,6 +2,7 @@
import hashlib
import json
+import shutil
from itertools import pairwise
from pathlib import Path
from typing import TypedDict
@@ -20,14 +21,21 @@ def test_one_increment(kind, expected):
assert next_version("6.2.66", kind) == expected
-def releases(tmp_path, monkeypatch):
- monkeypatch.chdir(tmp_path)
- git(tmp_path, "init", "-b", "main")
[email protected](scope="session")
+def releases_template(tmp_path_factory):
+ tmp_path = tmp_path_factory.mktemp("publish-release-template")
+ git(tmp_path, "init", "--template=", "-b", "main")
write(tmp_path, "src/file.py", "base")
base = commit(tmp_path)
git(tmp_path, "tag", "v1.2.3")
git(tmp_path, "update-ref", "refs/remotes/origin/main", base)
+ return tmp_path
+
+
+def releases(tmp_path, monkeypatch, releases_template):
+ shutil.copytree(releases_template, tmp_path, dirs_exist_ok=True)
+ monkeypatch.chdir(tmp_path)
state = FakeRemote(tmp_path)
monkeypatch.setattr(publishing, "command", state.command)
return tmp_path, state
diff --git a/tests/scripts/test_version_policy.py b/tests/scripts/test_version_policy.py
index 425460f0..4e83930f 100644
--- a/tests/scripts/test_version_policy.py
+++ b/tests/scripts/test_version_policy.py
@@ -1,6 +1,7 @@
"""Release intent is validated against committed paths, not edited numbers."""
import os
+import shutil
import subprocess
import sys
from pathlib import Path
@@ -40,9 +41,10 @@ def commit(repo: Path, title: str = "test change") -> str:
return git(repo, "rev-parse", "HEAD")
-def history(tmp_path):
- git(tmp_path, "init", "-b", "main")
[email protected](scope="session")
+def history_template(tmp_path_factory):
+ tmp_path = tmp_path_factory.mktemp("version-policy-template")
+ git(tmp_path, "init", "--template=", "-b", "main")
git(tmp_path, "config", "core.autocrlf", "false")
write(
tmp_path,
@@ -58,6 +60,15 @@ def history(tmp_path):
return tmp_path, commit(tmp_path)
+def history(tmp_path, history_template):
+ source, base = history_template
+ # Copy the immutable seed, including Git objects and refs. Every test can
+ # change its worktree, index, and history without affecting another test.
+ shutil.copytree(source, tmp_path, dirs_exist_ok=True)
+ return tmp_path, base
+
+
def check(repo: Path, base: str, title: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
diff --git a/tests/test_browser_artifacts.py b/tests/test_browser_artifacts.py
new file mode 100644
index 00000000..3731c871
--- /dev/null
+++ b/tests/test_browser_artifacts.py
@@ -0,0 +1,69 @@
+"""The real Playwright recorder retains requested evidence on every close path."""
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from pytest_playwright.pytest_playwright import ArtifactsRecorder
+
+from e2e.artifacts import discard_passed_artifacts
+
+
+ "outcome,tracing,screenshot,zip_count,png_count",
+ [
+ ("passed", "retain-on-failure", "only-on-failure", 0, 0),
+ ("failed", "retain-on-failure", "only-on-failure", 1, 1),
+ ("setup-failed", "retain-on-failure", "only-on-failure", 1, 1),
+ ("closed-before-failure", "retain-on-failure", "only-on-failure", 1, 1),
+ ("passed", "on", "on", 1, 1),
+ ("passed", "on", "only-on-failure", 1, 0),
+ ("passed", "retain-on-failure", "on", 0, 1),
+ ("failed", "off", "off", 0, 0),
+ ],
+)
+def test_artifact_retention(
+ tmp_path, outcome, tracing, screenshot, zip_count, png_count
+):
+ options = {
+ "--tracing": tracing,
+ "--screenshot": screenshot,
+ "--full-page-screenshot": True,
+ "--video": "off",
+ }
+ config = Mock()
+ config.getoption.side_effect = options.__getitem__
+ node = SimpleNamespace(nodeid="example")
+ if outcome in {"passed", "failed"}:
+ node.rep_call = SimpleNamespace(passed=outcome == "passed")
+ request = Mock(config=config, node=node)
+ output = tmp_path / "output"
+ context = Mock()
+ page = Mock()
+ context.pages = [page]
+
+ def save_trace(*, path=None):
+ if path is not None:
+ Path(path).write_bytes(b"trace")
+
+ def save_screenshot(*, path, **_kwargs):
+ Path(path).write_bytes(b"screenshot")
+
+ context.tracing.stop.side_effect = save_trace
+ page.screenshot.side_effect = save_screenshot
+ staging = TemporaryDirectory(dir=tmp_path)
+ with staging:
+ recorder = ArtifactsRecorder(config, request, str(output), Mock(), staging)
+ recorder = discard_passed_artifacts(recorder, request)
+ recorder.on_did_create_browser_context(context)
+ recorder.on_will_close_browser_context(context)
+ recorder.did_finish_test(failed=outcome != "passed")
+
+ assert len(list(output.glob("*.zip"))) == zip_count
+ assert len(list(output.glob("*.png"))) == png_count
+ assert list(Path(staging.name).iterdir()) == []
+ if outcome == "passed" and tracing != "on" and screenshot != "on":
+ context.tracing.stop.assert_called_once_with()
+ page.screenshot.assert_not_called()