Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
patch: reduce pytest and browser test overhead (#2044)
Ali Khokharcd7be2b22752
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 04b5616a..e5031ffb 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -137,7 +137,7 @@ jobs:runner-label: ${{ matrix.runner }}- name: Install Chromium- run: uv run --locked playwright install ${{ matrix.os == 'Linux' && '--with-deps chromium' || 'chromium' }}+ run: uv run --locked playwright install --only-shell ${{ matrix.os == 'Linux' && '--with-deps chromium' || 'chromium' }}- name: Run Playwrightrun: >-diff --git a/e2e/artifacts.py b/e2e/artifacts.pynew file mode 100644index 00000000..66399588--- /dev/null+++ b/e2e/artifacts.py@@ -0,0 +1,30 @@+"""Keep failure evidence without serializing artifacts already due for deletion."""++import pytest+++def discard_passed_artifacts(recorder, request):+ record_context = recorder.on_will_close_browser_context++ def close_context(context):+ report = getattr(request.node, "rep_call", None)+ if (+ report is not None+ and report.passed+ and request.config.getoption("--tracing") != "on"+ and request.config.getoption("--screenshot") != "on"+ ):+ # Stop recording without building a ZIP or PNG that the plugin+ # would immediately delete. Its normal video cleanup still runs.+ context.tracing.stop()+ else:+ # A context closed during setup or the test may precede a failure.+ record_context(context)++ recorder.on_will_close_browser_context = close_context+ return recorder++[email protected](name="_artifacts_recorder")+def failure_artifacts_recorder(_artifacts_recorder, request):+ return discard_passed_artifacts(_artifacts_recorder, request)diff --git a/e2e/conftest.py b/e2e/conftest.pyindex ea4cfa36..bf4a655d 100644--- a/e2e/conftest.py+++ b/e2e/conftest.py@@ -12,6 +12,7 @@ import pytestimport uvicornfrom playwright.sync_api import Page+from e2e.artifacts import failure_artifacts_recorder as failure_artifacts_recorderfrom e2e.code_support import CodeControlfrom e2e.server_shutdown import join_serverfrom free_claude_code.api.app import create_appdiff --git a/e2e/test_connected_accounts.py b/e2e/test_connected_accounts.pyindex 3a4b8925..b08c8b53 100644--- a/e2e/test_connected_accounts.py+++ b/e2e/test_connected_accounts.py@@ -413,8 +413,9 @@ def test_late_account_response_cannot_replace_discovered_model_count(page.route("**/admin/api/status", status)page.goto(f"{admin_base_url}/admin")page.wait_for_function("state.startup?.startup?.providers?.openai === 'ready'")+ pending = accounts.take_pending_status(page)accounts.hold_status.clear()- accounts.pending_status.pop().fulfill(json=account)+ pending.fulfill(json=account)card = page.locator('[data-provider="openai"]')expect(card.get_by_role("button", name="Disconnect", exact=True)).to_be_enabled()expect(card.locator(".provider-meta")).to_have_text("2 models available")@@ -429,8 +430,9 @@ def test_auth_status_refresh_preserves_proxy_edit_and_selection(proxy = dialog.locator("#field-OPENAI_PROXY")proxy.fill("http://pending-proxy:8080")proxy.evaluate("input => input.setSelectionRange(7, 14)")+ pending = accounts.take_pending_status(page)accounts.hold_status.clear()- accounts.pending_status.pop().fulfill(json=_status("openai", connected=True))+ pending.fulfill(json=_status("openai", connected=True))expect(page.locator('[data-provider="openai"]').get_by_role("button", name="Disconnect", exact=True@@ -466,13 +468,14 @@ def test_slow_initial_account_check_does_not_erase_a_later_save_warning()page.goto(f"{admin_base_url}/admin")dialog = open_provider(page, "nvidia_nim")- assert len(accounts.pending_status) == 1+ pending = accounts.take_pending_status(page)+ assert not accounts.pending_statusaccounts.hold_status.clear()dialog.locator("#field-NVIDIA_NIM_API_KEY").fill("new-key")dialog.get_by_role("button", name="Save", exact=True).click()expect(page.locator("#messageArea")).to_contain_text("Verification unavailable.")with page.expect_response("**/admin/api/providers/github_copilot/auth") as response:- accounts.pending_status.pop().fulfill(json=_status("github_copilot"))+ pending.fulfill(json=_status("github_copilot"))response.value.finished()page.evaluate("() => new Promise(requestAnimationFrame)")expect(page.locator("#messageArea")).to_contain_text("Verification unavailable.")diff --git a/e2e/test_provider_modals.py b/e2e/test_provider_modals.pyindex 74e2d8ce..6f6c88eb 100644--- a/e2e/test_provider_modals.py+++ b/e2e/test_provider_modals.py@@ -11,18 +11,38 @@ def test_provider_titles_link_to_websites_and_every_logo_loads() -> None:page.goto(f"{admin_base_url}/admin")expect(page.locator("#messageArea")).to_have_text("")- for card in page.locator(".provider-card").all():- link = card.locator(".provider-title a")- expect(link).to_have_count(1)- href = link.get_attribute("href")- assert href is not None and href.startswith("https://")- expect(link).to_have_attribute("target", "_blank")- expect(link).to_have_attribute("rel", "noopener noreferrer")- logo = link.locator("img")- expect(logo).to_have_count(1)- expect(logo).to_have_attribute("alt", "")- expect(logo).to_have_js_property("complete", True)- assert logo.evaluate("image => image.naturalWidth > 0")+ cards = page.locator(".provider-card")+ expect(cards.first).to_be_visible()+ page.wait_for_function("""() => Array.from(document.querySelectorAll(+ '.provider-card .provider-title a img'+ )).every(image => image.complete)""")+ # Read the rendered DOM together instead of tracing a browser round trip+ # for each attribute of every provider. Keep card identity in failures.+ snapshots = cards.evaluate_all("""cards => cards.map(card => ({+ provider: card.dataset.provider,+ links: Array.from(card.querySelectorAll('.provider-title a'), link => ({+ href: link.getAttribute('href'),+ target: link.getAttribute('target'),+ rel: link.getAttribute('rel'),+ logos: Array.from(link.querySelectorAll('img'), image => ({+ alt: image.getAttribute('alt'),+ complete: image.complete,+ naturalWidth: image.naturalWidth,+ })),+ })),+ }))""")+ assert snapshots+ for card in snapshots:+ assert len(card["links"]) == 1, card+ link = card["links"][0]+ assert link["href"] and link["href"].startswith("https://"), card+ assert link["target"] == "_blank", card+ assert link["rel"] == "noopener noreferrer", card+ assert len(link["logos"]) == 1, card+ logo = link["logos"][0]+ assert logo["alt"] == "", card+ assert logo["complete"] is True, card+ assert logo["naturalWidth"] > 0, cardexpect(page.locator('[data-provider="llamacpp"] .provider-title')).to_have_text("LLaMA.cpp")diff --git a/pyproject.toml b/pyproject.tomlindex 6ffc669e..370c8ea6 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -138,6 +138,7 @@ skip-magic-trailing-comma = false[tool.pytest.ini_options]pythonpath = ["src"]addopts = "-n auto --dist=worksteal"+tmp_path_retention_policy = "failed"filterwarnings = ["error",# Starlette's test client uses this deprecated alias, including in 1.6.0.diff --git a/tests/messaging/test_limiter.py b/tests/messaging/test_limiter.pyindex 3ed42f69..5cfdee58 100644--- a/tests/messaging/test_limiter.py+++ b/tests/messaging/test_limiter.py@@ -73,33 +73,23 @@ class TestMessagingRateLimiter:@pytest.mark.asyncioasync def test_compaction(self):- """- Verify multiple rapid requests with same dedup_key are compacted.- Logic ported from verify_limiter.py- """+ """Queued edits with the same key deliver only the latest content."""limiter = self.create_limiter(rate_limit=1, rate_window=1.0)- call_counts = {}+ edits = []async def mock_edit(msg_id, content):- call_counts[msg_id] = call_counts.get(msg_id, 0) + 1+ edits.append((msg_id, content))return f"done_{content}"- # Spam 5 edits+ # Submit the whole burst before yielding to the worker.for i in range(5):limiter.fire_and_forget(lambda i=i: mock_edit("msg1", f"update_{i}"), dedup_key="edit:msg1")- # Wait for processing- # 1st might go through immediately, subsequent ones queue and compact- await asyncio.sleep(2.5)-- # Expected: ~2 calls (first and last)- assert call_counts["msg1"] <= 2, (- f"Expected compaction to reduce calls, but got {call_counts.get('msg1', 0)}"- )- assert call_counts["msg1"] >= 1, "Expected at least one call"+ await asyncio.wait_for(asyncio.gather(*limiter._background_tasks), timeout=2.0)+ assert edits == [("msg1", "update_4")]@pytest.mark.asyncioasync def test_compaction_and_futures_resolution(self):@@ -114,7 +104,6 @@ class TestMessagingRateLimiter:async def mock_edit(mid, content):call_counts[mid] = call_counts.get(mid, 0) + 1- await asyncio.sleep(0.05)return f"result_{content}"async def task(i):@@ -122,20 +111,15 @@ class TestMessagingRateLimiter:lambda i=i: mock_edit(msg_id, f"v{i}"), dedup_key=f"edit:{msg_id}")- start_time = time.time()-# Enqueue 3 tasks concurrently- results = await asyncio.gather(task(1), task(2), task(3))-- duration = time.time() - start_time+ results = await asyncio.wait_for(+ asyncio.gather(task(1), task(2), task(3)), timeout=2.0+ )# All results should be the LAST one executedfor res in results:assert res == "result_v3", f"Expected result_v3, got {res}"- # Should be reasonably fast- assert duration < 2.0, "Execution took too long"-# Calls should be compactedassert call_counts[msg_id] <= 2, f"Too many actual calls: {call_counts[msg_id]}"@@ -279,9 +263,10 @@ class TestMessagingRateLimiter:raise ValueError("fire_and_forget failed")limiter.fire_and_forget(fail_task, dedup_key="fire_fail")- await asyncio.sleep(1.5)+ await asyncio.wait_for(asyncio.gather(*limiter._background_tasks), timeout=2.0)joined = " ".join(str(r.message) for r in caplog.records)+ assert "Final error in fire_and_forget for key fire_fail" in joinedassert "ValueError" in joinedassert "fire_and_forget failed" not in joineddiff --git a/tests/scripts/test_publish_release.py b/tests/scripts/test_publish_release.pyindex 51c56a9f..8b2f9393 100644--- a/tests/scripts/test_publish_release.py+++ b/tests/scripts/test_publish_release.py@@ -2,6 +2,7 @@import hashlibimport json+import shutilfrom itertools import pairwisefrom pathlib import Pathfrom typing import TypedDict@@ -20,14 +21,21 @@ def test_one_increment(kind, expected):assert next_version("6.2.66", kind) == expected-def releases(tmp_path, monkeypatch):- monkeypatch.chdir(tmp_path)- git(tmp_path, "init", "-b", "main")[email protected](scope="session")+def releases_template(tmp_path_factory):+ tmp_path = tmp_path_factory.mktemp("publish-release-template")+ git(tmp_path, "init", "--template=", "-b", "main")write(tmp_path, "src/file.py", "base")base = commit(tmp_path)git(tmp_path, "tag", "v1.2.3")git(tmp_path, "update-ref", "refs/remotes/origin/main", base)+ return tmp_path+++def releases(tmp_path, monkeypatch, releases_template):+ shutil.copytree(releases_template, tmp_path, dirs_exist_ok=True)+ monkeypatch.chdir(tmp_path)state = FakeRemote(tmp_path)monkeypatch.setattr(publishing, "command", state.command)return tmp_path, statediff --git a/tests/scripts/test_version_policy.py b/tests/scripts/test_version_policy.pyindex 425460f0..4e83930f 100644--- a/tests/scripts/test_version_policy.py+++ b/tests/scripts/test_version_policy.py@@ -1,6 +1,7 @@"""Release intent is validated against committed paths, not edited numbers."""import os+import shutilimport subprocessimport sysfrom pathlib import Path@@ -40,9 +41,10 @@ def commit(repo: Path, title: str = "test change") -> str:return git(repo, "rev-parse", "HEAD")-def history(tmp_path):- git(tmp_path, "init", "-b", "main")[email protected](scope="session")+def history_template(tmp_path_factory):+ tmp_path = tmp_path_factory.mktemp("version-policy-template")+ git(tmp_path, "init", "--template=", "-b", "main")git(tmp_path, "config", "core.autocrlf", "false")write(tmp_path,@@ -58,6 +60,15 @@ def history(tmp_path):return tmp_path, commit(tmp_path)+def history(tmp_path, history_template):+ source, base = history_template+ # Copy the immutable seed, including Git objects and refs. Every test can+ # change its worktree, index, and history without affecting another test.+ shutil.copytree(source, tmp_path, dirs_exist_ok=True)+ return tmp_path, base++def check(repo: Path, base: str, title: str) -> subprocess.CompletedProcess[str]:return subprocess.run([diff --git a/tests/test_browser_artifacts.py b/tests/test_browser_artifacts.pynew file mode 100644index 00000000..3731c871--- /dev/null+++ b/tests/test_browser_artifacts.py@@ -0,0 +1,69 @@+"""The real Playwright recorder retains requested evidence on every close path."""++from pathlib import Path+from tempfile import TemporaryDirectory+from types import SimpleNamespace+from unittest.mock import Mock++import pytest+from pytest_playwright.pytest_playwright import ArtifactsRecorder++from e2e.artifacts import discard_passed_artifacts+++ "outcome,tracing,screenshot,zip_count,png_count",+ [+ ("passed", "retain-on-failure", "only-on-failure", 0, 0),+ ("failed", "retain-on-failure", "only-on-failure", 1, 1),+ ("setup-failed", "retain-on-failure", "only-on-failure", 1, 1),+ ("closed-before-failure", "retain-on-failure", "only-on-failure", 1, 1),+ ("passed", "on", "on", 1, 1),+ ("passed", "on", "only-on-failure", 1, 0),+ ("passed", "retain-on-failure", "on", 0, 1),+ ("failed", "off", "off", 0, 0),+ ],+)+def test_artifact_retention(+ tmp_path, outcome, tracing, screenshot, zip_count, png_count+):+ options = {+ "--tracing": tracing,+ "--screenshot": screenshot,+ "--full-page-screenshot": True,+ "--video": "off",+ }+ config = Mock()+ config.getoption.side_effect = options.__getitem__+ node = SimpleNamespace(nodeid="example")+ if outcome in {"passed", "failed"}:+ node.rep_call = SimpleNamespace(passed=outcome == "passed")+ request = Mock(config=config, node=node)+ output = tmp_path / "output"+ context = Mock()+ page = Mock()+ context.pages = [page]++ def save_trace(*, path=None):+ if path is not None:+ Path(path).write_bytes(b"trace")++ def save_screenshot(*, path, **_kwargs):+ Path(path).write_bytes(b"screenshot")++ context.tracing.stop.side_effect = save_trace+ page.screenshot.side_effect = save_screenshot+ staging = TemporaryDirectory(dir=tmp_path)+ with staging:+ recorder = ArtifactsRecorder(config, request, str(output), Mock(), staging)+ recorder = discard_passed_artifacts(recorder, request)+ recorder.on_did_create_browser_context(context)+ recorder.on_will_close_browser_context(context)+ recorder.did_finish_test(failed=outcome != "passed")++ assert len(list(output.glob("*.zip"))) == zip_count+ assert len(list(output.glob("*.png"))) == png_count+ assert list(Path(staging.name).iterdir()) == []+ if outcome == "passed" and tracing != "on" and screenshot != "on":+ context.tracing.stop.assert_called_once_with()+ page.screenshot.assert_not_called()