Skip to content

Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.

15.5 KB
Raw Blame
"""End-to-end tests for the autoskill pipeline.

Exercises the full chain — fetch → redact → cluster → match → synthesize → write
— with real internal modules. External dependencies (screenpipe, LLM, embeddings)
are replaced with deterministic fakes so the suite runs offline in under a second.
"""

import json
import sys
from pathlib import Path

import httpx
import pytest

# Synthetic opaque value for auth-header round-trip tests. Not a credential.
_AUTH = "fake-test-value-1234"

from backends import LocalBackend
from run import run, main


# ---------- fixtures ----------

SKILL_TEMPLATE = "---\nname: {name}\ndescription: {description}\n---\n# {name}\n"


def _write_skill(root: Path, name: str, description: str) -> None:
    d = root / name
    d.mkdir(parents=True, exist_ok=True)
    (d / "SKILL.md").write_text(SKILL_TEMPLATE.format(name=name, description=description))


def _seed_skills_dir(root: Path) -> None:
    """A tiny but representative slice of the real skills/ layout."""
    _write_skill(root, "literature-review",
                 "Systematic literature reviews across PubMed arXiv bioRxiv with citations.")
    _write_skill(root, "citation-management",
                 "Find papers and format citations from Google Scholar and PubMed.")
    _write_skill(root, "scientific-writing",
                 "Write research manuscripts with IMRAD structure and citations.")
    _write_skill(root, "scientific-schematics",
                 "Create scientific diagrams figures and schematics for publications.")
    _write_skill(root, "latex-posters",
                 "Create academic conference posters in LaTeX.")
    _write_skill(root, "rdkit",
                 "Cheminformatics with RDKit molecules drug discovery.")


# ---------- fake screenpipe ----------

def _ocr(ts: str, app: str, title: str, text: str) -> dict:
    return {"type": "OCR", "content": {
        "timestamp": ts, "app_name": app, "window_name": title, "text": text,
    }}


def _realistic_day() -> list:
    """Three workflow patterns repeated twice each, plus noise."""
    events = []
    # Pattern 1: literature work (Chrome + Zotero) — reuse expected
    for hr in (9, 14):
        for m in range(0, 20):
            events.append(_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z",
                               "Chrome", "PubMed", "searching papers"))
    # Pattern 2: slides + schematics (Keynote + Preview) — novel expected
    for hr in (11, 16):
        for m in range(0, 20):
            events.append(_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z",
                               "Keynote", "deck", "slides for keynote talk"))
    # Pattern 3: manuscript + figures (VSCode + Preview) — compose expected
    for hr in (12, 17):
        for m in range(0, 20):
            events.append(_ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z",
                               "VSCode", "paper.tex", "writing manuscript with figures"))
    # Pattern 4 (lonely): should be dropped (only one session)
    for m in range(0, 10):
        events.append(_ocr(f"2026-04-17T22:{m:02d}:00Z",
                           "Lonely", "", "whatever"))
    return events


def _screenpipe_client(events: list) -> httpx.Client:
    # Paginate in chunks of 20 to exercise fetch_window's pagination loop.
    page_size = 20

    def handler(request):
        params = dict(request.url.params)
        offset = int(params.get("offset", "0"))
        chunk = events[offset:offset + page_size]
        return httpx.Response(200, json={
            "data": chunk,
            "pagination": {"limit": page_size, "offset": offset, "total": len(events)},
        })

    return httpx.Client(transport=httpx.MockTransport(handler), base_url="http://screenpipe.test")


# ---------- fake LM Studio ----------

class _FakeLMStudioHandler:
    """Returns different verdicts based on apps mentioned in the prompt."""

    def __init__(self):
        self.prompts = []

    def __call__(self, request):
        body = json.loads(request.read())
        prompt = body["messages"][0]["content"]
        self.prompts.append(prompt)

        if "Chrome" in prompt and "PubMed" in prompt:
            out = {"verdict": "reuse", "target": "literature-review"}
        elif "Keynote" in prompt:
            body = ("---\nname: keynote-talk-prep\n"
                    "description: Prepare and iterate on Keynote talks.\n---\n"
                    "# keynote-talk-prep\n")
            out = {"verdict": "novel", "name": "keynote-talk-prep", "skill_body": body}
        elif "VSCode" in prompt:
            body = ("---\nname: manuscript-with-figures\n"
                    "description: Chain scientific-writing + scientific-schematics.\n---\n"
                    "# manuscript-with-figures\n\n"
                    "Invoke scientific-writing then scientific-schematics.\n")
            out = {"verdict": "compose", "name": "manuscript-with-figures", "skill_body": body}
        else:
            out = {"verdict": "reuse", "target": "literature-review"}

        return httpx.Response(200, json={
            "choices": [{"finish_reason": "stop", "message": {"content": json.dumps(out)}}]
        })


def _keyword_embedder(text: str):
    keywords = ["paper", "pubmed", "literature", "citation", "writing",
                "schematic", "figure", "poster", "molecule", "slides"]
    return [1.0 if k in text.lower() else 0.0 for k in keywords]


def _base_config():
    return {
        "cluster": {"min_session_minutes": 0, "idle_gap_minutes": 10, "min_cluster_size": 2},
    }


# ---------- tests ----------

def test_full_pipeline_produces_report_and_drafts(tmp_path):
    skills_dir = tmp_path / "skills"
    _seed_skills_dir(skills_dir)

    fake_lmstudio = _FakeLMStudioHandler()
    lm_client = httpx.Client(
        transport=httpx.MockTransport(fake_lmstudio),
        base_url="http://localhost:1234/v1",
    )
    backend = LocalBackend(endpoint="http://localhost:1234/v1",
                           model="Gemma-4-31B-it", client=lm_client)

    out = run(
        _base_config(),
        start_time="2026-04-17T00:00:00Z", end_time="2026-04-17T23:59:59Z",
        out_dir=tmp_path / "_proposed",
        screenpipe_client=_screenpipe_client(_realistic_day()),
        backend=backend,
        embedder=_keyword_embedder,
        skills_dir=skills_dir,
        now=lambda: "2026-04-17T18-00-00",
    )

    # Report exists and mentions all three surviving clusters.
    report = (out / "report.md").read_text()
    assert "Chrome" in report
    assert "Keynote" in report
    assert "VSCode" in report
    assert "reuse" in report
    assert "novel" in report
    assert "compose" in report
    # Lonely cluster was dropped by min_cluster_size.
    assert "Lonely" not in report

    # Novel draft landed under new-skills/.
    novel_skill = out / "new-skills" / "keynote-talk-prep" / "SKILL.md"
    assert novel_skill.exists()
    assert "keynote-talk-prep" in novel_skill.read_text()

    # Compose draft landed under composition-recipes/.
    compose_skill = out / "composition-recipes" / "manuscript-with-figures" / "SKILL.md"
    assert compose_skill.exists()
    body = compose_skill.read_text()
    assert "scientific-writing" in body
    assert "scientific-schematics" in body

    # Reuse verdict writes no draft but does cite the matched existing skill.
    assert not (out / "new-skills" / "literature-review").exists()
    assert "literature-review" in report

    # Pagination actually happened (more than one fetch).
    # 120 events / 20 per page = 6 pages → ≥6 backend-independent calls.
    # (Can't assert directly without reaching into the transport; but the test
    # would fail above if pagination were broken since clusters would be empty.)


def test_pipeline_redaction_prevents_secrets_leaving_the_host(tmp_path):
    skills_dir = tmp_path / "skills"
    _seed_skills_dir(skills_dir)

    # Plant secrets in every OCR event.
    toxic = [
        _ocr(f"2026-04-17T{hr:02d}:{m:02d}:00Z", "Chrome", "Gmail",
             "email [email protected] password sk-abcdefghijklmnopqrstuvwxyz012345")
        for hr in (9, 14) for m in range(0, 20)
    ]

    fake_lmstudio = _FakeLMStudioHandler()
    lm_client = httpx.Client(
        transport=httpx.MockTransport(fake_lmstudio),
        base_url="http://localhost:1234/v1",
    )
    backend = LocalBackend(endpoint="http://localhost:1234/v1",
                           model="Gemma-4-31B-it", client=lm_client)

    run(
        _base_config(),
        start_time="s", end_time="e",
        out_dir=tmp_path / "_proposed",
        screenpipe_client=_screenpipe_client(toxic),
        backend=backend, embedder=_keyword_embedder, skills_dir=skills_dir,
        now=lambda: "ts",
    )

    assert fake_lmstudio.prompts, "backend should have been called"
    for prompt in fake_lmstudio.prompts:
        assert "[email protected]" not in prompt
        assert "sk-abcdefghijklmnopqrstuvwxyz012345" not in prompt


def test_auth_token_threads_through_run_into_fetch_window(tmp_path):
    """Integration: config's screenpipe.token reaches the outgoing HTTP request."""
    skills_dir = tmp_path / "skills"
    _seed_skills_dir(skills_dir)

    seen_auth = []

    def handler(request):
        seen_auth.append(request.headers.get("authorization"))
        return httpx.Response(200, json={
            "data": [], "pagination": {"limit": 20, "offset": 0, "total": 0},
        })

    client = httpx.Client(transport=httpx.MockTransport(handler),
                          base_url="http://screenpipe.test")

    run(
        _base_config(),
        start_time="2026-04-17T00:00:00Z", end_time="2026-04-17T23:59:59Z",
        out_dir=tmp_path / "_proposed",
        screenpipe_client=client,
        backend=None, embedder=_keyword_embedder, skills_dir=skills_dir,
        screenpipe_token=_AUTH,
        now=lambda: "ts",
    )

    assert seen_auth, "screenpipe should have been called"
    assert all(h == f"Bearer {_AUTH}" for h in seen_auth), seen_auth


def test_cli_dry_run_writes_plan_without_backend_or_embedding_model(tmp_path, monkeypatch):
    """Exercises scripts/run.py's main() via argv with --dry-run.

    --dry-run is the only path that avoids loading sentence-transformers and
    instantiating a backend, so we can smoke-test the CLI offline.
    """
    skills_dir = tmp_path / "skills"
    _seed_skills_dir(skills_dir)

    # Minimal config.yaml pointing at our fake screenpipe (via localhost base_url
    # that we'll monkeypatch httpx.Client to respect).
    config = tmp_path / "config.yaml"
    config.write_text(
        "backend: local\n"
        "local:\n"
        "  endpoint: http://localhost:1234/v1\n"
        "  model: Gemma-4-31B-it\n"
        "screenpipe:\n"
        "  url: http://localhost:3030\n"
        "cluster:\n"
        "  min_session_minutes: 0\n"
        "  idle_gap_minutes: 10\n"
        "  min_cluster_size: 2\n"
    )

    # Install a MockTransport-backed default for any httpx.Client the CLI builds.
    events = _realistic_day()

    def handler(request):
        params = dict(request.url.params)
        offset = int(params.get("offset", "0"))
        chunk = events[offset:offset + 20]
        return httpx.Response(200, json={
            "data": chunk,
            "pagination": {"limit": 20, "offset": offset, "total": len(events)},
        })

    original_client = httpx.Client
    monkeypatch.setattr(httpx, "Client", lambda *a, **kw: original_client(
        *a, **{**kw, "transport": httpx.MockTransport(handler)}
    ))

    # Need pyyaml for main() to load the config; install on demand and skip if unavailable.
    yaml = pytest.importorskip("yaml")

    out_dir = tmp_path / "_proposed"
    rc = main([
        "--start", "2026-04-17T00:00:00Z",
        "--end", "2026-04-17T23:59:59Z",
        "--config", str(config),
        "--skills-dir", str(skills_dir),
        "--out", str(out_dir),
        "--dry-run",
    ])
    assert rc == 0

    # One timestamped subdir was created under --out.
    subdirs = [p for p in out_dir.iterdir() if p.is_dir()]
    assert len(subdirs) == 1
    plan = (subdirs[0] / "plan.md").read_text()
    assert "Cluster" in plan
    assert "Chrome" in plan


def test_real_loopback_http_fixture_contracts(tmp_path, monkeypatch):
    """Actual HTTP on an ephemeral port, using only invented activity and tokens."""
    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
    from threading import Thread
    from urllib.parse import parse_qs, urlparse
    from doctor import default_llm_probe, default_screenpipe_probe

    events = _realistic_day()
    seen = []
    class Handler(BaseHTTPRequestHandler):
        def log_message(self, *args):
            pass

        def reply(self, data):
            encoded = json.dumps(data).encode()
            self.send_response(200)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(encoded)))
            self.end_headers()
            self.wfile.write(encoded)

        def do_GET(self):
            path = urlparse(self.path)
            seen.append(path.path)
            if path.path == "/health":
                return self.reply({"status": "healthy"})
            assert self.headers.get("Authorization") == "Bearer fixture-token"
            if path.path == "/v1/models":
                return self.reply({"data": [{"id": "fixture-model"}]})
            assert path.path == "/search"
            params = parse_qs(path.query)
            assert params["include_cloud"] == ["false"]
            assert params["filter_pii"] == ["false"]
            offset = int(params["offset"][0])
            return self.reply({"data": events[offset:offset + 20],
                               "pagination": {"offset": offset, "total": len(events)}})

        def do_POST(self):
            seen.append(self.path)
            assert self.path == "/v1/chat/completions"
            assert self.headers.get("Authorization") == "Bearer fixture-token"
            body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
            assert body["model"] == "fixture-model"
            assert body["stream"] is False
            return self.reply({"choices": [{"finish_reason": "stop", "message": {
                "content": json.dumps({"verdict": "reuse", "target": "literature-review"})}}]})

    server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
    thread = Thread(target=server.serve_forever, daemon=True)
    thread.start()
    url = f"http://127.0.0.1:{server.server_port}"
    monkeypatch.setenv("LM_API_TOKEN", "fixture-token")
    monkeypatch.setenv("SCREENPIPE_TOKEN", "fixture-token")
    config = {**_base_config(), "backend": "local", "screenpipe": {"url": url},
              "local": {"endpoint": url + "/v1", "model": "fixture-model"}}
    skills = tmp_path / "skills"
    _seed_skills_dir(skills)
    try:
        assert default_screenpipe_probe(config)[0] == "ok"
        assert default_llm_probe(config)[0] == "ok"
        with httpx.Client(base_url=url) as search, httpx.Client(base_url=url + "/v1") as llm:
            backend = LocalBackend(url + "/v1", "fixture-model", client=llm,
                                   api_key="fixture-token")
            out = run(config, start_time="2026-04-17T00:00:00Z", end_time="2026-04-18T00:00:00Z",
                      out_dir=tmp_path / "proposed", screenpipe_client=search,
                      backend=backend, embedder=_keyword_embedder, skills_dir=skills,
                      screenpipe_token="fixture-token", now=lambda: "fixture")
            assert "literature-review" in (out / "report.md").read_text()
        assert seen.count("/search") == 7
        assert seen.count("/v1/chat/completions") == 3
    finally:
        server.shutdown()
        server.server_close()
        thread.join(timeout=5)