Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

1.9 KB
Raw Blame
"""A failed or missing corpus workload must block release publication."""

from pathlib import Path
import yaml

WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows"


def load(name):
    result = yaml.safe_load((WORKFLOWS / name).read_text())
    if True in result:
        result["on"] = result.pop(True)
    return result


def test_release_requires_successful_rag_scale_shards():
    jobs = load("release.yml")["jobs"]
    assert (
        jobs["rag-index-scale-gate"]["uses"]
        == "$/.github/workflows/rag-index-scale.yml"
    )
    assert jobs["rag-index-scale-gate"]["needs"] == ["version-check"]
    assert "rag-index-scale-gate" in jobs["build"]["needs"]
    condition = jobs["build"]["if"]
    assert "needs.rag-index-scale-gate.result == 'success'" in condition
    assert "||" not in condition


def test_corpus_is_release_only_and_failures_cannot_be_hidden():
    workflow = load("rag-index-scale.yml")
    assert set(workflow["on"]) == {"workflow_call", "workflow_dispatch"}
    job = workflow["jobs"]["rag-index-scale"]
    assert set(job["strategy"]["matrix"]["index-type"]) == {"flat", "hnsw"}
    assert not job.get("continue-on-error")
    steps = job["steps"]
    workload = next(step for step in steps if step.get("id") == "workload")
    assert workload["shell"] == "bash"  # Actions enables -e -o pipefail.
    assert not workload.get("continue-on-error")
    assert "scripts/ci/rag_index_workload.py" in workload["run"]
    assert "--documents 128 --pages 16 --dimension 1024" in workload["run"]
    assert "--max-rss-mib 768" in workload["run"]
    required_report = next(
        step
        for step in steps
        if step["name"] == "Require a completed scale run"
    )
    assert required_report["if"] == "always()"
    assert not required_report.get("continue-on-error")
    assert any(
        step.get("if") == "always()"
        and "actions/upload-artifact@" in step.get("uses", "")
        for step in steps
    )