Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

4.7 KB
Raw Blame
"""The journal-data release gate must run its probes instead of skipping them."""

# allow: no-sut-import — exercises the actual workflow and its JUnit validator.

import os
from pathlib import Path
import subprocess
import sys
from xml.etree import ElementTree

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".github/workflows/journal-data-integration.yml"
REQUIRED = {
    "test_openalex_sources",
    "test_doaj_journals",
    "test_predatory_list",
    "test_jabref_abbreviations",
    "test_openalex_institutions",
    "test_build_journal_quality_db",
    "test_runtime_accessor_can_score_real_journal",
    "test_dashboard_queries_against_real_db",
}


@pytest.fixture(scope="module")
def job():
    return yaml.safe_load(WORKFLOW.read_text())["jobs"]["download-and-build"]


def _steps(job):
    return {s["id"]: s for s in job["steps"] if "id" in s}


def _run(script, directory, *, extra_env=None):
    return subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", script],
        cwd=directory,
        env={
            **os.environ,
            "PATH": f"{Path(sys.executable).parent}:{os.environ.get('PATH', '')}",
            **(extra_env or {}),
        },
        capture_output=True,
        text=True,
        timeout=20,
        check=False,
    )


def test_real_integration_mode_and_supported_python_are_explicit(job):
    assert job["env"]["LDR_TESTING_WITH_MOCKS"] == "false"
    assert "if" not in job and not job.get("continue-on-error")
    versions = [
        s["with"]["python-version"]
        for s in job["steps"]
        if "python-version" in s.get("with", {})
    ]
    assert versions == ["3.14", "3.14"]
    steps = _steps(job)
    tests = steps["tests"]
    assert tests["shell"] == "bash"
    assert "if" not in tests and not tests.get("continue-on-error")
    assert (
        "tests/integration/test_journal_quality_release_gate.py" in tests["run"]
    )
    assert "--junitxml=test-results/journal-quality.xml" in tests["run"]
    assert steps["validate-results"]["if"] == "always()"
    assert not steps["validate-results"].get("continue-on-error")
    upload = next(
        s
        for s in job["steps"]
        if s.get("uses", "").startswith("actions/upload-artifact@")
    )
    assert upload["if"] == "always()"
    assert upload["with"]["path"] == "test-results/journal-quality.*"


@pytest.mark.parametrize(
    "outcome",
    [
        "passed",
        "all-skipped",
        "one-skipped",
        "failure",
        "error",
        "missing-case",
        "empty",
        "missing-report",
        "malformed",
    ],
)
def test_actual_validator_requires_every_probe_to_pass(job, tmp_path, outcome):
    report = tmp_path / "test-results/journal-quality.xml"
    if outcome != "missing-report":
        report.parent.mkdir()
        if outcome == "malformed":
            report.write_text("<testsuites>")
        else:
            suite = ElementTree.Element("testsuite")
            names = sorted(REQUIRED)
            if outcome == "missing-case":
                names = names[:-1]
            elif outcome == "empty":
                names = []
            for i, name in enumerate(names):
                case = ElementTree.SubElement(suite, "testcase", name=name)
                if outcome == "all-skipped" or (
                    outcome == "one-skipped" and i == 0
                ):
                    ElementTree.SubElement(case, "skipped")
                elif outcome in {"failure", "error"} and i == 0:
                    ElementTree.SubElement(case, outcome)
            ElementTree.ElementTree(suite).write(report)
    result = _run(_steps(job)["validate-results"]["run"], tmp_path)
    assert (result.returncode == 0) is (outcome == "passed"), (
        result.stdout + result.stderr
    )


@pytest.mark.parametrize("pytest_status", [0, 1, 5])
def test_pytest_failure_cannot_be_hidden_by_diagnostic_tee(
    job, tmp_path, pytest_status
):
    binary = tmp_path / "bin"
    binary.mkdir()
    pdm = binary / "pdm"
    pdm.write_text(f"#!/bin/sh\nexit {pytest_status}\n")
    pdm.chmod(0o755)
    result = _run(
        _steps(job)["tests"]["run"],
        tmp_path,
        extra_env={"PATH": f"{binary}:{os.environ.get('PATH', '')}"},
    )
    assert result.returncode == pytest_status, result.stdout + result.stderr


def test_release_gate_keeps_journal_validation_required():
    workflow = yaml.safe_load(
        (ROOT / ".github/workflows/release-gate.yml").read_text()
    )
    jobs = workflow["jobs"]
    journal = jobs["journal-data-integration"]
    assert journal["uses"] == "$/.github/workflows/journal-data-integration.yml"
    assert "if" not in journal and not journal.get("continue-on-error")
    assert "journal-data-integration" in jobs["release-gate-summary"]["needs"]