~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""The journal-data release gate must run its probes instead of skipping them."""
# allow: no-sut-import — exercises the actual workflow and its JUnit validator.
import os
from pathlib import Path
import subprocess
import sys
from xml.etree import ElementTree
import pytest
import yaml
ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".github/workflows/journal-data-integration.yml"
REQUIRED = {
"test_openalex_sources",
"test_doaj_journals",
"test_predatory_list",
"test_jabref_abbreviations",
"test_openalex_institutions",
"test_build_journal_quality_db",
"test_runtime_accessor_can_score_real_journal",
"test_dashboard_queries_against_real_db",
}
@pytest.fixture(scope="module")
def job():
return yaml.safe_load(WORKFLOW.read_text())["jobs"]["download-and-build"]
def _steps(job):
return {s["id"]: s for s in job["steps"] if "id" in s}
def _run(script, directory, *, extra_env=None):
return subprocess.run(
["bash", "-e", "-o", "pipefail", "-c", script],
cwd=directory,
env={
**os.environ,
"PATH": f"{Path(sys.executable).parent}:{os.environ.get('PATH', '')}",
**(extra_env or {}),
},
capture_output=True,
text=True,
timeout=20,
check=False,
)
def test_real_integration_mode_and_supported_python_are_explicit(job):
assert job["env"]["LDR_TESTING_WITH_MOCKS"] == "false"
assert "if" not in job and not job.get("continue-on-error")
versions = [
s["with"]["python-version"]
for s in job["steps"]
if "python-version" in s.get("with", {})
]
assert versions == ["3.14", "3.14"]
steps = _steps(job)
tests = steps["tests"]
assert tests["shell"] == "bash"
assert "if" not in tests and not tests.get("continue-on-error")
assert (
"tests/integration/test_journal_quality_release_gate.py" in tests["run"]
)
assert "--junitxml=test-results/journal-quality.xml" in tests["run"]
assert steps["validate-results"]["if"] == "always()"
assert not steps["validate-results"].get("continue-on-error")
upload = next(
s
for s in job["steps"]
if s.get("uses", "").startswith("actions/upload-artifact@")
)
assert upload["if"] == "always()"
assert upload["with"]["path"] == "test-results/journal-quality.*"
@pytest.mark.parametrize(
"outcome",
[
"passed",
"all-skipped",
"one-skipped",
"failure",
"error",
"missing-case",
"empty",
"missing-report",
"malformed",
],
)
def test_actual_validator_requires_every_probe_to_pass(job, tmp_path, outcome):
report = tmp_path / "test-results/journal-quality.xml"
if outcome != "missing-report":
report.parent.mkdir()
if outcome == "malformed":
report.write_text("<testsuites>")
else:
suite = ElementTree.Element("testsuite")
names = sorted(REQUIRED)
if outcome == "missing-case":
names = names[:-1]
elif outcome == "empty":
names = []
for i, name in enumerate(names):
case = ElementTree.SubElement(suite, "testcase", name=name)
if outcome == "all-skipped" or (
outcome == "one-skipped" and i == 0
):
ElementTree.SubElement(case, "skipped")
elif outcome in {"failure", "error"} and i == 0:
ElementTree.SubElement(case, outcome)
ElementTree.ElementTree(suite).write(report)
result = _run(_steps(job)["validate-results"]["run"], tmp_path)
assert (result.returncode == 0) is (outcome == "passed"), (
result.stdout + result.stderr
)
@pytest.mark.parametrize("pytest_status", [0, 1, 5])
def test_pytest_failure_cannot_be_hidden_by_diagnostic_tee(
job, tmp_path, pytest_status
):
binary = tmp_path / "bin"
binary.mkdir()
pdm = binary / "pdm"
pdm.write_text(f"#!/bin/sh\nexit {pytest_status}\n")
pdm.chmod(0o755)
result = _run(
_steps(job)["tests"]["run"],
tmp_path,
extra_env={"PATH": f"{binary}:{os.environ.get('PATH', '')}"},
)
assert result.returncode == pytest_status, result.stdout + result.stderr
def test_release_gate_keeps_journal_validation_required():
workflow = yaml.safe_load(
(ROOT / ".github/workflows/release-gate.yml").read_text()
)
jobs = workflow["jobs"]
journal = jobs["journal-data-integration"]
assert journal["uses"] == "$/.github/workflows/journal-data-integration.yml"
assert "if" not in journal and not journal.get("continue-on-error")
assert "journal-data-integration" in jobs["release-gate-summary"]["needs"]