Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Skip product-claim gates for -dev publish tags (#93)

Christianf573cf712ce2
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index e787ad5..d467722 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -114,7 +114,7 @@ jobs:
unzip -q light.zip -d gate-input/light
unzip -q heavy.zip -d gate-input/heavy
python scripts/assemble_release_artifacts.py --light gate-input/light --heavy gate-input/heavy --output gate-artifacts
- python scripts/check_release_readiness.py gate-artifacts
+ python scripts/check_release_readiness.py gate-artifacts --tag "$GITHUB_REF_NAME"
- uses: actions/upload-artifact@v4
with:
name: release-evidence-${{ github.ref_name }}
diff --git a/deploy/release-artifacts/README.md b/deploy/release-artifacts/README.md
index c743f56..f820a78 100644
--- a/deploy/release-artifacts/README.md
+++ b/deploy/release-artifacts/README.md
@@ -1,7 +1,8 @@
# Production validation artifact contract
-Run `scripts/check_release_readiness.py DIR` before creating an RC tag. `DIR`
-must contain the following files produced against the exact candidate image:
+Run `scripts/check_release_readiness.py DIR` before creating an RC or stable
+tag. `DIR` must contain the following files produced against the exact
+candidate image:
- `smoke.json`: `light` and `heavy`, each with `unexpected_5xx: 0` and true
flow flags for authentication, brain creation, plain/structured ingestion,
@@ -18,6 +19,11 @@ The public ledger must also contain a representative LongMemEval result and a
WANDS representative Search row. LoCoMo and BEAM rows retain their explicit
sample scopes and are not promoted into broader claims by this gate.
+Dev tags (`vX.Y.Z-dev` / `vX.Y.Z-dev.N`) still require smoke, restore, and
+security on the exact image. They skip the public-ledger and latency SLO
+claims so a working image can publish before those product scores exist.
+Pass `--tag` so the publish workflow can apply that distinction.
+
`scripts/production_smoke.py exercise` produces profile-scoped smoke, latency,
and pre-restore state evidence; `verify-restore` produces the matching restore
report. `scripts/assemble_release_artifacts.py` combines successful light and
diff --git a/scripts/check_release_readiness.py b/scripts/check_release_readiness.py
index a6ef8dc..ae9cdb2 100644
--- a/scripts/check_release_readiness.py
+++ b/scripts/check_release_readiness.py
@@ -10,6 +10,11 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+from scripts.release_version import requires_product_claim_gates
+
COUNT_KEYS = ("brains", "nodes", "edges", "chunks", "vectors", "observations")
REQUIRED_FLOWS = {
"authentication",
@@ -109,18 +114,25 @@ def _check_security(data: dict, artifact_dir: Path) -> None:
raise RuntimeError("The exact image SBOM is missing")
+def evaluate(artifact_dir: Path, *, tag: str | None = None) -> None:
+ claim_gates = requires_product_claim_gates(tag)
+ if claim_gates:
+ _check_public_ledgers()
+ _check_smoke(_read(artifact_dir / "smoke.json"))
+ if claim_gates:
+ _check_latency(_read(artifact_dir / "latency.json"))
+ _check_restore("light", _read(artifact_dir / "restore-light.json"))
+ _check_restore("heavy", _read(artifact_dir / "restore-heavy.json"))
+ _check_security(_read(artifact_dir / "security.json"), artifact_dir)
+
+
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("artifact_dir", type=Path)
+ parser.add_argument("--tag")
args = parser.parse_args()
- artifact_dir = args.artifact_dir.resolve()
try:
- _check_public_ledgers()
- _check_smoke(_read(artifact_dir / "smoke.json"))
- _check_latency(_read(artifact_dir / "latency.json"))
- _check_restore("light", _read(artifact_dir / "restore-light.json"))
- _check_restore("heavy", _read(artifact_dir / "restore-heavy.json"))
- _check_security(_read(artifact_dir / "security.json"), artifact_dir)
+ evaluate(args.artifact_dir.resolve(), tag=args.tag)
except (KeyError, TypeError, ValueError, RuntimeError) as exc:
print(f"release gate failed: {exc}", file=sys.stderr)
return 1
diff --git a/scripts/release_version.py b/scripts/release_version.py
index 239a304..8e4c096 100644
--- a/scripts/release_version.py
+++ b/scripts/release_version.py
@@ -18,13 +18,17 @@ _DEV_TAG = re.compile(
class ReleaseVersion:
tag_version: str
project_version: str
+ channel: str
+
+
+PRODUCT_CLAIM_CHANNELS = frozenset({"stable", "rc"})
def parse_release_tag(tag: str) -> ReleaseVersion:
"""Return the container tag version and its PEP 440 project equivalent."""
if match := _STABLE_TAG.fullmatch(tag):
base = match.group("base")
- return ReleaseVersion(tag_version=base, project_version=base)
+ return ReleaseVersion(tag_version=base, project_version=base, channel="stable")
if match := _RC_TAG.fullmatch(tag):
base = match.group("base")
@@ -32,6 +36,7 @@ def parse_release_tag(tag: str) -> ReleaseVersion:
return ReleaseVersion(
tag_version=f"{base}-rc.{number}",
project_version=f"{base}rc{number}",
+ channel="rc",
)
if match := _DEV_TAG.fullmatch(tag):
@@ -43,6 +48,7 @@ def parse_release_tag(tag: str) -> ReleaseVersion:
return ReleaseVersion(
tag_version=tag_version,
project_version=f"{base}.dev{number}",
+ channel="dev",
)
raise ValueError(
@@ -51,6 +57,12 @@ def parse_release_tag(tag: str) -> ReleaseVersion:
)
+def requires_product_claim_gates(tag: str | None) -> bool:
+ if not tag:
+ return True
+ return parse_release_tag(tag).channel in PRODUCT_CLAIM_CHANNELS
+
+
def main(argv: list[str]) -> int:
if len(argv) != 2:
print(f"Usage: {argv[0]} <release-tag>", file=sys.stderr)
diff --git a/tests/test_deploy_production_contract.py b/tests/test_deploy_production_contract.py
index ad841e8..c3722a6 100644
--- a/tests/test_deploy_production_contract.py
+++ b/tests/test_deploy_production_contract.py
@@ -11,9 +11,10 @@ import pytest
import yaml
from deploy.backup_restore import verify_backup
-from scripts.check_release_readiness import _check_latency
+from scripts.check_release_readiness import _check_latency, evaluate
from scripts.openai_ci_stub import _embedding
from scripts.production_smoke import _latency_summary
+from scripts.release_version import parse_release_tag, requires_product_claim_gates
ROOT = Path(__file__).resolve().parents[1]
@@ -198,11 +199,91 @@ def test_release_workflow_enforces_artifact_gate_before_publish():
encoding="utf-8"
)
assert "check_release_readiness.py gate-artifacts" in workflow
+ assert '--tag "$GITHUB_REF_NAME"' in workflow
assert "needs: [validate-tag, verify-required-checks]" in workflow
assert 'python scripts/release_version.py "$GITHUB_REF_NAME"' in workflow
assert "steps.tag.outputs.project_version" in workflow
+def _gate_artifacts(tmp_path: Path, *, missing_flow: bool = False) -> Path:
+ tmp_path.mkdir(parents=True, exist_ok=True)
+ flows = {
+ "authentication": True,
+ "brain_creation": True,
+ "plain_ingestion": True,
+ "structured_ingestion": True,
+ "task_completion": True,
+ "context_retrieval": not missing_flow,
+ "search": True,
+ "console": True,
+ "mcp": True,
+ }
+ smoke = {
+ profile: {"flows": flows, "unexpected_5xx": 0} for profile in ("light", "heavy")
+ }
+ counts = {
+ "brains": 1,
+ "nodes": 2,
+ "edges": 1,
+ "chunks": 3,
+ "vectors": 3,
+ "observations": 0,
+ }
+ restore = {
+ "before": counts,
+ "after": counts,
+ "retrieval_spot_checks": [{"match": True} for _ in range(10)],
+ }
+ latency = {
+ profile: {
+ "context": {
+ "p50_ms": 5000,
+ "p95_ms": 5500,
+ "p99_ms": 5600,
+ "online_llm_retrieval_loops": 0,
+ },
+ "search": {
+ "p50_ms": 12,
+ "p95_ms": 20,
+ "p99_ms": 25,
+ "excludes_embed_query": True,
+ },
+ }
+ for profile in ("light", "heavy")
+ }
+ (tmp_path / "brainapi.spdx.json").write_text("{}\n", encoding="utf-8")
+ (tmp_path / "smoke.json").write_text(json.dumps(smoke), encoding="utf-8")
+ (tmp_path / "latency.json").write_text(json.dumps(latency), encoding="utf-8")
+ (tmp_path / "restore-light.json").write_text(json.dumps(restore), encoding="utf-8")
+ (tmp_path / "restore-heavy.json").write_text(json.dumps(restore), encoding="utf-8")
+ (tmp_path / "security.json").write_text(
+ json.dumps(
+ {
+ "high": 0,
+ "critical": 0,
+ "image_digest": "sha256:" + ("a" * 64),
+ "archive_sha256": "b" * 64,
+ "sbom": "brainapi.spdx.json",
+ }
+ ),
+ encoding="utf-8",
+ )
+ return tmp_path
+
+
+def test_dev_channel_skips_ledger_and_latency_claim_gates(tmp_path):
+ artifacts = _gate_artifacts(tmp_path)
+ assert parse_release_tag("v2.19.1-dev.0").channel == "dev"
+ assert requires_product_claim_gates("v2.19.1-dev.0") is False
+ evaluate(artifacts, tag="v2.19.1-dev.0")
+ with pytest.raises(RuntimeError, match="LongMemEval"):
+ evaluate(artifacts)
+ with pytest.raises(RuntimeError, match="LongMemEval"):
+ evaluate(artifacts, tag="v2.19.1-rc.1")
+ with pytest.raises(RuntimeError, match="missing flows"):
+ evaluate(_gate_artifacts(tmp_path / "broken", missing_flow=True), tag="v2.19.1-dev.0")
+
+
def test_mcp_has_only_one_lifespan_definition():
source = (ROOT / "src" / "services" / "mcp" / "app.py").read_text(
encoding="utf-8"
diff --git a/tests/test_release_version.py b/tests/test_release_version.py
index e5d1f05..8131272 100644
--- a/tests/test_release_version.py
+++ b/tests/test_release_version.py
@@ -8,10 +8,10 @@ from scripts.release_version import ReleaseVersion, parse_release_tag
@pytest.mark.parametrize(
("tag", "expected"),
[
- ("v2.19.0", ReleaseVersion("2.19.0", "2.19.0")),
- ("v2.19.0-rc.1", ReleaseVersion("2.19.0-rc.1", "2.19.0rc1")),
- ("v2.19.0-dev", ReleaseVersion("2.19.0-dev", "2.19.0.dev0")),
- ("v2.19.0-dev.3", ReleaseVersion("2.19.0-dev.3", "2.19.0.dev3")),
+ ("v2.19.0", ReleaseVersion("2.19.0", "2.19.0", "stable")),
+ ("v2.19.0-rc.1", ReleaseVersion("2.19.0-rc.1", "2.19.0rc1", "rc")),
+ ("v2.19.0-dev", ReleaseVersion("2.19.0-dev", "2.19.0.dev0", "dev")),
+ ("v2.19.0-dev.3", ReleaseVersion("2.19.0-dev.3", "2.19.0.dev3", "dev")),
],
)
def test_parse_release_tag(tag: str, expected: ReleaseVersion):