Skip to content

πŸ’« Toolkit to help you get started with SDD or any other process!

Validate community preset submissions before opening catalog PRs (#4787)

Manfred Riemcd5db8f9b077
diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py
new file mode 100644
index 000000000..ebc64a6cf
--- /dev/null
+++ b/.github/scripts/validate_community_preset.py
@@ -0,0 +1,532 @@
+"""Verify a community preset submission and its generated catalog changes.
+
+Exit 1: confirmed submission mismatch; 2: check blocked; 3: generated files
+need repair. The workflow decides labels and PR creation from these outcomes.
+"""
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urlsplit
+
+try:
+ import yaml
+except ImportError:
+ print("BLOCKED: PyYAML is unavailable; cannot inspect published preset.yml")
+ sys.exit(2)
+
+MAX_MANIFEST_COUNT = 100
+MAX_MANIFEST_SIZE = 1024 * 1024
+MAX_TOTAL_MANIFEST_SIZE = 10 * 1024 * 1024
+
+
+class SubmissionMismatch(Exception):
+ pass
+
+
+class Blocked(Exception):
+ pass
+
+
+class GeneratedError(Exception):
+ pass
+
+
+def read_text(
+ path: Path, context: str, error_type: type[Exception] = Blocked
+) -> str:
+ try:
+ return path.read_text(encoding="utf-8")
+ except (OSError, UnicodeError) as exc:
+ raise error_type(f"cannot read {context} ({path}): {exc}") from exc
+
+
+def read_json(path: Path, context: str, error_type: type[Exception]) -> dict:
+ try:
+ value = json.loads(read_text(path, context, error_type))
+ except json.JSONDecodeError as exc:
+ raise error_type(f"{context} is invalid JSON: {exc}") from exc
+ if not isinstance(value, dict):
+ raise error_type(f"{context} must be a JSON object")
+ return value
+
+
+def field(issue: dict, key: str) -> str:
+ value = issue.get(key)
+ if not isinstance(value, str) or not value.strip():
+ raise SubmissionMismatch(f"missing or invalid issue field: {key}")
+ return value.strip()
+
+
+def csv_values(value: str) -> list[str]:
+ return [part.strip() for part in value.split(",") if part.strip()]
+
+
+def repository_parts(url: str) -> tuple[str, str]:
+ parsed = urlsplit(url)
+ match = re.fullmatch(r"/([^/]+)/([^/]+)/?", parsed.path)
+ if parsed.scheme != "https" or parsed.netloc.lower() != "github.com" or not match:
+ raise SubmissionMismatch(f"invalid GitHub repository URL: {url}")
+ return match[1].lower(), match[2].lower()
+
+
+def release_tag(issue: dict) -> str:
+ owner, repo = repository_parts(field(issue, "repository"))
+ url = field(issue, "download_url")
+ parsed = urlsplit(url)
+ prefix = f"/{owner}/{repo}/"
+ path = parsed.path
+ if parsed.scheme != "https" or parsed.netloc.lower() != "github.com":
+ raise SubmissionMismatch(f"download URL is not a GitHub URL: {url}")
+ if not path.lower().startswith(prefix):
+ raise SubmissionMismatch("download URL does not belong to the submitted repository")
+ suffix = path[len(prefix):]
+ archive = re.fullmatch(r"archive/refs/tags/([^/]+)\.zip", suffix)
+ release = re.fullmatch(r"releases/download/([^/]+)/[^/]+\.zip", suffix)
+ if not (archive or release) or parsed.query or parsed.fragment:
+ raise SubmissionMismatch(f"download URL is not a pinned ZIP release: {url}")
+ tag = (archive or release)[1]
+ version = field(issue, "version")
+ if not re.fullmatch(r"\d+\.\d+\.\d+", version) or not re.fullmatch(
+ rf"(?:v?{re.escape(version)}|.+-v?{re.escape(version)})", tag
+ ):
+ raise SubmissionMismatch(
+ f"release tag {tag!r} does not match submitted version {version!r}"
+ )
+ return tag
+
+
+def normalized_release_asset(filename: str) -> str:
+ return re.sub(
+ r"(?<!\d)v?\d+\.\d+\.\d+(?!\d)",
+ "{version}",
+ filename,
+ flags=re.IGNORECASE,
+ )
+
+
+def published_manifest(archive_path: Path, preset_id: str) -> tuple[dict, bool]:
+ try:
+ with zipfile.ZipFile(archive_path) as archive:
+ matching = []
+ invalid = []
+ manifests = [
+ member for member in archive.infolist()
+ if not member.is_dir()
+ and member.filename.rsplit("/", 1)[-1] == "preset.yml"
+ ]
+ if len(manifests) > MAX_MANIFEST_COUNT:
+ raise SubmissionMismatch(
+ f"archive contains more than {MAX_MANIFEST_COUNT} preset.yml files"
+ )
+ total_size = sum(member.file_size for member in manifests)
+ if total_size > MAX_TOTAL_MANIFEST_SIZE:
+ raise SubmissionMismatch(
+ "archive preset.yml files exceed the "
+ f"{MAX_TOTAL_MANIFEST_SIZE // (1024 * 1024)} MiB total limit"
+ )
+ for member in manifests:
+ try:
+ if member.file_size > MAX_MANIFEST_SIZE:
+ raise ValueError("preset.yml exceeds 1 MiB")
+ with archive.open(member) as stream:
+ data = yaml.safe_load(stream.read().decode("utf-8"))
+ except (yaml.YAMLError, UnicodeError, ValueError) as exc:
+ invalid.append(f"{member.filename}: {exc}")
+ continue
+ if isinstance(data, dict) and isinstance(data.get("preset"), dict):
+ if data["preset"].get("id") == preset_id:
+ matching.append((member.filename, data))
+ except FileNotFoundError as exc:
+ raise Blocked(f"downloaded archive is unavailable at {archive_path}: {exc}") from exc
+ except PermissionError as exc:
+ raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc
+ except OSError as exc:
+ raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc
+ except (zipfile.BadZipFile, EOFError) as exc:
+ raise SubmissionMismatch(f"downloaded archive is not a readable ZIP: {exc}") from exc
+ if len(matching) != 1:
+ if not matching and invalid:
+ raise SubmissionMismatch(
+ f"no published preset.yml for {preset_id!r}; invalid manifests: "
+ + "; ".join(invalid)
+ )
+ raise SubmissionMismatch(
+ f"expected one published preset.yml for {preset_id!r}, found {len(matching)}"
+ )
+ return matching[0][1], len(manifests) == 1
+
+
+def required_extensions(manifest: dict) -> list[str]:
+ requires = manifest.get("requires")
+ if not isinstance(requires, dict):
+ raise SubmissionMismatch("published preset.yml has no requires mapping")
+ extensions = requires.get("extensions", [])
+ if not isinstance(extensions, list):
+ raise SubmissionMismatch("published requires.extensions must be a list")
+ result = []
+ for item in extensions:
+ if isinstance(item, str):
+ result.append(item)
+ elif isinstance(item, dict) and isinstance(item.get("id"), str):
+ if item.get("required", True) is not False:
+ result.append(item["id"])
+ else:
+ raise SubmissionMismatch("invalid published requires.extensions entry")
+ return result
+
+
+def check_readme(text: str, issue: dict, *, single_preset_archive: bool) -> None:
+ expected = field(issue, "download_url")
+ preset_id = field(issue, "preset_id")
+ owner, repo = repository_parts(field(issue, "repository"))
+ submitted_scope = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", release_tag(issue))
+ accepted_scopes = {preset_id}
+ if submitted_scope:
+ accepted_scopes.add(submitted_scope[1])
+ expected_path = urlsplit(expected).path.split("/")
+ accepted = False
+ for match in re.finditer(
+ r"(?<![\w-])specify[ \t]+preset[ \t]+add[ \t]+"
+ r"(?P<option>--from|--dev|[a-z][a-z0-9-]*)"
+ r"(?:[ \t]+(?P<value>[^\s`]+))?",
+ text,
+ ):
+ option = match["option"]
+ raw_value = match["value"] or ""
+ if option == "--from":
+ value = raw_value.strip("'\"<>(),.;")
+ if value == expected:
+ accepted = True
+ continue
+ parsed = urlsplit(value)
+ parts = parsed.path.split("/")
+ tag = (
+ parts[5] if len(parts) > 5 and parts[3] == "releases"
+ else parts[6].removesuffix(".zip")
+ if len(parts) > 6 and parts[3] == "archive"
+ else ""
+ )
+ scoped = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", tag)
+ same_repository = (
+ parsed.netloc.lower() == "github.com"
+ and parsed.path.lower().startswith(f"/{owner}/{repo}/")
+ )
+ # An unscoped tag alone does not identify a preset in a monorepo.
+ same_release_asset = (
+ len(expected_path) > 6
+ and expected_path[3:5] == ["releases", "download"]
+ and len(parts) > 6
+ and parts[3:5] == ["releases", "download"]
+ and normalized_release_asset(expected_path[6])
+ == normalized_release_asset(parts[6])
+ )
+ unscoped_archive = (
+ single_preset_archive
+ and submitted_scope is None
+ and expected_path[3:6] == ["archive", "refs", "tags"]
+ and parts[3:6] == ["archive", "refs", "tags"]
+ )
+ if (scoped and scoped[1] in accepted_scopes) or (
+ same_repository and not scoped and (same_release_asset or unscoped_archive)
+ ):
+ raise SubmissionMismatch(
+ f"README --from URL for {preset_id} differs from Download URL: {value}"
+ )
+ elif option == "--dev":
+ value = raw_value.strip("'\"")
+ if value and not value.startswith("-"):
+ accepted = True
+ elif option == preset_id:
+ accepted = True
+ if not accepted:
+ raise SubmissionMismatch(
+ "README has no valid specify preset add command for the submitted preset"
+ )
+
+
+def count_items(value: str) -> int:
+ return sum(line.lstrip().startswith("- ") for line in value.splitlines())
+
+
+def expected_values(issue: dict, manifest: dict, digest: str) -> dict:
+ pack = manifest.get("preset")
+ requires = manifest.get("requires")
+ if not isinstance(pack, dict) or not isinstance(requires, dict):
+ raise SubmissionMismatch("published preset.yml lacks preset or requires metadata")
+ extension_ids = csv_values(issue.get("required_extensions", ""))
+ actual_extensions = required_extensions(manifest)
+ comparisons = {
+ "preset.id": (pack.get("id"), field(issue, "preset_id")),
+ "preset.version": (pack.get("version"), field(issue, "version")),
+ "requires.speckit_version": (
+ requires.get("speckit_version"), field(issue, "speckit_version")
+ ),
+ "requires.extensions": (sorted(actual_extensions), sorted(extension_ids)),
+ }
+ for name, (actual, expected) in comparisons.items():
+ if actual != expected:
+ raise SubmissionMismatch(
+ f"published {name} {actual!r} differs from submitted {expected!r}"
+ )
+ dependencies = {"speckit_version": field(issue, "speckit_version")}
+ if extension_ids:
+ dependencies["extensions"] = extension_ids
+ provided = {
+ "templates": count_items(field(issue, "templates_provided")),
+ "commands": count_items(field(issue, "commands_provided")),
+ }
+ scripts = issue.get("scripts_count", "0")
+ if scripts:
+ try:
+ number = int(scripts)
+ except (ValueError, TypeError) as exc:
+ raise SubmissionMismatch("scripts_count must be a non-negative integer") from exc
+ if number < 0:
+ raise SubmissionMismatch("scripts_count must be a non-negative integer")
+ if number:
+ provided["scripts"] = number
+ return {
+ "id": field(issue, "preset_id"),
+ "name": field(issue, "preset_name"),
+ "version": field(issue, "version"),
+ "description": field(issue, "description"),
+ "author": field(issue, "author"),
+ "repository": field(issue, "repository"),
+ "homepage": field(issue, "repository"),
+ "download_url": field(issue, "download_url"),
+ "sha256": digest,
+ "documentation": field(issue, "documentation"),
+ "license": field(issue, "license"),
+ "requires": dependencies,
+ "provides": provided,
+ "tags": csv_values(field(issue, "tags")),
+ }
+
+
+def submission(args: argparse.Namespace) -> None:
+ issue = read_json(args.issue, "issue input", Blocked)
+ release_tag(issue)
+ manifest, single_preset_archive = published_manifest(
+ args.archive, field(issue, "preset_id")
+ )
+ check_readme(
+ read_text(args.readme, "fetched README"),
+ issue,
+ single_preset_archive=single_preset_archive,
+ )
+ try:
+ with args.archive.open("rb") as archive:
+ digest = hashlib.file_digest(archive, "sha256").hexdigest()
+ except OSError as exc:
+ raise Blocked(f"cannot hash downloaded archive: {exc}") from exc
+ expected = expected_values(issue, manifest, digest)
+ catalog = read_json(args.catalog, "original catalog", Blocked)
+ entries = catalog.get("presets")
+ if not isinstance(entries, dict):
+ raise Blocked("original catalog has no presets object")
+ previous = entries.get(expected["id"])
+ if previous is not None and not isinstance(previous, dict):
+ raise Blocked("original catalog entry is not an object")
+ if previous is not None and not isinstance(previous.get("created_at"), str):
+ raise Blocked("original catalog entry has no created_at to preserve")
+ original_rows = documentation_rows(
+ read_text(args.docs, "original documentation", Blocked), Blocked
+ )
+ expected_row = documentation_row(expected)
+ previous_row = None
+ if previous is not None:
+ previous_row = existing_documentation_row(original_rows, previous)
+ snapshot = {
+ "expected": expected,
+ "created_at": previous.get("created_at") if previous else None,
+ "expected_timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z"),
+ "documentation": {
+ "expected_row_count": original_rows.count(expected_row),
+ "previous_row": previous_row,
+ "previous_row_count": original_rows.count(previous_row)
+ if previous_row is not None else 0,
+ },
+ }
+ try:
+ args.snapshot.write_text(json.dumps(snapshot), encoding="utf-8")
+ except OSError as exc:
+ raise Blocked(f"cannot write verifier snapshot: {exc}") from exc
+ print("PASSED: published preset.yml, release tag, README, and issue fields agree")
+
+
+def documentation_row(entry: dict) -> str:
+ def cell(value: str) -> str:
+ return value.replace("\\", r"\\").replace("|", r"\|").replace("\n", " ")
+
+ provided = entry["provides"]
+ quantities = [
+ f"{count} {name[:-1] if count == 1 else name}"
+ for name, count in provided.items() if count
+ ]
+ dependencies = entry["requires"].get("extensions", [])
+ requires = ", ".join(f"{name} extension" for name in dependencies) or "β€”"
+ repo = entry["repository"].rstrip("/").rsplit("/", 1)[-1]
+ return (
+ f"| {cell(entry['name'])} | {cell(entry['description'])} | "
+ f"{', '.join(quantities)} | {requires} | "
+ f"[{repo}]({entry['repository']}) |"
+ )
+
+
+def markdown_table_cells(row: str) -> list[str]:
+ cells = []
+ cell = []
+ escaped = False
+ for character in row:
+ if escaped:
+ if character in ("\\", "|"):
+ cell.append(character)
+ else:
+ cell.extend(("\\", character))
+ escaped = False
+ elif character == "\\":
+ escaped = True
+ elif character == "|":
+ cells.append("".join(cell).strip())
+ cell = []
+ else:
+ cell.append(character)
+ if escaped:
+ cell.append("\\")
+ cells.append("".join(cell).strip())
+ return cells
+
+
+def documentation_rows(text: str, error_type: type[Exception]) -> list[str]:
+ lines = text.splitlines()
+ try:
+ start = next(i for i, line in enumerate(lines) if line.startswith("| Preset |"))
+ except StopIteration as exc:
+ raise error_type("documentation has no Community Presets table") from exc
+ rows = []
+ for line in lines[start + 2:]:
+ if not line.startswith("|"):
+ break
+ rows.append(line.strip())
+ return rows
+
+
+def existing_documentation_row(rows: list[str], entry: dict) -> str:
+ name = entry.get("name")
+ repository = entry.get("repository")
+ if not isinstance(name, str) or not isinstance(repository, str):
+ raise Blocked("original catalog entry lacks documentation row identity")
+ matches = []
+ for row in rows:
+ cells = markdown_table_cells(row)
+ if len(cells) < 3 or cells[1] != name:
+ continue
+ url_cell = cells[-2] if cells[-1] == "" else cells[-1]
+ link = re.fullmatch(r"\[[^\]]*\]\(([^)]+)\)", url_cell)
+ if link and link[1].rstrip("/") == repository.rstrip("/"):
+ matches.append(row)
+ if len(matches) != 1:
+ raise Blocked(
+ "original documentation must contain exactly one row matching the "
+ f"catalog name and repository for {name!r}; found {len(matches)}"
+ )
+ return matches[0]
+
+
+def generated(args: argparse.Namespace) -> None:
+ snapshot = read_json(args.snapshot, "verifier snapshot", Blocked)
+ expected = snapshot.get("expected")
+ if not isinstance(expected, dict) or not isinstance(expected.get("id"), str):
+ raise Blocked("verifier snapshot lacks validated expected values")
+ expected_timestamp = snapshot.get("expected_timestamp")
+ if not isinstance(expected_timestamp, str):
+ raise Blocked("verifier snapshot lacks the expected UTC timestamp")
+ catalog = read_json(args.catalog, "generated catalog", GeneratedError)
+ entries = catalog.get("presets")
+ if not isinstance(entries, dict):
+ raise GeneratedError("generated catalog has no presets object")
+ keys = list(entries)
+ if keys != sorted(keys):
+ raise GeneratedError("catalog preset IDs are not in alphabetical order")
+ entry = entries.get(expected["id"])
+ if not isinstance(entry, dict):
+ raise GeneratedError(f"catalog is missing entry {expected['id']}")
+ for key, value in expected.items():
+ if entry.get(key) != value:
+ raise GeneratedError(f"catalog {key} does not match validated value: {value!r}")
+ if snapshot.get("created_at") is not None:
+ if entry.get("created_at") != snapshot["created_at"]:
+ raise GeneratedError("catalog created_at was not preserved on update")
+ elif entry.get("created_at") != expected_timestamp:
+ raise GeneratedError("new catalog created_at does not match the expected UTC date")
+ if entry.get("updated_at") != expected_timestamp:
+ raise GeneratedError("catalog entry updated_at does not match the expected UTC date")
+ if catalog.get("updated_at") != expected_timestamp:
+ raise GeneratedError("catalog top-level updated_at does not match the expected UTC date")
+ documentation = snapshot.get("documentation")
+ if not isinstance(documentation, dict):
+ raise Blocked("verifier snapshot lacks original documentation state")
+ expected_row_count = documentation.get("expected_row_count")
+ previous_row = documentation.get("previous_row")
+ previous_row_count = documentation.get("previous_row_count")
+ if (
+ not isinstance(expected_row_count, int)
+ or isinstance(expected_row_count, bool)
+ or expected_row_count < 0
+ or previous_row is not None and not isinstance(previous_row, str)
+ or not isinstance(previous_row_count, int)
+ or isinstance(previous_row_count, bool)
+ or previous_row_count < 0
+ ):
+ raise Blocked("verifier snapshot has invalid original documentation state")
+ rows = documentation_rows(
+ read_text(args.docs, "generated documentation", GeneratedError),
+ GeneratedError,
+ )
+ names = [markdown_table_cells(row)[1] for row in rows]
+ if names != sorted(names, key=str.casefold):
+ raise GeneratedError("documentation preset names are not in alphabetical order")
+ expected_row = documentation_row(expected)
+ expected_generated_count = expected_row_count + (
+ 0 if previous_row == expected_row else 1
+ )
+ if rows.count(expected_row) != expected_generated_count:
+ raise GeneratedError(f"documentation row does not match validated values: {expected_row}")
+ if previous_row is not None and previous_row != expected_row:
+ expected_previous_count = max(0, previous_row_count - 1)
+ if rows.count(previous_row) != expected_previous_count:
+ raise GeneratedError("previous documentation row was not replaced")
+ print("PASSED: generated catalog and documentation match validated submission")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("phase", choices=("submission", "generated"))
+ for name in ("issue", "archive", "readme", "catalog", "docs", "snapshot"):
+ parser.add_argument(f"--{name}", required=True, type=Path)
+ args = parser.parse_args()
+ try:
+ if args.phase == "submission":
+ submission(args)
+ else:
+ generated(args)
+ except SubmissionMismatch as exc:
+ print(f"FAILED: {exc}")
+ return 1
+ except Blocked as exc:
+ print(f"BLOCKED: {exc}")
+ return 2
+ except GeneratedError as exc:
+ print(f"REPAIR: {exc}")
+ return 3
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.yml
index 5b6cb1081..5fd10b3cf 100644
--- a/.github/workflows/add-community-preset.lock.yml
+++ b/.github/workflows/add-community-preset.lock.yml
@@ -1,5 +1,5 @@
-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e692354fdb52cf49c3f46096529618b23eb09336fe58fe2392b77210b374b568","body_hash":"cd1920f05bf8608b46191a3802a1c2f6c595ab3ce28034116e3346d4988bed4a","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}}
-# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]}
+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"67f2a3313f59103002b9e362e4d16ad1c3ef4035afb6c96f3446a345e4da4852","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}}
+# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]}
# This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md
#
# ___ _ _
@@ -41,6 +41,7 @@
# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9)
# - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+# - actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
# - github/gh-aw-actions/setup@5e508589e03a7757a7e05b26e834292f5445bfb6 # v0.88.7
#
@@ -491,6 +492,15 @@ jobs:
with:
name: activation
path: /tmp/gh-aw
+ - continue-on-error: true
+ name: Set up Python for preset verification
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.13"
+ - continue-on-error: true
+ name: Install preset verifier dependency
+ run: python3 -m pip install 'PyYAML==6.0.3'
+
- name: Configure Git credentials
env:
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md
index f6ccf01c8..80608481e 100644
--- a/.github/workflows/add-community-preset.md
+++ b/.github/workflows/add-community-preset.md
@@ -39,6 +39,16 @@ permissions:
checkout:
fetch-depth: 0
+steps:
+ - name: Set up Python for preset verification
+ continue-on-error: true
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.13"
+ - name: Install preset verifier dependency
+ continue-on-error: true
+ run: python3 -m pip install 'PyYAML==6.0.3'
+
safe-outputs:
noop:
report-as-issue: false
@@ -130,7 +140,8 @@ deciding pass/fail:
### 2c. Repository validation
- Fetch the repository URL β€” confirm it exists and is publicly accessible
-- Confirm the repository contains a `preset.yml` file
+- Confirm the repository contains a `preset.yml` file (in the preset's
+ subdirectory for a monorepo)
- Confirm the repository contains a `LICENSE` file
> The README requirement is enforced once, in **Step 2d**, against the specific file the
@@ -170,10 +181,15 @@ preset** β€” not just any file named `README.md`, and not a product/framework pi
- `specify preset add --dev <path>`
A `specify preset add --from <url>` command only counts when its `<url>` **matches the
- submitted Download URL exactly**. A `--from` command pointing at a *different* URL does
- **not** satisfy the install-command requirement (treat it as if absent) β€” but the README
- may still pass on one of the other accepted forms (`specify preset add <preset-id>` or
- `specify preset add --dev <path>`).
+ submitted Download URL exactly**. If the README also contains a `--from` release URL
+ identifiable as this preset by its tag scope or matching release asset that differs
+ from the Download URL, **fail** even if another accepted command (`specify preset add
+ <preset-id>` or `specify preset add --dev <path>`) is present. Do not flag a different
+ preset's unscoped release URL in a monorepo as stale. Bare archive tags are only
+ compared when the downloaded archive contains one `preset.yml`: a bare tag alone
+ cannot identify which preset it belongs to in a multi-preset archive. A README
+ with only a valid `--dev` command remains acceptable. The verifier in Step 2g
+ enforces this comparison.
If **no** accepted `specify preset add ...` command is present, the README is treated as a
generic description/pitch rather than preset-usage documentation β€” **fail this check** and
@@ -258,6 +274,35 @@ substitute for fetching the archive. Never execute downloaded content.
- Confirm that all required checkboxes in the Testing Checklist and Submission
Requirements sections are checked (`[x]`)
+### 2g. Reproducible published-artifact and README comparison
+
+After the pinned download has returned HTTP 200 and the optional checksum comparison
+has succeeded, use the edit tool to save the fetched **exact documentation README**
+as `/tmp/gh-aw/preset-readme.md`. Use the edit tool to save
+`/tmp/gh-aw/preset-submission.json` as a JSON object with these keys copied from the
+issue form (not inferred from the README or archive):
+`preset_id`, `preset_name`, `version`, `description`, `author`, `repository`,
+`download_url`, `documentation`, `license`, `speckit_version`,
+`required_extensions` (empty string when absent), `templates_provided`,
+`commands_provided`, `scripts_count` (string `"0"` when absent), and `tags`.
+Retain multiline list values as strings. Do not interpolate issue or README text
+into shell commands. Run this fixed command unchanged:
+
+```bash
+python3 .github/scripts/validate_community_preset.py submission --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json
+```
+
+The verifier reads `preset.yml` from the downloaded ZIP, including preset-scoped
+paths in monorepos; it parses YAML without executing archive content. It checks
+the published ID, version, Spec Kit requirement, required extension IDs, release
+tag, and README install references against the issue, then records the validated
+values, current UTC date at submission validation, and existing `created_at`
+for the generated-file check. Exit 1 (`FAILED`)
+is a confirmed submission mismatch: report it on the issue under Failed, with
+no PR. Exit 2 (`BLOCKED`) means the verifier could not read the archive or other
+required inputs (including a missing Python dependency): report the exact error
+and run link under Blocked. Do not treat either exit as a pass.
+
### Validation outcome
Choose exactly one outcome below, in order. A check that could not run is
@@ -291,9 +336,10 @@ If there are no environment blockers and a completed check found a submission de
#### Passed
If there are no environment blockers and every required check completed and passed:
-1. Remove `validation-failed`
-2. Add the `validation-passed` label
-3. Continue to Step 3
+1. Remove any stale `validation-passed` and `validation-failed` labels from
+ earlier runs.
+2. Continue to Step 3. **Do not add `validation-passed` yet**: the generated
+ catalog and documentation must pass Step 5's verifier before success is recorded.
## Step 3 β€” Determine Add vs Update
@@ -329,7 +375,7 @@ Insert the entry in **alphabetical order by preset ID** within the
"repository": "<repository>",
"download_url": "<download_url>",
"sha256": "<actual_sha256>",
- "homepage": "<homepage or repository>",
+ "homepage": "<submitted repository URL>",
"documentation": "<documentation URL β€” the validated preset-usage README>",
"license": "<license>",
"requires": {
@@ -340,8 +386,8 @@ Insert the entry in **alphabetical order by preset ID** within the
"commands": <N>
},
"tags": ["<tag1>", "<tag2>"],
- "created_at": "<today>T00:00:00Z",
- "updated_at": "<today>T00:00:00Z"
+ "created_at": "<validated UTC date>T00:00:00Z",
+ "updated_at": "<validated UTC date>T00:00:00Z"
}
}
```
@@ -361,8 +407,11 @@ If the preset provides scripts, add `"scripts": <N>` inside `"provides"`.
### For an update
Replace only the changed fields (typically `version`, `download_url`,
-`description`, `provides`, `requires`, `tags`, `updated_at`). **Preserve**
-`created_at` from the existing entry.
+`description`, `homepage`, `provides`, `requires`, `tags`, `updated_at`). Set
+`homepage` to the submitted repository URL; the form has no separate homepage
+field. **Preserve**
+`created_at` from the existing entry. Use the verifier snapshot's validated UTC
+date for `updated_at`, even if the UTC date changes during the run.
### Counting templates and commands
@@ -372,16 +421,11 @@ Parse the "Templates Provided" and "Commands Provided" issue fields:
### After editing
-Update the **top-level `"updated_at"` timestamp** in the catalog to today's date
-in ISO 8601 format.
+Update the **top-level `"updated_at"` timestamp** in the catalog to the
+verifier snapshot's UTC date in ISO 8601 format.
-Validate the JSON by running:
-
-```bash
-python3 -c "import json; json.load(open('presets/catalog.community.json')); print('Valid JSON')"
-```
-
-If validation fails, fix the JSON and re-validate before continuing.
+The generated-file verifier in Step 5 parses and checks the JSON. Fix any
+catalog error it reports and rerun it before continuing.
## Step 5 β€” Update `docs/community/presets.md`
@@ -398,7 +442,8 @@ Insert a new row in **alphabetical order by preset name**:
For the Requires column:
- Use `β€”` if no extensions are required
-- List required extension names if any (e.g., `AIDE extension`)
+- List required extension IDs if any (e.g., `aide extension`), comma-separated
+- Omit zero-count kinds from Provides and use singular nouns for counts of one
If the preset provides scripts, include them: `<N> templates, <N> commands, <N> scripts`
@@ -406,9 +451,34 @@ If the preset provides scripts, include them: `<N> templates, <N> commands, <N>
Find the existing row and update any changed fields in-place.
+### Verify the generated files
+
+Before labeling success or requesting a PR, run this fixed command unchanged:
+
+```bash
+python3 .github/scripts/validate_community_preset.py generated --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json
+```
+
+The verifier checks JSON parsing, the validated catalog metadata (including
+`homepage` set to the submitted repository URL) and digest,
+the top-level and entry `updated_at` timestamps against the recorded UTC date
+(and `created_at` for new entries),
+alphabetical ID order, the documentation row's name, purpose, counts, extension
+requirements and repository link, alphabetical name order, and preservation of
+`created_at` on updates. Exit 3 (`REPAIR`) is an agent-generated catalog or
+documentation error, **not** a submitter defect: fix the files and re-run this
+command until it exits 0. Do not label `validation-failed` for an error in
+generated files. Exit 2 (`BLOCKED`) means an input or tool is unavailable: report
+the exact error and workflow run link to the maintainer, remove
+`validation-passed`, and stop without a PR. If a generated-file error cannot be
+corrected, remove `validation-passed`, stop without a PR, and report the problem for
+maintainer investigation.
+
## Step 6 β€” Create Pull Request
-Create a pull request with the changes. Use this branch naming convention:
+Only after the generated-file verifier exits 0, add the `validation-passed`
+label and request the configured draft pull request with the changes. Use this
+branch naming convention:
This repository-owned gh-aw maintenance workflow does not perform the contributor
open-PR count check or request confirmation. After successful validation and
diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py
new file mode 100644
index 000000000..bda905260
--- /dev/null
+++ b/tests/test_community_preset_validation.py
@@ -0,0 +1,735 @@
+"""Regression coverage for the repository-owned preset submission verifier."""
+
+import hashlib
+import json
+import subprocess
+import sys
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parent.parent
+VERIFIER = ROOT / ".github" / "scripts" / "validate_community_preset.py"
+WORKFLOW = ROOT / ".github" / "workflows" / "add-community-preset.md"
+
+
+def submission(tmp_path):
+ issue = {
+ "preset_id": "sample",
+ "preset_name": "Sample Preset",
+ "version": "1.2.3",
+ "description": "Sample usage",
+ "author": "Contributor",
+ "repository": "https://github.com/example/presets",
+ "download_url": "https://github.com/example/presets/releases/download/sample-v1.2.3/sample.zip",
+ "documentation": "https://github.com/example/presets/blob/main/sample/README.md",
+ "license": "MIT",
+ "speckit_version": ">=1.0.0",
+ "required_extensions": "aide, canon",
+ "templates_provided": "- spec-template.md",
+ "commands_provided": "- speckit.plan.md",
+ "scripts_count": "0",
+ "tags": "sample, example",
+ }
+ paths = {name: tmp_path / name for name in (
+ "issue.json", "archive.zip", "README.md", "catalog.json", "presets.md",
+ "snapshot.json",
+ )}
+ paths["README.md"].write_text(
+ f"specify preset add --from {issue['download_url']}\n", encoding="utf-8"
+ )
+ manifest = {
+ "preset": {"id": "sample", "version": "1.2.3"},
+ "requires": {"speckit_version": ">=1.0.0", "extensions": ["aide", "canon"]},
+ }
+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:
+ archive.writestr("presets-release/sample/preset.yml", yaml.safe_dump(manifest))
+ archive.writestr("presets-release/other/preset.yml", yaml.safe_dump({
+ "preset": {"id": "other", "version": "9.9.9"},
+ "requires": {"speckit_version": ">=0.1.0"},
+ }))
+ issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest()
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["catalog.json"].write_text(
+ json.dumps({"presets": {}}), encoding="utf-8"
+ )
+ paths["presets.md"].write_text(
+ "| Preset | Purpose | Provides | Requires | URL |\n"
+ "|--------|---------|----------|----------|-----|\n",
+ encoding="utf-8",
+ )
+ return issue, manifest, paths
+
+
+def run_verifier(paths, phase="submission"):
+ return subprocess.run(
+ [
+ sys.executable, str(VERIFIER), phase,
+ "--issue", str(paths["issue.json"]),
+ "--archive", str(paths["archive.zip"]),
+ "--readme", str(paths["README.md"]),
+ "--catalog", str(paths["catalog.json"]),
+ "--docs", str(paths["presets.md"]),
+ "--snapshot", str(paths["snapshot.json"]),
+ ],
+ capture_output=True, text=True, check=False,
+ )
+
+
+def write_archive(paths, manifest):
+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:
+ archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest))
+
+
+def write_generated(issue, paths, *, created_at=None):
+ timestamp = (
+ json.loads(paths["snapshot.json"].read_text(encoding="utf-8"))["expected_timestamp"]
+ if paths["snapshot.json"].exists()
+ else "2026-01-01T00:00:00Z"
+ )
+ entry = {
+ "id": issue["preset_id"], "name": issue["preset_name"],
+ "version": issue["version"], "description": issue["description"],
+ "author": issue["author"], "repository": issue["repository"],
+ "download_url": issue["download_url"], "sha256": issue["actual_sha256"],
+ "homepage": issue["repository"], "documentation": issue["documentation"],
+ "license": issue["license"],
+ "requires": {"speckit_version": issue["speckit_version"],
+ "extensions": ["aide", "canon"]},
+ "provides": {"templates": 1, "commands": 1},
+ "tags": ["sample", "example"],
+ "created_at": created_at if created_at is not None else timestamp,
+ "updated_at": timestamp,
+ }
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": entry["updated_at"], "presets": {"sample": entry},
+ }), encoding="utf-8")
+ preset_name = issue["preset_name"].replace("\\", r"\\").replace("|", r"\|")
+ description = issue["description"].replace("\\", r"\\").replace("|", r"\|")
+ paths["presets.md"].write_text(
+ "| Preset | Purpose | Provides | Requires | URL |\n"
+ "|--------|---------|----------|----------|-----|\n"
+ f"| {preset_name} | {description} | 1 template, 1 command | "
+ "aide extension, canon extension | "
+ "[presets](https://github.com/example/presets) |\n",
+ encoding="utf-8",
+ )
+ return entry
+
+
+def test_matching_monorepo_submission_and_generated_files_pass(submission):
+ issue, _, paths = submission
+ before = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z")
+ first = run_verifier(paths)
+ assert first.returncode == 0, first.stdout + first.stderr
+ snapshot = json.loads(paths["snapshot.json"].read_text(encoding="utf-8"))
+ after = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z")
+ assert snapshot["expected_timestamp"] in (before, after)
+ write_generated(issue, paths)
+ second = run_verifier(paths, "generated")
+ assert second.returncode == 0, second.stdout + second.stderr
+
+
+def test_update_preserving_created_at_passes(submission):
+ issue, _, paths = submission
+ original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": original["updated_at"], "presets": {"sample": original},
+ }), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+ write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ assert run_verifier(paths, "generated").returncode == 0
+
+
[email protected](("change", "message"), [
+ ({"preset": {"id": "sample", "version": "1.2.4"}}, "version"),
+ ({"requires": {"speckit_version": ">=2.0.0",
+ "extensions": ["aide", "canon"]}}, "speckit_version"),
+ ({"requires": {"speckit_version": ">=1.0.0",
+ "extensions": ["aide"]}}, "extensions"),
+])
+def test_published_manifest_mismatch_is_submission_failure(submission, change, message):
+ _, manifest, paths = submission
+ manifest.update(change)
+ write_archive(paths, manifest)
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert message in result.stdout
+ assert not paths["snapshot.json"].exists()
+
+
+def test_release_tag_mismatch_is_submission_failure(submission):
+ issue, _, paths = submission
+ issue["download_url"] = issue["download_url"].replace("v1.2.3", "v1.2.4")
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ assert run_verifier(paths).returncode == 1
+
+
+def test_stale_from_url_fails_even_with_valid_dev_command(submission):
+ issue, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README" in result.stdout
+
+
[email protected]("archive_url", [False, True])
+def test_stale_from_url_with_submitted_scoped_tag_fails(submission, archive_url):
+ issue, _, paths = submission
+ if archive_url:
+ issue["download_url"] = (
+ "https://github.com/example/presets/archive/refs/tags/"
+ "spec-kit-sample-v1.2.3.zip"
+ )
+ else:
+ issue["download_url"] = issue["download_url"].replace(
+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1, result.stdout + result.stderr
+ assert "README --from URL" in result.stdout
+
+
+def test_dev_only_readme_is_accepted(submission):
+ _, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n", encoding="utf-8"
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_dev_current_directory_path_is_accepted(submission):
+ _, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add --dev .\n", encoding="utf-8"
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_dev_option_without_path_is_rejected(submission):
+ _, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add --dev --priority 20\n", encoding="utf-8"
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README" in result.stdout
+
+
+def test_dev_without_path_does_not_consume_next_paragraph(submission):
+ _, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add --dev\n"
+ "Next paragraph describes the preset.\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README" in result.stdout
+
+
+def test_quoted_from_url_with_sentence_punctuation_is_accepted(submission):
+ issue, _, paths = submission
+ paths["README.md"].write_text(
+ f'Spec Kit install: `specify preset add --from "{issue["download_url"]}".`\n',
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_id_install_and_unrelated_monorepo_release_are_accepted(submission):
+ _, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add sample\n"
+ "specify preset add --from "
+ "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_unrelated_scoped_release_stays_accepted_with_submitted_scope(submission):
+ issue, _, paths = submission
+ issue["download_url"] = issue["download_url"].replace(
+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add sample\n"
+ "specify preset add --from "
+ "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+
+
[email protected]("archive_url", [False, True])
+def test_unrelated_unscoped_monorepo_release_stays_accepted(submission, archive_url):
+ issue, _, paths = submission
+ if archive_url:
+ issue["download_url"] = (
+ "https://github.com/example/presets/archive/refs/tags/"
+ "spec-kit-sample-v1.2.3.zip"
+ )
+ unrelated = (
+ "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip"
+ )
+ else:
+ issue["download_url"] = issue["download_url"].replace(
+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"
+ )
+ unrelated = (
+ "https://github.com/example/presets/releases/download/v2.0.0/other.zip"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ f"specify preset add --from {unrelated}\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_unrelated_unscoped_archive_stays_accepted_in_monorepo(submission):
+ issue, _, paths = submission
+ issue["download_url"] = (
+ "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ "specify preset add --from "
+ "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_stale_unscoped_archive_fails_for_single_preset(submission):
+ issue, manifest, paths = submission
+ write_archive(paths, manifest)
+ issue["download_url"] = (
+ "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ "specify preset add --from "
+ "https://github.com/example/presets/archive/refs/tags/v1.2.2.zip\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README --from URL" in result.stdout
+
+
+def test_matching_unscoped_archive_passes_for_single_preset(submission):
+ issue, manifest, paths = submission
+ write_archive(paths, manifest)
+ issue["download_url"] = (
+ "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ f"specify preset add --from {issue['download_url']}\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+
+
+def test_same_asset_on_unscoped_tag_is_reported_as_stale(submission):
+ issue, _, paths = submission
+ issue["download_url"] = issue["download_url"].replace(
+ "sample-v1.2.3", "v1.2.3"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README --from URL" in result.stdout
+
+
+def test_versioned_asset_on_unscoped_tag_is_reported_as_stale(submission):
+ issue, _, paths = submission
+ issue["download_url"] = (
+ "https://github.com/example/presets/releases/download/"
+ "v1.2.3/sample-1.2.3.zip"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ "specify preset add --from "
+ "https://github.com/example/presets/releases/download/"
+ "v1.2.2/sample-1.2.2.zip\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README --from URL" in result.stdout
+
+
+def test_same_asset_on_bare_tag_stays_stale_for_scoped_submission(submission):
+ issue, _, paths = submission
+ issue["download_url"] = issue["download_url"].replace(
+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"
+ )
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ "specify preset add --from "
+ "https://github.com/example/presets/releases/download/v1.2.2/sample.zip\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "README --from URL" in result.stdout
+
+
+def test_stale_scoped_release_in_another_repository_fails(submission):
+ _, _, paths = submission
+ paths["README.md"].write_text(
+ "specify preset add --dev ./sample\n"
+ "specify preset add --from "
+ "https://github.com/elsewhere/presets/releases/download/sample-v1.2.2/sample.zip\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 1
+
+
+def test_optional_manifest_extension_is_not_required(submission):
+ issue, manifest, paths = submission
+ manifest["requires"]["extensions"].append({
+ "id": "optional", "version": ">=1.0.0", "required": False,
+ })
+ write_archive(paths, manifest)
+ issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest()
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+
+
+def test_invalid_unrelated_monorepo_manifest_does_not_mask_match(submission):
+ _, manifest, paths = submission
+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:
+ archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest))
+ archive.writestr("release/other/preset.yml", "preset: [invalid\n")
+ assert run_verifier(paths).returncode == 0
+
+
+def test_archive_with_too_many_manifests_is_rejected_before_parsing(submission):
+ _, _, paths = submission
+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:
+ for index in range(101):
+ archive.writestr(f"release/{index}/preset.yml", "preset: [invalid\n")
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "more than 100 preset.yml files" in result.stdout
+ assert "invalid manifests" not in result.stdout
+
+
+def test_archive_with_excessive_total_manifest_size_is_rejected(submission):
+ _, manifest, paths = submission
+ padding = "#" * (1024 * 1024 - len(yaml.safe_dump(manifest)) - 2)
+ with zipfile.ZipFile(
+ paths["archive.zip"], "w", compression=zipfile.ZIP_DEFLATED
+ ) as archive:
+ for index in range(11):
+ archive.writestr(
+ f"release/{index}/preset.yml",
+ f"{yaml.safe_dump(manifest)}\n{padding}",
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 1
+ assert "10 MiB total limit" in result.stdout
+
+
+def test_missing_archive_is_blocked_not_failed(submission):
+ _, _, paths = submission
+ paths["archive.zip"].unlink()
+ result = run_verifier(paths)
+ assert result.returncode == 2
+ assert "BLOCKED" in result.stdout
+
+
+def test_missing_generated_documentation_is_repairable(submission):
+ issue, _, paths = submission
+ assert run_verifier(paths).returncode == 0
+ write_generated(issue, paths)
+ paths["presets.md"].unlink()
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 3
+ assert "REPAIR" in result.stdout
+
+
+def test_existing_entry_without_creation_date_blocks_update(submission):
+ issue, _, paths = submission
+ entry = write_generated(issue, paths)
+ del entry["created_at"]
+ paths["catalog.json"].write_text(
+ json.dumps({"presets": {"sample": entry}}), encoding="utf-8"
+ )
+ result = run_verifier(paths)
+ assert result.returncode == 2
+ assert "created_at" in result.stdout
+
+
[email protected]("timestamp_field", ["created_at", "updated_at"])
+def test_generated_new_entry_rejects_stale_dates(submission, timestamp_field):
+ issue, _, paths = submission
+ assert run_verifier(paths).returncode == 0
+ entry = write_generated(issue, paths)
+ entry[timestamp_field] = "2000-01-01T00:00:00Z"
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": entry["updated_at"], "presets": {"sample": entry},
+ }), encoding="utf-8")
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 3, result.stdout + result.stderr
+ assert timestamp_field in result.stdout
+
+
+def test_generated_update_rejects_matching_stale_updated_dates(submission):
+ issue, _, paths = submission
+ entry = write_generated(issue, paths)
+ assert run_verifier(paths).returncode == 0
+ entry["updated_at"] = "2000-01-01T00:00:00Z"
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": entry["updated_at"], "presets": {"sample": entry},
+ }), encoding="utf-8")
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 3, result.stdout + result.stderr
+ assert "updated_at" in result.stdout
+
+
+def test_generated_update_rejects_stale_homepage(submission):
+ issue, _, paths = submission
+ entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ entry["homepage"] = "https://example.com/old"
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": entry["updated_at"], "presets": {"sample": entry},
+ }), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+ entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ entry["homepage"] = "https://example.com/old"
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": entry["updated_at"], "presets": {"sample": entry},
+ }), encoding="utf-8")
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 3
+ assert "homepage" in result.stdout
+
+
+def test_generated_documentation_accepts_escaped_pipe_in_preset_name(submission):
+ issue, _, paths = submission
+ issue["preset_name"] = "Data | Governance"
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+ write_generated(issue, paths)
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_generated_documentation_accepts_backslash_before_pipe(submission):
+ issue, _, paths = submission
+ issue["preset_name"] = r"Data \| Governance"
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+ write_generated(issue, paths)
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_generated_documentation_allows_duplicate_display_names(submission):
+ issue, _, paths = submission
+ other_row = (
+ "| Sample Preset | Other usage | 1 command | β€” | "
+ "[other](https://github.com/example/other) |"
+ )
+ paths["catalog.json"].write_text(json.dumps({
+ "presets": {"other": {"name": issue["preset_name"]}},
+ }), encoding="utf-8")
+ paths["presets.md"].write_text(
+ "| Preset | Purpose | Provides | Requires | URL |\n"
+ "|--------|---------|----------|----------|-----|\n"
+ f"{other_row}\n",
+ encoding="utf-8",
+ )
+ assert run_verifier(paths).returncode == 0
+ entry = write_generated(issue, paths)
+ catalog = json.loads(paths["catalog.json"].read_text(encoding="utf-8"))
+ catalog["presets"] = {
+ "other": {"name": issue["preset_name"]},
+ "sample": entry,
+ }
+ paths["catalog.json"].write_text(json.dumps(catalog), encoding="utf-8")
+ paths["presets.md"].write_text(
+ "| Preset | Purpose | Provides | Requires | URL |\n"
+ "|--------|---------|----------|----------|-----|\n"
+ f"{other_row}\n"
+ "| Sample Preset | Sample usage | 1 template, 1 command | "
+ "aide extension, canon extension | "
+ "[presets](https://github.com/example/presets) |\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_generated_documentation_accepts_replaced_renamed_row(submission):
+ issue, _, paths = submission
+ original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ paths["presets.md"].write_text(
+ paths["presets.md"].read_text(encoding="utf-8").replace(
+ "Sample usage", "Legacy documentation wording"
+ ),
+ encoding="utf-8",
+ )
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": original["updated_at"], "presets": {"sample": original},
+ }), encoding="utf-8")
+ issue["preset_name"] = "Renamed Preset"
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+ write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_generated_documentation_rejects_stale_row_after_rename(submission):
+ issue, _, paths = submission
+ original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ paths["presets.md"].write_text(
+ paths["presets.md"].read_text(encoding="utf-8").replace(
+ "Sample usage", "Legacy documentation wording"
+ ),
+ encoding="utf-8",
+ )
+ previous_row = paths["presets.md"].read_text(encoding="utf-8").splitlines()[2]
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": original["updated_at"], "presets": {"sample": original},
+ }), encoding="utf-8")
+ issue["preset_name"] = "Renamed Preset"
+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")
+ assert run_verifier(paths).returncode == 0
+ write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")
+ paths["presets.md"].write_text(
+ paths["presets.md"].read_text(encoding="utf-8") + previous_row + "\n",
+ encoding="utf-8",
+ )
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 3
+ assert "previous documentation row" in result.stdout
+
+
[email protected](("damage", "message"), [
+ ("catalog-json", "catalog"),
+ ("catalog-order", "alphabetical"),
+ ("catalog-metadata", "version"),
+ ("homepage-missing", "homepage"),
+ ("homepage-stale", "homepage"),
+ ("catalog-timestamp", "top-level updated_at"),
+ ("docs-order", "alphabetical"),
+ ("docs-row", "documentation row"),
+ ("created-at", "created_at"),
+])
+def test_generated_defects_are_fixable_not_submission_failures(submission, damage, message):
+ issue, _, paths = submission
+ if damage == "created-at":
+ original = write_generated(issue, paths)
+ paths["catalog.json"].write_text(
+ json.dumps({"updated_at": original["updated_at"], "presets": {
+ "sample": original,
+ }}), encoding="utf-8"
+ )
+ assert run_verifier(paths).returncode == 0
+ entry = write_generated(issue, paths)
+ if damage == "catalog-json":
+ paths["catalog.json"].write_text("{", encoding="utf-8")
+ elif damage == "catalog-order":
+ paths["catalog.json"].write_text(json.dumps({
+ "updated_at": entry["updated_at"],
+ "presets": {"sample": entry, "aaa": {"name": "AAA"}},
+ }), encoding="utf-8")
+ elif damage == "catalog-metadata":
+ entry["version"] = "1.2.4"
+ paths["catalog.json"].write_text(
+ json.dumps({"updated_at": entry["updated_at"], "presets": {
+ "sample": entry,
+ }}), encoding="utf-8"
+ )
+ elif damage in ("homepage-missing", "homepage-stale"):
+ if damage == "homepage-missing":
+ del entry["homepage"]
+ else:
+ entry["homepage"] = "https://github.com/example/other"
+ paths["catalog.json"].write_text(
+ json.dumps({"updated_at": entry["updated_at"], "presets": {
+ "sample": entry,
+ }}), encoding="utf-8"
+ )
+ elif damage == "catalog-timestamp":
+ paths["catalog.json"].write_text(
+ json.dumps({"updated_at": "2020-01-01T00:00:00Z", "presets": {
+ "sample": entry,
+ }}), encoding="utf-8"
+ )
+ elif damage == "docs-order":
+ paths["presets.md"].write_text(
+ paths["presets.md"].read_text(encoding="utf-8")
+ + "| AAA | x | 1 command | β€” | [aaa](https://github.com/aaa/aaa) |\n",
+ encoding="utf-8",
+ )
+ elif damage == "docs-row":
+ paths["presets.md"].write_text(
+ paths["presets.md"].read_text(encoding="utf-8").replace(
+ "Sample usage", "Wrong purpose"
+ ), encoding="utf-8",
+ )
+ elif damage == "created-at":
+ entry["created_at"] = "2000-01-01T00:00:00Z"
+ paths["catalog.json"].write_text(
+ json.dumps({"updated_at": entry["updated_at"], "presets": {
+ "sample": entry,
+ }}), encoding="utf-8"
+ )
+ result = run_verifier(paths, "generated")
+ assert result.returncode == 3, result.stdout + result.stderr
+ assert message in result.stdout
+
+
+def test_workflow_gates_success_on_both_verifier_phases():
+ source = WORKFLOW.read_text(encoding="utf-8")
+ assert "python3 .github/scripts/validate_community_preset.py submission" in source
+ assert "python3 .github/scripts/validate_community_preset.py generated" in source
+ assert (source.index("validate_community_preset.py generated")
+ < source.index("## Step 6")
+ < source.index("add the `validation-passed`", source.index("## Step 6")))
+ frontmatter = yaml.safe_load(source.split("---", 2)[1])
+ assert [step["name"] for step in frontmatter["steps"]] == [
+ "Set up Python for preset verification",
+ "Install preset verifier dependency",
+ ]
+ compiled = yaml.safe_load(
+ (ROOT / ".github/workflows/add-community-preset.lock.yml").read_text(
+ encoding="utf-8"
+ )
+ )
+ steps = compiled["jobs"]["agent"]["steps"]
+ for setup in frontmatter["steps"]:
+ assert setup in steps
diff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.py
index 5268ac8c4..5b6d1d887 100644
--- a/tests/test_github_workflows.py
+++ b/tests/test_github_workflows.py
@@ -817,10 +817,21 @@ def test_community_archive_permission_failures_are_not_submission_failures(kind)
"If there are no environment blockers and every required check completed "
"and passed:"
)
- assert re.search(
- r"remove `validation-failed`.*add (?:the )?`validation-passed`",
- passed, re.IGNORECASE | re.DOTALL,
- )
+ if kind == "preset":
+ assert "Remove any stale `validation-passed` and `validation-failed`" in passed
+ assert "Do not add `validation-passed` yet" in passed
+ generated = source_text.split("### Verify the generated files", 1)[1].split(
+ "\n## Step 6", 1
+ )[0]
+ assert "validate_community_preset.py generated" in generated
+ assert "add the `validation-passed`" in source_text.split(
+ "\n## Step 6", 1
+ )[1]
+ else:
+ assert re.search(
+ r"remove `validation-failed`.*add (?:the )?`validation-passed`",
+ passed, re.IGNORECASE | re.DOTALL,
+ )
assert "validation-failed" in source["safe-outputs"]["remove-labels"]["allowed"]
assert "validation-failed" in _safe_output_config(compiled)["remove_labels"]["allowed"]
assert "validation-passed" in source["safe-outputs"]["remove-labels"]["allowed"]