π« Toolkit to help you get started with SDD or any other process!
Validate community preset submissions before opening catalog PRs (#4787)
Manfred Riemcd5db8f9b077
diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.pynew file mode 100644index 000000000..ebc64a6cf--- /dev/null+++ b/.github/scripts/validate_community_preset.py@@ -0,0 +1,532 @@+"""Verify a community preset submission and its generated catalog changes.++Exit 1: confirmed submission mismatch; 2: check blocked; 3: generated files+need repair. The workflow decides labels and PR creation from these outcomes.+"""++import argparse+import hashlib+import json+import re+import sys+import zipfile+from datetime import datetime, timezone+from pathlib import Path+from urllib.parse import urlsplit++try:+ import yaml+except ImportError:+ print("BLOCKED: PyYAML is unavailable; cannot inspect published preset.yml")+ sys.exit(2)++MAX_MANIFEST_COUNT = 100+MAX_MANIFEST_SIZE = 1024 * 1024+MAX_TOTAL_MANIFEST_SIZE = 10 * 1024 * 1024+++class SubmissionMismatch(Exception):+ pass+++class Blocked(Exception):+ pass+++class GeneratedError(Exception):+ pass+++def read_text(+ path: Path, context: str, error_type: type[Exception] = Blocked+) -> str:+ try:+ return path.read_text(encoding="utf-8")+ except (OSError, UnicodeError) as exc:+ raise error_type(f"cannot read {context} ({path}): {exc}") from exc+++def read_json(path: Path, context: str, error_type: type[Exception]) -> dict:+ try:+ value = json.loads(read_text(path, context, error_type))+ except json.JSONDecodeError as exc:+ raise error_type(f"{context} is invalid JSON: {exc}") from exc+ if not isinstance(value, dict):+ raise error_type(f"{context} must be a JSON object")+ return value+++def field(issue: dict, key: str) -> str:+ value = issue.get(key)+ if not isinstance(value, str) or not value.strip():+ raise SubmissionMismatch(f"missing or invalid issue field: {key}")+ return value.strip()+++def csv_values(value: str) -> list[str]:+ return [part.strip() for part in value.split(",") if part.strip()]+++def repository_parts(url: str) -> tuple[str, str]:+ parsed = urlsplit(url)+ match = re.fullmatch(r"/([^/]+)/([^/]+)/?", parsed.path)+ if parsed.scheme != "https" or parsed.netloc.lower() != "github.com" or not match:+ raise SubmissionMismatch(f"invalid GitHub repository URL: {url}")+ return match[1].lower(), match[2].lower()+++def release_tag(issue: dict) -> str:+ owner, repo = repository_parts(field(issue, "repository"))+ url = field(issue, "download_url")+ parsed = urlsplit(url)+ prefix = f"/{owner}/{repo}/"+ path = parsed.path+ if parsed.scheme != "https" or parsed.netloc.lower() != "github.com":+ raise SubmissionMismatch(f"download URL is not a GitHub URL: {url}")+ if not path.lower().startswith(prefix):+ raise SubmissionMismatch("download URL does not belong to the submitted repository")+ suffix = path[len(prefix):]+ archive = re.fullmatch(r"archive/refs/tags/([^/]+)\.zip", suffix)+ release = re.fullmatch(r"releases/download/([^/]+)/[^/]+\.zip", suffix)+ if not (archive or release) or parsed.query or parsed.fragment:+ raise SubmissionMismatch(f"download URL is not a pinned ZIP release: {url}")+ tag = (archive or release)[1]+ version = field(issue, "version")+ if not re.fullmatch(r"\d+\.\d+\.\d+", version) or not re.fullmatch(+ rf"(?:v?{re.escape(version)}|.+-v?{re.escape(version)})", tag+ ):+ raise SubmissionMismatch(+ f"release tag {tag!r} does not match submitted version {version!r}"+ )+ return tag+++def normalized_release_asset(filename: str) -> str:+ return re.sub(+ r"(?<!\d)v?\d+\.\d+\.\d+(?!\d)",+ "{version}",+ filename,+ flags=re.IGNORECASE,+ )+++def published_manifest(archive_path: Path, preset_id: str) -> tuple[dict, bool]:+ try:+ with zipfile.ZipFile(archive_path) as archive:+ matching = []+ invalid = []+ manifests = [+ member for member in archive.infolist()+ if not member.is_dir()+ and member.filename.rsplit("/", 1)[-1] == "preset.yml"+ ]+ if len(manifests) > MAX_MANIFEST_COUNT:+ raise SubmissionMismatch(+ f"archive contains more than {MAX_MANIFEST_COUNT} preset.yml files"+ )+ total_size = sum(member.file_size for member in manifests)+ if total_size > MAX_TOTAL_MANIFEST_SIZE:+ raise SubmissionMismatch(+ "archive preset.yml files exceed the "+ f"{MAX_TOTAL_MANIFEST_SIZE // (1024 * 1024)} MiB total limit"+ )+ for member in manifests:+ try:+ if member.file_size > MAX_MANIFEST_SIZE:+ raise ValueError("preset.yml exceeds 1 MiB")+ with archive.open(member) as stream:+ data = yaml.safe_load(stream.read().decode("utf-8"))+ except (yaml.YAMLError, UnicodeError, ValueError) as exc:+ invalid.append(f"{member.filename}: {exc}")+ continue+ if isinstance(data, dict) and isinstance(data.get("preset"), dict):+ if data["preset"].get("id") == preset_id:+ matching.append((member.filename, data))+ except FileNotFoundError as exc:+ raise Blocked(f"downloaded archive is unavailable at {archive_path}: {exc}") from exc+ except PermissionError as exc:+ raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc+ except OSError as exc:+ raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc+ except (zipfile.BadZipFile, EOFError) as exc:+ raise SubmissionMismatch(f"downloaded archive is not a readable ZIP: {exc}") from exc+ if len(matching) != 1:+ if not matching and invalid:+ raise SubmissionMismatch(+ f"no published preset.yml for {preset_id!r}; invalid manifests: "+ + "; ".join(invalid)+ )+ raise SubmissionMismatch(+ f"expected one published preset.yml for {preset_id!r}, found {len(matching)}"+ )+ return matching[0][1], len(manifests) == 1+++def required_extensions(manifest: dict) -> list[str]:+ requires = manifest.get("requires")+ if not isinstance(requires, dict):+ raise SubmissionMismatch("published preset.yml has no requires mapping")+ extensions = requires.get("extensions", [])+ if not isinstance(extensions, list):+ raise SubmissionMismatch("published requires.extensions must be a list")+ result = []+ for item in extensions:+ if isinstance(item, str):+ result.append(item)+ elif isinstance(item, dict) and isinstance(item.get("id"), str):+ if item.get("required", True) is not False:+ result.append(item["id"])+ else:+ raise SubmissionMismatch("invalid published requires.extensions entry")+ return result+++def check_readme(text: str, issue: dict, *, single_preset_archive: bool) -> None:+ expected = field(issue, "download_url")+ preset_id = field(issue, "preset_id")+ owner, repo = repository_parts(field(issue, "repository"))+ submitted_scope = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", release_tag(issue))+ accepted_scopes = {preset_id}+ if submitted_scope:+ accepted_scopes.add(submitted_scope[1])+ expected_path = urlsplit(expected).path.split("/")+ accepted = False+ for match in re.finditer(+ r"(?<![\w-])specify[ \t]+preset[ \t]+add[ \t]+"+ r"(?P<option>--from|--dev|[a-z][a-z0-9-]*)"+ r"(?:[ \t]+(?P<value>[^\s`]+))?",+ text,+ ):+ option = match["option"]+ raw_value = match["value"] or ""+ if option == "--from":+ value = raw_value.strip("'\"<>(),.;")+ if value == expected:+ accepted = True+ continue+ parsed = urlsplit(value)+ parts = parsed.path.split("/")+ tag = (+ parts[5] if len(parts) > 5 and parts[3] == "releases"+ else parts[6].removesuffix(".zip")+ if len(parts) > 6 and parts[3] == "archive"+ else ""+ )+ scoped = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", tag)+ same_repository = (+ parsed.netloc.lower() == "github.com"+ and parsed.path.lower().startswith(f"/{owner}/{repo}/")+ )+ # An unscoped tag alone does not identify a preset in a monorepo.+ same_release_asset = (+ len(expected_path) > 6+ and expected_path[3:5] == ["releases", "download"]+ and len(parts) > 6+ and parts[3:5] == ["releases", "download"]+ and normalized_release_asset(expected_path[6])+ == normalized_release_asset(parts[6])+ )+ unscoped_archive = (+ single_preset_archive+ and submitted_scope is None+ and expected_path[3:6] == ["archive", "refs", "tags"]+ and parts[3:6] == ["archive", "refs", "tags"]+ )+ if (scoped and scoped[1] in accepted_scopes) or (+ same_repository and not scoped and (same_release_asset or unscoped_archive)+ ):+ raise SubmissionMismatch(+ f"README --from URL for {preset_id} differs from Download URL: {value}"+ )+ elif option == "--dev":+ value = raw_value.strip("'\"")+ if value and not value.startswith("-"):+ accepted = True+ elif option == preset_id:+ accepted = True+ if not accepted:+ raise SubmissionMismatch(+ "README has no valid specify preset add command for the submitted preset"+ )+++def count_items(value: str) -> int:+ return sum(line.lstrip().startswith("- ") for line in value.splitlines())+++def expected_values(issue: dict, manifest: dict, digest: str) -> dict:+ pack = manifest.get("preset")+ requires = manifest.get("requires")+ if not isinstance(pack, dict) or not isinstance(requires, dict):+ raise SubmissionMismatch("published preset.yml lacks preset or requires metadata")+ extension_ids = csv_values(issue.get("required_extensions", ""))+ actual_extensions = required_extensions(manifest)+ comparisons = {+ "preset.id": (pack.get("id"), field(issue, "preset_id")),+ "preset.version": (pack.get("version"), field(issue, "version")),+ "requires.speckit_version": (+ requires.get("speckit_version"), field(issue, "speckit_version")+ ),+ "requires.extensions": (sorted(actual_extensions), sorted(extension_ids)),+ }+ for name, (actual, expected) in comparisons.items():+ if actual != expected:+ raise SubmissionMismatch(+ f"published {name} {actual!r} differs from submitted {expected!r}"+ )+ dependencies = {"speckit_version": field(issue, "speckit_version")}+ if extension_ids:+ dependencies["extensions"] = extension_ids+ provided = {+ "templates": count_items(field(issue, "templates_provided")),+ "commands": count_items(field(issue, "commands_provided")),+ }+ scripts = issue.get("scripts_count", "0")+ if scripts:+ try:+ number = int(scripts)+ except (ValueError, TypeError) as exc:+ raise SubmissionMismatch("scripts_count must be a non-negative integer") from exc+ if number < 0:+ raise SubmissionMismatch("scripts_count must be a non-negative integer")+ if number:+ provided["scripts"] = number+ return {+ "id": field(issue, "preset_id"),+ "name": field(issue, "preset_name"),+ "version": field(issue, "version"),+ "description": field(issue, "description"),+ "author": field(issue, "author"),+ "repository": field(issue, "repository"),+ "homepage": field(issue, "repository"),+ "download_url": field(issue, "download_url"),+ "sha256": digest,+ "documentation": field(issue, "documentation"),+ "license": field(issue, "license"),+ "requires": dependencies,+ "provides": provided,+ "tags": csv_values(field(issue, "tags")),+ }+++def submission(args: argparse.Namespace) -> None:+ issue = read_json(args.issue, "issue input", Blocked)+ release_tag(issue)+ manifest, single_preset_archive = published_manifest(+ args.archive, field(issue, "preset_id")+ )+ check_readme(+ read_text(args.readme, "fetched README"),+ issue,+ single_preset_archive=single_preset_archive,+ )+ try:+ with args.archive.open("rb") as archive:+ digest = hashlib.file_digest(archive, "sha256").hexdigest()+ except OSError as exc:+ raise Blocked(f"cannot hash downloaded archive: {exc}") from exc+ expected = expected_values(issue, manifest, digest)+ catalog = read_json(args.catalog, "original catalog", Blocked)+ entries = catalog.get("presets")+ if not isinstance(entries, dict):+ raise Blocked("original catalog has no presets object")+ previous = entries.get(expected["id"])+ if previous is not None and not isinstance(previous, dict):+ raise Blocked("original catalog entry is not an object")+ if previous is not None and not isinstance(previous.get("created_at"), str):+ raise Blocked("original catalog entry has no created_at to preserve")+ original_rows = documentation_rows(+ read_text(args.docs, "original documentation", Blocked), Blocked+ )+ expected_row = documentation_row(expected)+ previous_row = None+ if previous is not None:+ previous_row = existing_documentation_row(original_rows, previous)+ snapshot = {+ "expected": expected,+ "created_at": previous.get("created_at") if previous else None,+ "expected_timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z"),+ "documentation": {+ "expected_row_count": original_rows.count(expected_row),+ "previous_row": previous_row,+ "previous_row_count": original_rows.count(previous_row)+ if previous_row is not None else 0,+ },+ }+ try:+ args.snapshot.write_text(json.dumps(snapshot), encoding="utf-8")+ except OSError as exc:+ raise Blocked(f"cannot write verifier snapshot: {exc}") from exc+ print("PASSED: published preset.yml, release tag, README, and issue fields agree")+++def documentation_row(entry: dict) -> str:+ def cell(value: str) -> str:+ return value.replace("\\", r"\\").replace("|", r"\|").replace("\n", " ")++ provided = entry["provides"]+ quantities = [+ f"{count} {name[:-1] if count == 1 else name}"+ for name, count in provided.items() if count+ ]+ dependencies = entry["requires"].get("extensions", [])+ requires = ", ".join(f"{name} extension" for name in dependencies) or "β"+ repo = entry["repository"].rstrip("/").rsplit("/", 1)[-1]+ return (+ f"| {cell(entry['name'])} | {cell(entry['description'])} | "+ f"{', '.join(quantities)} | {requires} | "+ f"[{repo}]({entry['repository']}) |"+ )+++def markdown_table_cells(row: str) -> list[str]:+ cells = []+ cell = []+ escaped = False+ for character in row:+ if escaped:+ if character in ("\\", "|"):+ cell.append(character)+ else:+ cell.extend(("\\", character))+ escaped = False+ elif character == "\\":+ escaped = True+ elif character == "|":+ cells.append("".join(cell).strip())+ cell = []+ else:+ cell.append(character)+ if escaped:+ cell.append("\\")+ cells.append("".join(cell).strip())+ return cells+++def documentation_rows(text: str, error_type: type[Exception]) -> list[str]:+ lines = text.splitlines()+ try:+ start = next(i for i, line in enumerate(lines) if line.startswith("| Preset |"))+ except StopIteration as exc:+ raise error_type("documentation has no Community Presets table") from exc+ rows = []+ for line in lines[start + 2:]:+ if not line.startswith("|"):+ break+ rows.append(line.strip())+ return rows+++def existing_documentation_row(rows: list[str], entry: dict) -> str:+ name = entry.get("name")+ repository = entry.get("repository")+ if not isinstance(name, str) or not isinstance(repository, str):+ raise Blocked("original catalog entry lacks documentation row identity")+ matches = []+ for row in rows:+ cells = markdown_table_cells(row)+ if len(cells) < 3 or cells[1] != name:+ continue+ url_cell = cells[-2] if cells[-1] == "" else cells[-1]+ link = re.fullmatch(r"\[[^\]]*\]\(([^)]+)\)", url_cell)+ if link and link[1].rstrip("/") == repository.rstrip("/"):+ matches.append(row)+ if len(matches) != 1:+ raise Blocked(+ "original documentation must contain exactly one row matching the "+ f"catalog name and repository for {name!r}; found {len(matches)}"+ )+ return matches[0]+++def generated(args: argparse.Namespace) -> None:+ snapshot = read_json(args.snapshot, "verifier snapshot", Blocked)+ expected = snapshot.get("expected")+ if not isinstance(expected, dict) or not isinstance(expected.get("id"), str):+ raise Blocked("verifier snapshot lacks validated expected values")+ expected_timestamp = snapshot.get("expected_timestamp")+ if not isinstance(expected_timestamp, str):+ raise Blocked("verifier snapshot lacks the expected UTC timestamp")+ catalog = read_json(args.catalog, "generated catalog", GeneratedError)+ entries = catalog.get("presets")+ if not isinstance(entries, dict):+ raise GeneratedError("generated catalog has no presets object")+ keys = list(entries)+ if keys != sorted(keys):+ raise GeneratedError("catalog preset IDs are not in alphabetical order")+ entry = entries.get(expected["id"])+ if not isinstance(entry, dict):+ raise GeneratedError(f"catalog is missing entry {expected['id']}")+ for key, value in expected.items():+ if entry.get(key) != value:+ raise GeneratedError(f"catalog {key} does not match validated value: {value!r}")+ if snapshot.get("created_at") is not None:+ if entry.get("created_at") != snapshot["created_at"]:+ raise GeneratedError("catalog created_at was not preserved on update")+ elif entry.get("created_at") != expected_timestamp:+ raise GeneratedError("new catalog created_at does not match the expected UTC date")+ if entry.get("updated_at") != expected_timestamp:+ raise GeneratedError("catalog entry updated_at does not match the expected UTC date")+ if catalog.get("updated_at") != expected_timestamp:+ raise GeneratedError("catalog top-level updated_at does not match the expected UTC date")+ documentation = snapshot.get("documentation")+ if not isinstance(documentation, dict):+ raise Blocked("verifier snapshot lacks original documentation state")+ expected_row_count = documentation.get("expected_row_count")+ previous_row = documentation.get("previous_row")+ previous_row_count = documentation.get("previous_row_count")+ if (+ not isinstance(expected_row_count, int)+ or isinstance(expected_row_count, bool)+ or expected_row_count < 0+ or previous_row is not None and not isinstance(previous_row, str)+ or not isinstance(previous_row_count, int)+ or isinstance(previous_row_count, bool)+ or previous_row_count < 0+ ):+ raise Blocked("verifier snapshot has invalid original documentation state")+ rows = documentation_rows(+ read_text(args.docs, "generated documentation", GeneratedError),+ GeneratedError,+ )+ names = [markdown_table_cells(row)[1] for row in rows]+ if names != sorted(names, key=str.casefold):+ raise GeneratedError("documentation preset names are not in alphabetical order")+ expected_row = documentation_row(expected)+ expected_generated_count = expected_row_count + (+ 0 if previous_row == expected_row else 1+ )+ if rows.count(expected_row) != expected_generated_count:+ raise GeneratedError(f"documentation row does not match validated values: {expected_row}")+ if previous_row is not None and previous_row != expected_row:+ expected_previous_count = max(0, previous_row_count - 1)+ if rows.count(previous_row) != expected_previous_count:+ raise GeneratedError("previous documentation row was not replaced")+ print("PASSED: generated catalog and documentation match validated submission")+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("phase", choices=("submission", "generated"))+ for name in ("issue", "archive", "readme", "catalog", "docs", "snapshot"):+ parser.add_argument(f"--{name}", required=True, type=Path)+ args = parser.parse_args()+ try:+ if args.phase == "submission":+ submission(args)+ else:+ generated(args)+ except SubmissionMismatch as exc:+ print(f"FAILED: {exc}")+ return 1+ except Blocked as exc:+ print(f"BLOCKED: {exc}")+ return 2+ except GeneratedError as exc:+ print(f"REPAIR: {exc}")+ return 3+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.ymlindex 5b6cb1081..5fd10b3cf 100644--- a/.github/workflows/add-community-preset.lock.yml+++ b/.github/workflows/add-community-preset.lock.yml@@ -1,5 +1,5 @@-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e692354fdb52cf49c3f46096529618b23eb09336fe58fe2392b77210b374b568","body_hash":"cd1920f05bf8608b46191a3802a1c2f6c595ab3ce28034116e3346d4988bed4a","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}}-# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]}+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"67f2a3313f59103002b9e362e4d16ad1c3ef4035afb6c96f3446a345e4da4852","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}}+# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]}# This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md## ___ _ _@@ -41,6 +41,7 @@# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9)# - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+# - actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1# - github/gh-aw-actions/setup@5e508589e03a7757a7e05b26e834292f5445bfb6 # v0.88.7#@@ -491,6 +492,15 @@ jobs:with:name: activationpath: /tmp/gh-aw+ - continue-on-error: true+ name: Set up Python for preset verification+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.13"+ - continue-on-error: true+ name: Install preset verifier dependency+ run: python3 -m pip install 'PyYAML==6.0.3'+- name: Configure Git credentialsenv:GITHUB_REPOSITORY: ${{ github.repository }}diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.mdindex f6ccf01c8..80608481e 100644--- a/.github/workflows/add-community-preset.md+++ b/.github/workflows/add-community-preset.md@@ -39,6 +39,16 @@ permissions:checkout:fetch-depth: 0+steps:+ - name: Set up Python for preset verification+ continue-on-error: true+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.13"+ - name: Install preset verifier dependency+ continue-on-error: true+ run: python3 -m pip install 'PyYAML==6.0.3'+safe-outputs:noop:report-as-issue: false@@ -130,7 +140,8 @@ deciding pass/fail:### 2c. Repository validation- Fetch the repository URL β confirm it exists and is publicly accessible-- Confirm the repository contains a `preset.yml` file+- Confirm the repository contains a `preset.yml` file (in the preset's+ subdirectory for a monorepo)- Confirm the repository contains a `LICENSE` file> The README requirement is enforced once, in **Step 2d**, against the specific file the@@ -170,10 +181,15 @@ preset** β not just any file named `README.md`, and not a product/framework pi- `specify preset add --dev <path>`A `specify preset add --from <url>` command only counts when its `<url>` **matches the- submitted Download URL exactly**. A `--from` command pointing at a *different* URL does- **not** satisfy the install-command requirement (treat it as if absent) β but the README- may still pass on one of the other accepted forms (`specify preset add <preset-id>` or- `specify preset add --dev <path>`).+ submitted Download URL exactly**. If the README also contains a `--from` release URL+ identifiable as this preset by its tag scope or matching release asset that differs+ from the Download URL, **fail** even if another accepted command (`specify preset add+ <preset-id>` or `specify preset add --dev <path>`) is present. Do not flag a different+ preset's unscoped release URL in a monorepo as stale. Bare archive tags are only+ compared when the downloaded archive contains one `preset.yml`: a bare tag alone+ cannot identify which preset it belongs to in a multi-preset archive. A README+ with only a valid `--dev` command remains acceptable. The verifier in Step 2g+ enforces this comparison.If **no** accepted `specify preset add ...` command is present, the README is treated as ageneric description/pitch rather than preset-usage documentation β **fail this check** and@@ -258,6 +274,35 @@ substitute for fetching the archive. Never execute downloaded content.- Confirm that all required checkboxes in the Testing Checklist and SubmissionRequirements sections are checked (`[x]`)+### 2g. Reproducible published-artifact and README comparison++After the pinned download has returned HTTP 200 and the optional checksum comparison+has succeeded, use the edit tool to save the fetched **exact documentation README**+as `/tmp/gh-aw/preset-readme.md`. Use the edit tool to save+`/tmp/gh-aw/preset-submission.json` as a JSON object with these keys copied from the+issue form (not inferred from the README or archive):+`preset_id`, `preset_name`, `version`, `description`, `author`, `repository`,+`download_url`, `documentation`, `license`, `speckit_version`,+`required_extensions` (empty string when absent), `templates_provided`,+`commands_provided`, `scripts_count` (string `"0"` when absent), and `tags`.+Retain multiline list values as strings. Do not interpolate issue or README text+into shell commands. Run this fixed command unchanged:++```bash+python3 .github/scripts/validate_community_preset.py submission --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json+```++The verifier reads `preset.yml` from the downloaded ZIP, including preset-scoped+paths in monorepos; it parses YAML without executing archive content. It checks+the published ID, version, Spec Kit requirement, required extension IDs, release+tag, and README install references against the issue, then records the validated+values, current UTC date at submission validation, and existing `created_at`+for the generated-file check. Exit 1 (`FAILED`)+is a confirmed submission mismatch: report it on the issue under Failed, with+no PR. Exit 2 (`BLOCKED`) means the verifier could not read the archive or other+required inputs (including a missing Python dependency): report the exact error+and run link under Blocked. Do not treat either exit as a pass.+### Validation outcomeChoose exactly one outcome below, in order. A check that could not run is@@ -291,9 +336,10 @@ If there are no environment blockers and a completed check found a submission de#### PassedIf there are no environment blockers and every required check completed and passed:-1. Remove `validation-failed`-2. Add the `validation-passed` label-3. Continue to Step 3+1. Remove any stale `validation-passed` and `validation-failed` labels from+ earlier runs.+2. Continue to Step 3. **Do not add `validation-passed` yet**: the generated+ catalog and documentation must pass Step 5's verifier before success is recorded.## Step 3 β Determine Add vs Update@@ -329,7 +375,7 @@ Insert the entry in **alphabetical order by preset ID** within the"repository": "<repository>","download_url": "<download_url>","sha256": "<actual_sha256>",- "homepage": "<homepage or repository>",+ "homepage": "<submitted repository URL>","documentation": "<documentation URL β the validated preset-usage README>","license": "<license>","requires": {@@ -340,8 +386,8 @@ Insert the entry in **alphabetical order by preset ID** within the"commands": <N>},"tags": ["<tag1>", "<tag2>"],- "created_at": "<today>T00:00:00Z",- "updated_at": "<today>T00:00:00Z"+ "created_at": "<validated UTC date>T00:00:00Z",+ "updated_at": "<validated UTC date>T00:00:00Z"}}```@@ -361,8 +407,11 @@ If the preset provides scripts, add `"scripts": <N>` inside `"provides"`.### For an updateReplace only the changed fields (typically `version`, `download_url`,-`description`, `provides`, `requires`, `tags`, `updated_at`). **Preserve**-`created_at` from the existing entry.+`description`, `homepage`, `provides`, `requires`, `tags`, `updated_at`). Set+`homepage` to the submitted repository URL; the form has no separate homepage+field. **Preserve**+`created_at` from the existing entry. Use the verifier snapshot's validated UTC+date for `updated_at`, even if the UTC date changes during the run.### Counting templates and commands@@ -372,16 +421,11 @@ Parse the "Templates Provided" and "Commands Provided" issue fields:### After editing-Update the **top-level `"updated_at"` timestamp** in the catalog to today's date-in ISO 8601 format.+Update the **top-level `"updated_at"` timestamp** in the catalog to the+verifier snapshot's UTC date in ISO 8601 format.-Validate the JSON by running:--```bash-python3 -c "import json; json.load(open('presets/catalog.community.json')); print('Valid JSON')"-```--If validation fails, fix the JSON and re-validate before continuing.+The generated-file verifier in Step 5 parses and checks the JSON. Fix any+catalog error it reports and rerun it before continuing.## Step 5 β Update `docs/community/presets.md`@@ -398,7 +442,8 @@ Insert a new row in **alphabetical order by preset name**:For the Requires column:- Use `β` if no extensions are required-- List required extension names if any (e.g., `AIDE extension`)+- List required extension IDs if any (e.g., `aide extension`), comma-separated+- Omit zero-count kinds from Provides and use singular nouns for counts of oneIf the preset provides scripts, include them: `<N> templates, <N> commands, <N> scripts`@@ -406,9 +451,34 @@ If the preset provides scripts, include them: `<N> templates, <N> commands, <N>Find the existing row and update any changed fields in-place.+### Verify the generated files++Before labeling success or requesting a PR, run this fixed command unchanged:++```bash+python3 .github/scripts/validate_community_preset.py generated --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json+```++The verifier checks JSON parsing, the validated catalog metadata (including+`homepage` set to the submitted repository URL) and digest,+the top-level and entry `updated_at` timestamps against the recorded UTC date+(and `created_at` for new entries),+alphabetical ID order, the documentation row's name, purpose, counts, extension+requirements and repository link, alphabetical name order, and preservation of+`created_at` on updates. Exit 3 (`REPAIR`) is an agent-generated catalog or+documentation error, **not** a submitter defect: fix the files and re-run this+command until it exits 0. Do not label `validation-failed` for an error in+generated files. Exit 2 (`BLOCKED`) means an input or tool is unavailable: report+the exact error and workflow run link to the maintainer, remove+`validation-passed`, and stop without a PR. If a generated-file error cannot be+corrected, remove `validation-passed`, stop without a PR, and report the problem for+maintainer investigation.+## Step 6 β Create Pull Request-Create a pull request with the changes. Use this branch naming convention:+Only after the generated-file verifier exits 0, add the `validation-passed`+label and request the configured draft pull request with the changes. Use this+branch naming convention:This repository-owned gh-aw maintenance workflow does not perform the contributoropen-PR count check or request confirmation. After successful validation anddiff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.pynew file mode 100644index 000000000..bda905260--- /dev/null+++ b/tests/test_community_preset_validation.py@@ -0,0 +1,735 @@+"""Regression coverage for the repository-owned preset submission verifier."""++import hashlib+import json+import subprocess+import sys+import zipfile+from datetime import datetime, timezone+from pathlib import Path++import pytest+import yaml++ROOT = Path(__file__).resolve().parent.parent+VERIFIER = ROOT / ".github" / "scripts" / "validate_community_preset.py"+WORKFLOW = ROOT / ".github" / "workflows" / "add-community-preset.md"+++def submission(tmp_path):+ issue = {+ "preset_id": "sample",+ "preset_name": "Sample Preset",+ "version": "1.2.3",+ "description": "Sample usage",+ "author": "Contributor",+ "repository": "https://github.com/example/presets",+ "download_url": "https://github.com/example/presets/releases/download/sample-v1.2.3/sample.zip",+ "documentation": "https://github.com/example/presets/blob/main/sample/README.md",+ "license": "MIT",+ "speckit_version": ">=1.0.0",+ "required_extensions": "aide, canon",+ "templates_provided": "- spec-template.md",+ "commands_provided": "- speckit.plan.md",+ "scripts_count": "0",+ "tags": "sample, example",+ }+ paths = {name: tmp_path / name for name in (+ "issue.json", "archive.zip", "README.md", "catalog.json", "presets.md",+ "snapshot.json",+ )}+ paths["README.md"].write_text(+ f"specify preset add --from {issue['download_url']}\n", encoding="utf-8"+ )+ manifest = {+ "preset": {"id": "sample", "version": "1.2.3"},+ "requires": {"speckit_version": ">=1.0.0", "extensions": ["aide", "canon"]},+ }+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:+ archive.writestr("presets-release/sample/preset.yml", yaml.safe_dump(manifest))+ archive.writestr("presets-release/other/preset.yml", yaml.safe_dump({+ "preset": {"id": "other", "version": "9.9.9"},+ "requires": {"speckit_version": ">=0.1.0"},+ }))+ issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest()+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["catalog.json"].write_text(+ json.dumps({"presets": {}}), encoding="utf-8"+ )+ paths["presets.md"].write_text(+ "| Preset | Purpose | Provides | Requires | URL |\n"+ "|--------|---------|----------|----------|-----|\n",+ encoding="utf-8",+ )+ return issue, manifest, paths+++def run_verifier(paths, phase="submission"):+ return subprocess.run(+ [+ sys.executable, str(VERIFIER), phase,+ "--issue", str(paths["issue.json"]),+ "--archive", str(paths["archive.zip"]),+ "--readme", str(paths["README.md"]),+ "--catalog", str(paths["catalog.json"]),+ "--docs", str(paths["presets.md"]),+ "--snapshot", str(paths["snapshot.json"]),+ ],+ capture_output=True, text=True, check=False,+ )+++def write_archive(paths, manifest):+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:+ archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest))+++def write_generated(issue, paths, *, created_at=None):+ timestamp = (+ json.loads(paths["snapshot.json"].read_text(encoding="utf-8"))["expected_timestamp"]+ if paths["snapshot.json"].exists()+ else "2026-01-01T00:00:00Z"+ )+ entry = {+ "id": issue["preset_id"], "name": issue["preset_name"],+ "version": issue["version"], "description": issue["description"],+ "author": issue["author"], "repository": issue["repository"],+ "download_url": issue["download_url"], "sha256": issue["actual_sha256"],+ "homepage": issue["repository"], "documentation": issue["documentation"],+ "license": issue["license"],+ "requires": {"speckit_version": issue["speckit_version"],+ "extensions": ["aide", "canon"]},+ "provides": {"templates": 1, "commands": 1},+ "tags": ["sample", "example"],+ "created_at": created_at if created_at is not None else timestamp,+ "updated_at": timestamp,+ }+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": entry["updated_at"], "presets": {"sample": entry},+ }), encoding="utf-8")+ preset_name = issue["preset_name"].replace("\\", r"\\").replace("|", r"\|")+ description = issue["description"].replace("\\", r"\\").replace("|", r"\|")+ paths["presets.md"].write_text(+ "| Preset | Purpose | Provides | Requires | URL |\n"+ "|--------|---------|----------|----------|-----|\n"+ f"| {preset_name} | {description} | 1 template, 1 command | "+ "aide extension, canon extension | "+ "[presets](https://github.com/example/presets) |\n",+ encoding="utf-8",+ )+ return entry+++def test_matching_monorepo_submission_and_generated_files_pass(submission):+ issue, _, paths = submission+ before = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z")+ first = run_verifier(paths)+ assert first.returncode == 0, first.stdout + first.stderr+ snapshot = json.loads(paths["snapshot.json"].read_text(encoding="utf-8"))+ after = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z")+ assert snapshot["expected_timestamp"] in (before, after)+ write_generated(issue, paths)+ second = run_verifier(paths, "generated")+ assert second.returncode == 0, second.stdout + second.stderr+++def test_update_preserving_created_at_passes(submission):+ issue, _, paths = submission+ original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": original["updated_at"], "presets": {"sample": original},+ }), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+ write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ assert run_verifier(paths, "generated").returncode == 0++[email protected](("change", "message"), [+ ({"preset": {"id": "sample", "version": "1.2.4"}}, "version"),+ ({"requires": {"speckit_version": ">=2.0.0",+ "extensions": ["aide", "canon"]}}, "speckit_version"),+ ({"requires": {"speckit_version": ">=1.0.0",+ "extensions": ["aide"]}}, "extensions"),+])+def test_published_manifest_mismatch_is_submission_failure(submission, change, message):+ _, manifest, paths = submission+ manifest.update(change)+ write_archive(paths, manifest)+ result = run_verifier(paths)+ assert result.returncode == 1+ assert message in result.stdout+ assert not paths["snapshot.json"].exists()+++def test_release_tag_mismatch_is_submission_failure(submission):+ issue, _, paths = submission+ issue["download_url"] = issue["download_url"].replace("v1.2.3", "v1.2.4")+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ assert run_verifier(paths).returncode == 1+++def test_stale_from_url_fails_even_with_valid_dev_command(submission):+ issue, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README" in result.stdout++[email protected]("archive_url", [False, True])+def test_stale_from_url_with_submitted_scoped_tag_fails(submission, archive_url):+ issue, _, paths = submission+ if archive_url:+ issue["download_url"] = (+ "https://github.com/example/presets/archive/refs/tags/"+ "spec-kit-sample-v1.2.3.zip"+ )+ else:+ issue["download_url"] = issue["download_url"].replace(+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1, result.stdout + result.stderr+ assert "README --from URL" in result.stdout+++def test_dev_only_readme_is_accepted(submission):+ _, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n", encoding="utf-8"+ )+ assert run_verifier(paths).returncode == 0+++def test_dev_current_directory_path_is_accepted(submission):+ _, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add --dev .\n", encoding="utf-8"+ )+ assert run_verifier(paths).returncode == 0+++def test_dev_option_without_path_is_rejected(submission):+ _, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add --dev --priority 20\n", encoding="utf-8"+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README" in result.stdout+++def test_dev_without_path_does_not_consume_next_paragraph(submission):+ _, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add --dev\n"+ "Next paragraph describes the preset.\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README" in result.stdout+++def test_quoted_from_url_with_sentence_punctuation_is_accepted(submission):+ issue, _, paths = submission+ paths["README.md"].write_text(+ f'Spec Kit install: `specify preset add --from "{issue["download_url"]}".`\n',+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0+++def test_id_install_and_unrelated_monorepo_release_are_accepted(submission):+ _, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add sample\n"+ "specify preset add --from "+ "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0+++def test_unrelated_scoped_release_stays_accepted_with_submitted_scope(submission):+ issue, _, paths = submission+ issue["download_url"] = issue["download_url"].replace(+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add sample\n"+ "specify preset add --from "+ "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0++[email protected]("archive_url", [False, True])+def test_unrelated_unscoped_monorepo_release_stays_accepted(submission, archive_url):+ issue, _, paths = submission+ if archive_url:+ issue["download_url"] = (+ "https://github.com/example/presets/archive/refs/tags/"+ "spec-kit-sample-v1.2.3.zip"+ )+ unrelated = (+ "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip"+ )+ else:+ issue["download_url"] = issue["download_url"].replace(+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"+ )+ unrelated = (+ "https://github.com/example/presets/releases/download/v2.0.0/other.zip"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ f"specify preset add --from {unrelated}\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0+++def test_unrelated_unscoped_archive_stays_accepted_in_monorepo(submission):+ issue, _, paths = submission+ issue["download_url"] = (+ "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ "specify preset add --from "+ "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0+++def test_stale_unscoped_archive_fails_for_single_preset(submission):+ issue, manifest, paths = submission+ write_archive(paths, manifest)+ issue["download_url"] = (+ "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ "specify preset add --from "+ "https://github.com/example/presets/archive/refs/tags/v1.2.2.zip\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README --from URL" in result.stdout+++def test_matching_unscoped_archive_passes_for_single_preset(submission):+ issue, manifest, paths = submission+ write_archive(paths, manifest)+ issue["download_url"] = (+ "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ f"specify preset add --from {issue['download_url']}\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0+++def test_same_asset_on_unscoped_tag_is_reported_as_stale(submission):+ issue, _, paths = submission+ issue["download_url"] = issue["download_url"].replace(+ "sample-v1.2.3", "v1.2.3"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README --from URL" in result.stdout+++def test_versioned_asset_on_unscoped_tag_is_reported_as_stale(submission):+ issue, _, paths = submission+ issue["download_url"] = (+ "https://github.com/example/presets/releases/download/"+ "v1.2.3/sample-1.2.3.zip"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ "specify preset add --from "+ "https://github.com/example/presets/releases/download/"+ "v1.2.2/sample-1.2.2.zip\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README --from URL" in result.stdout+++def test_same_asset_on_bare_tag_stays_stale_for_scoped_submission(submission):+ issue, _, paths = submission+ issue["download_url"] = issue["download_url"].replace(+ "sample-v1.2.3", "spec-kit-sample-v1.2.3"+ )+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ "specify preset add --from "+ "https://github.com/example/presets/releases/download/v1.2.2/sample.zip\n",+ encoding="utf-8",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "README --from URL" in result.stdout+++def test_stale_scoped_release_in_another_repository_fails(submission):+ _, _, paths = submission+ paths["README.md"].write_text(+ "specify preset add --dev ./sample\n"+ "specify preset add --from "+ "https://github.com/elsewhere/presets/releases/download/sample-v1.2.2/sample.zip\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 1+++def test_optional_manifest_extension_is_not_required(submission):+ issue, manifest, paths = submission+ manifest["requires"]["extensions"].append({+ "id": "optional", "version": ">=1.0.0", "required": False,+ })+ write_archive(paths, manifest)+ issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest()+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+++def test_invalid_unrelated_monorepo_manifest_does_not_mask_match(submission):+ _, manifest, paths = submission+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:+ archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest))+ archive.writestr("release/other/preset.yml", "preset: [invalid\n")+ assert run_verifier(paths).returncode == 0+++def test_archive_with_too_many_manifests_is_rejected_before_parsing(submission):+ _, _, paths = submission+ with zipfile.ZipFile(paths["archive.zip"], "w") as archive:+ for index in range(101):+ archive.writestr(f"release/{index}/preset.yml", "preset: [invalid\n")+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "more than 100 preset.yml files" in result.stdout+ assert "invalid manifests" not in result.stdout+++def test_archive_with_excessive_total_manifest_size_is_rejected(submission):+ _, manifest, paths = submission+ padding = "#" * (1024 * 1024 - len(yaml.safe_dump(manifest)) - 2)+ with zipfile.ZipFile(+ paths["archive.zip"], "w", compression=zipfile.ZIP_DEFLATED+ ) as archive:+ for index in range(11):+ archive.writestr(+ f"release/{index}/preset.yml",+ f"{yaml.safe_dump(manifest)}\n{padding}",+ )+ result = run_verifier(paths)+ assert result.returncode == 1+ assert "10 MiB total limit" in result.stdout+++def test_missing_archive_is_blocked_not_failed(submission):+ _, _, paths = submission+ paths["archive.zip"].unlink()+ result = run_verifier(paths)+ assert result.returncode == 2+ assert "BLOCKED" in result.stdout+++def test_missing_generated_documentation_is_repairable(submission):+ issue, _, paths = submission+ assert run_verifier(paths).returncode == 0+ write_generated(issue, paths)+ paths["presets.md"].unlink()+ result = run_verifier(paths, "generated")+ assert result.returncode == 3+ assert "REPAIR" in result.stdout+++def test_existing_entry_without_creation_date_blocks_update(submission):+ issue, _, paths = submission+ entry = write_generated(issue, paths)+ del entry["created_at"]+ paths["catalog.json"].write_text(+ json.dumps({"presets": {"sample": entry}}), encoding="utf-8"+ )+ result = run_verifier(paths)+ assert result.returncode == 2+ assert "created_at" in result.stdout++[email protected]("timestamp_field", ["created_at", "updated_at"])+def test_generated_new_entry_rejects_stale_dates(submission, timestamp_field):+ issue, _, paths = submission+ assert run_verifier(paths).returncode == 0+ entry = write_generated(issue, paths)+ entry[timestamp_field] = "2000-01-01T00:00:00Z"+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": entry["updated_at"], "presets": {"sample": entry},+ }), encoding="utf-8")+ result = run_verifier(paths, "generated")+ assert result.returncode == 3, result.stdout + result.stderr+ assert timestamp_field in result.stdout+++def test_generated_update_rejects_matching_stale_updated_dates(submission):+ issue, _, paths = submission+ entry = write_generated(issue, paths)+ assert run_verifier(paths).returncode == 0+ entry["updated_at"] = "2000-01-01T00:00:00Z"+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": entry["updated_at"], "presets": {"sample": entry},+ }), encoding="utf-8")+ result = run_verifier(paths, "generated")+ assert result.returncode == 3, result.stdout + result.stderr+ assert "updated_at" in result.stdout+++def test_generated_update_rejects_stale_homepage(submission):+ issue, _, paths = submission+ entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ entry["homepage"] = "https://example.com/old"+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": entry["updated_at"], "presets": {"sample": entry},+ }), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+ entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ entry["homepage"] = "https://example.com/old"+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": entry["updated_at"], "presets": {"sample": entry},+ }), encoding="utf-8")+ result = run_verifier(paths, "generated")+ assert result.returncode == 3+ assert "homepage" in result.stdout+++def test_generated_documentation_accepts_escaped_pipe_in_preset_name(submission):+ issue, _, paths = submission+ issue["preset_name"] = "Data | Governance"+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+ write_generated(issue, paths)+ result = run_verifier(paths, "generated")+ assert result.returncode == 0, result.stdout + result.stderr+++def test_generated_documentation_accepts_backslash_before_pipe(submission):+ issue, _, paths = submission+ issue["preset_name"] = r"Data \| Governance"+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+ write_generated(issue, paths)+ result = run_verifier(paths, "generated")+ assert result.returncode == 0, result.stdout + result.stderr+++def test_generated_documentation_allows_duplicate_display_names(submission):+ issue, _, paths = submission+ other_row = (+ "| Sample Preset | Other usage | 1 command | β | "+ "[other](https://github.com/example/other) |"+ )+ paths["catalog.json"].write_text(json.dumps({+ "presets": {"other": {"name": issue["preset_name"]}},+ }), encoding="utf-8")+ paths["presets.md"].write_text(+ "| Preset | Purpose | Provides | Requires | URL |\n"+ "|--------|---------|----------|----------|-----|\n"+ f"{other_row}\n",+ encoding="utf-8",+ )+ assert run_verifier(paths).returncode == 0+ entry = write_generated(issue, paths)+ catalog = json.loads(paths["catalog.json"].read_text(encoding="utf-8"))+ catalog["presets"] = {+ "other": {"name": issue["preset_name"]},+ "sample": entry,+ }+ paths["catalog.json"].write_text(json.dumps(catalog), encoding="utf-8")+ paths["presets.md"].write_text(+ "| Preset | Purpose | Provides | Requires | URL |\n"+ "|--------|---------|----------|----------|-----|\n"+ f"{other_row}\n"+ "| Sample Preset | Sample usage | 1 template, 1 command | "+ "aide extension, canon extension | "+ "[presets](https://github.com/example/presets) |\n",+ encoding="utf-8",+ )+ result = run_verifier(paths, "generated")+ assert result.returncode == 0, result.stdout + result.stderr+++def test_generated_documentation_accepts_replaced_renamed_row(submission):+ issue, _, paths = submission+ original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ paths["presets.md"].write_text(+ paths["presets.md"].read_text(encoding="utf-8").replace(+ "Sample usage", "Legacy documentation wording"+ ),+ encoding="utf-8",+ )+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": original["updated_at"], "presets": {"sample": original},+ }), encoding="utf-8")+ issue["preset_name"] = "Renamed Preset"+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+ write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ result = run_verifier(paths, "generated")+ assert result.returncode == 0, result.stdout + result.stderr+++def test_generated_documentation_rejects_stale_row_after_rename(submission):+ issue, _, paths = submission+ original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ paths["presets.md"].write_text(+ paths["presets.md"].read_text(encoding="utf-8").replace(+ "Sample usage", "Legacy documentation wording"+ ),+ encoding="utf-8",+ )+ previous_row = paths["presets.md"].read_text(encoding="utf-8").splitlines()[2]+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": original["updated_at"], "presets": {"sample": original},+ }), encoding="utf-8")+ issue["preset_name"] = "Renamed Preset"+ paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8")+ assert run_verifier(paths).returncode == 0+ write_generated(issue, paths, created_at="2024-12-01T00:00:00Z")+ paths["presets.md"].write_text(+ paths["presets.md"].read_text(encoding="utf-8") + previous_row + "\n",+ encoding="utf-8",+ )+ result = run_verifier(paths, "generated")+ assert result.returncode == 3+ assert "previous documentation row" in result.stdout++[email protected](("damage", "message"), [+ ("catalog-json", "catalog"),+ ("catalog-order", "alphabetical"),+ ("catalog-metadata", "version"),+ ("homepage-missing", "homepage"),+ ("homepage-stale", "homepage"),+ ("catalog-timestamp", "top-level updated_at"),+ ("docs-order", "alphabetical"),+ ("docs-row", "documentation row"),+ ("created-at", "created_at"),+])+def test_generated_defects_are_fixable_not_submission_failures(submission, damage, message):+ issue, _, paths = submission+ if damage == "created-at":+ original = write_generated(issue, paths)+ paths["catalog.json"].write_text(+ json.dumps({"updated_at": original["updated_at"], "presets": {+ "sample": original,+ }}), encoding="utf-8"+ )+ assert run_verifier(paths).returncode == 0+ entry = write_generated(issue, paths)+ if damage == "catalog-json":+ paths["catalog.json"].write_text("{", encoding="utf-8")+ elif damage == "catalog-order":+ paths["catalog.json"].write_text(json.dumps({+ "updated_at": entry["updated_at"],+ "presets": {"sample": entry, "aaa": {"name": "AAA"}},+ }), encoding="utf-8")+ elif damage == "catalog-metadata":+ entry["version"] = "1.2.4"+ paths["catalog.json"].write_text(+ json.dumps({"updated_at": entry["updated_at"], "presets": {+ "sample": entry,+ }}), encoding="utf-8"+ )+ elif damage in ("homepage-missing", "homepage-stale"):+ if damage == "homepage-missing":+ del entry["homepage"]+ else:+ entry["homepage"] = "https://github.com/example/other"+ paths["catalog.json"].write_text(+ json.dumps({"updated_at": entry["updated_at"], "presets": {+ "sample": entry,+ }}), encoding="utf-8"+ )+ elif damage == "catalog-timestamp":+ paths["catalog.json"].write_text(+ json.dumps({"updated_at": "2020-01-01T00:00:00Z", "presets": {+ "sample": entry,+ }}), encoding="utf-8"+ )+ elif damage == "docs-order":+ paths["presets.md"].write_text(+ paths["presets.md"].read_text(encoding="utf-8")+ + "| AAA | x | 1 command | β | [aaa](https://github.com/aaa/aaa) |\n",+ encoding="utf-8",+ )+ elif damage == "docs-row":+ paths["presets.md"].write_text(+ paths["presets.md"].read_text(encoding="utf-8").replace(+ "Sample usage", "Wrong purpose"+ ), encoding="utf-8",+ )+ elif damage == "created-at":+ entry["created_at"] = "2000-01-01T00:00:00Z"+ paths["catalog.json"].write_text(+ json.dumps({"updated_at": entry["updated_at"], "presets": {+ "sample": entry,+ }}), encoding="utf-8"+ )+ result = run_verifier(paths, "generated")+ assert result.returncode == 3, result.stdout + result.stderr+ assert message in result.stdout+++def test_workflow_gates_success_on_both_verifier_phases():+ source = WORKFLOW.read_text(encoding="utf-8")+ assert "python3 .github/scripts/validate_community_preset.py submission" in source+ assert "python3 .github/scripts/validate_community_preset.py generated" in source+ assert (source.index("validate_community_preset.py generated")+ < source.index("## Step 6")+ < source.index("add the `validation-passed`", source.index("## Step 6")))+ frontmatter = yaml.safe_load(source.split("---", 2)[1])+ assert [step["name"] for step in frontmatter["steps"]] == [+ "Set up Python for preset verification",+ "Install preset verifier dependency",+ ]+ compiled = yaml.safe_load(+ (ROOT / ".github/workflows/add-community-preset.lock.yml").read_text(+ encoding="utf-8"+ )+ )+ steps = compiled["jobs"]["agent"]["steps"]+ for setup in frontmatter["steps"]:+ assert setup in stepsdiff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.pyindex 5268ac8c4..5b6d1d887 100644--- a/tests/test_github_workflows.py+++ b/tests/test_github_workflows.py@@ -817,10 +817,21 @@ def test_community_archive_permission_failures_are_not_submission_failures(kind)"If there are no environment blockers and every required check completed ""and passed:")- assert re.search(- r"remove `validation-failed`.*add (?:the )?`validation-passed`",- passed, re.IGNORECASE | re.DOTALL,- )+ if kind == "preset":+ assert "Remove any stale `validation-passed` and `validation-failed`" in passed+ assert "Do not add `validation-passed` yet" in passed+ generated = source_text.split("### Verify the generated files", 1)[1].split(+ "\n## Step 6", 1+ )[0]+ assert "validate_community_preset.py generated" in generated+ assert "add the `validation-passed`" in source_text.split(+ "\n## Step 6", 1+ )[1]+ else:+ assert re.search(+ r"remove `validation-failed`.*add (?:the )?`validation-passed`",+ passed, re.IGNORECASE | re.DOTALL,+ )assert "validation-failed" in source["safe-outputs"]["remove-labels"]["allowed"]assert "validation-failed" in _safe_output_config(compiled)["remove_labels"]["allowed"]assert "validation-passed" in source["safe-outputs"]["remove-labels"]["allowed"]