π« Toolkit to help you get started with SDD or any other process!
fix: use bounded read for integration catalog HTTP responses (#3763)
Quratulain-bilal89e204ca3c85
diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.pyindex da255ed86..ef776b18f 100644--- a/src/specify_cli/integrations/base.py+++ b/src/specify_cli/integrations/base.py@@ -1652,13 +1652,27 @@ class SkillsIntegration(IntegrationBase):command_name = src_file.stem # e.g. "plan"skill_name = f"speckit-{command_name.replace('.', '-')}"- # Parse frontmatter for description+ # Parse frontmatter for description. Locate the closing ``---`` on+ # its own line rather than with ``raw.split("---", 2)`` β a bare+ # substring split stops at the first ``---`` *anywhere*, including+ # one inside a value such as ``description: Separate sections+ # with ---``, which truncates the frontmatter and drops later keys.+ # The block between the delimiters is parsed unstripped so trailing+ # newlines in literal (``|``) block scalars survive.frontmatter: dict[str, Any] = {}if raw.startswith("---"):- parts = raw.split("---", 2)- if len(parts) >= 3:+ fm_lines = raw.splitlines(keepends=True)+ fm_close = next(+ (+ i+ for i in range(1, len(fm_lines))+ if fm_lines[i].rstrip() == "---"+ ),+ None,+ )+ if fm_close is not None:try:- fm = yaml.safe_load(parts[1])+ fm = yaml.safe_load("".join(fm_lines[1:fm_close]))if isinstance(fm, dict):frontmatter = fmexcept yaml.YAMLError:@@ -1673,11 +1687,27 @@ class SkillsIntegration(IntegrationBase):# Strip the processed frontmatter β we rebuild it for skills.# Preserve leading whitespace in the body to match release ZIP# output byte-for-byte (the template body starts with \n after- # the closing ---).+ # the closing ---). Scan for the closing ``---`` on its own line+ # rather than ``split("---", 2)`` so a ``---`` embedded in a value+ # does not truncate the frontmatter and spill it into the body.if processed_body.startswith("---"):- parts = processed_body.split("---", 2)- if len(parts) >= 3:- processed_body = parts[2]+ body_lines = processed_body.splitlines(keepends=True)+ close_idx = next(+ (+ i+ for i in range(1, len(body_lines))+ if body_lines[i].rstrip() == "---"+ ),+ None,+ )+ if close_idx is not None:+ # Keep whatever trails the ``---`` marker on the closing+ # line (normally just the newline) so the body stays+ # byte-for-byte identical to ``split("---", 2)[2]``. The+ # line-anchored check guarantees ``---`` sits at index 0.+ processed_body = body_lines[close_idx][3:] + "".join(+ body_lines[close_idx + 1 :]+ )# Select description β use the original template description# to stay byte-for-byte identical with release ZIP output.diff --git a/src/specify_cli/integrations/catalog.py b/src/specify_cli/integrations/catalog.pyindex 1ed98f5fd..1794caad8 100644--- a/src/specify_cli/integrations/catalog.py+++ b/src/specify_cli/integrations/catalog.py@@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional, Tupleimport yamlfrom packaging import version as pkg_version+from .._download_security import MAX_JSON_METADATA_BYTES, read_response_limitedfrom ..catalogs import CatalogEntry, CatalogStackBase@@ -200,7 +201,14 @@ class IntegrationCatalog(CatalogStackBase):final_url = resp.geturl()if final_url != entry.url:self._validate_catalog_url(final_url)- catalog_data = json.loads(resp.read())+ catalog_data = json.loads(+ read_response_limited(+ resp,+ max_bytes=MAX_JSON_METADATA_BYTES,+ error_type=IntegrationCatalogError,+ label=f"catalog from {entry.url}",+ )+ )shape_error = _catalog_shape_error(catalog_data)if shape_error is not None:diff --git a/tests/integrations/test_integration_catalog.py b/tests/integrations/test_integration_catalog.pyindex 2e9706e77..4688c6a21 100644--- a/tests/integrations/test_integration_catalog.py+++ b/tests/integrations/test_integration_catalog.py@@ -220,6 +220,33 @@ class TestActiveCatalogs:# ---------------------------------------------------------------------------+class _OversizedResponse:+ """Response stub that supports bounded streaming reads for oversized-catalog tests."""++ def __init__(self, data, url=""):+ self._data = json.dumps(data).encode()+ self._url = url if isinstance(url, str) else url.full_url+ self._pos = 0++ def read(self, n=-1):+ if n < 0:+ chunk = self._data[self._pos:]+ self._pos = len(self._data)+ return chunk+ chunk = self._data[self._pos : self._pos + n]+ self._pos += len(chunk)+ return chunk++ def geturl(self):+ return self._url++ def __enter__(self):+ return self++ def __exit__(self, *a):+ pass++class TestCatalogFetch:"""Tests that use a local HTTP server stub via monkeypatch."""@@ -230,9 +257,16 @@ class TestCatalogFetch:def __init__(self, data, url=""):self._data = json.dumps(data).encode()self._url = url if isinstance(url, str) else url.full_url+ self._pos = 0- def read(self):- return self._data+ def read(self, n=-1):+ if n < 0:+ chunk = self._data[self._pos:]+ self._pos = len(self._data)+ return chunk+ chunk = self._data[self._pos:self._pos + n]+ self._pos += len(chunk)+ return chunkdef geturl(self):return self._url@@ -395,6 +429,90 @@ class TestCatalogFetch:with pytest.raises(IntegrationCatalogError, match="Failed to fetch any integration catalog"):cat.search()+ def test_oversized_catalog_response_rejected(self, tmp_path, monkeypatch):+ """Response exceeding MAX_JSON_METADATA_BYTES is caught as IntegrationCatalogError.++ The per-entry error is logged as a warning and skipped (not fatal).+ When ALL catalogs are oversized, search() raises the aggregate error.+ """+ from specify_cli._download_security import MAX_JSON_METADATA_BYTES++ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("USERPROFILE", str(tmp_path))+ monkeypatch.delenv("SPECKIT_INTEGRATION_CATALOG_URL", raising=False)+ (tmp_path / ".specify").mkdir()+ cat = IntegrationCatalog(tmp_path)++ # Build a valid catalog dict whose JSON encoding exceeds the limit.+ oversized = {+ "schema_version": "1.0",+ "integrations": {},+ "padding": "x" * (MAX_JSON_METADATA_BYTES + 1),+ }++ import specify_cli.authentication.http as _auth_http++ def _oversized_urlopen(req, timeout=10):+ url = req if isinstance(req, str) else req.full_url+ return _OversizedResponse(oversized, url)++ monkeypatch.setattr(_auth_http.urllib.request, "urlopen", _oversized_urlopen)++ # Both default + community catalogs are oversized β all fail β aggregate error.+ # The per-entry IntegrationCatalogError (with "exceeds maximum size") is+ # logged as a warning; the aggregate raise has a different message.+ with pytest.raises(IntegrationCatalogError, match="Failed to fetch any integration catalog"):+ cat.search()++ def test_oversized_catalog_does_not_block_healthy_one(self, tmp_path, monkeypatch):+ """When one catalog is oversized, the healthy catalog still returns results."""+ from specify_cli._download_security import MAX_JSON_METADATA_BYTES++ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("USERPROFILE", str(tmp_path))+ monkeypatch.delenv("SPECKIT_INTEGRATION_CATALOG_URL", raising=False)+ specify = tmp_path / ".specify"+ specify.mkdir()++ healthy_catalog = {+ "schema_version": "1.0",+ "integrations": {+ "good-agent": {+ "id": "good-agent",+ "name": "Good Agent",+ "version": "1.0.0",+ "description": "A healthy integration",+ "author": "test-org",+ },+ },+ }+ oversized_catalog = {+ "schema_version": "1.0",+ "integrations": {},+ "padding": "x" * (MAX_JSON_METADATA_BYTES + 1),+ }+ cfg = specify / "integration-catalogs.yml"+ cfg.write_text(yaml.dump({"catalogs": [+ {"url": "https://healthy.example.com/catalog.json", "name": "healthy", "priority": 1, "install_allowed": True},+ {"url": "https://oversized.example.com/catalog.json", "name": "oversized", "priority": 2, "install_allowed": True},+ ]}))+ cat = IntegrationCatalog(tmp_path)++ import specify_cli.authentication.http as _auth_http++ def _multi_catalog_urlopen(req, timeout=10):+ url = req if isinstance(req, str) else req.full_url+ if "oversized" in url:+ return _OversizedResponse(oversized_catalog, url)+ return _OversizedResponse(healthy_catalog, url)++ monkeypatch.setattr(_auth_http.urllib.request, "urlopen", _multi_catalog_urlopen)++ # The oversized catalog is skipped; the healthy catalog's integrations are returned.+ results = cat.search()+ ids = [r["id"] for r in results]+ assert "good-agent" in ids+def test_clear_cache(self, tmp_path):(tmp_path / ".specify").mkdir()cat = IntegrationCatalog(tmp_path)@@ -592,8 +710,15 @@ class TestIntegrationListCatalog:def __init__(self, data, url=""):self._data = json.dumps(data).encode()self._url = url if isinstance(url, str) else url.full_url- def read(self):- return self._data+ self._pos = 0+ def read(self, n=-1):+ if n < 0:+ chunk = self._data[self._pos:]+ self._pos = len(self._data)+ return chunk+ chunk = self._data[self._pos:self._pos + n]+ self._pos += len(chunk)+ return chunkdef geturl(self):return self._urldef __enter__(self):diff --git a/tests/integrations/test_skill_frontmatter_quoting.py b/tests/integrations/test_skill_frontmatter_quoting.pyindex 739039ea3..b42ad8845 100644--- a/tests/integrations/test_skill_frontmatter_quoting.py+++ b/tests/integrations/test_skill_frontmatter_quoting.py@@ -34,6 +34,19 @@ description: "ding\\aling"Body of the command."""+# A description whose value contains an embedded ``---``. A substring split+# (``raw.split("---", 2)``) stops at this inner marker, truncating the parsed+# frontmatter β the closing document separator on its own line is the real+# boundary. See TestSkillFrontmatterEmbeddedDashes below.+DASHED_DESCRIPTION = "Separate sections with --- markers"+DASHED_TEMPLATE = """---+description: Separate sections with --- markers+name-marker: sentinel+---++Body of the command.+"""+def _parse_frontmatter(skill_file: Path) -> dict:content = skill_file.read_text(encoding="utf-8")@@ -90,6 +103,62 @@ class TestSkillFrontmatterQuoting:assert fm["description"] == CONTROL+def _parse_frontmatter_line_anchored(skill_file: Path) -> dict:+ """Parse SKILL.md frontmatter using the closing ``---`` on its own line.++ Unlike ``_parse_frontmatter`` (which uses ``split("---", 2)``), this is+ robust to a ``---`` embedded in a value, so it can validate that the+ generated frontmatter is itself well formed.+ """+ content = skill_file.read_text(encoding="utf-8")+ assert content.startswith("---\n")+ lines = content.splitlines(keepends=True)+ end = next(i for i in range(1, len(lines)) if lines[i].rstrip() == "---")+ return yaml.safe_load("".join(lines[1:end]))+++class TestSkillFrontmatterEmbeddedDashes:+ """A ``---`` inside a description value must not truncate parsing (#3634).++ The skills setup path parsed template frontmatter with+ ``raw.split("---", 2)``, which stops at the first ``---`` *anywhere* β+ including one inside a value such as ``description: ... --- ...``. That+ dropped every frontmatter key after the marker (so the description fell+ back to the generic default) and spilled the leftover frontmatter into+ the skill body. The parser must match the closing ``---`` on its own line.+ """++ def _generate(self, tmp_path, monkeypatch, template: str) -> Path:+ integration = get_integration("agy")+ monkeypatch.setattr(+ integration,+ "shared_commands_dir",+ lambda: _fake_templates(tmp_path, template),+ )+ manifest = IntegrationManifest("agy", tmp_path)+ created = integration.setup(tmp_path, manifest)+ skill_files = [f for f in created if f.name == "SKILL.md"]+ assert len(skill_files) == 1+ return skill_files[0]++ def test_dashed_description_is_preserved(self, tmp_path, monkeypatch):+ skill_file = self._generate(tmp_path, monkeypatch, DASHED_TEMPLATE)+ fm = _parse_frontmatter_line_anchored(skill_file)+ # Buggy split("---", 2) truncates the value to "Separate sections with"+ # (or drops it entirely, falling back to "Spec Kit: plan workflow").+ assert fm["description"] == DASHED_DESCRIPTION++ def test_leftover_frontmatter_not_spilled_into_body(self, tmp_path, monkeypatch):+ skill_file = self._generate(tmp_path, monkeypatch, DASHED_TEMPLATE)+ content = skill_file.read_text(encoding="utf-8")+ lines = content.splitlines(keepends=True)+ end = next(i for i in range(1, len(lines)) if lines[i].rstrip() == "---")+ body = "".join(lines[end + 1 :])+ # The template's trailing frontmatter key must not leak into the body.+ assert "name-marker: sentinel" not in body+ assert "Body of the command." in body++class TestHermesSkillFrontmatterQuoting:def test_multiline_description_survives(self, tmp_path, monkeypatch):home = tmp_path / "home"