💫 Toolkit to help you get started with SDD or any other process!
fix(integrations): wrap a non-UTF-8 catalog response (#4011)
Noor ul ain2a28f62e2572
diff --git a/src/specify_cli/integrations/catalog.py b/src/specify_cli/integrations/catalog.pyindex b3be8a84e..e18d30a6f 100644--- a/src/specify_cli/integrations/catalog.py+++ b/src/specify_cli/integrations/catalog.py@@ -237,6 +237,15 @@ class IntegrationCatalog(CatalogStackBase):raise IntegrationCatalogError(f"Failed to fetch catalog from {entry.url}: {exc}")+ except UnicodeDecodeError as exc:+ # A non-UTF-8 response body fails at .decode() before json.loads()+ # ever runs, so JSONDecodeError below does not cover it (the two are+ # sibling ValueError subclasses, not parent/child). Without this the+ # raw UnicodeDecodeError escapes _get_merged_integrations()'s+ # "warn and skip this catalog" handler and kills the whole command.+ raise IntegrationCatalogError(+ f"Catalog from {entry.url} is not valid UTF-8: {exc}"+ )except json.JSONDecodeError as exc:raise IntegrationCatalogError(f"Invalid JSON in catalog from {entry.url}: {exc}"diff --git a/tests/integrations/test_integration_catalog.py b/tests/integrations/test_integration_catalog.pyindex 68e8970c4..9b0263299 100644--- a/tests/integrations/test_integration_catalog.py+++ b/tests/integrations/test_integration_catalog.py@@ -392,6 +392,125 @@ class TestCatalogFetch:with pytest.raises(IntegrationCatalogError, match="exceeds maximum size"):cat._fetch_single_catalog(entry, force_refresh=True)+ def _patch_urlopen_bytes(self, monkeypatch, bodies):+ """Patch urlopen to serve raw *bodies* keyed by URL substring.++ Mirrors ``_patch_urlopen`` but passes the bytes through verbatim: these+ tests need a body that is not valid UTF-8, which ``json.dumps`` cannot+ produce.+ """++ class _RawResponse:+ def __init__(self, data, url):+ self._data = data+ self._url = url+ self._offset = 0++ def read(self, size=-1):+ if size == -1:+ chunk = self._data[self._offset:]+ self._offset = len(self._data)+ else:+ chunk = self._data[self._offset:self._offset + size]+ self._offset += len(chunk)+ return chunk++ def geturl(self):+ return self._url++ def __enter__(self):+ return self++ def __exit__(self, *a):+ pass++ def fake_urlopen(req, timeout=10):+ url = req if isinstance(req, str) else req.full_url+ for marker, body in bodies.items():+ if marker in url:+ return _RawResponse(body, url)+ raise AssertionError(f"unexpected URL requested: {url}")++ import specify_cli.authentication.http as _auth_http+ monkeypatch.setattr(_auth_http.urllib.request, "urlopen", fake_urlopen)++ def test_fetch_wraps_non_utf8_catalog_response(self, tmp_path, monkeypatch):+ """Regression: a non-UTF-8 response body must raise IntegrationCatalogError.++ ``.decode("utf-8")`` runs before ``json.loads``, so the resulting+ UnicodeDecodeError is not a JSONDecodeError and slipped past both+ handlers as a raw traceback.+ """+ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("USERPROFILE", str(tmp_path))+ monkeypatch.delenv("SPECKIT_INTEGRATION_CATALOG_URL", raising=False)+ (tmp_path / ".specify").mkdir(exist_ok=True)+ cat = IntegrationCatalog(tmp_path)++ self._patch_urlopen_bytes(+ monkeypatch,+ {"catalog.json": b'{"schema_version": "1.0", "name": "\xff\xfe"}'},+ )++ entry = IntegrationCatalogEntry(+ url="https://example.com/catalog.json",+ name="test",+ priority=1,+ install_allowed=True,+ )++ with pytest.raises(IntegrationCatalogError, match="not valid UTF-8"):+ cat._fetch_single_catalog(entry, force_refresh=True)++ def test_search_skips_non_utf8_catalog(self, tmp_path, monkeypatch, capsys):+ """A single non-UTF-8 catalog must not take down the whole search.++ ``_get_merged_integrations`` is built to warn and continue on a bad+ catalog; an unwrapped UnicodeDecodeError defeated that entirely.+ """+ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("USERPROFILE", str(tmp_path))+ monkeypatch.delenv("SPECKIT_INTEGRATION_CATALOG_URL", raising=False)+ specify = tmp_path / ".specify"+ specify.mkdir(exist_ok=True)+ (specify / "integration-catalogs.yml").write_text(+ "catalogs:\n"+ " - name: broken\n"+ " url: https://example.com/broken.json\n"+ " priority: 1\n"+ " - name: healthy\n"+ " url: https://example.com/healthy.json\n"+ " priority: 2\n",+ encoding="utf-8",+ )++ healthy = json.dumps(+ {+ "schema_version": "1.0",+ "integrations": {+ "acme-coder": {+ "name": "Acme Coder",+ "version": "1.0.0",+ "description": "Acme integration",+ }+ },+ }+ ).encode("utf-8")++ self._patch_urlopen_bytes(+ monkeypatch,+ {+ "broken.json": b'{"schema_version": "1.0", "name": "\xff\xfe"}',+ "healthy.json": healthy,+ },+ )++ cat = IntegrationCatalog(tmp_path)+ results = cat.search()++ assert "acme-coder" in [r["id"] for r in results]+ assert "broken" in capsys.readouterr().err+def test_search_by_tag(self, tmp_path, monkeypatch):monkeypatch.setenv("HOME", str(tmp_path))monkeypatch.setenv("USERPROFILE", str(tmp_path))