Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

feat(connector-checks): add HubSpot Tier 1 read checks to smoke V2 (#3759)

蓝鱼bluefish97ecd0b08bba
diff --git a/docs/operations/connector-smoke-v2.md b/docs/operations/connector-smoke-v2.md
index 98bf92f04..2183c2736 100644
--- a/docs/operations/connector-smoke-v2.md
+++ b/docs/operations/connector-smoke-v2.md
@@ -134,7 +134,7 @@ At run time (see [Running write checks](#running-write-checks)):
Review each write flow against these, which validation cannot check:
-- Cleanup and sweep use the connector's own tools, so a connector whose server cannot delete what its write check creates declares no write check until its Tier 1 issue settles how to clean up. Today that includes the HubSpot MCP server (#3572), the in-repo GitHub server (#3568), Slack's messages (#3743, split from its Tier 1 issue #3571), Gmail's drafts (#3567) and Teams' messages (#3569).
+- Cleanup and sweep use the connector's own tools, so a connector whose server cannot delete what its write check creates declares no write check until its Tier 1 issue settles how to clean up. Today that includes the HubSpot MCP server, which has no delete tool (#3572), the in-repo GitHub server (#3568), Slack's messages (#3743, split from its Tier 1 issue #3571), Gmail's drafts (#3567) and Teams' messages (#3569).
- The sweep's `find` lists the same container `create` writes into. Jira's search takes only JQL, so its sweep reads a query fixture rather than the project key `create` takes.
- Each step's target comes from its reference or a write-area fixture, never from a literal, or from the pin's key, next to one: validation sees that a reference is present, not which argument the tool treats as its target.
- The cleanup guard finds the object by its `CreatedRef` alone, not by a query fixture or a literal that sits next to it. Only then does it catch a wrongly declared path before anything is deleted.
@@ -187,7 +187,7 @@ A connector that has not adopted V2 keeps `web/connector_probes.py` as its singl
A connector adopts V2 by adding an entry to `CONNECTOR_CHECKS`. To keep its L3 call, it lists `l3_read_check(app_id)` among its reads.
-`notion`, `miro` and `atlassian` have adopted V2 with a `current_user` read check that sets `require_read_only_hint`, because their tools are vendor-owned: the check is skipped unless the server advertises the tool with `readOnlyHint: true`. The check takes the place of their unprobeable `l3_probe` row. `connector_probes.py` has no such guard, so the probe route still records them as `covered_by_l2`. The other remote MCP connectors keep the derived `covered_by_l2` row until their result shapes are verified against a test account; `fireflies` returns plain text, which no payload source reads. A remote (`url`) connector's tool list is vendor-owned and changes without a release, so per-tool coverage (#3565) does not apply to it: it declares the checks it has and no exemptions.
+`notion`, `miro` and `atlassian` have adopted V2 with a `current_user` read check that sets `require_read_only_hint`, because their tools are vendor-owned: the check is skipped unless the server advertises the tool with `readOnlyHint: true`. The check takes the place of their unprobeable `l3_probe` row. `connector_probes.py` has no such guard, so the probe route still records them as `covered_by_l2`. The other remote MCP connectors keep the derived `covered_by_l2` row until their result shapes are verified against a test account; `fireflies` returns plain text, which no payload source reads. `hubspot` has adopted V2 at the Tier 1 bar (#3572) with five read checks that assert shape (keys, types and lengths) and no account-specific value, since its test account has no seeded objects yet. A remote (`url`) connector's tool list is vendor-owned and changes without a release, so per-tool coverage (#3565) does not apply to it: it declares the checks it has and no exemptions.
The six document connectors (`google-docs`, `google-sheets`, `google-slides`, `powerpoint`, `word` and `excel`) declare a `read_fixture` read check that reads the object named by its fixture key: `google-docs.document_id`, `google-sheets.spreadsheet_id`, `google-slides.presentation_id`, `powerpoint.file_path`, `word.file_path` and `excel.file_path`. Each object is the seed's document or file in the identity's account (the Google and Microsoft 365 paragraphs below list what the checks assert of it), and a `file_path` is relative to that account's drive root, not a local task-workspace path. Without a value the check reports `skip` with `fixture_missing`, and the probe route, which does not take fixtures yet (#3284), still records these connectors as `needs_fixture`. `word`'s reads also need a tool output cap (`XAGENT_TOOL_MAX_OUTPUT_LENGTH`) of at least 16,384 characters.
diff --git a/src/xagent/web/connector_checks/registry.py b/src/xagent/web/connector_checks/registry.py
index e9ecc0b64..d846833ba 100644
--- a/src/xagent/web/connector_checks/registry.py
+++ b/src/xagent/web/connector_checks/registry.py
@@ -1124,6 +1124,80 @@ CONNECTOR_CHECKS: Mapping[str, ConnectorChecks] = {
),
),
),
+ # The test account has no seeded objects yet (#3572): a check asserts shape,
+ # and content only that every portal has (CONTACT and its email property).
+ "hubspot": ConnectorChecks(
+ "hubspot",
+ reads=(
+ ReadCheck(
+ "hubspot.search_contacts",
+ "search_crm_objects",
+ {"objectType": "CONTACT", "limit": 1, "properties": ["email"]},
+ payload="json_text",
+ assertions=(
+ HasKeys((), ("results", "total", "offset")),
+ EachHasKeys(("results",), ("id", "properties")),
+ Length(("results",), max=1),
+ ),
+ require_read_only_hint=True,
+ ),
+ ReadCheck(
+ "hubspot.query_crm_data",
+ "query_crm_data",
+ {"sql": "SELECT hs_object_id FROM contacts LIMIT 1"},
+ payload="json_text",
+ assertions=(
+ HasKeys((), ("results",)),
+ IsType(("results",), "array"),
+ ),
+ require_read_only_hint=True,
+ ),
+ ReadCheck(
+ "hubspot.object_types",
+ "discover_hubspot_schema",
+ {
+ "query": {
+ "_queryType": "GET_OBJECT_TYPES",
+ "typeNameFilter": ["CONTACT"],
+ "limit": 1,
+ }
+ },
+ payload="json_text",
+ assertions=(
+ HasKeys((), ("objectTypeResults",)),
+ HasKeys(("objectTypeResults",), ("results", "hasMore")),
+ EachHasKeys(
+ ("objectTypeResults", "results"),
+ ("typeName", "objectTypeId", "readAccess", "writeAccess"),
+ ),
+ IsType(("objectTypeResults", "hasMore"), "boolean"),
+ Length(("objectTypeResults", "results"), min=1),
+ Equals(("objectTypeResults", "results", 0, "typeName"), "CONTACT"),
+ ),
+ require_read_only_hint=True,
+ ),
+ ReadCheck(
+ "hubspot.search_properties",
+ "search_properties",
+ {"objectType": "CONTACT", "keywords": ["email"]},
+ payload="json_text",
+ assertions=(
+ HasKeys((), ("results",)),
+ EachHasKeys(("results",), ("name", "label")),
+ Length(("results",), min=1),
+ ),
+ require_read_only_hint=True,
+ ),
+ ReadCheck(
+ "hubspot.search_segments",
+ "query_segments",
+ {"operation": {"type": "SearchSegmentsQuery", "limit": 1}},
+ payload="json_text",
+ assertions=(HasKeys(("result",), ("segments",)),),
+ require_read_only_hint=True,
+ ),
+ ),
+ ),
"jira": _jira_checks(),
# Create-only (#3655): its one tool files a ticket no tool of the server
# can delete, and its L3 slot stays unprobeable for the same reason.
@@ -1582,7 +1656,6 @@ LEGACY_CONNECTORS: frozenset[str] = frozenset(
"google-maps",
"google-search-console",
"granola",
- "hubspot",
"instagram",
"intercom",
"linear",
diff --git a/tests/web/connector_checks/test_registry.py b/tests/web/connector_checks/test_registry.py
index 48b39b40e..0ac4fcb32 100644
--- a/tests/web/connector_checks/test_registry.py
+++ b/tests/web/connector_checks/test_registry.py
@@ -114,7 +114,7 @@ def test_the_l3_expect_keys_carry_over() -> None:
@pytest.mark.parametrize(
("app_id", "reason"),
[
- ("hubspot", "covered_by_l2"),
+ ("granola", "covered_by_l2"),
("google-maps", "sandboxed"),
("chrome-devtools", "execution_scoped"),
],
@@ -454,6 +454,66 @@ _TIER_2_READS: Mapping[str, tuple[str, dict[str, Any], tuple[Assertion, ...]]] =
}
+# HubSpot's Tier 1 read checks (#3572): the tool, its arguments, and every
+# assertion, so dropping an item key or a whole assertion fails here.
+_HUBSPOT_READS: Mapping[str, tuple[str, dict[str, Any], tuple[Assertion, ...]]] = {
+ "hubspot.search_contacts": (
+ "search_crm_objects",
+ {"objectType": "CONTACT", "limit": 1, "properties": ["email"]},
+ (
+ HasKeys((), ("results", "total", "offset")),
+ EachHasKeys(("results",), ("id", "properties")),
+ Length(("results",), max=1),
+ ),
+ ),
+ # The empty result's shape was not observed, so only ``results`` is
+ # asserted: an absent ``instructions`` key would fail a passing check.
+ "hubspot.query_crm_data": (
+ "query_crm_data",
+ {"sql": "SELECT hs_object_id FROM contacts LIMIT 1"},
+ (
+ HasKeys((), ("results",)),
+ IsType(("results",), "array"),
+ ),
+ ),
+ "hubspot.object_types": (
+ "discover_hubspot_schema",
+ {
+ "query": {
+ "_queryType": "GET_OBJECT_TYPES",
+ "typeNameFilter": ["CONTACT"],
+ "limit": 1,
+ }
+ },
+ (
+ HasKeys((), ("objectTypeResults",)),
+ HasKeys(("objectTypeResults",), ("results", "hasMore")),
+ EachHasKeys(
+ ("objectTypeResults", "results"),
+ ("typeName", "objectTypeId", "readAccess", "writeAccess"),
+ ),
+ IsType(("objectTypeResults", "hasMore"), "boolean"),
+ Length(("objectTypeResults", "results"), min=1),
+ Equals(("objectTypeResults", "results", 0, "typeName"), "CONTACT"),
+ ),
+ ),
+ "hubspot.search_properties": (
+ "search_properties",
+ {"objectType": "CONTACT", "keywords": ["email"]},
+ (
+ HasKeys((), ("results",)),
+ EachHasKeys(("results",), ("name", "label")),
+ Length(("results",), min=1),
+ ),
+ ),
+ "hubspot.search_segments": (
+ "query_segments",
+ {"operation": {"type": "SearchSegmentsQuery", "limit": 1}},
+ (HasKeys(("result",), ("segments",)),),
+ ),
+}
+
+
def _read_check(check_id: str) -> ReadCheck:
app_id = check_id.split(".", 1)[0]
check = next(c for c in CONNECTOR_CHECKS[app_id].reads if c.id == check_id)
@@ -496,6 +556,149 @@ def test_a_tier_2_read_check_is_declared_exactly(
assert check.assertions == assertions
[email protected](("check_id", "expected"), sorted(_HUBSPOT_READS.items()))
+def test_a_hubspot_read_check_is_declared_exactly(
+ check_id: str, expected: tuple[str, dict[str, Any], tuple[Assertion, ...]]
+) -> None:
+ tool, arguments, assertions = expected
+ check = _read_check(check_id)
+ assert (check.tool, dict(check.arguments)) == (tool, arguments)
+ assert check.assertions == assertions
+
+
+# Result shapes each check accepts, with synthetic values: an empty list too,
+# except where every portal has a match.
+_HUBSPOT_OUTPUTS: Mapping[str, tuple[dict[str, Any], ...]] = {
+ "hubspot.search_contacts": (
+ {
+ "results": [
+ {
+ "id": 1,
+ "properties": {"hs_object_id": "1", "email": "[email protected]"},
+ "displayName": "A",
+ }
+ ],
+ "total": 3,
+ "urlTemplate": "https://app.hubspot.com/contacts/0/record/0-1/{id}",
+ "offset": 1,
+ },
+ # An empty result omits urlTemplate, which no check asserts.
+ {"results": [], "total": 0, "offset": 0},
+ ),
+ "hubspot.query_crm_data": (
+ {
+ "results": [
+ {
+ "content": (
+ '{"objectTypeId":"0-1","properties":{"hs_object_id":"1"}}'
+ )
+ }
+ ],
+ "instructions": "Showing 1 of 3 total objects.",
+ },
+ {"results": []},
+ ),
+ "hubspot.object_types": (
+ {
+ "objectTypeResults": {
+ "results": [
+ {
+ "typeName": "CONTACT",
+ "objectTypeId": "0-1",
+ "typeCategory": "HUBSPOT_DEFINED_OBJECTS",
+ "readAccess": "AVAILABLE",
+ "writeAccess": "AVAILABLE",
+ "activationStatus": "PERMANENTLY_ACTIVATED",
+ }
+ ],
+ "hasMore": False,
+ }
+ },
+ ),
+ "hubspot.search_properties": (
+ {
+ "results": [
+ {
+ "name": "email",
+ "label": "Email",
+ "description": "A contact's email address",
+ "matchScore": 0.87,
+ }
+ ]
+ },
+ ),
+ "hubspot.search_segments": (
+ {
+ "result": {
+ "segments": {
+ "segments": [
+ {
+ "listId": 9,
+ "objectTypeId": "0-1",
+ "listType": "STATIC",
+ "name": "Example",
+ }
+ ],
+ "totalCount": 1,
+ "hasMore": False,
+ },
+ "__typename": "x",
+ "type": "SegmentSearchResult",
+ }
+ },
+ {"result": {"segments": {}}},
+ ),
+}
+
+
+ ("check_id", "payload"),
+ [
+ (check_id, payload)
+ for check_id, payloads in sorted(_HUBSPOT_OUTPUTS.items())
+ for payload in payloads
+ ],
+)
+def test_a_hubspot_read_output_satisfies_its_declaration(
+ check_id: str, payload: dict[str, Any]
+) -> None:
+ _assert_check_output(check_id, json.dumps(payload))
+
+
+ ("check_id", "payload", "reason"),
+ [
+ (
+ "hubspot.object_types",
+ {"objectTypeResults": {"results": [], "hasMore": False}},
+ "length",
+ ),
+ (
+ "hubspot.object_types",
+ {
+ "objectTypeResults": {
+ "results": [
+ {
+ "typeName": "COMPANY",
+ "objectTypeId": "0-2",
+ "readAccess": "AVAILABLE",
+ "writeAccess": "AVAILABLE",
+ }
+ ],
+ "hasMore": False,
+ }
+ },
+ "equals",
+ ),
+ ("hubspot.search_properties", {"results": []}, "length"),
+ ],
+)
+def test_a_hubspot_check_fails_without_what_every_portal_has(
+ check_id: str, payload: dict[str, Any], reason: str
+) -> None:
+ _assert_check_fails(check_id, json.dumps(payload), reason)
+
+
@pytest.mark.parametrize(
"app_id", ["google-ads", "outlook", "planner", "sharepoint", "zendesk", "zoom"]
)
@@ -511,7 +714,7 @@ def test_a_tier_2_l3_check_is_the_probe_derived_one(app_id: str) -> None:
assert l3_checks == [expected]
[email protected]("app_id", ["acme", "hubspot"])
[email protected]("app_id", ["acme", "granola"])
def test_the_declared_l3_helper_rejects_anything_but_a_read_check(app_id: str) -> None:
with pytest.raises(DeclarationError, match=app_id):
_declared_l3(app_id)
diff --git a/tests/web/connector_checks/test_tool_coverage.py b/tests/web/connector_checks/test_tool_coverage.py
index 64ff45bb7..4632a6ff2 100644
--- a/tests/web/connector_checks/test_tool_coverage.py
+++ b/tests/web/connector_checks/test_tool_coverage.py
@@ -43,7 +43,6 @@ _PINNED_LEGACY = frozenset(
"google-maps",
"google-search-console",
"granola",
- "hubspot",
"instagram",
"intercom",
"linear",