The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(tools): isolate catalog ownership (#17165)
Dottaf7210cd31f03
diff --git a/doc/plans/2026-10-21-tool-access-boundaries.md b/doc/plans/2026-10-21-tool-access-boundaries.mdindex f4506339ca..d1090a78b1 100644--- a/doc/plans/2026-10-21-tool-access-boundaries.md+++ b/doc/plans/2026-10-21-tool-access-boundaries.md@@ -202,3 +202,9 @@ boundaries pass. No new supervisor, background loop or runtime cache is created.- Separate state validation/consumption, authorization initiation, direct OAuth callback, managed-cloud callback, Vercel callback, catalog completion and explicit identity promotion. Shared setup policy, personal identity checks and organization-grant persistence have explicit owners.- `tool-access.ts`: 12,019 → 8,997 lines. The largest new module is 700 lines. Callback-specific state consumption order, membership locks, transaction-scoped secret writes and promotion cleanup stay intact.- Before/after: all 470 public service and real HTTP/database integration cases pass. Server typecheck, module-boundary and whitespace checks pass; all 182 previous function/method bodies match by normalized syntax comparison.++### Catalog ownership++- Separate guarded remote HTTP, descriptor risk policy, discovery, publication, default access, cached reads, health, attention and aggregator account synchronization.+- `tool-access.ts`: 8,997 → 6,418 lines. New owners range from 39 to 492 lines. Remote options remain live; catalog flights and aggregator sync jobs retain per-service lifetime. Publication locks and transaction-scoped default access remain unchanged.+- Before/after: 470 public service/HTTP integration cases plus 44 catalog-publication, HTTP rebinding and aggregator-sync cases pass. Server typecheck and module-boundary checks pass. All 157 previous function/method bodies match by normalized syntax comparison.diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.tsindex 5fc47d6a84..5f3f5e4f51 100644--- a/server/src/services/tool-access.ts+++ b/server/src/services/tool-access.ts@@ -1,3 +1,16 @@+import { createToolRemoteHttp } from "./tool-access/remote-http.js";+import { classifyRisk, normalizeToolDescriptor, type McpToolDescriptor } from "./tool-access/catalog/descriptors.js";+export { classifyRisk, isGmailToolPermanentlyBlocked, isGoogleWorkspaceToolAllowed, type McpToolDescriptor } from "./tool-access/catalog/descriptors.js";+import { toCatalogEntry } from "./tool-access/catalog/projection.js";+import { toConnection, toConnectionGrant, toConnectionInstall } from "./tool-access/connections/projection.js";+import { isOAuthEndpointRejection } from "./tool-access/catalog/failures.js";+import { createCatalogDefaultAccess, toPolicy } from "./tool-access/catalog/default-access.js";+import { createToolCatalogDiscovery } from "./tool-access/catalog/discovery.js";+import { createConnectionHealth } from "./tool-access/catalog/health.js";+import { createToolAppAttention } from "./tool-access/catalog/attention.js";+import { createToolCatalogPublication } from "./tool-access/catalog/publication.js";+import { createToolCatalogReader } from "./tool-access/catalog/reader.js";+import { createAggregatorAccounts } from "./tool-access/catalog/aggregator-accounts.js";import { actorBinding } from "./tool-access/actor.js";import { toApplication } from "./tool-access/applications/projection.js";import { groupedActions } from "./tool-access/catalog/action-summary.js";@@ -11,10 +24,9 @@ import { createDirectOAuthCallback } from "./tool-access/oauth/direct-callback.jimport { createCloudOAuthCallback } from "./tool-access/oauth/cloud-callback.js";import { createVercelOAuthCallback } from "./tool-access/oauth/vercel-callback.js";import { createOAuthAccessFinalization } from "./tool-access/oauth/access-finalization.js";-import { createVercelCredentialService, vercelConnectHttpError, vercelCredentialFor, vercelConnectResourcesFor, redactedVercelGrant } from "./tool-access/vercel-credentials.js";+import { createVercelCredentialService, vercelConnectHttpError, vercelConnectResourcesFor } from "./tool-access/vercel-credentials.js";import { createConnectionCredentialAccess } from "./tool-access/credential-access.js";-import { singleFlight } from "./tool-access/single-flight.js";-import { connectionSetupUrl, connectionReconnectUrl } from "./tool-access/identifiers.js";+import { createOAuthCredentialStore, actorForSecret } from "./tool-access/oauth/credential-store.js";import { managedConnectorProfile } from "./tool-access/oauth/client-settings.js";import { createOAuthClientRegistration } from "./tool-access/oauth/client-registration.js";@@ -28,13 +40,13 @@ import { createToolActivityService } from "./tool-access/activity/service.js";import { createToolActionReviewQueue } from "./tool-access/activity/review-queue.js";import { createToolAccessAudit } from "./tool-access/audit.js";import { createConnectionTokenBroker } from "./tool-access/token-broker/service.js";-import { assertConfiguredTokenBrokerEndpointsAllowed } from "./tool-access/token-broker/endpoint-policy.js";+import { createToolProfileService } from "./tool-access/profiles/service.js";import { createToolProfileResolver } from "./tool-access/profiles/resolution.js";-import { toProfile, toProfileEntry, toProfileBinding, pendingNewToolsForProfile, profileCatalogColumns } from "./tool-access/profiles/projection.js";+import { toProfile, toProfileEntry, toProfileBinding } from "./tool-access/profiles/projection.js";import { lockManagedAppPolicies, lockToolAssignmentAgents, lockToolConnectionRuntimeInputs } from "./tool-access/assignment-locks.js";export { lockToolConnectionRuntimeInputs } from "./tool-access/assignment-locks.js";-import type { ActorInfo, ToolAccessMutationDb } from "./tool-access/types.js";+import type { ActorInfo } from "./tool-access/types.js";import { normalizeKey, connectionUid } from "./tool-access/identifiers.js";import { createOAuthEndpointPolicy } from "./tool-access/oauth/endpoint-policy.js";import { createOAuthDiscovery } from "./tool-access/oauth/discovery.js";@@ -44,15 +56,7 @@ import { oauthConfig } from "./tool-access/oauth/metadata.js";import type { OAuthClientRegistrationSource } from "./tool-access/oauth/types.js";export { OAUTH_CLIENT_ID_METADATA_DOCUMENT_PATH, resolveOAuthClientIdMetadataDocumentUrl, oauthClientIdMetadataDocument } from "./tool-access/oauth/client-metadata.js";-import {- remoteEndpoint,- connectionMethodFor,- connectionMethodForConnection,- credentialFieldsFor,- normalizeConnectionMethodConfig,- projectedConnectionHeaders,- projectedConnectionToolInputSchema,-} from "./tool-access/connection-methods.js";+import { connectionMethodFor, connectionMethodForConnection, credentialFieldsFor, normalizeConnectionMethodConfig, projectedConnectionToolInputSchema } from "./tool-access/connection-methods.js";export {normalizeConnectionMethodConfig,projectedConnectionHeaders,@@ -64,30 +68,21 @@ export {import { loadGitHubTokenRepositories } from "./tool-access/github-access-discovery.js";export { loadGitHubGrantMetadata, loadGitHubTokenRepositories } from "./tool-access/github-access-discovery.js";-import { stableHash, descriptorHash } from "./tool-catalog-hash.js";+import { stableHash } from "./tool-catalog-hash.js";-import { DAYTONA_TOOLS, daytonaClient, daytonaRisk, isDaytonaConnection, isDaytonaToolName } from "./daytona.js";+import { isDaytonaConnection } from "./daytona.js";import { connectorToolPolicy } from "./connectors/tool-policy.js";import { chatCredentialMutationLease } from "./chat-credential-mutation-lease.js";import { notifyChatEndpointWork } from "./chat-work-notifications.js";import { removeSlackRegistration } from "./chat-slack-registration-cleanup.js";import { chatEndpoints } from "@paperclipai/db";-import { AGGREGATOR_NAMES, isAppAggregator, type AggregatorAppsResponse, type ArcadeDiscoverySetupInput } from "@paperclipai/shared/aggregator-apps";-import { resolveAggregatorApp, type AppCatalogAggregator } from "@paperclipai/shared/aggregator-app-catalog";-import { AggregatorDiscoveryUnavailableError, discoverArcadeApps, discoverExecutorApps, type DiscoveredApp } from "./aggregator-app-discovery.js";-import { COMPOSIO_APP_TOOLKITS, findComposioCatalogApp } from "@paperclipai/shared/aggregator-app-catalog";-import type { ComposioAppAccount, ComposioAppAccountInput, ComposioAppSetupInput, ComposioAppSetupResult, ComposioAppSnapshot, ComposioAppsResponse } from "@paperclipai/shared";-import { composioAppAccounts, composioAppSetupResult } from "./composio-app-setup.js";+import { defaultConnectionAgentInstructions } from "@paperclipai/shared";-import { GOOGLE_WORKSPACE_SERVICES, googleWorkspaceServiceGranted, googleWorkspaceToolName, googleWorkspaceToolTarget, googleWorkspaceGrantedScopes, isGoogleWorkspaceToolGranted, type GoogleWorkspaceService } from "@paperclipai/shared";-import { isInsufficientConnectionScope, INSUFFICIENT_CONNECTION_SCOPE_MESSAGE } from "./connection-permission-errors.js";-import { mcpDiscoveryHttpFailure, retainMcpConnectionFailure, withMcpConnectionFailure } from "./mcp-connection-failure.js";-import { BROWSER_USE_TOOLS } from "@paperclipai/shared";-import { browserUseClient, isBrowserUseConnection } from "./browser-use-client.js";+import { isBrowserUseConnection } from "./browser-use-client.js";import { browserUseService } from "./browser-use.js";-import { COGNEE_STDIO_TEMPLATE, cogneeCloudUrl } from "./cognee-connection.js";-import { isMemoryConnectorId, isRemoteMcpConnectorMethod, connectionPurposeTransportSchema } from "@paperclipai/shared";+import { COGNEE_STDIO_TEMPLATE } from "./cognee-connection.js";+import { isRemoteMcpConnectorMethod } from "@paperclipai/shared";import { githubBotRequest } from "./chat-github-client.js";import { syncConnectionCredentialBindings } from "./connection-credential-bindings.js";@@ -99,13 +94,13 @@ import { canBrowseProjectRepositoryGrant, mergeProjectRepository } from "./projeimport { measureRunPhase } from "./execution-preparation-timing.js";-import { createHash, randomUUID } from "node:crypto";+import { randomUUID } from "node:crypto";import { readFileSync } from "node:fs";-import { and, asc, desc, eq, inArray, isNotNull, isNull, lt, lte, max, ne, notInArray, or, sql } from "drizzle-orm";+import { and, asc, desc, eq, inArray, isNull, max, ne, notInArray, sql } from "drizzle-orm";import type { Db } from "@paperclipai/db";-import { agents, connectionGrantMembers, connectionGrantDelegations, connectionGrants, connectionTokenIssuances, authUsers, companyMemberships, companySecretBindings, companySecrets, principalPermissionGrants, userSecretDefinitions, userSecretDeclarations, plugins, toolAccessAuditEvents, toolApplications, toolActionRequests, toolCatalogEntries, toolConnectionInstalls, toolConnectionAppSnapshots, toolConnectionAppSyncs, toolConnections, toolOauthStates, toolStdioCommandTemplates, toolCallEvents, toolInvocations, toolPolicies, toolGatewaySessions, toolMcpGateways, toolMcpGatewayTokens, toolProfileBindings, toolProfileEntries, toolProfiles, toolRuntimeSlots } from "@paperclipai/db";-import type { AppDefinition, ConnectionGrantKind, CreateToolApplication, CreateToolConnection, ConnectToolApp, ConnectToolAppResult, CreateToolStdioCommandTemplate, FinishToolApp, FinishToolAppResult, CreateToolProfileBindingForProfile, CreateToolProfileEntryForProfile, DeploymentExposure, DeploymentMode, ImportMcpJson, McpConnectionCredentialRef, McpJsonImportPreview, ToolApplication, ToolCatalogEntry, ToolCatalogRefreshResult, ToolConnection, ToolConnectionInstall, ToolConnectionInstallSnapshot, ToolConnectionRemovalResult, ToolConnectionRemovalSummary, ToolConnectionHealthCheckResult, ToolConnectionHealthStatus, ToolConnectionAuthKind, ToolConnectionCredentialPolicy, ToolConnectionCredentialSource, ToolConnectionTransport, ToolAppsAttentionResponse, ToolAppMetadataPreflightResult, ToolExampleInstallResult, ToolExampleSmokeCheck, ToolExampleSmokeResult, ToolExampleSummary, ToolProfileBinding, ToolProfileEntry, ToolPolicyDecision, ToolPolicy, ToolRiskLevel, ToolStdioCommandTemplate, UpdateToolApplication, UpdateToolConnection, PutToolConnectionInstalls, VercelConnectCredentialReference } from "@paperclipai/shared";-import { CLASS3_STATIC_LEASE_ALLOWLIST, GOOGLE_WORKSPACE_CONNECTOR_PROFILES, getAppDefinitionForUrl, getAvailableConnectionMethods, getConnectableAppDefinition, isGoogleWorkspaceConnectorProfileId, isToolConnectionAttentionHealth, recommendedDefaultsForApp, type GoogleWorkspaceConnectorProfileId } from "@paperclipai/shared";+import { agents, connectionGrantMembers, connectionGrantDelegations, connectionGrants, connectionTokenIssuances, authUsers, companyMemberships, companySecretBindings, companySecrets, principalPermissionGrants, userSecretDefinitions, userSecretDeclarations, plugins, toolAccessAuditEvents, toolApplications, toolCatalogEntries, toolConnectionInstalls, toolConnections, toolOauthStates, toolStdioCommandTemplates, toolCallEvents, toolPolicies, toolGatewaySessions, toolMcpGateways, toolMcpGatewayTokens, toolProfileBindings, toolProfileEntries, toolProfiles, toolRuntimeSlots } from "@paperclipai/db";+import type { AppDefinition, ConnectionGrantKind, CreateToolApplication, CreateToolConnection, ConnectToolApp, ConnectToolAppResult, CreateToolStdioCommandTemplate, FinishToolApp, FinishToolAppResult, CreateToolProfileBindingForProfile, CreateToolProfileEntryForProfile, DeploymentExposure, DeploymentMode, ImportMcpJson, McpConnectionCredentialRef, McpJsonImportPreview, ToolApplication, ToolCatalogEntry, ToolCatalogRefreshResult, ToolConnection, ToolConnectionInstall, ToolConnectionInstallSnapshot, ToolConnectionRemovalResult, ToolConnectionRemovalSummary, ToolConnectionHealthCheckResult, ToolConnectionAuthKind, ToolConnectionCredentialPolicy, ToolConnectionCredentialSource, ToolConnectionTransport, ToolAppMetadataPreflightResult, ToolExampleInstallResult, ToolExampleSmokeCheck, ToolExampleSmokeResult, ToolExampleSummary, ToolProfileBinding, ToolProfileEntry, ToolPolicyDecision, ToolStdioCommandTemplate, UpdateToolApplication, UpdateToolConnection, PutToolConnectionInstalls, VercelConnectCredentialReference } from "@paperclipai/shared";+import { CLASS3_STATIC_LEASE_ALLOWLIST, getAppDefinitionForUrl, getAvailableConnectionMethods, getConnectableAppDefinition, recommendedDefaultsForApp } from "@paperclipai/shared";import {checkMcpRemoteHeaderName,checkMcpRemoteHeaderValue,@@ -123,17 +118,10 @@ import {import { isUniqueViolation } from "../db-errors.js";import { logger } from "../middleware/logger.js";import { logActivity } from "./activity-log.js";-import { isSlackMcpAccessDisabledResponse } from "./slack-mcp-error.js";-import { initializeMcpHttpSession, McpHttpInitializationError, getMcpHttpSession, forgetMcpHttpSessions, readMcpHttpResponse, mcpHttpRequestHeaders } from "./mcp-http.js";-import {- assertPublicRemoteHttpEndpoint,- parseRemoteHttpEndpoint,- type RemoteHttpEndpointLookup,-} from "./remote-http-endpoint-guard.js";-import {- guardedRemoteHttpFetch,- type GuardedRemoteHttpFetchOptions,-} from "./remote-http-fetch.js";++import { forgetMcpHttpSessions } from "./mcp-http.js";+import { type RemoteHttpEndpointLookup } from "./remote-http-endpoint-guard.js";+import {REMOTE_URL_SECRET_CONFIG_PATH,remoteUrlCredentialMatchesPublicUrl,@@ -141,10 +129,10 @@ import {} from "./remote-url-credentials.js";import { secretService } from "./secrets.js";import { connectionCredentialConfigPath as credentialRefConfigPath, connectionSecretsUsedByOtherConsumers, validateConnectionGrantSecretOwnership, writeConnectionCredential } from "./connection-credentials.js";-import { agentmailApi } from "./agentmail-api.js";+import type { ConfigureRailwaySsh, RailwaySshSetup } from "@paperclipai/shared";import { generateRailwaySshKey, RAILWAY_SSH_SECRET_PATH, validateRailwayKnownHosts } from "./railway-ssh.js";-import { createRailwayClient, discoverRailwayWorkspace, isRailwayConnection, isRailwayEndpoint, isRailwayToolBlocked, normalizeRailwayToolName, RAILWAY_TOOLS, RAILWAY_TOOL_PREFIX, RailwayError } from "./railway.js";+import { isRailwayConnection } from "./railway.js";import { toolAccessPolicyService } from "./tool-access-policy.js";import { googleChatToolDescription } from "./google-chat-tool-policy.js";@@ -152,11 +140,8 @@ import { gmailArchiveToolDescription } from "./gmail-archive-tool-policy.js";import { createToolRuntimeSupervisor } from "./tool-runtime-supervisor.js";-import { isRetiredComposioConnection, RETIRED_COMPOSIO_MESSAGE } from "@paperclipai/shared";import { appWithPaperclipCloudConnectorAvailability, paperclipCloudConnectorCapabilitiesFromEnv, createPaperclipCloudConnector, isPaperclipCloudConnectorStrategy, paperclipCloudConnectorConfigFromEnv, type PaperclipCloudConnector } from "./paperclip-cloud-connector.js";import { createVercelConnectClient, deriveVercelConnectSubject, vercelConnectIntegrationStatus, vercelTokenRequest, type VercelConnectClient } from "./vercel-connect.js";-const REMOTE_HTTP_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);-const MAX_REMOTE_HTTP_REDIRECTS = 5;type ToolAccessServiceOptions = {deploymentMode?: DeploymentMode;@@ -181,14 +166,6 @@ type ToolAccessServiceOptions = {vercelConnectClient?: VercelConnectClient | null;};-export type McpToolDescriptor = {- name: string;- title?: string | null;- description?: string | null;- inputSchema?: Record<string, unknown>;- annotations?: Record<string, unknown>;-};-const GOOGLE_SHEETS_SPREADSHEET_SCHEMA = {type: "object",properties: {@@ -677,102 +654,6 @@ function assertClass3ToolCredentialRefAllowed(ref: {}}-function toConnection(row: typeof toolConnections.$inferSelect): ToolConnection {- connectionPurposeTransportSchema.parse(row);- const retired = isRetiredComposioConnection(row);- return {- id: row.id,- companyId: row.companyId,- applicationId: row.applicationId,- name: row.name,- uid: row.uid,- connectionKind: row.connectionKind,- connectionPurpose: row.connectionPurpose,- ownership: row.ownership,- transport: row.transport,- authKind: row.authKind,- credentialSource: row.credentialSource,- externalCredential: row.externalCredential ?? null,- credentialPolicy: row.credentialPolicy,- status: row.status,- enabled: row.enabled && !retired,- agentInstructions: row.agentInstructions ?? null,- config: row.config ?? {},- transportConfig: row.transportConfig ?? {},- credentialRefs: row.credentialRefs ?? [],- credentialSecretRefs: row.credentialSecretRefs ?? [],- healthStatus: retired ? "error" : row.healthStatus,- healthMessage: retired ? RETIRED_COMPOSIO_MESSAGE : row.healthMessage,- healthCheckedAt: row.healthCheckedAt,- lastHealthAt: row.lastHealthAt,- lastCatalogRefreshAt: row.lastCatalogRefreshAt,- lastError: row.lastError,- createdByAgentId: row.createdByAgentId,- createdByUserId: row.createdByUserId,- createdAt: row.createdAt,- updatedAt: row.updatedAt,- };-}--function toConnectionGrant(row: typeof connectionGrants.$inferSelect) {- return {- ...row,- externalCredential: redactedVercelGrant(row.externalCredential),- };-}--function toConnectionInstall(- row: typeof toolConnectionInstalls.$inferSelect,-): ToolConnectionInstall {- return {- id: row.id,- companyId: row.companyId,- connectionId: row.connectionId,- targetType: row.targetType,- targetId: row.targetId,- createdByAgentId: row.createdByAgentId,- createdByUserId: row.createdByUserId,- createdAt: row.createdAt,- };-}--function toCatalogEntry(- row: typeof toolCatalogEntries.$inferSelect,-): ToolCatalogEntry {- return {- id: row.id,- companyId: row.companyId,- applicationId: row.applicationId,- connectionId: row.connectionId,- entryKind: row.entryKind,- name: row.name,- toolName: row.toolName,- title: row.title,- description: row.description,- inputSchema: row.inputSchema ?? {},- outputSchema: row.outputSchema ?? null,- annotations: row.annotations ?? {},- riskLevel: row.riskLevel,- isReadOnly: row.isReadOnly,- isWrite: row.isWrite,- isDestructive: row.isDestructive,- status: row.status,- addedAt: row.firstSeenAt,- version: row.version,- versionHash: row.versionHash,- schemaHash: row.schemaHash,- firstSeenAt: row.firstSeenAt,- lastSeenAt: row.lastSeenAt,- reviewedAt: row.reviewedAt,- reviewedByAgentId: row.reviewedByAgentId,- reviewedByUserId: row.reviewedByUserId,- quarantinedAt: row.quarantinedAt,- quarantineReason: row.quarantineReason,- createdAt: row.createdAt,- updatedAt: row.updatedAt,- };-}-function toCatalogEntryForConnection(row: typeof toolCatalogEntries.$inferSelect,connection: typeof toolConnections.$inferSelect,@@ -864,25 +745,6 @@ function toStdioCommandTemplate(};}-function toPolicy(row: typeof toolPolicies.$inferSelect): ToolPolicy {- return {- id: row.id,- companyId: row.companyId,- name: row.name,- description: row.description,- policyType: row.policyType,- priority: row.priority,- enabled: row.enabled,- selectors: row.selectors ?? {},- conditions: row.conditions ?? null,- config: row.config ?? null,- createdByAgentId: row.createdByAgentId,- createdByUserId: row.createdByUserId,- createdAt: row.createdAt,- updatedAt: row.updatedAt,- };-}-function connectionSetupMutationFingerprint(row: typeof toolConnections.$inferSelect,): string {@@ -902,419 +764,6 @@ function connectionSetupMutationFingerprint(});}-function normalizeToolDescriptor(tool: unknown): McpToolDescriptor | null {- const record = asRecord(tool);- if (typeof record.name !== "string" || record.name.trim().length === 0)- return null;- return {- name: record.name.trim(),- title: typeof record.title === "string" ? record.title : null,- description:- typeof record.description === "string" ? record.description : null,- inputSchema: asRecord(record.inputSchema ?? record.input_schema),- annotations: asRecord(record.annotations),- };-}--// Match a verb anywhere it forms a name segment, not just at the leading edge.-// Real MCP servers namespace and style tool names many ways:-// "github:create_issue", "notion:update_page", "slack:postMessage", "set_value".-// A leading-anchor regex (/^(create|...)/) misses every namespaced/camelCase-// form and silently classifies writes as read-only. We normalise camelCase to-// snake_case first so "postMessage" -> "post_message", then match the verb when-// it is delimiter- or word-bounded. This mirrors the gateway classifier in-// tool-gateway.ts (inferToolRisk) so the two stay consistent.-function verbMatches(toolName: string, verbs: string): boolean {- const normalized = toolName- .replace(/([a-z0-9])([A-Z])/g, "$1_$2")- .toLowerCase();- return new RegExp(`\\b(${verbs})\\b|(^|[:._-])(${verbs})([:._-]|$)`).test(- normalized,- );-}--const NOTION_READ_TOOLS = new Set([- "notion-fetch",- "notion-get-async-task",- "notion-get-comments",- "notion-get-teams",- "notion-get-users",- "notion-query-data-sources",- "notion-query-database-view",- "notion-query-meeting-notes",- "notion-search",-]);--const NOTION_WRITE_TOOLS = new Set([- "notion-convert-page-to-skill",- "notion-create-comment",- "notion-create-database",- "notion-create-folder",- "notion-create-pages",- "notion-create-view",- "notion-duplicate-page",- "notion-move-pages",- "notion-update-data-source",- "notion-update-page",- "notion-update-view",-]);--const SHOPIFY_DESTRUCTIVE_TOOLS = new Set([- "cancel-cart",- "cancel-checkout",- "complete-checkout",-]);--function normalizedProviderToolName(toolName: string): string {- return toolName- .replace(/([a-z0-9])([A-Z])/g, "$1-$2")- .toLowerCase()- .replace(/[:._-]+/g, "-");-}--// Aggregators can add arbitrarily powerful tools without changing their MCP-// endpoint. Only these reviewed, exact capabilities are read-only. Unknown or-// renamed actions remain enabled by the usual access rules, but have write risk.-const AGGREGATOR_READ_TOOLS = new Map<string, ReadonlySet<string>>([- ["executor", new Set(["skills"])],- ["composio", new Set(["COMPOSIO_SEARCH_TOOLS", "COMPOSIO_GET_TOOL_SCHEMAS", "COMPOSIO_SEARCH_SKILLS", "COMPOSIO_USE_SKILL"])],- ["arcade", new Set(["Github.GetRepository"])],- ["zapier", new Set()],-]);--export function classifyRisk(- tool: McpToolDescriptor,- sourceTemplateKey?: string | null,-): ToolRiskLevel {- if (sourceTemplateKey === "google-workspace") {- const target = googleWorkspaceToolTarget(tool.name);- if (!target) return "destructive";- const risk = classifyRisk({ ...tool, name: target.upstreamName }, target.appSlug);- if (risk === "destructive" || target.leaf === "delete_event" || target.leaf === "send_message") return "destructive";- if (target.write) return "write";- return risk;- }- const annotations = tool.annotations ?? {};- if (annotations.destructiveHint === true || annotations.destructive === true)- return "destructive";- const reviewedRisk = connectorToolPolicy(sourceTemplateKey).classifyRisk?.(tool);- if (reviewedRisk) return reviewedRisk;- const normalizedToolName = normalizedProviderToolName(tool.name);- if (isMemoryConnectorId(sourceTemplateKey)) {- if (verbMatches(tool.name, "delete|remove|destroy|forget|prune|reset")) return "destructive";- // Supermemory uses one add_memory tool for both save and forget.- if (sourceTemplateKey === "supermemory" && normalizedToolName === "add-memory") return "destructive";- if (verbMatches(tool.name, "add|remember|save|record|ingest|cognify|update|create|set|upload|select|rename|share")) return "write";- if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";- const reads = ["get", "list", "search", "recall", "query", "retrieve", "who"];- return verbMatches(tool.name, reads.join("|")) ? "read" : "write";- }- const reviewedReads = AGGREGATOR_READ_TOOLS.get(sourceTemplateKey ?? "");- if (reviewedReads) {- if (verbMatches(tool.name, "delete|remove|destroy|unpublish")) return "destructive";- if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";- return reviewedReads.has(tool.name) ? "read" : "write";- }- if (sourceTemplateKey === "browser-use-cloud") {- return BROWSER_USE_TOOLS.find(t => t.name === tool.name)?.annotations.readOnlyHint ? "read" : "destructive";- }- // Fireflies sharing, moving, and access revocation are mutations even when- // a provider omits annotations or mistakenly advertises a read hint.- if (sourceTemplateKey === "fireflies" && [- "fireflies-share-meeting", "fireflies-revoke-meeting-access", "fireflies-move-meeting",- ].includes(normalizedToolName)) return "write";- // Gmail archive removes a system label. Keep it behind the archive method's- // destructive Ask-first default even when the preview provider omits hints.- if (sourceTemplateKey === "gmail" && normalizedToolName === "unlabel-thread")- return "destructive";- if (sourceTemplateKey === "posthog" && normalizedToolName === "exec")- return "destructive";- // Superagent mirrors its REST API, and several mutations use verbs the- // generic classifier reads as reads: billable triage_finding, scans,- // restore_agent_builtin_rule, and credential-revoking revoke_agent_client.- // Only list/get tools and tools the provider marks read-only are reads.- if (sourceTemplateKey === "superagent") {- if (verbMatches(tool.name, "delete|remove|destroy|revoke")) return "destructive";- if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";- return /^(list|get)-/.test(normalizedToolName) || annotations.readOnlyHint === true ? "read" : "write";- }- // Ditto's translate, link and unlink tools replace saved text or developer- // IDs, but the generic classifier reads those verbs as reads. Only- // list/get/search tools, suggest_edit, and tools Ditto marks read-only are- // reads.- if (sourceTemplateKey === "ditto") {- if (verbMatches(tool.name, "delete|remove|destroy")) return "destructive";- if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";- return /^(list|get|search)-/.test(normalizedToolName) ||- normalizedToolName === "suggest-edit" ||- annotations.readOnlyHint === true- ? "read"- : "write";- }- if (- sourceTemplateKey === "shopify" &&- SHOPIFY_DESTRUCTIVE_TOOLS.has(normalizedToolName)- )- return "destructive";- // Notion's hosted MCP catalog contains mutations whose names do not use one- // of the generic create/update/delete verbs (move, duplicate, and convert).- // Keep all reviewed tools explicit so provider changes are visible in code,- // while an annotation may still escalate a known read to a write.- if (- sourceTemplateKey === "notion" &&- NOTION_WRITE_TOOLS.has(normalizedToolName)- )- return "write";- if (annotations.readOnlyHint === false || annotations.writeHint === true)- return "write";- if (- sourceTemplateKey === "notion" &&- NOTION_READ_TOOLS.has(normalizedToolName)- )- return "read";- if (verbMatches(tool.name, "delete|remove|destroy|unpublish"))- return "destructive";- if (- verbMatches(- tool.name,- "create|update|write|set|send|publish|post|mutate|mark|archive",- )- )- return "write";- // PostHog exposes a broad and evolving catalog. Unknown tools must never be- // silently treated as reads; provider annotations can opt known reads in.- if (sourceTemplateKey === "posthog")- return annotations.readOnlyHint === true ? "read" : "write";- return "read";-}--export function isGmailToolPermanentlyBlocked(- tool: McpToolDescriptor,-): boolean {- if (googleWorkspaceToolLeafName(tool.name) === "unlabel-thread") return false;- const riskLevel = classifyRisk(tool, "gmail");- return (- verbMatches(- tool.name,- "send|trash|spam|delete|remove|destroy|execute|run",- ) ||- (normalizedProviderToolName(tool.name).includes("label") &&- riskLevel !== "read")- );-}--const GOOGLE_WORKSPACE_READ_TOOLS: Record<string, ReadonlySet<string>> = {- gmail: new Set([- "get-message",- "get-thread",- "get-draft",- "list-drafts",- "list-labels",- "search-threads",- "list-threads",- "search-messages",- ]),- "google-drive": new Set([- "download-file-content",- "get-file-metadata",- "get-file-permissions",- "list-recent-files",- "read-file-content",- "search-files",- ]),- "google-docs": new Set(["read-doc"]),- "google-sheets": new Set(["get-values", "get-spreadsheet"]),- "google-slides": new Set(["read-presentation"]),- "google-calendar": new Set([- "get-event",- "list-calendars",- "list-events",- "search-events",- "suggest-time",- ]),- "google-chat": new Set([- "search-conversations",- "list-messages",- "search-messages",- ]),- "google-people": new Set([- "search-directory-people",- "search-contacts",- "get-user-profile",- ]),- "google-workspace-search": new Set(["search-corpus"]),-};--function googleWorkspaceToolLeafName(name: string): string {- return (name.split(/[.:/]/).pop() ?? name)- .replace(/([a-z0-9])([A-Z])/g, "$1-$2")- .replace(/_/g, "-")- .toLowerCase();-}--export function isGoogleWorkspaceToolAllowed(- profileId: GoogleWorkspaceConnectorProfileId,- tool: McpToolDescriptor,-): boolean {- if (profileId === "workspace.all") return googleWorkspaceToolTarget(tool.name) !== null;- const profile = GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profileId];- const toolName = googleWorkspaceToolLeafName(tool.name);- const readTools =- GOOGLE_WORKSPACE_READ_TOOLS[profile.appSlug] ?? new Set<string>();- return (- readTools.has(toolName) ||- profile.writeTools.some(- (writeTool) => googleWorkspaceToolLeafName(writeTool) === toolName,- )- );-}--/**- * Did this error come from the OAuth endpoint gate (PAP-17099)? Such a refusal- * is Paperclip's own decision about an unsafe address, so it must keep its code- * and its 422 instead of being folded into a generic upstream failure.- */-function isOAuthEndpointRejection(error: unknown): boolean {- if (!(error instanceof HttpError)) return false;- const code = asRecord(error.details).code;- return typeof code === "string" && code.endsWith("_endpoint_rejected");-}--function healthFailureHttpStatus(failure: {- status: ToolConnectionHealthStatus;- code: string;-}): number {- if (failure.status === "missing_secret") return 422;- if (failure.code === "oauth_challenge") return 422;- if (failure.code === "oauth_refresh_missing") return 422;- if (failure.code === "oauth_reauthorization_required") return 422;- if (failure.code === "oauth_insufficient_scope") return 422;- if (failure.code === "slack_mcp_access_disabled") return 422;- if (failure.code === "user_authorization_required") return 422;- if (failure.code === "composio_broker_retired") return 422;- if (failure.code === "tool_connection_transport_unsupported") return 422;- if (failure.code === "cognee_access_unverified" || failure.code === "memory_api_key_rejected") return 422;- if (failure.code.endsWith("_endpoint_rejected")) return 422;- return 502;-}--function unsupportedToolConnectionTransport() {- return unprocessable(- "This connection has no supported tool integration. Add a supported account or MCP connection from Connectors.",- { code: "tool_connection_transport_unsupported" },- );-}--function sanitizeHttpFailure(error: unknown): {- status: ToolConnectionHealthStatus;- message: string;- code: string;-} {- if (error instanceof HttpError) {- const code = asRecord(error.details).code;- if (code === "cognee_access_unverified" || code === "memory_api_key_rejected") {- return { status: "error", message: error.message, code };- }- if (code === "grant_credential_invalid" || code === "oauth_insufficient_scope") {- return { status: code === "grant_credential_invalid" ? "missing_secret" : "degraded", message: error.message, code };- }- if (code === "slack_mcp_access_disabled" || code === "google_workspace_services_unavailable") {- return { status: "error", message: error.message, code };- }- if (code === "user_authorization_required") {- return { status: "error", message: error.message, code };- }- if (code === "tool_connection_transport_unsupported") {- return { status: "error", message: error.message, code };- }- if (code === "composio_broker_retired") {- return { status: "error", message: error.message, code };- }- if (typeof code === "string" && code.startsWith("remote_http_")) {- return { status: "error", message: error.message, code };- }- if (isOAuthEndpointRejection(error)) {- return { status: "error", message: error.message, code: String(code) };- }- if (code === "oauth_challenge") {- return {- status: "error",- message: "This app needs you to sign in.",- code: "oauth_challenge",- };- }- if (code === "oauth_refresh_missing") {- return {- status: "failed",- message: "OAuth credentials have expired and need to be reconnected.",- code: "oauth_refresh_missing",- };- }- if (code === "oauth_reauthorization_required") {- return {- status: "error",- message: "OAuth authorization expired. Reconnect this app to continue.",- code: "oauth_reauthorization_required",- };- }- if (typeof code === "string" && code.startsWith("vercel_connect_")) {- return {- status:- code === "vercel_connect_unavailable" ||- code === "vercel_connect_auth_failed" ||- code === "vercel_connect_installation_required"- ? "degraded"- : "error",- message: error.message,- code,- };- }- if (- code === "oauth_refresh_in_progress" ||- code === "oauth_refresh_superseded" ||- code === "oauth_refresh_outcome_unknown"- ) {- return {- status: "error",- message: error.message,- code,- };- }- if (- code === "binding_missing" ||- code === "secret_deleted" ||- code === "secret_inactive" ||- code === "version_missing"- ) {- return {- status: "missing_secret",- message: "A configured credential secret could not be resolved.",- code: String(code),- };- }- if (error.status === 404 && /secret/i.test(error.message)) {- return {- status: "missing_secret",- message: "A configured credential secret could not be resolved.",- code: "secret_missing",- };- }- return { status: "error", message: error.message, code: "paperclip_error" };- }- if (error instanceof Error) {- return {- status: "error",- message: error.message.slice(0, 240),- code: "runtime_error",- };- }- return {- status: "error",- message: "Connection check failed.",- code: "runtime_error",- };-}-function readStdioTemplateId(config: Record<string, unknown>): string {const templateId = config.templateId;if (typeof templateId !== "string" || templateId.trim().length === 0) {@@ -1358,6 +807,8 @@ export function toolAccessService(options: ToolAccessServiceOptions = {},) {const secrets = secretService(db);+ const remoteHttp = createToolRemoteHttp(options);+ const { allowPrivateRemoteEndpoints, assertRemoteHttpUrlAllowed, fetchRemoteHttpUrl, assertRemoteEndpointAllowed, assertRemoteConnectionEndpointsAllowed, requestRemoteHttpEndpoint } = remoteHttp;const { assertOAuthEndpointUrl, safeOAuthEndpointUrl } = createOAuthEndpointPolicy(allowPrivateRemoteEndpoints);const { oauthEndpointsForConnection, discoverOAuthEndpoints, preflightConnectionMetadata } = createOAuthDiscovery({fetchRemoteHttpUrl, assertRemoteEndpointAllowed, assertOAuthEndpointUrl, safeOAuthEndpointUrl,@@ -1365,9 +816,6 @@ export function toolAccessService(const { requestOAuthClientRegistration, exchangeOAuthToken } = createOAuthClientProtocol({fetchRemoteHttpUrl, assertOAuthEndpointUrl,});- const composioSyncJobs = new Map<string, Promise<void>>();- const composioSyncTtlMs = 5 * 60_000;- const composioSyncLeaseMs = 4 * 60_000;const policySvc = toolAccessPolicyService(db);const now = options.now ?? (() => new Date());const audit = createToolAccessAudit(db);@@ -1419,6 +867,17 @@ export function toolAccessService(const setupPolicy = createConnectionSetupPolicy(db);const { assertConnectorConnectionSetupEnabled, assertExperimentalConnectorSetupEnabled } = setupPolicy;const fixedPersonalIdentityForReconnect = createPersonalConnectionIdentity(db);+ const runtimeSlots = createToolRuntimeSlots(db, runtimeSupervisor);+ const { ensureRuntimeSlot } = runtimeSlots;+ const defaultCatalogAccess = createCatalogDefaultAccess(db);+ const { enableCatalogEntriesByDefault, upsertAskFirstPolicies } = defaultCatalogAccess;+ const catalogDiscovery = createToolCatalogDiscovery(db, { now, audit, secrets, credentials: credentialAccess, vercel: vercelCredentials, vercelConnect, setupPolicy, requestRemoteHttpEndpoint, resolveOAuthGrantSecret, discoverOAuthEndpoints, localTools });+ const connectionHealth = createConnectionHealth(db, { now, audit, getConnectionRow, discovery: catalogDiscovery, setupPolicy, resolveCredentialHeaders, refreshManagedGitHubGrantAccess, ensureRuntimeSlot, stdioTemplateId });+ const { updateConnectionHealth, checkConnectionHealth, sweepConnectionHealth } = connectionHealth;+ const refreshCatalog = createToolCatalogPublication(db, { now, audit, getConnectionRow, discoverTools: catalogDiscovery.discoverTools, assertConnectorConnectionSetupEnabled, updateConnectionHealth, vaultGrantForConnection, ensureRuntimeSlot, enableCatalogEntriesByDefault });+ const catalogReader = createToolCatalogReader(db, options, now, getConnectionRow, refreshCatalog, toCatalogEntryForConnection);+ const listAppsNeedingAttention = createToolAppAttention(db, now);+ const aggregatorAccounts = createAggregatorAccounts(db, { options, secrets, audit, credentials: credentialAccess, getConnectionRow, requestRemoteHttpEndpoint, remoteTools: catalogDiscovery.remoteTools });const oauthState = createOAuthStateStore(db);const { peekOAuthState } = oauthState;const finishOAuthCatalogWithRecommendedDefaults = createOAuthCatalogCompletion(db, finishGalleryAppConnection);@@ -1427,124 +886,9 @@ export function toolAccessService(const completePaperclipCloudConnectorCallback = createCloudOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, currentCloudConnector });const completeVercelConnectCallback = createVercelOAuthCallback(db, { now, getConnectionRow, state: oauthState, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, vercelConnect });const finalizeOAuthAccess = createOAuthAccessFinalization(db, { getConnectionRow, secrets, store: credentialStore, assertAgentsInCompany, finishGalleryAppConnection });- const runtimeSlots = createToolRuntimeSlots(db, runtimeSupervisor);- const { ensureRuntimeSlot } = runtimeSlots;const runtimeHealth = createToolRuntimeHealth(db, options, now, trustedRuntimeHost);const toolActivity = createToolActivityService(db, getConnectionRow);const toolActionReviewQueue = createToolActionReviewQueue(db);- const catalogRefreshFlights = new Map<string, Promise<unknown>>();- const catalogCacheTtlMs = Math.max(- 0,- options.catalogCacheTtlMs ?? 15 * 60 * 1000,- );-- function allowPrivateRemoteEndpoints() {- return (- options.deploymentMode !== "authenticated" ||- options.deploymentExposure !== "public"- );- }-- async function assertRemoteHttpUrlAllowed(value: string): Promise<string> {- const endpoint = parseRemoteHttpEndpoint(value, (message, code) =>- badRequest(message, { code }),- );- await assertPublicRemoteHttpEndpoint(- endpoint,- {- allowPrivateNetwork: allowPrivateRemoteEndpoints(),- lookup: options.remoteHttpEndpointLookup,- },- (message, code) => badRequest(message, { code }),- );- return endpoint.toString();- }-- function remoteHttpFetchOptions(): GuardedRemoteHttpFetchOptions {- return {- allowPrivateNetwork: allowPrivateRemoteEndpoints(),- lookup: options.remoteHttpEndpointLookup,- error: (message, code) => badRequest(message, { code }),- };- }-- async function requestRemoteHttpEndpoint(- endpoint: URL,- init: RequestInit,- ): Promise<Response> {- return options.remoteHttpRequest- ? options.remoteHttpRequest(endpoint.toString(), {- ...init,- redirect: "manual",- })- : guardedRemoteHttpFetch(endpoint, init, remoteHttpFetchOptions());- }-- /**- * Fetch an operator-supplied remote URL with the egress guard bound to the- * connection itself.- *- * `guardedRemoteHttpFetch` resolves the hostname once and dials the approved- * address, so a name server that answers public-then-private cannot move the- * connection onto a loopback or metadata address after validation- * (PAP-17098). Redirects stay manual and run the full guard again on the next- * hop, because a `Location` is just as attacker-controlled as the first URL.- */- async function fetchRemoteHttpUrl(- value: string,- init: RequestInit = {},- ): Promise<Response> {- let currentUrl = value;- const method = (init.method ?? "GET").toUpperCase();- for (- let redirectCount = 0;- redirectCount <= MAX_REMOTE_HTTP_REDIRECTS;- redirectCount += 1- ) {- const endpoint = parseRemoteHttpEndpoint(currentUrl, (message, code) =>- badRequest(message, { code }),- );- const response = await requestRemoteHttpEndpoint(endpoint, init);- const location = REMOTE_HTTP_REDIRECT_STATUSES.has(response.status)- ? (response.headers?.get?.("location") ?? null)- : null;- if (!location) return response;- if (method !== "GET" && method !== "HEAD") {- throw new HttpError(- 502,- "Remote OAuth endpoint redirected unexpectedly",- { code: "oauth_redirect_rejected" },- );- }- if (redirectCount >= MAX_REMOTE_HTTP_REDIRECTS) {- throw new HttpError(- 502,- "Remote OAuth endpoint redirected too many times",- { code: "oauth_redirect_limit" },- );- }- currentUrl = new URL(location, endpoint).toString();- }- throw new HttpError(- 502,- "Remote OAuth endpoint redirected too many times",- { code: "oauth_redirect_limit" },- );- }-- async function assertRemoteEndpointAllowed(- config: Record<string, unknown>,- ): Promise<string> {- return assertRemoteHttpUrlAllowed(remoteEndpoint(config));- }-- async function assertRemoteConnectionEndpointsAllowed(- config: Record<string, unknown>,- ): Promise<string> {- const endpoint = await assertRemoteEndpointAllowed(config);- await assertConfiguredTokenBrokerEndpointsAllowed(config);- return endpoint;- }function trustedRuntimeHost() {return (@@ -1614,420 +958,54 @@ export function toolAccessService(return (await resolveStdioTemplate(companyId, config)).templateId;}- function shouldQuarantineNewEntries(- connection: typeof toolConnections.$inferSelect,- ): boolean {- return isDaytonaConnection(connection) || asRecord(connection.config).quarantineNewEntries === true;- }-- function isAttentionHealthStatus(- status: ToolConnectionHealthStatus,- ): boolean {- return isToolConnectionAttentionHealth(status);- }-- async function composioCredentialKey(connection: typeof toolConnections.$inferSelect, actor: ActorInfo) {- const usesConnectionRefs = ["arcade", "executor"].includes(String(connection.config.sourceTemplateKey)) && connection.authKind !== "oauth" && connection.credentialPolicy === "shared";- const grant = usesConnectionRefs ? null : await vaultGrantForConnection(connection, actor);- const refs = grant?.credentialSecretRefs ?? connection.credentialSecretRefs;- const discovery = asRecord(asRecord(connection.config.aggregatorDiscovery)[actor.actorId!]);- const ids = [...new Set([...refs.map(ref => ref.secretId), ...(typeof discovery.secretId === "string" ? [discovery.secretId] : [])])].sort();- const versions = ids.length ? await db.select({ id: companySecrets.id, version: companySecrets.latestVersion }).from(companySecrets)- .where(and(eq(companySecrets.companyId, connection.companyId), inArray(companySecrets.id, ids))).orderBy(asc(companySecrets.id)) : [];- return createHash("sha256").update(JSON.stringify({ grant: grant?.id, refs, versions,- ...(Object.keys(discovery).length ? { discovery } : {}),- ...(connection.config.sourceTemplateKey === "arcade" || connection.config.sourceTemplateKey === "executor"- ? { headers: projectedConnectionHeaders(connection), managementUrl: connection.config.managementUrl } : {}), endpoint: remoteEndpoint(connection.config), policy: connection.credentialPolicy, connectedAt: asRecord(connection.config.oauth)?.connectedAt })).digest("hex");+ async function assertApplication(companyId: string, applicationId: string) {+ const [row] = await db+ .select()+ .from(toolApplications)+ .where(+ and(+ eq(toolApplications.id, applicationId),+ eq(toolApplications.companyId, companyId),+ ),+ );+ if (!row) throw notFound("Tool application not found");+ return row;}- async function cachedComposioApps(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<ComposioAppSnapshot[]> {- const credentialKey = await composioCredentialKey(connection, actor);- const rows = await db.select().from(toolConnectionAppSnapshots).where(and(- eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connection.id),- eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),- ));- return rows.map(row => ({ connectionId: row.connectionId, toolkit: row.toolkit, status: row.status, accounts: row.accounts,- checkedAt: row.checkedAt.toISOString(), errorAt: row.errorAt?.toISOString() ?? null }));+ async function assertOptionalAgent(+ companyId: string,+ agentId: string | null | undefined,+ label: string,+ ) {+ if (!agentId) return;+ const [row] = await db+ .select({ id: agents.id })+ .from(agents)+ .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));+ if (!row) throw unprocessable(`${label} must belong to the same company`);}- async function composioAppsResponse(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<ComposioAppsResponse> {- const credentialKey = await composioCredentialKey(connection, actor);- const [sync] = await db.select().from(toolConnectionAppSyncs).where(and(- eq(toolConnectionAppSyncs.companyId, connection.companyId), eq(toolConnectionAppSyncs.connectionId, connection.id),- eq(toolConnectionAppSyncs.userId, actor.actorId!), eq(toolConnectionAppSyncs.credentialKey, credentialKey),- ));- const interrupted = sync?.status === "syncing" && sync.updatedAt.getTime() < Date.now() - composioSyncLeaseMs;- return { apps: await cachedComposioApps(connection, actor), sync: {- status: interrupted || sync?.status === "unsupported" ? "error" : sync?.status ?? "idle", coverage: "supported_catalog",- checked: sync?.checked ?? 0, total: sync?.total ?? 0, failed: sync?.failed ?? 0,- lastCompletedAt: sync?.lastCompletedAt?.toISOString() ?? null,- error: interrupted ? "App checks were interrupted. Refresh to try again."- : (sync?.status === "error" || sync?.status === "unsupported") ? "Couldn’t finish checking Composio apps. Refresh to try again." : null,- } };+ async function assertOptionalPlugin(pluginId: string | null | undefined) {+ if (!pluginId) return;+ const [row] = await db+ .select({ id: plugins.id })+ .from(plugins)+ .where(eq(plugins.id, pluginId));+ if (!row) throw unprocessable("Tool application plugin was not found");}- async function openComposioGateway(connectionId: string, actor: ActorInfo) {- const connection = await getConnectionRow(connectionId);- if (actor.actorType !== "user" || !actor.actorId) throw forbidden("A human connection manager must configure this app");- if (connection.config.sourceTemplateKey !== "composio" || connection.transport !== "mcp_remote"- || connection.status !== "active" || !connection.enabled) throw unprocessable("Connect your Composio gateway first");- const catalog = await db.select().from(toolCatalogEntries).where(and(- eq(toolCatalogEntries.companyId, connection.companyId), eq(toolCatalogEntries.connectionId, connection.id),- ));- if (!catalog.some(tool => tool.toolName === "COMPOSIO_MANAGE_CONNECTIONS" && tool.status === "active")) {- throw unprocessable("Refresh your Composio gateway's actions before configuring this app");- }- const endpoint = await resolvedRemoteEndpoint(connection, actor);- const headers = { ...projectedConnectionHeaders(connection), ...(await resolveCredentialHeaders(connection, actor)) };- const credentialKey = await composioCredentialKey(await getConnectionRow(connectionId), actor);- const send = (init: RequestInit) => requestRemoteHttpEndpoint(new URL(endpoint), { ...init, signal: AbortSignal.timeout(30_000) });- let sessionHeaders = connection.config.mcpSessionRequired === true- ? await getMcpHttpSession({ send, headers, scope: `${connection.id}:app-setup:${actor.actorId}`, requestId: "paperclip-app-setup" }) : headers;- async function manage(toolkits: { name: string; action: "list" | "add" | "rename" | "remove"; account_id?: string; alias?: string }[], tolerateToolkitErrors = false) {- const checkedAt = new Date();- const id = `paperclip-app-setup-${randomUUID()}`;- const request = () => send({ method: "POST", headers: mcpHttpRequestHeaders(sessionHeaders), body: JSON.stringify({- jsonrpc: "2.0", id, method: "tools/call", params: { name: "COMPOSIO_MANAGE_CONNECTIONS", arguments: { toolkits } },- }) });- const markFailed = async () => {- const names = toolkits.filter(tool => tool.action === "list").map(tool => tool.name);- if (names.length) await db.update(toolConnectionAppSnapshots).set({ errorAt: checkedAt }).where(and(- eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId),- eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),- inArray(toolConnectionAppSnapshots.toolkit, names), lte(toolConnectionAppSnapshots.checkedAt, checkedAt),- ));- };- try {- let response = await request();- // Only reads retry. An uncertain account mutation must never repeat automatically.- if (toolkits.every(tool => tool.action === "list") && (response.status === 400 || response.status === 404)) {- await response.body?.cancel();- sessionHeaders = await initializeMcpHttpSession({ send, headers, requestId: id });- response = await request();- }- if (!response.ok) throw new HttpError(502, toolkits.every(tool => tool.action === "list")- ? "Composio could not check this app. Reconnect the gateway if its sign-in expired."- : "Composio has not confirmed this change. Refresh the accounts before trying again.", { code: "composio_app_setup_failed", status: response.status });- const payload = await readMcpHttpResponse(response, id);- if (await composioCredentialKey(await getConnectionRow(connectionId), actor) !== credentialKey) {- throw conflict("The Composio account changed while checking apps. Refresh to check the current account.");- }- const failedToolkits: string[] = [];- for (const tool of toolkits.filter(tool => tool.action === "list")) {- let accounts: ComposioAppAccount[];- try {- composioAppSetupResult(payload, tool.name);- const parsed = composioAppAccounts(payload, tool.name);- if (!parsed) throw unprocessable("Composio did not return a complete account list. Refresh the accounts before trying again.");- accounts = parsed;- } catch (error) {- await db.update(toolConnectionAppSnapshots).set({ errorAt: checkedAt }).where(and(- eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId),- eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),- eq(toolConnectionAppSnapshots.toolkit, tool.name), lte(toolConnectionAppSnapshots.checkedAt, checkedAt),- ));- if (!tolerateToolkitErrors) throw error;- failedToolkits.push(tool.name);- continue;- }- const status = accounts.some(account => account.status === "ACTIVE") ? "connected" as const : "not_connected" as const;- await db.insert(toolConnectionAppSnapshots).values({ companyId: connection.companyId, connectionId, userId: actor.actorId!,- credentialKey, toolkit: tool.name, accounts, status, checkedAt, errorAt: null,- }).onConflictDoUpdate({ target: [toolConnectionAppSnapshots.companyId, toolConnectionAppSnapshots.connectionId, toolConnectionAppSnapshots.userId, toolConnectionAppSnapshots.toolkit],- set: { accounts, status, checkedAt, credentialKey, errorAt: null }, setWhere: lte(toolConnectionAppSnapshots.checkedAt, checkedAt) });- }- return { payload, failedToolkits };- } catch (error) { await markFailed(); throw error; }- }- return { connection, catalog, credentialKey, manage: async (...args: Parameters<typeof manage>) => (await manage(...args)).payload, checkBatch: manage };- }-- async function startComposioAppSync(connectionId: string, force: boolean, actor: ActorInfo) {- const opened = await openComposioGateway(connectionId, actor);- const { connection, credentialKey } = opened;- const existing = await cachedComposioApps(connection, actor);- const catalog = options.composioAppToolkits ?? COMPOSIO_APP_TOOLKITS;- const toolkits = [...new Set([...existing.filter(app => app.accounts.length > 0).map(app => app.toolkit), ...catalog])];- const leaseId = randomUUID();- const startedAt = new Date();- const [lease] = await db.insert(toolConnectionAppSyncs).values({ companyId: connection.companyId, connectionId,- userId: actor.actorId!, credentialKey, leaseId, status: "syncing", total: toolkits.length, updatedAt: startedAt,- }).onConflictDoUpdate({ target: [toolConnectionAppSyncs.companyId, toolConnectionAppSyncs.connectionId, toolConnectionAppSyncs.userId, toolConnectionAppSyncs.credentialKey],- set: { leaseId, status: "syncing", checked: 0, total: toolkits.length, failed: 0, updatedAt: startedAt },- setWhere: and(- or(ne(toolConnectionAppSyncs.status, "syncing"), lt(toolConnectionAppSyncs.updatedAt, new Date(Date.now() - composioSyncLeaseMs))),- force ? sql`true` : or(ne(toolConnectionAppSyncs.status, "ready"), isNull(toolConnectionAppSyncs.lastCompletedAt),- lt(toolConnectionAppSyncs.lastCompletedAt, new Date(Date.now() - composioSyncTtlMs))),- ),- }).returning();- if (lease) {- const updateLease = async (values: Partial<typeof toolConnectionAppSyncs.$inferInsert>) => {- const updated = await db.update(toolConnectionAppSyncs).set({ ...values, updatedAt: new Date() })- .where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId))).returning({ id: toolConnectionAppSyncs.id });- if (!updated.length) throw conflict("App checks were replaced by a newer sync.");- };- const job = (async () => {- let failed = 0;- try {- for (let start = 0; start < toolkits.length; start += 32) {- if (Date.now() - startedAt.getTime() > composioSyncLeaseMs) throw new Error("Sync deadline exceeded");- await updateLease({});- const current = start === 0 ? opened : await openComposioGateway(connectionId, actor);- if (current.credentialKey !== credentialKey) throw conflict("Composio credentials changed during sync.");- const batch = toolkits.slice(start, start + 32);- const result = await current.checkBatch(batch.map(name => ({ name, action: "list" })), true);- failed += result.failedToolkits.length;- await updateLease({ checked: start + batch.length, failed });- }- await updateLease({ status: failed > 0 ? "error" : "ready", lastCompletedAt: new Date(), failed });- } catch {- const [activeLease] = await db.select({ id: toolConnectionAppSyncs.id }).from(toolConnectionAppSyncs)- .where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId)));- if (!activeLease) return;- // Preserve observations, but never present a failed check as current authorization.- await db.update(toolConnectionAppSnapshots).set({ errorAt: new Date() }).where(and(- eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId),- eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),- lte(toolConnectionAppSnapshots.checkedAt, startedAt),- ));- await updateLease({ status: "error", failed }).catch(() => undefined);- }- })();- composioSyncJobs.set(leaseId, job);- void job.finally(() => composioSyncJobs.delete(leaseId)).catch(() => undefined);- }- return composioAppsResponse(await getConnectionRow(connectionId), actor);- }-- function aggregatorProvider(connection: typeof toolConnections.$inferSelect): AppCatalogAggregator {- const provider = connection.config.sourceTemplateKey;- if (!isAppAggregator(provider) || connection.transport !== "mcp_remote") throw notFound("Aggregator gateway not found");- return provider;- }-- function assertDiscoveryActor(actor: ActorInfo) {- if (actor.actorType !== "user" || !actor.actorId) throw forbidden("A human connection manager must check accounts");- }-- async function aggregatorDiscoveryState(connection: typeof toolConnections.$inferSelect, actor: ActorInfo, retryUnsupported = false): Promise<AggregatorAppsResponse["discovery"]> {- const provider = aggregatorProvider(connection);- if (connection.status !== "active" || !connection.enabled) return { availability: "disabled", message: "Restore this gateway to check its accounts." };- if (provider === "arcade") {- const discovery = asRecord(asRecord(connection.config.aggregatorDiscovery)[actor.actorId!]);- const headers = projectedConnectionHeaders(connection);- const grant = connection.authKind === "oauth" || connection.credentialPolicy !== "shared" ? await vaultGrantForConnection(connection, actor) : null;- const refs = grant?.credentialSecretRefs ?? connection.credentialSecretRefs;- const hasUserId = Boolean(Object.entries(headers).find(([key]) => key.toLowerCase() === "arcade-user-id")?.[1])- || refs.some(ref => ref.configPath.toLowerCase() === "headers.arcade-user-id");- if ((!discovery.secretId || !discovery.userId) && !(hasUserId && connection.authKind === "api_key")) {- return { availability: "setup_required", message: "Set up account sync to see your Arcade apps here." };- }- }- const catalog = await db.select({ name: toolCatalogEntries.toolName }).from(toolCatalogEntries).where(and(- eq(toolCatalogEntries.companyId, connection.companyId), eq(toolCatalogEntries.connectionId, connection.id), eq(toolCatalogEntries.status, "active"),- ));- if ((provider === "executor" && !catalog.some(tool => ["integrations", "execute"].includes(tool.name)))- || (provider === "composio" && !catalog.some(tool => tool.name === "COMPOSIO_MANAGE_CONNECTIONS"))) {- return { availability: "unsupported", message: `${AGGREGATOR_NAMES[provider]} account discovery is unavailable on this gateway. Refresh its actions or check its setup.` };- }- if (!retryUnsupported && provider === "executor") {- const credentialKey = await composioCredentialKey(connection, actor);- const [sync] = await db.select({ status: toolConnectionAppSyncs.status }).from(toolConnectionAppSyncs).where(and(- eq(toolConnectionAppSyncs.companyId, connection.companyId), eq(toolConnectionAppSyncs.connectionId, connection.id),- eq(toolConnectionAppSyncs.userId, actor.actorId!), eq(toolConnectionAppSyncs.credentialKey, credentialKey),- ));- if (sync?.status === "unsupported") return { availability: "unsupported", message: "Executor account discovery is unavailable on this server. You can still use the gateway’s actions." };- }- return { availability: "available", message: null };- }-- async function aggregatorAppsResponse(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<AggregatorAppsResponse> {- assertDiscoveryActor(actor);- const provider = aggregatorProvider(connection);- const result = await composioAppsResponse(connection, actor);- const discovery = await aggregatorDiscoveryState(connection, actor);- return { provider, discovery, apps: result.apps.map(snapshot => {- const app = resolveAggregatorApp(provider, snapshot.toolkit, snapshot.accounts[0]?.appName);- return { ...snapshot, provider, freshness: discovery.availability !== "available" || snapshot.errorAt || result.sync.status === "error" || connection.status !== "active" || !connection.enabled || Date.now() - new Date(snapshot.checkedAt).getTime() > composioSyncTtlMs ? "stale" as const : "fresh" as const, appSlug: app.slug.startsWith(`${provider}:`) ? `${provider}:${connection.id}:${snapshot.toolkit}` : snapshot.accounts[0]?.appSlug ?? app.slug, appName: snapshot.accounts[0]?.appName ?? app.name };- }), sync: { ...result.sync, coverage: provider === "composio" ? "supported_catalog" : provider === "arcade" ? "gateway_tools" : "visible_accounts",- error: result.sync.error ? `Couldn’t finish checking ${AGGREGATOR_NAMES[provider]} apps. Refresh to try again.` : null,- } };- }-- async function discoverAggregatorInventory(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<DiscoveredApp[]> {- const provider = aggregatorProvider(connection);- const tools = await remoteTools(connection, undefined, actor);- connection = await getConnectionRow(connection.id);- const headers = { ...projectedConnectionHeaders(connection), ...(await resolveCredentialHeaders(connection, actor)) };- const endpoint = new URL(await resolvedRemoteEndpoint(connection, actor));- if (provider === "arcade") {- // The extra project credential never joins invocation headers or tool grants.- if (endpoint.hostname !== "api.arcade.dev" || endpoint.protocol !== "https:") throw unprocessable("Account sync requires an Arcade Cloud gateway");- const discovery = asRecord(asRecord(connection.config.aggregatorDiscovery)[actor.actorId!]);- let key: string | undefined;- let userId: string | undefined;- if (typeof discovery.secretId === "string" && typeof discovery.userId === "string") {- const [secret] = await db.select().from(companySecrets).where(and(eq(companySecrets.id, discovery.secretId), eq(companySecrets.companyId, connection.companyId), eq(companySecrets.ownerUserId, actor.actorId!)));- if (!secret?.userSecretDefinitionId) throw forbidden("Your Arcade discovery credential is unavailable");- const resolved = await secrets.resolveUserSecretValue(connection.companyId, { definitionId: secret.userSecretDefinitionId, responsibleUserId: actor.actorId!, required: true }, {- actorType: "user", actorId: actor.actorId!, consumerType: "tool_connection", consumerId: connection.id, responsibleUserId: actor.actorId!,- });- key = resolved?.value;- userId = discovery.userId;- } else {- userId = Object.entries(headers).find(([name]) => name.toLowerCase() === "arcade-user-id")?.[1];- if (connection.authKind === "api_key") key = Object.entries(headers).find(([name]) => name.toLowerCase() === "authorization")?.[1].replace(/^Bearer\s+/i, "");- }- if (!key || !userId) throw unprocessable("Set up Arcade account sync first");- return discoverArcadeApps({ userId, gatewayTools: tools.map(tool => tool.name), request: async path => {- const response = await requestRemoteHttpEndpoint(new URL(path, endpoint.origin), { method: "GET", headers: { Authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(30_000) });- if (!response.ok) { await response.body?.cancel(); throw new HttpError(502, "Arcade could not list your accounts. Check the project API key and Arcade user ID."); }- return response.json();- } });- }- const send = (init: RequestInit) => requestRemoteHttpEndpoint(endpoint, { ...init, signal: AbortSignal.timeout(30_000) });- let sessionHeaders = connection.config.mcpSessionRequired === true ? await getMcpHttpSession({ send, headers, scope: `${connection.id}:inventory:${actor.actorId}`, requestId: "paperclip-account-inventory" }) : headers;- return discoverExecutorApps({ toolNames: tools.map(tool => tool.name), managementUrl: typeof connection.config.managementUrl === "string" ? connection.config.managementUrl : null,- call: async (name, args) => {- const id = `paperclip-account-inventory-${randomUUID()}`;- const request = () => send({ method: "POST", headers: mcpHttpRequestHeaders(sessionHeaders), body: JSON.stringify({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: args } }) });- let response = await request();- if (response.status === 400 || response.status === 404) {- await response.body?.cancel(); sessionHeaders = await initializeMcpHttpSession({ send, headers, requestId: id }); response = await request();- }- if (!response.ok) { await response.body?.cancel(); throw new HttpError(502, "Executor could not list your accounts. Reconnect if sign-in expired."); }- return readMcpHttpResponse(response, id);- },- });- }-- async function syncAggregatorApps(connectionId: string, force: boolean, actor: ActorInfo): Promise<AggregatorAppsResponse> {- assertDiscoveryActor(actor);- let connection = await getConnectionRow(connectionId);- const provider = aggregatorProvider(connection);- const discovery = await aggregatorDiscoveryState(connection, actor, force);- if (discovery.availability !== "available") return aggregatorAppsResponse(connection, actor);- if (provider === "composio") {- await startComposioAppSync(connectionId, force, actor);- return aggregatorAppsResponse(await getConnectionRow(connectionId), actor);- }- // Resolve/refresh the caller's gateway credentials before fingerprinting the lease.- await resolveCredentialHeaders(connection, actor);- connection = await getConnectionRow(connectionId);- const credentialKey = await composioCredentialKey(connection, actor);- const leaseId = randomUUID();- const startedAt = new Date();- const [lease] = await db.insert(toolConnectionAppSyncs).values({ companyId: connection.companyId, connectionId, userId: actor.actorId!, credentialKey, leaseId, status: "syncing", updatedAt: startedAt })- .onConflictDoUpdate({ target: [toolConnectionAppSyncs.companyId, toolConnectionAppSyncs.connectionId, toolConnectionAppSyncs.userId, toolConnectionAppSyncs.credentialKey],- set: { leaseId, status: "syncing", checked: 0, total: 0, failed: 0, updatedAt: startedAt },- setWhere: and(or(ne(toolConnectionAppSyncs.status, "syncing"), lt(toolConnectionAppSyncs.updatedAt, new Date(Date.now() - composioSyncLeaseMs))),- force ? sql`true` : or(ne(toolConnectionAppSyncs.status, "ready"), isNull(toolConnectionAppSyncs.lastCompletedAt), lt(toolConnectionAppSyncs.lastCompletedAt, new Date(Date.now() - composioSyncTtlMs)))),- }).returning();- if (lease) {- const job = (async () => {- try {- const apps = await discoverAggregatorInventory(connection, actor);- await db.transaction(async tx => {- const [current] = await tx.select().from(toolConnections).where(and(eq(toolConnections.id, connectionId), eq(toolConnections.companyId, connection.companyId))).for("update");- const [active] = await tx.select().from(toolConnectionAppSyncs).where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId))).for("update");- if (!current || !active || current.status !== "active" || !current.enabled || Date.now() - startedAt.getTime() > composioSyncLeaseMs- || await composioCredentialKey(current, actor) !== credentialKey) throw conflict("The gateway changed during account discovery");- const checkedAt = new Date();- // Commit the entire enumeration at once; a failed later page cannot delete accounts.- await tx.update(toolConnectionAppSnapshots).set({ accounts: [], status: "not_connected", checkedAt, errorAt: null }).where(and(- eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId), eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),- ));- for (const app of apps) await tx.insert(toolConnectionAppSnapshots).values({ companyId: connection.companyId, connectionId, userId: actor.actorId!, credentialKey,- toolkit: app.toolkit, accounts: app.accounts, status: app.accounts.some(account => account.status === "ACTIVE") ? "connected" : "not_connected", checkedAt,- }).onConflictDoUpdate({ target: [toolConnectionAppSnapshots.companyId, toolConnectionAppSnapshots.connectionId, toolConnectionAppSnapshots.userId, toolConnectionAppSnapshots.toolkit],- set: { accounts: app.accounts, status: app.accounts.some(account => account.status === "ACTIVE") ? "connected" : "not_connected", checkedAt, errorAt: null, credentialKey },- });- await tx.update(toolConnectionAppSyncs).set({ status: "ready", checked: apps.length, total: apps.length, failed: 0, updatedAt: checkedAt, lastCompletedAt: checkedAt }).where(eq(toolConnectionAppSyncs.id, lease.id));- });- } catch (error) {- await db.transaction(async tx => {- const [active] = await tx.select().from(toolConnectionAppSyncs).where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId))).for("update");- if (!active) return;- await tx.update(toolConnectionAppSnapshots).set({ errorAt: new Date() }).where(and(eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId), eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey)));- await tx.update(toolConnectionAppSyncs).set({ status: error instanceof AggregatorDiscoveryUnavailableError ? "unsupported" : "error", failed: 1, updatedAt: new Date() }).where(eq(toolConnectionAppSyncs.id, lease.id));- });- }- })();- composioSyncJobs.set(leaseId, job);- void job.finally(() => composioSyncJobs.delete(leaseId)).catch(() => undefined);- }- return aggregatorAppsResponse(await getConnectionRow(connectionId), actor);- }-- async function configureArcadeDiscovery(connectionId: string, input: ArcadeDiscoverySetupInput, actor: ActorInfo) {- assertDiscoveryActor(actor);- const companyId = (await getConnectionRow(connectionId)).companyId;- await db.transaction(async tx => {- const [connection] = await tx.select().from(toolConnections).where(eq(toolConnections.id, connectionId)).for("update");- if (!connection || aggregatorProvider(connection) !== "arcade" || connection.status === "archived") throw notFound("Arcade gateway not found");- const all = asRecord(connection.config.aggregatorDiscovery);- const before = asRecord(all[actor.actorId!]);- const { secret } = await writeConnectionCredential(tx, {- companyId: connection.companyId, connectionName: connection.name, configPath: "aggregatorDiscovery.apiKey", label: "Arcade account sync API key",- value: input.apiKey, ownerUserId: actor.actorId!, actor: actorForSecret(actor),- existingRef: typeof before.secretId === "string" ? { secretId: before.secretId, configPath: "aggregatorDiscovery.apiKey", versionSelector: "latest" } : undefined,- definitionKey: `arcade_discovery.${connection.id}`,- });- const config = { ...connection.config, aggregatorDiscovery: { ...all, [actor.actorId!]: { userId: input.userId, secretId: secret.id } } };- await tx.update(toolConnections).set({ config, transportConfig: config, updatedAt: new Date() }).where(eq(toolConnections.id, connectionId));- });- await logActivity(db, { companyId, actorType: "user", actorId: actor.actorId!, action: "tool_connection.account_sync_configured", entityType: "tool_connection", entityId: connectionId, details: { provider: "arcade" } });- return syncAggregatorApps(connectionId, true, actor);- }-- async function assertApplication(companyId: string, applicationId: string) {- const [row] = await db- .select()- .from(toolApplications)- .where(- and(- eq(toolApplications.id, applicationId),- eq(toolApplications.companyId, companyId),- ),- );- if (!row) throw notFound("Tool application not found");- return row;- }-- async function assertOptionalAgent(- companyId: string,- agentId: string | null | undefined,- label: string,- ) {- if (!agentId) return;- const [row] = await db- .select({ id: agents.id })- .from(agents)- .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));- if (!row) throw unprocessable(`${label} must belong to the same company`);- }-- async function assertOptionalPlugin(pluginId: string | null | undefined) {- if (!pluginId) return;- const [row] = await db- .select({ id: plugins.id })- .from(plugins)- .where(eq(plugins.id, pluginId));- if (!row) throw unprocessable("Tool application plugin was not found");- }-- async function assertSecretRefs(- companyId: string,- refs: Array<{- secretId: string;- configPath?: string | null;- projectionClass?: string | null;- projectionAllowlistKey?: string | null;- }>,- ) {- if (refs.length === 0) return;- for (const ref of refs) {- assertClass3ToolCredentialRefAllowed(ref);+ async function assertSecretRefs(+ companyId: string,+ refs: Array<{+ secretId: string;+ configPath?: string | null;+ projectionClass?: string | null;+ projectionAllowlistKey?: string | null;+ }>,+ ) {+ if (refs.length === 0) return;+ for (const ref of refs) {+ assertClass3ToolCredentialRefAllowed(ref);}const secretIds = [...new Set(refs.map((ref) => ref.secretId))];for (const secretId of secretIds) {@@ -2172,190 +1150,6 @@ export function toolAccessService(return profile;}- async function enableCatalogEntriesByDefault(input: {- connection: typeof toolConnections.$inferSelect;- newCatalogEntryIds: string[];- activeCatalogEntryIds: string[];- restoreDraftDefaults?: boolean;- actor?: ActorInfo;- }, dbClient: ToolAccessMutationDb = db) {- // Catalog discovery also runs while the setup wizard is still a draft.- // Access is not granted until the operator finishes that wizard, so a- // draft refresh must never manufacture a profile or company-wide binding.- // Active legacy connections may still need the managed profile created on- // their first refresh. A reconnect is the one draft exception: removal- // deliberately clears the old selections, so reconnecting restores the- // documented defaults without activating the connection itself.- if (input.connection.status !== "active" && !input.restoreDraftDefaults)- return;- const profileKey = `app:${input.connection.id}`;- let [profile] = await dbClient- .select()- .from(toolProfiles)- .where(- and(- eq(toolProfiles.companyId, input.connection.companyId),- eq(toolProfiles.profileKey, profileKey),- ),- )- .limit(1).for("no key update");- await lockManagedAppPolicies(dbClient, input.connection.companyId, input.connection.id);- const createdProfile = !profile;- const resetToRecommendedDefaults = !profile || profile.status !== "active";- if (resetToRecommendedDefaults) await lockToolAssignmentAgents(dbClient, input.connection.companyId, [{ targetType: "company", targetId: input.connection.companyId }]);- if (!profile) {- const [sameName] = await dbClient- .select({ id: toolProfiles.id })- .from(toolProfiles)- .where(- and(- eq(toolProfiles.companyId, input.connection.companyId),- eq(toolProfiles.name, input.connection.name),- ),- )- .limit(1);- const profileName = sameName- ? `${input.connection.name} (${input.connection.id.replace(/-/g, "").slice(0, 8)})`- : input.connection.name;- [profile] = await dbClient- .insert(toolProfiles)- .values({- companyId: input.connection.companyId,- profileKey,- name: profileName,- description: `Access profile for ${input.connection.name}.`,- status: "active",- defaultAction: "deny",- metadata: {- source: "app_gallery_finish",- connectionId: input.connection.id,- },- })- .returning();- await dbClient.insert(toolProfileBindings).values({- companyId: input.connection.companyId,- profileId: profile.id,- targetType: "company",- targetId: input.connection.companyId,- priority: 100,- metadata: { source: "app_gallery_finish" },- createdByAgentId:- input.actor?.actorType === "agent"- ? (input.actor.actorId ?? null)- : null,- createdByUserId:- input.actor?.actorType === "user"- ? (input.actor.actorId ?? null)- : null,- });- } else if (resetToRecommendedDefaults) {- // Removing an app may retain its profile row when another record still- // references it (for example, an MCP gateway). Reconnecting revives the- // same connection id, so restore that retained profile as a fresh- // all-agents assignment instead of leaving every read action Off.- [profile] = await dbClient- .update(toolProfiles)- .set({- name: input.connection.name,- description: `Access profile for ${input.connection.name}.`,- status: "active",- defaultAction: "deny",- metadata: {- source: "app_gallery_finish",- connectionId: input.connection.id,- },- updatedAt: new Date(),- })- .where(eq(toolProfiles.id, profile.id))- .returning();- await dbClient- .delete(toolProfileBindings)- .where(- and(- eq(toolProfileBindings.companyId, input.connection.companyId),- eq(toolProfileBindings.profileId, profile.id),- ),- );- await dbClient- .delete(toolProfileEntries)- .where(- and(- eq(toolProfileEntries.companyId, input.connection.companyId),- eq(toolProfileEntries.profileId, profile.id),- ),- );- await dbClient.insert(toolProfileBindings).values({- companyId: input.connection.companyId,- profileId: profile.id,- targetType: "company",- targetId: input.connection.companyId,- priority: 100,- metadata: { source: "app_gallery_finish" },- createdByAgentId:- input.actor?.actorType === "agent"- ? (input.actor.actorId ?? null)- : null,- createdByUserId:- input.actor?.actorType === "user"- ? (input.actor.actorId ?? null)- : null,- });- }-- if (resetToRecommendedDefaults) {- // Old setup defaults created app-scoped Ask first policies for writes.- // They outrank profile allows, so a revived connection must explicitly- // retire them to make every action Allowed. Policies outside this app's- // managed profile are intentionally untouched.- await upsertAskFirstPolicies({- companyId: input.connection.companyId,- connection: input.connection,- askFirstEntries: [],- actor: input.actor,- }, dbClient);- }-- // A new or revived connection starts with every discovered action enabled.- // Later refreshes extend an active managed profile only for genuinely new- // actions, so an action the operator deliberately turned off remains off.- const candidateIds = [- ...new Set(- createdProfile || resetToRecommendedDefaults- ? input.activeCatalogEntryIds- : input.newCatalogEntryIds,- ),- ];- if (candidateIds.length === 0) return;- const existingEntries = await dbClient- .select({ catalogEntryId: toolProfileEntries.catalogEntryId })- .from(toolProfileEntries)- .where(- and(- eq(toolProfileEntries.companyId, input.connection.companyId),- eq(toolProfileEntries.profileId, profile.id),- inArray(toolProfileEntries.catalogEntryId, candidateIds),- ),- );- const configuredIds = new Set(- existingEntries.flatMap((entry) =>- entry.catalogEntryId ? [entry.catalogEntryId] : [],- ),- );- const entryIds = candidateIds.filter((id) => !configuredIds.has(id));- if (entryIds.length === 0) return;- await dbClient.insert(toolProfileEntries).values(- entryIds.map((catalogEntryId) => ({- companyId: input.connection.companyId,- profileId: profile.id,- selectorType: "catalog_entry" as const,- effect: "include" as const,- applicationId: input.connection.applicationId,- connectionId: input.connection.id,- catalogEntryId,- })),- );- }-async function listConnectionInstalls(connectionId: string,companyId?: string,@@ -2952,1235 +1746,131 @@ export function toolAccessService();let secretsRevoked = 0;for (const secretId of owned) {- const removed = await secrets.remove(secretId);- if (removed) secretsRevoked += 1;- }-- const credentialRefsCleared =- connection.credentialRefs.length + connection.credentialSecretRefs.length;- const [cleared] = await db- .update(toolConnections)- .set({- credentialRefs: [],- credentialSecretRefs: [],- config: withoutInlineOAuthTokens(connection.config),- transportConfig: withoutInlineOAuthTokens(connection.transportConfig),- updatedAt: now,- })- .where(eq(toolConnections.id, connection.id))- .returning();- if (- grantRows.some((grant) => (grant.credentialSecretRefs ?? []).length > 0)- ) {- await db- .update(connectionGrants)- .set({ credentialSecretRefs: [], updatedAt: now })- .where(- and(- eq(connectionGrants.companyId, connection.companyId),- eq(connectionGrants.connectionId, connection.id),- ),- );- }-- return {- connection: toConnection(cleared ?? archived.connection),- removal: {- secretsRevoked,- secretsRetainedShared: retained.length,- credentialRefsCleared,- secretBindingsRemoved: removedSecretBindings.length,- grantsRevoked: grantsToRevoke.length,- installsRemoved: removedInstalls.length,- appProfile: appProfileOutcome,- appProfileEntriesRemoved,- appProfileBindingsRemoved,- catalogEntriesMarkedRemoved: removedCatalogEntries.length,- oauthStatesDiscarded: discardedOAuthStates.length,- tokenIssuanceHashesCleared: clearedIssuanceHashes.length,- runtimeSlotsStopped,- gatewayTokensRevoked,- gatewaySessionsRevoked,- applicationArchived: archived.applicationArchived,- externalCredentialCleanup,- },- };- }-- async function remoteTools(- connection: typeof toolConnections.$inferSelect,- credentialHeaders?: Record<string, string>,- actor?: ActorInfo,- workspaceEndpoint?: string,- unavailableServices: string[] = [],- ): Promise<McpToolDescriptor[]> {- assertSupportedConnection(connection);- await assertConnectorConnectionSetupEnabled(connection);- let headers = credentialHeaders ?? {- ...projectedConnectionHeaders(connection),- ...(await resolveCredentialHeaders(connection, actor)),- };- if (connection.config.sourceTemplateKey === "google-workspace" && !workspaceEndpoint) {- const grant = await vaultGrantForConnection(connection, actor);- const scopes = googleWorkspaceGrantedScopes(grant);- const services = (Object.keys(GOOGLE_WORKSPACE_SERVICES) as GoogleWorkspaceService[])- .filter((service) => googleWorkspaceServiceGranted(service, scopes));- // One credential authority, no child connections or implied service grants.- const catalogs = await Promise.allSettled(services.map(async (service) => {- const profile = GOOGLE_WORKSPACE_CONNECTOR_PROFILES[GOOGLE_WORKSPACE_SERVICES[service]];- const tools = await remoteTools(connection, headers, actor, profile.serverUrl);- return tools.map((tool) => ({ ...tool, name: googleWorkspaceToolName(service, tool.name) }))- .filter((tool) => isGoogleWorkspaceToolGranted(tool.name, scopes));- }));- for (const [index, catalog] of catalogs.entries()) {- if (catalog.status === "fulfilled") continue;- const service = services[index]!;- unavailableServices.push(service);- await audit({ companyId: connection.companyId, connectionId: connection.id,- action: "tool_connection.catalog_refresh", outcome: "failure", actor,- reasonCode: "google_workspace_service_unavailable", details: { service } });- }- const currentGrant = await vaultGrantForConnection(connection, actor);- if (!currentGrant) throw unprocessable("OAuth authorization must be reconnected", { code: "oauth_reauthorization_required" });- if (catalogs.length > 0 && catalogs.every((catalog) => catalog.status === "rejected")) {- throw new HttpError(502, `Google services unavailable: ${unavailableServices.join(", ")}. Retry refreshing actions.`,- { code: "google_workspace_services_unavailable" });- }- // A service's authentication retry may have refreshed into a narrower- // grant. Never publish actions using the pre-refresh permission snapshot.- const currentScopes = googleWorkspaceGrantedScopes(currentGrant);- return catalogs.flatMap((catalog) => catalog.status === "fulfilled" ? catalog.value : [])- .filter((tool) => isGoogleWorkspaceToolGranted(tool.name, currentScopes));- }- const endpoint = workspaceEndpoint ?? await resolvedRemoteEndpoint(connection, actor);- // Pinned to the address the guard approved: `config.url` is operator-supplied,- // so a second DNS resolution here would reopen the rebinding window that- // PAP-17098 closed for the OAuth endpoints.- let listRequestId = "paperclip-catalog-refresh";- let sessionHeaders = headers;- const sendRemote = (init: RequestInit) => withMcpConnectionFailure(() => requestRemoteHttpEndpoint(new URL(endpoint), init));- const sendToolsList = (requestHeaders: Record<string, string>, cursor?: string) => {- sessionHeaders = requestHeaders;- return sendRemote({ method: "POST", headers: mcpHttpRequestHeaders(requestHeaders),- body: JSON.stringify({ jsonrpc: "2.0", id: listRequestId, method: "tools/list", params: cursor ? { cursor } : {} }) });- };- const connectionConfig = asRecord(connection.config);- const sourceTemplateKey = typeof connectionConfig?.sourceTemplateKey === "string"- ? connectionConfig.sourceTemplateKey- : null;- const connectionMethodKey = typeof connectionConfig?.connectionMethodKey === "string"- ? connectionConfig.connectionMethodKey- : null;- const curatedMethodRequiresMcpSession = Boolean(- sourceTemplateKey && connectionMethodKey &&- getConnectableAppDefinition(sourceTemplateKey)?.methods.some((method) =>- method.key === connectionMethodKey && method.defaults?.mcpSessionRequired === true- ),- );- let usedInitializedSession = connectionConfig?.mcpSessionRequired === true || curatedMethodRequiresMcpSession;- let response: Response;- if (usedInitializedSession) {- try {- sessionHeaders = await getMcpHttpSession({ send: sendRemote, headers,- scope: `${connection.id}:catalog:${actor?.actorType}:${actor?.actorId}:${endpoint}`,- requestId: listRequestId });- response = await sendToolsList(sessionHeaders);- } catch (error) {- if (!(error instanceof McpHttpInitializationError) || !error.response) throw error;- response = error.response;- }- } else {- response = await sendToolsList(headers);- // `tools/list` is read-only, so a 400 can safely be retried after the MCP- // initialize handshake. Stateful servers such as Supabase require the- // returned Mcp-Session-Id on every non-initialization request.- if (response.status === 400) {- try {- const sessionHeaders = await initializeMcpHttpSession({- send: sendRemote,- headers,- requestId: "paperclip-catalog-refresh",- });- response = await sendToolsList(sessionHeaders);- usedInitializedSession = response.ok;- } catch {- // Preserve the original HTTP failure below when this was not an MCP- // session requirement after all.- }- }- }- if (response.status === 404 && new Headers(sessionHeaders).has("mcp-session-id")) {- // MCP uses 404 for an expired server session. Discovery is read-only, so- // discard the stale session and retry it once with a new handshake.- forgetMcpHttpSessions(connection.id);- await response.body?.cancel().catch(() => undefined);- sessionHeaders = await getMcpHttpSession({ send: sendRemote, headers,- scope: `${connection.id}:catalog:${actor?.actorType}:${actor?.actorId}:${endpoint}`,- requestId: listRequestId });- response = await sendToolsList(sessionHeaders);- if (response.status === 404) forgetMcpHttpSessions(connection.id);- }- if (- usedInitializedSession &&- connectionConfig?.mcpSessionRequired !== true- ) {- const nextConfig = { ...connection.config, mcpSessionRequired: true };- await db- .update(toolConnections)- .set({- config: nextConfig,- transportConfig: nextConfig,- updatedAt: now(),- })- .where(- and(- eq(toolConnections.id, connection.id),- eq(toolConnections.companyId, connection.companyId),- ),- );- }- if (isInsufficientConnectionScope(response)) {- await response.body?.cancel().catch(() => undefined);- throw unprocessable(INSUFFICIENT_CONNECTION_SCOPE_MESSAGE, { code: "oauth_insufficient_scope", connectionId: connection.id });- }- if (- response.status === 401 &&- connection.credentialSource === "vercel_connect"- ) {- const grant = await vercelGrantForConnection(connection, actor);- const credential = vercelCredentialFor(connection);- const request = vercelTokenRequest({- credential,- grant,- connectionId: connection.id,- companyId: connection.companyId,- resources: vercelConnectResourcesFor(connection),- });- vercelConnect?.evict(request);- headers = {- ...projectedConnectionHeaders(connection),- ...(await resolveVercelCredentialHeaders(connection, grant, {- forceRefresh: true,- })),- };- response = await sendToolsList(headers);- }- if (- response.status === 401 &&- connection.authKind === "oauth" &&- connection.credentialSource === "paperclip_vault"- ) {- headers = {- ...projectedConnectionHeaders(connection),- ...(await resolveCredentialHeaders(connection, actor, {- forceRefresh: true,- })),- };- response = await sendToolsList(headers);- if (- response.status === 401 &&- isPaperclipCloudConnectorStrategy(oauthConfig(connection).strategy)- ) {- const grant = await vaultGrantForConnection(connection, actor);- if (grant) {- await db- .update(connectionGrants)- .set({ status: "needs_reauthorization", updatedAt: now() })- .where(- and(- eq(connectionGrants.id, grant.id),- eq(connectionGrants.companyId, connection.companyId),- ),- );- }- }- }- if (!response.ok) {- if ((response.status === 401 || response.status === 403)- && connection.authKind === "api_key" && isMemoryConnectorId(connection.config.sourceTemplateKey)) {- throw unprocessable("The provider rejected this API key. Check the key and its account access, then try again.", { code: "memory_api_key_rejected" });- }- if (await isSlackMcpAccessDisabledResponse(endpoint, response)) {- throw unprocessable(- "Slack MCP access is disabled for this app. Ask the Slack app owner to enable MCP access, then refresh this connection.",- { code: "slack_mcp_access_disabled", setupUrl: connectionSetupUrl(connection) },- );- }- const authenticate = response.headers.get("www-authenticate") ?? "";- if (- response.status === 401 &&- /bearer|oauth|authorization/i.test(authenticate)- ) {- const endpoints = await discoverOAuthEndpoints(- connection,- authenticate,- );- if (endpoints) {- const nextConfig = {- ...connection.config,- oauth: {- ...oauthConfig(connection),- provider: endpoints.provider,- authorizationUrl: endpoints.authorizationUrl,- tokenUrl: endpoints.tokenUrl,- registrationUrl: endpoints.registrationUrl ?? null,- metadataUrl: endpoints.metadataUrl ?? null,- scopes: endpoints.scopes,- codeChallengeMethodsSupported:- endpoints.codeChallengeMethodsSupported ?? [],- tokenEndpointAuthMethodsSupported:- endpoints.tokenEndpointAuthMethodsSupported ?? [],- grantType: endpoints.grantType ?? "authorization_code",- issuer: endpoints.issuer ?? null,- resource: endpoints.resource ?? null,- clientIdMetadataDocumentSupported:- endpoints.clientIdMetadataDocumentSupported === true,- discoveredAt: new Date().toISOString(),- },- };- await db- .update(toolConnections)- .set({- // Record the discovered auth kind now, so a URL-only connection that- // is waiting on sign-in reads as OAuth everywhere rather than- // falling back to `authKind: none` semantics.- authKind: "oauth",- config: nextConfig,- transportConfig: nextConfig,- updatedAt: new Date(),- })- .where(eq(toolConnections.id, connection.id));- }- throw unprocessable("This app needs you to sign in.", {- code: "oauth_challenge",- status: response.status,- setupUrl: connectionSetupUrl(connection),- reconnectUrl: connectionReconnectUrl(connection),- oauthSupported: Boolean(endpoints),- });- }- throw mcpDiscoveryHttpFailure(response, `Remote app returned HTTP ${response.status}`);- }- const descriptors: McpToolDescriptor[] = [];- const seenCursors = new Set<string>();- for (let page = 0; ; page += 1) {- const payload = await withMcpConnectionFailure(() => readMcpHttpResponse(response, listRequestId));- const record = asRecord(payload);- if (record.error) throw new HttpError(502, "Remote MCP tool discovery failed", { code: "mcp_catalog_error" });- const result = asRecord(record.result);- if (!Array.isArray(result.tools)) throw new HttpError(502, "Remote MCP returned an invalid tool catalog", { code: "mcp_catalog_invalid" });- descriptors.push(...result.tools.map((tool) => normalizeToolDescriptor(tool)).filter((tool): tool is McpToolDescriptor => Boolean(tool)));- const cursor = typeof result.nextCursor === "string" ? result.nextCursor : null;- if (!cursor) break;- if (seenCursors.has(cursor) || page >= 99 || descriptors.length > 20_000) throw new HttpError(502, "Remote MCP tool catalog pagination did not finish", { code: "mcp_catalog_pagination" });- seenCursors.add(cursor);- listRequestId = `paperclip-catalog-refresh-${page + 1}`;- response = await sendToolsList(sessionHeaders, cursor);- if (!response.ok) throw mcpDiscoveryHttpFailure(response, "Remote MCP catalog page could not be read");- }- if (!isRailwayConnection(connection)) return descriptors;- if (descriptors.some((tool) => normalizeRailwayToolName(tool.name).startsWith(RAILWAY_TOOL_PREFIX))) {- throw unprocessable("Railway advertised a reserved Paperclip action name. Refresh is blocked pending review.", { code: "railway_tool_name_collision" });- }- let apiStatus = "available";- let apiMessage = "Direct Railway service, log, and deployment tools are available.";- try {- const railwayOptions = {- authorization: headers.Authorization ?? "",- request: (url: string, init: RequestInit) => requestRemoteHttpEndpoint(new URL(url), init),- signal: AbortSignal.timeout(15_000),- };- const workspaceId = await discoverRailwayWorkspace(railwayOptions);- await createRailwayClient(railwayOptions).probe(workspaceId);- } catch (error) {- apiStatus = "unavailable";- apiMessage = error instanceof RailwayError ? error.message : "Railway API access could not be verified. Refresh actions or reconnect Railway.";- }- // API interoperability is verified with the actual credential; the presence- // of an OAuth token alone never enables the additional capability surface.- const nextConfig = { ...connection.config, railwayApiStatus: apiStatus, railwayApiMessage: apiMessage };- await db.update(toolConnections).set({ config: nextConfig, transportConfig: nextConfig, updatedAt: now() }).where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId)));- connection.config = nextConfig;- connection.transportConfig = nextConfig;- return apiStatus === "available" ? [...descriptors, ...RAILWAY_TOOLS] : descriptors;- }-- async function validateCogneeConnection(connection: typeof toolConnections.$inferSelect, actor?: ActorInfo, probe = true) {- if (connection.config.templateId !== "paperclip.cognee-cloud") return;- const grant = await vaultGrantForConnection(connection, actor);- const refs = grant?.credentialSecretRefs ?? connection.credentialSecretRefs;- const values: Record<string, string> = {};- for (const key of COGNEE_STDIO_TEMPLATE.envKeys) {- const ref = refs.find((candidate) => candidate.configPath === `env.${key}`);- if (!ref) throw unprocessable("Reconnect Cognee to restore its Cloud credentials", { code: "missing_secret" });- values[key] = grant- ? (await resolveOAuthGrantSecret(connection, grant, ref, actor, undefined)).value- : await secrets.resolveSecretValue(connection.companyId, ref.secretId, ref.versionSelector ?? "latest", {- consumerType: "tool_connection", consumerId: connection.id, configPath: ref.configPath,- actorType: "system", actorId: null,- });- }- let base: URL;- try { base = cogneeCloudUrl(values.COGNEE_BASE_URL!); }- catch { throw badRequest("Copy the tenant API Base URL from Cognee’s API Keys page."); }- // Scheduled health checks validate configuration and vault access. A transient- // Cloud probe must not withdraw an already assigned local tool catalog.- if (!probe) return;- const response = await requestRemoteHttpEndpoint(new URL("/api/v1/datasets/", base), {- method: "GET", headers: { "X-Api-Key": values.COGNEE_API_KEY! }, signal: AbortSignal.timeout(15_000),- });- await response.body?.cancel();- if (!response.ok) throw unprocessable("Cognee Cloud could not verify access. Check the tenant URL, API key, and workspace subscription.", { code: "cognee_access_unverified" });- }-- async function localTools(- connection: typeof toolConnections.$inferSelect,- ): Promise<McpToolDescriptor[]> {- const template = await resolveStdioTemplate(- connection.companyId,- connection.config,- );- return template.tools.map((tool) => ({- name: tool.name,- title: tool.title ?? null,- description: tool.description ?? null,- inputSchema: tool.inputSchema ?? { type: "object", properties: {} },- annotations: tool.annotations ?? {},- }));- }-- function isAgentMailConnection(connection: typeof toolConnections.$inferSelect) {- return connection.transport === "rest_api" && connection.config.provider === "agentmail";- }-- async function validateAgentMailConnection(- connection: typeof toolConnections.$inferSelect,- ) {- const configPath = connection.config.emailCredential- ? "credentials.controlKey"- : "credentials.apiKey";- const ref = connection.credentialSecretRefs.find(- (candidate) => candidate.configPath === configPath,- );- if (!ref) {- throw unprocessable("Reconnect AgentMail to restore its API key", {- code: "missing_secret",- });- }- const key = await secrets.resolveSecretValue(- connection.companyId,- ref.secretId,- ref.versionSelector ?? "latest",- {- consumerType: "tool_connection",- consumerId: connection.id,- configPath,- actorType: "system",- actorId: null,- },- );- await agentmailApi(key).whoami();- }-- async function discoverTools(- connection: typeof toolConnections.$inferSelect,- credentialHeaders?: Record<string, string>,- actor?: ActorInfo,- unavailableServices: string[] = [],- ): Promise<McpToolDescriptor[]> {- assertSupportedConnection(connection);- await assertConnectorConnectionSetupEnabled(connection);- if (connection.connectionPurpose === "ai") throw unprocessable("AI connections provide runtime authentication, not tool actions");- if (isDaytonaConnection(connection)) {- const headers = credentialHeaders ?? await resolveCredentialHeaders(connection, actor);- await daytonaClient(headers, (url, init) => requestRemoteHttpEndpoint(new URL(url), init), asRecord(connection.config.methodConfig).organizationId).probe();- return DAYTONA_TOOLS;- }- if (isBrowserUseConnection(connection)) {- const headers = credentialHeaders ?? await resolveCredentialHeaders(connection, actor);- await browserUseClient(headers, (url, init) => requestRemoteHttpEndpoint(new URL(url), init), connection.id).probe();- return BROWSER_USE_TOOLS;- }- if (isAgentMailConnection(connection)) {- await validateAgentMailConnection(connection);- return [];- }- if (connection.transport === "mcp_remote")- return remoteTools(connection, credentialHeaders, actor, undefined, unavailableServices);- if (connection.transport !== "local_stdio") {- throw unsupportedToolConnectionTransport();- }- await resolveCredentialHeaders(connection);- await validateCogneeConnection(connection, actor);- return localTools(connection);- }-- async function annotateAiGrantHealth(connections: ToolConnection[]) {- const aiConnections = connections.filter(connection => connection.connectionPurpose === "ai");- if (!aiConnections.length) return;- const grants = await db.select({ connectionId: connectionGrants.connectionId, status: connectionGrants.status })- .from(connectionGrants).where(and(eq(connectionGrants.companyId, aiConnections[0].companyId),- inArray(connectionGrants.connectionId, aiConnections.map(connection => connection.id))));- for (const connection of aiConnections) {- const identities = grants.filter(grant => grant.connectionId === connection.id);- if (identities.length && identities.every(grant => grant.status === "revoked")) {- connection.healthStatus = "missing_secret";- connection.healthMessage = "This credential was revoked. Reconnect the account to restore access.";- connection.requiresReauthorization = true;- }- }- }-- async function annotateGitHubAuthorization(- connections: ToolConnection[],- viewerUserId?: string,- ) {- const github = connections.filter(- (connection) =>- asRecord(connection.config).sourceTemplateKey === "github",- );- if (!github.length) return;- const grants = await db- .select({- connectionId: connectionGrants.connectionId,- status: connectionGrants.status,- kind: connectionGrants.kind,- subjectUserId: connectionGrants.subjectUserId,- })- .from(connectionGrants)- .where(- and(- eq(connectionGrants.companyId, github[0].companyId),- inArray(- connectionGrants.connectionId,- github.map((connection) => connection.id),- ),- ),- );- for (const connection of github) {- const userId = viewerUserId ?? connection.createdByUserId;- const eligible = grants.filter(- (grant) =>- grant.connectionId === connection.id &&- (connection.credentialPolicy !== "per_user" ||- (grant.kind === "user" && grant.subjectUserId === userId)),- );- connection.requiresReauthorization = !eligible.some(- (grant) => grant.status === "active",- );- }- }-- async function updateConnectionHealth(- connection: typeof toolConnections.$inferSelect,- status: ToolConnectionHealthStatus,- message: string | null,- ) {- const now = new Date();- const [updated] = await db- .update(toolConnections)- .set({- healthStatus: status,- healthMessage: message,- healthCheckedAt: now,- lastHealthAt: now,- lastError: status === "ok" ? null : message,- updatedAt: now,- })- .where(eq(toolConnections.id, connection.id))- .returning();- if (connection.transport === "local_stdio") {- await db- .update(toolRuntimeSlots)- .set({- healthStatus: status,- healthMessage: message,- lastHealthCheckAt: now,- updatedAt: now,- })- .where(eq(toolRuntimeSlots.connectionId, connection.id));- }- return updated;- }-- async function checkConnectionHealth(- connectionId: string,- actor?: ActorInfo,- options: { allowUnauthenticatedProbe?: boolean } = {},- ): Promise<ToolConnectionHealthCheckResult> {- const connection = await getConnectionRow(connectionId);- if (connection.connectionPurpose === "ai") return { connection: toConnection(connection), runtimeSlot: null };- await assertConnectorConnectionSetupEnabled(connection);- const unavailableServices: string[] = [];- try {- assertSupportedConnection(connection);- const config = asRecord(connection.config);- const oauth = asRecord(config.oauth);- if (- config.sourceTemplateKey === "github" &&- oauth.connectorProfile === "github.code"- ) {- const activeGrants = await db- .select()- .from(connectionGrants)- .where(- and(- eq(connectionGrants.companyId, connection.companyId),- eq(connectionGrants.connectionId, connection.id),- eq(connectionGrants.status, "active"),- ),- );- const actorGrant =- actor?.actorType === "user"- ? activeGrants.find(- (grant) =>- grant.kind === "user" &&- grant.subjectUserId === actor.actorId,- )- : null;- const grantsToCheck =- connection.credentialPolicy === "per_user" &&- actor?.actorType === "user"- ? actorGrant- ? [actorGrant]- : []- : actorGrant- ? [actorGrant]- : activeGrants;- if (grantsToCheck.length === 0)- throw unprocessable("GitHub authorization must be connected", {- code: "oauth_reauthorization_required",- });- for (const grant of grantsToCheck)- await refreshManagedGitHubGrantAccess(connection, grant, actor);- } else if (isBrowserUseConnection(connection) || isDaytonaConnection(connection)) {- await discoverTools(connection, undefined, actor);- } else if (isAgentMailConnection(connection)) {- await validateAgentMailConnection(connection);- } else if (connection.transport === "mcp_remote") {- const canProbeWithoutAuthorization =- options.allowUnauthenticatedProbe === true &&- connection.status === "draft" &&- connection.authKind === "none" &&- connection.credentialPolicy === "per_user" &&- actor?.actorType === "user" &&- actor.actorId === connection.createdByUserId;- const credentialHeaders =- connection.credentialSource === "vercel_connect"- ? await resolveCredentialHeaders(connection, actor, {- forceRefresh: true,- })- : canProbeWithoutAuthorization- ? {}- : undefined;- await remoteTools(connection, credentialHeaders, actor, undefined, unavailableServices);- } else if (connection.transport === "local_stdio") {- await resolveCredentialHeaders(connection);- await stdioTemplateId(connection.companyId, connection.config);- await validateCogneeConnection(connection, actor, false);- } else {- throw unsupportedToolConnectionTransport();- }- const updated = await updateConnectionHealth(- connection,- "ok",- unavailableServices.length > 0- ? `Connected. Google services unavailable: ${unavailableServices.join(", ")}. Other services remain available; retry refreshing actions.`- : config.sourceTemplateKey === "github" &&- oauth.connectorProfile === "github.code"- ? "GitHub account, installation, and repository access are available."- : isDaytonaConnection(connection)- ? "Daytona API access is connected."- : isBrowserUseConnection(connection)- ? "Browser Use API key is connected."- : isAgentMailConnection(connection)- ? "AgentMail API key is connected."- : connection.transport === "local_stdio"- ? "Approved stdio template is ready."- : "Remote MCP server responded to tools/list.",- );- const runtimeSlot = await ensureRuntimeSlot(updated);- await audit({- companyId: connection.companyId,- connectionId: connection.id,- action: "tool_connection.health_check",- outcome: "success",- actor,- details: { transport: connection.transport },- });- return { connection: toConnection(updated), runtimeSlot };- } catch (error) {- if (- error instanceof HttpError &&- asRecord(error.details).code === "github_access_changed"- )- throw error;- const failure = sanitizeHttpFailure(error);- const updated = await updateConnectionHealth(- connection,- failure.status,- failure.message,- );- const runtimeSlot =- connection.transport === "local_stdio"- ? await ensureRuntimeSlot(updated)- : null;- await audit({- companyId: connection.companyId,- connectionId: connection.id,- action: "tool_connection.health_check",- outcome: "failure",- reasonCode: failure.code,- actor,- details: { status: failure.status, transport: connection.transport },- });- throw retainMcpConnectionFailure(error, new HttpError(healthFailureHttpStatus(failure), failure.message, {- code: failure.code,- connection: toConnection(updated),- runtimeSlot,- setupUrl: connectionSetupUrl(connection),- reconnectUrl: connectionReconnectUrl(connection),- }));- }- }-- async function refreshCatalog(- connectionId: string,- actor?: ActorInfo,- refreshOptions: {- enableAllByDefault?: boolean;- restoreDraftDefaults?: boolean;- /** Keep first managed OAuth discovery quarantined without changing generic draft refresh semantics. */- quarantineManagedOAuthDraft?: boolean;- skipDefaultProfileSync?: boolean;- credentialHeaders?: Record<string, string>;- } = {},- ): Promise<ToolCatalogRefreshResult> {- const connection = await getConnectionRow(connectionId);- if (connection.connectionPurpose === "ai") throw unprocessable("AI connections do not have a tool catalog");- await assertConnectorConnectionSetupEnabled(connection);- const refreshedAt = now();- const unavailableServices: string[] = [];- let descriptors: McpToolDescriptor[];- try {- descriptors = await discoverTools(- connection,- refreshOptions.credentialHeaders,- actor,- unavailableServices,- );- } catch (error) {- if (- error instanceof HttpError &&- asRecord(error.details).code === "github_access_changed"- )- throw error;- const failure = sanitizeHttpFailure(error);- const updated = await updateConnectionHealth(- connection,- failure.status,- failure.message,- );- await audit({- companyId: connection.companyId,- connectionId: connection.id,- action: "tool_connection.catalog_refresh",- outcome: "failure",- reasonCode: failure.code,- details: { status: failure.status },- actor,- });- throw retainMcpConnectionFailure(error, new HttpError(healthFailureHttpStatus(failure), failure.message, {- code: failure.code,- setupUrl: connectionSetupUrl(connection),- reconnectUrl: connectionReconnectUrl(connection),- }));- }-- const publish = <T>(write: (client: ToolAccessMutationDb) => Promise<T>): Promise<T> => db.transaction(async tx => {- await tx.select({ id: toolApplications.id }).from(toolApplications)- .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, connection.companyId)))- .for("update");- await tx.select({ id: toolConnections.id }).from(toolConnections)- .where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId)))- .for("update");- return write(tx);- });- descriptors = [...new Map(descriptors.map(descriptor => [descriptor.name, descriptor])).values()];- const existingRows = await db- .select()- .from(toolCatalogEntries)- .where(eq(toolCatalogEntries.connectionId, connection.id));- const existingByName = new Map(- existingRows.map((entry) => [entry.toolName, entry]),- );- const nativeDaytona = isDaytonaConnection(connection);- if (connection.config.sourceTemplateKey === "google-workspace"- || nativeDaytona || isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey)) {- const discoveredNames = new Set(descriptors.map((descriptor) => descriptor.name));- // The catalog is shared, but discovery uses only the caller's grant. A- // narrower personal consent must not remove actions another active grant- // still supports. Never combine scopes from different grants into authority.- let otherWorkspaceGrants: string[][] = [];- let selectedWorkspaceScopes: string[] = [];- if (connection.config.sourceTemplateKey === "google-workspace") {- const selected = await vaultGrantForConnection(connection, actor);- selectedWorkspaceScopes = googleWorkspaceGrantedScopes(selected);- const grants = await db.select().from(connectionGrants).where(and(- eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id),- eq(connectionGrants.status, "active"),- ));- otherWorkspaceGrants = grants.filter((grant) => grant.id !== selected?.id).map(googleWorkspaceGrantedScopes);- }- // Preserve only consent-filtered actions, not failed-service catalogs or- // tools a successful, sufficiently authorized discovery actually removed.- const removedStatus = nativeDaytona ? "removed" : "disabled";- const removedIds = existingRows.filter((entry) => !discoveredNames.has(entry.toolName) && entry.status !== removedStatus- && !(otherWorkspaceGrants.some((scopes) => isGoogleWorkspaceToolGranted(entry.toolName, scopes))- && !isGoogleWorkspaceToolGranted(entry.toolName, selectedWorkspaceScopes)- && !unavailableServices.includes(googleWorkspaceToolTarget(entry.toolName)?.service ?? ""))).map((entry) => entry.id);- if (removedIds.length) await publish(tx => tx.update(toolCatalogEntries)- .set({ status: removedStatus, quarantineReason: nativeDaytona ? "provider_tool_removed" : "mcp_tool_removed", updatedAt: refreshedAt })- .where(and(eq(toolCatalogEntries.companyId, connection.companyId), eq(toolCatalogEntries.connectionId, connection.id), inArray(toolCatalogEntries.id, removedIds))));- }- // Retired native actions are absent from discovery, but old catalog rows- // still need to show as disabled. Gateway denial also applies before refresh.- const blockedRailwayEntryIds = isRailwayEndpoint(connection.config.url)- ? existingRows.filter((entry) => isRailwayToolBlocked(entry.toolName)).map((entry) => entry.id)- : [];- if (blockedRailwayEntryIds.length > 0) {- await publish(tx => tx.update(toolCatalogEntries).set({ status: "disabled", updatedAt: refreshedAt })- .where(and(eq(toolCatalogEntries.connectionId, connection.id), inArray(toolCatalogEntries.id, blockedRailwayEntryIds))));- }- const catalogValues: Array<typeof toolCatalogEntries.$inferInsert> = [];- let quarantinedCount = 0;- const sourceTemplateKey =- typeof asRecord(connection.config).sourceTemplateKey === "string"- ? String(asRecord(connection.config).sourceTemplateKey)- : null;- const sourceApp = sourceTemplateKey- ? getConnectableAppDefinition(sourceTemplateKey)- : null;- const sourceMethod = sourceApp- ? connectionMethodForConnection(sourceApp, connection)- : null;- const sourceCapabilityKey = sourceMethod?.capabilityProfile?.key;- const googleProfileValue =- sourceMethod?.connectorProfile ??- sourceApp?.methods.find(- (candidate) =>- candidate.connectorProfile &&- candidate.capabilityProfile?.key === sourceCapabilityKey,- )?.connectorProfile;- const googleProfile =- googleProfileValue &&- isGoogleWorkspaceConnectorProfileId(googleProfileValue)- ? googleProfileValue- : null;- const connectorPolicy = connectorToolPolicy(sourceTemplateKey, connection);- const preserveReviewedCatalog = connectorPolicy.quarantineNewEntries && existingRows.length > 0;- const quarantineOnRefresh =- (!refreshOptions.enableAllByDefault || preserveReviewedCatalog) &&- shouldQuarantineNewEntries(connection) &&- (connection.status === "active" ||- preserveReviewedCatalog ||- sourceTemplateKey === "posthog" ||- refreshOptions.quarantineManagedOAuthDraft === true);- const safeDefault = asRecord(connection.config).safeDefault === true;- for (const descriptor of descriptors) {- const riskLevel = classifyRisk(descriptor, connectorPolicy.sourceTemplateKey);- const hash = descriptorHash(descriptor, riskLevel);- const schemaHash = stableHash(descriptor.inputSchema ?? {});- const existing = existingByName.get(descriptor.name);- const changed =- existing &&- (existing.versionHash !== hash || existing.schemaHash !== schemaHash);- const shouldQuarantine =- quarantineOnRefresh &&- (!existing || changed) &&- existing?.status !== "disabled" &&- (!safeDefault || riskLevel !== "read" || nativeDaytona);- const providerPermanentlyBlocked = Boolean(- googleProfile &&- !isGoogleWorkspaceToolAllowed(googleProfile, descriptor),- ) || (isRailwayEndpoint(connection.config.url) && isRailwayToolBlocked(descriptor.name));- const status = providerPermanentlyBlocked- ? "disabled"- : shouldQuarantine- ? "quarantined"- : existing?.status === "disabled" && existing.quarantineReason !== "mcp_tool_removed"- ? "disabled"- : quarantineOnRefresh && existing?.status === "quarantined"- ? "quarantined"- : "active";- if (shouldQuarantine && !providerPermanentlyBlocked) quarantinedCount += 1;-- catalogValues.push({- id: existing?.id, companyId: connection.companyId, applicationId: connection.applicationId,- connectionId: connection.id, name: existing?.name ?? descriptor.name, toolName: descriptor.name, entryKind: "tool",- title: descriptor.title ?? null, description: descriptor.description ?? null,- inputSchema: descriptor.inputSchema ?? {}, annotations: descriptor.annotations ?? {}, riskLevel,- isReadOnly: riskLevel === "read", isWrite: riskLevel === "write", isDestructive: riskLevel === "destructive",- status, versionHash: hash, schemaHash, firstSeenAt: existing?.firstSeenAt ?? refreshedAt, lastSeenAt: refreshedAt,- quarantinedAt: existing- ? status === "quarantined" ? shouldQuarantine ? refreshedAt : existing.quarantinedAt : null- : shouldQuarantine ? refreshedAt : null,- quarantineReason: existing- ? status === "quarantined" ? shouldQuarantine ? "pending_review" : existing.quarantineReason : null- : shouldQuarantine ? "pending_review" : null,- updatedAt: existing ? refreshedAt : new Date(),- });- }- const catalogRows: Array<typeof toolCatalogEntries.$inferSelect> = [];- const newCatalogEntryIds = new Set<string>();- for (let index = 0; index < catalogValues.length; index += 500) {- const chunk = catalogValues.slice(index, index + 500);- catalogRows.push(...await publish(async tx => {- const current = await tx.select({ name: toolCatalogEntries.name, status: toolCatalogEntries.status })- .from(toolCatalogEntries).where(and(eq(toolCatalogEntries.connectionId, connection.id), inArray(toolCatalogEntries.name, chunk.map(entry => entry.name))));- const existingNonQuarantinedNames = new Set(current.filter(entry => entry.status !== "quarantined").map(entry => entry.name));- const rows = await tx.insert(toolCatalogEntries).values(chunk)- .onConflictDoUpdate({ target: [toolCatalogEntries.connectionId, toolCatalogEntries.name], set: {- title: sql`excluded.title`, description: sql`excluded.description`, inputSchema: sql`excluded.input_schema`,- annotations: sql`excluded.annotations`, riskLevel: sql`excluded.risk_level`, isReadOnly: sql`excluded.is_read_only`,- isWrite: sql`excluded.is_write`, isDestructive: sql`excluded.is_destructive`,- status: sql`CASE WHEN ${toolCatalogEntries.status} = 'disabled' AND ${toolCatalogEntries.quarantineReason} IS DISTINCT FROM 'mcp_tool_removed' THEN 'disabled' ELSE excluded.status END`,- versionHash: sql`excluded.version_hash`, schemaHash: sql`excluded.schema_hash`, lastSeenAt: sql`excluded.last_seen_at`,- quarantinedAt: sql`excluded.quarantined_at`, quarantineReason: sql`excluded.quarantine_reason`, updatedAt: sql`excluded.updated_at`,- } }).returning();- for (const row of rows) if (!existingNonQuarantinedNames.has(row.name)) newCatalogEntryIds.add(row.id);- return rows;- }));+ const removed = await secrets.remove(secretId);+ if (removed) secretsRevoked += 1;}- const catalogByName = new Map(catalogRows.map(row => [row.toolName, row]));- const updatedEntries = descriptors.map(descriptor => toCatalogEntry(catalogByName.get(descriptor.name)!));-- const preserveQuarantine = connectorPolicy.quarantineNewEntries;- const normalizedConfig = preserveQuarantine- ? { ...connection.config, quarantineNewEntries: true }- : refreshOptions.enableAllByDefault- ? { ...connection.config, quarantineNewEntries: false }- : connection.config;- const normalizedTransportConfig = preserveQuarantine- ? { ...connection.transportConfig, quarantineNewEntries: true }- : refreshOptions.enableAllByDefault- ? { ...connection.transportConfig, quarantineNewEntries: false }- : connection.transportConfig;- const [updatedConnection] = await db++ const credentialRefsCleared =+ connection.credentialRefs.length + connection.credentialSecretRefs.length;+ const [cleared] = await db.update(toolConnections).set({- config: normalizedConfig,- transportConfig: normalizedTransportConfig,- healthStatus: "ok",- healthMessage: unavailableServices.length > 0- ? `Actions refreshed. Google services unavailable: ${unavailableServices.join(", ")}. Other services remain available; retry refreshing actions.`- : isAgentMailConnection(connection)- ? "AgentMail API key is connected."- : "Tool catalog refreshed.",- healthCheckedAt: refreshedAt,- lastHealthAt: refreshedAt,- lastCatalogRefreshAt: refreshedAt,- lastError: null,- updatedAt: refreshedAt,+ credentialRefs: [],+ credentialSecretRefs: [],+ config: withoutInlineOAuthTokens(connection.config),+ transportConfig: withoutInlineOAuthTokens(connection.transportConfig),+ updatedAt: now,}).where(eq(toolConnections.id, connection.id)).returning();-- if (connection.transport === "local_stdio") {- await ensureRuntimeSlot(updatedConnection);+ if (+ grantRows.some((grant) => (grant.credentialSecretRefs ?? []).length > 0)+ ) {await db- .update(toolRuntimeSlots)- .set({- healthStatus: "ok",- healthMessage: "Approved stdio template is ready.",- lastHealthCheckAt: refreshedAt,- updatedAt: refreshedAt,- })- .where(eq(toolRuntimeSlots.connectionId, connection.id));- }-- const activeEntries = updatedEntries.filter(- (entry) => entry.status === "active",- );- const preserveMcpAccess = connection.config.mcpPreserveAccess === true- && isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey);- if (!refreshOptions.skipDefaultProfileSync || preserveMcpAccess) {- await publish(tx => enableCatalogEntriesByDefault({- connection: updatedConnection,- // Discovery must not re-enable actions the operator turned Off,- // including curated MCP connections during API-key replacement.- newCatalogEntryIds: activeEntries- .filter((entry) => newCatalogEntryIds.has(entry.id))- .map((entry) => entry.id),- activeCatalogEntryIds: activeEntries.map((entry) => entry.id),- restoreDraftDefaults: refreshOptions.restoreDraftDefaults || preserveMcpAccess,- actor,- }, tx));+ .update(connectionGrants)+ .set({ credentialSecretRefs: [], updatedAt: now })+ .where(+ and(+ eq(connectionGrants.companyId, connection.companyId),+ eq(connectionGrants.connectionId, connection.id),+ ),+ );}- await audit({- companyId: connection.companyId,- connectionId: connection.id,- action: "tool_connection.catalog_refresh",- outcome: "success",- details: { discoveredCount: descriptors.length, quarantinedCount },- actor,- });return {- connection: toConnection(updatedConnection),- catalog: updatedEntries,- discoveredCount: descriptors.length,- quarantinedCount,+ connection: toConnection(cleared ?? archived.connection),+ removal: {+ secretsRevoked,+ secretsRetainedShared: retained.length,+ credentialRefsCleared,+ secretBindingsRemoved: removedSecretBindings.length,+ grantsRevoked: grantsToRevoke.length,+ installsRemoved: removedInstalls.length,+ appProfile: appProfileOutcome,+ appProfileEntriesRemoved,+ appProfileBindingsRemoved,+ catalogEntriesMarkedRemoved: removedCatalogEntries.length,+ oauthStatesDiscarded: discardedOAuthStates.length,+ tokenIssuanceHashesCleared: clearedIssuanceHashes.length,+ runtimeSlotsStopped,+ gatewayTokensRevoked,+ gatewaySessionsRevoked,+ applicationArchived: archived.applicationArchived,+ externalCredentialCleanup,+ },};}- async function listAppsNeedingAttention(- companyId: string,- ): Promise<ToolAppsAttentionResponse> {- const generatedAt = now();- const [- connections,- quarantinedEntries,- pendingActionRequests,- invocations,- profiles,- profileEntries,- activeCatalog,- ] = await Promise.all([- db- .select()- .from(toolConnections)- .where(- and(- eq(toolConnections.companyId, companyId),- ne(toolConnections.status, "archived"),- ),- ),- db- .select({ connectionId: toolCatalogEntries.connectionId })- .from(toolCatalogEntries)- .where(- and(- eq(toolCatalogEntries.companyId, companyId),- eq(toolCatalogEntries.status, "quarantined"),- ),- ),- db- .select({ invocationId: toolActionRequests.invocationId })- .from(toolActionRequests)- .where(- and(- eq(toolActionRequests.companyId, companyId),- eq(toolActionRequests.status, "pending"),- isNotNull(toolActionRequests.signedArguments),- ),- ),- db- .select({ id: toolInvocations.id, connectionId: toolInvocations.connectionId })- .from(toolInvocations)- .where(eq(toolInvocations.companyId, companyId)),- db- .select()- .from(toolProfiles)- .where(eq(toolProfiles.companyId, companyId)),- db- .select()- .from(toolProfileEntries)- .where(eq(toolProfileEntries.companyId, companyId)),- db- .select(profileCatalogColumns)- .from(toolCatalogEntries)- .where(- and(- eq(toolCatalogEntries.companyId, companyId),- eq(toolCatalogEntries.status, "active"),- ),- ),- ]);- const quarantinedCountByConnection = new Map<string, number>();- for (const entry of quarantinedEntries) {- quarantinedCountByConnection.set(- entry.connectionId,- (quarantinedCountByConnection.get(entry.connectionId) ?? 0) + 1,- );- }- const invocationConnectionById = new Map(- invocations.map((invocation) => [invocation.id, invocation.connectionId]),- );- const pendingActionRequestCountByConnection = new Map<string, number>();- for (const request of pendingActionRequests) {- const connectionId = invocationConnectionById.get(request.invocationId);- if (!connectionId) continue;- pendingActionRequestCountByConnection.set(- connectionId,- (pendingActionRequestCountByConnection.get(connectionId) ?? 0) + 1,- );- }- const entriesByProfile = new Map<- string,- Array<typeof toolProfileEntries.$inferSelect>- >();- for (const entry of profileEntries) {- const list = entriesByProfile.get(entry.profileId) ?? [];- list.push(entry);- entriesByProfile.set(entry.profileId, list);- }- const connectionsById = new Map(- connections.map((connection) => [connection.id, connection]),+ async function localTools(+ connection: typeof toolConnections.$inferSelect,+ ): Promise<McpToolDescriptor[]> {+ const template = await resolveStdioTemplate(+ connection.companyId,+ connection.config,);- const pendingProfilesByConnection = new Map<- string,- Map<- string,- { profileId: string; profileName: string; pendingCount: number }- >- >();- for (const profile of profiles) {- const tools = pendingNewToolsForProfile({- profile,- entries: entriesByProfile.get(profile.id) ?? [],- catalog: activeCatalog,- connectionsById,- });- for (const tool of tools) {- const profileCounts =- pendingProfilesByConnection.get(tool.connectionId) ?? new Map();- const existing = profileCounts.get(profile.id) ?? {- profileId: profile.id,- profileName: profile.name,- pendingCount: 0,- };- existing.pendingCount += 1;- profileCounts.set(profile.id, existing);- pendingProfilesByConnection.set(tool.connectionId, profileCounts);+ return template.tools.map((tool) => ({+ name: tool.name,+ title: tool.title ?? null,+ description: tool.description ?? null,+ inputSchema: tool.inputSchema ?? { type: "object", properties: {} },+ annotations: tool.annotations ?? {},+ }));+ }++ async function annotateAiGrantHealth(connections: ToolConnection[]) {+ const aiConnections = connections.filter(connection => connection.connectionPurpose === "ai");+ if (!aiConnections.length) return;+ const grants = await db.select({ connectionId: connectionGrants.connectionId, status: connectionGrants.status })+ .from(connectionGrants).where(and(eq(connectionGrants.companyId, aiConnections[0].companyId),+ inArray(connectionGrants.connectionId, aiConnections.map(connection => connection.id))));+ for (const connection of aiConnections) {+ const identities = grants.filter(grant => grant.connectionId === connection.id);+ if (identities.length && identities.every(grant => grant.status === "revoked")) {+ connection.healthStatus = "missing_secret";+ connection.healthMessage = "This credential was revoked. Reconnect the account to restore access.";+ connection.requiresReauthorization = true;}}- const apps = connections.flatMap((connection) => {- const healthNeedsAttention = isAttentionHealthStatus(- connection.healthStatus,- );- const quarantinedCatalogEntryCount =- quarantinedCountByConnection.get(connection.id) ?? 0;- const pendingActionRequestCount =- pendingActionRequestCountByConnection.get(connection.id) ?? 0;- const newToolsPendingProfiles = [- ...(pendingProfilesByConnection.get(connection.id)?.values() ?? []),- ].sort(- (a, b) =>- b.pendingCount - a.pendingCount ||- a.profileName.localeCompare(b.profileName),- );- const newToolsPendingReviewCount = newToolsPendingProfiles.reduce(- (sum, profile) => sum + profile.pendingCount,- 0,- );- const reasons = [- ...(healthNeedsAttention ? ["health" as const] : []),- ...(quarantinedCatalogEntryCount > 0- ? ["quarantined_catalog_entries" as const]- : []),- ...(pendingActionRequestCount > 0- ? ["pending_action_requests" as const]- : []),- ...(newToolsPendingReviewCount > 0- ? ["profile_new_tools" as const]- : []),- ];- return reasons.length > 0- ? [- {- connection: toConnection(connection),- healthNeedsAttention,- quarantinedCatalogEntryCount,- pendingActionRequestCount,- newToolsPendingReviewCount,- newToolsPendingProfiles,- reasons,- },- ]- : [];- });- return {- generatedAt,- apps,- totals: {- connections: apps.length,- health: apps.filter((app) => app.healthNeedsAttention).length,- quarantinedCatalogEntries: apps.reduce(- (sum, app) => sum + app.quarantinedCatalogEntryCount,- 0,- ),- pendingActionRequests: apps.reduce(- (sum, app) => sum + app.pendingActionRequestCount,- 0,- ),- newToolsPendingReview: apps.reduce(- (sum, app) => sum + app.newToolsPendingReviewCount,- 0,- ),- newToolsPendingProfiles: apps.reduce(- (sum, app) => sum + app.newToolsPendingProfiles.length,- 0,- ),- },- };}- async function sweepConnectionHealth(- input: { staleAfterMs?: number; limit?: number } = {},+ async function annotateGitHubAuthorization(+ connections: ToolConnection[],+ viewerUserId?: string,) {- const generatedAt = now();- const staleAfterMs = input.staleAfterMs ?? 15 * 60 * 1000;- const limit = input.limit ?? 25;- const cutoff = new Date(generatedAt.getTime() - staleAfterMs);- // Legacy plugin backfills use a remote transport as a placeholder, but- // their tools run in the plugin worker and have no remote MCP endpoint.- // Select only due IDs in SQL so each scheduler tick does not decode every- // active connection's config and credential metadata.- const due = await db- .select({ id: toolConnections.id })- .from(toolConnections)- .innerJoin(toolApplications, and(- eq(toolApplications.id, toolConnections.applicationId),- eq(toolApplications.companyId, toolConnections.companyId),- ))+ const github = connections.filter(+ (connection) =>+ asRecord(connection.config).sourceTemplateKey === "github",+ );+ if (!github.length) return;+ const grants = await db+ .select({+ connectionId: connectionGrants.connectionId,+ status: connectionGrants.status,+ kind: connectionGrants.kind,+ subjectUserId: connectionGrants.subjectUserId,+ })+ .from(connectionGrants).where(and(- eq(toolConnections.enabled, true),- eq(toolConnections.status, "active"),- ne(toolConnections.transport, "chat_sdk"),- ne(toolConnections.transport, "runtime_auth"),- ne(toolApplications.type, "paperclip_plugin"),- or(isNull(toolConnections.healthCheckedAt), lte(toolConnections.healthCheckedAt, cutoff)),+ eq(connectionGrants.companyId, github[0].companyId),+ inArray(+ connectionGrants.connectionId,+ github.map((connection) => connection.id),+ ),),- )- .orderBy(- sql`${toolConnections.healthCheckedAt} asc nulls first`,- asc(toolConnections.createdAt),- asc(toolConnections.id),- )- .limit(limit);- let healthy = 0;- let failed = 0;- const failedConnectionIds: string[] = [];- for (const connection of due) {- try {- await checkConnectionHealth(connection.id, {- actorType: "system",- actorId: "tool_health_sweep",- });- healthy += 1;- } catch {- failed += 1;- failedConnectionIds.push(connection.id);- }+ );+ for (const connection of github) {+ const userId = viewerUserId ?? connection.createdByUserId;+ const eligible = grants.filter(+ (grant) =>+ grant.connectionId === connection.id &&+ (connection.credentialPolicy !== "per_user" ||+ (grant.kind === "user" && grant.subjectUserId === userId)),+ );+ connection.requiresReauthorization = !eligible.some(+ (grant) => grant.status === "active",+ );}- return {- checked: due.length,- healthy,- failed,- failedConnectionIds,- };}function findExample(exampleId: string): ToolExampleDefinition {@@ -4688,14 +2378,6 @@ export function toolAccessService(return fields;}- function policyNameForApp(- connection: typeof toolConnections.$inferSelect,- entry: typeof toolCatalogEntries.$inferSelect,- ) {- const base = `Ask first ${connection.id.slice(0, 8)} ${entry.toolName}`;- return base.length <= 160 ? base : base.slice(0, 160);- }-function nextAvailableConnectionName(requestedName: string,existingNames: readonly string[],@@ -6226,108 +3908,6 @@ export function toolAccessService(}}- async function upsertAskFirstPolicies(- input: {- companyId: string;- connection: typeof toolConnections.$inferSelect;- askFirstEntries: Array<typeof toolCatalogEntries.$inferSelect>;- actor?: ActorInfo;- disableStale?: boolean;- },- dbClient: ToolAccessMutationDb = db,- ): Promise<ToolPolicy[]> {- const existingPolicies = await dbClient- .select()- .from(toolPolicies)- .where(- and(- eq(toolPolicies.companyId, input.companyId),- eq(toolPolicies.policyType, "require_approval"),- ),- );- const managedPolicies = existingPolicies.filter((policy) => {- const config = asRecord(policy.config);- return (- config.source === "app_gallery_finish" &&- config.connectionId === input.connection.id- );- });- const policiesByCatalogEntryId = new Map<- string,- typeof toolPolicies.$inferSelect- >();- for (const policy of managedPolicies) {- const config = asRecord(policy.config);- if (typeof config.catalogEntryId === "string") {- policiesByCatalogEntryId.set(config.catalogEntryId, policy);- }- }- const askFirstIds = new Set(input.askFirstEntries.map((entry) => entry.id));- const results: ToolPolicy[] = [];- for (const entry of input.askFirstEntries) {- const config = {- source: "app_gallery_finish",- connectionId: input.connection.id,- catalogEntryId: entry.id,- };- const existing = policiesByCatalogEntryId.get(entry.id);- if (existing) {- const [updated] = await dbClient- .update(toolPolicies)- .set({- name: policyNameForApp(input.connection, entry),- description: `Ask first before running ${entry.toolName}.`,- enabled: true,- selectors: { catalogEntryId: entry.id },- config,- updatedAt: new Date(),- })- .where(eq(toolPolicies.id, existing.id))- .returning();- results.push(toPolicy(updated));- } else {- const [created] = await dbClient- .insert(toolPolicies)- .values({- companyId: input.companyId,- name: policyNameForApp(input.connection, entry),- description: `Ask first before running ${entry.toolName}.`,- policyType: "require_approval",- priority: 50,- enabled: true,- selectors: { catalogEntryId: entry.id },- config,- createdByAgentId:- input.actor?.actorType === "agent"- ? (input.actor.actorId ?? null)- : null,- createdByUserId:- input.actor?.actorType === "user"- ? (input.actor.actorId ?? null)- : null,- })- .returning();- results.push(toPolicy(created));- }- }- if (input.disableStale !== false) {- const stalePolicies = managedPolicies.filter((policy) => {- const config = asRecord(policy.config);- return (- typeof config.catalogEntryId === "string" &&- !askFirstIds.has(config.catalogEntryId)- );- });- for (const policy of stalePolicies) {- await dbClient- .update(toolPolicies)- .set({ enabled: false, updatedAt: new Date() })- .where(eq(toolPolicies.id, policy.id));- }- }- return results;- }-async function finishGalleryAppConnection(companyId: string,connectionId: string,@@ -7047,100 +4627,7 @@ export function toolAccessService(.returning();return toStdioCommandTemplate(row);},-- async setupComposioApp(connectionId: string, toolkit: string, input: ComposioAppSetupInput, actor: ActorInfo): Promise<ComposioAppSetupResult> {- if (!findComposioCatalogApp(toolkit)) throw notFound("This Composio app is not in the catalog");- const { connection, manage } = await openComposioGateway(connectionId, actor);- let result = composioAppSetupResult(await manage([{ name: toolkit, action: "list" }]), toolkit);- if (input.action === "start" && result.status !== "connected") {- result = composioAppSetupResult(await manage([{ name: toolkit, action: "add" }]), toolkit);- if (result.status === "not_connected") throw unprocessable("Composio did not return a sign-in link for this app. Manage it in Composio to check its setup requirements.");- }- if (input.action === "complete") {- if (result.status !== "connected") throw conflict("Finish connecting this app in Composio, then check again");- const currentConnection = await getConnectionRow(connectionId);- if (currentConnection.status !== "active" || !currentConnection.enabled) {- throw conflict("The gateway is no longer active. Restore it before connecting this app.");- }- }- await audit({ companyId: connection.companyId, connectionId, action: "tool_connection.upstream_app_setup", outcome: "success", actor,- details: { toolkit, action: input.action, status: result.status } });- return result;- },-- listAggregatorApps: async (connectionId: string, actor: ActorInfo) => aggregatorAppsResponse(await getConnectionRow(connectionId), actor),- syncAggregatorApps,- refreshAggregatorApps: async (connectionId: string, requested: string[], actor: ActorInfo) => {- const connection = await getConnectionRow(connectionId);- if (aggregatorProvider(connection) === "composio" && requested.length) {- const { manage } = await openComposioGateway(connectionId, actor);- if (requested.some(toolkit => !findComposioCatalogApp(toolkit))) throw notFound("This Composio app is not in the catalog");- await manage(requested.map(name => ({ name, action: "list" })));- return aggregatorAppsResponse(connection, actor);- }- return syncAggregatorApps(connectionId, true, actor);- },- configureArcadeDiscovery,- syncComposioApps: startComposioAppSync,-- async listComposioApps(connectionId: string, actor: ActorInfo) {- const connection = await getConnectionRow(connectionId);- if (actor.actorType !== "user" || !actor.actorId) throw forbidden("Board access required");- if (connection.config.sourceTemplateKey !== "composio") throw notFound("Composio gateway not found");- return composioAppsResponse(connection, actor);- },-- async refreshComposioApps(connectionId: string, requested: string[], actor: ActorInfo) {- if (requested.some(toolkit => !findComposioCatalogApp(toolkit))) throw notFound("This Composio app is not in the catalog");- const { connection, manage } = await openComposioGateway(connectionId, actor);- const existing = await cachedComposioApps(connection, actor);- // Migrate observations from the original direct setup flow on the first visit.- const history = await db.select({ details: toolAccessAuditEvents.details }).from(toolAccessAuditEvents).where(and(- eq(toolAccessAuditEvents.companyId, connection.companyId), eq(toolAccessAuditEvents.connectionId, connectionId),- eq(toolAccessAuditEvents.actorId, actor.actorId!), eq(toolAccessAuditEvents.action, "tool_connection.upstream_app_setup"),- )).orderBy(desc(toolAccessAuditEvents.createdAt)).limit(128);- const known = [...existing.filter(app => app.accounts.length > 0).map(app => app.toolkit),- ...history.map(row => row.details.toolkit).filter((toolkit): toolkit is string => typeof toolkit === "string" && Boolean(findComposioCatalogApp(toolkit)))];- const toolkits = [...new Set([...requested, ...known])];- for (let start = 0; start < toolkits.length; start += 32) {- await manage(toolkits.slice(start, start + 32).map(name => ({ name, action: "list" })));- }- return composioAppsResponse(connection, actor);- },-- async manageComposioAppAccount(connectionId: string, toolkit: string, input: ComposioAppAccountInput, actor: ActorInfo) {- if (!findComposioCatalogApp(toolkit)) throw notFound("This Composio app is not in the catalog");- const { connection, manage } = await openComposioGateway(connectionId, actor);- if (input.action !== "add") {- const before = composioAppAccounts(await manage([{ name: toolkit, action: "list" }]), toolkit);- if (!before?.some(account => account.id === input.accountId)) throw notFound("This account is no longer available in the selected Composio app");- }- const details = { toolkit, ...(input.action !== "add" ? { accountId: input.accountId } : {}) };- await logActivity(db, { companyId: connection.companyId, actorType: "user", actorId: actor.actorId!,- action: `tool_connection.upstream_account_${input.action}_requested`, entityType: "tool_connection", entityId: connectionId, details });- let payload: unknown;- try {- payload = await manage([{ name: toolkit, action: input.action,- ...(input.action !== "add" ? { account_id: input.accountId } : {}), ...(input.action === "rename" ? { alias: input.alias } : {}) }]);- } catch (error) {- await audit({ companyId: connection.companyId, connectionId, actor, action: `tool_connection.upstream_account_${input.action}`,- outcome: "failure", reasonCode: "provider_unconfirmed", details });- throw error;- }- const result = composioAppSetupResult(payload, toolkit); // Reject provider errors before reporting success.- if (input.action === "add" && !result.authorizationUrl) throw unprocessable("Composio did not return a sign-in link. Check the app in Composio.");- if (input.action !== "add") {- const after = composioAppAccounts(await manage([{ name: toolkit, action: "list" }]), toolkit);- if (!after || (input.action === "remove" ? after.some(account => account.id === input.accountId)- : !after.some(account => account.id === input.accountId && account.alias === input.alias))) {- throw conflict("Composio has not confirmed this change. Refresh the accounts before trying again.");- }- }- await logActivity(db, { companyId: connection.companyId, actorType: "user", actorId: actor.actorId!,- action: `tool_connection.upstream_account_${input.action}`, entityType: "tool_connection", entityId: connectionId,- details: { toolkit, ...(input.action !== "add" ? { accountId: input.accountId } : {}) } });- return { ...result, apps: await cachedComposioApps(connection, actor) };- },+ ...aggregatorAccounts,connectGalleryApp,@@ -8781,63 +6268,7 @@ export function toolAccessService(sweepConnectionHealth,sweepGitHubConnectionContinuity,-- listCatalog: async (- connectionId: string,- companyId?: string,- actor?: ActorInfo,- ): Promise<ToolCatalogEntry[]> => {- const connection = await getConnectionRow(connectionId, companyId);- let rows = await db- .select()- .from(toolCatalogEntries)- .where(eq(toolCatalogEntries.connectionId, connection.id))- .orderBy(desc(toolCatalogEntries.updatedAt));- // Native provider catalogs change with the deployed code. Upgrade saved- // Daytona connections too; their first small catalog must not survive- // merely because REST discovery has no remote MCP cache timer.- const nativeDaytona = isDaytonaConnection(connection);- const rowsByName = new Map(rows.map((row) => [row.toolName, row]));- const daytonaCatalogChanged = nativeDaytona && connection.status === "active" && (- rows.some((row) => row.status !== "removed" && !isDaytonaToolName(row.toolName))- || DAYTONA_TOOLS.some((tool) => {- const existing = rowsByName.get(tool.name);- return !existing || existing.versionHash !== descriptorHash(tool, daytonaRisk(tool.name));- })- );- const cacheExpired = daytonaCatalogChanged || (- connection.transport === "mcp_remote" &&- connection.status !== "archived" &&- (rows.length === 0 ||- !connection.lastCatalogRefreshAt ||- connection.lastCatalogRefreshAt.getTime() <=- now().getTime() - catalogCacheTtlMs));- if (cacheExpired) {- try {- await singleFlight(catalogRefreshFlights, connection.id, () =>- refreshCatalog(connection.id, nativeDaytona && actor ? actor : {- actorType: "system",- actorId: "tool_catalog_cache",- }),- );- rows = await db- .select()- .from(toolCatalogEntries)- .where(eq(toolCatalogEntries.connectionId, connection.id))- .orderBy(desc(toolCatalogEntries.updatedAt));- } catch (error) {- // A stale catalog remains useful when the remote server is temporarily- // unavailable. Empty caches still fail so callers never mistake “no- // actions discovered” for a successful lookup.- if (rows.length === 0) throw error;- }- }- // Even failed discovery must not keep retired native tools visible.- // Their database rows remain available to historical invocation records.- return rows- .filter((row) => !nativeDaytona || (row.status !== "removed" && isDaytonaToolName(row.toolName)))- .map((row) => toCatalogEntryForConnection(row, connection));- },+ ...catalogReader,...toolActionReviewQueue,...profileService,...profileResolver,diff --git a/server/src/services/tool-access/catalog/aggregator-accounts.ts b/server/src/services/tool-access/catalog/aggregator-accounts.tsnew file mode 100644index 0000000000..d83e43c25d--- /dev/null+++ b/server/src/services/tool-access/catalog/aggregator-accounts.ts@@ -0,0 +1,492 @@+import { actorForSecret } from "../oauth/credential-store.js";+import type { ActorInfo } from "../types.js";+import { remoteEndpoint, projectedConnectionHeaders } from "../connection-methods.js";+import { AGGREGATOR_NAMES, isAppAggregator, type AggregatorAppsResponse, type ArcadeDiscoverySetupInput } from "@paperclipai/shared/aggregator-apps";+import { resolveAggregatorApp, type AppCatalogAggregator } from "@paperclipai/shared/aggregator-app-catalog";+import { AggregatorDiscoveryUnavailableError, discoverArcadeApps, discoverExecutorApps, type DiscoveredApp } from "../../aggregator-app-discovery.js";+import { COMPOSIO_APP_TOOLKITS, findComposioCatalogApp } from "@paperclipai/shared/aggregator-app-catalog";+import type { ComposioAppAccount, ComposioAppAccountInput, ComposioAppSetupInput, ComposioAppSetupResult, ComposioAppSnapshot, ComposioAppsResponse } from "@paperclipai/shared";+import { composioAppAccounts, composioAppSetupResult } from "../../composio-app-setup.js";+import { createHash, randomUUID } from "node:crypto";+import { and, asc, desc, eq, inArray, isNull, lt, lte, ne, or, sql } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { companySecrets, toolAccessAuditEvents, toolCatalogEntries, toolConnectionAppSnapshots, toolConnectionAppSyncs, toolConnections } from "@paperclipai/db";+import { conflict, forbidden, HttpError, notFound, unprocessable } from "../../../errors.js";+import { logActivity } from "../../activity-log.js";+import { initializeMcpHttpSession, getMcpHttpSession, readMcpHttpResponse, mcpHttpRequestHeaders } from "../../mcp-http.js";+import { secretService } from "../../secrets.js";+import { writeConnectionCredential } from "../../connection-credentials.js";++import type { ToolAccessAudit } from "../audit.js";+import type { ConnectionCredentialAccess } from "../credential-access.js";+import type { ToolRemoteHttp } from "../remote-http.js";+import type { ToolCatalogDiscovery } from "./discovery.js";++function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++type AggregatorDependencies = {+ options: { composioAppToolkits?: readonly string[] };+ secrets: ReturnType<typeof secretService>;+ audit: ToolAccessAudit;+ credentials: ConnectionCredentialAccess;+ getConnectionRow(idOrUid: string, companyId?: string): Promise<typeof toolConnections.$inferSelect>;+ requestRemoteHttpEndpoint: ToolRemoteHttp["requestRemoteHttpEndpoint"];+ remoteTools: ToolCatalogDiscovery["remoteTools"];+};++export function createAggregatorAccounts(db: Db, { options, secrets, audit, credentials, getConnectionRow, requestRemoteHttpEndpoint, remoteTools }: AggregatorDependencies) {+ const { vaultGrantForConnection, resolveCredentialHeaders, resolvedRemoteEndpoint } = credentials;+ const composioSyncJobs = new Map<string, Promise<void>>();++ const composioSyncTtlMs = 5 * 60_000;++ const composioSyncLeaseMs = 4 * 60_000;++ async function composioCredentialKey(connection: typeof toolConnections.$inferSelect, actor: ActorInfo) {+ const usesConnectionRefs = ["arcade", "executor"].includes(String(connection.config.sourceTemplateKey)) && connection.authKind !== "oauth" && connection.credentialPolicy === "shared";+ const grant = usesConnectionRefs ? null : await vaultGrantForConnection(connection, actor);+ const refs = grant?.credentialSecretRefs ?? connection.credentialSecretRefs;+ const discovery = asRecord(asRecord(connection.config.aggregatorDiscovery)[actor.actorId!]);+ const ids = [...new Set([...refs.map(ref => ref.secretId), ...(typeof discovery.secretId === "string" ? [discovery.secretId] : [])])].sort();+ const versions = ids.length ? await db.select({ id: companySecrets.id, version: companySecrets.latestVersion }).from(companySecrets)+ .where(and(eq(companySecrets.companyId, connection.companyId), inArray(companySecrets.id, ids))).orderBy(asc(companySecrets.id)) : [];+ return createHash("sha256").update(JSON.stringify({ grant: grant?.id, refs, versions,+ ...(Object.keys(discovery).length ? { discovery } : {}),+ ...(connection.config.sourceTemplateKey === "arcade" || connection.config.sourceTemplateKey === "executor"+ ? { headers: projectedConnectionHeaders(connection), managementUrl: connection.config.managementUrl } : {}), endpoint: remoteEndpoint(connection.config), policy: connection.credentialPolicy, connectedAt: asRecord(connection.config.oauth)?.connectedAt })).digest("hex");+ }++ async function cachedComposioApps(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<ComposioAppSnapshot[]> {+ const credentialKey = await composioCredentialKey(connection, actor);+ const rows = await db.select().from(toolConnectionAppSnapshots).where(and(+ eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connection.id),+ eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),+ ));+ return rows.map(row => ({ connectionId: row.connectionId, toolkit: row.toolkit, status: row.status, accounts: row.accounts,+ checkedAt: row.checkedAt.toISOString(), errorAt: row.errorAt?.toISOString() ?? null }));+ }++ async function composioAppsResponse(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<ComposioAppsResponse> {+ const credentialKey = await composioCredentialKey(connection, actor);+ const [sync] = await db.select().from(toolConnectionAppSyncs).where(and(+ eq(toolConnectionAppSyncs.companyId, connection.companyId), eq(toolConnectionAppSyncs.connectionId, connection.id),+ eq(toolConnectionAppSyncs.userId, actor.actorId!), eq(toolConnectionAppSyncs.credentialKey, credentialKey),+ ));+ const interrupted = sync?.status === "syncing" && sync.updatedAt.getTime() < Date.now() - composioSyncLeaseMs;+ return { apps: await cachedComposioApps(connection, actor), sync: {+ status: interrupted || sync?.status === "unsupported" ? "error" : sync?.status ?? "idle", coverage: "supported_catalog",+ checked: sync?.checked ?? 0, total: sync?.total ?? 0, failed: sync?.failed ?? 0,+ lastCompletedAt: sync?.lastCompletedAt?.toISOString() ?? null,+ error: interrupted ? "App checks were interrupted. Refresh to try again."+ : (sync?.status === "error" || sync?.status === "unsupported") ? "Couldn’t finish checking Composio apps. Refresh to try again." : null,+ } };+ }++ async function openComposioGateway(connectionId: string, actor: ActorInfo) {+ const connection = await getConnectionRow(connectionId);+ if (actor.actorType !== "user" || !actor.actorId) throw forbidden("A human connection manager must configure this app");+ if (connection.config.sourceTemplateKey !== "composio" || connection.transport !== "mcp_remote"+ || connection.status !== "active" || !connection.enabled) throw unprocessable("Connect your Composio gateway first");+ const catalog = await db.select().from(toolCatalogEntries).where(and(+ eq(toolCatalogEntries.companyId, connection.companyId), eq(toolCatalogEntries.connectionId, connection.id),+ ));+ if (!catalog.some(tool => tool.toolName === "COMPOSIO_MANAGE_CONNECTIONS" && tool.status === "active")) {+ throw unprocessable("Refresh your Composio gateway's actions before configuring this app");+ }+ const endpoint = await resolvedRemoteEndpoint(connection, actor);+ const headers = { ...projectedConnectionHeaders(connection), ...(await resolveCredentialHeaders(connection, actor)) };+ const credentialKey = await composioCredentialKey(await getConnectionRow(connectionId), actor);+ const send = (init: RequestInit) => requestRemoteHttpEndpoint(new URL(endpoint), { ...init, signal: AbortSignal.timeout(30_000) });+ let sessionHeaders = connection.config.mcpSessionRequired === true+ ? await getMcpHttpSession({ send, headers, scope: `${connection.id}:app-setup:${actor.actorId}`, requestId: "paperclip-app-setup" }) : headers;+ async function manage(toolkits: { name: string; action: "list" | "add" | "rename" | "remove"; account_id?: string; alias?: string }[], tolerateToolkitErrors = false) {+ const checkedAt = new Date();+ const id = `paperclip-app-setup-${randomUUID()}`;+ const request = () => send({ method: "POST", headers: mcpHttpRequestHeaders(sessionHeaders), body: JSON.stringify({+ jsonrpc: "2.0", id, method: "tools/call", params: { name: "COMPOSIO_MANAGE_CONNECTIONS", arguments: { toolkits } },+ }) });+ const markFailed = async () => {+ const names = toolkits.filter(tool => tool.action === "list").map(tool => tool.name);+ if (names.length) await db.update(toolConnectionAppSnapshots).set({ errorAt: checkedAt }).where(and(+ eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId),+ eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),+ inArray(toolConnectionAppSnapshots.toolkit, names), lte(toolConnectionAppSnapshots.checkedAt, checkedAt),+ ));+ };+ try {+ let response = await request();+ // Only reads retry. An uncertain account mutation must never repeat automatically.+ if (toolkits.every(tool => tool.action === "list") && (response.status === 400 || response.status === 404)) {+ await response.body?.cancel();+ sessionHeaders = await initializeMcpHttpSession({ send, headers, requestId: id });+ response = await request();+ }+ if (!response.ok) throw new HttpError(502, toolkits.every(tool => tool.action === "list")+ ? "Composio could not check this app. Reconnect the gateway if its sign-in expired."+ : "Composio has not confirmed this change. Refresh the accounts before trying again.", { code: "composio_app_setup_failed", status: response.status });+ const payload = await readMcpHttpResponse(response, id);+ if (await composioCredentialKey(await getConnectionRow(connectionId), actor) !== credentialKey) {+ throw conflict("The Composio account changed while checking apps. Refresh to check the current account.");+ }+ const failedToolkits: string[] = [];+ for (const tool of toolkits.filter(tool => tool.action === "list")) {+ let accounts: ComposioAppAccount[];+ try {+ composioAppSetupResult(payload, tool.name);+ const parsed = composioAppAccounts(payload, tool.name);+ if (!parsed) throw unprocessable("Composio did not return a complete account list. Refresh the accounts before trying again.");+ accounts = parsed;+ } catch (error) {+ await db.update(toolConnectionAppSnapshots).set({ errorAt: checkedAt }).where(and(+ eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId),+ eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),+ eq(toolConnectionAppSnapshots.toolkit, tool.name), lte(toolConnectionAppSnapshots.checkedAt, checkedAt),+ ));+ if (!tolerateToolkitErrors) throw error;+ failedToolkits.push(tool.name);+ continue;+ }+ const status = accounts.some(account => account.status === "ACTIVE") ? "connected" as const : "not_connected" as const;+ await db.insert(toolConnectionAppSnapshots).values({ companyId: connection.companyId, connectionId, userId: actor.actorId!,+ credentialKey, toolkit: tool.name, accounts, status, checkedAt, errorAt: null,+ }).onConflictDoUpdate({ target: [toolConnectionAppSnapshots.companyId, toolConnectionAppSnapshots.connectionId, toolConnectionAppSnapshots.userId, toolConnectionAppSnapshots.toolkit],+ set: { accounts, status, checkedAt, credentialKey, errorAt: null }, setWhere: lte(toolConnectionAppSnapshots.checkedAt, checkedAt) });+ }+ return { payload, failedToolkits };+ } catch (error) { await markFailed(); throw error; }+ }+ return { connection, catalog, credentialKey, manage: async (...args: Parameters<typeof manage>) => (await manage(...args)).payload, checkBatch: manage };+ }++ async function startComposioAppSync(connectionId: string, force: boolean, actor: ActorInfo) {+ const opened = await openComposioGateway(connectionId, actor);+ const { connection, credentialKey } = opened;+ const existing = await cachedComposioApps(connection, actor);+ const catalog = options.composioAppToolkits ?? COMPOSIO_APP_TOOLKITS;+ const toolkits = [...new Set([...existing.filter(app => app.accounts.length > 0).map(app => app.toolkit), ...catalog])];+ const leaseId = randomUUID();+ const startedAt = new Date();+ const [lease] = await db.insert(toolConnectionAppSyncs).values({ companyId: connection.companyId, connectionId,+ userId: actor.actorId!, credentialKey, leaseId, status: "syncing", total: toolkits.length, updatedAt: startedAt,+ }).onConflictDoUpdate({ target: [toolConnectionAppSyncs.companyId, toolConnectionAppSyncs.connectionId, toolConnectionAppSyncs.userId, toolConnectionAppSyncs.credentialKey],+ set: { leaseId, status: "syncing", checked: 0, total: toolkits.length, failed: 0, updatedAt: startedAt },+ setWhere: and(+ or(ne(toolConnectionAppSyncs.status, "syncing"), lt(toolConnectionAppSyncs.updatedAt, new Date(Date.now() - composioSyncLeaseMs))),+ force ? sql`true` : or(ne(toolConnectionAppSyncs.status, "ready"), isNull(toolConnectionAppSyncs.lastCompletedAt),+ lt(toolConnectionAppSyncs.lastCompletedAt, new Date(Date.now() - composioSyncTtlMs))),+ ),+ }).returning();+ if (lease) {+ const updateLease = async (values: Partial<typeof toolConnectionAppSyncs.$inferInsert>) => {+ const updated = await db.update(toolConnectionAppSyncs).set({ ...values, updatedAt: new Date() })+ .where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId))).returning({ id: toolConnectionAppSyncs.id });+ if (!updated.length) throw conflict("App checks were replaced by a newer sync.");+ };+ const job = (async () => {+ let failed = 0;+ try {+ for (let start = 0; start < toolkits.length; start += 32) {+ if (Date.now() - startedAt.getTime() > composioSyncLeaseMs) throw new Error("Sync deadline exceeded");+ await updateLease({});+ const current = start === 0 ? opened : await openComposioGateway(connectionId, actor);+ if (current.credentialKey !== credentialKey) throw conflict("Composio credentials changed during sync.");+ const batch = toolkits.slice(start, start + 32);+ const result = await current.checkBatch(batch.map(name => ({ name, action: "list" })), true);+ failed += result.failedToolkits.length;+ await updateLease({ checked: start + batch.length, failed });+ }+ await updateLease({ status: failed > 0 ? "error" : "ready", lastCompletedAt: new Date(), failed });+ } catch {+ const [activeLease] = await db.select({ id: toolConnectionAppSyncs.id }).from(toolConnectionAppSyncs)+ .where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId)));+ if (!activeLease) return;+ // Preserve observations, but never present a failed check as current authorization.+ await db.update(toolConnectionAppSnapshots).set({ errorAt: new Date() }).where(and(+ eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId),+ eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),+ lte(toolConnectionAppSnapshots.checkedAt, startedAt),+ ));+ await updateLease({ status: "error", failed }).catch(() => undefined);+ }+ })();+ composioSyncJobs.set(leaseId, job);+ void job.finally(() => composioSyncJobs.delete(leaseId)).catch(() => undefined);+ }+ return composioAppsResponse(await getConnectionRow(connectionId), actor);+ }++ function aggregatorProvider(connection: typeof toolConnections.$inferSelect): AppCatalogAggregator {+ const provider = connection.config.sourceTemplateKey;+ if (!isAppAggregator(provider) || connection.transport !== "mcp_remote") throw notFound("Aggregator gateway not found");+ return provider;+ }++ function assertDiscoveryActor(actor: ActorInfo) {+ if (actor.actorType !== "user" || !actor.actorId) throw forbidden("A human connection manager must check accounts");+ }++ async function aggregatorDiscoveryState(connection: typeof toolConnections.$inferSelect, actor: ActorInfo, retryUnsupported = false): Promise<AggregatorAppsResponse["discovery"]> {+ const provider = aggregatorProvider(connection);+ if (connection.status !== "active" || !connection.enabled) return { availability: "disabled", message: "Restore this gateway to check its accounts." };+ if (provider === "arcade") {+ const discovery = asRecord(asRecord(connection.config.aggregatorDiscovery)[actor.actorId!]);+ const headers = projectedConnectionHeaders(connection);+ const grant = connection.authKind === "oauth" || connection.credentialPolicy !== "shared" ? await vaultGrantForConnection(connection, actor) : null;+ const refs = grant?.credentialSecretRefs ?? connection.credentialSecretRefs;+ const hasUserId = Boolean(Object.entries(headers).find(([key]) => key.toLowerCase() === "arcade-user-id")?.[1])+ || refs.some(ref => ref.configPath.toLowerCase() === "headers.arcade-user-id");+ if ((!discovery.secretId || !discovery.userId) && !(hasUserId && connection.authKind === "api_key")) {+ return { availability: "setup_required", message: "Set up account sync to see your Arcade apps here." };+ }+ }+ const catalog = await db.select({ name: toolCatalogEntries.toolName }).from(toolCatalogEntries).where(and(+ eq(toolCatalogEntries.companyId, connection.companyId), eq(toolCatalogEntries.connectionId, connection.id), eq(toolCatalogEntries.status, "active"),+ ));+ if ((provider === "executor" && !catalog.some(tool => ["integrations", "execute"].includes(tool.name)))+ || (provider === "composio" && !catalog.some(tool => tool.name === "COMPOSIO_MANAGE_CONNECTIONS"))) {+ return { availability: "unsupported", message: `${AGGREGATOR_NAMES[provider]} account discovery is unavailable on this gateway. Refresh its actions or check its setup.` };+ }+ if (!retryUnsupported && provider === "executor") {+ const credentialKey = await composioCredentialKey(connection, actor);+ const [sync] = await db.select({ status: toolConnectionAppSyncs.status }).from(toolConnectionAppSyncs).where(and(+ eq(toolConnectionAppSyncs.companyId, connection.companyId), eq(toolConnectionAppSyncs.connectionId, connection.id),+ eq(toolConnectionAppSyncs.userId, actor.actorId!), eq(toolConnectionAppSyncs.credentialKey, credentialKey),+ ));+ if (sync?.status === "unsupported") return { availability: "unsupported", message: "Executor account discovery is unavailable on this server. You can still use the gateway’s actions." };+ }+ return { availability: "available", message: null };+ }++ async function aggregatorAppsResponse(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<AggregatorAppsResponse> {+ assertDiscoveryActor(actor);+ const provider = aggregatorProvider(connection);+ const result = await composioAppsResponse(connection, actor);+ const discovery = await aggregatorDiscoveryState(connection, actor);+ return { provider, discovery, apps: result.apps.map(snapshot => {+ const app = resolveAggregatorApp(provider, snapshot.toolkit, snapshot.accounts[0]?.appName);+ return { ...snapshot, provider, freshness: discovery.availability !== "available" || snapshot.errorAt || result.sync.status === "error" || connection.status !== "active" || !connection.enabled || Date.now() - new Date(snapshot.checkedAt).getTime() > composioSyncTtlMs ? "stale" as const : "fresh" as const, appSlug: app.slug.startsWith(`${provider}:`) ? `${provider}:${connection.id}:${snapshot.toolkit}` : snapshot.accounts[0]?.appSlug ?? app.slug, appName: snapshot.accounts[0]?.appName ?? app.name };+ }), sync: { ...result.sync, coverage: provider === "composio" ? "supported_catalog" : provider === "arcade" ? "gateway_tools" : "visible_accounts",+ error: result.sync.error ? `Couldn’t finish checking ${AGGREGATOR_NAMES[provider]} apps. Refresh to try again.` : null,+ } };+ }++ async function discoverAggregatorInventory(connection: typeof toolConnections.$inferSelect, actor: ActorInfo): Promise<DiscoveredApp[]> {+ const provider = aggregatorProvider(connection);+ const tools = await remoteTools(connection, undefined, actor);+ connection = await getConnectionRow(connection.id);+ const headers = { ...projectedConnectionHeaders(connection), ...(await resolveCredentialHeaders(connection, actor)) };+ const endpoint = new URL(await resolvedRemoteEndpoint(connection, actor));+ if (provider === "arcade") {+ // The extra project credential never joins invocation headers or tool grants.+ if (endpoint.hostname !== "api.arcade.dev" || endpoint.protocol !== "https:") throw unprocessable("Account sync requires an Arcade Cloud gateway");+ const discovery = asRecord(asRecord(connection.config.aggregatorDiscovery)[actor.actorId!]);+ let key: string | undefined;+ let userId: string | undefined;+ if (typeof discovery.secretId === "string" && typeof discovery.userId === "string") {+ const [secret] = await db.select().from(companySecrets).where(and(eq(companySecrets.id, discovery.secretId), eq(companySecrets.companyId, connection.companyId), eq(companySecrets.ownerUserId, actor.actorId!)));+ if (!secret?.userSecretDefinitionId) throw forbidden("Your Arcade discovery credential is unavailable");+ const resolved = await secrets.resolveUserSecretValue(connection.companyId, { definitionId: secret.userSecretDefinitionId, responsibleUserId: actor.actorId!, required: true }, {+ actorType: "user", actorId: actor.actorId!, consumerType: "tool_connection", consumerId: connection.id, responsibleUserId: actor.actorId!,+ });+ key = resolved?.value;+ userId = discovery.userId;+ } else {+ userId = Object.entries(headers).find(([name]) => name.toLowerCase() === "arcade-user-id")?.[1];+ if (connection.authKind === "api_key") key = Object.entries(headers).find(([name]) => name.toLowerCase() === "authorization")?.[1].replace(/^Bearer\s+/i, "");+ }+ if (!key || !userId) throw unprocessable("Set up Arcade account sync first");+ return discoverArcadeApps({ userId, gatewayTools: tools.map(tool => tool.name), request: async path => {+ const response = await requestRemoteHttpEndpoint(new URL(path, endpoint.origin), { method: "GET", headers: { Authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(30_000) });+ if (!response.ok) { await response.body?.cancel(); throw new HttpError(502, "Arcade could not list your accounts. Check the project API key and Arcade user ID."); }+ return response.json();+ } });+ }+ const send = (init: RequestInit) => requestRemoteHttpEndpoint(endpoint, { ...init, signal: AbortSignal.timeout(30_000) });+ let sessionHeaders = connection.config.mcpSessionRequired === true ? await getMcpHttpSession({ send, headers, scope: `${connection.id}:inventory:${actor.actorId}`, requestId: "paperclip-account-inventory" }) : headers;+ return discoverExecutorApps({ toolNames: tools.map(tool => tool.name), managementUrl: typeof connection.config.managementUrl === "string" ? connection.config.managementUrl : null,+ call: async (name, args) => {+ const id = `paperclip-account-inventory-${randomUUID()}`;+ const request = () => send({ method: "POST", headers: mcpHttpRequestHeaders(sessionHeaders), body: JSON.stringify({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: args } }) });+ let response = await request();+ if (response.status === 400 || response.status === 404) {+ await response.body?.cancel(); sessionHeaders = await initializeMcpHttpSession({ send, headers, requestId: id }); response = await request();+ }+ if (!response.ok) { await response.body?.cancel(); throw new HttpError(502, "Executor could not list your accounts. Reconnect if sign-in expired."); }+ return readMcpHttpResponse(response, id);+ },+ });+ }++ async function syncAggregatorApps(connectionId: string, force: boolean, actor: ActorInfo): Promise<AggregatorAppsResponse> {+ assertDiscoveryActor(actor);+ let connection = await getConnectionRow(connectionId);+ const provider = aggregatorProvider(connection);+ const discovery = await aggregatorDiscoveryState(connection, actor, force);+ if (discovery.availability !== "available") return aggregatorAppsResponse(connection, actor);+ if (provider === "composio") {+ await startComposioAppSync(connectionId, force, actor);+ return aggregatorAppsResponse(await getConnectionRow(connectionId), actor);+ }+ // Resolve/refresh the caller's gateway credentials before fingerprinting the lease.+ await resolveCredentialHeaders(connection, actor);+ connection = await getConnectionRow(connectionId);+ const credentialKey = await composioCredentialKey(connection, actor);+ const leaseId = randomUUID();+ const startedAt = new Date();+ const [lease] = await db.insert(toolConnectionAppSyncs).values({ companyId: connection.companyId, connectionId, userId: actor.actorId!, credentialKey, leaseId, status: "syncing", updatedAt: startedAt })+ .onConflictDoUpdate({ target: [toolConnectionAppSyncs.companyId, toolConnectionAppSyncs.connectionId, toolConnectionAppSyncs.userId, toolConnectionAppSyncs.credentialKey],+ set: { leaseId, status: "syncing", checked: 0, total: 0, failed: 0, updatedAt: startedAt },+ setWhere: and(or(ne(toolConnectionAppSyncs.status, "syncing"), lt(toolConnectionAppSyncs.updatedAt, new Date(Date.now() - composioSyncLeaseMs))),+ force ? sql`true` : or(ne(toolConnectionAppSyncs.status, "ready"), isNull(toolConnectionAppSyncs.lastCompletedAt), lt(toolConnectionAppSyncs.lastCompletedAt, new Date(Date.now() - composioSyncTtlMs)))),+ }).returning();+ if (lease) {+ const job = (async () => {+ try {+ const apps = await discoverAggregatorInventory(connection, actor);+ await db.transaction(async tx => {+ const [current] = await tx.select().from(toolConnections).where(and(eq(toolConnections.id, connectionId), eq(toolConnections.companyId, connection.companyId))).for("update");+ const [active] = await tx.select().from(toolConnectionAppSyncs).where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId))).for("update");+ if (!current || !active || current.status !== "active" || !current.enabled || Date.now() - startedAt.getTime() > composioSyncLeaseMs+ || await composioCredentialKey(current, actor) !== credentialKey) throw conflict("The gateway changed during account discovery");+ const checkedAt = new Date();+ // Commit the entire enumeration at once; a failed later page cannot delete accounts.+ await tx.update(toolConnectionAppSnapshots).set({ accounts: [], status: "not_connected", checkedAt, errorAt: null }).where(and(+ eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId), eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey),+ ));+ for (const app of apps) await tx.insert(toolConnectionAppSnapshots).values({ companyId: connection.companyId, connectionId, userId: actor.actorId!, credentialKey,+ toolkit: app.toolkit, accounts: app.accounts, status: app.accounts.some(account => account.status === "ACTIVE") ? "connected" : "not_connected", checkedAt,+ }).onConflictDoUpdate({ target: [toolConnectionAppSnapshots.companyId, toolConnectionAppSnapshots.connectionId, toolConnectionAppSnapshots.userId, toolConnectionAppSnapshots.toolkit],+ set: { accounts: app.accounts, status: app.accounts.some(account => account.status === "ACTIVE") ? "connected" : "not_connected", checkedAt, errorAt: null, credentialKey },+ });+ await tx.update(toolConnectionAppSyncs).set({ status: "ready", checked: apps.length, total: apps.length, failed: 0, updatedAt: checkedAt, lastCompletedAt: checkedAt }).where(eq(toolConnectionAppSyncs.id, lease.id));+ });+ } catch (error) {+ await db.transaction(async tx => {+ const [active] = await tx.select().from(toolConnectionAppSyncs).where(and(eq(toolConnectionAppSyncs.id, lease.id), eq(toolConnectionAppSyncs.leaseId, leaseId))).for("update");+ if (!active) return;+ await tx.update(toolConnectionAppSnapshots).set({ errorAt: new Date() }).where(and(eq(toolConnectionAppSnapshots.companyId, connection.companyId), eq(toolConnectionAppSnapshots.connectionId, connectionId), eq(toolConnectionAppSnapshots.userId, actor.actorId!), eq(toolConnectionAppSnapshots.credentialKey, credentialKey)));+ await tx.update(toolConnectionAppSyncs).set({ status: error instanceof AggregatorDiscoveryUnavailableError ? "unsupported" : "error", failed: 1, updatedAt: new Date() }).where(eq(toolConnectionAppSyncs.id, lease.id));+ });+ }+ })();+ composioSyncJobs.set(leaseId, job);+ void job.finally(() => composioSyncJobs.delete(leaseId)).catch(() => undefined);+ }+ return aggregatorAppsResponse(await getConnectionRow(connectionId), actor);+ }++ async function configureArcadeDiscovery(connectionId: string, input: ArcadeDiscoverySetupInput, actor: ActorInfo) {+ assertDiscoveryActor(actor);+ const companyId = (await getConnectionRow(connectionId)).companyId;+ await db.transaction(async tx => {+ const [connection] = await tx.select().from(toolConnections).where(eq(toolConnections.id, connectionId)).for("update");+ if (!connection || aggregatorProvider(connection) !== "arcade" || connection.status === "archived") throw notFound("Arcade gateway not found");+ const all = asRecord(connection.config.aggregatorDiscovery);+ const before = asRecord(all[actor.actorId!]);+ const { secret } = await writeConnectionCredential(tx, {+ companyId: connection.companyId, connectionName: connection.name, configPath: "aggregatorDiscovery.apiKey", label: "Arcade account sync API key",+ value: input.apiKey, ownerUserId: actor.actorId!, actor: actorForSecret(actor),+ existingRef: typeof before.secretId === "string" ? { secretId: before.secretId, configPath: "aggregatorDiscovery.apiKey", versionSelector: "latest" } : undefined,+ definitionKey: `arcade_discovery.${connection.id}`,+ });+ const config = { ...connection.config, aggregatorDiscovery: { ...all, [actor.actorId!]: { userId: input.userId, secretId: secret.id } } };+ await tx.update(toolConnections).set({ config, transportConfig: config, updatedAt: new Date() }).where(eq(toolConnections.id, connectionId));+ });+ await logActivity(db, { companyId, actorType: "user", actorId: actor.actorId!, action: "tool_connection.account_sync_configured", entityType: "tool_connection", entityId: connectionId, details: { provider: "arcade" } });+ return syncAggregatorApps(connectionId, true, actor);+ }+ return {+ syncAggregatorApps, configureArcadeDiscovery,+ async setupComposioApp(connectionId: string, toolkit: string, input: ComposioAppSetupInput, actor: ActorInfo): Promise<ComposioAppSetupResult> {+ if (!findComposioCatalogApp(toolkit)) throw notFound("This Composio app is not in the catalog");+ const { connection, manage } = await openComposioGateway(connectionId, actor);+ let result = composioAppSetupResult(await manage([{ name: toolkit, action: "list" }]), toolkit);+ if (input.action === "start" && result.status !== "connected") {+ result = composioAppSetupResult(await manage([{ name: toolkit, action: "add" }]), toolkit);+ if (result.status === "not_connected") throw unprocessable("Composio did not return a sign-in link for this app. Manage it in Composio to check its setup requirements.");+ }+ if (input.action === "complete") {+ if (result.status !== "connected") throw conflict("Finish connecting this app in Composio, then check again");+ const currentConnection = await getConnectionRow(connectionId);+ if (currentConnection.status !== "active" || !currentConnection.enabled) {+ throw conflict("The gateway is no longer active. Restore it before connecting this app.");+ }+ }+ await audit({ companyId: connection.companyId, connectionId, action: "tool_connection.upstream_app_setup", outcome: "success", actor,+ details: { toolkit, action: input.action, status: result.status } });+ return result;+ },+ listAggregatorApps: async (connectionId: string, actor: ActorInfo) => aggregatorAppsResponse(await getConnectionRow(connectionId), actor),+ refreshAggregatorApps: async (connectionId: string, requested: string[], actor: ActorInfo) => {+ const connection = await getConnectionRow(connectionId);+ if (aggregatorProvider(connection) === "composio" && requested.length) {+ const { manage } = await openComposioGateway(connectionId, actor);+ if (requested.some(toolkit => !findComposioCatalogApp(toolkit))) throw notFound("This Composio app is not in the catalog");+ await manage(requested.map(name => ({ name, action: "list" })));+ return aggregatorAppsResponse(connection, actor);+ }+ return syncAggregatorApps(connectionId, true, actor);+ },+ syncComposioApps: startComposioAppSync,+ async listComposioApps(connectionId: string, actor: ActorInfo) {+ const connection = await getConnectionRow(connectionId);+ if (actor.actorType !== "user" || !actor.actorId) throw forbidden("Board access required");+ if (connection.config.sourceTemplateKey !== "composio") throw notFound("Composio gateway not found");+ return composioAppsResponse(connection, actor);+ },+ async refreshComposioApps(connectionId: string, requested: string[], actor: ActorInfo) {+ if (requested.some(toolkit => !findComposioCatalogApp(toolkit))) throw notFound("This Composio app is not in the catalog");+ const { connection, manage } = await openComposioGateway(connectionId, actor);+ const existing = await cachedComposioApps(connection, actor);+ // Migrate observations from the original direct setup flow on the first visit.+ const history = await db.select({ details: toolAccessAuditEvents.details }).from(toolAccessAuditEvents).where(and(+ eq(toolAccessAuditEvents.companyId, connection.companyId), eq(toolAccessAuditEvents.connectionId, connectionId),+ eq(toolAccessAuditEvents.actorId, actor.actorId!), eq(toolAccessAuditEvents.action, "tool_connection.upstream_app_setup"),+ )).orderBy(desc(toolAccessAuditEvents.createdAt)).limit(128);+ const known = [...existing.filter(app => app.accounts.length > 0).map(app => app.toolkit),+ ...history.map(row => row.details.toolkit).filter((toolkit): toolkit is string => typeof toolkit === "string" && Boolean(findComposioCatalogApp(toolkit)))];+ const toolkits = [...new Set([...requested, ...known])];+ for (let start = 0; start < toolkits.length; start += 32) {+ await manage(toolkits.slice(start, start + 32).map(name => ({ name, action: "list" })));+ }+ return composioAppsResponse(connection, actor);+ },+ async manageComposioAppAccount(connectionId: string, toolkit: string, input: ComposioAppAccountInput, actor: ActorInfo) {+ if (!findComposioCatalogApp(toolkit)) throw notFound("This Composio app is not in the catalog");+ const { connection, manage } = await openComposioGateway(connectionId, actor);+ if (input.action !== "add") {+ const before = composioAppAccounts(await manage([{ name: toolkit, action: "list" }]), toolkit);+ if (!before?.some(account => account.id === input.accountId)) throw notFound("This account is no longer available in the selected Composio app");+ }+ const details = { toolkit, ...(input.action !== "add" ? { accountId: input.accountId } : {}) };+ await logActivity(db, { companyId: connection.companyId, actorType: "user", actorId: actor.actorId!,+ action: `tool_connection.upstream_account_${input.action}_requested`, entityType: "tool_connection", entityId: connectionId, details });+ let payload: unknown;+ try {+ payload = await manage([{ name: toolkit, action: input.action,+ ...(input.action !== "add" ? { account_id: input.accountId } : {}), ...(input.action === "rename" ? { alias: input.alias } : {}) }]);+ } catch (error) {+ await audit({ companyId: connection.companyId, connectionId, actor, action: `tool_connection.upstream_account_${input.action}`,+ outcome: "failure", reasonCode: "provider_unconfirmed", details });+ throw error;+ }+ const result = composioAppSetupResult(payload, toolkit); // Reject provider errors before reporting success.+ if (input.action === "add" && !result.authorizationUrl) throw unprocessable("Composio did not return a sign-in link. Check the app in Composio.");+ if (input.action !== "add") {+ const after = composioAppAccounts(await manage([{ name: toolkit, action: "list" }]), toolkit);+ if (!after || (input.action === "remove" ? after.some(account => account.id === input.accountId)+ : !after.some(account => account.id === input.accountId && account.alias === input.alias))) {+ throw conflict("Composio has not confirmed this change. Refresh the accounts before trying again.");+ }+ }+ await logActivity(db, { companyId: connection.companyId, actorType: "user", actorId: actor.actorId!,+ action: `tool_connection.upstream_account_${input.action}`, entityType: "tool_connection", entityId: connectionId,+ details: { toolkit, ...(input.action !== "add" ? { accountId: input.accountId } : {}) } });+ return { ...result, apps: await cachedComposioApps(connection, actor) };+ }+ };+}diff --git a/server/src/services/tool-access/catalog/attention.ts b/server/src/services/tool-access/catalog/attention.tsnew file mode 100644index 0000000000..40cc2cac05--- /dev/null+++ b/server/src/services/tool-access/catalog/attention.ts@@ -0,0 +1,207 @@+import { pendingNewToolsForProfile, profileCatalogColumns } from "../profiles/projection.js";+import { and, eq, isNotNull, ne } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { toolActionRequests, toolCatalogEntries, toolConnections, toolInvocations, toolProfileEntries, toolProfiles } from "@paperclipai/db";+import type { ToolConnectionHealthStatus, ToolAppsAttentionResponse } from "@paperclipai/shared";+import { isToolConnectionAttentionHealth } from "@paperclipai/shared";+import { toConnection } from "../connections/projection.js";+function isAttentionHealthStatus(+ status: ToolConnectionHealthStatus,+): boolean {+ return isToolConnectionAttentionHealth(status);+}++export function createToolAppAttention(db: Db, now: () => Date) {+ async function listAppsNeedingAttention(+ companyId: string,+ ): Promise<ToolAppsAttentionResponse> {+ const generatedAt = now();+ const [+ connections,+ quarantinedEntries,+ pendingActionRequests,+ invocations,+ profiles,+ profileEntries,+ activeCatalog,+ ] = await Promise.all([+ db+ .select()+ .from(toolConnections)+ .where(+ and(+ eq(toolConnections.companyId, companyId),+ ne(toolConnections.status, "archived"),+ ),+ ),+ db+ .select({ connectionId: toolCatalogEntries.connectionId })+ .from(toolCatalogEntries)+ .where(+ and(+ eq(toolCatalogEntries.companyId, companyId),+ eq(toolCatalogEntries.status, "quarantined"),+ ),+ ),+ db+ .select({ invocationId: toolActionRequests.invocationId })+ .from(toolActionRequests)+ .where(+ and(+ eq(toolActionRequests.companyId, companyId),+ eq(toolActionRequests.status, "pending"),+ isNotNull(toolActionRequests.signedArguments),+ ),+ ),+ db+ .select({ id: toolInvocations.id, connectionId: toolInvocations.connectionId })+ .from(toolInvocations)+ .where(eq(toolInvocations.companyId, companyId)),+ db+ .select()+ .from(toolProfiles)+ .where(eq(toolProfiles.companyId, companyId)),+ db+ .select()+ .from(toolProfileEntries)+ .where(eq(toolProfileEntries.companyId, companyId)),+ db+ .select(profileCatalogColumns)+ .from(toolCatalogEntries)+ .where(+ and(+ eq(toolCatalogEntries.companyId, companyId),+ eq(toolCatalogEntries.status, "active"),+ ),+ ),+ ]);+ const quarantinedCountByConnection = new Map<string, number>();+ for (const entry of quarantinedEntries) {+ quarantinedCountByConnection.set(+ entry.connectionId,+ (quarantinedCountByConnection.get(entry.connectionId) ?? 0) + 1,+ );+ }+ const invocationConnectionById = new Map(+ invocations.map((invocation) => [invocation.id, invocation.connectionId]),+ );+ const pendingActionRequestCountByConnection = new Map<string, number>();+ for (const request of pendingActionRequests) {+ const connectionId = invocationConnectionById.get(request.invocationId);+ if (!connectionId) continue;+ pendingActionRequestCountByConnection.set(+ connectionId,+ (pendingActionRequestCountByConnection.get(connectionId) ?? 0) + 1,+ );+ }+ const entriesByProfile = new Map<+ string,+ Array<typeof toolProfileEntries.$inferSelect>+ >();+ for (const entry of profileEntries) {+ const list = entriesByProfile.get(entry.profileId) ?? [];+ list.push(entry);+ entriesByProfile.set(entry.profileId, list);+ }+ const connectionsById = new Map(+ connections.map((connection) => [connection.id, connection]),+ );+ const pendingProfilesByConnection = new Map<+ string,+ Map<+ string,+ { profileId: string; profileName: string; pendingCount: number }+ >+ >();+ for (const profile of profiles) {+ const tools = pendingNewToolsForProfile({+ profile,+ entries: entriesByProfile.get(profile.id) ?? [],+ catalog: activeCatalog,+ connectionsById,+ });+ for (const tool of tools) {+ const profileCounts =+ pendingProfilesByConnection.get(tool.connectionId) ?? new Map();+ const existing = profileCounts.get(profile.id) ?? {+ profileId: profile.id,+ profileName: profile.name,+ pendingCount: 0,+ };+ existing.pendingCount += 1;+ profileCounts.set(profile.id, existing);+ pendingProfilesByConnection.set(tool.connectionId, profileCounts);+ }+ }+ const apps = connections.flatMap((connection) => {+ const healthNeedsAttention = isAttentionHealthStatus(+ connection.healthStatus,+ );+ const quarantinedCatalogEntryCount =+ quarantinedCountByConnection.get(connection.id) ?? 0;+ const pendingActionRequestCount =+ pendingActionRequestCountByConnection.get(connection.id) ?? 0;+ const newToolsPendingProfiles = [+ ...(pendingProfilesByConnection.get(connection.id)?.values() ?? []),+ ].sort(+ (a, b) =>+ b.pendingCount - a.pendingCount ||+ a.profileName.localeCompare(b.profileName),+ );+ const newToolsPendingReviewCount = newToolsPendingProfiles.reduce(+ (sum, profile) => sum + profile.pendingCount,+ 0,+ );+ const reasons = [+ ...(healthNeedsAttention ? ["health" as const] : []),+ ...(quarantinedCatalogEntryCount > 0+ ? ["quarantined_catalog_entries" as const]+ : []),+ ...(pendingActionRequestCount > 0+ ? ["pending_action_requests" as const]+ : []),+ ...(newToolsPendingReviewCount > 0+ ? ["profile_new_tools" as const]+ : []),+ ];+ return reasons.length > 0+ ? [+ {+ connection: toConnection(connection),+ healthNeedsAttention,+ quarantinedCatalogEntryCount,+ pendingActionRequestCount,+ newToolsPendingReviewCount,+ newToolsPendingProfiles,+ reasons,+ },+ ]+ : [];+ });+ return {+ generatedAt,+ apps,+ totals: {+ connections: apps.length,+ health: apps.filter((app) => app.healthNeedsAttention).length,+ quarantinedCatalogEntries: apps.reduce(+ (sum, app) => sum + app.quarantinedCatalogEntryCount,+ 0,+ ),+ pendingActionRequests: apps.reduce(+ (sum, app) => sum + app.pendingActionRequestCount,+ 0,+ ),+ newToolsPendingReview: apps.reduce(+ (sum, app) => sum + app.newToolsPendingReviewCount,+ 0,+ ),+ newToolsPendingProfiles: apps.reduce(+ (sum, app) => sum + app.newToolsPendingProfiles.length,+ 0,+ ),+ },+ };+ }+ return listAppsNeedingAttention;+}diff --git a/server/src/services/tool-access/catalog/default-access.ts b/server/src/services/tool-access/catalog/default-access.tsnew file mode 100644index 0000000000..c9a459dfaf--- /dev/null+++ b/server/src/services/tool-access/catalog/default-access.ts@@ -0,0 +1,329 @@+import { lockManagedAppPolicies, lockToolAssignmentAgents } from "../assignment-locks.js";+import type { ActorInfo, ToolAccessMutationDb } from "../types.js";+import { and, eq, inArray } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { toolCatalogEntries, toolConnections, toolPolicies, toolProfileBindings, toolProfileEntries, toolProfiles } from "@paperclipai/db";+import type { ToolPolicy } from "@paperclipai/shared";++function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++export function toPolicy(row: typeof toolPolicies.$inferSelect): ToolPolicy {+ return {+ id: row.id,+ companyId: row.companyId,+ name: row.name,+ description: row.description,+ policyType: row.policyType,+ priority: row.priority,+ enabled: row.enabled,+ selectors: row.selectors ?? {},+ conditions: row.conditions ?? null,+ config: row.config ?? null,+ createdByAgentId: row.createdByAgentId,+ createdByUserId: row.createdByUserId,+ createdAt: row.createdAt,+ updatedAt: row.updatedAt,+ };+}++function policyNameForApp(+ connection: typeof toolConnections.$inferSelect,+ entry: typeof toolCatalogEntries.$inferSelect,+) {+ const base = `Ask first ${connection.id.slice(0, 8)} ${entry.toolName}`;+ return base.length <= 160 ? base : base.slice(0, 160);+}++export function createCatalogDefaultAccess(db: Db) {+ async function enableCatalogEntriesByDefault(input: {+ connection: typeof toolConnections.$inferSelect;+ newCatalogEntryIds: string[];+ activeCatalogEntryIds: string[];+ restoreDraftDefaults?: boolean;+ actor?: ActorInfo;+ }, dbClient: ToolAccessMutationDb = db) {+ // Catalog discovery also runs while the setup wizard is still a draft.+ // Access is not granted until the operator finishes that wizard, so a+ // draft refresh must never manufacture a profile or company-wide binding.+ // Active legacy connections may still need the managed profile created on+ // their first refresh. A reconnect is the one draft exception: removal+ // deliberately clears the old selections, so reconnecting restores the+ // documented defaults without activating the connection itself.+ if (input.connection.status !== "active" && !input.restoreDraftDefaults)+ return;+ const profileKey = `app:${input.connection.id}`;+ let [profile] = await dbClient+ .select()+ .from(toolProfiles)+ .where(+ and(+ eq(toolProfiles.companyId, input.connection.companyId),+ eq(toolProfiles.profileKey, profileKey),+ ),+ )+ .limit(1).for("no key update");+ await lockManagedAppPolicies(dbClient, input.connection.companyId, input.connection.id);+ const createdProfile = !profile;+ const resetToRecommendedDefaults = !profile || profile.status !== "active";+ if (resetToRecommendedDefaults) await lockToolAssignmentAgents(dbClient, input.connection.companyId, [{ targetType: "company", targetId: input.connection.companyId }]);+ if (!profile) {+ const [sameName] = await dbClient+ .select({ id: toolProfiles.id })+ .from(toolProfiles)+ .where(+ and(+ eq(toolProfiles.companyId, input.connection.companyId),+ eq(toolProfiles.name, input.connection.name),+ ),+ )+ .limit(1);+ const profileName = sameName+ ? `${input.connection.name} (${input.connection.id.replace(/-/g, "").slice(0, 8)})`+ : input.connection.name;+ [profile] = await dbClient+ .insert(toolProfiles)+ .values({+ companyId: input.connection.companyId,+ profileKey,+ name: profileName,+ description: `Access profile for ${input.connection.name}.`,+ status: "active",+ defaultAction: "deny",+ metadata: {+ source: "app_gallery_finish",+ connectionId: input.connection.id,+ },+ })+ .returning();+ await dbClient.insert(toolProfileBindings).values({+ companyId: input.connection.companyId,+ profileId: profile.id,+ targetType: "company",+ targetId: input.connection.companyId,+ priority: 100,+ metadata: { source: "app_gallery_finish" },+ createdByAgentId:+ input.actor?.actorType === "agent"+ ? (input.actor.actorId ?? null)+ : null,+ createdByUserId:+ input.actor?.actorType === "user"+ ? (input.actor.actorId ?? null)+ : null,+ });+ } else if (resetToRecommendedDefaults) {+ // Removing an app may retain its profile row when another record still+ // references it (for example, an MCP gateway). Reconnecting revives the+ // same connection id, so restore that retained profile as a fresh+ // all-agents assignment instead of leaving every read action Off.+ [profile] = await dbClient+ .update(toolProfiles)+ .set({+ name: input.connection.name,+ description: `Access profile for ${input.connection.name}.`,+ status: "active",+ defaultAction: "deny",+ metadata: {+ source: "app_gallery_finish",+ connectionId: input.connection.id,+ },+ updatedAt: new Date(),+ })+ .where(eq(toolProfiles.id, profile.id))+ .returning();+ await dbClient+ .delete(toolProfileBindings)+ .where(+ and(+ eq(toolProfileBindings.companyId, input.connection.companyId),+ eq(toolProfileBindings.profileId, profile.id),+ ),+ );+ await dbClient+ .delete(toolProfileEntries)+ .where(+ and(+ eq(toolProfileEntries.companyId, input.connection.companyId),+ eq(toolProfileEntries.profileId, profile.id),+ ),+ );+ await dbClient.insert(toolProfileBindings).values({+ companyId: input.connection.companyId,+ profileId: profile.id,+ targetType: "company",+ targetId: input.connection.companyId,+ priority: 100,+ metadata: { source: "app_gallery_finish" },+ createdByAgentId:+ input.actor?.actorType === "agent"+ ? (input.actor.actorId ?? null)+ : null,+ createdByUserId:+ input.actor?.actorType === "user"+ ? (input.actor.actorId ?? null)+ : null,+ });+ }++ if (resetToRecommendedDefaults) {+ // Old setup defaults created app-scoped Ask first policies for writes.+ // They outrank profile allows, so a revived connection must explicitly+ // retire them to make every action Allowed. Policies outside this app's+ // managed profile are intentionally untouched.+ await upsertAskFirstPolicies({+ companyId: input.connection.companyId,+ connection: input.connection,+ askFirstEntries: [],+ actor: input.actor,+ }, dbClient);+ }++ // A new or revived connection starts with every discovered action enabled.+ // Later refreshes extend an active managed profile only for genuinely new+ // actions, so an action the operator deliberately turned off remains off.+ const candidateIds = [+ ...new Set(+ createdProfile || resetToRecommendedDefaults+ ? input.activeCatalogEntryIds+ : input.newCatalogEntryIds,+ ),+ ];+ if (candidateIds.length === 0) return;+ const existingEntries = await dbClient+ .select({ catalogEntryId: toolProfileEntries.catalogEntryId })+ .from(toolProfileEntries)+ .where(+ and(+ eq(toolProfileEntries.companyId, input.connection.companyId),+ eq(toolProfileEntries.profileId, profile.id),+ inArray(toolProfileEntries.catalogEntryId, candidateIds),+ ),+ );+ const configuredIds = new Set(+ existingEntries.flatMap((entry) =>+ entry.catalogEntryId ? [entry.catalogEntryId] : [],+ ),+ );+ const entryIds = candidateIds.filter((id) => !configuredIds.has(id));+ if (entryIds.length === 0) return;+ await dbClient.insert(toolProfileEntries).values(+ entryIds.map((catalogEntryId) => ({+ companyId: input.connection.companyId,+ profileId: profile.id,+ selectorType: "catalog_entry" as const,+ effect: "include" as const,+ applicationId: input.connection.applicationId,+ connectionId: input.connection.id,+ catalogEntryId,+ })),+ );+ }++ async function upsertAskFirstPolicies(+ input: {+ companyId: string;+ connection: typeof toolConnections.$inferSelect;+ askFirstEntries: Array<typeof toolCatalogEntries.$inferSelect>;+ actor?: ActorInfo;+ disableStale?: boolean;+ },+ dbClient: ToolAccessMutationDb = db,+ ): Promise<ToolPolicy[]> {+ const existingPolicies = await dbClient+ .select()+ .from(toolPolicies)+ .where(+ and(+ eq(toolPolicies.companyId, input.companyId),+ eq(toolPolicies.policyType, "require_approval"),+ ),+ );+ const managedPolicies = existingPolicies.filter((policy) => {+ const config = asRecord(policy.config);+ return (+ config.source === "app_gallery_finish" &&+ config.connectionId === input.connection.id+ );+ });+ const policiesByCatalogEntryId = new Map<+ string,+ typeof toolPolicies.$inferSelect+ >();+ for (const policy of managedPolicies) {+ const config = asRecord(policy.config);+ if (typeof config.catalogEntryId === "string") {+ policiesByCatalogEntryId.set(config.catalogEntryId, policy);+ }+ }+ const askFirstIds = new Set(input.askFirstEntries.map((entry) => entry.id));+ const results: ToolPolicy[] = [];+ for (const entry of input.askFirstEntries) {+ const config = {+ source: "app_gallery_finish",+ connectionId: input.connection.id,+ catalogEntryId: entry.id,+ };+ const existing = policiesByCatalogEntryId.get(entry.id);+ if (existing) {+ const [updated] = await dbClient+ .update(toolPolicies)+ .set({+ name: policyNameForApp(input.connection, entry),+ description: `Ask first before running ${entry.toolName}.`,+ enabled: true,+ selectors: { catalogEntryId: entry.id },+ config,+ updatedAt: new Date(),+ })+ .where(eq(toolPolicies.id, existing.id))+ .returning();+ results.push(toPolicy(updated));+ } else {+ const [created] = await dbClient+ .insert(toolPolicies)+ .values({+ companyId: input.companyId,+ name: policyNameForApp(input.connection, entry),+ description: `Ask first before running ${entry.toolName}.`,+ policyType: "require_approval",+ priority: 50,+ enabled: true,+ selectors: { catalogEntryId: entry.id },+ config,+ createdByAgentId:+ input.actor?.actorType === "agent"+ ? (input.actor.actorId ?? null)+ : null,+ createdByUserId:+ input.actor?.actorType === "user"+ ? (input.actor.actorId ?? null)+ : null,+ })+ .returning();+ results.push(toPolicy(created));+ }+ }+ if (input.disableStale !== false) {+ const stalePolicies = managedPolicies.filter((policy) => {+ const config = asRecord(policy.config);+ return (+ typeof config.catalogEntryId === "string" &&+ !askFirstIds.has(config.catalogEntryId)+ );+ });+ for (const policy of stalePolicies) {+ await dbClient+ .update(toolPolicies)+ .set({ enabled: false, updatedAt: new Date() })+ .where(eq(toolPolicies.id, policy.id));+ }+ }+ return results;+ }+ return { enableCatalogEntriesByDefault, upsertAskFirstPolicies };+}+export type CatalogDefaultAccess = ReturnType<typeof createCatalogDefaultAccess>;diff --git a/server/src/services/tool-access/catalog/descriptors.ts b/server/src/services/tool-access/catalog/descriptors.tsnew file mode 100644index 0000000000..7644a70001--- /dev/null+++ b/server/src/services/tool-access/catalog/descriptors.ts@@ -0,0 +1,295 @@+import { isDaytonaConnection } from "../../daytona.js";+import { googleWorkspaceToolTarget } from "@paperclipai/shared";+import { BROWSER_USE_TOOLS } from "@paperclipai/shared";+import { isMemoryConnectorId } from "@paperclipai/shared";+import { toolConnections } from "@paperclipai/db";+import type { ToolRiskLevel } from "@paperclipai/shared";+import { GOOGLE_WORKSPACE_CONNECTOR_PROFILES, type GoogleWorkspaceConnectorProfileId } from "@paperclipai/shared";+import { connectorToolPolicy } from "../../connectors/tool-policy.js";++function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++export type McpToolDescriptor = {+ name: string;+ title?: string | null;+ description?: string | null;+ inputSchema?: Record<string, unknown>;+ annotations?: Record<string, unknown>;+};++export function normalizeToolDescriptor(tool: unknown): McpToolDescriptor | null {+ const record = asRecord(tool);+ if (typeof record.name !== "string" || record.name.trim().length === 0)+ return null;+ return {+ name: record.name.trim(),+ title: typeof record.title === "string" ? record.title : null,+ description:+ typeof record.description === "string" ? record.description : null,+ inputSchema: asRecord(record.inputSchema ?? record.input_schema),+ annotations: asRecord(record.annotations),+ };+}++// Match a verb anywhere it forms a name segment, not just at the leading edge.+// Real MCP servers namespace and style tool names many ways:+// "github:create_issue", "notion:update_page", "slack:postMessage", "set_value".+// A leading-anchor regex (/^(create|...)/) misses every namespaced/camelCase+// form and silently classifies writes as read-only. We normalise camelCase to+// snake_case first so "postMessage" -> "post_message", then match the verb when+// it is delimiter- or word-bounded. This mirrors the gateway classifier in+// tool-gateway.ts (inferToolRisk) so the two stay consistent.+function verbMatches(toolName: string, verbs: string): boolean {+ const normalized = toolName+ .replace(/([a-z0-9])([A-Z])/g, "$1_$2")+ .toLowerCase();+ return new RegExp(`\\b(${verbs})\\b|(^|[:._-])(${verbs})([:._-]|$)`).test(+ normalized,+ );+}++const NOTION_READ_TOOLS = new Set([+ "notion-fetch",+ "notion-get-async-task",+ "notion-get-comments",+ "notion-get-teams",+ "notion-get-users",+ "notion-query-data-sources",+ "notion-query-database-view",+ "notion-query-meeting-notes",+ "notion-search",+]);++const NOTION_WRITE_TOOLS = new Set([+ "notion-convert-page-to-skill",+ "notion-create-comment",+ "notion-create-database",+ "notion-create-folder",+ "notion-create-pages",+ "notion-create-view",+ "notion-duplicate-page",+ "notion-move-pages",+ "notion-update-data-source",+ "notion-update-page",+ "notion-update-view",+]);++const SHOPIFY_DESTRUCTIVE_TOOLS = new Set([+ "cancel-cart",+ "cancel-checkout",+ "complete-checkout",+]);++function normalizedProviderToolName(toolName: string): string {+ return toolName+ .replace(/([a-z0-9])([A-Z])/g, "$1-$2")+ .toLowerCase()+ .replace(/[:._-]+/g, "-");+}++// Aggregators can add arbitrarily powerful tools without changing their MCP+// endpoint. Only these reviewed, exact capabilities are read-only. Unknown or+// renamed actions remain enabled by the usual access rules, but have write risk.+const AGGREGATOR_READ_TOOLS = new Map<string, ReadonlySet<string>>([+ ["executor", new Set(["skills"])],+ ["composio", new Set(["COMPOSIO_SEARCH_TOOLS", "COMPOSIO_GET_TOOL_SCHEMAS", "COMPOSIO_SEARCH_SKILLS", "COMPOSIO_USE_SKILL"])],+ ["arcade", new Set(["Github.GetRepository"])],+ ["zapier", new Set()],+]);++export function classifyRisk(+ tool: McpToolDescriptor,+ sourceTemplateKey?: string | null,+): ToolRiskLevel {+ if (sourceTemplateKey === "google-workspace") {+ const target = googleWorkspaceToolTarget(tool.name);+ if (!target) return "destructive";+ const risk = classifyRisk({ ...tool, name: target.upstreamName }, target.appSlug);+ if (risk === "destructive" || target.leaf === "delete_event" || target.leaf === "send_message") return "destructive";+ if (target.write) return "write";+ return risk;+ }+ const annotations = tool.annotations ?? {};+ if (annotations.destructiveHint === true || annotations.destructive === true)+ return "destructive";+ const reviewedRisk = connectorToolPolicy(sourceTemplateKey).classifyRisk?.(tool);+ if (reviewedRisk) return reviewedRisk;+ const normalizedToolName = normalizedProviderToolName(tool.name);+ if (isMemoryConnectorId(sourceTemplateKey)) {+ if (verbMatches(tool.name, "delete|remove|destroy|forget|prune|reset")) return "destructive";+ // Supermemory uses one add_memory tool for both save and forget.+ if (sourceTemplateKey === "supermemory" && normalizedToolName === "add-memory") return "destructive";+ if (verbMatches(tool.name, "add|remember|save|record|ingest|cognify|update|create|set|upload|select|rename|share")) return "write";+ if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";+ const reads = ["get", "list", "search", "recall", "query", "retrieve", "who"];+ return verbMatches(tool.name, reads.join("|")) ? "read" : "write";+ }+ const reviewedReads = AGGREGATOR_READ_TOOLS.get(sourceTemplateKey ?? "");+ if (reviewedReads) {+ if (verbMatches(tool.name, "delete|remove|destroy|unpublish")) return "destructive";+ if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";+ return reviewedReads.has(tool.name) ? "read" : "write";+ }+ if (sourceTemplateKey === "browser-use-cloud") {+ return BROWSER_USE_TOOLS.find(t => t.name === tool.name)?.annotations.readOnlyHint ? "read" : "destructive";+ }+ // Fireflies sharing, moving, and access revocation are mutations even when+ // a provider omits annotations or mistakenly advertises a read hint.+ if (sourceTemplateKey === "fireflies" && [+ "fireflies-share-meeting", "fireflies-revoke-meeting-access", "fireflies-move-meeting",+ ].includes(normalizedToolName)) return "write";+ // Gmail archive removes a system label. Keep it behind the archive method's+ // destructive Ask-first default even when the preview provider omits hints.+ if (sourceTemplateKey === "gmail" && normalizedToolName === "unlabel-thread")+ return "destructive";+ if (sourceTemplateKey === "posthog" && normalizedToolName === "exec")+ return "destructive";+ // Superagent mirrors its REST API, and several mutations use verbs the+ // generic classifier reads as reads: billable triage_finding, scans,+ // restore_agent_builtin_rule, and credential-revoking revoke_agent_client.+ // Only list/get tools and tools the provider marks read-only are reads.+ if (sourceTemplateKey === "superagent") {+ if (verbMatches(tool.name, "delete|remove|destroy|revoke")) return "destructive";+ if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";+ return /^(list|get)-/.test(normalizedToolName) || annotations.readOnlyHint === true ? "read" : "write";+ }+ // Ditto's translate, link and unlink tools replace saved text or developer+ // IDs, but the generic classifier reads those verbs as reads. Only+ // list/get/search tools, suggest_edit, and tools Ditto marks read-only are+ // reads.+ if (sourceTemplateKey === "ditto") {+ if (verbMatches(tool.name, "delete|remove|destroy")) return "destructive";+ if (annotations.readOnlyHint === false || annotations.writeHint === true) return "write";+ return /^(list|get|search)-/.test(normalizedToolName) ||+ normalizedToolName === "suggest-edit" ||+ annotations.readOnlyHint === true+ ? "read"+ : "write";+ }+ if (+ sourceTemplateKey === "shopify" &&+ SHOPIFY_DESTRUCTIVE_TOOLS.has(normalizedToolName)+ )+ return "destructive";+ // Notion's hosted MCP catalog contains mutations whose names do not use one+ // of the generic create/update/delete verbs (move, duplicate, and convert).+ // Keep all reviewed tools explicit so provider changes are visible in code,+ // while an annotation may still escalate a known read to a write.+ if (+ sourceTemplateKey === "notion" &&+ NOTION_WRITE_TOOLS.has(normalizedToolName)+ )+ return "write";+ if (annotations.readOnlyHint === false || annotations.writeHint === true)+ return "write";+ if (+ sourceTemplateKey === "notion" &&+ NOTION_READ_TOOLS.has(normalizedToolName)+ )+ return "read";+ if (verbMatches(tool.name, "delete|remove|destroy|unpublish"))+ return "destructive";+ if (+ verbMatches(+ tool.name,+ "create|update|write|set|send|publish|post|mutate|mark|archive",+ )+ )+ return "write";+ // PostHog exposes a broad and evolving catalog. Unknown tools must never be+ // silently treated as reads; provider annotations can opt known reads in.+ if (sourceTemplateKey === "posthog")+ return annotations.readOnlyHint === true ? "read" : "write";+ return "read";+}++export function isGmailToolPermanentlyBlocked(+ tool: McpToolDescriptor,+): boolean {+ if (googleWorkspaceToolLeafName(tool.name) === "unlabel-thread") return false;+ const riskLevel = classifyRisk(tool, "gmail");+ return (+ verbMatches(+ tool.name,+ "send|trash|spam|delete|remove|destroy|execute|run",+ ) ||+ (normalizedProviderToolName(tool.name).includes("label") &&+ riskLevel !== "read")+ );+}++const GOOGLE_WORKSPACE_READ_TOOLS: Record<string, ReadonlySet<string>> = {+ gmail: new Set([+ "get-message",+ "get-thread",+ "get-draft",+ "list-drafts",+ "list-labels",+ "search-threads",+ "list-threads",+ "search-messages",+ ]),+ "google-drive": new Set([+ "download-file-content",+ "get-file-metadata",+ "get-file-permissions",+ "list-recent-files",+ "read-file-content",+ "search-files",+ ]),+ "google-docs": new Set(["read-doc"]),+ "google-sheets": new Set(["get-values", "get-spreadsheet"]),+ "google-slides": new Set(["read-presentation"]),+ "google-calendar": new Set([+ "get-event",+ "list-calendars",+ "list-events",+ "search-events",+ "suggest-time",+ ]),+ "google-chat": new Set([+ "search-conversations",+ "list-messages",+ "search-messages",+ ]),+ "google-people": new Set([+ "search-directory-people",+ "search-contacts",+ "get-user-profile",+ ]),+ "google-workspace-search": new Set(["search-corpus"]),+};++function googleWorkspaceToolLeafName(name: string): string {+ return (name.split(/[.:/]/).pop() ?? name)+ .replace(/([a-z0-9])([A-Z])/g, "$1-$2")+ .replace(/_/g, "-")+ .toLowerCase();+}++export function isGoogleWorkspaceToolAllowed(+ profileId: GoogleWorkspaceConnectorProfileId,+ tool: McpToolDescriptor,+): boolean {+ if (profileId === "workspace.all") return googleWorkspaceToolTarget(tool.name) !== null;+ const profile = GOOGLE_WORKSPACE_CONNECTOR_PROFILES[profileId];+ const toolName = googleWorkspaceToolLeafName(tool.name);+ const readTools =+ GOOGLE_WORKSPACE_READ_TOOLS[profile.appSlug] ?? new Set<string>();+ return (+ readTools.has(toolName) ||+ profile.writeTools.some(+ (writeTool) => googleWorkspaceToolLeafName(writeTool) === toolName,+ )+ );+}++export function shouldQuarantineNewEntries(+ connection: typeof toolConnections.$inferSelect,+): boolean {+ return isDaytonaConnection(connection) || asRecord(connection.config).quarantineNewEntries === true;+}diff --git a/server/src/services/tool-access/catalog/discovery.ts b/server/src/services/tool-access/catalog/discovery.tsnew file mode 100644index 0000000000..deb7343a5a--- /dev/null+++ b/server/src/services/tool-access/catalog/discovery.ts@@ -0,0 +1,454 @@+import { assertSupportedConnection } from "../connections/setup-policy.js";+import { vercelCredentialFor, vercelConnectResourcesFor } from "../vercel-credentials.js";+import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js";+import type { ActorInfo } from "../types.js";+import { createOAuthDiscovery } from "../oauth/discovery.js";+import { oauthConfig } from "../oauth/metadata.js";+import { projectedConnectionHeaders } from "../connection-methods.js";+import { DAYTONA_TOOLS, daytonaClient, isDaytonaConnection } from "../../daytona.js";+import { GOOGLE_WORKSPACE_SERVICES, googleWorkspaceServiceGranted, googleWorkspaceToolName, googleWorkspaceGrantedScopes, isGoogleWorkspaceToolGranted, type GoogleWorkspaceService } from "@paperclipai/shared";+import { isInsufficientConnectionScope, INSUFFICIENT_CONNECTION_SCOPE_MESSAGE } from "../../connection-permission-errors.js";+import { mcpDiscoveryHttpFailure, withMcpConnectionFailure } from "../../mcp-connection-failure.js";+import { BROWSER_USE_TOOLS } from "@paperclipai/shared";+import { browserUseClient, isBrowserUseConnection } from "../../browser-use-client.js";+import { COGNEE_STDIO_TEMPLATE, cogneeCloudUrl } from "../../cognee-connection.js";+import { isMemoryConnectorId } from "@paperclipai/shared";+import { and, eq } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { connectionGrants, toolConnections } from "@paperclipai/db";+import { GOOGLE_WORKSPACE_CONNECTOR_PROFILES, getConnectableAppDefinition } from "@paperclipai/shared";+import { badRequest, HttpError, unprocessable } from "../../../errors.js";+import { isSlackMcpAccessDisabledResponse } from "../../slack-mcp-error.js";+import { initializeMcpHttpSession, McpHttpInitializationError, getMcpHttpSession, forgetMcpHttpSessions, readMcpHttpResponse, mcpHttpRequestHeaders } from "../../mcp-http.js";+import { secretService } from "../../secrets.js";+import { agentmailApi } from "../../agentmail-api.js";+import { createRailwayClient, discoverRailwayWorkspace, isRailwayConnection, normalizeRailwayToolName, RAILWAY_TOOLS, RAILWAY_TOOL_PREFIX, RailwayError } from "../../railway.js";+import { isPaperclipCloudConnectorStrategy } from "../../paperclip-cloud-connector.js";+import { vercelTokenRequest, type VercelConnectClient } from "../../vercel-connect.js";++import type { ToolAccessAudit } from "../audit.js";+import type { ConnectionCredentialAccess } from "../credential-access.js";+import type { VercelCredentialService } from "../vercel-credentials.js";+import type { OAuthCredentialStore } from "../oauth/credential-store.js";+import type { ConnectionSetupPolicy } from "../connections/setup-policy.js";+import type { ToolRemoteHttp } from "../remote-http.js";+import { normalizeToolDescriptor, type McpToolDescriptor } from "./descriptors.js";+import { unsupportedToolConnectionTransport } from "./failures.js";+function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++export function isAgentMailConnection(connection: typeof toolConnections.$inferSelect) {+ return connection.transport === "rest_api" && connection.config.provider === "agentmail";+}++type DiscoveryDependencies = {+ now: () => Date;+ audit: ToolAccessAudit;+ secrets: ReturnType<typeof secretService>;+ credentials: ConnectionCredentialAccess;+ vercel: VercelCredentialService;+ vercelConnect: VercelConnectClient | null;+ setupPolicy: ConnectionSetupPolicy;+ requestRemoteHttpEndpoint: ToolRemoteHttp["requestRemoteHttpEndpoint"];+ resolveOAuthGrantSecret: OAuthCredentialStore["resolveOAuthGrantSecret"];+ discoverOAuthEndpoints: ReturnType<typeof createOAuthDiscovery>["discoverOAuthEndpoints"];+ localTools(connection: typeof toolConnections.$inferSelect): Promise<McpToolDescriptor[]>;+};++export function createToolCatalogDiscovery(db: Db, { now, audit, secrets, credentials, vercel, vercelConnect, setupPolicy, requestRemoteHttpEndpoint, resolveOAuthGrantSecret, discoverOAuthEndpoints, localTools }: DiscoveryDependencies) {+ const { vaultGrantForConnection, resolveCredentialHeaders, resolvedRemoteEndpoint } = credentials;+ const { resolveVercelCredentialHeaders, vercelGrantForConnection } = vercel;+ const { assertConnectorConnectionSetupEnabled } = setupPolicy;+ async function remoteTools(+ connection: typeof toolConnections.$inferSelect,+ credentialHeaders?: Record<string, string>,+ actor?: ActorInfo,+ workspaceEndpoint?: string,+ unavailableServices: string[] = [],+ ): Promise<McpToolDescriptor[]> {+ assertSupportedConnection(connection);+ await assertConnectorConnectionSetupEnabled(connection);+ let headers = credentialHeaders ?? {+ ...projectedConnectionHeaders(connection),+ ...(await resolveCredentialHeaders(connection, actor)),+ };+ if (connection.config.sourceTemplateKey === "google-workspace" && !workspaceEndpoint) {+ const grant = await vaultGrantForConnection(connection, actor);+ const scopes = googleWorkspaceGrantedScopes(grant);+ const services = (Object.keys(GOOGLE_WORKSPACE_SERVICES) as GoogleWorkspaceService[])+ .filter((service) => googleWorkspaceServiceGranted(service, scopes));+ // One credential authority, no child connections or implied service grants.+ const catalogs = await Promise.allSettled(services.map(async (service) => {+ const profile = GOOGLE_WORKSPACE_CONNECTOR_PROFILES[GOOGLE_WORKSPACE_SERVICES[service]];+ const tools = await remoteTools(connection, headers, actor, profile.serverUrl);+ return tools.map((tool) => ({ ...tool, name: googleWorkspaceToolName(service, tool.name) }))+ .filter((tool) => isGoogleWorkspaceToolGranted(tool.name, scopes));+ }));+ for (const [index, catalog] of catalogs.entries()) {+ if (catalog.status === "fulfilled") continue;+ const service = services[index]!;+ unavailableServices.push(service);+ await audit({ companyId: connection.companyId, connectionId: connection.id,+ action: "tool_connection.catalog_refresh", outcome: "failure", actor,+ reasonCode: "google_workspace_service_unavailable", details: { service } });+ }+ const currentGrant = await vaultGrantForConnection(connection, actor);+ if (!currentGrant) throw unprocessable("OAuth authorization must be reconnected", { code: "oauth_reauthorization_required" });+ if (catalogs.length > 0 && catalogs.every((catalog) => catalog.status === "rejected")) {+ throw new HttpError(502, `Google services unavailable: ${unavailableServices.join(", ")}. Retry refreshing actions.`,+ { code: "google_workspace_services_unavailable" });+ }+ // A service's authentication retry may have refreshed into a narrower+ // grant. Never publish actions using the pre-refresh permission snapshot.+ const currentScopes = googleWorkspaceGrantedScopes(currentGrant);+ return catalogs.flatMap((catalog) => catalog.status === "fulfilled" ? catalog.value : [])+ .filter((tool) => isGoogleWorkspaceToolGranted(tool.name, currentScopes));+ }+ const endpoint = workspaceEndpoint ?? await resolvedRemoteEndpoint(connection, actor);+ // Pinned to the address the guard approved: `config.url` is operator-supplied,+ // so a second DNS resolution here would reopen the rebinding window that+ // PAP-17098 closed for the OAuth endpoints.+ let listRequestId = "paperclip-catalog-refresh";+ let sessionHeaders = headers;+ const sendRemote = (init: RequestInit) => withMcpConnectionFailure(() => requestRemoteHttpEndpoint(new URL(endpoint), init));+ const sendToolsList = (requestHeaders: Record<string, string>, cursor?: string) => {+ sessionHeaders = requestHeaders;+ return sendRemote({ method: "POST", headers: mcpHttpRequestHeaders(requestHeaders),+ body: JSON.stringify({ jsonrpc: "2.0", id: listRequestId, method: "tools/list", params: cursor ? { cursor } : {} }) });+ };+ const connectionConfig = asRecord(connection.config);+ const sourceTemplateKey = typeof connectionConfig?.sourceTemplateKey === "string"+ ? connectionConfig.sourceTemplateKey+ : null;+ const connectionMethodKey = typeof connectionConfig?.connectionMethodKey === "string"+ ? connectionConfig.connectionMethodKey+ : null;+ const curatedMethodRequiresMcpSession = Boolean(+ sourceTemplateKey && connectionMethodKey &&+ getConnectableAppDefinition(sourceTemplateKey)?.methods.some((method) =>+ method.key === connectionMethodKey && method.defaults?.mcpSessionRequired === true+ ),+ );+ let usedInitializedSession = connectionConfig?.mcpSessionRequired === true || curatedMethodRequiresMcpSession;+ let response: Response;+ if (usedInitializedSession) {+ try {+ sessionHeaders = await getMcpHttpSession({ send: sendRemote, headers,+ scope: `${connection.id}:catalog:${actor?.actorType}:${actor?.actorId}:${endpoint}`,+ requestId: listRequestId });+ response = await sendToolsList(sessionHeaders);+ } catch (error) {+ if (!(error instanceof McpHttpInitializationError) || !error.response) throw error;+ response = error.response;+ }+ } else {+ response = await sendToolsList(headers);+ // `tools/list` is read-only, so a 400 can safely be retried after the MCP+ // initialize handshake. Stateful servers such as Supabase require the+ // returned Mcp-Session-Id on every non-initialization request.+ if (response.status === 400) {+ try {+ const sessionHeaders = await initializeMcpHttpSession({+ send: sendRemote,+ headers,+ requestId: "paperclip-catalog-refresh",+ });+ response = await sendToolsList(sessionHeaders);+ usedInitializedSession = response.ok;+ } catch {+ // Preserve the original HTTP failure below when this was not an MCP+ // session requirement after all.+ }+ }+ }+ if (response.status === 404 && new Headers(sessionHeaders).has("mcp-session-id")) {+ // MCP uses 404 for an expired server session. Discovery is read-only, so+ // discard the stale session and retry it once with a new handshake.+ forgetMcpHttpSessions(connection.id);+ await response.body?.cancel().catch(() => undefined);+ sessionHeaders = await getMcpHttpSession({ send: sendRemote, headers,+ scope: `${connection.id}:catalog:${actor?.actorType}:${actor?.actorId}:${endpoint}`,+ requestId: listRequestId });+ response = await sendToolsList(sessionHeaders);+ if (response.status === 404) forgetMcpHttpSessions(connection.id);+ }+ if (+ usedInitializedSession &&+ connectionConfig?.mcpSessionRequired !== true+ ) {+ const nextConfig = { ...connection.config, mcpSessionRequired: true };+ await db+ .update(toolConnections)+ .set({+ config: nextConfig,+ transportConfig: nextConfig,+ updatedAt: now(),+ })+ .where(+ and(+ eq(toolConnections.id, connection.id),+ eq(toolConnections.companyId, connection.companyId),+ ),+ );+ }+ if (isInsufficientConnectionScope(response)) {+ await response.body?.cancel().catch(() => undefined);+ throw unprocessable(INSUFFICIENT_CONNECTION_SCOPE_MESSAGE, { code: "oauth_insufficient_scope", connectionId: connection.id });+ }+ if (+ response.status === 401 &&+ connection.credentialSource === "vercel_connect"+ ) {+ const grant = await vercelGrantForConnection(connection, actor);+ const credential = vercelCredentialFor(connection);+ const request = vercelTokenRequest({+ credential,+ grant,+ connectionId: connection.id,+ companyId: connection.companyId,+ resources: vercelConnectResourcesFor(connection),+ });+ vercelConnect?.evict(request);+ headers = {+ ...projectedConnectionHeaders(connection),+ ...(await resolveVercelCredentialHeaders(connection, grant, {+ forceRefresh: true,+ })),+ };+ response = await sendToolsList(headers);+ }+ if (+ response.status === 401 &&+ connection.authKind === "oauth" &&+ connection.credentialSource === "paperclip_vault"+ ) {+ headers = {+ ...projectedConnectionHeaders(connection),+ ...(await resolveCredentialHeaders(connection, actor, {+ forceRefresh: true,+ })),+ };+ response = await sendToolsList(headers);+ if (+ response.status === 401 &&+ isPaperclipCloudConnectorStrategy(oauthConfig(connection).strategy)+ ) {+ const grant = await vaultGrantForConnection(connection, actor);+ if (grant) {+ await db+ .update(connectionGrants)+ .set({ status: "needs_reauthorization", updatedAt: now() })+ .where(+ and(+ eq(connectionGrants.id, grant.id),+ eq(connectionGrants.companyId, connection.companyId),+ ),+ );+ }+ }+ }+ if (!response.ok) {+ if ((response.status === 401 || response.status === 403)+ && connection.authKind === "api_key" && isMemoryConnectorId(connection.config.sourceTemplateKey)) {+ throw unprocessable("The provider rejected this API key. Check the key and its account access, then try again.", { code: "memory_api_key_rejected" });+ }+ if (await isSlackMcpAccessDisabledResponse(endpoint, response)) {+ throw unprocessable(+ "Slack MCP access is disabled for this app. Ask the Slack app owner to enable MCP access, then refresh this connection.",+ { code: "slack_mcp_access_disabled", setupUrl: connectionSetupUrl(connection) },+ );+ }+ const authenticate = response.headers.get("www-authenticate") ?? "";+ if (+ response.status === 401 &&+ /bearer|oauth|authorization/i.test(authenticate)+ ) {+ const endpoints = await discoverOAuthEndpoints(+ connection,+ authenticate,+ );+ if (endpoints) {+ const nextConfig = {+ ...connection.config,+ oauth: {+ ...oauthConfig(connection),+ provider: endpoints.provider,+ authorizationUrl: endpoints.authorizationUrl,+ tokenUrl: endpoints.tokenUrl,+ registrationUrl: endpoints.registrationUrl ?? null,+ metadataUrl: endpoints.metadataUrl ?? null,+ scopes: endpoints.scopes,+ codeChallengeMethodsSupported:+ endpoints.codeChallengeMethodsSupported ?? [],+ tokenEndpointAuthMethodsSupported:+ endpoints.tokenEndpointAuthMethodsSupported ?? [],+ grantType: endpoints.grantType ?? "authorization_code",+ issuer: endpoints.issuer ?? null,+ resource: endpoints.resource ?? null,+ clientIdMetadataDocumentSupported:+ endpoints.clientIdMetadataDocumentSupported === true,+ discoveredAt: new Date().toISOString(),+ },+ };+ await db+ .update(toolConnections)+ .set({+ // Record the discovered auth kind now, so a URL-only connection that+ // is waiting on sign-in reads as OAuth everywhere rather than+ // falling back to `authKind: none` semantics.+ authKind: "oauth",+ config: nextConfig,+ transportConfig: nextConfig,+ updatedAt: new Date(),+ })+ .where(eq(toolConnections.id, connection.id));+ }+ throw unprocessable("This app needs you to sign in.", {+ code: "oauth_challenge",+ status: response.status,+ setupUrl: connectionSetupUrl(connection),+ reconnectUrl: connectionReconnectUrl(connection),+ oauthSupported: Boolean(endpoints),+ });+ }+ throw mcpDiscoveryHttpFailure(response, `Remote app returned HTTP ${response.status}`);+ }+ const descriptors: McpToolDescriptor[] = [];+ const seenCursors = new Set<string>();+ for (let page = 0; ; page += 1) {+ const payload = await withMcpConnectionFailure(() => readMcpHttpResponse(response, listRequestId));+ const record = asRecord(payload);+ if (record.error) throw new HttpError(502, "Remote MCP tool discovery failed", { code: "mcp_catalog_error" });+ const result = asRecord(record.result);+ if (!Array.isArray(result.tools)) throw new HttpError(502, "Remote MCP returned an invalid tool catalog", { code: "mcp_catalog_invalid" });+ descriptors.push(...result.tools.map((tool) => normalizeToolDescriptor(tool)).filter((tool): tool is McpToolDescriptor => Boolean(tool)));+ const cursor = typeof result.nextCursor === "string" ? result.nextCursor : null;+ if (!cursor) break;+ if (seenCursors.has(cursor) || page >= 99 || descriptors.length > 20_000) throw new HttpError(502, "Remote MCP tool catalog pagination did not finish", { code: "mcp_catalog_pagination" });+ seenCursors.add(cursor);+ listRequestId = `paperclip-catalog-refresh-${page + 1}`;+ response = await sendToolsList(sessionHeaders, cursor);+ if (!response.ok) throw mcpDiscoveryHttpFailure(response, "Remote MCP catalog page could not be read");+ }+ if (!isRailwayConnection(connection)) return descriptors;+ if (descriptors.some((tool) => normalizeRailwayToolName(tool.name).startsWith(RAILWAY_TOOL_PREFIX))) {+ throw unprocessable("Railway advertised a reserved Paperclip action name. Refresh is blocked pending review.", { code: "railway_tool_name_collision" });+ }+ let apiStatus = "available";+ let apiMessage = "Direct Railway service, log, and deployment tools are available.";+ try {+ const railwayOptions = {+ authorization: headers.Authorization ?? "",+ request: (url: string, init: RequestInit) => requestRemoteHttpEndpoint(new URL(url), init),+ signal: AbortSignal.timeout(15_000),+ };+ const workspaceId = await discoverRailwayWorkspace(railwayOptions);+ await createRailwayClient(railwayOptions).probe(workspaceId);+ } catch (error) {+ apiStatus = "unavailable";+ apiMessage = error instanceof RailwayError ? error.message : "Railway API access could not be verified. Refresh actions or reconnect Railway.";+ }+ // API interoperability is verified with the actual credential; the presence+ // of an OAuth token alone never enables the additional capability surface.+ const nextConfig = { ...connection.config, railwayApiStatus: apiStatus, railwayApiMessage: apiMessage };+ await db.update(toolConnections).set({ config: nextConfig, transportConfig: nextConfig, updatedAt: now() }).where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId)));+ connection.config = nextConfig;+ connection.transportConfig = nextConfig;+ return apiStatus === "available" ? [...descriptors, ...RAILWAY_TOOLS] : descriptors;+ }++ async function validateCogneeConnection(connection: typeof toolConnections.$inferSelect, actor?: ActorInfo, probe = true) {+ if (connection.config.templateId !== "paperclip.cognee-cloud") return;+ const grant = await vaultGrantForConnection(connection, actor);+ const refs = grant?.credentialSecretRefs ?? connection.credentialSecretRefs;+ const values: Record<string, string> = {};+ for (const key of COGNEE_STDIO_TEMPLATE.envKeys) {+ const ref = refs.find((candidate) => candidate.configPath === `env.${key}`);+ if (!ref) throw unprocessable("Reconnect Cognee to restore its Cloud credentials", { code: "missing_secret" });+ values[key] = grant+ ? (await resolveOAuthGrantSecret(connection, grant, ref, actor, undefined)).value+ : await secrets.resolveSecretValue(connection.companyId, ref.secretId, ref.versionSelector ?? "latest", {+ consumerType: "tool_connection", consumerId: connection.id, configPath: ref.configPath,+ actorType: "system", actorId: null,+ });+ }+ let base: URL;+ try { base = cogneeCloudUrl(values.COGNEE_BASE_URL!); }+ catch { throw badRequest("Copy the tenant API Base URL from Cognee’s API Keys page."); }+ // Scheduled health checks validate configuration and vault access. A transient+ // Cloud probe must not withdraw an already assigned local tool catalog.+ if (!probe) return;+ const response = await requestRemoteHttpEndpoint(new URL("/api/v1/datasets/", base), {+ method: "GET", headers: { "X-Api-Key": values.COGNEE_API_KEY! }, signal: AbortSignal.timeout(15_000),+ });+ await response.body?.cancel();+ if (!response.ok) throw unprocessable("Cognee Cloud could not verify access. Check the tenant URL, API key, and workspace subscription.", { code: "cognee_access_unverified" });+ }++ async function validateAgentMailConnection(+ connection: typeof toolConnections.$inferSelect,+ ) {+ const configPath = connection.config.emailCredential+ ? "credentials.controlKey"+ : "credentials.apiKey";+ const ref = connection.credentialSecretRefs.find(+ (candidate) => candidate.configPath === configPath,+ );+ if (!ref) {+ throw unprocessable("Reconnect AgentMail to restore its API key", {+ code: "missing_secret",+ });+ }+ const key = await secrets.resolveSecretValue(+ connection.companyId,+ ref.secretId,+ ref.versionSelector ?? "latest",+ {+ consumerType: "tool_connection",+ consumerId: connection.id,+ configPath,+ actorType: "system",+ actorId: null,+ },+ );+ await agentmailApi(key).whoami();+ }++ async function discoverTools(+ connection: typeof toolConnections.$inferSelect,+ credentialHeaders?: Record<string, string>,+ actor?: ActorInfo,+ unavailableServices: string[] = [],+ ): Promise<McpToolDescriptor[]> {+ assertSupportedConnection(connection);+ await assertConnectorConnectionSetupEnabled(connection);+ if (connection.connectionPurpose === "ai") throw unprocessable("AI connections provide runtime authentication, not tool actions");+ if (isDaytonaConnection(connection)) {+ const headers = credentialHeaders ?? await resolveCredentialHeaders(connection, actor);+ await daytonaClient(headers, (url, init) => requestRemoteHttpEndpoint(new URL(url), init), asRecord(connection.config.methodConfig).organizationId).probe();+ return DAYTONA_TOOLS;+ }+ if (isBrowserUseConnection(connection)) {+ const headers = credentialHeaders ?? await resolveCredentialHeaders(connection, actor);+ await browserUseClient(headers, (url, init) => requestRemoteHttpEndpoint(new URL(url), init), connection.id).probe();+ return BROWSER_USE_TOOLS;+ }+ if (isAgentMailConnection(connection)) {+ await validateAgentMailConnection(connection);+ return [];+ }+ if (connection.transport === "mcp_remote")+ return remoteTools(connection, credentialHeaders, actor, undefined, unavailableServices);+ if (connection.transport !== "local_stdio") {+ throw unsupportedToolConnectionTransport();+ }+ await resolveCredentialHeaders(connection);+ await validateCogneeConnection(connection, actor);+ return localTools(connection);+ }+ return { remoteTools, validateCogneeConnection, validateAgentMailConnection, discoverTools };+}+export type ToolCatalogDiscovery = ReturnType<typeof createToolCatalogDiscovery>;diff --git a/server/src/services/tool-access/catalog/failures.ts b/server/src/services/tool-access/catalog/failures.tsnew file mode 100644index 0000000000..8a5eb3b352--- /dev/null+++ b/server/src/services/tool-access/catalog/failures.ts@@ -0,0 +1,154 @@+import type { ToolConnectionHealthStatus } from "@paperclipai/shared";+import { HttpError, unprocessable } from "../../../errors.js";++function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++/**+ * Did this error come from the OAuth endpoint gate (PAP-17099)? Such a refusal+ * is Paperclip's own decision about an unsafe address, so it must keep its code+ * and its 422 instead of being folded into a generic upstream failure.+ */+export function isOAuthEndpointRejection(error: unknown): boolean {+ if (!(error instanceof HttpError)) return false;+ const code = asRecord(error.details).code;+ return typeof code === "string" && code.endsWith("_endpoint_rejected");+}++export function healthFailureHttpStatus(failure: {+ status: ToolConnectionHealthStatus;+ code: string;+}): number {+ if (failure.status === "missing_secret") return 422;+ if (failure.code === "oauth_challenge") return 422;+ if (failure.code === "oauth_refresh_missing") return 422;+ if (failure.code === "oauth_reauthorization_required") return 422;+ if (failure.code === "oauth_insufficient_scope") return 422;+ if (failure.code === "slack_mcp_access_disabled") return 422;+ if (failure.code === "user_authorization_required") return 422;+ if (failure.code === "composio_broker_retired") return 422;+ if (failure.code === "tool_connection_transport_unsupported") return 422;+ if (failure.code === "cognee_access_unverified" || failure.code === "memory_api_key_rejected") return 422;+ if (failure.code.endsWith("_endpoint_rejected")) return 422;+ return 502;+}++export function unsupportedToolConnectionTransport() {+ return unprocessable(+ "This connection has no supported tool integration. Add a supported account or MCP connection from Connectors.",+ { code: "tool_connection_transport_unsupported" },+ );+}++export function sanitizeHttpFailure(error: unknown): {+ status: ToolConnectionHealthStatus;+ message: string;+ code: string;+} {+ if (error instanceof HttpError) {+ const code = asRecord(error.details).code;+ if (code === "cognee_access_unverified" || code === "memory_api_key_rejected") {+ return { status: "error", message: error.message, code };+ }+ if (code === "grant_credential_invalid" || code === "oauth_insufficient_scope") {+ return { status: code === "grant_credential_invalid" ? "missing_secret" : "degraded", message: error.message, code };+ }+ if (code === "slack_mcp_access_disabled" || code === "google_workspace_services_unavailable") {+ return { status: "error", message: error.message, code };+ }+ if (code === "user_authorization_required") {+ return { status: "error", message: error.message, code };+ }+ if (code === "tool_connection_transport_unsupported") {+ return { status: "error", message: error.message, code };+ }+ if (code === "composio_broker_retired") {+ return { status: "error", message: error.message, code };+ }+ if (typeof code === "string" && code.startsWith("remote_http_")) {+ return { status: "error", message: error.message, code };+ }+ if (isOAuthEndpointRejection(error)) {+ return { status: "error", message: error.message, code: String(code) };+ }+ if (code === "oauth_challenge") {+ return {+ status: "error",+ message: "This app needs you to sign in.",+ code: "oauth_challenge",+ };+ }+ if (code === "oauth_refresh_missing") {+ return {+ status: "failed",+ message: "OAuth credentials have expired and need to be reconnected.",+ code: "oauth_refresh_missing",+ };+ }+ if (code === "oauth_reauthorization_required") {+ return {+ status: "error",+ message: "OAuth authorization expired. Reconnect this app to continue.",+ code: "oauth_reauthorization_required",+ };+ }+ if (typeof code === "string" && code.startsWith("vercel_connect_")) {+ return {+ status:+ code === "vercel_connect_unavailable" ||+ code === "vercel_connect_auth_failed" ||+ code === "vercel_connect_installation_required"+ ? "degraded"+ : "error",+ message: error.message,+ code,+ };+ }+ if (+ code === "oauth_refresh_in_progress" ||+ code === "oauth_refresh_superseded" ||+ code === "oauth_refresh_outcome_unknown"+ ) {+ return {+ status: "error",+ message: error.message,+ code,+ };+ }+ if (+ code === "binding_missing" ||+ code === "secret_deleted" ||+ code === "secret_inactive" ||+ code === "version_missing"+ ) {+ return {+ status: "missing_secret",+ message: "A configured credential secret could not be resolved.",+ code: String(code),+ };+ }+ if (error.status === 404 && /secret/i.test(error.message)) {+ return {+ status: "missing_secret",+ message: "A configured credential secret could not be resolved.",+ code: "secret_missing",+ };+ }+ return { status: "error", message: error.message, code: "paperclip_error" };+ }+ if (error instanceof Error) {+ return {+ status: "error",+ message: error.message.slice(0, 240),+ code: "runtime_error",+ };+ }+ return {+ status: "error",+ message: "Connection check failed.",+ code: "runtime_error",+ };+}diff --git a/server/src/services/tool-access/catalog/health.ts b/server/src/services/tool-access/catalog/health.tsnew file mode 100644index 0000000000..e05be18a92--- /dev/null+++ b/server/src/services/tool-access/catalog/health.ts@@ -0,0 +1,273 @@+import { assertSupportedConnection } from "../connections/setup-policy.js";+import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js";+import { createGitHubGrantContinuity } from "../github-grant-continuity.js";+import { createToolRuntimeSlots } from "../runtime/slots.js";+import type { ActorInfo } from "../types.js";+import { isDaytonaConnection } from "../../daytona.js";+import { retainMcpConnectionFailure } from "../../mcp-connection-failure.js";+import { isBrowserUseConnection } from "../../browser-use-client.js";+import { and, asc, eq, isNull, lte, ne, or, sql } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { connectionGrants, toolApplications, toolConnections, toolRuntimeSlots } from "@paperclipai/db";+import type { ToolConnectionHealthCheckResult, ToolConnectionHealthStatus } from "@paperclipai/shared";+import { HttpError, unprocessable } from "../../../errors.js";++import type { ToolAccessAudit } from "../audit.js";+import type { ConnectionCredentialAccess } from "../credential-access.js";+import type { ConnectionSetupPolicy } from "../connections/setup-policy.js";+import { toConnection } from "../connections/projection.js";+import { isAgentMailConnection, type ToolCatalogDiscovery } from "./discovery.js";+import { healthFailureHttpStatus, sanitizeHttpFailure, unsupportedToolConnectionTransport } from "./failures.js";+function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++type HealthDependencies = {+ now: () => Date;+ audit: ToolAccessAudit;+ getConnectionRow(idOrUid: string, companyId?: string): Promise<typeof toolConnections.$inferSelect>;+ discovery: ToolCatalogDiscovery;+ setupPolicy: ConnectionSetupPolicy;+ resolveCredentialHeaders: ConnectionCredentialAccess["resolveCredentialHeaders"];+ refreshManagedGitHubGrantAccess: ReturnType<typeof createGitHubGrantContinuity>["refreshManagedGitHubGrantAccess"];+ ensureRuntimeSlot: ReturnType<typeof createToolRuntimeSlots>["ensureRuntimeSlot"];+ stdioTemplateId(companyId: string, config: Record<string, unknown>): Promise<string>;+};++export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, discovery, setupPolicy, resolveCredentialHeaders, refreshManagedGitHubGrantAccess, ensureRuntimeSlot, stdioTemplateId }: HealthDependencies) {+ const { discoverTools, remoteTools, validateAgentMailConnection, validateCogneeConnection } = discovery;+ const { assertConnectorConnectionSetupEnabled } = setupPolicy;+ async function updateConnectionHealth(+ connection: typeof toolConnections.$inferSelect,+ status: ToolConnectionHealthStatus,+ message: string | null,+ ) {+ const now = new Date();+ const [updated] = await db+ .update(toolConnections)+ .set({+ healthStatus: status,+ healthMessage: message,+ healthCheckedAt: now,+ lastHealthAt: now,+ lastError: status === "ok" ? null : message,+ updatedAt: now,+ })+ .where(eq(toolConnections.id, connection.id))+ .returning();+ if (connection.transport === "local_stdio") {+ await db+ .update(toolRuntimeSlots)+ .set({+ healthStatus: status,+ healthMessage: message,+ lastHealthCheckAt: now,+ updatedAt: now,+ })+ .where(eq(toolRuntimeSlots.connectionId, connection.id));+ }+ return updated;+ }++ async function checkConnectionHealth(+ connectionId: string,+ actor?: ActorInfo,+ options: { allowUnauthenticatedProbe?: boolean } = {},+ ): Promise<ToolConnectionHealthCheckResult> {+ const connection = await getConnectionRow(connectionId);+ if (connection.connectionPurpose === "ai") return { connection: toConnection(connection), runtimeSlot: null };+ await assertConnectorConnectionSetupEnabled(connection);+ const unavailableServices: string[] = [];+ try {+ assertSupportedConnection(connection);+ const config = asRecord(connection.config);+ const oauth = asRecord(config.oauth);+ if (+ config.sourceTemplateKey === "github" &&+ oauth.connectorProfile === "github.code"+ ) {+ const activeGrants = await db+ .select()+ .from(connectionGrants)+ .where(+ and(+ eq(connectionGrants.companyId, connection.companyId),+ eq(connectionGrants.connectionId, connection.id),+ eq(connectionGrants.status, "active"),+ ),+ );+ const actorGrant =+ actor?.actorType === "user"+ ? activeGrants.find(+ (grant) =>+ grant.kind === "user" &&+ grant.subjectUserId === actor.actorId,+ )+ : null;+ const grantsToCheck =+ connection.credentialPolicy === "per_user" &&+ actor?.actorType === "user"+ ? actorGrant+ ? [actorGrant]+ : []+ : actorGrant+ ? [actorGrant]+ : activeGrants;+ if (grantsToCheck.length === 0)+ throw unprocessable("GitHub authorization must be connected", {+ code: "oauth_reauthorization_required",+ });+ for (const grant of grantsToCheck)+ await refreshManagedGitHubGrantAccess(connection, grant, actor);+ } else if (isBrowserUseConnection(connection) || isDaytonaConnection(connection)) {+ await discoverTools(connection, undefined, actor);+ } else if (isAgentMailConnection(connection)) {+ await validateAgentMailConnection(connection);+ } else if (connection.transport === "mcp_remote") {+ const canProbeWithoutAuthorization =+ options.allowUnauthenticatedProbe === true &&+ connection.status === "draft" &&+ connection.authKind === "none" &&+ connection.credentialPolicy === "per_user" &&+ actor?.actorType === "user" &&+ actor.actorId === connection.createdByUserId;+ const credentialHeaders =+ connection.credentialSource === "vercel_connect"+ ? await resolveCredentialHeaders(connection, actor, {+ forceRefresh: true,+ })+ : canProbeWithoutAuthorization+ ? {}+ : undefined;+ await remoteTools(connection, credentialHeaders, actor, undefined, unavailableServices);+ } else if (connection.transport === "local_stdio") {+ await resolveCredentialHeaders(connection);+ await stdioTemplateId(connection.companyId, connection.config);+ await validateCogneeConnection(connection, actor, false);+ } else {+ throw unsupportedToolConnectionTransport();+ }+ const updated = await updateConnectionHealth(+ connection,+ "ok",+ unavailableServices.length > 0+ ? `Connected. Google services unavailable: ${unavailableServices.join(", ")}. Other services remain available; retry refreshing actions.`+ : config.sourceTemplateKey === "github" &&+ oauth.connectorProfile === "github.code"+ ? "GitHub account, installation, and repository access are available."+ : isDaytonaConnection(connection)+ ? "Daytona API access is connected."+ : isBrowserUseConnection(connection)+ ? "Browser Use API key is connected."+ : isAgentMailConnection(connection)+ ? "AgentMail API key is connected."+ : connection.transport === "local_stdio"+ ? "Approved stdio template is ready."+ : "Remote MCP server responded to tools/list.",+ );+ const runtimeSlot = await ensureRuntimeSlot(updated);+ await audit({+ companyId: connection.companyId,+ connectionId: connection.id,+ action: "tool_connection.health_check",+ outcome: "success",+ actor,+ details: { transport: connection.transport },+ });+ return { connection: toConnection(updated), runtimeSlot };+ } catch (error) {+ if (+ error instanceof HttpError &&+ asRecord(error.details).code === "github_access_changed"+ )+ throw error;+ const failure = sanitizeHttpFailure(error);+ const updated = await updateConnectionHealth(+ connection,+ failure.status,+ failure.message,+ );+ const runtimeSlot =+ connection.transport === "local_stdio"+ ? await ensureRuntimeSlot(updated)+ : null;+ await audit({+ companyId: connection.companyId,+ connectionId: connection.id,+ action: "tool_connection.health_check",+ outcome: "failure",+ reasonCode: failure.code,+ actor,+ details: { status: failure.status, transport: connection.transport },+ });+ throw retainMcpConnectionFailure(error, new HttpError(healthFailureHttpStatus(failure), failure.message, {+ code: failure.code,+ connection: toConnection(updated),+ runtimeSlot,+ setupUrl: connectionSetupUrl(connection),+ reconnectUrl: connectionReconnectUrl(connection),+ }));+ }+ }++ async function sweepConnectionHealth(+ input: { staleAfterMs?: number; limit?: number } = {},+ ) {+ const generatedAt = now();+ const staleAfterMs = input.staleAfterMs ?? 15 * 60 * 1000;+ const limit = input.limit ?? 25;+ const cutoff = new Date(generatedAt.getTime() - staleAfterMs);+ // Legacy plugin backfills use a remote transport as a placeholder, but+ // their tools run in the plugin worker and have no remote MCP endpoint.+ // Select only due IDs in SQL so each scheduler tick does not decode every+ // active connection's config and credential metadata.+ const due = await db+ .select({ id: toolConnections.id })+ .from(toolConnections)+ .innerJoin(toolApplications, and(+ eq(toolApplications.id, toolConnections.applicationId),+ eq(toolApplications.companyId, toolConnections.companyId),+ ))+ .where(+ and(+ eq(toolConnections.enabled, true),+ eq(toolConnections.status, "active"),+ ne(toolConnections.transport, "chat_sdk"),+ ne(toolConnections.transport, "runtime_auth"),+ ne(toolApplications.type, "paperclip_plugin"),+ or(isNull(toolConnections.healthCheckedAt), lte(toolConnections.healthCheckedAt, cutoff)),+ ),+ )+ .orderBy(+ sql`${toolConnections.healthCheckedAt} asc nulls first`,+ asc(toolConnections.createdAt),+ asc(toolConnections.id),+ )+ .limit(limit);+ let healthy = 0;+ let failed = 0;+ const failedConnectionIds: string[] = [];+ for (const connection of due) {+ try {+ await checkConnectionHealth(connection.id, {+ actorType: "system",+ actorId: "tool_health_sweep",+ });+ healthy += 1;+ } catch {+ failed += 1;+ failedConnectionIds.push(connection.id);+ }+ }+ return {+ checked: due.length,+ healthy,+ failed,+ failedConnectionIds,+ };+ }+ return { updateConnectionHealth, checkConnectionHealth, sweepConnectionHealth };+}+export type ConnectionHealth = ReturnType<typeof createConnectionHealth>;diff --git a/server/src/services/tool-access/catalog/projection.ts b/server/src/services/tool-access/catalog/projection.tsnew file mode 100644index 0000000000..89fdc05229--- /dev/null+++ b/server/src/services/tool-access/catalog/projection.ts@@ -0,0 +1,39 @@+import { toolCatalogEntries } from "@paperclipai/db";+import type { ToolCatalogEntry } from "@paperclipai/shared";++export function toCatalogEntry(+ row: typeof toolCatalogEntries.$inferSelect,+): ToolCatalogEntry {+ return {+ id: row.id,+ companyId: row.companyId,+ applicationId: row.applicationId,+ connectionId: row.connectionId,+ entryKind: row.entryKind,+ name: row.name,+ toolName: row.toolName,+ title: row.title,+ description: row.description,+ inputSchema: row.inputSchema ?? {},+ outputSchema: row.outputSchema ?? null,+ annotations: row.annotations ?? {},+ riskLevel: row.riskLevel,+ isReadOnly: row.isReadOnly,+ isWrite: row.isWrite,+ isDestructive: row.isDestructive,+ status: row.status,+ addedAt: row.firstSeenAt,+ version: row.version,+ versionHash: row.versionHash,+ schemaHash: row.schemaHash,+ firstSeenAt: row.firstSeenAt,+ lastSeenAt: row.lastSeenAt,+ reviewedAt: row.reviewedAt,+ reviewedByAgentId: row.reviewedByAgentId,+ reviewedByUserId: row.reviewedByUserId,+ quarantinedAt: row.quarantinedAt,+ quarantineReason: row.quarantineReason,+ createdAt: row.createdAt,+ updatedAt: row.updatedAt,+ };+}diff --git a/server/src/services/tool-access/catalog/publication.ts b/server/src/services/tool-access/catalog/publication.tsnew file mode 100644index 0000000000..f60d00f213--- /dev/null+++ b/server/src/services/tool-access/catalog/publication.ts@@ -0,0 +1,338 @@+import { connectorToolPolicy } from "../../connectors/tool-policy.js";+import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js";+import { createToolRuntimeSlots } from "../runtime/slots.js";+import type { ActorInfo, ToolAccessMutationDb } from "../types.js";+import { connectionMethodForConnection } from "../connection-methods.js";+import { stableHash, descriptorHash } from "../../tool-catalog-hash.js";+import { isDaytonaConnection } from "../../daytona.js";+import { googleWorkspaceToolTarget, googleWorkspaceGrantedScopes, isGoogleWorkspaceToolGranted } from "@paperclipai/shared";+import { retainMcpConnectionFailure } from "../../mcp-connection-failure.js";+import { isRemoteMcpConnectorMethod } from "@paperclipai/shared";+import { and, eq, inArray, sql } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { connectionGrants, toolApplications, toolCatalogEntries, toolConnections, toolRuntimeSlots } from "@paperclipai/db";+import type { ToolCatalogRefreshResult } from "@paperclipai/shared";+import { getConnectableAppDefinition, isGoogleWorkspaceConnectorProfileId } from "@paperclipai/shared";+import { HttpError, unprocessable } from "../../../errors.js";+import { isRailwayEndpoint, isRailwayToolBlocked } from "../../railway.js";++import type { ToolAccessAudit } from "../audit.js";+import type { ConnectionCredentialAccess } from "../credential-access.js";+import type { ConnectionSetupPolicy } from "../connections/setup-policy.js";+import type { CatalogDefaultAccess } from "./default-access.js";+import type { ConnectionHealth } from "./health.js";+import { isAgentMailConnection, type ToolCatalogDiscovery } from "./discovery.js";+import { classifyRisk, shouldQuarantineNewEntries, isGoogleWorkspaceToolAllowed, type McpToolDescriptor } from "./descriptors.js";+import { toConnection } from "../connections/projection.js";+import { toCatalogEntry } from "./projection.js";+import { healthFailureHttpStatus, sanitizeHttpFailure } from "./failures.js";+function asRecord(value: unknown): Record<string, unknown> {+ if (value && typeof value === "object" && !Array.isArray(value))+ return value as Record<string, unknown>;+ return {};+}++type PublicationDependencies = {+ now: () => Date;+ audit: ToolAccessAudit;+ getConnectionRow(idOrUid: string, companyId?: string): Promise<typeof toolConnections.$inferSelect>;+ discoverTools: ToolCatalogDiscovery["discoverTools"];+ assertConnectorConnectionSetupEnabled: ConnectionSetupPolicy["assertConnectorConnectionSetupEnabled"];+ updateConnectionHealth: ConnectionHealth["updateConnectionHealth"];+ vaultGrantForConnection: ConnectionCredentialAccess["vaultGrantForConnection"];+ ensureRuntimeSlot: ReturnType<typeof createToolRuntimeSlots>["ensureRuntimeSlot"];+ enableCatalogEntriesByDefault: CatalogDefaultAccess["enableCatalogEntriesByDefault"];+};++export function createToolCatalogPublication(db: Db, { now, audit, getConnectionRow, discoverTools, assertConnectorConnectionSetupEnabled, updateConnectionHealth, vaultGrantForConnection, ensureRuntimeSlot, enableCatalogEntriesByDefault }: PublicationDependencies) {+ async function refreshCatalog(+ connectionId: string,+ actor?: ActorInfo,+ refreshOptions: {+ enableAllByDefault?: boolean;+ restoreDraftDefaults?: boolean;+ /** Keep first managed OAuth discovery quarantined without changing generic draft refresh semantics. */+ quarantineManagedOAuthDraft?: boolean;+ skipDefaultProfileSync?: boolean;+ credentialHeaders?: Record<string, string>;+ } = {},+ ): Promise<ToolCatalogRefreshResult> {+ const connection = await getConnectionRow(connectionId);+ if (connection.connectionPurpose === "ai") throw unprocessable("AI connections do not have a tool catalog");+ await assertConnectorConnectionSetupEnabled(connection);+ const refreshedAt = now();+ const unavailableServices: string[] = [];+ let descriptors: McpToolDescriptor[];+ try {+ descriptors = await discoverTools(+ connection,+ refreshOptions.credentialHeaders,+ actor,+ unavailableServices,+ );+ } catch (error) {+ if (+ error instanceof HttpError &&+ asRecord(error.details).code === "github_access_changed"+ )+ throw error;+ const failure = sanitizeHttpFailure(error);+ const updated = await updateConnectionHealth(+ connection,+ failure.status,+ failure.message,+ );+ await audit({+ companyId: connection.companyId,+ connectionId: connection.id,+ action: "tool_connection.catalog_refresh",+ outcome: "failure",+ reasonCode: failure.code,+ details: { status: failure.status },+ actor,+ });+ throw retainMcpConnectionFailure(error, new HttpError(healthFailureHttpStatus(failure), failure.message, {+ code: failure.code,+ setupUrl: connectionSetupUrl(connection),+ reconnectUrl: connectionReconnectUrl(connection),+ }));+ }++ const publish = <T>(write: (client: ToolAccessMutationDb) => Promise<T>): Promise<T> => db.transaction(async tx => {+ await tx.select({ id: toolApplications.id }).from(toolApplications)+ .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, connection.companyId)))+ .for("update");+ await tx.select({ id: toolConnections.id }).from(toolConnections)+ .where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId)))+ .for("update");+ return write(tx);+ });+ descriptors = [...new Map(descriptors.map(descriptor => [descriptor.name, descriptor])).values()];+ const existingRows = await db+ .select()+ .from(toolCatalogEntries)+ .where(eq(toolCatalogEntries.connectionId, connection.id));+ const existingByName = new Map(+ existingRows.map((entry) => [entry.toolName, entry]),+ );+ const nativeDaytona = isDaytonaConnection(connection);+ if (connection.config.sourceTemplateKey === "google-workspace"+ || nativeDaytona || isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey)) {+ const discoveredNames = new Set(descriptors.map((descriptor) => descriptor.name));+ // The catalog is shared, but discovery uses only the caller's grant. A+ // narrower personal consent must not remove actions another active grant+ // still supports. Never combine scopes from different grants into authority.+ let otherWorkspaceGrants: string[][] = [];+ let selectedWorkspaceScopes: string[] = [];+ if (connection.config.sourceTemplateKey === "google-workspace") {+ const selected = await vaultGrantForConnection(connection, actor);+ selectedWorkspaceScopes = googleWorkspaceGrantedScopes(selected);+ const grants = await db.select().from(connectionGrants).where(and(+ eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id),+ eq(connectionGrants.status, "active"),+ ));+ otherWorkspaceGrants = grants.filter((grant) => grant.id !== selected?.id).map(googleWorkspaceGrantedScopes);+ }+ // Preserve only consent-filtered actions, not failed-service catalogs or+ // tools a successful, sufficiently authorized discovery actually removed.+ const removedStatus = nativeDaytona ? "removed" : "disabled";+ const removedIds = existingRows.filter((entry) => !discoveredNames.has(entry.toolName) && entry.status !== removedStatus+ && !(otherWorkspaceGrants.some((scopes) => isGoogleWorkspaceToolGranted(entry.toolName, scopes))+ && !isGoogleWorkspaceToolGranted(entry.toolName, selectedWorkspaceScopes)+ && !unavailableServices.includes(googleWorkspaceToolTarget(entry.toolName)?.service ?? ""))).map((entry) => entry.id);+ if (removedIds.length) await publish(tx => tx.update(toolCatalogEntries)+ .set({ status: removedStatus, quarantineReason: nativeDaytona ? "provider_tool_removed" : "mcp_tool_removed", updatedAt: refreshedAt })+ .where(and(eq(toolCatalogEntries.companyId, connection.companyId), eq(toolCatalogEntries.connectionId, connection.id), inArray(toolCatalogEntries.id, removedIds))));+ }+ // Retired native actions are absent from discovery, but old catalog rows+ // still need to show as disabled. Gateway denial also applies before refresh.+ const blockedRailwayEntryIds = isRailwayEndpoint(connection.config.url)+ ? existingRows.filter((entry) => isRailwayToolBlocked(entry.toolName)).map((entry) => entry.id)+ : [];+ if (blockedRailwayEntryIds.length > 0) {+ await publish(tx => tx.update(toolCatalogEntries).set({ status: "disabled", updatedAt: refreshedAt })+ .where(and(eq(toolCatalogEntries.connectionId, connection.id), inArray(toolCatalogEntries.id, blockedRailwayEntryIds))));+ }+ const catalogValues: Array<typeof toolCatalogEntries.$inferInsert> = [];+ let quarantinedCount = 0;+ const sourceTemplateKey =+ typeof asRecord(connection.config).sourceTemplateKey === "string"+ ? String(asRecord(connection.config).sourceTemplateKey)+ : null;+ const sourceApp = sourceTemplateKey+ ? getConnectableAppDefinition(sourceTemplateKey)+ : null;+ const sourceMethod = sourceApp+ ? connectionMethodForConnection(sourceApp, connection)+ : null;+ const sourceCapabilityKey = sourceMethod?.capabilityProfile?.key;+ const googleProfileValue =+ sourceMethod?.connectorProfile ??+ sourceApp?.methods.find(+ (candidate) =>+ candidate.connectorProfile &&+ candidate.capabilityProfile?.key === sourceCapabilityKey,+ )?.connectorProfile;+ const googleProfile =+ googleProfileValue &&+ isGoogleWorkspaceConnectorProfileId(googleProfileValue)+ ? googleProfileValue+ : null;+ const connectorPolicy = connectorToolPolicy(sourceTemplateKey, connection);+ const preserveReviewedCatalog = connectorPolicy.quarantineNewEntries && existingRows.length > 0;+ const quarantineOnRefresh =+ (!refreshOptions.enableAllByDefault || preserveReviewedCatalog) &&+ shouldQuarantineNewEntries(connection) &&+ (connection.status === "active" ||+ preserveReviewedCatalog ||+ sourceTemplateKey === "posthog" ||+ refreshOptions.quarantineManagedOAuthDraft === true);+ const safeDefault = asRecord(connection.config).safeDefault === true;+ for (const descriptor of descriptors) {+ const riskLevel = classifyRisk(descriptor, connectorPolicy.sourceTemplateKey);+ const hash = descriptorHash(descriptor, riskLevel);+ const schemaHash = stableHash(descriptor.inputSchema ?? {});+ const existing = existingByName.get(descriptor.name);+ const changed =+ existing &&+ (existing.versionHash !== hash || existing.schemaHash !== schemaHash);+ const shouldQuarantine =+ quarantineOnRefresh &&+ (!existing || changed) &&+ existing?.status !== "disabled" &&+ (!safeDefault || riskLevel !== "read" || nativeDaytona);+ const providerPermanentlyBlocked = Boolean(+ googleProfile &&+ !isGoogleWorkspaceToolAllowed(googleProfile, descriptor),+ ) || (isRailwayEndpoint(connection.config.url) && isRailwayToolBlocked(descriptor.name));+ const status = providerPermanentlyBlocked+ ? "disabled"+ : shouldQuarantine+ ? "quarantined"+ : existing?.status === "disabled" && existing.quarantineReason !== "mcp_tool_removed"+ ? "disabled"+ : quarantineOnRefresh && existing?.status === "quarantined"+ ? "quarantined"+ : "active";+ if (shouldQuarantine && !providerPermanentlyBlocked) quarantinedCount += 1;++ catalogValues.push({+ id: existing?.id, companyId: connection.companyId, applicationId: connection.applicationId,+ connectionId: connection.id, name: existing?.name ?? descriptor.name, toolName: descriptor.name, entryKind: "tool",+ title: descriptor.title ?? null, description: descriptor.description ?? null,+ inputSchema: descriptor.inputSchema ?? {}, annotations: descriptor.annotations ?? {}, riskLevel,+ isReadOnly: riskLevel === "read", isWrite: riskLevel === "write", isDestructive: riskLevel === "destructive",+ status, versionHash: hash, schemaHash, firstSeenAt: existing?.firstSeenAt ?? refreshedAt, lastSeenAt: refreshedAt,+ quarantinedAt: existing+ ? status === "quarantined" ? shouldQuarantine ? refreshedAt : existing.quarantinedAt : null+ : shouldQuarantine ? refreshedAt : null,+ quarantineReason: existing+ ? status === "quarantined" ? shouldQuarantine ? "pending_review" : existing.quarantineReason : null+ : shouldQuarantine ? "pending_review" : null,+ updatedAt: existing ? refreshedAt : new Date(),+ });+ }+ const catalogRows: Array<typeof toolCatalogEntries.$inferSelect> = [];+ const newCatalogEntryIds = new Set<string>();+ for (let index = 0; index < catalogValues.length; index += 500) {+ const chunk = catalogValues.slice(index, index + 500);+ catalogRows.push(...await publish(async tx => {+ const current = await tx.select({ name: toolCatalogEntries.name, status: toolCatalogEntries.status })+ .from(toolCatalogEntries).where(and(eq(toolCatalogEntries.connectionId, connection.id), inArray(toolCatalogEntries.name, chunk.map(entry => entry.name))));+ const existingNonQuarantinedNames = new Set(current.filter(entry => entry.status !== "quarantined").map(entry => entry.name));+ const rows = await tx.insert(toolCatalogEntries).values(chunk)+ .onConflictDoUpdate({ target: [toolCatalogEntries.connectionId, toolCatalogEntries.name], set: {+ title: sql`excluded.title`, description: sql`excluded.description`, inputSchema: sql`excluded.input_schema`,+ annotations: sql`excluded.annotations`, riskLevel: sql`excluded.risk_level`, isReadOnly: sql`excluded.is_read_only`,+ isWrite: sql`excluded.is_write`, isDestructive: sql`excluded.is_destructive`,+ status: sql`CASE WHEN ${toolCatalogEntries.status} = 'disabled' AND ${toolCatalogEntries.quarantineReason} IS DISTINCT FROM 'mcp_tool_removed' THEN 'disabled' ELSE excluded.status END`,+ versionHash: sql`excluded.version_hash`, schemaHash: sql`excluded.schema_hash`, lastSeenAt: sql`excluded.last_seen_at`,+ quarantinedAt: sql`excluded.quarantined_at`, quarantineReason: sql`excluded.quarantine_reason`, updatedAt: sql`excluded.updated_at`,+ } }).returning();+ for (const row of rows) if (!existingNonQuarantinedNames.has(row.name)) newCatalogEntryIds.add(row.id);+ return rows;+ }));+ }+ const catalogByName = new Map(catalogRows.map(row => [row.toolName, row]));+ const updatedEntries = descriptors.map(descriptor => toCatalogEntry(catalogByName.get(descriptor.name)!));++ const preserveQuarantine = connectorPolicy.quarantineNewEntries;+ const normalizedConfig = preserveQuarantine+ ? { ...connection.config, quarantineNewEntries: true }+ : refreshOptions.enableAllByDefault+ ? { ...connection.config, quarantineNewEntries: false }+ : connection.config;+ const normalizedTransportConfig = preserveQuarantine+ ? { ...connection.transportConfig, quarantineNewEntries: true }+ : refreshOptions.enableAllByDefault+ ? { ...connection.transportConfig, quarantineNewEntries: false }+ : connection.transportConfig;+ const [updatedConnection] = await db+ .update(toolConnections)+ .set({+ config: normalizedConfig,+ transportConfig: normalizedTransportConfig,+ healthStatus: "ok",+ healthMessage: unavailableServices.length > 0+ ? `Actions refreshed. Google services unavailable: ${unavailableServices.join(", ")}. Other services remain available; retry refreshing actions.`+ : isAgentMailConnection(connection)+ ? "AgentMail API key is connected."+ : "Tool catalog refreshed.",+ healthCheckedAt: refreshedAt,+ lastHealthAt: refreshedAt,+ lastCatalogRefreshAt: refreshedAt,+ lastError: null,+ updatedAt: refreshedAt,+ })+ .where(eq(toolConnections.id, connection.id))+ .returning();++ if (connection.transport === "local_stdio") {+ await ensureRuntimeSlot(updatedConnection);+ await db+ .update(toolRuntimeSlots)+ .set({+ healthStatus: "ok",+ healthMessage: "Approved stdio template is ready.",+ lastHealthCheckAt: refreshedAt,+ updatedAt: refreshedAt,+ })+ .where(eq(toolRuntimeSlots.connectionId, connection.id));+ }++ const activeEntries = updatedEntries.filter(+ (entry) => entry.status === "active",+ );+ const preserveMcpAccess = connection.config.mcpPreserveAccess === true+ && isRemoteMcpConnectorMethod(connection.config.sourceTemplateKey, connection.config.connectionMethodKey);+ if (!refreshOptions.skipDefaultProfileSync || preserveMcpAccess) {+ await publish(tx => enableCatalogEntriesByDefault({+ connection: updatedConnection,+ // Discovery must not re-enable actions the operator turned Off,+ // including curated MCP connections during API-key replacement.+ newCatalogEntryIds: activeEntries+ .filter((entry) => newCatalogEntryIds.has(entry.id))+ .map((entry) => entry.id),+ activeCatalogEntryIds: activeEntries.map((entry) => entry.id),+ restoreDraftDefaults: refreshOptions.restoreDraftDefaults || preserveMcpAccess,+ actor,+ }, tx));+ }+ await audit({+ companyId: connection.companyId,+ connectionId: connection.id,+ action: "tool_connection.catalog_refresh",+ outcome: "success",+ details: { discoveredCount: descriptors.length, quarantinedCount },+ actor,+ });++ return {+ connection: toConnection(updatedConnection),+ catalog: updatedEntries,+ discoveredCount: descriptors.length,+ quarantinedCount,+ };+ }+ return refreshCatalog;+}diff --git a/server/src/services/tool-access/catalog/reader.ts b/server/src/services/tool-access/catalog/reader.tsnew file mode 100644index 0000000000..b7c1d30012--- /dev/null+++ b/server/src/services/tool-access/catalog/reader.ts@@ -0,0 +1,75 @@+import { singleFlight } from "../single-flight.js";+import type { ActorInfo } from "../types.js";+import { descriptorHash } from "../../tool-catalog-hash.js";+import { DAYTONA_TOOLS, daytonaRisk, isDaytonaConnection, isDaytonaToolName } from "../../daytona.js";+import { desc, eq } from "drizzle-orm";+import type { Db } from "@paperclipai/db";+import { toolCatalogEntries, toolConnections } from "@paperclipai/db";+import type { ToolCatalogEntry } from "@paperclipai/shared";++import type { RefreshConnectionCatalog } from "./contracts.js";+export function createToolCatalogReader(db: Db, options: { catalogCacheTtlMs?: number }, now: () => Date, getConnectionRow: (idOrUid: string, companyId?: string) => Promise<typeof toolConnections.$inferSelect>, refreshCatalog: RefreshConnectionCatalog, toCatalogEntryForConnection: (row: typeof toolCatalogEntries.$inferSelect, connection: typeof toolConnections.$inferSelect) => ToolCatalogEntry) {+ const catalogRefreshFlights = new Map<string, Promise<unknown>>();+ const catalogCacheTtlMs = Math.max(+ 0,+ options.catalogCacheTtlMs ?? 15 * 60 * 1000,+ );+ return {+ listCatalog: async (+ connectionId: string,+ companyId?: string,+ actor?: ActorInfo,+ ): Promise<ToolCatalogEntry[]> => {+ const connection = await getConnectionRow(connectionId, companyId);+ let rows = await db+ .select()+ .from(toolCatalogEntries)+ .where(eq(toolCatalogEntries.connectionId, connection.id))+ .orderBy(desc(toolCatalogEntries.updatedAt));+ // Native provider catalogs change with the deployed code. Upgrade saved+ // Daytona connections too; their first small catalog must not survive+ // merely because REST discovery has no remote MCP cache timer.+ const nativeDaytona = isDaytonaConnection(connection);+ const rowsByName = new Map(rows.map((row) => [row.toolName, row]));+ const daytonaCatalogChanged = nativeDaytona && connection.status === "active" && (+ rows.some((row) => row.status !== "removed" && !isDaytonaToolName(row.toolName))+ || DAYTONA_TOOLS.some((tool) => {+ const existing = rowsByName.get(tool.name);+ return !existing || existing.versionHash !== descriptorHash(tool, daytonaRisk(tool.name));+ })+ );+ const cacheExpired = daytonaCatalogChanged || (+ connection.transport === "mcp_remote" &&+ connection.status !== "archived" &&+ (rows.length === 0 ||+ !connection.lastCatalogRefreshAt ||+ connection.lastCatalogRefreshAt.getTime() <=+ now().getTime() - catalogCacheTtlMs));+ if (cacheExpired) {+ try {+ await singleFlight(catalogRefreshFlights, connection.id, () =>+ refreshCatalog(connection.id, nativeDaytona && actor ? actor : {+ actorType: "system",+ actorId: "tool_catalog_cache",+ }),+ );+ rows = await db+ .select()+ .from(toolCatalogEntries)+ .where(eq(toolCatalogEntries.connectionId, connection.id))+ .orderBy(desc(toolCatalogEntries.updatedAt));+ } catch (error) {+ // A stale catalog remains useful when the remote server is temporarily+ // unavailable. Empty caches still fail so callers never mistake “no+ // actions discovered” for a successful lookup.+ if (rows.length === 0) throw error;+ }+ }+ // Even failed discovery must not keep retired native tools visible.+ // Their database rows remain available to historical invocation records.+ return rows+ .filter((row) => !nativeDaytona || (row.status !== "removed" && isDaytonaToolName(row.toolName)))+ .map((row) => toCatalogEntryForConnection(row, connection));+ }+ };+}diff --git a/server/src/services/tool-access/connections/projection.ts b/server/src/services/tool-access/connections/projection.tsnew file mode 100644index 0000000000..f0652389db--- /dev/null+++ b/server/src/services/tool-access/connections/projection.ts@@ -0,0 +1,64 @@+import { redactedVercelGrant } from "../vercel-credentials.js";+import { connectionPurposeTransportSchema } from "@paperclipai/shared";+import { connectionGrants, toolConnectionInstalls, toolConnections } from "@paperclipai/db";+import type { ToolConnection, ToolConnectionInstall } from "@paperclipai/shared";+import { isRetiredComposioConnection, RETIRED_COMPOSIO_MESSAGE } from "@paperclipai/shared";++export function toConnection(row: typeof toolConnections.$inferSelect): ToolConnection {+ connectionPurposeTransportSchema.parse(row);+ const retired = isRetiredComposioConnection(row);+ return {+ id: row.id,+ companyId: row.companyId,+ applicationId: row.applicationId,+ name: row.name,+ uid: row.uid,+ connectionKind: row.connectionKind,+ connectionPurpose: row.connectionPurpose,+ ownership: row.ownership,+ transport: row.transport,+ authKind: row.authKind,+ credentialSource: row.credentialSource,+ externalCredential: row.externalCredential ?? null,+ credentialPolicy: row.credentialPolicy,+ status: row.status,+ enabled: row.enabled && !retired,+ agentInstructions: row.agentInstructions ?? null,+ config: row.config ?? {},+ transportConfig: row.transportConfig ?? {},+ credentialRefs: row.credentialRefs ?? [],+ credentialSecretRefs: row.credentialSecretRefs ?? [],+ healthStatus: retired ? "error" : row.healthStatus,+ healthMessage: retired ? RETIRED_COMPOSIO_MESSAGE : row.healthMessage,+ healthCheckedAt: row.healthCheckedAt,+ lastHealthAt: row.lastHealthAt,+ lastCatalogRefreshAt: row.lastCatalogRefreshAt,+ lastError: row.lastError,+ createdByAgentId: row.createdByAgentId,+ createdByUserId: row.createdByUserId,+ createdAt: row.createdAt,+ updatedAt: row.updatedAt,+ };+}++export function toConnectionGrant(row: typeof connectionGrants.$inferSelect) {+ return {+ ...row,+ externalCredential: redactedVercelGrant(row.externalCredential),+ };+}++export function toConnectionInstall(+ row: typeof toolConnectionInstalls.$inferSelect,+): ToolConnectionInstall {+ return {+ id: row.id,+ companyId: row.companyId,+ connectionId: row.connectionId,+ targetType: row.targetType,+ targetId: row.targetId,+ createdByAgentId: row.createdByAgentId,+ createdByUserId: row.createdByUserId,+ createdAt: row.createdAt,+ };+}diff --git a/server/src/services/tool-access/remote-http.ts b/server/src/services/tool-access/remote-http.tsnew file mode 100644index 0000000000..46fea27423--- /dev/null+++ b/server/src/services/tool-access/remote-http.ts@@ -0,0 +1,129 @@+import { assertConfiguredTokenBrokerEndpointsAllowed } from "./token-broker/endpoint-policy.js";+import { remoteEndpoint } from "./connection-methods.js";+import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";+import { badRequest, HttpError } from "../../errors.js";+import { assertPublicRemoteHttpEndpoint, parseRemoteHttpEndpoint, type RemoteHttpEndpointLookup } from "../remote-http-endpoint-guard.js";+import { guardedRemoteHttpFetch, type GuardedRemoteHttpFetchOptions } from "../remote-http-fetch.js";++const REMOTE_HTTP_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);++const MAX_REMOTE_HTTP_REDIRECTS = 5;++type RemoteHttpOptions = {+ deploymentMode?: DeploymentMode;+ deploymentExposure?: DeploymentExposure;+ remoteHttpEndpointLookup?: RemoteHttpEndpointLookup;+ remoteHttpRequest?: (url: string, init: RequestInit) => Promise<Response>;+};++export function createToolRemoteHttp(options: RemoteHttpOptions) {+ function allowPrivateRemoteEndpoints() {+ return (+ options.deploymentMode !== "authenticated" ||+ options.deploymentExposure !== "public"+ );+ }++ async function assertRemoteHttpUrlAllowed(value: string): Promise<string> {+ const endpoint = parseRemoteHttpEndpoint(value, (message, code) =>+ badRequest(message, { code }),+ );+ await assertPublicRemoteHttpEndpoint(+ endpoint,+ {+ allowPrivateNetwork: allowPrivateRemoteEndpoints(),+ lookup: options.remoteHttpEndpointLookup,+ },+ (message, code) => badRequest(message, { code }),+ );+ return endpoint.toString();+ }++ function remoteHttpFetchOptions(): GuardedRemoteHttpFetchOptions {+ return {+ allowPrivateNetwork: allowPrivateRemoteEndpoints(),+ lookup: options.remoteHttpEndpointLookup,+ error: (message, code) => badRequest(message, { code }),+ };+ }++ async function requestRemoteHttpEndpoint(+ endpoint: URL,+ init: RequestInit,+ ): Promise<Response> {+ return options.remoteHttpRequest+ ? options.remoteHttpRequest(endpoint.toString(), {+ ...init,+ redirect: "manual",+ })+ : guardedRemoteHttpFetch(endpoint, init, remoteHttpFetchOptions());+ }++ /**+ * Fetch an operator-supplied remote URL with the egress guard bound to the+ * connection itself.+ *+ * `guardedRemoteHttpFetch` resolves the hostname once and dials the approved+ * address, so a name server that answers public-then-private cannot move the+ * connection onto a loopback or metadata address after validation+ * (PAP-17098). Redirects stay manual and run the full guard again on the next+ * hop, because a `Location` is just as attacker-controlled as the first URL.+ */+async function fetchRemoteHttpUrl(+ value: string,+ init: RequestInit = {},+ ): Promise<Response> {+ let currentUrl = value;+ const method = (init.method ?? "GET").toUpperCase();+ for (+ let redirectCount = 0;+ redirectCount <= MAX_REMOTE_HTTP_REDIRECTS;+ redirectCount += 1+ ) {+ const endpoint = parseRemoteHttpEndpoint(currentUrl, (message, code) =>+ badRequest(message, { code }),+ );+ const response = await requestRemoteHttpEndpoint(endpoint, init);+ const location = REMOTE_HTTP_REDIRECT_STATUSES.has(response.status)+ ? (response.headers?.get?.("location") ?? null)+ : null;+ if (!location) return response;+ if (method !== "GET" && method !== "HEAD") {+ throw new HttpError(+ 502,+ "Remote OAuth endpoint redirected unexpectedly",+ { code: "oauth_redirect_rejected" },+ );+ }+ if (redirectCount >= MAX_REMOTE_HTTP_REDIRECTS) {+ throw new HttpError(+ 502,+ "Remote OAuth endpoint redirected too many times",+ { code: "oauth_redirect_limit" },+ );+ }+ currentUrl = new URL(location, endpoint).toString();+ }+ throw new HttpError(+ 502,+ "Remote OAuth endpoint redirected too many times",+ { code: "oauth_redirect_limit" },+ );+ }++ async function assertRemoteEndpointAllowed(+ config: Record<string, unknown>,+ ): Promise<string> {+ return assertRemoteHttpUrlAllowed(remoteEndpoint(config));+ }++ async function assertRemoteConnectionEndpointsAllowed(+ config: Record<string, unknown>,+ ): Promise<string> {+ const endpoint = await assertRemoteEndpointAllowed(config);+ await assertConfiguredTokenBrokerEndpointsAllowed(config);+ return endpoint;+ }+ return { allowPrivateRemoteEndpoints, assertRemoteHttpUrlAllowed, requestRemoteHttpEndpoint, fetchRemoteHttpUrl, assertRemoteEndpointAllowed, assertRemoteConnectionEndpointsAllowed };+}+export type ToolRemoteHttp = ReturnType<typeof createToolRemoteHttp>;