Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

[codex] Enrich relay authorization diagnostics (#2977)

Julius Marminge3ea6adf17d55
diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts
index 505d1f31d..0ca7e16ab 100644
--- a/infra/relay/src/environments/EnvironmentConnector.test.ts
+++ b/infra/relay/src/environments/EnvironmentConnector.test.ts
@@ -22,6 +22,7 @@ import * as Redacted from "effect/Redacted";
import * as Result from "effect/Result";
import * as Schema from "effect/Schema";
import * as TestClock from "effect/testing/TestClock";
+import * as Tracer from "effect/Tracer";
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
import * as EnvironmentLinks from "./EnvironmentLinks.ts";
@@ -50,6 +51,9 @@ const decodeHealthRequestBody = Schema.decodeUnknownSync(
const decodeMintRequestBody = Schema.decodeUnknownSync(
Schema.fromJsonString(RelayCloudMintCredentialRequest),
);
+const isEnvironmentConnectNotAuthorized = Schema.is(
+ EnvironmentConnector.EnvironmentConnectNotAuthorized,
+);
function requestBodyText(request: HttpClientRequest.HttpClientRequest): string {
return request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "{}";
@@ -280,7 +284,13 @@ describe("EnvironmentConnector", () => {
expect(Result.isFailure(result)).toBe(true);
if (Result.isFailure(result)) {
- expect(result.failure).toBeInstanceOf(EnvironmentConnector.EnvironmentConnectNotAuthorized);
+ expect(isEnvironmentConnectNotAuthorized(result.failure)).toBe(true);
+ if (isEnvironmentConnectNotAuthorized(result.failure)) {
+ expect(result.failure).toMatchObject({
+ operation: "status",
+ reason: "endpoint_provider_not_managed",
+ });
+ }
}
expect(requestCount).toBe(0);
}).pipe(
@@ -300,6 +310,14 @@ describe("EnvironmentConnector", () => {
it.effect("rejects stale managed endpoints before sending a mint request", () => {
let requestCount = 0;
+ const spans: Array<Tracer.NativeSpan> = [];
+ const tracer = Tracer.make({
+ span: (options) => {
+ const span = new Tracer.NativeSpan(options);
+ spans.push(span);
+ return span;
+ },
+ });
const execute = () =>
Effect.sync(() => {
requestCount += 1;
@@ -318,8 +336,27 @@ describe("EnvironmentConnector", () => {
expect(Result.isFailure(result)).toBe(true);
if (Result.isFailure(result)) {
- expect(result.failure).toBeInstanceOf(EnvironmentConnector.EnvironmentConnectNotAuthorized);
+ expect(isEnvironmentConnectNotAuthorized(result.failure)).toBe(true);
+ if (isEnvironmentConnectNotAuthorized(result.failure)) {
+ expect(result.failure).toMatchObject({
+ operation: "connect",
+ reason: "managed_endpoint_mismatch",
+ });
+ }
}
+ const resolutionSpan = spans.find(
+ (span) => span.name === "relay.environment_connector.resolve_managed_endpoint",
+ );
+ expect(Object.fromEntries(resolutionSpan?.attributes ?? [])).toMatchObject({
+ "relay.authorization.allocation_hostname": "env.example.test",
+ "relay.authorization.allocation_has_ready_at": true,
+ "relay.authorization.allocation_has_tunnel_id": true,
+ "relay.authorization.allocation_has_dns_record_id": true,
+ "relay.authorization.linked_http_base_url": "https://attacker.example.test/",
+ "relay.authorization.linked_ws_base_url": "wss://attacker.example.test/ws",
+ "relay.authorization.resolved_http_base_url": "https://env.example.test/",
+ "relay.authorization.resolved_ws_base_url": "wss://env.example.test/ws",
+ });
expect(requestCount).toBe(0);
}).pipe(
Effect.provide(
@@ -333,6 +370,7 @@ describe("EnvironmentConnector", () => {
}),
}),
),
+ Effect.provideService(Tracer.Tracer, tracer),
);
});
@@ -355,7 +393,13 @@ describe("EnvironmentConnector", () => {
expect(Result.isFailure(result)).toBe(true);
if (Result.isFailure(result)) {
- expect(result.failure).toBeInstanceOf(EnvironmentConnector.EnvironmentConnectNotAuthorized);
+ expect(isEnvironmentConnectNotAuthorized(result.failure)).toBe(true);
+ if (isEnvironmentConnectNotAuthorized(result.failure)) {
+ expect(result.failure).toMatchObject({
+ operation: "status",
+ reason: "managed_endpoint_allocation_not_ready",
+ });
+ }
}
expect(requestCount).toBe(0);
}).pipe(
diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts
index 8c9804496..dac5c81f6 100644
--- a/infra/relay/src/environments/EnvironmentConnector.ts
+++ b/infra/relay/src/environments/EnvironmentConnector.ts
@@ -40,15 +40,54 @@ import { FetchHttpClient, HttpClient } from "effect/unstable/http";
import * as EnvironmentLinks from "./EnvironmentLinks.ts";
import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
import * as RelayConfiguration from "../Config.ts";
+import { isManagedEndpointHostname } from "../deploymentConfig.ts";
+
+export const EnvironmentConnectNotAuthorizedReason = Schema.Literals([
+ "client_proof_key_thumbprint_missing",
+ "environment_link_not_found",
+ "endpoint_provider_not_managed",
+ "managed_endpoint_allocation_not_found",
+ "managed_endpoint_base_domain_not_configured",
+ "managed_endpoint_allocation_not_ready",
+ "managed_endpoint_hostname_invalid",
+ "managed_endpoint_mismatch",
+]);
+export type EnvironmentConnectNotAuthorizedReason =
+ typeof EnvironmentConnectNotAuthorizedReason.Type;
+
+function environmentConnectNotAuthorizedReasonMessage(
+ reason: EnvironmentConnectNotAuthorizedReason,
+): string {
+ switch (reason) {
+ case "client_proof_key_thumbprint_missing":
+ return "the client proof key thumbprint is missing";
+ case "environment_link_not_found":
+ return "no active environment link was found";
+ case "endpoint_provider_not_managed":
+ return "the linked endpoint is not relay-managed";
+ case "managed_endpoint_allocation_not_found":
+ return "no managed endpoint allocation was found";
+ case "managed_endpoint_base_domain_not_configured":
+ return "the managed endpoint base domain is not configured";
+ case "managed_endpoint_allocation_not_ready":
+ return "the managed endpoint allocation is incomplete";
+ case "managed_endpoint_hostname_invalid":
+ return "the managed endpoint hostname is invalid";
+ case "managed_endpoint_mismatch":
+ return "the linked endpoint does not match its managed allocation";
+ }
+}
export class EnvironmentConnectNotAuthorized extends Schema.TaggedErrorClass<EnvironmentConnectNotAuthorized>()(
"EnvironmentConnectNotAuthorized",
{
environmentId: Schema.String,
+ operation: Schema.Literals(["connect", "status"]),
+ reason: EnvironmentConnectNotAuthorizedReason,
},
) {
override get message(): string {
- return `Environment '${this.environmentId}' is not authorized to connect`;
+ return `Environment '${this.environmentId}' is not authorized for ${this.operation}: ${environmentConnectNotAuthorizedReasonMessage(this.reason)}`;
}
}
@@ -257,30 +296,91 @@ const make = Effect.gen(function* () {
const resolveManagedEndpoint = Effect.fn("relay.environment_connector.resolve_managed_endpoint")(
function* (input: {
readonly userId: string;
+ readonly operation: "connect" | "status";
readonly link: EnvironmentLinks.RelayLinkedEnvironmentRecord;
}) {
if (input.link.endpoint.providerKind !== "cloudflare_tunnel") {
+ yield* Effect.annotateCurrentSpan({
+ "relay.authorization.endpoint_provider_kind": input.link.endpoint.providerKind,
+ });
return yield* new EnvironmentConnectNotAuthorized({
environmentId: input.link.environmentId,
+ operation: input.operation,
+ reason: "endpoint_provider_not_managed",
});
}
const allocation = yield* allocations.get({
userId: input.userId,
environmentId: input.link.environmentId,
});
- const endpoint = allocation
- ? ManagedEndpointAllocations.resolveReadyManagedEndpoint({
- allocation,
- baseDomain: settings.managedEndpointBaseDomain,
- })
- : null;
+ if (!allocation) {
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.link.environmentId,
+ operation: input.operation,
+ reason: "managed_endpoint_allocation_not_found",
+ });
+ }
+ const allocationAttributes = {
+ "relay.authorization.allocation_hostname": allocation.hostname,
+ "relay.authorization.allocation_has_ready_at": allocation.readyAt !== null,
+ "relay.authorization.allocation_has_tunnel_id": allocation.tunnelId !== null,
+ "relay.authorization.allocation_has_dns_record_id": allocation.dnsRecordId !== null,
+ } as const;
+ if (!settings.managedEndpointBaseDomain) {
+ yield* Effect.annotateCurrentSpan(allocationAttributes);
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.link.environmentId,
+ operation: input.operation,
+ reason: "managed_endpoint_base_domain_not_configured",
+ });
+ }
+ if (
+ allocation.readyAt === null ||
+ allocation.tunnelId === null ||
+ allocation.dnsRecordId === null
+ ) {
+ yield* Effect.annotateCurrentSpan(allocationAttributes);
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.link.environmentId,
+ operation: input.operation,
+ reason: "managed_endpoint_allocation_not_ready",
+ });
+ }
+ if (!isManagedEndpointHostname(allocation.hostname, settings.managedEndpointBaseDomain)) {
+ yield* Effect.annotateCurrentSpan({
+ ...allocationAttributes,
+ "relay.authorization.managed_endpoint_base_domain": settings.managedEndpointBaseDomain,
+ });
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.link.environmentId,
+ operation: input.operation,
+ reason: "managed_endpoint_hostname_invalid",
+ });
+ }
+ const endpoint = ManagedEndpointAllocations.resolveReadyManagedEndpoint({
+ allocation,
+ baseDomain: settings.managedEndpointBaseDomain,
+ });
if (
endpoint === null ||
endpoint.httpBaseUrl !== input.link.endpoint.httpBaseUrl ||
endpoint.wsBaseUrl !== input.link.endpoint.wsBaseUrl
) {
+ yield* Effect.annotateCurrentSpan({
+ ...allocationAttributes,
+ "relay.authorization.linked_http_base_url": input.link.endpoint.httpBaseUrl,
+ "relay.authorization.linked_ws_base_url": input.link.endpoint.wsBaseUrl,
+ ...(endpoint
+ ? {
+ "relay.authorization.resolved_http_base_url": endpoint.httpBaseUrl,
+ "relay.authorization.resolved_ws_base_url": endpoint.wsBaseUrl,
+ }
+ : {}),
+ });
return yield* new EnvironmentConnectNotAuthorized({
environmentId: input.link.environmentId,
+ operation: input.operation,
+ reason: "managed_endpoint_mismatch",
});
}
return endpoint;
@@ -295,9 +395,17 @@ const make = Effect.gen(function* () {
});
const link = yield* links.getForUser(input);
if (!link) {
- return yield* new EnvironmentConnectNotAuthorized({ environmentId: input.environmentId });
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.environmentId,
+ operation: "status",
+ reason: "environment_link_not_found",
+ });
}
- const endpoint = yield* resolveManagedEndpoint({ userId: input.userId, link });
+ const endpoint = yield* resolveManagedEndpoint({
+ userId: input.userId,
+ operation: "status",
+ link,
+ });
const now = yield* DateTime.now;
const expiresAt = DateTime.add(now, { minutes: 2 });
const nonce = yield* crypto.randomUUIDv4.pipe(
@@ -404,13 +512,25 @@ const make = Effect.gen(function* () {
...(input.deviceId ? { "relay.mobile.device_id": input.deviceId } : {}),
});
if (input.clientProofKeyThumbprint.trim().length === 0) {
- return yield* new EnvironmentConnectNotAuthorized({ environmentId: input.environmentId });
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.environmentId,
+ operation: "connect",
+ reason: "client_proof_key_thumbprint_missing",
+ });
}
const link = yield* links.getForUser(input);
if (!link) {
- return yield* new EnvironmentConnectNotAuthorized({ environmentId: input.environmentId });
+ return yield* new EnvironmentConnectNotAuthorized({
+ environmentId: input.environmentId,
+ operation: "connect",
+ reason: "environment_link_not_found",
+ });
}
- const endpoint = yield* resolveManagedEndpoint({ userId: input.userId, link });
+ const endpoint = yield* resolveManagedEndpoint({
+ userId: input.userId,
+ operation: "connect",
+ link,
+ });
const now = yield* DateTime.now;
const expiresAt = DateTime.add(now, { minutes: 2 });
const nonce = yield* crypto.randomUUIDv4.pipe(
diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts
index b9c4abf2c..45dde8d39 100644
--- a/infra/relay/src/http/Api.ts
+++ b/infra/relay/src/http/Api.ts
@@ -830,7 +830,7 @@ const currentTraceId = Effect.currentParentSpan.pipe(
Effect.orElseSucceed(() => "unavailable"),
);
-const COMMON_AUTH_INVALID_REASONS = [
+const RelayCommonPersistenceError = Schema.Union([
Devices.DeviceRegistrationPersistenceError,
Devices.DeviceUnregistrationPersistenceError,
Devices.DeviceListPersistenceError,
@@ -850,9 +850,9 @@ const COMMON_AUTH_INVALID_REASONS = [
AgentActivityRows.AgentActivityRowListPersistenceError,
LiveActivities.LiveActivityDeliveryMarkPersistenceError,
DeliveryAttempts.DeliveryAttemptRecordPersistenceError,
-] as const;
-type RelayCommonPersistenceError = InstanceType<(typeof COMMON_AUTH_INVALID_REASONS)[number]>;
-const isRelayCommonPersistenceError = Schema.is(Schema.Union(COMMON_AUTH_INVALID_REASONS));
+]);
+type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type;
+const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError);
type MapRelayCommonApiError<E> =
| Exclude<E, HttpApiError.Unauthorized | RelayCommonPersistenceError>
diff --git a/infra/relay/src/observability.test.ts b/infra/relay/src/observability.test.ts
index d74fb2900..ff543672f 100644
--- a/infra/relay/src/observability.test.ts
+++ b/infra/relay/src/observability.test.ts
@@ -9,7 +9,7 @@ import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest";
import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
import type { OtlpTracer } from "effect/unstable/observability";
-import { EnvironmentMintRequestFailed } from "./environments/EnvironmentConnector.ts";
+import { EnvironmentConnectNotAuthorized } from "./environments/EnvironmentConnector.ts";
import { makeRelayTraceLayer } from "./observability.ts";
interface ExportedRequest {
@@ -43,10 +43,10 @@ it.effect("exports schema error fields as span attributes", () =>
);
yield* Effect.fail(
- new EnvironmentMintRequestFailed({
+ new EnvironmentConnectNotAuthorized({
environmentId: "environment-1",
operation: "connect",
- cause: new Error("upstream unavailable"),
+ reason: "managed_endpoint_allocation_not_ready",
}),
).pipe(
Effect.withSpan("relay.test.schema_error"),
@@ -76,11 +76,10 @@ it.effect("exports schema error fields as span attributes", () =>
expect(request.authorization).toBe("Bearer test-token");
expect(request.dataset).toBe("relay-test-traces");
expect(attributes).toMatchObject({
- "error.type": "EnvironmentMintRequestFailed",
+ "error.type": "EnvironmentConnectNotAuthorized",
"error.environmentId": "environment-1",
"error.operation": "connect",
- "error.cause.name": "Error",
- "error.cause.message": "upstream unavailable",
+ "error.reason": "managed_endpoint_allocation_not_ready",
});
}).pipe(Effect.provide(NodeHttpServer.layerTest), Effect.scoped),
);