Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(connect): reboots no longer strand the relay link, 403s now say why (#4988)

Theo Browne1877237c7ae9
diff --git a/apps/mobile/src/features/cloud/linkEnvironment.ts b/apps/mobile/src/features/cloud/linkEnvironment.ts
index 5d619c688..958827ee4 100644
--- a/apps/mobile/src/features/cloud/linkEnvironment.ts
+++ b/apps/mobile/src/features/cloud/linkEnvironment.ts
@@ -134,7 +134,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string {
case "RelayEnvironmentLinkProofInvalidError":
return `Relay rejected the environment link proof (${error.reason}).`;
case "RelayEnvironmentConnectNotAuthorizedError":
- return "Relay rejected the environment connection request.";
+ // "Not authorized" covers non-auth causes too; surface the reason so a
+ // missing link doesn't read as a credential problem.
+ if (error.reason === "environment_link_not_found") {
+ return "Relay has no active link for this environment. The environment server may not have re-established its link yet.";
+ }
+ return error.reason
+ ? `Relay rejected the environment connection request (${error.reason}).`
+ : "Relay rejected the environment connection request.";
case "RelayEnvironmentEndpointUnavailableError":
return `Relay could not reach the environment endpoint (${error.reason}).`;
case "RelayEnvironmentEndpointTimedOutError":
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index e0d36e99b..853bb0b11 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -1,6 +1,8 @@
import { EnvironmentHttpApi } from "@t3tools/contracts";
+import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
+import * as Schedule from "effect/Schedule";
import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";
import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
@@ -535,7 +537,25 @@ export const makeServerLayer = Layer.unwrap(
yield* Effect.forkScoped(
Effect.sleep("250 millis").pipe(
Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)),
- Effect.retry({ times: 4 }),
+ // On reboot this races NIC/DNS bring-up, so back off exponentially
+ // (capped at 30s) instead of burning all retries in a second.
+ // Bounded overall so a permanently broken setup still surfaces the
+ // warning below. Bad-request/unauthorized/conflict are
+ // deterministic failures (malformed origin, not linked yet, linked
+ // to a different cloud account) that no amount of retrying
+ // converges.
+ Effect.retry({
+ while: (error) =>
+ error._tag !== "EnvironmentHttpBadRequestError" &&
+ error._tag !== "EnvironmentHttpUnauthorizedError" &&
+ error._tag !== "EnvironmentHttpConflictError",
+ schedule: Schedule.exponential("1 second").pipe(
+ Schedule.modifyDelay(({ duration }) =>
+ Effect.succeed(Duration.min(duration, Duration.seconds(30))),
+ ),
+ Schedule.upTo({ duration: "10 minutes" }),
+ ),
+ }),
Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")),
Effect.catch((cause) =>
Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
diff --git a/apps/web/src/cloud/linkEnvironment.ts b/apps/web/src/cloud/linkEnvironment.ts
index 22ff986af..a245cbc54 100644
--- a/apps/web/src/cloud/linkEnvironment.ts
+++ b/apps/web/src/cloud/linkEnvironment.ts
@@ -145,7 +145,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string {
case "RelayEnvironmentLinkProofInvalidError":
return `Relay rejected the environment link proof (${error.reason}).`;
case "RelayEnvironmentConnectNotAuthorizedError":
- return "Relay rejected the environment connection request.";
+ // "Not authorized" covers non-auth causes too; surface the reason so a
+ // missing link doesn't read as a credential problem.
+ if (error.reason === "environment_link_not_found") {
+ return "Relay has no active link for this environment. The environment server may not have re-established its link yet.";
+ }
+ return error.reason
+ ? `Relay rejected the environment connection request (${error.reason}).`
+ : "Relay rejected the environment connection request.";
case "RelayEnvironmentEndpointUnavailableError":
return `Relay could not reach the environment endpoint (${error.reason}).`;
case "RelayEnvironmentEndpointTimedOutError":
diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts
index db662aee9..d840f809e 100644
--- a/infra/relay/src/environments/EnvironmentConnector.ts
+++ b/infra/relay/src/environments/EnvironmentConnector.ts
@@ -13,6 +13,7 @@ import {
RelayEnvironmentMintResponse,
RelayEnvironmentMintResponseProofPayload,
RelayCloudMintCredentialProofPayload,
+ RelayEnvironmentConnectNotAuthorizedReason,
type RelayEnvironmentConnectResponse,
type RelayEnvironmentStatusResponse,
} from "@t3tools/contracts/relay";
@@ -44,21 +45,8 @@ import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
import * as RelayConfiguration from "../Config.ts";
import { isManagedEndpointHostname } from "../deploymentConfig.ts";
-export const EnvironmentConnectNotAuthorizedReason = Schema.Literals([
- "client_proof_key_thumbprint_missing",
- "environment_link_not_found",
- "endpoint_provider_not_managed",
- "managed_endpoint_allocation_not_found",
- "managed_endpoint_base_domain_not_configured",
- "managed_endpoint_allocation_not_ready",
- "managed_endpoint_hostname_invalid",
- "managed_endpoint_mismatch",
-]);
-export type EnvironmentConnectNotAuthorizedReason =
- typeof EnvironmentConnectNotAuthorizedReason.Type;
-
function environmentConnectNotAuthorizedReasonMessage(
- reason: EnvironmentConnectNotAuthorizedReason,
+ reason: RelayEnvironmentConnectNotAuthorizedReason,
): string {
switch (reason) {
case "client_proof_key_thumbprint_missing":
@@ -85,7 +73,7 @@ export class EnvironmentConnectNotAuthorized extends Schema.TaggedErrorClass<Env
{
environmentId: Schema.String,
operation: Schema.Literals(["connect", "status"]),
- reason: EnvironmentConnectNotAuthorizedReason,
+ reason: RelayEnvironmentConnectNotAuthorizedReason,
},
) {
override get message(): string {
diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts
index 7510b4a8d..50bcff665 100644
--- a/infra/relay/src/http/Api.ts
+++ b/infra/relay/src/http/Api.ts
@@ -778,9 +778,10 @@ export const dpopClientApi = HttpApiBuilder.group(
},
mapRelayCommonApiErrors("invalid_dpop"),
mapErrorTags({
- EnvironmentConnectNotAuthorized: (_error, traceId) =>
+ EnvironmentConnectNotAuthorized: (error, traceId) =>
new RelayEnvironmentConnectNotAuthorizedError({
code: "environment_connect_not_authorized",
+ reason: error.reason,
traceId,
}),
EnvironmentMintRequestFailed: (_error, traceId) =>
@@ -820,9 +821,10 @@ export const dpopClientApi = HttpApiBuilder.group(
},
mapRelayCommonApiErrors("invalid_dpop"),
mapErrorTags({
- EnvironmentConnectNotAuthorized: (_error, traceId) =>
+ EnvironmentConnectNotAuthorized: (error, traceId) =>
new RelayEnvironmentConnectNotAuthorizedError({
code: "environment_connect_not_authorized",
+ reason: error.reason,
traceId,
}),
EnvironmentMintRequestFailed: (_error, traceId) =>
diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts
index ff9a9e3ac..52f7d7d43 100644
--- a/packages/contracts/src/relay.ts
+++ b/packages/contracts/src/relay.ts
@@ -371,16 +371,34 @@ export class RelayEnvironmentLinkProofInvalidError extends Schema.TaggedErrorCla
}
}
+export const RelayEnvironmentConnectNotAuthorizedReason = Schema.Literals([
+ "client_proof_key_thumbprint_missing",
+ "environment_link_not_found",
+ "endpoint_provider_not_managed",
+ "managed_endpoint_allocation_not_found",
+ "managed_endpoint_base_domain_not_configured",
+ "managed_endpoint_allocation_not_ready",
+ "managed_endpoint_hostname_invalid",
+ "managed_endpoint_mismatch",
+]);
+export type RelayEnvironmentConnectNotAuthorizedReason =
+ typeof RelayEnvironmentConnectNotAuthorizedReason.Type;
+
export class RelayEnvironmentConnectNotAuthorizedError extends Schema.TaggedErrorClass<RelayEnvironmentConnectNotAuthorizedError>()(
"RelayEnvironmentConnectNotAuthorizedError",
{
code: Schema.Literal("environment_connect_not_authorized"),
+ // Optional so responses from relays deployed before the reason was
+ // threaded through still decode.
+ reason: Schema.optional(RelayEnvironmentConnectNotAuthorizedReason),
traceId: TrimmedNonEmptyString,
},
{ httpApiStatus: 403 },
) {
override get message(): string {
- return "Relay environment connection is not authorized";
+ return this.reason
+ ? `Relay environment connection is not authorized: ${this.reason}`
+ : "Relay environment connection is not authorized";
}
}