Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
[codex] Audit managed endpoint error context (#3245)
Julius Marminged7ff7e733364
diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.tsnew file mode 100644index 000000000..1a3c01d1e--- /dev/null+++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts@@ -0,0 +1,93 @@+import { describe, expect, it } from "@effect/vitest";+import * as Effect from "effect/Effect";+import * as Layer from "effect/Layer";++import * as RelayDb from "../db.ts";+import { relayManagedEndpointAllocations } from "../persistence/schema.ts";+import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";++const layerWithDb = (db: RelayDb.RelayDb["Service"]) =>+ ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db)));++describe("ManagedEndpointAllocations", () => {+ it.effect("retains database failures with allocation operation and identity", () => {+ const cause = new Error("database unavailable");+ const fakeDb = {+ select: () => ({+ from: (table: unknown) => {+ expect(table).toBe(relayManagedEndpointAllocations);+ return {+ where: () => ({+ limit: () => Effect.fail(cause),+ }),+ };+ },+ }),+ } as unknown as RelayDb.RelayDb["Service"];++ return Effect.gen(function* () {+ const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;+ const error = yield* Effect.flip(+ allocations.get({ userId: "user-1", environmentId: "environment-1" }),+ );++ expect(error).toMatchObject({+ _tag: "ManagedEndpointAllocationPersistenceError",+ operation: "get",+ stage: "database-request",+ userId: "user-1",+ environmentId: "environment-1",+ });+ expect(error.cause).toBe(cause);+ }).pipe(Effect.provide(layerWithDb(fakeDb)));+ });++ it.effect("reports an unresolved reservation without manufacturing a cause", () => {+ const fakeDb = {+ insert: (table: unknown) => {+ expect(table).toBe(relayManagedEndpointAllocations);+ return {+ values: () => ({+ onConflictDoNothing: () => ({+ returning: () => Effect.succeed([]),+ }),+ }),+ };+ },+ select: () => ({+ from: (table: unknown) => {+ expect(table).toBe(relayManagedEndpointAllocations);+ return {+ where: () => ({+ limit: () => Effect.succeed([]),+ }),+ };+ },+ }),+ } as unknown as RelayDb.RelayDb["Service"];++ return Effect.gen(function* () {+ const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;+ const error = yield* Effect.flip(+ allocations.reserve({+ userId: "user-1",+ environmentId: "environment-1",+ hostname: "environment-1.example.test",+ tunnelName: "environment-1-tunnel",+ }),+ );++ expect(error).toMatchObject({+ _tag: "ManagedEndpointAllocationPersistenceError",+ operation: "reserve",+ stage: "resolve-reservation",+ userId: "user-1",+ environmentId: "environment-1",+ hostname: "environment-1.example.test",+ tunnelName: "environment-1-tunnel",+ });+ expect(error.cause).toBeUndefined();+ expect(error.message).toContain("'resolve-reservation'");+ }).pipe(Effect.provide(layerWithDb(fakeDb)));+ });+});diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.tsindex 440f1d48d..c951ee03c 100644--- a/infra/relay/src/environments/ManagedEndpointAllocations.ts+++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts@@ -38,15 +38,29 @@ export function resolveReadyManagedEndpoint(input: {export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErrorClass<ManagedEndpointAllocationPersistenceError>()("ManagedEndpointAllocationPersistenceError",- { cause: Schema.Defect() },+ {+ operation: Schema.Literals([+ "get",+ "reserve",+ "record-tunnel",+ "record-dns",+ "mark-ready",+ "remove",+ ]),+ stage: Schema.Literals(["database-request", "resolve-reservation"]),+ userId: Schema.String,+ environmentId: Schema.String,+ hostname: Schema.optionalKey(Schema.String),+ tunnelName: Schema.optionalKey(Schema.String),+ tunnelId: Schema.optionalKey(Schema.String),+ dnsRecordId: Schema.optionalKey(Schema.String),+ cause: Schema.optional(Schema.Defect()),+ },) {override get message(): string {- return "Failed to persist managed endpoint allocation";+ return `Managed endpoint allocation '${this.operation}' failed during '${this.stage}' for user '${this.userId}', environment '${this.environmentId}'`;}}-const isManagedEndpointAllocationPersistenceError = Schema.is(- ManagedEndpointAllocationPersistenceError,-);interface ManagedEndpointAllocationKey {readonly userId: string;@@ -106,11 +120,6 @@ const whereAllocation = (input: ManagedEndpointAllocationKey) =>eq(relayManagedEndpointAllocations.environmentId, input.environmentId),);-const persistenceError = (cause: unknown) =>- isManagedEndpointAllocationPersistenceError(cause)- ? cause- : new ManagedEndpointAllocationPersistenceError({ cause });-const make = Effect.gen(function* () {const db = yield* RelayDb.RelayDb;@@ -125,7 +134,15 @@ const make = Effect.gen(function* () {.limit(1).pipe(Effect.map((rows) => rows[0] ?? null),- Effect.mapError(persistenceError),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "get",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),);}),reserve: Effect.fn("relay.managed_endpoint_allocations.reserve")(function* (@@ -140,7 +157,18 @@ const make = Effect.gen(function* () {updatedAt: now,}).onConflictDoNothing()- .returning(allocationSelection);+ .returning(allocationSelection)+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "reserve",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),+ );const allocation =inserted[0] ??@@ -149,16 +177,29 @@ const make = Effect.gen(function* () {.from(relayManagedEndpointAllocations).where(whereAllocation(input)).limit(1)- .pipe(Effect.map((rows) => rows[0])));+ .pipe(+ Effect.map((rows) => rows[0]),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "reserve",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),+ ));if (allocation === undefined) {return yield* new ManagedEndpointAllocationPersistenceError({- cause: new Error("Managed endpoint allocation was not persisted."),+ operation: "reserve",+ stage: "resolve-reservation",+ ...input,});}return allocation;- }, Effect.mapError(persistenceError)),+ }),recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* (input: RecordManagedEndpointTunnelInput,) {@@ -168,8 +209,19 @@ const make = Effect.gen(function* () {tunnelId: input.tunnelId,updatedAt: DateTime.formatIso(yield* DateTime.now),})- .where(whereAllocation(input));- }, Effect.mapError(persistenceError)),+ .where(whereAllocation(input))+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "record-tunnel",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),+ );+ }),recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* (input: RecordManagedEndpointDnsInput,) {@@ -179,8 +231,19 @@ const make = Effect.gen(function* () {dnsRecordId: input.dnsRecordId,updatedAt: DateTime.formatIso(yield* DateTime.now),})- .where(whereAllocation(input));- }, Effect.mapError(persistenceError)),+ .where(whereAllocation(input))+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "record-dns",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),+ );+ }),markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* (input: ManagedEndpointAllocationKey,) {@@ -191,13 +254,37 @@ const make = Effect.gen(function* () {readyAt: now,updatedAt: now,})- .where(whereAllocation(input));- }, Effect.mapError(persistenceError)),+ .where(whereAllocation(input))+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "mark-ready",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),+ );+ }),remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* (input: ManagedEndpointAllocationKey,) {- yield* db.delete(relayManagedEndpointAllocations).where(whereAllocation(input));- }, Effect.mapError(persistenceError)),+ yield* db+ .delete(relayManagedEndpointAllocations)+ .where(whereAllocation(input))+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointAllocationPersistenceError({+ operation: "remove",+ stage: "database-request",+ ...input,+ cause,+ }),+ ),+ );+ }),});});diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.tsindex 7b8f0cc28..479be4123 100644--- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts+++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts@@ -130,6 +130,9 @@ function makeDnsClient(calls.push({ operation: "updateRecord", input: { dnsRecordId, request } });if (!currentRecords.some((record) => record.id === dnsRecordId)) {return yield* new ManagedEndpointProvider.ManagedEndpointDnsClientError({+ operation: "update-record",+ hostname: request.name,+ dnsRecordId,cause: `DNS record ${dnsRecordId} does not exist.`,});}@@ -398,7 +401,13 @@ describe("ManagedEndpointProvider", () => {expect(dnsCalls).toHaveLength(0);expect(result._tag).toBe("Failure");if (result._tag === "Failure") {- expect(result.failure._tag).toBe("ManagedEndpointOriginNotAllowed");+ expect(result.failure).toMatchObject({+ _tag: "ManagedEndpointOriginNotAllowed",+ userId: "user_ABC",+ environmentId: "env_ABC",+ host: "192.168.1.10",+ port: 3773,+ });}}).pipe(Effect.provide(providerLayer(makeTunnelClient(), makeDnsClient(dnsCalls))));});@@ -593,6 +602,8 @@ describe("ManagedEndpointProvider", () => {const tunnelCalls: TunnelCall[] = [];let deleteAttempts = 0;const failure = new ManagedEndpointProvider.ManagedEndpointTunnelClientError({+ operation: "delete",+ tunnelId: "tunnel-id",cause: "Cloudflare tunnel deletion failed",});const tunnels = makePersistentTunnelClient(tunnelCalls);@@ -618,6 +629,16 @@ describe("ManagedEndpointProvider", () => {});const first = yield* Effect.result(provider.deprovision(key));expect(first._tag).toBe("Failure");+ if (first._tag === "Failure") {+ expect(first.failure).toMatchObject({+ _tag: "ManagedEndpointDeprovisioningFailed",+ stage: "delete-tunnel",+ userId: key.userId,+ environmentId: key.environmentId,+ tunnelId: "tunnel-id",+ });+ expect(first.failure.cause).toBe(failure);+ }yield* provider.deprovision(key);expect(allocationCalls.map((call) => call.operation)).toEqual([@@ -639,13 +660,23 @@ describe("ManagedEndpointProvider", () => {...makeTunnelClient(),delete: () =>Effect.fail(- new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ cause: notFound }),+ new ManagedEndpointProvider.ManagedEndpointTunnelClientError({+ operation: "delete",+ tunnelId: "tunnel-id",+ cause: notFound,+ }),),});const dnsClient = ManagedEndpointProvider.ManagedEndpointDnsClient.of({...makeDnsClient(),deleteRecord: () =>- Effect.fail(new ManagedEndpointProvider.ManagedEndpointDnsClientError({ cause: notFound })),+ Effect.fail(+ new ManagedEndpointProvider.ManagedEndpointDnsClientError({+ operation: "delete-record",+ dnsRecordId: "created-record-id",+ cause: notFound,+ }),+ ),});const layer = providerLayer(tunnelClient, dnsClient, makeAllocations(allocationCalls));@@ -690,6 +721,8 @@ describe("ManagedEndpointProvider", () => {it.effect("recovers when DNS creation reports failure after the record became visible", () => {const dnsCalls: DnsCall[] = [];const failure = new ManagedEndpointProvider.ManagedEndpointDnsClientError({+ operation: "create-record",+ hostname: expectedManagedHostname("env_ABC"),cause: "ambiguous Cloudflare DNS response",});let records: ReadonlyArray<{ readonly id: string }> = [];@@ -732,8 +765,43 @@ describe("ManagedEndpointProvider", () => {}).pipe(Effect.provide(providerLayer(makeTunnelClient(), dnsClient)));});+ it.effect("reports malformed tunnel responses without manufacturing a cause", () => {+ const dnsCalls: DnsCall[] = [];+ const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({+ ...makeTunnelClient(),+ create: () => Effect.succeed({ id: "returned-tunnel-id", name: null }),+ });++ return Effect.gen(function* () {+ const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;+ const error = yield* Effect.flip(+ provider.provision({+ userId: "user_ABC",+ environmentId: "env_ABC",+ origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },+ }),+ );++ expect(error).toMatchObject({+ _tag: "ManagedEndpointProvisioningFailed",+ stage: "validate-tunnel-response",+ userId: "user_ABC",+ environmentId: "env_ABC",+ hostname: expectedManagedHostname("env_ABC"),+ tunnelName: expectedManagedTunnelName("env_ABC"),+ returnedTunnelId: "returned-tunnel-id",+ });+ if (error._tag === "ManagedEndpointProvisioningFailed") {+ expect(error.cause).toBeUndefined();+ }+ expect(dnsCalls).toHaveLength(0);+ }).pipe(Effect.provide(providerLayer(tunnelClient, makeDnsClient(dnsCalls))));+ });+it.effect("fails provisioning when the DNS client fails", () => {const failure = new ManagedEndpointProvider.ManagedEndpointDnsClientError({+ operation: "list-records",+ hostname: expectedManagedHostname("env_ABC"),cause: "Cloudflare DNS failure",});const dnsClient = ManagedEndpointProvider.ManagedEndpointDnsClient.of({@@ -753,8 +821,18 @@ describe("ManagedEndpointProvider", () => {}),);- expect(error._tag).toBe("ManagedEndpointProvisioningFailed");- expect(error.cause).toBe(failure);+ expect(error).toMatchObject({+ _tag: "ManagedEndpointProvisioningFailed",+ stage: "ensure-dns-record",+ userId: "user_ABC",+ environmentId: "env_ABC",+ hostname: expectedManagedHostname("env_ABC"),+ tunnelName: expectedManagedTunnelName("env_ABC"),+ tunnelId: "tunnel-id",+ });+ if (error._tag === "ManagedEndpointProvisioningFailed") {+ expect(error.cause).toBe(failure);+ }}).pipe(Effect.provide(providerLayer(makeTunnelClient(), dnsClient)));});});diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.tsindex 2de9d1966..bb2dd4b0c 100644--- a/infra/relay/src/environments/ManagedEndpointProvider.ts+++ b/infra/relay/src/environments/ManagedEndpointProvider.ts@@ -7,7 +7,6 @@ import * as Effect from "effect/Effect";import * as Encoding from "effect/Encoding";import * as Layer from "effect/Layer";import * as Option from "effect/Option";-import * as Result from "effect/Result";import * as Schema from "effect/Schema";import type {@@ -27,40 +26,86 @@ import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";export class ManagedEndpointProvisioningNotConfigured extends Schema.TaggedErrorClass<ManagedEndpointProvisioningNotConfigured>()("ManagedEndpointProvisioningNotConfigured",- {},+ {+ userId: Schema.String,+ environmentId: Schema.String,+ missingSettings: Schema.Array(+ Schema.Literals(["managedEndpointBaseDomain", "managedEndpointNamespace"]),+ ),+ },) {override get message(): string {- return "Managed endpoint provisioning is not configured";+ return `Managed endpoint provisioning is not configured for user '${this.userId}', environment '${this.environmentId}': missing ${this.missingSettings.join(", ")}`;}}+const ManagedEndpointProvisioningStage = Schema.Literals([+ "derive-environment-hash",+ "reserve-allocation",+ "ensure-tunnel",+ "validate-tunnel-response",+ "record-tunnel",+ "configure-tunnel",+ "ensure-dns-record",+ "record-dns",+ "get-tunnel-token",+ "mark-allocation-ready",+]);+export class ManagedEndpointProvisioningFailed extends Schema.TaggedErrorClass<ManagedEndpointProvisioningFailed>()("ManagedEndpointProvisioningFailed",- { cause: Schema.Defect() },+ {+ stage: ManagedEndpointProvisioningStage,+ userId: Schema.String,+ environmentId: Schema.String,+ hostname: Schema.optionalKey(Schema.String),+ tunnelName: Schema.optionalKey(Schema.String),+ tunnelId: Schema.optionalKey(Schema.String),+ dnsRecordId: Schema.optionalKey(Schema.String),+ returnedTunnelName: Schema.optionalKey(Schema.String),+ returnedTunnelId: Schema.optionalKey(Schema.String),+ cause: Schema.optional(Schema.Defect()),+ },) {override get message(): string {- return "Managed endpoint provisioning failed";+ return `Managed endpoint provisioning failed during '${this.stage}' for user '${this.userId}', environment '${this.environmentId}'`;}}+const ManagedEndpointDeprovisioningStage = Schema.Literals([+ "load-allocation",+ "delete-dns-record",+ "delete-tunnel",+ "remove-allocation",+]);+export class ManagedEndpointDeprovisioningFailed extends Schema.TaggedErrorClass<ManagedEndpointDeprovisioningFailed>()("ManagedEndpointDeprovisioningFailed",- { cause: Schema.Defect() },+ {+ stage: ManagedEndpointDeprovisioningStage,+ userId: Schema.String,+ environmentId: Schema.String,+ tunnelId: Schema.optionalKey(Schema.String),+ dnsRecordId: Schema.optionalKey(Schema.String),+ cause: Schema.Defect(),+ },) {override get message(): string {- return "Managed endpoint deprovisioning failed";+ return `Managed endpoint deprovisioning failed during '${this.stage}' for user '${this.userId}', environment '${this.environmentId}'`;}}export class ManagedEndpointOriginNotAllowed extends Schema.TaggedErrorClass<ManagedEndpointOriginNotAllowed>()("ManagedEndpointOriginNotAllowed",{+ userId: Schema.String,+ environmentId: Schema.String,host: Schema.String,port: Schema.Number,},) {override get message(): string {- return `Managed endpoint origin '${this.host}:${this.port}' is not allowed`;+ return `Managed endpoint origin '${this.host}:${this.port}' is not allowed for user '${this.userId}', environment '${this.environmentId}'`;}}@@ -94,12 +139,26 @@ interface ManagedEndpointTunnel {readonly name?: string | null;}+const ManagedEndpointTunnelClientOperation = Schema.Literals([+ "list",+ "create",+ "put-configuration",+ "get-token",+ "delete",+]);+export class ManagedEndpointTunnelClientError extends Schema.TaggedErrorClass<ManagedEndpointTunnelClientError>()("ManagedEndpointTunnelClientError",- { cause: Schema.Defect() },+ {+ operation: ManagedEndpointTunnelClientOperation,+ tunnelName: Schema.optionalKey(Schema.String),+ tunnelId: Schema.optionalKey(Schema.String),+ cause: Schema.Defect(),+ },) {override get message(): string {- return "Managed endpoint tunnel provider request failed";+ const target = this.tunnelId ?? this.tunnelName;+ return `Managed endpoint tunnel provider '${this.operation}' request failed${target === undefined ? "" : ` for '${target}'`}`;}}@@ -147,12 +206,25 @@ interface ManagedEndpointCnameRecordInput {readonly proxied: true;}+const ManagedEndpointDnsClientOperation = Schema.Literals([+ "list-records",+ "create-record",+ "update-record",+ "delete-record",+]);+export class ManagedEndpointDnsClientError extends Schema.TaggedErrorClass<ManagedEndpointDnsClientError>()("ManagedEndpointDnsClientError",- { cause: Schema.Defect() },+ {+ operation: ManagedEndpointDnsClientOperation,+ hostname: Schema.optionalKey(Schema.String),+ dnsRecordId: Schema.optionalKey(Schema.String),+ cause: Schema.Defect(),+ },) {override get message(): string {- return "Managed endpoint DNS provider request failed";+ const target = this.dnsRecordId ?? this.hostname;+ return `Managed endpoint DNS provider '${this.operation}' request failed${target === undefined ? "" : ` for '${target}'`}`;}}@@ -183,13 +255,26 @@ export const layerDnsClient = (client: ManagedEndpointDnsClient["Service"]) =>const requireCloudflareSettings = Effect.fnUntraced(function* (settings: RelayConfiguration.RelayConfiguration["Service"],+ input: { readonly userId: string; readonly environmentId: string },) {- if (!settings.managedEndpointBaseDomain || !settings.managedEndpointNamespace) {- return yield* new ManagedEndpointProvisioningNotConfigured();+ const baseDomain = settings.managedEndpointBaseDomain;+ const namespace = settings.managedEndpointNamespace;+ const missingSettings: Array<"managedEndpointBaseDomain" | "managedEndpointNamespace"> = [];+ if (!baseDomain) {+ missingSettings.push("managedEndpointBaseDomain");+ }+ if (!namespace) {+ missingSettings.push("managedEndpointNamespace");+ }+ if (!baseDomain || !namespace) {+ return yield* new ManagedEndpointProvisioningNotConfigured({+ ...input,+ missingSettings,+ });}return {- baseDomain: settings.managedEndpointBaseDomain,- namespace: settings.managedEndpointNamespace,+ baseDomain,+ namespace,};});@@ -231,10 +316,19 @@ function isNotFoundCause(cause: unknown): boolean {return "cause" in cause && isNotFoundCause(cause.cause);}-const ignoreNotFound = <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<void, E> =>+type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError;++const ignoreNotFound = <A>(+ effect: Effect.Effect<A, ManagedEndpointClientError>,+): Effect.Effect<void, ManagedEndpointClientError> =>effect.pipe(Effect.asVoid,- Effect.catch((cause) => (isNotFoundCause(cause) ? Effect.void : Effect.fail(cause))),+ Effect.catchTags({+ ManagedEndpointTunnelClientError: (error) =>+ isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error),+ ManagedEndpointDnsClientError: (error) =>+ isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error),+ }),);const make = Effect.gen(function* () {@@ -289,25 +383,26 @@ const make = Effect.gen(function* () {}return yield* dns.createRecord(dnsRecord).pipe(Effect.map((record) => record.id),- Effect.catch((createError) =>- Effect.gen(function* () {- let records = yield* dns.listRecords(hostname);- for (let attempt = 0; records.length === 0 && attempt < 4; attempt++) {- yield* Effect.sleep("200 millis");- records = yield* dns.listRecords(hostname);- }- return records;- }).pipe(- Effect.flatMap((records) =>- records.length > 0- ? updateExistingDnsRecords(records, preferredDnsRecordId, dnsRecord)- : Effect.fail(createError),- ),- Effect.flatMap((dnsRecordId) =>- dnsRecordId === null ? Effect.fail(createError) : Effect.succeed(dnsRecordId),+ Effect.catchTags({+ ManagedEndpointDnsClientError: (createError) =>+ Effect.gen(function* () {+ let records = yield* dns.listRecords(hostname);+ for (let attempt = 0; records.length === 0 && attempt < 4; attempt++) {+ yield* Effect.sleep("200 millis");+ records = yield* dns.listRecords(hostname);+ }+ return records;+ }).pipe(+ Effect.flatMap((records) =>+ records.length > 0+ ? updateExistingDnsRecords(records, preferredDnsRecordId, dnsRecord)+ : Effect.fail(createError),+ ),+ Effect.flatMap((dnsRecordId) =>+ dnsRecordId === null ? Effect.fail(createError) : Effect.succeed(dnsRecordId),+ ),),- ),- ),+ }),);});@@ -317,25 +412,59 @@ const make = Effect.gen(function* () {"relay.user_id": input.userId,"relay.environment_id": input.environmentId,});- const allocation = yield* allocations- .get(input)- .pipe(Effect.mapError((cause) => new ManagedEndpointDeprovisioningFailed({ cause })));+ const allocation = yield* allocations.get(input).pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDeprovisioningFailed({+ ...input,+ stage: "load-allocation",+ cause,+ }),+ ),+ );if (allocation === null) {return;}- if (allocation.dnsRecordId !== null) {- yield* ignoreNotFound(dns.deleteRecord(allocation.dnsRecordId)).pipe(- Effect.mapError((cause) => new ManagedEndpointDeprovisioningFailed({ cause })),+ const dnsRecordId = allocation.dnsRecordId;+ if (dnsRecordId !== null) {+ yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDeprovisioningFailed({+ ...input,+ stage: "delete-dns-record",+ dnsRecordId,+ cause,+ }),+ ),);}- if (allocation.tunnelId !== null) {- yield* ignoreNotFound(tunnels.delete(allocation.tunnelId)).pipe(- Effect.mapError((cause) => new ManagedEndpointDeprovisioningFailed({ cause })),+ const tunnelId = allocation.tunnelId;+ if (tunnelId !== null) {+ yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDeprovisioningFailed({+ ...input,+ stage: "delete-tunnel",+ tunnelId,+ cause,+ }),+ ),);}- yield* allocations- .remove(input)- .pipe(Effect.mapError((cause) => new ManagedEndpointDeprovisioningFailed({ cause })));+ yield* allocations.remove(input).pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDeprovisioningFailed({+ ...input,+ stage: "remove-allocation",+ ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }),+ ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }),+ cause,+ }),+ ),+ );}),provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) {yield* Effect.annotateCurrentSpan({@@ -346,11 +475,13 @@ const make = Effect.gen(function* () {});if (!isLoopbackOrigin(input.origin)) {return yield* new ManagedEndpointOriginNotAllowed({+ userId: input.userId,+ environmentId: input.environmentId,host: input.origin.localHttpHost,port: input.origin.localHttpPort,});}- const cf = yield* requireCloudflareSettings(config);+ const cf = yield* requireCloudflareSettings(config, input);const environmentHash = yield* crypto.digest("SHA-256",@@ -360,19 +491,45 @@ const make = Effect.gen(function* () {).pipe(Effect.map(Encoding.encodeHex),- Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "derive-environment-hash",+ cause,+ }),+ ),);+ const requestedHostname = managedEndpointHostname(+ cf.namespace,+ cf.baseDomain,+ environmentHash,+ );+ const requestedTunnelName = managedEndpointTunnelName(cf.namespace, environmentHash);const allocation = yield* allocations.reserve({userId: input.userId,environmentId: input.environmentId,- hostname: managedEndpointHostname(cf.namespace, cf.baseDomain, environmentHash),- tunnelName: managedEndpointTunnelName(cf.namespace, environmentHash),+ hostname: requestedHostname,+ tunnelName: requestedTunnelName,})- .pipe(Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })));+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "reserve-allocation",+ hostname: requestedHostname,+ tunnelName: requestedTunnelName,+ cause,+ }),+ ),+ );const { hostname, tunnelName } = allocation;- const tunnel = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe(+ const tunnelResponse = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe(Effect.map((tunnels) => tunnels.result),Effect.map(Arr.findFirst((tunnel) => tunnel.name === tunnelName)),Effect.flatMap(@@ -381,20 +538,50 @@ const make = Effect.gen(function* () {onNone: () => tunnels.create({ name: tunnelName, configSrc: "cloudflare" }),}),),- Effect.filterMapOrFail((tunnel) =>- tunnel.id && tunnel.name- ? Result.succeed({ id: tunnel.id, name: tunnel.name })- : Result.fail(new ManagedEndpointProvisioningFailed({ cause: tunnel })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "ensure-tunnel",+ hostname,+ tunnelName,+ cause,+ }),),- Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })),);+ if (!tunnelResponse.id || !tunnelResponse.name) {+ return yield* new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "validate-tunnel-response",+ hostname,+ tunnelName,+ ...(tunnelResponse.id ? { returnedTunnelId: tunnelResponse.id } : {}),+ ...(tunnelResponse.name ? { returnedTunnelName: tunnelResponse.name } : {}),+ });+ }+ const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name };yield* allocations.recordTunnel({userId: input.userId,environmentId: input.environmentId,tunnelId: tunnel.id,})- .pipe(Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })));+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "record-tunnel",+ hostname,+ tunnelName,+ tunnelId: tunnel.id,+ cause,+ }),+ ),+ );yield* tunnels.putConfiguration(tunnel.id, {@@ -406,7 +593,20 @@ const make = Effect.gen(function* () {{ service: "http_status:404" },],})- .pipe(Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })));+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "configure-tunnel",+ hostname,+ tunnelName,+ tunnelId: tunnel.id,+ cause,+ }),+ ),+ );const dnsRecord = {type: "CNAME",@@ -417,7 +617,19 @@ const make = Effect.gen(function* () {} as const;const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe(- Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "ensure-dns-record",+ hostname,+ tunnelName,+ tunnelId: tunnel.id,+ ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }),+ cause,+ }),+ ),);yield* allocations.recordDns({@@ -425,17 +637,57 @@ const make = Effect.gen(function* () {environmentId: input.environmentId,dnsRecordId,})- .pipe(Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })));+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "record-dns",+ hostname,+ tunnelName,+ tunnelId: tunnel.id,+ dnsRecordId,+ cause,+ }),+ ),+ );- const connectorToken = yield* tunnels- .getToken(tunnel.id)- .pipe(Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })));+ const connectorToken = yield* tunnels.getToken(tunnel.id).pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "get-tunnel-token",+ hostname,+ tunnelName,+ tunnelId: tunnel.id,+ dnsRecordId,+ cause,+ }),+ ),+ );yield* allocations.markReady({userId: input.userId,environmentId: input.environmentId,})- .pipe(Effect.mapError((cause) => new ManagedEndpointProvisioningFailed({ cause })));+ .pipe(+ Effect.mapError(+ (cause) =>+ new ManagedEndpointProvisioningFailed({+ userId: input.userId,+ environmentId: input.environmentId,+ stage: "mark-allocation-ready",+ hostname,+ tunnelName,+ tunnelId: tunnel.id,+ dnsRecordId,+ cause,+ }),+ ),+ );return {endpoint: managedEndpointForHostname(hostname),@@ -464,27 +716,62 @@ export const layerCloudflareBindings = (ManagedEndpointTunnelClient.of({list: (request) =>tunnelClient.list(request).pipe(- Effect.mapError((cause) => new ManagedEndpointTunnelClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointTunnelClientError({+ operation: "list",+ tunnelName: request.name,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),create: (request) =>tunnelClient.create(request).pipe(- Effect.mapError((cause) => new ManagedEndpointTunnelClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointTunnelClientError({+ operation: "create",+ tunnelName: request.name,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),putConfiguration: (tunnelId, config) =>tunnelClient.putConfiguration(tunnelId, config).pipe(- Effect.mapError((cause) => new ManagedEndpointTunnelClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointTunnelClientError({+ operation: "put-configuration",+ tunnelId,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),getToken: (tunnelId) =>tunnelClient.getToken(tunnelId).pipe(- Effect.mapError((cause) => new ManagedEndpointTunnelClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointTunnelClientError({+ operation: "get-token",+ tunnelId,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),delete: (tunnelId) =>tunnelClient.delete(tunnelId).pipe(- Effect.mapError((cause) => new ManagedEndpointTunnelClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointTunnelClientError({+ operation: "delete",+ tunnelId,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),}),@@ -500,23 +787,52 @@ export const layerCloudflareBindings = (normalizeHostname(record.name) === normalizeHostname(hostname),),),- Effect.mapError((cause) => new ManagedEndpointDnsClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDnsClientError({+ operation: "list-records",+ hostname,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),createRecord: (request) =>dnsClient.createDnsRecord(request).pipe(Effect.map((response) => ({ id: response.id })),- Effect.mapError((cause) => new ManagedEndpointDnsClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDnsClientError({+ operation: "create-record",+ hostname: request.name,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),updateRecord: (dnsRecordId, request) =>dnsClient.updateDnsRecord(dnsRecordId, request).pipe(- Effect.mapError((cause) => new ManagedEndpointDnsClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDnsClientError({+ operation: "update-record",+ hostname: request.name,+ dnsRecordId,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),deleteRecord: (dnsRecordId) =>dnsClient.deleteDnsRecord(dnsRecordId).pipe(- Effect.mapError((cause) => new ManagedEndpointDnsClientError({ cause })),+ Effect.mapError(+ (cause) =>+ new ManagedEndpointDnsClientError({+ operation: "delete-record",+ dnsRecordId,+ cause,+ }),+ ),Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),),}),