The open-source agent harness - the runtime layer that turns an LLM into a working agent.
AGE-1831: enhance DaytonaSandboxProvider with build failure handling (#258)
Sujai Kumar Gupta69237db843c2
diff --git a/.changeset/20260817052904-daytona-snapshot-register-status.md b/.changeset/20260817052904-daytona-snapshot-register-status.mdnew file mode 100644index 000000000..c2ab94b93--- /dev/null+++ b/.changeset/20260817052904-daytona-snapshot-register-status.md@@ -0,0 +1,6 @@+---+'@truefoundry/trueforge-core': patch+'@truefoundry/trueforge': patch+---++Await Daytona snapshot registration on sandbox provider configure so auth failures return 422 instead of a false pending status, and keep GET status refreshes persisted.diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.tsindex 014d3ad81..540ab56e4 100644--- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts+++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts@@ -17,6 +17,8 @@ import { DEFAULT_PREVIEW_URL_EXPIRY_SECONDS, DEFAULT_SANDBOX_NATS_WS_PORT } fromimport type { ExecResult, SandboxBuild, SandboxExecParams, SandboxFileInfo, SandboxProvider } from './Provider';const SANDBOX_NOT_FOUND_STATUS = 404;+/** Another replica already registered this build name; its create is the one that counts. */+const SNAPSHOT_CONFLICT_STATUS = 409;const SANDBOX_STATE_STARTED = 'started';const BUILD_STATE_ACTIVE = 'active';@@ -25,6 +27,12 @@ const BUILD_STATE_ERROR = 'error';const BUILD_STATE_BUILD_FAILED = 'build_failed';const IMAGE_BUILD_NAME_PREFIX = 'trueforge-build-';+/** Same default the Daytona SDK applies when `DaytonaConfig.apiUrl` is omitted. */+const DEFAULT_DAYTONA_API_URL = 'https://app.daytona.io/api';++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null;+}/*** Digest portion of a container image reference (the tag/digest after the final `:`)@@ -60,6 +68,13 @@ function httpUrlToWsUrl(url: string): string {export interface DaytonaSandboxProviderOptions {/** Caller-owned Daytona SDK client (credentials / lifetime). */client: Daytona;+ /**+ * Same API key the client was built with; used for the register-only snapshot POST because the+ * SDK's `snapshot.create` polls to a terminal state instead of returning once registered.+ */+ apiKey: string;+ /** Daytona API base URL (including `/api`). Defaults to the SDK's public cloud endpoint. */+ apiUrl?: string | undefined;tenantName: string;/** Release-owned sandbox image reference; built into a Daytona snapshot and cloned per sandbox. */sandboxImage: string;@@ -94,6 +109,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {private readonly fileMaxBytesForDownload: number;private readonly natsBridgePort: number;private readonly previewUrlExpirySeconds: number;+ private readonly apiKey: string;+ private readonly apiUrl: string;private readonly logger: Logger;private readonly daytona: Daytona;private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();@@ -102,6 +119,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {constructor(options: DaytonaSandboxProviderOptions) {this.daytona = options.client;+ this.apiKey = options.apiKey;+ this.apiUrl = options.apiUrl ?? DEFAULT_DAYTONA_API_URL;this.tenantName = options.tenantName;this.imageUri = options.sandboxImage;this.buildRef = options.buildRef ?? deriveImageBuildName(imageDigest(options.sandboxImage));@@ -243,7 +262,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {}}- private toBuild(state: Snapshot['state'], errorReason: string | null): SandboxBuild {+ private toBuild(state: string, errorReason: string | null): SandboxBuild {const metadata = { build_ref: this.buildRef, image_uri: this.imageUri };switch (state) {case BUILD_STATE_ACTIVE:@@ -257,6 +276,45 @@ export class DaytonaSandboxProvider implements SandboxProvider {}}+ /**+ * Registers the snapshot and returns its initial state without waiting for the build.+ *+ * The SDK's `snapshot.create` issues this same POST and then polls until the snapshot is active+ * or failed (minutes on a cold image pull). Configure only needs the registration result, so+ * credential errors surface on the request while build progress stays observable via `getSnapshot`.+ */+ private async registerSnapshot(): Promise<{ state: string; errorReason: string | null }> {+ let response: Response;+ try {+ response = await fetch(`${this.apiUrl}/snapshots`, {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${this.apiKey}`,+ 'Content-Type': 'application/json',+ },+ body: JSON.stringify({ name: this.buildRef, imageName: this.imageUri }),+ });+ } catch (error) {+ throw new Error('Daytona snapshot registration request failed.', { cause: error });+ }++ const body: unknown = await response.json().catch(() => null);+ if (!response.ok) {+ const message =+ isRecord(body) && typeof body['message'] === 'string'+ ? body['message']+ : `Daytona snapshot registration failed (${String(response.status)})`;+ throw new DaytonaError(message, response.status);+ }+ if (!isRecord(body) || typeof body['state'] !== 'string') {+ throw new DaytonaError("Failed to register snapshot. Daytona didn't return a snapshot state.");+ }+ return {+ state: body['state'],+ errorReason: typeof body['errorReason'] === 'string' ? body['errorReason'] : null,+ };+ }+async buildImage(): Promise<SandboxBuild> {const existing = await this.getSnapshot(this.buildRef);if (existing) {@@ -272,17 +330,24 @@ export class DaytonaSandboxProvider implements SandboxProvider {// re-saving settings (which calls buildImage) could never retry. Drop it, then recreate.await this.deleteFailedBuild(existing);}- // Fire the create in the background: the SDK's snapshot.create polls until the snapshot is- // terminal (minutes on a cold image pull), but Daytona registers it on the first request, so- // progress is observed via getImageBuildStatus. A concurrent-create conflict is harmless.- void this.daytona.snapshot.create({ name: this.buildRef, image: this.imageUri }).catch((error: unknown) => {- this.logger.error(`Daytona snapshot create failed: name=${this.buildRef}`, extractErrorLogFields(error));- });- return {- status: 'pending',- reason: 'Sandbox image build started.',- metadata: { build_ref: this.buildRef, image_uri: this.imageUri },- };++ try {+ const registered = await this.registerSnapshot();+ // Registration returns pending almost always; map whatever Daytona sent so a fast+ // active/error still surfaces correctly without a follow-up GET.+ return this.toBuild(registered.state, registered.errorReason);+ } catch (error) {+ // A losing concurrent create is not a build failure: the winner owns the deterministic name.+ if (error instanceof DaytonaError && error.statusCode === SNAPSHOT_CONFLICT_STATUS) {+ this.logger.info(`Daytona snapshot already created concurrently: name=${this.buildRef}`);+ return {+ status: 'pending',+ reason: 'Sandbox image build started by another server replica.',+ metadata: { build_ref: this.buildRef, image_uri: this.imageUri },+ };+ }+ throw error;+ }}async getImageBuildStatus(): Promise<SandboxBuild> {diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.tsnew file mode 100644index 000000000..26414fb24--- /dev/null+++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts@@ -0,0 +1,80 @@+import { Daytona, DaytonaError } from '@daytona/sdk';+import { DaytonaSandboxProvider } from '../../../src/core/sandbox/provider/DaytonaProvider';+import { makeSilentLogger } from '../harnessMocks';++const NOT_FOUND_STATUS = 404;+const CONFLICT_STATUS = 409;+const FORBIDDEN_STATUS = 403;+const API_URL = 'https://daytona.test/api';++/**+ * Builds a provider whose snapshot lookup reports "not built yet" so `buildImage` always reaches+ * the register-only POST.+ */+function makeProvider(): DaytonaSandboxProvider {+ // useDeprecatedPolling keeps the constructor from opening the event-stream WebSocket.+ const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });+ jest.spyOn(client.snapshot, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS));++ return new DaytonaSandboxProvider({+ client,+ apiKey: 'dtn-test',+ apiUrl: API_URL,+ tenantName: 'test-tenant',+ sandboxImage: 'registry.example.com/sandbox:029ea5ff',+ timeoutMs: 1000,+ autoStopIntervalInMinutes: 5,+ autoArchiveIntervalInMinutes: 60,+ autoDeleteIntervalInMinutes: 7200,+ fileMaxBytesForDownload: 1024,+ logger: makeSilentLogger(),+ });+}++function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction<typeof globalThis.fetch> {+ return jest+ .spyOn(globalThis, 'fetch')+ .mockResolvedValue(new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }));+}++afterEach(() => {+ jest.restoreAllMocks();+});++describe('DaytonaSandboxProvider register-only snapshot create', () => {+ it('awaits the register POST and returns pending without polling to active', async () => {+ const fetchMock = mockFetch({+ status: 200,+ body: { id: 'snap-1', name: 'trueforge-build-029ea5ff', state: 'pending', errorReason: null },+ });++ const build = await makeProvider().buildImage();++ expect(build.status).toBe('pending');+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(fetchMock.mock.calls[0]?.[0]).toBe(`${API_URL}/snapshots`);+ const init = fetchMock.mock.calls[0]?.[1];+ expect(init?.method).toBe('POST');+ expect(JSON.parse(String(init?.body))).toEqual({+ name: 'trueforge-build-029ea5ff',+ imageName: 'registry.example.com/sandbox:029ea5ff',+ });+ });++ it('treats a concurrent-create conflict as pending, not a thrown failure', async () => {+ mockFetch({ status: CONFLICT_STATUS, body: { statusCode: CONFLICT_STATUS, message: 'Conflict' } });++ const build = await makeProvider().buildImage();++ expect(build.status).toBe('pending');+ });++ it('throws on Access denied so PUT can map it to 422', async () => {+ mockFetch({ status: FORBIDDEN_STATUS, body: { statusCode: FORBIDDEN_STATUS, message: 'Access denied' } });++ await expect(makeProvider().buildImage()).rejects.toMatchObject({+ message: 'Access denied',+ statusCode: FORBIDDEN_STATUS,+ });+ });+});diff --git a/packages/trueforge/src/apis/sandboxProviders.ts b/packages/trueforge/src/apis/sandboxProviders.tsindex 8f4e982a0..13b54b362 100644--- a/packages/trueforge/src/apis/sandboxProviders.ts+++ b/packages/trueforge/src/apis/sandboxProviders.ts@@ -14,7 +14,7 @@ import type { PutSandboxProviderRequest, SandboxProviderManifest } from '../scheimport { MissingStoredSecretError, resolveStoredSecretValue, toRedactedSecretValue } from '../utils/secretRedaction';import { TENANT_ID } from './sessions';-/** Cap the Daytona build kickoff so a slow/unreachable provider can't hold the request (or DB txn) open. */+/** Cap the Daytona register round-trip so a slow/unreachable provider can't hold the request (or DB txn) open. */const BUILD_REQUEST_TIMEOUT_MS = 3_000;export interface SandboxProvidersRouterDeps<TTransaction> {diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.tsindex 2770bf9cd..9189f7742 100644--- a/packages/trueforge/src/sandbox/providerUtils.ts+++ b/packages/trueforge/src/sandbox/providerUtils.ts@@ -3,7 +3,7 @@ import { Daytona, DaytonaError } from '@daytona/sdk';import { DaytonaSandboxProvider, SANDBOX_IMAGE_URI, type SandboxBuild } from '@truefoundry/trueforge-core/core';import type { Logger } from 'winston';import configuration from '../config';-import type { ISandboxProviderStore } from '../db/sandboxProviderStore';+import type { ISandboxProviderStore, SandboxProviderRecord } from '../db/sandboxProviderStore';import {toDaytonaSandboxProviderInput,type SandboxBuildMetadata,@@ -38,6 +38,7 @@ export function toDaytonaSandboxProvider({const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest);return new DaytonaSandboxProvider({client: new Daytona({ apiKey }),+ apiKey,...settings,tenantName: tenant_id,sandboxImage: build_metadata?.['image_uri'] ?? SANDBOX_IMAGE_URI,@@ -56,6 +57,14 @@ export function toSandboxStatus(build: SandboxBuild): SandboxStatus {};}+function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus {+ return {+ status: record.status,+ status_reason: record.status_reason,+ build_metadata: record.build_metadata,+ };+}+// Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead.const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000;@@ -73,11 +82,7 @@ export async function checkSnapshotStatus({return undefined;}- const persisted: SandboxStatus = {- status: record.status,- status_reason: record.status_reason,- build_metadata: record.build_metadata,- };+ const persisted = sandboxStatusFromRecord(record);const readyIsFresh =record.status === 'ready' && Date.now() - Date.parse(record.updated_at) < READY_REVALIDATE_INTERVAL_MS;@@ -99,6 +104,6 @@ export async function checkSnapshotStatus({build = await provider.getImageBuildStatus();}const next = toSandboxStatus(build);- await store.updateSandboxStatus({ tenant_id, ...next });- return next;+ const updated = await store.updateSandboxStatus({ tenant_id, ...next });+ return updated ? sandboxStatusFromRecord(updated) : next;}