Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

AGE-1831: enhance DaytonaSandboxProvider with build failure handling (#258)

Sujai Kumar Gupta69237db843c2
diff --git a/.changeset/20260817052904-daytona-snapshot-register-status.md b/.changeset/20260817052904-daytona-snapshot-register-status.md
new file mode 100644
index 000000000..c2ab94b93
--- /dev/null
+++ b/.changeset/20260817052904-daytona-snapshot-register-status.md
@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge-core': patch
+'@truefoundry/trueforge': patch
+---
+
+Await Daytona snapshot registration on sandbox provider configure so auth failures return 422 instead of a false pending status, and keep GET status refreshes persisted.
diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
index 014d3ad81..540ab56e4 100644
--- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
+++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
@@ -17,6 +17,8 @@ import { DEFAULT_PREVIEW_URL_EXPIRY_SECONDS, DEFAULT_SANDBOX_NATS_WS_PORT } from
import type { ExecResult, SandboxBuild, SandboxExecParams, SandboxFileInfo, SandboxProvider } from './Provider';
const SANDBOX_NOT_FOUND_STATUS = 404;
+/** Another replica already registered this build name; its create is the one that counts. */
+const SNAPSHOT_CONFLICT_STATUS = 409;
const SANDBOX_STATE_STARTED = 'started';
const BUILD_STATE_ACTIVE = 'active';
@@ -25,6 +27,12 @@ const BUILD_STATE_ERROR = 'error';
const BUILD_STATE_BUILD_FAILED = 'build_failed';
const IMAGE_BUILD_NAME_PREFIX = 'trueforge-build-';
+/** Same default the Daytona SDK applies when `DaytonaConfig.apiUrl` is omitted. */
+const DEFAULT_DAYTONA_API_URL = 'https://app.daytona.io/api';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === 'object' && value !== null;
+}
/**
* Digest portion of a container image reference (the tag/digest after the final `:`)
@@ -60,6 +68,13 @@ function httpUrlToWsUrl(url: string): string {
export interface DaytonaSandboxProviderOptions {
/** Caller-owned Daytona SDK client (credentials / lifetime). */
client: Daytona;
+ /**
+ * Same API key the client was built with; used for the register-only snapshot POST because the
+ * SDK's `snapshot.create` polls to a terminal state instead of returning once registered.
+ */
+ apiKey: string;
+ /** Daytona API base URL (including `/api`). Defaults to the SDK's public cloud endpoint. */
+ apiUrl?: string | undefined;
tenantName: string;
/** Release-owned sandbox image reference; built into a Daytona snapshot and cloned per sandbox. */
sandboxImage: string;
@@ -94,6 +109,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
private readonly fileMaxBytesForDownload: number;
private readonly natsBridgePort: number;
private readonly previewUrlExpirySeconds: number;
+ private readonly apiKey: string;
+ private readonly apiUrl: string;
private readonly logger: Logger;
private readonly daytona: Daytona;
private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();
@@ -102,6 +119,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
constructor(options: DaytonaSandboxProviderOptions) {
this.daytona = options.client;
+ this.apiKey = options.apiKey;
+ this.apiUrl = options.apiUrl ?? DEFAULT_DAYTONA_API_URL;
this.tenantName = options.tenantName;
this.imageUri = options.sandboxImage;
this.buildRef = options.buildRef ?? deriveImageBuildName(imageDigest(options.sandboxImage));
@@ -243,7 +262,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
}
}
- private toBuild(state: Snapshot['state'], errorReason: string | null): SandboxBuild {
+ private toBuild(state: string, errorReason: string | null): SandboxBuild {
const metadata = { build_ref: this.buildRef, image_uri: this.imageUri };
switch (state) {
case BUILD_STATE_ACTIVE:
@@ -257,6 +276,45 @@ export class DaytonaSandboxProvider implements SandboxProvider {
}
}
+ /**
+ * Registers the snapshot and returns its initial state without waiting for the build.
+ *
+ * The SDK's `snapshot.create` issues this same POST and then polls until the snapshot is active
+ * or failed (minutes on a cold image pull). Configure only needs the registration result, so
+ * credential errors surface on the request while build progress stays observable via `getSnapshot`.
+ */
+ private async registerSnapshot(): Promise<{ state: string; errorReason: string | null }> {
+ let response: Response;
+ try {
+ response = await fetch(`${this.apiUrl}/snapshots`, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${this.apiKey}`,
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({ name: this.buildRef, imageName: this.imageUri }),
+ });
+ } catch (error) {
+ throw new Error('Daytona snapshot registration request failed.', { cause: error });
+ }
+
+ const body: unknown = await response.json().catch(() => null);
+ if (!response.ok) {
+ const message =
+ isRecord(body) && typeof body['message'] === 'string'
+ ? body['message']
+ : `Daytona snapshot registration failed (${String(response.status)})`;
+ throw new DaytonaError(message, response.status);
+ }
+ if (!isRecord(body) || typeof body['state'] !== 'string') {
+ throw new DaytonaError("Failed to register snapshot. Daytona didn't return a snapshot state.");
+ }
+ return {
+ state: body['state'],
+ errorReason: typeof body['errorReason'] === 'string' ? body['errorReason'] : null,
+ };
+ }
+
async buildImage(): Promise<SandboxBuild> {
const existing = await this.getSnapshot(this.buildRef);
if (existing) {
@@ -272,17 +330,24 @@ export class DaytonaSandboxProvider implements SandboxProvider {
// re-saving settings (which calls buildImage) could never retry. Drop it, then recreate.
await this.deleteFailedBuild(existing);
}
- // Fire the create in the background: the SDK's snapshot.create polls until the snapshot is
- // terminal (minutes on a cold image pull), but Daytona registers it on the first request, so
- // progress is observed via getImageBuildStatus. A concurrent-create conflict is harmless.
- void this.daytona.snapshot.create({ name: this.buildRef, image: this.imageUri }).catch((error: unknown) => {
- this.logger.error(`Daytona snapshot create failed: name=${this.buildRef}`, extractErrorLogFields(error));
- });
- return {
- status: 'pending',
- reason: 'Sandbox image build started.',
- metadata: { build_ref: this.buildRef, image_uri: this.imageUri },
- };
+
+ try {
+ const registered = await this.registerSnapshot();
+ // Registration returns pending almost always; map whatever Daytona sent so a fast
+ // active/error still surfaces correctly without a follow-up GET.
+ return this.toBuild(registered.state, registered.errorReason);
+ } catch (error) {
+ // A losing concurrent create is not a build failure: the winner owns the deterministic name.
+ if (error instanceof DaytonaError && error.statusCode === SNAPSHOT_CONFLICT_STATUS) {
+ this.logger.info(`Daytona snapshot already created concurrently: name=${this.buildRef}`);
+ return {
+ status: 'pending',
+ reason: 'Sandbox image build started by another server replica.',
+ metadata: { build_ref: this.buildRef, image_uri: this.imageUri },
+ };
+ }
+ throw error;
+ }
}
async getImageBuildStatus(): Promise<SandboxBuild> {
diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts
new file mode 100644
index 000000000..26414fb24
--- /dev/null
+++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts
@@ -0,0 +1,80 @@
+import { Daytona, DaytonaError } from '@daytona/sdk';
+import { DaytonaSandboxProvider } from '../../../src/core/sandbox/provider/DaytonaProvider';
+import { makeSilentLogger } from '../harnessMocks';
+
+const NOT_FOUND_STATUS = 404;
+const CONFLICT_STATUS = 409;
+const FORBIDDEN_STATUS = 403;
+const API_URL = 'https://daytona.test/api';
+
+/**
+ * Builds a provider whose snapshot lookup reports "not built yet" so `buildImage` always reaches
+ * the register-only POST.
+ */
+function makeProvider(): DaytonaSandboxProvider {
+ // useDeprecatedPolling keeps the constructor from opening the event-stream WebSocket.
+ const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+ jest.spyOn(client.snapshot, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS));
+
+ return new DaytonaSandboxProvider({
+ client,
+ apiKey: 'dtn-test',
+ apiUrl: API_URL,
+ tenantName: 'test-tenant',
+ sandboxImage: 'registry.example.com/sandbox:029ea5ff',
+ timeoutMs: 1000,
+ autoStopIntervalInMinutes: 5,
+ autoArchiveIntervalInMinutes: 60,
+ autoDeleteIntervalInMinutes: 7200,
+ fileMaxBytesForDownload: 1024,
+ logger: makeSilentLogger(),
+ });
+}
+
+function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction<typeof globalThis.fetch> {
+ return jest
+ .spyOn(globalThis, 'fetch')
+ .mockResolvedValue(new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }));
+}
+
+afterEach(() => {
+ jest.restoreAllMocks();
+});
+
+describe('DaytonaSandboxProvider register-only snapshot create', () => {
+ it('awaits the register POST and returns pending without polling to active', async () => {
+ const fetchMock = mockFetch({
+ status: 200,
+ body: { id: 'snap-1', name: 'trueforge-build-029ea5ff', state: 'pending', errorReason: null },
+ });
+
+ const build = await makeProvider().buildImage();
+
+ expect(build.status).toBe('pending');
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(fetchMock.mock.calls[0]?.[0]).toBe(`${API_URL}/snapshots`);
+ const init = fetchMock.mock.calls[0]?.[1];
+ expect(init?.method).toBe('POST');
+ expect(JSON.parse(String(init?.body))).toEqual({
+ name: 'trueforge-build-029ea5ff',
+ imageName: 'registry.example.com/sandbox:029ea5ff',
+ });
+ });
+
+ it('treats a concurrent-create conflict as pending, not a thrown failure', async () => {
+ mockFetch({ status: CONFLICT_STATUS, body: { statusCode: CONFLICT_STATUS, message: 'Conflict' } });
+
+ const build = await makeProvider().buildImage();
+
+ expect(build.status).toBe('pending');
+ });
+
+ it('throws on Access denied so PUT can map it to 422', async () => {
+ mockFetch({ status: FORBIDDEN_STATUS, body: { statusCode: FORBIDDEN_STATUS, message: 'Access denied' } });
+
+ await expect(makeProvider().buildImage()).rejects.toMatchObject({
+ message: 'Access denied',
+ statusCode: FORBIDDEN_STATUS,
+ });
+ });
+});
diff --git a/packages/trueforge/src/apis/sandboxProviders.ts b/packages/trueforge/src/apis/sandboxProviders.ts
index 8f4e982a0..13b54b362 100644
--- a/packages/trueforge/src/apis/sandboxProviders.ts
+++ b/packages/trueforge/src/apis/sandboxProviders.ts
@@ -14,7 +14,7 @@ import type { PutSandboxProviderRequest, SandboxProviderManifest } from '../sche
import { MissingStoredSecretError, resolveStoredSecretValue, toRedactedSecretValue } from '../utils/secretRedaction';
import { TENANT_ID } from './sessions';
-/** Cap the Daytona build kickoff so a slow/unreachable provider can't hold the request (or DB txn) open. */
+/** Cap the Daytona register round-trip so a slow/unreachable provider can't hold the request (or DB txn) open. */
const BUILD_REQUEST_TIMEOUT_MS = 3_000;
export interface SandboxProvidersRouterDeps<TTransaction> {
diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.ts
index 2770bf9cd..9189f7742 100644
--- a/packages/trueforge/src/sandbox/providerUtils.ts
+++ b/packages/trueforge/src/sandbox/providerUtils.ts
@@ -3,7 +3,7 @@ import { Daytona, DaytonaError } from '@daytona/sdk';
import { DaytonaSandboxProvider, SANDBOX_IMAGE_URI, type SandboxBuild } from '@truefoundry/trueforge-core/core';
import type { Logger } from 'winston';
import configuration from '../config';
-import type { ISandboxProviderStore } from '../db/sandboxProviderStore';
+import type { ISandboxProviderStore, SandboxProviderRecord } from '../db/sandboxProviderStore';
import {
toDaytonaSandboxProviderInput,
type SandboxBuildMetadata,
@@ -38,6 +38,7 @@ export function toDaytonaSandboxProvider({
const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest);
return new DaytonaSandboxProvider({
client: new Daytona({ apiKey }),
+ apiKey,
...settings,
tenantName: tenant_id,
sandboxImage: build_metadata?.['image_uri'] ?? SANDBOX_IMAGE_URI,
@@ -56,6 +57,14 @@ export function toSandboxStatus(build: SandboxBuild): SandboxStatus {
};
}
+function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus {
+ return {
+ status: record.status,
+ status_reason: record.status_reason,
+ build_metadata: record.build_metadata,
+ };
+}
+
// Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead.
const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000;
@@ -73,11 +82,7 @@ export async function checkSnapshotStatus({
return undefined;
}
- const persisted: SandboxStatus = {
- status: record.status,
- status_reason: record.status_reason,
- build_metadata: record.build_metadata,
- };
+ const persisted = sandboxStatusFromRecord(record);
const readyIsFresh =
record.status === 'ready' && Date.now() - Date.parse(record.updated_at) < READY_REVALIDATE_INTERVAL_MS;
@@ -99,6 +104,6 @@ export async function checkSnapshotStatus({
build = await provider.getImageBuildStatus();
}
const next = toSandboxStatus(build);
- await store.updateSandboxStatus({ tenant_id, ...next });
- return next;
+ const updated = await store.updateSandboxStatus({ tenant_id, ...next });
+ return updated ? sandboxStatusFromRecord(updated) : next;
}