Skip to content

The open-source app everyone uses to manage agents at work

feat(server): provision managed sandbox environments from the managed config (#10324)

Devin Foley273315a4d01a
diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.ts
index eff92293fd..4b3ef48810 100644
--- a/server/src/__tests__/environment-service.test.ts
+++ b/server/src/__tests__/environment-service.test.ts
@@ -630,6 +630,147 @@ describeEmbeddedPostgres("environmentService leases", () => {
expect((rows[0]?.metadata as Record<string, unknown>)?.managedKubernetesSandbox).toBe(true);
});
+ it("ensures and refreshes a managed sandbox environment for an arbitrary provider", async () => {
+ const created = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona",
+ description: "Managed Daytona sandbox environment.",
+ provider: "daytona",
+ config: { target: "us" },
+ });
+
+ expect(created.driver).toBe("sandbox");
+ expect(created.name).toBe("Daytona");
+ expect(created.config.provider).toBe("daytona");
+ expect(created.config.target).toBe("us");
+ expect(created.metadata?.managedByPaperclip).toBe(true);
+ expect(created.metadata?.managedSandboxProvider).toBe("daytona");
+
+ // Idempotent: a second call refreshes config and name in place, and a
+ // description omitted from the spec is cleared, not pinned forever.
+ const refreshed = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona (EU)",
+ provider: "daytona",
+ config: { target: "eu" },
+ });
+ expect(refreshed.id).toBe(created.id);
+ expect(refreshed.name).toBe("Daytona (EU)");
+ expect(refreshed.config.target).toBe("eu");
+ expect(refreshed.description).toBeNull();
+
+ const rows = await db
+ .select()
+ .from(environments)
+ .where(eq(environments.driver, "sandbox"));
+ expect(rows).toHaveLength(1);
+ });
+
+ it("adopts the managed slot on a provider switch and drops the stale kubernetes marker", async () => {
+ const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" });
+ expect(kubernetes.metadata?.managedKubernetesSandbox).toBe(true);
+
+ const daytona = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona",
+ provider: "daytona",
+ config: { target: "us" },
+ });
+
+ expect(daytona.id).toBe(kubernetes.id);
+ expect(daytona.name).toBe("Daytona");
+ expect(daytona.config.provider).toBe("daytona");
+ expect(daytona.config.backend).toBeUndefined();
+ expect(daytona.metadata?.managedSandboxProvider).toBe("daytona");
+ expect(daytona.metadata?.managedKubernetesSandbox).toBeUndefined();
+ expect(await svc.findKubernetesEnvironment()).toBeNull();
+
+ // And back: the kubernetes wrapper re-adopts the same row.
+ const restored = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" });
+ expect(restored.id).toBe(kubernetes.id);
+ expect(restored.metadata?.managedKubernetesSandbox).toBe(true);
+ });
+
+ it("archives the managed sandbox row only for its own provider and reactivates on ensure", async () => {
+ // Nothing provisioned yet: archiving is a no-op.
+ expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull();
+
+ const created = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona",
+ provider: "daytona",
+ config: { target: "us" },
+ });
+ expect(created.status).toBe("active");
+
+ // Another provider's unavailability leaves this provider's row alone.
+ expect(await svc.archiveManagedSandboxEnvironment({ provider: "kubernetes" })).toBeNull();
+
+ const archived = await svc.archiveManagedSandboxEnvironment({ provider: "daytona" });
+ expect(archived?.id).toBe(created.id);
+ expect(archived?.status).toBe("archived");
+
+ // Already archived: a repeat call is a no-op.
+ expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull();
+
+ // The next healthy boot's ensure re-activates the same row.
+ const restored = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona",
+ provider: "daytona",
+ config: { target: "us" },
+ });
+ expect(restored.id).toBe(created.id);
+ expect(restored.status).toBe("active");
+ });
+
+ it("adopts an existing unmanaged sandbox row holding the desired name", async () => {
+ const handMade = await svc.create({
+ name: "Daytona",
+ driver: "sandbox",
+ status: "active",
+ config: { provider: "daytona", target: "us" },
+ });
+ expect(handMade.metadata?.managedByPaperclip).toBeUndefined();
+
+ const adopted = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona",
+ provider: "daytona",
+ config: { target: "eu" },
+ });
+ expect(adopted.id).toBe(handMade.id);
+ expect(adopted.config.target).toBe("eu");
+ expect(adopted.metadata?.managedByPaperclip).toBe(true);
+ expect(adopted.metadata?.managedSandboxProvider).toBe("daytona");
+
+ const rows = await db
+ .select()
+ .from(environments)
+ .where(eq(environments.driver, "sandbox"));
+ expect(rows).toHaveLength(1);
+ });
+
+ it("keeps the current name when the desired name belongs to another row", async () => {
+ await svc.create({
+ name: "Daytona",
+ driver: "ssh",
+ status: "active",
+ config: {
+ host: "fixture.example.test",
+ port: 22,
+ username: "fixture",
+ remoteWorkspacePath: "/srv/paperclip",
+ },
+ });
+ const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true });
+
+ // The managed slot is adopted, but the rename would collide with the ssh
+ // row on environments_name_idx; the ensure keeps the existing name.
+ const adopted = await svc.ensureManagedSandboxEnvironment({
+ name: "Daytona",
+ provider: "daytona",
+ config: { target: "us" },
+ });
+ expect(adopted.id).toBe(kubernetes.id);
+ expect(adopted.name).toBe(kubernetes.name);
+ expect(adopted.config.provider).toBe("daytona");
+ });
+
it("returns a conflict when creating a second environment with the same name", async () => {
await seedEnvironment();
diff --git a/server/src/__tests__/managed-config.test.ts b/server/src/__tests__/managed-config.test.ts
index 1717b33288..370999e348 100644
--- a/server/src/__tests__/managed-config.test.ts
+++ b/server/src/__tests__/managed-config.test.ts
@@ -53,6 +53,7 @@ describe("parseManagedConfigEnv", () => {
catalogVersion: "2026.720.0",
features: { enableApps: false, enablePipelines: true },
plugins: { autoInstall: ["daytona", "kubernetes"] },
+ environments: [],
});
});
@@ -66,6 +67,7 @@ describe("parseManagedConfigEnv", () => {
catalogVersion: "2026.720.0",
features: {},
plugins: { autoInstall: [] },
+ environments: [],
});
});
@@ -223,6 +225,115 @@ describe("parseManagedConfigEnv", () => {
});
});
+describe("parseManagedConfigEnv environments section", () => {
+ const entry = (overrides: Record<string, unknown> = {}) => ({
+ name: "Daytona",
+ provider: "daytona",
+ config: { target: "us" },
+ ...overrides,
+ });
+
+ it("defaults to an empty list when the section is absent (pre-section documents keep booting)", () => {
+ const config = parseManagedConfigEnv(envWith(validDoc()));
+ expect(config?.environments).toEqual([]);
+ });
+
+ it("parses a declared environment and freezes it", () => {
+ const config = parseManagedConfigEnv(
+ envWith(validDoc({ environments: [entry({ description: "Managed Daytona sandbox." })] })),
+ );
+ expect(config?.environments).toHaveLength(1);
+ const spec = config?.environments[0];
+ expect(spec?.name).toBe("Daytona");
+ expect(spec?.description).toBe("Managed Daytona sandbox.");
+ expect(spec?.provider).toBe("daytona");
+ expect(spec?.config).toEqual({ target: "us" });
+ expect(Object.isFrozen(config?.environments)).toBe(true);
+ expect(Object.isFrozen(spec)).toBe(true);
+ expect(Object.isFrozen(spec?.config)).toBe(true);
+ });
+
+ it("treats config and description as optional", () => {
+ const config = parseManagedConfigEnv(
+ envWith(validDoc({ environments: [{ name: "Daytona", provider: "daytona" }] })),
+ );
+ expect(config?.environments[0]?.config).toEqual({});
+ expect(config?.environments[0]?.description).toBeUndefined();
+ });
+
+ it("rejects a non-array section and non-object entries", () => {
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: {} }))),
+ ).toThrow(/"environments" must be an array/);
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: ["daytona"] }))),
+ ).toThrow(/"environments\[0\]" must be an object/);
+ });
+
+ it("rejects more than one entry (single managed sandbox slot)", () => {
+ expect(() =>
+ parseManagedConfigEnv(
+ envWith(validDoc({ environments: [entry(), entry({ name: "Other", provider: "kubernetes" })] })),
+ ),
+ ).toThrow(/at most one entry/);
+ });
+
+ it("rejects unknown entry keys", () => {
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ envVars: {} })] }))),
+ ).toThrow(/unknown key "envVars"/);
+ });
+
+ it("rejects malformed names, descriptions, and providers", () => {
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ name: "" })] }))),
+ ).toThrow(/"environments\[0\].name"/);
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ name: " Daytona" })] }))),
+ ).toThrow(/"environments\[0\].name"/);
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ description: " " })] }))),
+ ).toThrow(/"environments\[0\].description"/);
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ provider: 7 })] }))),
+ ).toThrow(/"environments\[0\].provider"/);
+ });
+
+ it("rejects a provider that plugins.autoInstall does not provision", () => {
+ expect(() =>
+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ provider: "modal" })] }))),
+ ).toThrow(/not in "plugins.autoInstall"/);
+ });
+
+ it("rejects a config that sets provider", () => {
+ expect(() =>
+ parseManagedConfigEnv(
+ envWith(validDoc({ environments: [entry({ config: { provider: "daytona" } })] })),
+ ),
+ ).toThrow(/must not set "provider"/);
+ });
+
+ it("rejects secret-bearing config keys at any depth (secrets travel as env vars)", () => {
+ expect(() =>
+ parseManagedConfigEnv(
+ envWith(validDoc({ environments: [entry({ config: { apiKey: "not-a-real-key" } })] })),
+ ),
+ ).toThrow(/looks secret-bearing/);
+ expect(() =>
+ parseManagedConfigEnv(
+ envWith(validDoc({ environments: [entry({ config: { auth: { accessToken: "t" } } })] })),
+ ),
+ ).toThrow(/auth.accessToken/);
+ expect(() =>
+ parseManagedConfigEnv(
+ envWith(
+ validDoc({ environments: [entry({ config: { adapters: [{ clientSecret: "s" }] } })] }),
+ ),
+ ),
+ ).toThrow(/adapters\[0\].clientSecret/);
+ });
+});
+
describe("getManagedInstanceConfig", () => {
it("caches by raw env value and reparses when it changes", () => {
const raw = validDoc();
diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts
index f7a277fd71..d309533dde 100644
--- a/server/src/__tests__/server-startup-feedback-export.test.ts
+++ b/server/src/__tests__/server-startup-feedback-export.test.ts
@@ -206,6 +206,7 @@ vi.mock("../services/index.js", () => ({
})),
feedbackService: feedbackServiceFactoryMock,
bootstrapExecutionPolicyFromEnv: vi.fn(async () => null),
+ applyManagedEnvironments: vi.fn(async () => null),
environmentCustomImageService: environmentCustomImagesServiceFactoryMock,
heartbeatService: heartbeatServiceFactoryMock,
issueService: vi.fn(() => ({ update: vi.fn(async () => null) })),
diff --git a/server/src/app.ts b/server/src/app.ts
index d6f6829684..5eeeb751c0 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -590,7 +590,13 @@ export async function createApp(
// swallowed per plugin so the server ALWAYS finishes booting. A degraded
// boot (a provider unavailable, some agents cannot run) is strictly
// preferable to a crash loop.
- void ensureBundledPlugins(
+ //
+ // The chain is not awaited here (createApp stays fast), but the settled
+ // promise is exposed via `app.locals.bundledPluginsStartup` so boot steps
+ // that must not outrun plugin availability — managed sandbox environments
+ // (`applyManagedEnvironments`) run before the heartbeat resumes queued
+ // runs — can sequence on it. It never rejects.
+ const bundledPluginsStartup = ensureBundledPlugins(
bundledPluginInstalls,
{ registry: pluginRegistry, loader, lifecycle, logger },
// Managed mode reinstalls soft-uninstalled bundles (the control plane
@@ -609,6 +615,7 @@ export async function createApp(
}).catch((err) => {
logger.error({ err }, "Failed to load ready plugins on startup");
});
+ app.locals.bundledPluginsStartup = bundledPluginsStartup;
let appServicesShutdown = false;
const shutdownAppServices = () => {
if (appServicesShutdown) return;
diff --git a/server/src/index.ts b/server/src/index.ts
index 8161c2cd37..3908d1cfc1 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -41,6 +41,7 @@ import { setupEnvironmentCustomImageTerminalWebSocketServer } from "./realtime/e
import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js";
import {
feedbackService,
+ applyManagedEnvironments,
backfillPrincipalAccessCompatibility,
backfillLegacyToolOAuthTokens,
bootstrapExecutionPolicyFromEnv,
@@ -860,6 +861,31 @@ export async function startServer(): Promise<StartedServer> {
throw err;
}
+ // Ensure sandbox environments declared in the managed-config document
+ // (`environments` section) before the heartbeat resumes queued runs. The
+ // document already parsed fail-closed above; the ensure step itself is
+ // fail-safe per entry (a degraded boot beats a fleet-wide crash loop), but
+ // a contradictory deployment that also forces PAPERCLIP_EXECUTION_MODE
+ // throws here and fails startup. `pluginsReady` sequences the ensure after
+ // the bundled-plugin install/load pass so a declared environment never
+ // activates before its provider driver is registered; the worker manager
+ // additionally gates each entry on a live plugin worker (and archives the
+ // row of a provider that did not come up).
+ try {
+ const bundledPluginsStartup = (app as { locals?: { bundledPluginsStartup?: Promise<unknown> } })
+ .locals?.bundledPluginsStartup;
+ const managedEnvironmentsResult = await applyManagedEnvironments(db as any, managedConfig, {
+ pluginsReady: bundledPluginsStartup,
+ workerManager: pluginWorkerManager,
+ });
+ if (managedEnvironmentsResult) {
+ logger.warn(managedEnvironmentsResult, "managed sandbox environments ensured from managed config");
+ }
+ } catch (err) {
+ logger.error({ err }, "failed to apply managed environments from managed config");
+ throw err;
+ }
+
let drainHeartbeatRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<unknown>) | null = null;
let prepareHotRestartShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<{ skipDrain: boolean }>) | null = null;
let heartbeatSchedulerStopped = false;
diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts
index 6487f7a1b9..d9a4466168 100644
--- a/server/src/services/environments.ts
+++ b/server/src/services/environments.ts
@@ -73,6 +73,26 @@ export interface KubernetesEnvironmentConfigInput {
[key: string]: unknown;
}
+/**
+ * Input to `ensureManagedSandboxEnvironment`. Provider-agnostic: `provider`
+ * is the sandbox plugin's driver key and is forced into `config.provider`;
+ * the rest of `config` is stored verbatim for the plugin to validate at
+ * lease time.
+ */
+export interface ManagedSandboxEnvironmentInput {
+ name: string;
+ description?: string;
+ /** Sandbox provider key (the plugin's driverKey, e.g. "kubernetes", "daytona"). */
+ provider: string;
+ config?: Record<string, unknown>;
+ /**
+ * Extra metadata markers stamped on the managed row (e.g. the legacy
+ * kubernetes marker `managedKubernetesSandbox` that
+ * `findKubernetesEnvironment` keys on).
+ */
+ extraMetadata?: Record<string, unknown>;
+}
+
function cloneRecord(value: unknown, fallback: Record<string, unknown> | null = null): Record<string, unknown> | null {
if (!value || typeof value !== "object" || Array.isArray(value)) return fallback;
return { ...(value as Record<string, unknown>) };
@@ -181,6 +201,177 @@ function countFromRows(rows: Array<{ count: number | string | null | undefined }
}
export function environmentService(db: Db) {
+ /** The single Paperclip-managed sandbox row (`environments_managed_sandbox_idx`), if present. */
+ const findManagedSandboxRow = () =>
+ db
+ .select()
+ .from(environments)
+ .where(eq(environments.driver, "sandbox"))
+ .then(
+ (rows) =>
+ rows.find(
+ (row) =>
+ (row.metadata as Record<string, unknown> | null)?.managedByPaperclip === true,
+ ) ?? null,
+ );
+
+ /**
+ * Idempotently ensure THE Paperclip-managed sandbox environment for this
+ * instance, configured for an arbitrary sandbox provider plugin. Mirrors
+ * `ensureLocalEnvironment`; the partial unique index
+ * `environments_managed_sandbox_idx` enforces at most one managed sandbox
+ * row per instance, so this function owns that single slot regardless of
+ * provider:
+ *
+ * - An existing managed row is adopted and refreshed (name, description,
+ * config, provider) on every call, so operator/control-plane changes flow
+ * via redeploy without recreating the row — including a provider switch,
+ * which also drops a stale provider-specific metadata marker.
+ * - An existing UNmanaged sandbox row holding the desired name is adopted
+ * and stamped as managed, so a row created by hand before the instance
+ * became config-managed converges instead of colliding on
+ * `environments_name_idx` on every boot.
+ */
+ const ensureManagedSandboxEnvironment = async (
+ input: ManagedSandboxEnvironmentInput,
+ ): Promise<Environment> => {
+ const desiredConfig: Record<string, unknown> = {
+ ...(input.config ?? {}),
+ provider: input.provider,
+ };
+ const desiredMetadata: Record<string, unknown> = {
+ managedByPaperclip: true,
+ managedSandboxProvider: input.provider,
+ ...(input.extraMetadata ?? {}),
+ };
+
+ const adopt = async (row: EnvironmentRow): Promise<Environment> => {
+ const metadata: Record<string, unknown> = { ...(row.metadata ?? {}), ...desiredMetadata };
+ // A provider switch must not leave the previous provider's marker
+ // behind (`findKubernetesEnvironment` keys on it).
+ if (desiredMetadata[KUBERNETES_MANAGED_MARKER] !== true) {
+ delete metadata[KUBERNETES_MANAGED_MARKER];
+ }
+ const now = new Date();
+ const runUpdate = (values: { name?: string }) =>
+ db
+ .update(environments)
+ .set({
+ ...values,
+ // The row mirrors the managed spec: omitting `description` clears
+ // a previously configured one rather than pinning it forever.
+ description: input.description ?? null,
+ config: desiredConfig,
+ metadata,
+ status: "active",
+ updatedAt: now,
+ })
+ .where(eq(environments.id, row.id))
+ .returning()
+ .then((rows) => rows[0] ?? row);
+ const updated = await runUpdate({ name: input.name }).catch((error: unknown) => {
+ // Another row already holds the desired name; keep the current name
+ // rather than failing a boot-time ensure over a display label.
+ if (hasConstraintName(error, "environments_name_idx")) {
+ return runUpdate({});
+ }
+ throw error;
+ });
+ return toEnvironment(updated);
+ };
+
+ const existing = await findManagedSandboxRow();
+ if (existing) return adopt(existing);
+
+ // The partial unique index `environments_managed_sandbox_idx` enforces
+ // "at most one Paperclip-managed sandbox row per instance" at the DB
+ // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent
+ // callers can race the INSERT; losers re-read the surviving row.
+ const now = new Date();
+ const inserted = await db
+ .insert(environments)
+ .values({
+ name: input.name,
+ description: input.description ?? null,
+ driver: "sandbox",
+ status: "active",
+ config: desiredConfig,
+ envVars: {},
+ metadata: desiredMetadata,
+ createdAt: now,
+ updatedAt: now,
+ })
+ .onConflictDoNothing({
+ target: [environments.driver],
+ where:
+ sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`,
+ })
+ .returning()
+ .then((rows) => rows[0] ?? null)
+ .catch((error) => {
+ if (
+ hasConstraintName(error, "environments_name_idx")
+ || hasConstraintName(error, "environments_managed_sandbox_idx")
+ ) {
+ return null;
+ }
+ throw error;
+ });
+ if (inserted) return toEnvironment(inserted);
+
+ // Either a concurrent caller won the managed slot, or an unmanaged row
+ // holds the desired name. Adopt whichever exists.
+ const winner = await findManagedSandboxRow();
+ if (winner) return adopt(winner);
+ const sameName = await db
+ .select()
+ .from(environments)
+ .where(eq(environments.name, input.name))
+ .then((rows) => rows[0] ?? null);
+ if (sameName) {
+ if (sameName.driver !== "sandbox") {
+ throw new Error(
+ `Failed to ensure managed sandbox environment: environment "${input.name}" already exists with driver "${sameName.driver}"`,
+ );
+ }
+ return adopt(sameName);
+ }
+ throw new Error("Failed to ensure managed sandbox environment");
+ };
+
+ /**
+ * Archive the Paperclip-managed sandbox row when its provider became
+ * unavailable (plugin missing, not ready, or its worker not running), so
+ * run scheduling stops selecting an environment whose lease acquisition
+ * cannot succeed (`resolveEnvironment` rejects non-active rows).
+ *
+ * Scoped to the row provisioned for the SAME provider: a row that a
+ * provider switch left on a different provider is not touched (the ensure
+ * path adopts it once the new provider is healthy). Reactivation is
+ * automatic — the next successful `ensureManagedSandboxEnvironment` stamps
+ * the row `active` again.
+ *
+ * Returns the archived environment, or null when there is no active
+ * managed row for this provider.
+ */
+ const archiveManagedSandboxEnvironment = async (
+ input: { provider: string },
+ ): Promise<Environment | null> => {
+ const existing = await findManagedSandboxRow();
+ if (!existing || existing.status !== "active") return null;
+ const rowProvider = (existing.metadata as Record<string, unknown> | null)
+ ?.managedSandboxProvider;
+ if (rowProvider !== input.provider) return null;
+ const archived = await db
+ .update(environments)
+ .set({ status: "archived", updatedAt: new Date() })
+ // Guarded on status so a concurrent re-activation is not clobbered.
+ .where(and(eq(environments.id, existing.id), eq(environments.status, "active")))
+ .returning()
+ .then((rows) => rows[0] ?? null);
+ return archived ? toEnvironment(archived) : null;
+ };
+
return {
list: async (
companyIdOrFilters?: string | EnvironmentListFilters,
@@ -256,15 +447,16 @@ export function environmentService(db: Db) {
return toEnvironment(existing);
},
+ ensureManagedSandboxEnvironment,
+
+ archiveManagedSandboxEnvironment,
+
/**
- * Idempotently ensure a managed Kubernetes sandbox environment exists for a
- * instance, configured from instance/operator-supplied config. Mirrors
- * `ensureLocalEnvironment`, but there is no DB unique index for sandbox
- * drivers, so idempotency is by metadata marker + driver lookup.
- *
- * The environment is `driver: "sandbox"` with `config.provider:
- * "kubernetes"` so it resolves to the first-party Kubernetes sandbox
- * provider. On subsequent calls the config is refreshed (so operators can
+ * Idempotently ensure a managed Kubernetes sandbox environment exists for
+ * an instance, configured from instance/operator-supplied config. A thin
+ * wrapper over `ensureManagedSandboxEnvironment` that pins the provider to
+ * "kubernetes" and stamps the legacy marker `findKubernetesEnvironment`
+ * keys on. On subsequent calls the config is refreshed (so operators can
* update egress/runtimeClass via gitops without recreating the row).
*/
ensureKubernetesEnvironment: async (
@@ -272,94 +464,13 @@ export function environmentService(db: Db) {
maybeConfig?: KubernetesEnvironmentConfigInput,
): Promise<Environment> => {
const config = resolveKubernetesConfig(companyIdOrConfig, maybeConfig);
- const desiredConfig: Record<string, unknown> = {
- ...config,
+ return ensureManagedSandboxEnvironment({
+ name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME,
+ description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION,
provider: KUBERNETES_PROVIDER_KEY,
- };
- const desiredMetadata: Record<string, unknown> = {
- managedByPaperclip: true,
- [KUBERNETES_MANAGED_MARKER]: true,
- };
-
- const existing = await db
- .select()
- .from(environments)
- .where(eq(environments.driver, "sandbox"))
- .then((rows) =>
- rows.find(
- (row) =>
- (row.metadata as Record<string, unknown> | null)?.[KUBERNETES_MANAGED_MARKER] === true,
- ) ?? null,
- );
-
- const now = new Date();
- if (existing) {
- const updated = await db
- .update(environments)
- .set({
- config: desiredConfig,
- metadata: { ...(existing.metadata ?? {}), ...desiredMetadata },
- status: "active",
- updatedAt: now,
- })
- .where(eq(environments.id, existing.id))
- .returning()
- .then((rows) => rows[0] ?? existing);
- return toEnvironment(updated);
- }
-
- // The partial unique index `environments_managed_sandbox_idx` enforces
- // "at most one Paperclip-managed sandbox row per instance" at the DB
- // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent
- // callers can race the INSERT; losers re-read the surviving row.
- const inserted = await db
- .insert(environments)
- .values({
- name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME,
- description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION,
- driver: "sandbox",
- status: "active",
- config: desiredConfig,
- envVars: {},
- metadata: desiredMetadata,
- createdAt: now,
- updatedAt: now,
- })
- .onConflictDoNothing({
- target: [environments.driver],
- where:
- sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`,
- })
- .returning()
- .then((rows) => rows[0] ?? null)
- .catch((error) => {
- if (
- hasConstraintName(error, "environments_name_idx")
- || hasConstraintName(error, "environments_managed_sandbox_idx")
- ) {
- return null;
- }
- throw error;
- });
- if (inserted) return toEnvironment(inserted);
-
- const winner = await db
- .select()
- .from(environments)
- .where(eq(environments.driver, "sandbox"))
- .then(
- (rows) =>
- rows.find(
- (candidate) =>
- (candidate.metadata as Record<string, unknown> | null)?.[
- KUBERNETES_MANAGED_MARKER
- ] === true,
- ) ?? null,
- );
- if (!winner) {
- throw new Error("Failed to ensure kubernetes environment");
- }
- return toEnvironment(winner);
+ config,
+ extraMetadata: { [KUBERNETES_MANAGED_MARKER]: true },
+ });
},
/**
diff --git a/server/src/services/index.ts b/server/src/services/index.ts
index 5f05bcf00d..08e0386c6d 100644
--- a/server/src/services/index.ts
+++ b/server/src/services/index.ts
@@ -116,9 +116,11 @@ export {
managedFeatureKeySet,
parseManagedConfigEnv,
MANAGED_CONFIG_ENV_KEY,
+ type ManagedEnvironmentSpec,
type ManagedInstanceConfig,
} from "./managed-config.js";
export { bootstrapExecutionPolicyFromEnv } from "./execution-policy-bootstrap.js";
+export { applyManagedEnvironments } from "./managed-environments.js";
export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js";
export { companyPortabilityService } from "./company-portability.js";
export { teamsCatalogService } from "./teams-catalog.js";
diff --git a/server/src/services/managed-config.ts b/server/src/services/managed-config.ts
index 87c37b5332..b723930201 100644
--- a/server/src/services/managed-config.ts
+++ b/server/src/services/managed-config.ts
@@ -9,7 +9,10 @@
* "mode": "cloud",
* "catalogVersion": "2026.720.0",
* "features": { "<feature-key>": true | false, ... },
- * "plugins": { "autoInstall": ["daytona", "kubernetes"] }
+ * "plugins": { "autoInstall": ["daytona", "kubernetes"] },
+ * "environments": [
+ * { "name": "Daytona", "provider": "daytona", "config": { "target": "us" } }
+ * ]
* }
*
* Parsing follows the `execution-policy-bootstrap.ts` doctrine: a pure
@@ -47,6 +50,52 @@ export interface ManagedInstanceConfig {
catalogVersion: string;
features: Readonly<Partial<Record<ManagedExperimentalFeatureKey, boolean>>>;
plugins: { readonly autoInstall: readonly string[] };
+ /** Sandbox environments the control plane provisions at boot (empty when the section is absent). */
+ environments: readonly ManagedEnvironmentSpec[];
+}
+
+/**
+ * One declared managed sandbox environment. Provider-agnostic: `provider` is
+ * the sandbox plugin's driver key (== its `plugins.autoInstall` catalog key),
+ * and `config` is stored verbatim in the environment row for the plugin to
+ * validate at lease time — the same split `ensureKubernetesEnvironment` has
+ * always used, generalized to any bundled sandbox provider.
+ */
+export interface ManagedEnvironmentSpec {
+ /** Display name of the instance-level environment row (unique per instance). */
+ name: string;
+ description?: string;
+ /** Sandbox provider key (the plugin's driverKey, e.g. "daytona"). */
+ provider: string;
+ /** Provider config stored in `environment.config`; never carries secrets. */
+ config: Readonly<Record<string, unknown>>;
+}
+
+/**
+ * Managed-config documents deliver NO secrets, ever. Provider credentials
+ * reach a managed instance as process environment variables (every bundled
+ * sandbox provider falls back to its env var when `config` omits the key,
+ * e.g. `DAYTONA_API_KEY`), so any secret-looking config key in the document
+ * is a misrouted credential and fails startup.
+ */
+const SECRET_LIKE_CONFIG_KEY_PATTERN = /(api[-_]?key|token|secret|password|credential)/i;
+
+function findSecretLikeConfigKey(value: Record<string, unknown>, path: string): string | null {
+ for (const [key, child] of Object.entries(value)) {
+ const childPath = path.length > 0 ? `${path}.${key}` : key;
+ if (SECRET_LIKE_CONFIG_KEY_PATTERN.test(key)) return childPath;
+ if (isPlainObject(child)) {
+ const nested = findSecretLikeConfigKey(child, childPath);
+ if (nested) return nested;
+ } else if (Array.isArray(child)) {
+ for (const [index, element] of child.entries()) {
+ if (!isPlainObject(element)) continue;
+ const nested = findSecretLikeConfigKey(element, `${childPath}[${index}]`);
+ if (nested) return nested;
+ }
+ }
+ }
+ return null;
}
let cachedFeatureKeys: ReadonlySet<string> | null = null;
@@ -106,10 +155,10 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon
fail(`must be a JSON object (got ${describeJsonValue(doc)})`);
}
- const allowedTopLevelKeys = new Set(["v", "mode", "catalogVersion", "features", "plugins"]);
+ const allowedTopLevelKeys = new Set(["v", "mode", "catalogVersion", "features", "plugins", "environments"]);
for (const key of Object.keys(doc)) {
if (!allowedTopLevelKeys.has(key)) {
- fail(`has unknown top-level key "${key}" (allowed: v, mode, catalogVersion, features, plugins)`);
+ fail(`has unknown top-level key "${key}" (allowed: v, mode, catalogVersion, features, plugins, environments)`);
}
}
@@ -195,12 +244,93 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceCon
autoInstall.push(entry);
}
+ // `environments` is OPTIONAL, unlike `features` and `plugins`: documents
+ // delivered before the section existed must keep booting newer builds (a
+ // fleet image roll cannot be lockstepped with a config re-delivery), and
+ // absence is not a dropped security control — it simply declares no managed
+ // environments. When present, the section is validated fail-closed like
+ // everything else.
+ const environmentSpecs: ManagedEnvironmentSpec[] = [];
+ if (doc.environments !== undefined) {
+ if (!Array.isArray(doc.environments)) {
+ fail(`"environments" must be an array of environment objects (got ${describeJsonValue(doc.environments)})`);
+ }
+ // The DB enforces at most ONE Paperclip-managed sandbox row per instance
+ // (partial unique index `environments_managed_sandbox_idx`); every entry
+ // here provisions that row, so a longer list can never be satisfied.
+ if (doc.environments.length > 1) {
+ fail(
+ `"environments" supports at most one entry: each entry provisions the single Paperclip-managed sandbox environment (DB invariant environments_managed_sandbox_idx)`,
+ );
+ }
+ for (const [index, entry] of doc.environments.entries()) {
+ if (!isPlainObject(entry)) {
+ fail(`"environments[${index}]" must be an object (got ${describeJsonValue(entry)})`);
+ }
+ const allowedEntryKeys = new Set(["name", "description", "provider", "config"]);
+ for (const key of Object.keys(entry)) {
+ if (!allowedEntryKeys.has(key)) {
+ fail(`"environments[${index}]" has unknown key "${key}" (allowed: name, description, provider, config)`);
+ }
+ }
+ if (typeof entry.name !== "string" || entry.name.length === 0 || entry.name.trim() !== entry.name) {
+ fail(
+ `"environments[${index}].name" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.name)})`,
+ );
+ }
+ if (
+ entry.description !== undefined
+ && (typeof entry.description !== "string" || entry.description.trim().length === 0)
+ ) {
+ fail(`"environments[${index}].description" must be a non-empty string when present (got ${describeJsonValue(entry.description)})`);
+ }
+ if (typeof entry.provider !== "string" || entry.provider.length === 0 || entry.provider.trim() !== entry.provider) {
+ fail(
+ `"environments[${index}].provider" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.provider)})`,
+ );
+ }
+ // Coherence: on a managed instance the control plane is the only plugin
+ // install path, so an environment whose provider plugin is not
+ // auto-installed could never serve a lease. Catch the skew at parse time.
+ if (!autoInstall.includes(entry.provider)) {
+ fail(
+ `"environments[${index}].provider" is "${entry.provider}", which is not in "plugins.autoInstall"; a managed environment requires its provider plugin to be provisioned`,
+ );
+ }
+ const config: Record<string, unknown> = {};
+ if (entry.config !== undefined) {
+ if (!isPlainObject(entry.config)) {
+ fail(`"environments[${index}].config" must be an object (got ${describeJsonValue(entry.config)})`);
+ }
+ if (entry.config.provider !== undefined) {
+ fail(`"environments[${index}].config" must not set "provider"; it is forced from the entry's provider key`);
+ }
+ const secretLikeKey = findSecretLikeConfigKey(entry.config, "");
+ if (secretLikeKey) {
+ fail(
+ `"environments[${index}].config" key "${secretLikeKey}" looks secret-bearing; credentials are delivered to managed instances as process environment variables (the provider's documented env fallback), never in the managed-config document`,
+ );
+ }
+ Object.assign(config, entry.config);
+ }
+ environmentSpecs.push(
+ Object.freeze({
+ name: entry.name,
+ ...(entry.description !== undefined ? { description: entry.description } : {}),
+ provider: entry.provider,
+ config: Object.freeze(config),
+ }) as ManagedEnvironmentSpec,
+ );
+ }
+ }
+
return Object.freeze({
v: SUPPORTED_MANAGED_CONFIG_VERSION,
mode: "cloud",
catalogVersion: doc.catalogVersion,
features: Object.freeze(features),
plugins: Object.freeze({ autoInstall: Object.freeze(autoInstall) }),
+ environments: Object.freeze(environmentSpecs),
}) as ManagedInstanceConfig;
}
diff --git a/server/src/services/managed-environments.test.ts b/server/src/services/managed-environments.test.ts
new file mode 100644
index 0000000000..4857c26a8b
--- /dev/null
+++ b/server/src/services/managed-environments.test.ts
@@ -0,0 +1,415 @@
+import { describe, expect, it, vi } from "vitest";
+import type { Db } from "@paperclipai/db";
+import { MANAGED_CONFIG_ENV_KEY, parseManagedConfigEnv } from "./managed-config.js";
+import {
+ applyManagedEnvironments,
+ type ApplyManagedEnvironmentsOptions,
+} from "./managed-environments.js";
+
+const noDb = null as unknown as Db;
+
+function parsedConfig(overrides: Record<string, unknown> = {}) {
+ const config = parseManagedConfigEnv({
+ [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({
+ v: 1,
+ mode: "cloud",
+ catalogVersion: "2026.720.0",
+ features: {},
+ plugins: { autoInstall: ["daytona"] },
+ ...overrides,
+ }),
+ });
+ if (!config) throw new Error("expected a parsed managed config");
+ return config;
+}
+
+function environmentRow(overrides: Record<string, unknown> = {}) {
+ const now = new Date();
+ return {
+ id: "env-1",
+ name: "Daytona",
+ description: null,
+ driver: "sandbox",
+ status: "active",
+ config: {},
+ envVars: {},
+ metadata: {},
+ createdAt: now,
+ updatedAt: now,
+ ...overrides,
+ };
+}
+
+function readyDriverResolver(status = "ready") {
+ return vi.fn(async () => ({
+ plugin: { id: "plugin-1", pluginKey: "sandbox-providers/daytona", status },
+ }));
+}
+
+function runningWorkerManager(running = true) {
+ return { isRunning: vi.fn(() => running), getWorker: vi.fn(() => undefined) };
+}
+
+type WorkerManagerSeam = NonNullable<ApplyManagedEnvironmentsOptions["workerManager"]>;
+type RecoveryHandle = NonNullable<ReturnType<WorkerManagerSeam["getWorker"]>>;
+
+/** A worker handle fake that records `ready` listeners so tests can fire them. */
+function recoveryHandle() {
+ const readyListeners: Array<(payload: { pluginId: string }) => void> = [];
+ const handle: RecoveryHandle = {
+ on: vi.fn((_event, listener) => {
+ readyListeners.push(listener);
+ }),
+ off: vi.fn((_event, listener) => {
+ const at = readyListeners.indexOf(listener);
+ if (at !== -1) readyListeners.splice(at, 1);
+ }),
+ };
+ return {
+ handle,
+ fireReady: () => {
+ for (const listener of [...readyListeners]) listener({ pluginId: "plugin-1" });
+ },
+ };
+}
+
+/** Lets the fire-and-forget recovery ensure settle. */
+function tick() {
+ return new Promise((resolve) => setImmediate(resolve));
+}
+
+type EnvironmentsSeam = NonNullable<ApplyManagedEnvironmentsOptions["environments"]>;
+
+function environmentsSeam(overrides: Partial<EnvironmentsSeam> = {}): EnvironmentsSeam {
+ return {
+ ensureManagedSandboxEnvironment:
+ overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(environmentRow()),
+ archiveManagedSandboxEnvironment:
+ overrides.archiveManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null),
+ };
+}
+
+describe("applyManagedEnvironments", () => {
+ it("no-ops for self-hosted instances and for documents without environments", async () => {
+ expect(await applyManagedEnvironments(noDb, null)).toBeNull();
+ expect(await applyManagedEnvironments(noDb, parsedConfig())).toBeNull();
+ });
+
+ it("refuses startup when a forced execution mode also claims the managed sandbox slot", async () => {
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+ await expect(
+ applyManagedEnvironments(noDb, config, {
+ env: { PAPERCLIP_EXECUTION_MODE: "kubernetes" },
+ }),
+ ).rejects.toThrow(/mutually exclusive/);
+ });
+
+ it("ensures each declared environment through the provider-agnostic service call", async () => {
+ const ensureManagedSandboxEnvironment = vi
+ .fn()
+ .mockResolvedValue(environmentRow());
+ const config = parsedConfig({
+ environments: [
+ {
+ name: "Daytona",
+ description: "Managed Daytona sandbox.",
+ provider: "daytona",
+ config: { target: "us" },
+ },
+ ],
+ });
+
+ const resolveSandboxProviderDriver = readyDriverResolver();
+ const workerManager = runningWorkerManager();
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager,
+ environments: environmentsSeam({ ensureManagedSandboxEnvironment }),
+ resolveSandboxProviderDriver,
+ });
+
+ expect(result).toEqual({ ensured: 1, failed: 0 });
+ expect(resolveSandboxProviderDriver).toHaveBeenCalledWith({ db: noDb, driverKey: "daytona" });
+ expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1");
+ expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);
+ expect(ensureManagedSandboxEnvironment).toHaveBeenCalledWith({
+ name: "Daytona",
+ description: "Managed Daytona sandbox.",
+ provider: "daytona",
+ config: { target: "us" },
+ });
+ // The frozen parsed config must not leak into the service (the row's
+ // config is mutated downstream when the provider key is forced in).
+ const passedConfig = ensureManagedSandboxEnvironment.mock.calls[0]?.[0]?.config;
+ expect(Object.isFrozen(passedConfig)).toBe(false);
+ });
+
+ it("waits for the bundled-plugin startup pass before ensuring anything", async () => {
+ const ensureManagedSandboxEnvironment = vi
+ .fn()
+ .mockResolvedValue(environmentRow());
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ let releasePlugins!: () => void;
+ const pluginsReady = new Promise<void>((resolve) => {
+ releasePlugins = resolve;
+ });
+
+ const pending = applyManagedEnvironments(noDb, config, {
+ env: {},
+ pluginsReady,
+ workerManager: runningWorkerManager(),
+ environments: environmentsSeam({ ensureManagedSandboxEnvironment }),
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+
+ // Give the ensure every chance to (incorrectly) run early.
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(ensureManagedSandboxEnvironment).not.toHaveBeenCalled();
+
+ releasePlugins();
+ expect(await pending).toEqual({ ensured: 1, failed: 0 });
+ expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);
+ });
+
+ it("skips an entry whose provider plugin is missing and archives its stale row", async () => {
+ const environments = environmentsSeam({
+ archiveManagedSandboxEnvironment: vi.fn().mockResolvedValue(environmentRow({ status: "archived" })),
+ });
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager: runningWorkerManager(),
+ environments,
+ resolveSandboxProviderDriver: vi.fn(async () => null),
+ });
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();
+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({
+ provider: "daytona",
+ });
+ });
+
+ it("skips (and counts failed) an entry whose provider plugin is not ready", async () => {
+ const environments = environmentsSeam();
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager: runningWorkerManager(),
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver("disabled"),
+ });
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();
+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({
+ provider: "daytona",
+ });
+ });
+
+ it("skips an entry whose plugin record is ready but whose worker is not running", async () => {
+ const environments = environmentsSeam();
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const workerManager = runningWorkerManager(false);
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager,
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1");
+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();
+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({
+ provider: "daytona",
+ });
+ // No registered handle → no respawn is coming; degraded until next boot.
+ expect(workerManager.getWorker).toHaveBeenCalledWith("plugin-1");
+ });
+
+ it("reactivates the archived environment once when the provider worker recovers", async () => {
+ const environments = environmentsSeam();
+ const config = parsedConfig({
+ environments: [
+ { name: "Daytona", provider: "daytona", config: { target: "us" } },
+ ],
+ });
+
+ const { handle, fireReady } = recoveryHandle();
+ const workerManager = {
+ isRunning: vi.fn(() => false),
+ getWorker: vi.fn(() => handle),
+ };
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager,
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+
+ // The boot pass itself stays degraded: archived, counted failed.
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({
+ provider: "daytona",
+ });
+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();
+
+ // Worker manager respawns the worker → the handle re-emits `ready`.
+ fireReady();
+ fireReady();
+ await tick();
+
+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);
+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledWith({
+ name: "Daytona",
+ description: undefined,
+ provider: "daytona",
+ config: { target: "us" },
+ });
+ expect(handle.off).toHaveBeenCalledTimes(1);
+ });
+
+ it("re-checks isRunning after subscribing so a recovery during the archive is not missed", async () => {
+ const environments = environmentsSeam();
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const { handle } = recoveryHandle();
+ // Down at the gate check, already back up by the time the listener is
+ // registered — the `ready` event has been missed.
+ const workerManager = {
+ isRunning: vi
+ .fn()
+ .mockReturnValueOnce(false)
+ .mockReturnValue(true),
+ getWorker: vi.fn(() => handle),
+ };
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager,
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+ await tick();
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not subscribe for recovery when the plugin record is missing or not ready", async () => {
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ for (const resolveSandboxProviderDriver of [
+ vi.fn(async () => null),
+ readyDriverResolver("disabled"),
+ ]) {
+ const workerManager = runningWorkerManager(false);
+ await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager,
+ environments: environmentsSeam(),
+ resolveSandboxProviderDriver,
+ });
+ expect(workerManager.getWorker).not.toHaveBeenCalled();
+ }
+ });
+
+ it("logs, not throws, when the recovery re-ensure fails", async () => {
+ const environments = environmentsSeam({
+ ensureManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")),
+ });
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const { handle, fireReady } = recoveryHandle();
+ const workerManager = {
+ isRunning: vi.fn(() => false),
+ getWorker: vi.fn(() => handle),
+ };
+ await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager,
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+
+ fireReady();
+ await tick();
+ // The rejection is swallowed into a log line; reaching this point without
+ // an unhandled rejection is the assertion.
+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);
+ });
+
+ it("fails closed when no worker manager is provided", async () => {
+ const environments = environmentsSeam();
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();
+ });
+
+ it("counts a skipped entry once even when archiving its stale row fails", async () => {
+ const environments = environmentsSeam({
+ archiveManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")),
+ });
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager: runningWorkerManager(),
+ environments,
+ resolveSandboxProviderDriver: vi.fn(async () => null),
+ });
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledTimes(1);
+ });
+
+ it("is fail-safe per entry: an ensure failure is counted, not thrown", async () => {
+ const environments = environmentsSeam({
+ ensureManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")),
+ });
+ const config = parsedConfig({
+ environments: [{ name: "Daytona", provider: "daytona" }],
+ });
+
+ const result = await applyManagedEnvironments(noDb, config, {
+ env: {},
+ workerManager: runningWorkerManager(),
+ environments,
+ resolveSandboxProviderDriver: readyDriverResolver(),
+ });
+
+ expect(result).toEqual({ ensured: 0, failed: 1 });
+ expect(environments.archiveManagedSandboxEnvironment).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/services/managed-environments.ts b/server/src/services/managed-environments.ts
new file mode 100644
index 0000000000..fbdf2c6985
--- /dev/null
+++ b/server/src/services/managed-environments.ts
@@ -0,0 +1,239 @@
+/**
+ * Managed-environment bootstrap (harness → app contract).
+ *
+ * Managed-cloud instances may declare sandbox environments in the
+ * `environments` section of `PAPERCLIP_MANAGED_CONFIG` (parsed fail-closed in
+ * `managed-config.ts`). On boot, each declared environment is idempotently
+ * ensured as the instance-level Paperclip-managed sandbox row via the
+ * provider-agnostic `ensureManagedSandboxEnvironment` — the control plane
+ * provisions, tenants use, for any bundled sandbox provider plugin.
+ *
+ * The failure posture mirrors bundled plugin provisioning
+ * (`bundled-plugins.ts`), deliberately split:
+ *
+ * 1. **Validation fails closed at parse time** (`managed-config.ts`): a
+ * malformed section refuses startup with a precise error.
+ * 2. **The DB ensure step is fail-safe per entry**: an ensure failure is
+ * logged and boot continues degraded (environment unavailable) rather
+ * than crash-looping a fleet.
+ *
+ * Ensuring is additionally synchronized with provider availability: the
+ * caller's `pluginsReady` promise (the bundled-plugin install/load pass) is
+ * awaited first, and an entry whose provider plugin is not installed, `ready`,
+ * AND running a live worker (`workerManager.isRunning`; a `ready` record whose
+ * activation failed has no worker and cannot serve leases) afterwards is
+ * skipped (counted failed) instead of being written as an active row —
+ * otherwise the heartbeat would resume queued runs against an environment
+ * whose lease acquisition cannot succeed yet. When such an entry was
+ * provisioned by an earlier boot, its still-active row is archived
+ * (`archiveManagedSandboxEnvironment`) for the same reason. Re-activation
+ * happens on the next healthy boot's ensure, or earlier: when the plugin
+ * record is `ready` and only the worker is down (a crash in restart-backoff),
+ * a one-shot `ready` listener on the worker handle re-runs the ensure as soon
+ * as the worker manager respawns the worker, so a transient crash does not
+ * leave the environment archived until someone restarts the server.
+ *
+ * Removing an entry from the document stops future refreshes but never
+ * deletes or archives the row — there is intentionally no unprovision path
+ * here, matching `plugins.autoInstall` semantics (leases may still reference
+ * the row; withdrawal is an explicit operator action). Archiving above is
+ * scoped to a declared-but-unavailable provider, not to document removal.
+ *
+ * Provider credentials are never part of the declared config: every bundled
+ * sandbox provider falls back to its documented process environment variable
+ * (e.g. `DAYTONA_API_KEY`) when `config` omits the key, so the deployment
+ * delivers secrets as env vars and the managed document stays secret-free.
+ */
+
+import type { Db } from "@paperclipai/db";
+import { logger } from "../middleware/logger.js";
+import { environmentService } from "./environments.js";
+import type { ManagedInstanceConfig } from "./managed-config.js";
+import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js";
+import { resolvePluginSandboxProviderDriverByKey } from "./plugin-environment-driver.js";
+import type { PluginWorkerManager } from "./plugin-worker-manager.js";
+
+export interface ApplyManagedEnvironmentsOptions {
+ env?: Record<string, string | undefined>;
+ /**
+ * Resolves when the bundled-plugin startup pass (install + load; the
+ * `ensureBundledPlugins` chain in `createApp`) has finished. Awaited before
+ * any environment is ensured so an active row never precedes its provider
+ * driver. The promise never rejects (the chain catches internally).
+ */
+ pluginsReady?: Promise<unknown>;
+ /**
+ * The app's plugin worker manager, consulted per entry so an environment is
+ * only ensured while its provider plugin has a live worker (a `ready`
+ * registry record whose activation failed cannot serve leases). Fails
+ * closed: without a worker manager every entry is treated as unavailable.
+ *
+ * `getWorker` is `PluginWorkerManager.getWorker` narrowed to the `ready`
+ * subscription used for post-archive recovery: a worker in crash-restart
+ * backoff re-emits `ready` when the manager respawns it, and the listener
+ * re-ensures the archived environment (see `applyManagedEnvironments`).
+ */
+ workerManager?: Pick<PluginWorkerManager, "isRunning"> & {
+ getWorker(pluginId: string):
+ | {
+ on(event: "ready", listener: (payload: { pluginId: string }) => void): void;
+ off(event: "ready", listener: (payload: { pluginId: string }) => void): void;
+ }
+ | undefined;
+ };
+ /** Test seam: overrides the environment service built from `db`. */
+ environments?: Pick<
+ ReturnType<typeof environmentService>,
+ "ensureManagedSandboxEnvironment" | "archiveManagedSandboxEnvironment"
+ >;
+ /** Test seam: overrides the sandbox-provider plugin driver lookup. */
+ resolveSandboxProviderDriver?: (input: {
+ db: Db;
+ driverKey: string;
+ }) => Promise<{ plugin: { id: string; pluginKey: string; status: string } } | null>;
+}
+
+/**
+ * Ensure every environment declared in the managed-config document. Returns
+ * null when there is nothing to do (self-hosted, or no `environments`
+ * section); otherwise the ensured/failed counts. Idempotent; safe to call on
+ * every boot.
+ */
+export async function applyManagedEnvironments(
+ db: Db,
+ managedConfig: ManagedInstanceConfig | null,
+ opts: ApplyManagedEnvironmentsOptions = {},
+): Promise<{ ensured: number; failed: number } | null> {
+ if (!managedConfig || managedConfig.environments.length === 0) return null;
+
+ // The forced-execution-mode bootstrap (`PAPERCLIP_EXECUTION_MODE=kubernetes`)
+ // and this one both own the single Paperclip-managed sandbox row
+ // (`environments_managed_sandbox_idx`). Configuring both is contradictory;
+ // refuse startup rather than let bootstrap ordering pick a winner.
+ const env = opts.env ?? process.env;
+ if (parseExecutionPolicyBootstrapEnv(env)) {
+ throw new Error(
+ `PAPERCLIP_EXECUTION_MODE and the PAPERCLIP_MANAGED_CONFIG "environments" section are mutually exclusive: both manage the single instance sandbox environment`,
+ );
+ }
+
+ // The heartbeat resumes queued runs right after this bootstrap step, and
+ // lease acquisition fails hard on a provider whose plugin is missing or not
+ // ready. Wait for the bundled-plugin startup pass to finish, then refuse to
+ // ensure (and in particular to re-activate) a row whose provider driver did
+ // not come up — a degraded boot without the row beats an active environment
+ // that fails every lease until the plugin recovers.
+ await opts.pluginsReady;
+
+ const resolveDriver = opts.resolveSandboxProviderDriver ?? resolvePluginSandboxProviderDriverByKey;
+ const environments = opts.environments ?? environmentService(db);
+
+ // Recovery path for a `ready` plugin whose worker was down at check time:
+ // that shape is usually a crash in restart-backoff, and the manager's
+ // respawn re-emits `ready` on the same handle. A one-shot listener re-runs
+ // the idempotent ensure so the recovered provider becomes selectable again
+ // without waiting for the next boot (this pass has already archived the
+ // row). The post-subscribe `isRunning` re-check closes the race where the
+ // worker recovered between the gate check and the subscription. No handle
+ // means no respawn is coming (activation never started a worker), so the
+ // degraded-until-next-boot posture stands.
+ const scheduleRecoveryReactivation = (
+ spec: ManagedInstanceConfig["environments"][number],
+ pluginId: string,
+ ): void => {
+ const manager = opts.workerManager;
+ const handle = manager?.getWorker(pluginId);
+ if (!handle) return;
+ let reactivated = false;
+ const reactivate = (): void => {
+ if (reactivated) return;
+ reactivated = true;
+ handle.off("ready", reactivate);
+ void environments
+ .ensureManagedSandboxEnvironment({
+ name: spec.name,
+ description: spec.description,
+ provider: spec.provider,
+ config: { ...spec.config },
+ })
+ .then((environment) => {
+ logger.info(
+ { environmentId: environment.id, name: spec.name, provider: spec.provider },
+ "managed sandbox environment reactivated after provider worker recovery",
+ );
+ })
+ .catch((err: unknown) => {
+ logger.error(
+ { err, name: spec.name, provider: spec.provider },
+ "failed to reactivate managed sandbox environment after provider worker recovery (degraded: environment unavailable until next boot)",
+ );
+ });
+ };
+ handle.on("ready", reactivate);
+ if (manager?.isRunning(pluginId)) reactivate();
+ };
+
+ let ensured = 0;
+ let failed = 0;
+ for (const spec of managedConfig.environments) {
+ try {
+ const resolved = await resolveDriver({ db, driverKey: spec.provider });
+ // `ready` in the registry is necessary but not sufficient: activation
+ // can fail after install leaves the record `ready`, in which case no
+ // worker is running and the driver cannot serve leases.
+ const workerRunning =
+ resolved != null && (opts.workerManager?.isRunning(resolved.plugin.id) ?? false);
+ if (!resolved || resolved.plugin.status !== "ready" || !workerRunning) {
+ failed += 1;
+ // A row provisioned by an earlier boot must not stay active either —
+ // archive it so run scheduling stops selecting it. Best-effort: an
+ // archive failure is logged, and the entry is already counted failed.
+ const archived = await environments
+ .archiveManagedSandboxEnvironment({ provider: spec.provider })
+ .catch((archiveErr: unknown) => {
+ logger.error(
+ { err: archiveErr, name: spec.name, provider: spec.provider },
+ "failed to archive the managed sandbox environment of an unavailable provider",
+ );
+ return null;
+ });
+ logger.error(
+ {
+ name: spec.name,
+ provider: spec.provider,
+ pluginKey: resolved?.plugin.pluginKey ?? null,
+ pluginStatus: resolved?.plugin.status ?? null,
+ workerRunning,
+ archivedEnvironmentId: archived?.id ?? null,
+ },
+ "managed sandbox environment provider plugin is not installed, ready, and running; skipping ensure and archiving any previously provisioned row (degraded: environment unavailable)",
+ );
+ // Only a `ready` record can recover without another boot: the worker
+ // manager restarts crashed workers, but nothing (re)installs a missing
+ // or non-ready plugin at runtime.
+ if (resolved && resolved.plugin.status === "ready") {
+ scheduleRecoveryReactivation(spec, resolved.plugin.id);
+ }
+ continue;
+ }
+ const environment = await environments.ensureManagedSandboxEnvironment({
+ name: spec.name,
+ description: spec.description,
+ provider: spec.provider,
+ config: { ...spec.config },
+ });
+ ensured += 1;
+ logger.info(
+ { environmentId: environment.id, name: environment.name, provider: spec.provider },
+ "managed sandbox environment ensured",
+ );
+ } catch (err) {
+ failed += 1;
+ logger.error(
+ { err, name: spec.name, provider: spec.provider },
+ "failed to ensure managed sandbox environment; continuing boot (degraded: environment unavailable)",
+ );
+ }
+ }
+ return { ensured, failed };
+}