Skip to content

The open-source app everyone uses to manage agents at work

fix(server): stamp built commit into service.version (#11748)

Nicky Leach5a1ce7aed823
diff --git a/Dockerfile b/Dockerfile
index a449c77942..109305a323 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,6 +54,14 @@ COPY --from=deps /app /app
COPY . .
RUN pnpm --filter @paperclipai/ui build
RUN pnpm --filter @paperclipai/plugin-sdk build
+# The server build runs scripts/write-build-stamp.mjs, which stamps the built
+# commit into dist/build-info.json. The build context has no .git, so the
+# script reads PAPERCLIP_BUILD_COMMIT instead. Docker exposes an ARG to the
+# next RUN as an environment variable, so declare it here — in the build
+# stage — before the server build. The production stage below declares the
+# same ARG again for the runtime fallback; an ARG goes out of scope at the
+# end of its stage. Empty for local `docker build`, which then writes no stamp.
+ARG PAPERCLIP_BUILD_COMMIT=""
RUN pnpm --filter @paperclipai/server build
RUN test -f server/dist/index.js || (echo "ERROR: server build output missing" && exit 1)
diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts
index 1de56bb9be..3e5d3444d4 100644
--- a/cli/src/__tests__/worktree.test.ts
+++ b/cli/src/__tests__/worktree.test.ts
@@ -623,7 +623,7 @@ describe("worktree helpers", () => {
fs.rmSync(tempRoot, { recursive: true, force: true });
await tempDb.cleanup();
}
- });
+ }, 30000);
it("ensure-seeded seeds once and fast-exits on the verified manifest", async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-"));
diff --git a/doc/observability.md b/doc/observability.md
index 0af08e29c2..15101f47d9 100644
--- a/doc/observability.md
+++ b/doc/observability.md
@@ -60,6 +60,31 @@ export OTEL_SERVICE_NAME="paperclip"
export OTEL_SERVICE_VERSION="2026.5.0"
```
+### `service.version` resolution order
+
+The `service.version` span attribute reports the commit the running server was
+built from. The server resolves it in this order and uses the first source that
+returns a value:
+
+1. **The build stamp.** The server `build` script writes the commit SHA into
+ `dist/build-info.json`. The stamp wins so the reported version tracks the
+ true built commit and cannot go stale across rebuilds. The build script
+ reads the commit from `git rev-parse --short HEAD` first. A Docker image
+ build excludes `.git`, so the build script reads the `PAPERCLIP_BUILD_COMMIT`
+ environment variable instead. Pass the built commit in that variable so the
+ image stamp records the true commit.
+2. **A runtime `git rev-parse --short HEAD`.** This covers `tsx src/index.ts`
+ dev mode, where the server runs from the source checkout and writes no
+ stamp. A failure here is not fatal.
+3. **The `OTEL_SERVICE_VERSION` environment variable.** This is the fallback
+ for a build with no stamp and no reachable git — for example a tarball
+ build. `OTEL_SERVICE_VERSION` is a Paperclip-specific variable, not an
+ OpenTelemetry SDK variable, so Paperclip controls this precedence.
+4. **`"unknown"`** when no source returns a value.
+
+The server logs the resolved `service.version` once at startup, so an operator
+can confirm the value.
+
If `OTEL_EXPORTER_OTLP_PROTOCOL` is set to an unrecognized value, Paperclip
logs a single warning and falls back to gRPC.
diff --git a/server/package.json b/server/package.json
index 854e80cbc4..2811c72530 100644
--- a/server/package.json
+++ b/server/package.json
@@ -35,7 +35,7 @@
"dev": "tsx src/index.ts",
"dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx ./scripts/dev-watch.ts",
"prepare:ui-dist": "bash ../scripts/prepare-server-ui-dist.sh",
- "build": "tsc && mkdir -p dist/onboarding-assets dist/built-ins && cp -R src/onboarding-assets/. dist/onboarding-assets/ && cp -R src/built-ins/. dist/built-ins/",
+ "build": "tsc && mkdir -p dist/onboarding-assets dist/built-ins && cp -R src/onboarding-assets/. dist/onboarding-assets/ && cp -R src/built-ins/. dist/built-ins/ && node scripts/write-build-stamp.mjs",
"prepack": "pnpm run prepare:ui-dist",
"postpack": "rm -rf ui-dist",
"clean": "rm -rf dist",
diff --git a/server/scripts/write-build-stamp.mjs b/server/scripts/write-build-stamp.mjs
new file mode 100644
index 0000000000..bceb09455f
--- /dev/null
+++ b/server/scripts/write-build-stamp.mjs
@@ -0,0 +1,87 @@
+// Write the build stamp for the server.
+//
+// The server `build` script runs this after `tsc`. It writes the commit SHA
+// into `dist/build-info.json`. The instrumentation module reads that stamp to
+// report `service.version`, so the value tracks the true built commit.
+//
+// The build resolves the commit in two steps:
+// 1. `git rev-parse --short HEAD` in the server directory.
+// 2. The `PAPERCLIP_BUILD_COMMIT` environment variable.
+// A Docker image build excludes `.git`, so the git lookup fails there. The
+// image build passes the commit in `PAPERCLIP_BUILD_COMMIT` instead, so the
+// stamp still records the true built commit.
+//
+// The build must not fail when no commit is available. A missing `git`, a
+// checkout with no `.git`, and an unset `PAPERCLIP_BUILD_COMMIT` together write
+// no stamp and exit 0.
+
+import { execFileSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const serverDir = join(scriptDir, "..");
+const distDir = join(serverDir, "dist");
+const outFile = join(distDir, "build-info.json");
+
+/**
+ * Resolve the commit for the build stamp. Prefer the git commit. Fall back to
+ * the supplied commit — the value a Docker image build passes in
+ * `PAPERCLIP_BUILD_COMMIT` when `.git` is absent. Return null when neither
+ * source gives a non-empty value.
+ *
+ * @param {unknown} gitCommit The `git rev-parse` result, or null on failure.
+ * @param {unknown} suppliedCommit The `PAPERCLIP_BUILD_COMMIT` value.
+ * @returns {string | null}
+ */
+export function resolveBuildCommit(gitCommit, suppliedCommit) {
+ const git = typeof gitCommit === "string" ? gitCommit.trim() : "";
+ if (git) return git;
+ const supplied = typeof suppliedCommit === "string" ? suppliedCommit.trim() : "";
+ if (supplied) return supplied;
+ return null;
+}
+
+/**
+ * Read the short commit SHA with `git rev-parse --short HEAD` in the server
+ * directory. Return the SHA, or null on any failure.
+ *
+ * @returns {string | null}
+ */
+function readGitCommit() {
+ try {
+ const out = execFileSync("git", ["rev-parse", "--short", "HEAD"], {
+ cwd: serverDir,
+ stdio: ["ignore", "pipe", "ignore"],
+ })
+ .toString()
+ .trim();
+ return out.length > 0 ? out : null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Resolve the commit and write the build stamp. Write no stamp and return when
+ * no commit is available, so the build continues.
+ */
+function main() {
+ const commit = resolveBuildCommit(readGitCommit(), process.env.PAPERCLIP_BUILD_COMMIT);
+
+ if (!commit) {
+ console.log("[build-stamp] no commit available; wrote no build stamp");
+ return;
+ }
+
+ mkdirSync(distDir, { recursive: true });
+ writeFileSync(outFile, `${JSON.stringify({ commit }, null, 2)}\n`);
+ console.log(`[build-stamp] wrote ${outFile} commit=${commit}`);
+}
+
+// Run only when node invokes this file directly (the `build` script). A test
+// that imports `resolveBuildCommit` does not run `main`.
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ main();
+}
diff --git a/server/src/__tests__/docker-build-stamp.test.ts b/server/src/__tests__/docker-build-stamp.test.ts
new file mode 100644
index 0000000000..f013d30ec6
--- /dev/null
+++ b/server/src/__tests__/docker-build-stamp.test.ts
@@ -0,0 +1,58 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+
+/**
+ * Drift guard for the Docker build-stamp wiring.
+ *
+ * The server build runs scripts/write-build-stamp.mjs, which stamps the built
+ * commit into dist/build-info.json. The build context has no .git, so the
+ * script reads PAPERCLIP_BUILD_COMMIT instead. Docker exposes an ARG to the
+ * next RUN as an environment variable, but an ARG goes out of scope at the end
+ * of its stage. So the build stage must declare `ARG PAPERCLIP_BUILD_COMMIT`
+ * before the server build; the production ARG alone stamps nothing, because
+ * the server build already ran in the earlier stage.
+ *
+ * This guard fails if a refactor drops the build-stage ARG, moves it after the
+ * server build, or removes the build-arg the docker workflow passes.
+ */
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
+const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8");
+const workflow = readFileSync(path.join(repoRoot, ".github", "workflows", "docker.yml"), "utf8");
+
+/**
+ * Return the text of the Dockerfile stage that starts at the named target.
+ * A stage runs from its `FROM ... AS <name>` line to the next `FROM` line.
+ */
+function stageBody(source: string, stageName: string): string {
+ const froms = [...source.matchAll(/^FROM .*$/gm)];
+ const startIdx = froms.findIndex((m) => new RegExp(`\\bAS ${stageName}\\b`).test(m[0]));
+ expect(startIdx, `Dockerfile must declare a '${stageName}' stage`).toBeGreaterThanOrEqual(0);
+ const start = froms[startIdx].index ?? 0;
+ const end = froms[startIdx + 1]?.index ?? source.length;
+ return source.slice(start, end);
+}
+
+describe("docker build-stamp wiring", () => {
+ it("declares PAPERCLIP_BUILD_COMMIT in the build stage before the server build", () => {
+ const build = stageBody(dockerfile, "build");
+ const argIdx = build.search(/^ARG PAPERCLIP_BUILD_COMMIT\b/m);
+ const serverBuildIdx = build.search(/^RUN pnpm --filter @paperclipai\/server build\b/m);
+ expect(argIdx, "build stage must declare ARG PAPERCLIP_BUILD_COMMIT").toBeGreaterThanOrEqual(0);
+ expect(serverBuildIdx, "build stage must run the server build").toBeGreaterThanOrEqual(0);
+ expect(
+ argIdx,
+ "ARG PAPERCLIP_BUILD_COMMIT must precede the server build so the stamp script reads it",
+ ).toBeLessThan(serverBuildIdx);
+ });
+
+ it("passes PAPERCLIP_BUILD_COMMIT as a build-arg for both image targets", () => {
+ const argLines = [...workflow.matchAll(/^\s*PAPERCLIP_BUILD_COMMIT=.*$/gm)];
+ expect(
+ argLines.length,
+ "the docker workflow must pass PAPERCLIP_BUILD_COMMIT for the production and cloud builds",
+ ).toBeGreaterThanOrEqual(2);
+ });
+});
diff --git a/server/src/__tests__/service-version.test.ts b/server/src/__tests__/service-version.test.ts
new file mode 100644
index 0000000000..fd7a95abca
--- /dev/null
+++ b/server/src/__tests__/service-version.test.ts
@@ -0,0 +1,51 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { readGitCommit, resolveServiceVersion } from "../instrumentation.js";
+
+describe("resolveServiceVersion", () => {
+ it("prefers the build stamp over every other source", () => {
+ expect(resolveServiceVersion("aaaaaaa", "bbbbbbb", "2026.5.0")).toBe("aaaaaaa");
+ });
+
+ it("uses the runtime git commit when no build stamp exists", () => {
+ expect(resolveServiceVersion(null, "bbbbbbb", "2026.5.0")).toBe("bbbbbbb");
+ });
+
+ it("uses OTEL_SERVICE_VERSION when no stamp and no git commit exist", () => {
+ expect(resolveServiceVersion(null, null, "2026.5.0")).toBe("2026.5.0");
+ });
+
+ it("falls back to 'unknown' when every source is absent", () => {
+ expect(resolveServiceVersion(null, null, undefined)).toBe("unknown");
+ });
+
+ it("treats an empty env value as absent", () => {
+ expect(resolveServiceVersion(null, null, "")).toBe("unknown");
+ });
+});
+
+describe("readGitCommit", () => {
+ it("resolves git from the module checkout, not the process launch directory", () => {
+ // This test needs reachable git metadata in the checkout that holds the
+ // module. Skip the assertion when the environment has no git.
+ const baseline = readGitCommit();
+ if (baseline === null) return;
+
+ const original = process.cwd();
+ const launchDir = mkdtempSync(join(tmpdir(), "paperclip-no-git-"));
+ try {
+ // Move the process into a directory with no repository. The lookup still
+ // returns the checkout commit, because it reads the module directory, not
+ // the launch directory.
+ process.chdir(launchDir);
+ expect(readGitCommit()).toBe(baseline);
+ } finally {
+ process.chdir(original);
+ rmSync(launchDir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts
index 882ec1114a..7f902c85de 100644
--- a/server/src/__tests__/workspace-runtime.test.ts
+++ b/server/src/__tests__/workspace-runtime.test.ts
@@ -5881,6 +5881,10 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () =>
afterEach(async () => {
await resetRuntimeServicesForTests();
+ // Service control writes activity_log rows. Delete them before the company
+ // delete so a lingering foreign-key row cannot block the company delete and
+ // leak rows into the next test.
+ await db.delete(activityLog);
await db.delete(workspaceRuntimeServices);
await db.delete(executionWorkspaces);
await db.delete(projectWorkspaces);
@@ -6915,6 +6919,11 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
});
afterEach(async () => {
+ // Startup reconciliation writes activity_log rows (for example, exposure
+ // reservation drift). Delete those rows before the company delete. A stale
+ // activity_log row holds a foreign key to the company and makes the company
+ // delete fail, which leaks rows into the next test.
+ await db.delete(activityLog);
await db.delete(workspaceRuntimeServices);
await db.delete(executionWorkspaces);
await db.delete(projectWorkspaces);
diff --git a/server/src/__tests__/write-build-stamp.test.ts b/server/src/__tests__/write-build-stamp.test.ts
new file mode 100644
index 0000000000..eb7d55e63a
--- /dev/null
+++ b/server/src/__tests__/write-build-stamp.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+
+import { resolveBuildCommit } from "../../scripts/write-build-stamp.mjs";
+
+describe("resolveBuildCommit", () => {
+ it("prefers the git commit over the supplied environment commit", () => {
+ expect(resolveBuildCommit("aaaaaaa", "bbbbbbb")).toBe("aaaaaaa");
+ });
+
+ it("falls back to PAPERCLIP_BUILD_COMMIT when git gives no commit", () => {
+ // A Docker image build excludes `.git`, so the git lookup returns null. The
+ // image build passes the commit in the environment instead.
+ expect(resolveBuildCommit(null, "bbbbbbb")).toBe("bbbbbbb");
+ });
+
+ it("trims the supplied commit", () => {
+ expect(resolveBuildCommit(null, " bbbbbbb\n")).toBe("bbbbbbb");
+ });
+
+ it("returns null when neither git nor the environment gives a commit", () => {
+ expect(resolveBuildCommit(null, undefined)).toBe(null);
+ });
+
+ it("treats an empty supplied commit as absent", () => {
+ expect(resolveBuildCommit(null, "")).toBe(null);
+ expect(resolveBuildCommit(null, " ")).toBe(null);
+ });
+});
diff --git a/server/src/instrumentation.ts b/server/src/instrumentation.ts
index c1524c815d..5f7a31bc96 100644
--- a/server/src/instrumentation.ts
+++ b/server/src/instrumentation.ts
@@ -26,6 +26,8 @@
// exit via `shutdownInstrumentation()`, which index.ts awaits in its signal
// handler before `process.exit`.
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
import { createRequire } from "node:module";
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
@@ -354,6 +356,72 @@ async function importExporter(protocol: ExporterProtocol): Promise<{
}
}
+/**
+ * Read the commit SHA from the build stamp. The server `build` script writes
+ * `dist/build-info.json` next to the compiled module. Return the SHA, or null
+ * when the stamp is absent or unreadable. In `tsx` dev mode the module runs
+ * from `src`, where no stamp exists, so this returns null and the caller falls
+ * back to a runtime git lookup.
+ */
+export function readBuildStamp(): string | null {
+ try {
+ const stampUrl = new URL("./build-info.json", import.meta.url);
+ const raw = readFileSync(stampUrl, "utf8");
+ const parsed = JSON.parse(raw) as { commit?: unknown };
+ if (typeof parsed.commit === "string" && parsed.commit.length > 0) {
+ return parsed.commit;
+ }
+ return null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Read the current commit SHA with `git rev-parse --short HEAD`. Return the
+ * SHA, or null on any failure. This covers dev mode, where the process runs
+ * from a git checkout. A missing `git` or a checkout with no `.git` returns
+ * null and is not fatal.
+ *
+ * The lookup runs in the directory of this module, not the directory the
+ * server process started in. `import.meta.url` points at `src` in dev mode and
+ * `dist` in a built server; both sit inside the Paperclip checkout. A server
+ * launched from an unrelated directory, or from inside another repository,
+ * would otherwise report a wrong commit or fall back.
+ */
+export function readGitCommit(): string | null {
+ try {
+ const out = execFileSync("git", ["rev-parse", "--short", "HEAD"], {
+ cwd: new URL("./", import.meta.url),
+ stdio: ["ignore", "pipe", "ignore"],
+ })
+ .toString()
+ .trim();
+ return out.length > 0 ? out : null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Resolve the `service.version` span attribute. The order is:
+ * 1. The build stamp — the commit the running server was built from.
+ * 2. A runtime git lookup — covers `tsx src/index.ts` dev mode.
+ * 3. The `OTEL_SERVICE_VERSION` environment variable.
+ * 4. "unknown".
+ * The build stamp wins over the environment variable, so a stale
+ * `OTEL_SERVICE_VERSION` cannot mask the true built commit. `OTEL_SERVICE_VERSION`
+ * is a Paperclip-specific variable, not an OpenTelemetry SDK variable, so
+ * Paperclip controls this precedence.
+ */
+export function resolveServiceVersion(
+ buildStamp: string | null,
+ gitCommit: string | null,
+ envVersion: string | undefined,
+): string {
+ return buildStamp || gitCommit || envVersion || "unknown";
+}
+
async function bootstrapOtel(endpoint: string): Promise<void> {
const { protocol, packageName: exporterPackage } = resolveProtocol();
@@ -379,10 +447,19 @@ async function bootstrapOtel(endpoint: string): Promise<void> {
const { resourceFromAttributes } = resources;
const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } = semconv;
+ const serviceVersion = resolveServiceVersion(
+ readBuildStamp(),
+ readGitCommit(),
+ process.env.OTEL_SERVICE_VERSION,
+ );
+ // Log the resolved value once so an operator can confirm the built commit.
+ // eslint-disable-next-line no-console
+ console.log(`[paperclip] OpenTelemetry service.version=${serviceVersion}`);
+
const sdk = new NodeSDK({
resource: resourceFromAttributes({
[ATTR_SERVICE_NAME]: process.env.OTEL_SERVICE_NAME || "paperclip",
- [ATTR_SERVICE_VERSION]: process.env.OTEL_SERVICE_VERSION || "unknown",
+ [ATTR_SERVICE_VERSION]: serviceVersion,
}),
// For the HTTP protocols OTEL_EXPORTER_OTLP_ENDPOINT is a *base* URL
// and the exporter appends /v1/traces only when it reads the env var
diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts
index d1a4f7801c..ade0786489 100644
--- a/server/src/services/workspace-runtime.ts
+++ b/server/src/services/workspace-runtime.ts
@@ -7702,19 +7702,29 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
}
if (adoptedRecord) {
const adoptedUrl = adoptedRecord.url ?? row.backendUrl ?? row.url ?? null;
+ const adoptedHealthInput = {
+ db,
+ serviceName: row.serviceName,
+ command: row.command,
+ provider: "local_process",
+ port: adoptedRecord.port ?? row.port,
+ cwd: row.cwd,
+ executionWorkspaceId: row.executionWorkspaceId ?? null,
+ companyId: row.companyId,
+ };
+ // A surviving service can be slow to answer one probe when the host is
+ // busy at startup. One timeout is not enough evidence to terminate it.
+ // Confirm an unhealthy verdict with a second probe after a short bounded
+ // delay, the same way the reuse path protects a shared runtime.
+ let adoptedHealthy = await isRuntimeServiceUrlHealthy(adoptedUrl, adoptedHealthInput);
+ if (!adoptedHealthy) {
+ await delay(250);
+ adoptedHealthy = await isRuntimeServiceUrlHealthy(adoptedUrl, adoptedHealthInput);
+ }
if (
backfillDecision.action === "reprovision"
|| !exposureHealthMatches
- || !(await isRuntimeServiceUrlHealthy(adoptedUrl, {
- db,
- serviceName: row.serviceName,
- command: row.command,
- provider: "local_process",
- port: adoptedRecord.port ?? row.port,
- cwd: row.cwd,
- executionWorkspaceId: row.executionWorkspaceId ?? null,
- companyId: row.companyId,
- }))
+ || !adoptedHealthy
) {
if (backfillDecision.action === "reprovision") backfilled += 1;
await terminateLocalService(adoptedRecord);