Skip to content

Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.

ci: Post Deploy Sync retries a colliding Inngest sync instead of cancelling a waiting run

Sidwebworksfda276ddcb2a
diff --git a/.claude/rules/sst-deployment-infrastructure.md b/.claude/rules/sst-deployment-infrastructure.md
index 6b4f908680..aeefbbe16b 100644
--- a/.claude/rules/sst-deployment-infrastructure.md
+++ b/.claude/rules/sst-deployment-infrastructure.md
@@ -136,9 +136,12 @@ registration handshake, not Carbon-specific. Two independent triggers exist:
has no `/api/inngest` route to sync. For a workspace with a `project_id`
(hosted Supabase) it then copies the ERP's `/templates/*.html` into the
project's Auth emails (`ci/src/auth-emails.ts`): hosted Auth cannot
- download templates the way self-hosted Auth does. One sync at a time
- (job-level `concurrency: post-deploy-sync`): each ERP deploy starts a run
- that syncs every workspace, and two concurrent PUTs to one ERP collide.
+ download templates the way self-hosted Auth does. Each ERP deploy starts a
+ run that syncs every workspace, and Inngest answers a PUT that lands with
+ another one with a 400, so the Inngest PUT retries with jittered backoff
+ (`withRetry`, `ci/src/retry.ts`). There is no concurrency group: GitHub
+ keeps one waiting job per group and cancels the rest, which marked the
+ deploy's run cancelled.
Both hit the same handler (`apps/erp/app/routes/api+/inngest.ts`, `serve()`
from `inngest/remix`) — the registration target URL is fixed by that route's
diff --git a/.github/workflows/post-deploy.yml b/.github/workflows/post-deploy.yml
index c7a293e2de..afc7aac52a 100644
--- a/.github/workflows/post-deploy.yml
+++ b/.github/workflows/post-deploy.yml
@@ -24,14 +24,6 @@ jobs:
github.event_name != 'deployment_status' ||
(github.event.deployment_status.state == 'success' &&
startsWith(github.event.deployment.environment, 'Production – carbon'))
- # One sync at a time: every run syncs every workspace, and two at once
- # collide (Inngest answers the second PUT with a 400). On the job, not
- # the workflow, so a skipped run (an MES or docs deploy) never takes
- # the slot. A newer run replaces a waiting one; it syncs everything
- # the waiting one would have, later.
- concurrency:
- group: post-deploy-sync
- cancel-in-progress: false
runs-on: ubuntu-latest
steps:
diff --git a/ci/src/post-deploy.ts b/ci/src/post-deploy.ts
index 24ca6b87f8..991ef093f9 100644
--- a/ci/src/post-deploy.ts
+++ b/ci/src/post-deploy.ts
@@ -7,6 +7,7 @@ import { $ } from "execa";
import { pushAuthEmails } from "./auth-emails";
import { client } from "./client";
import { SUPABASE_ACCESS_TOKEN } from "./env";
+import { withRetry } from "./retry";
type Workspace = {
id: number;
@@ -25,7 +26,13 @@ function stepsFor(workspace: Workspace): Step[] {
{
name: `Inngest sync for workspace ${id} (${url_erp})`,
// --fail: an HTTP error is a failed sync, not a successful curl.
- run: () => $`curl --fail --silent --show-error -X PUT https://${url_erp}/api/inngest`
+ // Every ERP deploy starts a sync, and Inngest answers a PUT that lands
+ // with another one with a 400, so a failed sync is retried.
+ run: () =>
+ withRetry(
+ () =>
+ $`curl --fail --silent --show-error -X PUT https://${url_erp}/api/inngest`
+ )
}
];
// A hosted Supabase project keeps Auth's emails as settings: copy in the
diff --git a/ci/src/retry.test.ts b/ci/src/retry.test.ts
new file mode 100644
index 0000000000..026ea298d6
--- /dev/null
+++ b/ci/src/retry.test.ts
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,
+// including ports, remain AGPLv3; serving them over a network requires releasing their source.
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { withRetry } from "./retry";
+
+test("retries with jittered, growing waits until the call succeeds", async () => {
+ const waits: number[] = [];
+ let calls = 0;
+ const result = await withRetry(
+ async () => {
+ if (++calls < 3) throw new Error("400");
+ return "ok";
+ },
+ { baseMs: 1000, random: () => 0.5, wait: async (ms) => waits.push(ms) }
+ );
+ assert.equal(result, "ok");
+ assert.deepEqual(waits, [500, 1000]);
+});
+
+test("throws the last error once the attempts run out", async () => {
+ let calls = 0;
+ await assert.rejects(
+ withRetry(
+ async () => {
+ calls++;
+ throw new Error("400");
+ },
+ { attempts: 3, wait: async () => {} }
+ ),
+ /400/
+ );
+ assert.equal(calls, 3);
+});
diff --git a/ci/src/retry.ts b/ci/src/retry.ts
new file mode 100644
index 0000000000..7ba115e25e
--- /dev/null
+++ b/ci/src/retry.ts
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,
+// including ports, remain AGPLv3; serving them over a network requires releasing their source.
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Runs `run` until it succeeds, up to `attempts` times. Each wait is a random
+ * time between 0 and `baseMs × 2^n` ("full jitter"): several deploys that fail
+ * together retry at different moments instead of colliding again.
+ */
+export async function withRetry<T>(
+ run: () => Promise<T>,
+ {
+ attempts = 6,
+ baseMs = 2000,
+ wait = sleep,
+ random = Math.random
+ }: {
+ attempts?: number;
+ baseMs?: number;
+ wait?: (ms: number) => Promise<unknown>;
+ random?: () => number;
+ } = {}
+): Promise<T> {
+ for (let attempt = 1; ; attempt++) {
+ try {
+ return await run();
+ } catch (error) {
+ if (attempt >= attempts) throw error;
+ await wait(random() * baseMs * 2 ** (attempt - 1));
+ }
+ }
+}