Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.
TypeScript ai-agentsbill-of-materialserpinventory-managementleanmanufacturingmanufacturing-execution-systemmcp-servermesmrpopen-source-erppostgresqlproduction-planningqmsquality-management-systemreact-routerself-hostedsupabasetraceabilitytypescript
ci: Post Deploy Sync retries a colliding Inngest sync instead of cancelling a waiting run
Sidwebworksfda276ddcb2a
diff --git a/.claude/rules/sst-deployment-infrastructure.md b/.claude/rules/sst-deployment-infrastructure.mdindex 6b4f908680..aeefbbe16b 100644--- a/.claude/rules/sst-deployment-infrastructure.md+++ b/.claude/rules/sst-deployment-infrastructure.md@@ -136,9 +136,12 @@ registration handshake, not Carbon-specific. Two independent triggers exist:has no `/api/inngest` route to sync. For a workspace with a `project_id`(hosted Supabase) it then copies the ERP's `/templates/*.html` into theproject's Auth emails (`ci/src/auth-emails.ts`): hosted Auth cannot- download templates the way self-hosted Auth does. One sync at a time- (job-level `concurrency: post-deploy-sync`): each ERP deploy starts a run- that syncs every workspace, and two concurrent PUTs to one ERP collide.+ download templates the way self-hosted Auth does. Each ERP deploy starts a+ run that syncs every workspace, and Inngest answers a PUT that lands with+ another one with a 400, so the Inngest PUT retries with jittered backoff+ (`withRetry`, `ci/src/retry.ts`). There is no concurrency group: GitHub+ keeps one waiting job per group and cancels the rest, which marked the+ deploy's run cancelled.Both hit the same handler (`apps/erp/app/routes/api+/inngest.ts`, `serve()`from `inngest/remix`) — the registration target URL is fixed by that route'sdiff --git a/.github/workflows/post-deploy.yml b/.github/workflows/post-deploy.ymlindex c7a293e2de..afc7aac52a 100644--- a/.github/workflows/post-deploy.yml+++ b/.github/workflows/post-deploy.yml@@ -24,14 +24,6 @@ jobs:github.event_name != 'deployment_status' ||(github.event.deployment_status.state == 'success' &&startsWith(github.event.deployment.environment, 'Production – carbon'))- # One sync at a time: every run syncs every workspace, and two at once- # collide (Inngest answers the second PUT with a 400). On the job, not- # the workflow, so a skipped run (an MES or docs deploy) never takes- # the slot. A newer run replaces a waiting one; it syncs everything- # the waiting one would have, later.- concurrency:- group: post-deploy-sync- cancel-in-progress: falseruns-on: ubuntu-lateststeps:diff --git a/ci/src/post-deploy.ts b/ci/src/post-deploy.tsindex 24ca6b87f8..991ef093f9 100644--- a/ci/src/post-deploy.ts+++ b/ci/src/post-deploy.ts@@ -7,6 +7,7 @@ import { $ } from "execa";import { pushAuthEmails } from "./auth-emails";import { client } from "./client";import { SUPABASE_ACCESS_TOKEN } from "./env";+import { withRetry } from "./retry";type Workspace = {id: number;@@ -25,7 +26,13 @@ function stepsFor(workspace: Workspace): Step[] {{name: `Inngest sync for workspace ${id} (${url_erp})`,// --fail: an HTTP error is a failed sync, not a successful curl.- run: () => $`curl --fail --silent --show-error -X PUT https://${url_erp}/api/inngest`+ // Every ERP deploy starts a sync, and Inngest answers a PUT that lands+ // with another one with a 400, so a failed sync is retried.+ run: () =>+ withRetry(+ () =>+ $`curl --fail --silent --show-error -X PUT https://${url_erp}/api/inngest`+ )}];// A hosted Supabase project keeps Auth's emails as settings: copy in thediff --git a/ci/src/retry.test.ts b/ci/src/retry.test.tsnew file mode 100644index 0000000000..026ea298d6--- /dev/null+++ b/ci/src/retry.test.ts@@ -0,0 +1,37 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,+// including ports, remain AGPLv3; serving them over a network requires releasing their source.++import assert from "node:assert/strict";+import { test } from "node:test";++import { withRetry } from "./retry";++test("retries with jittered, growing waits until the call succeeds", async () => {+ const waits: number[] = [];+ let calls = 0;+ const result = await withRetry(+ async () => {+ if (++calls < 3) throw new Error("400");+ return "ok";+ },+ { baseMs: 1000, random: () => 0.5, wait: async (ms) => waits.push(ms) }+ );+ assert.equal(result, "ok");+ assert.deepEqual(waits, [500, 1000]);+});++test("throws the last error once the attempts run out", async () => {+ let calls = 0;+ await assert.rejects(+ withRetry(+ async () => {+ calls++;+ throw new Error("400");+ },+ { attempts: 3, wait: async () => {} }+ ),+ /400/+ );+ assert.equal(calls, 3);+});diff --git a/ci/src/retry.ts b/ci/src/retry.tsnew file mode 100644index 0000000000..7ba115e25e--- /dev/null+++ b/ci/src/retry.ts@@ -0,0 +1,34 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,+// including ports, remain AGPLv3; serving them over a network requires releasing their source.++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++/**+ * Runs `run` until it succeeds, up to `attempts` times. Each wait is a random+ * time between 0 and `baseMs × 2^n` ("full jitter"): several deploys that fail+ * together retry at different moments instead of colliding again.+ */+export async function withRetry<T>(+ run: () => Promise<T>,+ {+ attempts = 6,+ baseMs = 2000,+ wait = sleep,+ random = Math.random+ }: {+ attempts?: number;+ baseMs?: number;+ wait?: (ms: number) => Promise<unknown>;+ random?: () => number;+ } = {}+): Promise<T> {+ for (let attempt = 1; ; attempt++) {+ try {+ return await run();+ } catch (error) {+ if (attempt >= attempts) throw error;+ await wait(random() * baseMs * 2 ** (attempt - 1));+ }+ }+}