Skip to content

A one-of-a-kind resume builder that keeps your privacy in mind. Completely secure, customizable, portable, open-source and free forever. Try it out today!

fix: wait for production readiness before cache purge

Amruth Pillai9714c34bbbde
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index 26c4b8184..c1eaf0c45 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -448,6 +448,34 @@ jobs:
cd docker
./manage_stack.sh up reactive_resume
+ - name: Wait for healthy production version
+ if: ${{ needs.plan.outputs.release == 'true' && steps.version.outputs.prerelease != 'true' && (env.DEPLOY == 'true' || env.PURGE_CLOUDFLARE == 'true') }}
+ timeout-minutes: 16
+ env:
+ VERSION: ${{ steps.version.outputs.version }}
+ COMMIT_SHA: ${{ steps.version.outputs.sha }}
+ run: |
+ set -euo pipefail
+ matches=0
+ for attempt in $(seq 1 60); do
+ if curl --fail --silent --show-error --max-time 10 \
+ -H 'Cache-Control: no-cache' "https://rxresu.me/api/health?revision=$COMMIT_SHA&probe=$attempt" \
+ | jq --exit-status --arg version "$VERSION" --arg sha "$COMMIT_SHA" \
+ '.service == "reactive-resume" and .status == "healthy" and .version == $version and .revision == $sha' > /dev/null; then
+ matches=$((matches + 1))
+ if [[ "$matches" -ge 3 ]]; then
+ echo "Production is healthy at version $VERSION ($COMMIT_SHA)."
+ exit 0
+ fi
+ else
+ matches=0
+ fi
+ echo "Waiting for healthy version $VERSION ($COMMIT_SHA): probe $attempt/60, $matches/3 consecutive matches."
+ sleep 5
+ done
+ echo "::error::Production did not become healthy at version $VERSION ($COMMIT_SHA); cache purge withheld."
+ exit 1
+
- name: Purge production Cloudflare cache
if: ${{ needs.plan.outputs.release == 'true' && steps.version.outputs.prerelease != 'true' && env.PURGE_CLOUDFLARE == 'true' }}
env:
diff --git a/.github/workflows/mobile-ota-update.yml b/.github/workflows/mobile-ota-update.yml
index 5671eb639..f81372868 100644
--- a/.github/workflows/mobile-ota-update.yml
+++ b/.github/workflows/mobile-ota-update.yml
@@ -95,4 +95,4 @@ jobs:
- name: Publish production OTA update
working-directory: apps/mobile
- run: eas update --channel production --environment production --message "Docker $COMMIT_SHA" --upload-source-maps --non-interactive
+ run: eas update --channel production --environment production --message "Docker $COMMIT_SHA" --non-interactive
diff --git a/tooling/docker-publishing.test.ts b/tooling/docker-publishing.test.ts
index b4bea7912..855a202f6 100644
--- a/tooling/docker-publishing.test.ts
+++ b/tooling/docker-publishing.test.ts
@@ -86,6 +86,82 @@ it("validates release tags against the built version and distinguishes prereleas
}
});
+const commit = "a".repeat(40);
+const healthy = { service: "reactive-resume", status: "healthy", version: "6.0.0", revision: commit };
+
+it.each([
+ ["matching release", [healthy], 0, 3],
+ ["previous version", [{ ...healthy, version: "5.0.0" }], 1, 60],
+ ["previous commit with same version", [{ ...healthy, revision: "b".repeat(40) }], 1, 60],
+ ["unhealthy release", [{ ...healthy, status: "unhealthy" }], 1, 60],
+ ["wrong service", [{ ...healthy, service: "other" }], 1, 60],
+ ["invalid JSON", ["not JSON"], 1, 60],
+ ["HTTP failure", ["HTTP_ERROR"], 1, 60],
+ [
+ "rolling deployment and transient failures reset consecutive matches",
+ [
+ healthy,
+ healthy,
+ { ...healthy, version: "5.0.0" },
+ healthy,
+ healthy,
+ "HTTP_ERROR",
+ healthy,
+ healthy,
+ "not JSON",
+ healthy,
+ healthy,
+ healthy,
+ ],
+ 0,
+ 12,
+ ],
+] as const)("waits for published release before cache purge: %s", (_name, responses, status, probes) => {
+ const workflow = readFileSync(new URL("../.github/workflows/docker-publish.yml", import.meta.url), "utf8");
+ const script = workflow.match(
+ / - name: Deploy production stack[\s\S]*? - name: Wait for healthy production version[\s\S]*? run: \|\n([\s\S]*?)(?=\n - name: Purge production Cloudflare cache)/,
+ )?.[1];
+ if (!script) throw new Error("Production readiness check must run between deployment and cache purge");
+ const directory = mkdtempSync(join(tmpdir(), "docker-readiness-"));
+ try {
+ writeFileSync(
+ join(directory, "responses"),
+ responses.map((response) => (typeof response === "string" ? response : JSON.stringify(response))).join("\n") +
+ "\n",
+ );
+ writeFileSync(join(directory, "count"), "0");
+ writeFileSync(join(directory, "sleep"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 });
+ writeFileSync(
+ join(directory, "curl"),
+ `#!/usr/bin/env bash
+set -eu
+count=$(cat "$FIXTURE/count")
+count=$((count + 1))
+echo "$count" > "$FIXTURE/count"
+line=$(sed -n "${"$"}{count}p" "$FIXTURE/responses")
+if [[ -z "$line" ]]; then line=$(tail -n 1 "$FIXTURE/responses"); fi
+if [[ "$line" == HTTP_ERROR ]]; then exit 22; fi
+printf '%s\\n' "$line"
+`,
+ { mode: 0o755 },
+ );
+ const result = spawnSync("bash", ["-eu", "-c", script], {
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ PATH: `${directory}:${process.env.PATH}`,
+ FIXTURE: directory,
+ COMMIT_SHA: commit,
+ VERSION: healthy.version,
+ },
+ });
+ expect(result.status, result.stderr).toBe(status);
+ expect(Number(readFileSync(join(directory, "count"), "utf8"))).toBe(probes);
+ } finally {
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
+
it.each([
["complete GHCR index", "", false, false, 0, 1],
["missing GHCR source", "ghcr.io/example/app", false, false, 1, 0],