Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(web): keep a deploy from deleting a running build's chunks (#7817)
Eric Hayes6c843dd48ce6
diff --git a/apps/web/scripts/retain-recent-builds.sh b/apps/web/scripts/retain-recent-builds.shnew file mode 100644index 0000000000..20be5ebfe4--- /dev/null+++ b/apps/web/scripts/retain-recent-builds.sh@@ -0,0 +1,134 @@+#!/usr/bin/env bash+# Remove published assets that no recent build refers to.+#+# This replaces `aws s3 sync --delete`. That flag deleted the previous build's+# content-hashed files the moment a new build landed, which breaks tabs that+# are still open: the app fetches most of itself lazily - views, modals, block+# components, the fold worker and its WASM - so a tab that has not needed a+# chunk yet asks for it long after the build that owns it stopped being+# current. The client moves tabs to a new build deliberately gently (it will+# not interrupt someone typing), so that window is minutes to hours, not+# seconds. Deleting on deploy turned every such fetch into a 404; for the fold+# worker that surfaced as an unexplained worker failure, because a module+# worker whose script 404s reports an error with no message.+#+# So each deploy records the keys it publishes, and this prunes anything the+# last <keep-builds> records do not, between them, still refer to. Whatever+# the records say, nothing the build being published right now is ever+# removed.+#+# <keep-builds> counts the build being published, so 10 keeps this one and the+# nine before it.+set -euo pipefail++if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then+ echo "usage: retain-recent-builds.sh <dist-root> <s3-prefix> [keep-builds]" >&2+ exit 2+fi++dist_root=${1%/}+s3_prefix=${2%/}+keep_builds=${3:-10}+if ! [[ "$keep_builds" =~ ^[1-9][0-9]*$ ]]; then+ echo "keep-builds must be a positive integer" >&2+ exit 2+fi++s3_location=${s3_prefix#s3://}+if [ "$s3_location" = "$s3_prefix" ]; then+ echo "S3 prefix must start with s3://" >&2+ exit 2+fi+bucket=${s3_location%%/*}+if [ -z "$bucket" ]; then+ echo "S3 prefix must include a bucket" >&2+ exit 2+fi+if [[ "$s3_location" == */* ]]; then+ prefix=${s3_location#*/}+else+ prefix=''+fi+key_prefix=${prefix:+$prefix/}+records_prefix=${key_prefix}_builds++# The build number the shell advertises as `macro-bundle-build`, so a record+# sorts by the same clock the service worker compares builds with.+manifest=$dist_root/bundle-manifest.json+if [ ! -f "$manifest" ]; then+ echo "no bundle manifest at $manifest" >&2+ exit 1+fi+build=$(sed -n 's/.*"bundleBuild"[[:space:]]*:[[:space:]]*\([0-9]\{1,\}\).*/\1/p' "$manifest" | head -1)+if [ -z "$build" ]; then+ echo "bundle manifest has no bundleBuild" >&2+ exit 1+fi++work=$(mktemp -d)+trap 'rm -rf "$work"' EXIT+published=$work/published+retained=$work/retained+existing=$work/existing++# What this build publishes, as the keys the sync wrote them to.+(cd "$dist_root" && find . -type f) |+ sed "s|^\./|$key_prefix|" |+ LC_ALL=C sort -u >"$published"+# Every build publishes a shell. Its absence means the output handed to this+# script is not a build, and pruning against it would retire live assets.+if ! LC_ALL=C grep -qx "${key_prefix}index.html" "$published"; then+ echo "no ${key_prefix}index.html under $dist_root; refusing to prune" >&2+ exit 1+fi++aws s3 cp "$published" "s3://$bucket/$records_prefix/$build.txt" >/dev/null++# Newest builds first, so the tail of the list is what falls out of retention.+mapfile -t records < <(+ aws s3api list-objects-v2 --bucket "$bucket" --prefix "$records_prefix/" \+ --query 'Contents[].Key' --output text |+ tr '\t' '\n' |+ sed -n 's|.*/\([0-9]\{1,\}\)\.txt$|\1|p' |+ LC_ALL=C sort -rn+)++# This build's own keys first: a record that cannot be read back must never be+# able to make the live build look prunable.+cp "$published" "$retained"+kept=0+for record in "${records[@]}"; do+ kept=$((kept + 1))+ if [ "$kept" -le "$keep_builds" ]; then+ aws s3 cp "s3://$bucket/$records_prefix/$record.txt" - >>"$retained"+ else+ aws s3 rm "s3://$bucket/$records_prefix/$record.txt" >/dev/null+ fi+done+LC_ALL=C sort -u "$retained" -o "$retained"++list_args=(s3api list-objects-v2 --bucket "$bucket")+if [ -n "$prefix" ]; then+ list_args+=(--prefix "$key_prefix")+fi+list_args+=(--query 'Contents[].Key' --output text)+aws "${list_args[@]}" |+ tr '\t' '\n' |+ sed '/^$/d; /^None$/d' |+ LC_ALL=C sort -u >"$existing"++# Keys nothing retained refers to. `index.html` is republished every deploy so+# it is always retained; the exclusions below are for objects this script does+# not own - the records themselves, the archive uploaded as its own resource,+# and the cache WASM, which `prune-old-brotli-from-s3.sh` retires on its own+# schedule because the generic sync never uploads it.+while IFS= read -r key; do+ [ -n "$key" ] || continue+ case "$key" in+ "$records_prefix"/*) continue ;;+ *app-archive.zip | app-archive/*) continue ;;+ *cache_wasm_bg*) continue ;;+ *index.html) continue ;;+ esac+ aws s3 rm "s3://$bucket/$key" >/dev/null+done < <(LC_ALL=C comm -23 "$existing" "$retained")diff --git a/apps/web/scripts/retain-recent-builds.test.ts b/apps/web/scripts/retain-recent-builds.test.tsnew file mode 100644index 0000000000..fcd03a30ce--- /dev/null+++ b/apps/web/scripts/retain-recent-builds.test.ts@@ -0,0 +1,286 @@+import { spawnSync } from 'node:child_process';+import {+ chmodSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ rmSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { afterEach, describe, expect, it } from 'vitest';++const script = resolve(import.meta.dirname, 'retain-recent-builds.sh');++const temporaryDirectories: string[] = [];+afterEach(() => {+ for (const directory of temporaryDirectories.splice(0)) {+ rmSync(directory, { recursive: true, force: true });+ }+});++type Bucket = {+ /** Build number to the keys that build published. */+ records: Record<string, string[]>;+ /** Every key currently in the bucket under the prefix. */+ objects: string[];+};++/**+ * Runs the script against a fake `aws` that answers from `bucket` and records+ * what it was asked to delete.+ */+function run(+ distFiles: Record<string, string>,+ bucket: Bucket,+ keepBuilds: number+) {+ const root = mkdtempSync(join(tmpdir(), 'retain-builds-'));+ temporaryDirectories.push(root);+ const dist = join(root, 'dist');+ const bin = join(root, 'bin');+ const state = join(root, 'bucket.json');+ const deletions = join(root, 'deleted.txt');+ const uploads = join(root, 'uploaded.txt');+ mkdirSync(bin, { recursive: true });+ for (const [path, contents] of Object.entries(distFiles)) {+ const full = join(dist, path);+ mkdirSync(resolve(full, '..'), { recursive: true });+ writeFileSync(full, contents);+ }+ writeFileSync(state, JSON.stringify(bucket));+ writeFileSync(deletions, '');+ writeFileSync(uploads, '');++ // Only the three shapes the script uses: list keys, read a record, delete.+ const fakeAws = join(bin, 'aws');+ writeFileSync(+ fakeAws,+ `#!/usr/bin/env node+const { readFileSync, writeFileSync, appendFileSync } = require('node:fs');+const bucket = JSON.parse(readFileSync(${JSON.stringify(state)}, 'utf8'));+const argv = process.argv.slice(2);+if (argv[0] === 's3api' && argv[1] === 'list-objects-v2') {+ const prefix = argv[argv.indexOf('--prefix') + 1] ?? '';+ const keys = prefix.endsWith('_builds/')+ ? Object.keys(bucket.records).map((build) => prefix + build + '.txt')+ : bucket.objects.filter((key) => key.startsWith(prefix));+ process.stdout.write(keys.length ? keys.join('\\t') + '\\n' : 'None\\n');+ process.exit(0);+}+if (argv[0] === 's3' && argv[1] === 'cp' && argv[3] === '-') {+ const build = argv[2].replace(/.*\\/(\\d+)\\.txt$/, '$1');+ process.stdout.write((bucket.records[build] ?? []).join('\\n') + '\\n');+ process.exit(0);+}+if (argv[0] === 's3' && argv[1] === 'cp') {+ // Uploading a record makes it visible to the listing, as S3 would.+ const build = argv[3].replace(/.*\\/(\\d+)\\.txt$/, '$1');+ bucket.records[build] = readFileSync(argv[2], 'utf8').split('\\n').filter(Boolean);+ writeFileSync(${JSON.stringify(state)}, JSON.stringify(bucket));+ appendFileSync(${JSON.stringify(uploads)}, argv[3] + '\\n');+ process.exit(0);+}+if (argv[0] === 's3' && argv[1] === 'rm') {+ appendFileSync(${JSON.stringify(deletions)}, argv[2] + '\\n');+ process.exit(0);+}+process.stderr.write('unexpected aws call: ' + argv.join(' ') + '\\n');+process.exit(1);+`+ );+ chmodSync(fakeAws, 0o755);++ const result = spawnSync(+ 'bash',+ [script, dist, 's3://example/app', String(keepBuilds)],+ {+ encoding: 'utf8',+ env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },+ }+ );+ const lines = (path: string) =>+ readFileSync(path, 'utf8').split('\n').filter(Boolean);+ return {+ result,+ deleted: lines(deletions).map((uri) => uri.replace('s3://example/', '')),+ uploaded: lines(uploads),+ };+}++const manifest = (build: number) =>+ JSON.stringify({ schemaVersion: 3, bundleBuild: build });++describe('retention wiring', () => {+ const infrastructure = readFileSync(+ resolve(import.meta.dirname, '../../../infra/stacks/web-app/index.ts'),+ 'utf8'+ );++ it('does not let the generic sync delete a superseded build', () => {+ const sync = infrastructure.slice(+ infrastructure.indexOf('aws s3 sync ./output')+ );+ expect(sync.slice(0, sync.indexOf('`'))).not.toContain('--delete');+ });++ it('prunes last, after every publishing step has succeeded', () => {+ const sync = infrastructure.indexOf('aws s3 sync ./output');+ const indexPublish = infrastructure.indexOf(+ 'index-html-object-metadata-command'+ );+ // Anchored on the invocations: both scripts are named in comments too.+ const cacheWasmPrune = infrastructure.indexOf(+ 'bash ../../../apps/web/scripts/cache-wasm/prune-old-brotli-from-s3.sh'+ );+ const retain = infrastructure.indexOf(+ 'bash ../../../apps/web/scripts/retain-recent-builds.sh'+ );+ expect(sync).toBeLessThan(indexPublish);+ expect(indexPublish).toBeLessThan(cacheWasmPrune);+ expect(cacheWasmPrune).toBeLessThan(retain);+ // Serialized behind the other prune so a failure above retires nothing.+ expect(infrastructure).toContain('dependsOn: [pruneOldCacheWasmCommand]');+ });+});++describe('retaining recent builds', () => {+ it('keeps what recent builds still refer to and drops what none do', () => {+ const { result, deleted } = run(+ {+ 'bundle-manifest.json': manifest(300),+ 'index.html': '<html></html>',+ 'app-new.js': '',+ 'fold.worker-new.js': '',+ },+ {+ records: {+ '100': ['app/index.html', 'app/app-ancient.js'],+ '200': ['app/index.html', 'app/app-previous.js'],+ },+ objects: [+ 'app/index.html',+ 'app/app-ancient.js',+ 'app/app-previous.js',+ 'app/app-new.js',+ 'app/fold.worker-new.js',+ ],+ },+ 2+ );++ expect(result.stderr).toBe('');+ expect(result.status).toBe(0);+ // The window counts the build being published, so 2 is 300 and 200; build+ // 100 falls out, and its record goes with its assets.+ expect(deleted).toEqual(['app/_builds/100.txt', 'app/app-ancient.js']);+ });++ it('keeps the previous build so an open tab can still fetch it lazily', () => {+ const { deleted } = run(+ {+ 'bundle-manifest.json': manifest(300),+ 'index.html': '',+ 'fold.worker-new.js': '',+ },+ {+ records: { '200': ['app/fold.worker-previous.js'] },+ objects: ['app/fold.worker-previous.js', 'app/fold.worker-new.js'],+ },+ 10+ );++ // The exact failure this change exists for: a tab on build 200 has not+ // constructed its fold worker yet when build 300 lands.+ expect(deleted).toEqual([]);+ });++ it('never deletes what the build being published just wrote', () => {+ const { deleted } = run(+ {+ 'bundle-manifest.json': manifest(300),+ 'index.html': '',+ 'app-new.js': '',+ },+ // No record survives to vouch for the live build.+ { records: {}, objects: ['app/app-new.js', 'app/index.html'] },+ 1+ );++ expect(deleted).toEqual([]);+ });++ it('leaves objects it does not own alone', () => {+ const { deleted } = run(+ { 'bundle-manifest.json': manifest(300), 'index.html': '' },+ {+ records: {},+ objects: [+ 'app/index.html',+ 'app/app-archive.zip',+ 'app/assets/cache_wasm_bg-hash.wasm',+ 'app/assets/orphan-hash.js',+ ],+ },+ 1+ );++ // The archive is its own resource and the cache WASM has its own prune.+ expect(deleted).toEqual(['app/assets/orphan-hash.js']);+ });++ it('retires records beyond the retention window with their assets', () => {+ const { deleted, uploaded } = run(+ { 'bundle-manifest.json': manifest(400), 'index.html': '' },+ {+ records: {+ '100': ['app/a.js'],+ '200': ['app/b.js'],+ '300': ['app/c.js'],+ },+ objects: ['app/index.html', 'app/a.js', 'app/b.js', 'app/c.js'],+ },+ 2+ );++ expect(uploaded).toEqual(['s3://example/app/_builds/400.txt']);+ // A window of 2 is the build being published (400) and the one before it+ // (300), so 200 and 100 are retired along with what only they referred to.+ expect(deleted).toEqual(+ expect.arrayContaining([+ 'app/_builds/200.txt',+ 'app/_builds/100.txt',+ 'app/a.js',+ 'app/b.js',+ ])+ );+ expect(deleted).not.toContain('app/c.js');+ expect(deleted).not.toContain('app/index.html');+ });++ it('refuses to prune output that is not a build', () => {+ const { result, deleted } = run(+ { 'bundle-manifest.json': manifest(300) },+ { records: {}, objects: ['app/app-live.js'] },+ 10+ );++ // A shell-less dist would otherwise read as "no build refers to anything",+ // and take the live assets with it.+ expect(result.status).not.toBe(0);+ expect(result.stderr).toContain('refusing to prune');+ expect(deleted).toEqual([]);+ });++ it('refuses to run without a bundle manifest', () => {+ const { result } = run(+ { 'index.html': '' },+ { records: {}, objects: [] },+ 10+ );++ expect(result.status).not.toBe(0);+ expect(result.stderr).toContain('bundle manifest');+ });+});diff --git a/apps/web/src/lib/core/agent-fold/client.test.ts b/apps/web/src/lib/core/agent-fold/client.test.tsindex 6c07ea4854..175243ebdb 100644--- a/apps/web/src/lib/core/agent-fold/client.test.ts+++ b/apps/web/src/lib/core/agent-fold/client.test.ts@@ -93,6 +93,57 @@ describe('agent fold preloading', () => {await expect(pushed).resolves.toEqual([]);});+ // The three ways a worker can fail are distinguishable, and the browser+ // tells them apart only by the error event's message. Measured in Chrome: a+ // script that 404s reports no message at all; one that loads and throws+ // reports what it threw; a failure inside the worker never reaches `error`+ // and comes back as an ordinary reply.+ it('reports a script that never loaded as the build being gone', async () => {+ const { AgentFoldWorkerUnavailable, pushSession } = await import(+ './client'+ );+ const pushed = pushSession('session', [{ kind: 'snapshot', rows: [] }]);++ workers[0]!.dispatchEvent(new ErrorEvent('error', { message: '' }));++ const error = await pushed.catch((reason: unknown) => reason);+ expect(error).toBeInstanceOf(AgentFoldWorkerUnavailable);+ expect((error as Error).message).toContain('no longer served');+ // The chunk names the build, which is the whole diagnosis.+ expect((error as Error).message).toContain('fold.worker');+ });++ it('reports a script that loaded and threw as a startup failure', async () => {+ const { AgentFoldWorkerUnavailable, pushSession } = await import(+ './client'+ );+ const pushed = pushSession('session', [{ kind: 'snapshot', rows: [] }]);++ workers[0]!.dispatchEvent(+ new ErrorEvent('error', { message: 'Uncaught Error: wasm exploded' })+ );++ const error = await pushed.catch((reason: unknown) => reason);+ expect(error).not.toBeInstanceOf(AgentFoldWorkerUnavailable);+ expect((error as Error).message).toContain('wasm exploded');+ });++ it('rejects every fold in flight from one worker failure', async () => {+ const { pushSession } = await import('./client');+ const first = pushSession('a', [{ kind: 'snapshot', rows: [] }]);+ const second = pushSession('b', [{ kind: 'snapshot', rows: [] }]);++ workers[0]!.dispatchEvent(new ErrorEvent('error', { message: '' }));++ // One cause, two reports: why a single failure looks like many in Datadog.+ const [a, b] = await Promise.all([+ first.catch((reason: unknown) => reason),+ second.catch((reason: unknown) => reason),+ ]);+ expect((a as Error).name).toBe('AgentFoldWorkerUnavailable');+ expect((b as Error).name).toBe('AgentFoldWorkerUnavailable');+ });+it('contains constructor failures without preventing a later real load', async () => {vi.spyOn(console, 'warn').mockImplementation(() => {});vi.stubGlobal(diff --git a/apps/web/src/lib/core/agent-fold/client.ts b/apps/web/src/lib/core/agent-fold/client.tsindex b06402995d..c616bd6345 100644--- a/apps/web/src/lib/core/agent-fold/client.ts+++ b/apps/web/src/lib/core/agent-fold/client.ts@@ -26,6 +26,43 @@ interface Pending {reject: (error: Error) => void;}+/**+ * The worker's script never ran.+ *+ * A dedicated worker reports a script it could not load as an `error` event+ * carrying no message at all; one that loaded and then threw reports what it+ * threw, and anything failing *inside* the worker comes back as an ordinary+ * `{ok: false}` reply. So an empty message is a precise signal, and it has+ * one overwhelmingly likely cause: this tab is running a build whose assets+ * are no longer served. Deploys replace the content-hashed bundle and the+ * worker chunk is fetched lazily, on the first fold, which is often after+ * the build that owns it has been superseded.+ *+ * Named rather than folded into a generic failure so it groups on its own in+ * error tracking: the fix for it is a deploy/retention question, not a bug+ * in the fold.+ */+export class AgentFoldWorkerUnavailable extends Error {+ constructor(readonly workerUrl: string) {+ super(+ `agent fold worker script could not be loaded (${workerUrl}) - this tab is running a build that is no longer served; reloading will pick up the current one`+ );+ this.name = 'AgentFoldWorkerUnavailable';+ }+}++/**+ * What an `error` event from the worker means, as precisely as the browser+ * lets us put it. The empty-message case is {@link AgentFoldWorkerUnavailable};+ * anything else threw while starting up and says so itself.+ */+function workerStartupFailure(url: string, message: string): Error {+ if (!message) return new AgentFoldWorkerUnavailable(url);+ const error = new Error(`agent fold worker failed to start: ${message}`);+ error.name = 'AgentFoldWorkerStartupFailed';+ return error;+}+let worker: Worker | undefined;const pending = new Map<number, Pending>();let nextId = 0;@@ -37,9 +74,10 @@ let nextId = 0;function ensureWorker(): Worker {if (worker) return worker;- const started = new Worker(new URL('./fold.worker.ts', import.meta.url), {- type: 'module',- });+ // Held so a failure can name the file that did not load: its content hash+ // identifies the build this tab is running, which is the whole diagnosis.+ const url = new URL('./fold.worker.ts', import.meta.url);+ const started = new Worker(url, { type: 'module' });started.addEventListener('message', (event: MessageEvent<FoldResponse>) => {const response = event.data;@@ -55,7 +93,9 @@ function ensureWorker(): Worker {started.addEventListener('error', (event) => {// The worker itself failed, so nothing in flight will ever be answered.- const error = new Error(`agent fold worker failed: ${event.message}`);+ // One error event takes down every fold in flight, which is why a single+ // failure reports as many: the count is concurrent folds, not causes.+ const error = workerStartupFailure(url.href, event.message ?? '');for (const waiting of pending.values()) waiting.reject(error);pending.clear();// Dropped so the next call starts a fresh one rather than waiting on adiff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.tsindex cc81dcc0f8..f060ff39c8 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts@@ -15,6 +15,14 @@ const fold = vi.hoisted(() => ({pushSession: vi.fn(),readSession: vi.fn(),closeSession: vi.fn(),+ // The real class, so `instanceof` classifies a load failure the way the+ // app does rather than the way a stand-in happens to.+ AgentFoldWorkerUnavailable: class AgentFoldWorkerUnavailable extends Error {+ constructor(readonly workerUrl: string) {+ super(`agent fold worker script could not be loaded (${workerUrl})`);+ this.name = 'AgentFoldWorkerUnavailable';+ }+ },}));const harness = vi.hoisted(() => ({get: vi.fn(),@@ -1107,6 +1115,14 @@ describe('AgentSession', () => {fold.readSession.mockRejectedValueOnce(new Error('worker died'));await expect(live.load()).rejects.toThrow('worker died');+ // A worker whose script never loaded is counted apart from a fold that+ // went wrong: nothing is wrong with the fold, and the fix is a deploy+ // retention question.+ fold.readSession.mockRejectedValueOnce(+ new fold.AgentFoldWorkerUnavailable('/app/fold.worker-abc.js')+ );+ await expect(live.load()).rejects.toThrow('could not be loaded');+expect(loadSpans().map((span) => [span.attributes['agent.session.load.outcome'],@@ -1117,6 +1133,7 @@ describe('AgentSession', () => {['failed', 'log_fetch'],['failed', 'access_denied'],['failed', 'fold'],+ ['failed', 'worker_unavailable'],]);live.release();});diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.tsindex 7d1bcda8be..835f92c86c 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts@@ -20,6 +20,7 @@*/import {+ AgentFoldWorkerUnavailable,closeSession,type FoldInput,pushSession,@@ -727,7 +728,11 @@ export class AgentSession {this.trace.folded(foldStartedAt);this.setTurn((await readSession(this.id)).metadata.turn);} catch (error) {- this.trace.failing('fold');+ this.trace.failing(+ error instanceof AgentFoldWorkerUnavailable+ ? 'worker_unavailable'+ : 'fold'+ );throw error;}if (this.resyncRequested) void this.resync();diff --git a/apps/web/src/lib/core/agent-session/load-telemetry.ts b/apps/web/src/lib/core/agent-session/load-telemetry.tsindex 830ed159cd..25c1dc34e2 100644--- a/apps/web/src/lib/core/agent-session/load-telemetry.ts+++ b/apps/web/src/lib/core/agent-session/load-telemetry.ts@@ -57,6 +57,13 @@ export type LoadFailure =| 'log_fetch'/** The fold worker failed on the snapshot. */| 'fold'+ /**+ * The fold worker's script never loaded, so no snapshot could be folded at+ * all. Almost always this tab's build being replaced while it was open, so+ * it is counted apart from `fold`: nothing is wrong with the fold, and the+ * fix is a deploy-retention one.+ */+ | 'worker_unavailable'/** Nothing above; a bug in the classification. */| 'unknown';diff --git a/infra/stacks/web-app/index.ts b/infra/stacks/web-app/index.tsindex 29ce494173..9607cc3deb 100644--- a/infra/stacks/web-app/index.ts+++ b/infra/stacks/web-app/index.ts@@ -69,6 +69,11 @@ const publicAccessBlock = new aws.s3.BucketPublicAccessBlock(const buildOutputPath = './output/app';const appArchiveOutputPath = './output/app-archive';const cacheWasmRetentionDays = 7;+// How many builds of content-hashed assets stay published. A tab keeps+// running the build it loaded until it is reloaded, and the client defers+// that while someone is using the app, so a deploy must not be able to pull+// a still-open build's lazily fetched chunks out from under it.+const assetBuildRetention = 10;const shellQuote = (value: string): string =>"'" + value.split("'").join("'\\''") + "'";execSync('rm -rf ./output', { stdio: 'inherit' });@@ -115,7 +120,12 @@ const syncAssetsCommand = new command.local.Command(// raw bytes in dist/archive for Tauri and local preview, but exclude both// raw and sidecar from generic sync. The targeted uploader stores Brotli// bytes at the original .wasm key with application/wasm + Content-Encoding br.- create: pulumi.interpolate`bash ../../../apps/web/scripts/cache-wasm/upload-brotli-to-s3.sh ./output/app s3://${webAppAssets.bucket}/app public-read && aws s3 sync ./output s3://${webAppAssets.bucket} --acl public-read --delete --exclude "app/app-archive.zip" --exclude "app-archive/*" --exclude "*cache_wasm_bg*.wasm" --exclude "*cache_wasm_bg*.wasm.br"`,+ // No --delete: it removed the previous build's content-hashed files as+ // soon as a new build landed, and a tab still running that build fetches+ // most of itself lazily, so those chunks are asked for long afterwards.+ // `retain-recent-builds.sh` deletes what the last few builds no longer+ // refer to instead, once everything below has published.+ create: pulumi.interpolate`bash ../../../apps/web/scripts/cache-wasm/upload-brotli-to-s3.sh ./output/app s3://${webAppAssets.bucket}/app public-read && aws s3 sync ./output s3://${webAppAssets.bucket} --acl public-read --exclude "app/app-archive.zip" --exclude "app-archive/*" --exclude "*cache_wasm_bg*.wasm" --exclude "*cache_wasm_bg*.wasm.br"`,triggers: [Date.now()],},{@@ -144,7 +154,7 @@ const indexHtmlObjectMetadataCommand = new command.local.Command(// Prune only after the current WASM, generic assets, index content, and index// metadata have all published. Any earlier failure preserves all prior keys.-new command.local.Command(+const pruneOldCacheWasmCommand = new command.local.Command('prune-old-cache-wasm-command',{create: pulumi.interpolate`bash ../../../apps/web/scripts/cache-wasm/prune-old-brotli-from-s3.sh ./output/app s3://${webAppAssets.bucket}/app ${cacheWasmRetentionDays}`,@@ -156,6 +166,21 @@ new command.local.Command(});+// What the generic sync's --delete used to do, delayed by a few builds. Runs+// last, and after the cache WASM prune, so a failure anywhere above leaves+// every prior key in place rather than half-retiring a build.+new command.local.Command(+ 'retain-recent-builds-command',+ {+ create: pulumi.interpolate`bash ../../../apps/web/scripts/retain-recent-builds.sh ./output/app s3://${webAppAssets.bucket}/app ${assetBuildRetention}`,+ triggers: [Date.now()],+ },+ {+ dependsOn: [pruneOldCacheWasmCommand],+ replaceOnChanges: ['*'],+ }+);+// First, create an IAM role and attach the AWSLambdaBasicExecutionRole policyconst lambdaRole = new aws.iam.Role('content-encoding-header-lambda-role', {assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({