Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(web): keep a deploy from deleting a running build's chunks (#7817)

Eric Hayes6c843dd48ce6
diff --git a/apps/web/scripts/retain-recent-builds.sh b/apps/web/scripts/retain-recent-builds.sh
new file mode 100644
index 0000000000..20be5ebfe4
--- /dev/null
+++ b/apps/web/scripts/retain-recent-builds.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# Remove published assets that no recent build refers to.
+#
+# This replaces `aws s3 sync --delete`. That flag deleted the previous build's
+# content-hashed files the moment a new build landed, which breaks tabs that
+# are still open: the app fetches most of itself lazily - views, modals, block
+# components, the fold worker and its WASM - so a tab that has not needed a
+# chunk yet asks for it long after the build that owns it stopped being
+# current. The client moves tabs to a new build deliberately gently (it will
+# not interrupt someone typing), so that window is minutes to hours, not
+# seconds. Deleting on deploy turned every such fetch into a 404; for the fold
+# worker that surfaced as an unexplained worker failure, because a module
+# worker whose script 404s reports an error with no message.
+#
+# So each deploy records the keys it publishes, and this prunes anything the
+# last <keep-builds> records do not, between them, still refer to. Whatever
+# the records say, nothing the build being published right now is ever
+# removed.
+#
+# <keep-builds> counts the build being published, so 10 keeps this one and the
+# nine before it.
+set -euo pipefail
+
+if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
+ echo "usage: retain-recent-builds.sh <dist-root> <s3-prefix> [keep-builds]" >&2
+ exit 2
+fi
+
+dist_root=${1%/}
+s3_prefix=${2%/}
+keep_builds=${3:-10}
+if ! [[ "$keep_builds" =~ ^[1-9][0-9]*$ ]]; then
+ echo "keep-builds must be a positive integer" >&2
+ exit 2
+fi
+
+s3_location=${s3_prefix#s3://}
+if [ "$s3_location" = "$s3_prefix" ]; then
+ echo "S3 prefix must start with s3://" >&2
+ exit 2
+fi
+bucket=${s3_location%%/*}
+if [ -z "$bucket" ]; then
+ echo "S3 prefix must include a bucket" >&2
+ exit 2
+fi
+if [[ "$s3_location" == */* ]]; then
+ prefix=${s3_location#*/}
+else
+ prefix=''
+fi
+key_prefix=${prefix:+$prefix/}
+records_prefix=${key_prefix}_builds
+
+# The build number the shell advertises as `macro-bundle-build`, so a record
+# sorts by the same clock the service worker compares builds with.
+manifest=$dist_root/bundle-manifest.json
+if [ ! -f "$manifest" ]; then
+ echo "no bundle manifest at $manifest" >&2
+ exit 1
+fi
+build=$(sed -n 's/.*"bundleBuild"[[:space:]]*:[[:space:]]*\([0-9]\{1,\}\).*/\1/p' "$manifest" | head -1)
+if [ -z "$build" ]; then
+ echo "bundle manifest has no bundleBuild" >&2
+ exit 1
+fi
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+published=$work/published
+retained=$work/retained
+existing=$work/existing
+
+# What this build publishes, as the keys the sync wrote them to.
+(cd "$dist_root" && find . -type f) |
+ sed "s|^\./|$key_prefix|" |
+ LC_ALL=C sort -u >"$published"
+# Every build publishes a shell. Its absence means the output handed to this
+# script is not a build, and pruning against it would retire live assets.
+if ! LC_ALL=C grep -qx "${key_prefix}index.html" "$published"; then
+ echo "no ${key_prefix}index.html under $dist_root; refusing to prune" >&2
+ exit 1
+fi
+
+aws s3 cp "$published" "s3://$bucket/$records_prefix/$build.txt" >/dev/null
+
+# Newest builds first, so the tail of the list is what falls out of retention.
+mapfile -t records < <(
+ aws s3api list-objects-v2 --bucket "$bucket" --prefix "$records_prefix/" \
+ --query 'Contents[].Key' --output text |
+ tr '\t' '\n' |
+ sed -n 's|.*/\([0-9]\{1,\}\)\.txt$|\1|p' |
+ LC_ALL=C sort -rn
+)
+
+# This build's own keys first: a record that cannot be read back must never be
+# able to make the live build look prunable.
+cp "$published" "$retained"
+kept=0
+for record in "${records[@]}"; do
+ kept=$((kept + 1))
+ if [ "$kept" -le "$keep_builds" ]; then
+ aws s3 cp "s3://$bucket/$records_prefix/$record.txt" - >>"$retained"
+ else
+ aws s3 rm "s3://$bucket/$records_prefix/$record.txt" >/dev/null
+ fi
+done
+LC_ALL=C sort -u "$retained" -o "$retained"
+
+list_args=(s3api list-objects-v2 --bucket "$bucket")
+if [ -n "$prefix" ]; then
+ list_args+=(--prefix "$key_prefix")
+fi
+list_args+=(--query 'Contents[].Key' --output text)
+aws "${list_args[@]}" |
+ tr '\t' '\n' |
+ sed '/^$/d; /^None$/d' |
+ LC_ALL=C sort -u >"$existing"
+
+# Keys nothing retained refers to. `index.html` is republished every deploy so
+# it is always retained; the exclusions below are for objects this script does
+# not own - the records themselves, the archive uploaded as its own resource,
+# and the cache WASM, which `prune-old-brotli-from-s3.sh` retires on its own
+# schedule because the generic sync never uploads it.
+while IFS= read -r key; do
+ [ -n "$key" ] || continue
+ case "$key" in
+ "$records_prefix"/*) continue ;;
+ *app-archive.zip | app-archive/*) continue ;;
+ *cache_wasm_bg*) continue ;;
+ *index.html) continue ;;
+ esac
+ aws s3 rm "s3://$bucket/$key" >/dev/null
+done < <(LC_ALL=C comm -23 "$existing" "$retained")
diff --git a/apps/web/scripts/retain-recent-builds.test.ts b/apps/web/scripts/retain-recent-builds.test.ts
new file mode 100644
index 0000000000..fcd03a30ce
--- /dev/null
+++ b/apps/web/scripts/retain-recent-builds.test.ts
@@ -0,0 +1,286 @@
+import { spawnSync } from 'node:child_process';
+import {
+ chmodSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+
+const script = resolve(import.meta.dirname, 'retain-recent-builds.sh');
+
+const temporaryDirectories: string[] = [];
+afterEach(() => {
+ for (const directory of temporaryDirectories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
+
+type Bucket = {
+ /** Build number to the keys that build published. */
+ records: Record<string, string[]>;
+ /** Every key currently in the bucket under the prefix. */
+ objects: string[];
+};
+
+/**
+ * Runs the script against a fake `aws` that answers from `bucket` and records
+ * what it was asked to delete.
+ */
+function run(
+ distFiles: Record<string, string>,
+ bucket: Bucket,
+ keepBuilds: number
+) {
+ const root = mkdtempSync(join(tmpdir(), 'retain-builds-'));
+ temporaryDirectories.push(root);
+ const dist = join(root, 'dist');
+ const bin = join(root, 'bin');
+ const state = join(root, 'bucket.json');
+ const deletions = join(root, 'deleted.txt');
+ const uploads = join(root, 'uploaded.txt');
+ mkdirSync(bin, { recursive: true });
+ for (const [path, contents] of Object.entries(distFiles)) {
+ const full = join(dist, path);
+ mkdirSync(resolve(full, '..'), { recursive: true });
+ writeFileSync(full, contents);
+ }
+ writeFileSync(state, JSON.stringify(bucket));
+ writeFileSync(deletions, '');
+ writeFileSync(uploads, '');
+
+ // Only the three shapes the script uses: list keys, read a record, delete.
+ const fakeAws = join(bin, 'aws');
+ writeFileSync(
+ fakeAws,
+ `#!/usr/bin/env node
+const { readFileSync, writeFileSync, appendFileSync } = require('node:fs');
+const bucket = JSON.parse(readFileSync(${JSON.stringify(state)}, 'utf8'));
+const argv = process.argv.slice(2);
+if (argv[0] === 's3api' && argv[1] === 'list-objects-v2') {
+ const prefix = argv[argv.indexOf('--prefix') + 1] ?? '';
+ const keys = prefix.endsWith('_builds/')
+ ? Object.keys(bucket.records).map((build) => prefix + build + '.txt')
+ : bucket.objects.filter((key) => key.startsWith(prefix));
+ process.stdout.write(keys.length ? keys.join('\\t') + '\\n' : 'None\\n');
+ process.exit(0);
+}
+if (argv[0] === 's3' && argv[1] === 'cp' && argv[3] === '-') {
+ const build = argv[2].replace(/.*\\/(\\d+)\\.txt$/, '$1');
+ process.stdout.write((bucket.records[build] ?? []).join('\\n') + '\\n');
+ process.exit(0);
+}
+if (argv[0] === 's3' && argv[1] === 'cp') {
+ // Uploading a record makes it visible to the listing, as S3 would.
+ const build = argv[3].replace(/.*\\/(\\d+)\\.txt$/, '$1');
+ bucket.records[build] = readFileSync(argv[2], 'utf8').split('\\n').filter(Boolean);
+ writeFileSync(${JSON.stringify(state)}, JSON.stringify(bucket));
+ appendFileSync(${JSON.stringify(uploads)}, argv[3] + '\\n');
+ process.exit(0);
+}
+if (argv[0] === 's3' && argv[1] === 'rm') {
+ appendFileSync(${JSON.stringify(deletions)}, argv[2] + '\\n');
+ process.exit(0);
+}
+process.stderr.write('unexpected aws call: ' + argv.join(' ') + '\\n');
+process.exit(1);
+`
+ );
+ chmodSync(fakeAws, 0o755);
+
+ const result = spawnSync(
+ 'bash',
+ [script, dist, 's3://example/app', String(keepBuilds)],
+ {
+ encoding: 'utf8',
+ env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
+ }
+ );
+ const lines = (path: string) =>
+ readFileSync(path, 'utf8').split('\n').filter(Boolean);
+ return {
+ result,
+ deleted: lines(deletions).map((uri) => uri.replace('s3://example/', '')),
+ uploaded: lines(uploads),
+ };
+}
+
+const manifest = (build: number) =>
+ JSON.stringify({ schemaVersion: 3, bundleBuild: build });
+
+describe('retention wiring', () => {
+ const infrastructure = readFileSync(
+ resolve(import.meta.dirname, '../../../infra/stacks/web-app/index.ts'),
+ 'utf8'
+ );
+
+ it('does not let the generic sync delete a superseded build', () => {
+ const sync = infrastructure.slice(
+ infrastructure.indexOf('aws s3 sync ./output')
+ );
+ expect(sync.slice(0, sync.indexOf('`'))).not.toContain('--delete');
+ });
+
+ it('prunes last, after every publishing step has succeeded', () => {
+ const sync = infrastructure.indexOf('aws s3 sync ./output');
+ const indexPublish = infrastructure.indexOf(
+ 'index-html-object-metadata-command'
+ );
+ // Anchored on the invocations: both scripts are named in comments too.
+ const cacheWasmPrune = infrastructure.indexOf(
+ 'bash ../../../apps/web/scripts/cache-wasm/prune-old-brotli-from-s3.sh'
+ );
+ const retain = infrastructure.indexOf(
+ 'bash ../../../apps/web/scripts/retain-recent-builds.sh'
+ );
+ expect(sync).toBeLessThan(indexPublish);
+ expect(indexPublish).toBeLessThan(cacheWasmPrune);
+ expect(cacheWasmPrune).toBeLessThan(retain);
+ // Serialized behind the other prune so a failure above retires nothing.
+ expect(infrastructure).toContain('dependsOn: [pruneOldCacheWasmCommand]');
+ });
+});
+
+describe('retaining recent builds', () => {
+ it('keeps what recent builds still refer to and drops what none do', () => {
+ const { result, deleted } = run(
+ {
+ 'bundle-manifest.json': manifest(300),
+ 'index.html': '<html></html>',
+ 'app-new.js': '',
+ 'fold.worker-new.js': '',
+ },
+ {
+ records: {
+ '100': ['app/index.html', 'app/app-ancient.js'],
+ '200': ['app/index.html', 'app/app-previous.js'],
+ },
+ objects: [
+ 'app/index.html',
+ 'app/app-ancient.js',
+ 'app/app-previous.js',
+ 'app/app-new.js',
+ 'app/fold.worker-new.js',
+ ],
+ },
+ 2
+ );
+
+ expect(result.stderr).toBe('');
+ expect(result.status).toBe(0);
+ // The window counts the build being published, so 2 is 300 and 200; build
+ // 100 falls out, and its record goes with its assets.
+ expect(deleted).toEqual(['app/_builds/100.txt', 'app/app-ancient.js']);
+ });
+
+ it('keeps the previous build so an open tab can still fetch it lazily', () => {
+ const { deleted } = run(
+ {
+ 'bundle-manifest.json': manifest(300),
+ 'index.html': '',
+ 'fold.worker-new.js': '',
+ },
+ {
+ records: { '200': ['app/fold.worker-previous.js'] },
+ objects: ['app/fold.worker-previous.js', 'app/fold.worker-new.js'],
+ },
+ 10
+ );
+
+ // The exact failure this change exists for: a tab on build 200 has not
+ // constructed its fold worker yet when build 300 lands.
+ expect(deleted).toEqual([]);
+ });
+
+ it('never deletes what the build being published just wrote', () => {
+ const { deleted } = run(
+ {
+ 'bundle-manifest.json': manifest(300),
+ 'index.html': '',
+ 'app-new.js': '',
+ },
+ // No record survives to vouch for the live build.
+ { records: {}, objects: ['app/app-new.js', 'app/index.html'] },
+ 1
+ );
+
+ expect(deleted).toEqual([]);
+ });
+
+ it('leaves objects it does not own alone', () => {
+ const { deleted } = run(
+ { 'bundle-manifest.json': manifest(300), 'index.html': '' },
+ {
+ records: {},
+ objects: [
+ 'app/index.html',
+ 'app/app-archive.zip',
+ 'app/assets/cache_wasm_bg-hash.wasm',
+ 'app/assets/orphan-hash.js',
+ ],
+ },
+ 1
+ );
+
+ // The archive is its own resource and the cache WASM has its own prune.
+ expect(deleted).toEqual(['app/assets/orphan-hash.js']);
+ });
+
+ it('retires records beyond the retention window with their assets', () => {
+ const { deleted, uploaded } = run(
+ { 'bundle-manifest.json': manifest(400), 'index.html': '' },
+ {
+ records: {
+ '100': ['app/a.js'],
+ '200': ['app/b.js'],
+ '300': ['app/c.js'],
+ },
+ objects: ['app/index.html', 'app/a.js', 'app/b.js', 'app/c.js'],
+ },
+ 2
+ );
+
+ expect(uploaded).toEqual(['s3://example/app/_builds/400.txt']);
+ // A window of 2 is the build being published (400) and the one before it
+ // (300), so 200 and 100 are retired along with what only they referred to.
+ expect(deleted).toEqual(
+ expect.arrayContaining([
+ 'app/_builds/200.txt',
+ 'app/_builds/100.txt',
+ 'app/a.js',
+ 'app/b.js',
+ ])
+ );
+ expect(deleted).not.toContain('app/c.js');
+ expect(deleted).not.toContain('app/index.html');
+ });
+
+ it('refuses to prune output that is not a build', () => {
+ const { result, deleted } = run(
+ { 'bundle-manifest.json': manifest(300) },
+ { records: {}, objects: ['app/app-live.js'] },
+ 10
+ );
+
+ // A shell-less dist would otherwise read as "no build refers to anything",
+ // and take the live assets with it.
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain('refusing to prune');
+ expect(deleted).toEqual([]);
+ });
+
+ it('refuses to run without a bundle manifest', () => {
+ const { result } = run(
+ { 'index.html': '' },
+ { records: {}, objects: [] },
+ 10
+ );
+
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain('bundle manifest');
+ });
+});
diff --git a/apps/web/src/lib/core/agent-fold/client.test.ts b/apps/web/src/lib/core/agent-fold/client.test.ts
index 6c07ea4854..175243ebdb 100644
--- a/apps/web/src/lib/core/agent-fold/client.test.ts
+++ b/apps/web/src/lib/core/agent-fold/client.test.ts
@@ -93,6 +93,57 @@ describe('agent fold preloading', () => {
await expect(pushed).resolves.toEqual([]);
});
+ // The three ways a worker can fail are distinguishable, and the browser
+ // tells them apart only by the error event's message. Measured in Chrome: a
+ // script that 404s reports no message at all; one that loads and throws
+ // reports what it threw; a failure inside the worker never reaches `error`
+ // and comes back as an ordinary reply.
+ it('reports a script that never loaded as the build being gone', async () => {
+ const { AgentFoldWorkerUnavailable, pushSession } = await import(
+ './client'
+ );
+ const pushed = pushSession('session', [{ kind: 'snapshot', rows: [] }]);
+
+ workers[0]!.dispatchEvent(new ErrorEvent('error', { message: '' }));
+
+ const error = await pushed.catch((reason: unknown) => reason);
+ expect(error).toBeInstanceOf(AgentFoldWorkerUnavailable);
+ expect((error as Error).message).toContain('no longer served');
+ // The chunk names the build, which is the whole diagnosis.
+ expect((error as Error).message).toContain('fold.worker');
+ });
+
+ it('reports a script that loaded and threw as a startup failure', async () => {
+ const { AgentFoldWorkerUnavailable, pushSession } = await import(
+ './client'
+ );
+ const pushed = pushSession('session', [{ kind: 'snapshot', rows: [] }]);
+
+ workers[0]!.dispatchEvent(
+ new ErrorEvent('error', { message: 'Uncaught Error: wasm exploded' })
+ );
+
+ const error = await pushed.catch((reason: unknown) => reason);
+ expect(error).not.toBeInstanceOf(AgentFoldWorkerUnavailable);
+ expect((error as Error).message).toContain('wasm exploded');
+ });
+
+ it('rejects every fold in flight from one worker failure', async () => {
+ const { pushSession } = await import('./client');
+ const first = pushSession('a', [{ kind: 'snapshot', rows: [] }]);
+ const second = pushSession('b', [{ kind: 'snapshot', rows: [] }]);
+
+ workers[0]!.dispatchEvent(new ErrorEvent('error', { message: '' }));
+
+ // One cause, two reports: why a single failure looks like many in Datadog.
+ const [a, b] = await Promise.all([
+ first.catch((reason: unknown) => reason),
+ second.catch((reason: unknown) => reason),
+ ]);
+ expect((a as Error).name).toBe('AgentFoldWorkerUnavailable');
+ expect((b as Error).name).toBe('AgentFoldWorkerUnavailable');
+ });
+
it('contains constructor failures without preventing a later real load', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {});
vi.stubGlobal(
diff --git a/apps/web/src/lib/core/agent-fold/client.ts b/apps/web/src/lib/core/agent-fold/client.ts
index b06402995d..c616bd6345 100644
--- a/apps/web/src/lib/core/agent-fold/client.ts
+++ b/apps/web/src/lib/core/agent-fold/client.ts
@@ -26,6 +26,43 @@ interface Pending {
reject: (error: Error) => void;
}
+/**
+ * The worker's script never ran.
+ *
+ * A dedicated worker reports a script it could not load as an `error` event
+ * carrying no message at all; one that loaded and then threw reports what it
+ * threw, and anything failing *inside* the worker comes back as an ordinary
+ * `{ok: false}` reply. So an empty message is a precise signal, and it has
+ * one overwhelmingly likely cause: this tab is running a build whose assets
+ * are no longer served. Deploys replace the content-hashed bundle and the
+ * worker chunk is fetched lazily, on the first fold, which is often after
+ * the build that owns it has been superseded.
+ *
+ * Named rather than folded into a generic failure so it groups on its own in
+ * error tracking: the fix for it is a deploy/retention question, not a bug
+ * in the fold.
+ */
+export class AgentFoldWorkerUnavailable extends Error {
+ constructor(readonly workerUrl: string) {
+ super(
+ `agent fold worker script could not be loaded (${workerUrl}) - this tab is running a build that is no longer served; reloading will pick up the current one`
+ );
+ this.name = 'AgentFoldWorkerUnavailable';
+ }
+}
+
+/**
+ * What an `error` event from the worker means, as precisely as the browser
+ * lets us put it. The empty-message case is {@link AgentFoldWorkerUnavailable};
+ * anything else threw while starting up and says so itself.
+ */
+function workerStartupFailure(url: string, message: string): Error {
+ if (!message) return new AgentFoldWorkerUnavailable(url);
+ const error = new Error(`agent fold worker failed to start: ${message}`);
+ error.name = 'AgentFoldWorkerStartupFailed';
+ return error;
+}
+
let worker: Worker | undefined;
const pending = new Map<number, Pending>();
let nextId = 0;
@@ -37,9 +74,10 @@ let nextId = 0;
function ensureWorker(): Worker {
if (worker) return worker;
- const started = new Worker(new URL('./fold.worker.ts', import.meta.url), {
- type: 'module',
- });
+ // Held so a failure can name the file that did not load: its content hash
+ // identifies the build this tab is running, which is the whole diagnosis.
+ const url = new URL('./fold.worker.ts', import.meta.url);
+ const started = new Worker(url, { type: 'module' });
started.addEventListener('message', (event: MessageEvent<FoldResponse>) => {
const response = event.data;
@@ -55,7 +93,9 @@ function ensureWorker(): Worker {
started.addEventListener('error', (event) => {
// The worker itself failed, so nothing in flight will ever be answered.
- const error = new Error(`agent fold worker failed: ${event.message}`);
+ // One error event takes down every fold in flight, which is why a single
+ // failure reports as many: the count is concurrent folds, not causes.
+ const error = workerStartupFailure(url.href, event.message ?? '');
for (const waiting of pending.values()) waiting.reject(error);
pending.clear();
// Dropped so the next call starts a fresh one rather than waiting on a
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
index cc81dcc0f8..f060ff39c8 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
@@ -15,6 +15,14 @@ const fold = vi.hoisted(() => ({
pushSession: vi.fn(),
readSession: vi.fn(),
closeSession: vi.fn(),
+ // The real class, so `instanceof` classifies a load failure the way the
+ // app does rather than the way a stand-in happens to.
+ AgentFoldWorkerUnavailable: class AgentFoldWorkerUnavailable extends Error {
+ constructor(readonly workerUrl: string) {
+ super(`agent fold worker script could not be loaded (${workerUrl})`);
+ this.name = 'AgentFoldWorkerUnavailable';
+ }
+ },
}));
const harness = vi.hoisted(() => ({
get: vi.fn(),
@@ -1107,6 +1115,14 @@ describe('AgentSession', () => {
fold.readSession.mockRejectedValueOnce(new Error('worker died'));
await expect(live.load()).rejects.toThrow('worker died');
+ // A worker whose script never loaded is counted apart from a fold that
+ // went wrong: nothing is wrong with the fold, and the fix is a deploy
+ // retention question.
+ fold.readSession.mockRejectedValueOnce(
+ new fold.AgentFoldWorkerUnavailable('/app/fold.worker-abc.js')
+ );
+ await expect(live.load()).rejects.toThrow('could not be loaded');
+
expect(
loadSpans().map((span) => [
span.attributes['agent.session.load.outcome'],
@@ -1117,6 +1133,7 @@ describe('AgentSession', () => {
['failed', 'log_fetch'],
['failed', 'access_denied'],
['failed', 'fold'],
+ ['failed', 'worker_unavailable'],
]);
live.release();
});
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.ts
index 7d1bcda8be..835f92c86c 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts
@@ -20,6 +20,7 @@
*/
import {
+ AgentFoldWorkerUnavailable,
closeSession,
type FoldInput,
pushSession,
@@ -727,7 +728,11 @@ export class AgentSession {
this.trace.folded(foldStartedAt);
this.setTurn((await readSession(this.id)).metadata.turn);
} catch (error) {
- this.trace.failing('fold');
+ this.trace.failing(
+ error instanceof AgentFoldWorkerUnavailable
+ ? 'worker_unavailable'
+ : 'fold'
+ );
throw error;
}
if (this.resyncRequested) void this.resync();
diff --git a/apps/web/src/lib/core/agent-session/load-telemetry.ts b/apps/web/src/lib/core/agent-session/load-telemetry.ts
index 830ed159cd..25c1dc34e2 100644
--- a/apps/web/src/lib/core/agent-session/load-telemetry.ts
+++ b/apps/web/src/lib/core/agent-session/load-telemetry.ts
@@ -57,6 +57,13 @@ export type LoadFailure =
| 'log_fetch'
/** The fold worker failed on the snapshot. */
| 'fold'
+ /**
+ * The fold worker's script never loaded, so no snapshot could be folded at
+ * all. Almost always this tab's build being replaced while it was open, so
+ * it is counted apart from `fold`: nothing is wrong with the fold, and the
+ * fix is a deploy-retention one.
+ */
+ | 'worker_unavailable'
/** Nothing above; a bug in the classification. */
| 'unknown';
diff --git a/infra/stacks/web-app/index.ts b/infra/stacks/web-app/index.ts
index 29ce494173..9607cc3deb 100644
--- a/infra/stacks/web-app/index.ts
+++ b/infra/stacks/web-app/index.ts
@@ -69,6 +69,11 @@ const publicAccessBlock = new aws.s3.BucketPublicAccessBlock(
const buildOutputPath = './output/app';
const appArchiveOutputPath = './output/app-archive';
const cacheWasmRetentionDays = 7;
+// How many builds of content-hashed assets stay published. A tab keeps
+// running the build it loaded until it is reloaded, and the client defers
+// that while someone is using the app, so a deploy must not be able to pull
+// a still-open build's lazily fetched chunks out from under it.
+const assetBuildRetention = 10;
const shellQuote = (value: string): string =>
"'" + value.split("'").join("'\\''") + "'";
execSync('rm -rf ./output', { stdio: 'inherit' });
@@ -115,7 +120,12 @@ const syncAssetsCommand = new command.local.Command(
// raw bytes in dist/archive for Tauri and local preview, but exclude both
// raw and sidecar from generic sync. The targeted uploader stores Brotli
// bytes at the original .wasm key with application/wasm + Content-Encoding br.
- create: pulumi.interpolate`bash ../../../apps/web/scripts/cache-wasm/upload-brotli-to-s3.sh ./output/app s3://${webAppAssets.bucket}/app public-read && aws s3 sync ./output s3://${webAppAssets.bucket} --acl public-read --delete --exclude "app/app-archive.zip" --exclude "app-archive/*" --exclude "*cache_wasm_bg*.wasm" --exclude "*cache_wasm_bg*.wasm.br"`,
+ // No --delete: it removed the previous build's content-hashed files as
+ // soon as a new build landed, and a tab still running that build fetches
+ // most of itself lazily, so those chunks are asked for long afterwards.
+ // `retain-recent-builds.sh` deletes what the last few builds no longer
+ // refer to instead, once everything below has published.
+ create: pulumi.interpolate`bash ../../../apps/web/scripts/cache-wasm/upload-brotli-to-s3.sh ./output/app s3://${webAppAssets.bucket}/app public-read && aws s3 sync ./output s3://${webAppAssets.bucket} --acl public-read --exclude "app/app-archive.zip" --exclude "app-archive/*" --exclude "*cache_wasm_bg*.wasm" --exclude "*cache_wasm_bg*.wasm.br"`,
triggers: [Date.now()],
},
{
@@ -144,7 +154,7 @@ const indexHtmlObjectMetadataCommand = new command.local.Command(
// Prune only after the current WASM, generic assets, index content, and index
// metadata have all published. Any earlier failure preserves all prior keys.
-new command.local.Command(
+const pruneOldCacheWasmCommand = new command.local.Command(
'prune-old-cache-wasm-command',
{
create: pulumi.interpolate`bash ../../../apps/web/scripts/cache-wasm/prune-old-brotli-from-s3.sh ./output/app s3://${webAppAssets.bucket}/app ${cacheWasmRetentionDays}`,
@@ -156,6 +166,21 @@ new command.local.Command(
}
);
+// What the generic sync's --delete used to do, delayed by a few builds. Runs
+// last, and after the cache WASM prune, so a failure anywhere above leaves
+// every prior key in place rather than half-retiring a build.
+new command.local.Command(
+ 'retain-recent-builds-command',
+ {
+ create: pulumi.interpolate`bash ../../../apps/web/scripts/retain-recent-builds.sh ./output/app s3://${webAppAssets.bucket}/app ${assetBuildRetention}`,
+ triggers: [Date.now()],
+ },
+ {
+ dependsOn: [pruneOldCacheWasmCommand],
+ replaceOnChanges: ['*'],
+ }
+);
+
// First, create an IAM role and attach the AWSLambdaBasicExecutionRole policy
const lambdaRole = new aws.iam.Role('content-encoding-header-lambda-role', {
assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({