Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(web): delete retired assets in batches on deploy (#9698)
teo7de7083b7ee5
diff --git a/apps/web/scripts/retain-recent-builds.sh b/apps/web/scripts/retain-recent-builds.shindex 6276467e0c..cd6b9f91a5 100644--- a/apps/web/scripts/retain-recent-builds.sh+++ b/apps/web/scripts/retain-recent-builds.sh@@ -121,6 +121,8 @@ aws "${list_args[@]}" |# not own - the records themselves, the archive uploaded as its own resource,# and the cache WASM, which `prune-old-brotli-from-s3.sh` retires on its own# schedule because the generic sync never uploads it.+prunable=$work/prunable+: >"$prunable"while IFS= read -r key; do[ -n "$key" ] || continuecase "$key" in@@ -129,5 +131,22 @@ while IFS= read -r key; do*cache_wasm_bg*) continue ;;*index.html) continue ;;esac- aws s3 rm "s3://$bucket/$key" >/dev/null+ printf '%s\n' "$key" >>"$prunable"done < <(LC_ALL=C comm -23 "$existing" "$retained")++# One request per 1000 keys, the DeleteObjects limit: starting the CLI once per+# key took minutes on every deploy. This call exits 0 even when S3 refuses keys,+# so the response has to be read for the failure to stop the deploy.+if [ -s "$prunable" ]; then+ split -l 1000 "$prunable" "$work/batch."+fi+for batch in "$work"/batch.*; do+ [ -e "$batch" ] || continue+ jq -Rn '{Objects: [inputs | {Key: .}], Quiet: true}' <"$batch" >"$batch.json"+ errors=$(aws s3api delete-objects --bucket "$bucket" --delete "file://$batch.json" \+ --query 'Errors' --output json)+ if [ "$errors" != "null" ] && [ "$errors" != "[]" ]; then+ echo "S3 refused to delete some keys: $errors" >&2+ exit 1+ fi+donediff --git a/apps/web/scripts/retain-recent-builds.test.ts b/apps/web/scripts/retain-recent-builds.test.tsindex 2c8aa4114a..dea82a048b 100644--- a/apps/web/scripts/retain-recent-builds.test.ts+++ b/apps/web/scripts/retain-recent-builds.test.ts@@ -25,6 +25,8 @@ type Bucket = {records: Record<string, string[]>;/** Every key currently in the bucket under the prefix. */objects: string[];+ /** Keys S3 refuses to delete, reported in a response that still exits 0. */+ refuse?: string[];};/**@@ -43,6 +45,7 @@ function run(const state = join(root, 'bucket.json');const deletions = join(root, 'deleted.txt');const uploads = join(root, 'uploaded.txt');+ const batches = join(root, 'batches.txt');mkdirSync(bin, { recursive: true });for (const [path, contents] of Object.entries(distFiles)) {const full = join(dist, path);@@ -52,8 +55,9 @@ function run(writeFileSync(state, JSON.stringify(bucket));writeFileSync(deletions, '');writeFileSync(uploads, '');+ writeFileSync(batches, '');- // Only the three shapes the script uses: list keys, read a record, delete.+ // Only the shapes the script uses: list keys, read a record, delete.const fakeAws = join(bin, 'aws');writeFileSync(fakeAws,@@ -66,8 +70,11 @@ if (argv[0] === 's3api' && argv[1] === 'list-objects-v2') {const keys = prefix.endsWith('_builds/')? Object.keys(bucket.records).map((build) => prefix + build + '.txt'): bucket.objects.filter((key) => key.startsWith(prefix));- process.stdout.write(keys.length ? keys.join('\\t') + '\\n' : 'None\\n');- process.exit(0);+ // Exit after the write: process.exit() truncates output to a pipe.+ process.stdout.write(keys.length ? keys.join('\\t') + '\\n' : 'None\\n', () =>+ process.exit(0)+ );+ return;}if (argv[0] === 's3' && argv[1] === 'cp' && argv[3] === '-') {const build = argv[2].replace(/.*\\/(\\d+)\\.txt$/, '$1');@@ -86,6 +93,24 @@ if (argv[0] === 's3' && argv[1] === 'rm') {appendFileSync(${JSON.stringify(deletions)}, argv[2] + '\\n');process.exit(0);}+if (argv[0] === 's3api' && argv[1] === 'delete-objects') {+ const path = argv[argv.indexOf('--delete') + 1].replace('file://', '');+ const keys = JSON.parse(readFileSync(path, 'utf8')).Objects.map((o) => o.Key);+ appendFileSync(${JSON.stringify(batches)}, keys.length + '\\n');+ const refused = keys.filter((key) => (bucket.refuse ?? []).includes(key));+ for (const key of keys) {+ if (!refused.includes(key)) {+ appendFileSync(${JSON.stringify(deletions)}, 's3://example/' + key + '\\n');+ }+ }+ // S3 reports a refused key in the response and the CLI still exits 0.+ process.stdout.write(+ refused.length+ ? JSON.stringify(refused.map((Key) => ({ Key, Code: 'AccessDenied' })))+ : 'null'+ );+ process.exit(0);+}process.stderr.write('unexpected aws call: ' + argv.join(' ') + '\\n');process.exit(1);`@@ -106,6 +131,7 @@ process.exit(1);result,deleted: lines(deletions).map((uri) => uri.replace('s3://example/', '')),uploaded: lines(uploads),+ batchSizes: lines(batches).map(Number),};}@@ -226,6 +252,48 @@ describe('retaining recent builds', () => {expect(deleted).not.toContain('app/index.html');});+ it('deletes in batches no larger than S3 accepts', () => {+ const orphans = Array.from({ length: 2500 }, (_, i) => `app/orphan-${i}.js`);+ const { result, deleted, batchSizes } = run(+ { 'bundle-manifest.json': manifest(300), 'index.html': '' },+ { records: {}, objects: ['app/index.html', ...orphans] },+ 1+ );++ expect(result.status).toBe(0);+ expect(batchSizes).toEqual([1000, 1000, 500]);+ expect(deleted.toSorted()).toEqual(orphans.toSorted());+ });++ it('sends no delete request when nothing is prunable', () => {+ const { result, batchSizes } = run(+ { 'bundle-manifest.json': manifest(300), 'index.html': '' },+ { records: {}, objects: ['app/index.html'] },+ 1+ );++ expect(result.status).toBe(0);+ expect(batchSizes).toEqual([]);+ });++ it('fails the deploy when S3 refuses to delete a key', () => {+ const { result, deleted } = run(+ { 'bundle-manifest.json': manifest(300), 'index.html': '' },+ {+ records: {},+ objects: ['app/index.html', 'app/a.js', 'app/b.js'],+ refuse: ['app/b.js'],+ },+ 1+ );++ // DeleteObjects exits 0 with per-key errors; the old per-key call failed+ // the script, so a refusal must not start passing silently.+ expect(result.status).not.toBe(0);+ expect(result.stderr).toContain('app/b.js');+ expect(deleted).toEqual(['app/a.js']);+ });+it('refuses to prune output that is not a build', () => {const { result, deleted } = run({ 'bundle-manifest.json': manifest(300) },