Skip to content

The agent that grows with you

fix(webapp): keep the previous bundle's lazy chunks across a rebuild

dokterdokf22598136dbf
diff --git a/scripts/build/freshness.mjs b/scripts/build/freshness.mjs
index 089dd4626fb..22c6721c343 100644
--- a/scripts/build/freshness.mjs
+++ b/scripts/build/freshness.mjs
@@ -102,10 +102,10 @@ export function sourceEntries(source, product) {
return entries
}
-function outputHash(out) {
+function outputHash(out, ignored = []) {
// Native binaries can be signed after compilation. Their ABI validation is
// owned by native preparation; renderer/main/preload bytes must stay intact.
- return treeHash(out, readdirSync(out).sort(), name => name === receiptName || name === '.hermes-product',
+ return treeHash(out, readdirSync(out).sort(), name => name === receiptName || name === '.hermes-product' || ignored.includes(name),
name => !name.split('/').includes('node_modules') && !name.startsWith('native/'))
}
@@ -201,6 +201,16 @@ export function recordProduct({ source, product, out, inputs, stampClock }) {
}) + '\n')
}
+/** Validate output bytes against their receipt, independently of current inputs or host, and of
+ * the marker ``hermes update`` leaves in a product it could not rebuild (hermes_cli/webapp.py),
+ * which still fails productCurrent. */
+export function outputIntact(out, product) {
+ try {
+ const saved = JSON.parse(readFileSync(join(out, receiptName), 'utf8'))
+ return saved.schema === 1 && saved.product === product && saved.outputHash === outputHash(out, ['.hermes-stale'])
+ } catch { return false }
+}
+
/** The receipt in ``out`` when it was written for ``product`` by this platform, arch and Node. */
function hostReceipt(out, product) {
const saved = JSON.parse(readFileSync(join(out, receiptName), 'utf8'))
diff --git a/scripts/build/webapp.mjs b/scripts/build/webapp.mjs
index 4464bf6e2f3..85365fcb771 100644
--- a/scripts/build/webapp.mjs
+++ b/scripts/build/webapp.mjs
@@ -1,11 +1,58 @@
#!/usr/bin/env node
// Pure browser-renderer compilation. The workspace is prepared by the caller
// (apps/desktop/scripts/build-webapp.mjs); the receipt certifies the checkout.
-import { existsSync, readdirSync } from 'node:fs'
-import { join, relative, resolve } from 'node:path'
+import { constants, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync } from 'node:fs'
+import { dirname, isAbsolute, join, relative, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { productOutput, withProduct, workspaceTool } from './frontend-common.mjs'
-import { buildInputs, recordProduct, sourceHash } from './freshness.mjs'
+import { buildInputs, outputIntact, recordProduct, sourceHash } from './freshness.mjs'
+
+// Vite's build manifest (apps/desktop/vite.config.ts names it too): the files a generation emitted.
+const manifestName = 'renderer-manifest.json'
+
+/** Keep in ``next`` the lazy chunks an open tab of the previous generation still loads: only missing
+ * regular files from a generation intact against its receipt, never through a symlink or over a file.
+ * A manifest omits the chunks its generation inherited, so retention stays one generation back. */
+export function keepPreviousGeneration(previous, next) {
+ let root
+ let chunks
+ try {
+ if (!outputIntact(previous, 'webapp')) return []
+ root = realpathSync(previous)
+ chunks = Object.values(JSON.parse(readFileSync(join(previous, manifestName), 'utf8')) ?? {})
+ } catch {
+ return [] // no previous generation, or one without a manifest
+ }
+ const list = value => (Array.isArray(value) ? value : [])
+ const names = new Set(chunks.flatMap(chunk => [chunk?.file, ...list(chunk?.css), ...list(chunk?.assets)]))
+ const kept = []
+ for (const name of names) {
+ if (typeof name !== 'string' || !name || isAbsolute(name) || name.split(/[\\/]/).includes('..')) continue
+ const from = join(previous, name)
+ const to = join(next, name)
+ try {
+ // A symlink anywhere on the way would resolve elsewhere than the name it was listed under.
+ if (existsSync(to) || !lstatSync(from).isFile() || relative(root, realpathSync(from)) !== join(name)) continue
+ } catch {
+ continue // missing, or not a name this filesystem can hold
+ }
+ try {
+ mkdirSync(dirname(to), { recursive: true })
+ } catch {
+ continue // e.g. a new file stands where the name's directory would go
+ }
+ try {
+ copyFileSync(from, to, constants.COPYFILE_EXCL)
+ kept.push(name)
+ } catch (error) {
+ // Only ever this copy's own partial file (EEXIST: one stood there after all).
+ try {
+ if (error?.code !== 'EEXIST') rmSync(to, { force: true })
+ } catch { /* a failed cleanup must not fail the build either */ }
+ }
+ }
+ return kept
+}
export async function buildWebapp({ source, workspace, out }) {
if (!workspace) throw new Error('A prepared Webapp workspace is required')
@@ -28,9 +75,11 @@ export async function buildWebapp({ source, workspace, out }) {
root: join(workspace, app),
configLoader: 'runner',
cacheDir: join(scratch, 'vite-cache'),
- build: { outDir: product, emptyOutDir: true },
+ build: { outDir: product, emptyOutDir: true, manifest: manifestName },
})
if (!existsSync(join(product, 'index.html'))) throw new Error('Webapp build did not produce index.html')
+ // Before the receipt, whose output hash covers the kept files too.
+ keepPreviousGeneration(out, product)
recordProduct({ source, product: 'webapp', out: product, inputs })
}, { source })
return { out, index: join(out, 'index.html') }
diff --git a/tests-js/product-builders.test.mjs b/tests-js/product-builders.test.mjs
index e53d868591d..fbe64095625 100644
--- a/tests-js/product-builders.test.mjs
+++ b/tests-js/product-builders.test.mjs
@@ -1,5 +1,5 @@
import { execFileSync } from 'node:child_process'
-import { copyFileSync, mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, symlinkSync, statSync, utimesSync } from 'node:fs'
+import { copyFileSync, mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, rmSync, symlinkSync, statSync, utimesSync } from 'node:fs'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { createRequire } from 'node:module'
@@ -446,6 +446,60 @@ test('the Webapp receipt certifies its checkout inputs, not the workspace it com
expect(current()).toBe(false)
}, 60_000)
+test('a Webapp rebuild keeps the previous generation for tabs that booted it, never an older one', async () => {
+ const { productCurrent } = await import('../scripts/build/freshness.mjs')
+ const { buildWebapp, keepPreviousGeneration } = await import('../scripts/build/webapp.mjs')
+ const { mirrorWebappInputs } = await import('../apps/desktop/scripts/build-webapp.mjs')
+ const base = fixture()
+ const source = path.join(base, 'source')
+ const workspace = path.join(base, 'workspace')
+ const out = path.join(source, 'apps/desktop/dist-webapp')
+ webappSource(source)
+ dependency(workspace, '', 'vite')
+ const chunk = marker => readdirSync(path.join(out, 'assets'))
+ .find(name => readFileSync(path.join(out, 'assets', name), 'utf8').includes(marker))
+ for (const generation of ['gen-one', 'gen-two', 'gen-three']) {
+ // The lazy chunk an open tab loads only when it needs it.
+ put(source, 'apps/desktop/src/lazy.ts', `export const text = '${generation}-marker'`)
+ put(source, 'apps/desktop/src/main.ts', 'import("./lazy").then(m => { document.body.textContent = m.text })')
+ mirrorWebappInputs(source, workspace)
+ await buildWebapp({ source, workspace, out })
+ expect(productCurrent({ source, product: 'webapp', out }), generation).toBe(true)
+ if (generation === 'gen-two') {
+ // An update that could not rebuild gen-two marks it (hermes_cli/webapp.py mark_webapp_stale):
+ // the marker still forces the rebuild, and must not cost gen-two's tabs their chunks.
+ writeFileSync(path.join(out, '.hermes-stale'), 'hermes update could not rebuild this bundle\n')
+ expect(productCurrent({ source, product: 'webapp', out })).toBe(false)
+ }
+ }
+ // A tab booted on gen-two still finds its chunk after gen-three published; gen-one's is gone.
+ expect(chunk('gen-three-marker')).toBeTruthy()
+ expect(chunk('gen-two-marker')).toBeTruthy()
+ expect(chunk('gen-one-marker')).toBeUndefined()
+ // Only regular files inside an intact previous generation are kept, never over a new file, and a
+ // bad entry is skipped rather than failing the build.
+ const { buildInputs, outputIntact, recordProduct } = await import('../scripts/build/freshness.mjs')
+ const [previous, next, outside] = [fixture(), fixture(), fixture()]
+ put(outside, 'secret.txt', 'outside')
+ symlinkSync(outside, path.join(previous, 'linked'), 'junction')
+ // ``x`` is a directory in the previous generation and a file in the new one.
+ for (const [name, content] of [['index.html', 'old'], ['assets/ok.js', 'old chunk'], ['assets/same.js', 'old same'], ['x/kept.js', 'old']]) put(previous, name, content)
+ for (const [name, content] of [['index.html', 'new'], ['assets/same.js', 'new same'], ['assets/only-new.js', 'new'], ['x', 'new file']]) put(next, name, content)
+ put(previous, 'renderer-manifest.json', JSON.stringify({ a: { file: 'assets/ok.js', css: ['../escape.css', '/abs.css'],
+ assets: ['index.html', 'assets/same.js', 'assets/only-new.js', 'linked/secret.txt', 'bad\0name.js', 'assets/ok.js/child', 'x/kept.js'] } }))
+ const keep = () => keepPreviousGeneration(previous, next)
+ recordProduct({ source: base, product: 'webapp', out: previous, inputs: buildInputs(base, 'webapp') })
+ expect(keep()).toEqual(['assets/ok.js'])
+ expect(['index.html', 'assets/same.js', 'assets/only-new.js', 'x'].map(name => readFileSync(path.join(next, name), 'utf8'))).toEqual(['new', 'new same', 'new', 'new file'])
+ expect(existsSync(path.join(next, 'linked'))).toBe(false)
+ recordProduct({ source: base, product: 'webapp', out: next, inputs: buildInputs(base, 'webapp') })
+ expect(outputIntact(next, 'webapp')).toBe(true)
+ // A previous generation whose bytes no longer match its receipt gives nothing.
+ rmSync(path.join(next, 'assets/ok.js'))
+ put(previous, 'assets/ok.js', 'damaged')
+ expect(keep()).toEqual([])
+}, 120_000)
+
test('Webapp builds install privately and never disturb the native Desktop installation', async () => {
const { buildSourceWebapp } = await import('../apps/desktop/scripts/build-webapp.mjs')
const base = fixture()