Bento, the office suite that fits in a file
TypeScript mit-licenseofficeoffice-365office-suiteoffice-toolsoffice365open-sourcepowerpointself-hostedsingle-page-appslidesslideshareslideshowslideshow-galleryslideshow-makerslideshows
home/webext DocStore: a late write never replaces a newer one; orphaned chunks are swept (#670)
nyblnetd1a9265c04b7
diff --git a/home/webext/src/store.js b/home/webext/src/store.jsindex fac8f12a..8e2793f6 100644--- a/home/webext/src/store.js+++ b/home/webext/src/store.js@@ -25,7 +25,8 @@// TRANSPORT. Runtime messaging is JSON, so bytes cross the relay↔worker hop// as base64, and a value over CHUNK bytes travels as numbered chunks under a// transaction id, committed in one step: a reader sees the old value or the-// new one, never half. Every message re-derives the partition and holds no+// new one, never half. A transfer that began before the value now committed+// is refused as 'superseded', so a late write never replaces a newer one. Every message re-derives the partition and holds no// state between messages (the worker is evicted at will); chunks of a// transfer that never committed are swept by `gc`.//@@ -127,7 +128,10 @@ export async function put(path, p, deps) {if (b64Len(p.data) > CHUNK) return { ok: false, reason: 'chunk too large' }if (p.i * CHUNK >= MAX_VALUE) return { ok: false, reason: 'too large' }await deps.db.put(cKey(path, p.name, p.tx, p.i), p.data)- await deps.db.put(`t${S}${path}${S}${p.name}${S}${p.tx}`, { at: deps.now() }) // for the stale sweep+ // `start` (first chunk) orders transfers for commit; `at` (latest chunk) is for the stale sweep+ const tk = `t${S}${path}${S}${p.name}${S}${p.tx}`+ const t = await deps.db.get(tk)+ await deps.db.put(tk, { start: t?.start ?? deps.now(), at: deps.now() })return { ok: true }}@@ -143,6 +147,14 @@ export async function commit(path, p, deps) {}if (size > MAX_VALUE) { await dropTx(path, p.name, p.tx, deps); return { ok: false, reason: 'too large' } }const old = await deps.db.get(mKey(path, p.name))+ // A LATE transfer — one that began before the value now committed, e.g. a+ // write the page gave up waiting for and has since sent again — must not+ // replace the newer value. Refused, and its chunks dropped; the newer one stands.+ const begun = (await deps.db.get(`t${S}${path}${S}${p.name}${S}${p.tx}`))?.start+ if (old && old.tx !== p.tx && begun != null && (old.at ?? 0) > begun) {+ await dropTx(path, p.name, p.tx, deps)+ return { ok: false, reason: 'superseded' }+ }await deps.db.put(mKey(path, p.name), { tx: p.tx, n: p.n, size, at: deps.now() })await deps.db.del(`t${S}${path}${S}${p.name}${S}${p.tx}`)if (old && old.tx !== p.tx) await dropTx(path, p.name, old.tx, deps)@@ -330,6 +342,15 @@ export async function gc(deps) {await deps.db.del(`x${S}${path}`)dropped.push({ path, why })}+ // Chunks no manifest points to and no live transfer owns: left when two+ // commits interleaved (each dropped the value it read as old, not the other's).+ const live = new Set((await deps.db.range(`t${S}`, `t${S}${END}`)).map(([k]) => k.slice(2)))+ for (const [k] of await deps.db.range(`c${S}`, `c${S}${END}`)) {+ const [, path, name, tx] = k.split(S)+ if (live.has(`${path}${S}${name}${S}${tx}`)) continue+ const m = await deps.db.get(mKey(path, name))+ if (!m || m.tx !== tx) await deps.db.del(k)+ }for (const [k, v] of await deps.db.range(`t${S}`, `t${S}${END}`)) {if ((v?.at ?? 0) < now - STALE_TX_MS) {const [, path, name, tx] = k.split(S)diff --git a/scripts/test-webext-store.ts b/scripts/test-webext-store.tsindex 3aab540c..4a5071e3 100644--- a/scripts/test-webext-store.ts+++ b/scripts/test-webext-store.ts@@ -103,6 +103,47 @@ console.log('\n— chunked values: committed whole or not at all')ok((await bg.storeOp('store.put', A, { name: 'memberkey', tx, i: 0, data: st.toB64(new Uint8Array(st.CHUNK + 1)) }, d)).reason === 'chunk too large', 'a chunk over CHUNK is refused')}+console.log('\n— late and interleaved writes (a page that gave up waiting sends again)')+{+ const d = deps()+ // a slow transfer begins, the page times out and sends a fresh value, which commits first+ await bg.storeOp('store.put', A, { name: 'recovery', tx: 'tx-slow-0001', i: 0, data: enc('OLD ') }, d)+ clock += 1000+ ok((await bg.storeOp('store.set', A, { name: 'recovery', data: enc('NEW') }, d)).ok, 'the resend commits')+ clock += 1000+ await bg.storeOp('store.put', A, { name: 'recovery', tx: 'tx-slow-0001', i: 1, data: enc('VALUE') }, d)+ const late = await bg.storeOp('store.commit', A, { name: 'recovery', tx: 'tx-slow-0001', n: 2 }, d)+ ok(late.ok === false && late.reason === 'superseded', 'the late transfer, begun before the committed value, is refused as superseded')+ ok(dec((await bg.storeOp('store.get', A, { name: 'recovery' }, d)).inline) === 'NEW', 'and the newer value stands')+ ok(![...d.db.m.keys()].some((k) => k.includes('tx-slow-0001')), 'the refused transfer leaves no chunks')+ clock += 1000+ ok((await bg.storeOp('store.set', A, { name: 'recovery', data: enc('NEWER') }, d)).ok && dec((await bg.storeOp('store.get', A, { name: 'recovery' }, d)).inline) === 'NEWER', 'ordinary successive writes still replace each other')+}+{+ // two commits that read the same "old" value: each drops that one, neither the other's+ const d = deps()+ await bg.storeOp('store.set', A, { name: 'recovery', data: enc('X') }, d)+ const db = d.db+ const realGet = db.get+ let held: any = null+ // make commit #1 read the manifest, then let commit #2 run to completion before #1 continues+ for (const tx of ['tx-race-aaaa', 'tx-race-bbbb']) await bg.storeOp('store.put', A, { name: 'recovery', tx, i: 0, data: enc(tx) }, d)+ let gate: any+ const paused = new Promise((r) => { gate = r })+ db.get = async (k: string) => { const v = await realGet(k); if (!held && k.startsWith('m\u0000') && k.endsWith('recovery')) { held = v; await paused } return v }+ const first = bg.storeOp('store.commit', A, { name: 'recovery', tx: 'tx-race-aaaa', n: 1 }, d)+ await new Promise((r) => setTimeout(r, 0))+ db.get = realGet+ await bg.storeOp('store.commit', A, { name: 'recovery', tx: 'tx-race-bbbb', n: 1 }, d)+ gate()+ await first+ clock += 2 * 3600 * 1000+ await st.gc(d)+ const m = await db.get(`m\u0000${'/Users/you/Decks/Q3.bento.html'}\u0000recovery`)+ const chunks = [...db.m.keys()].filter((k) => k.startsWith('c\u0000'))+ ok(chunks.length === m.n && chunks.every((k) => k.includes(m.tx)), `after the sweep, only the committed value's chunks remain (${chunks.length})`)+}+console.log('\n— the sweep'){const DAY = 24 * 3600 * 1000