Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

CLI: cache the Store catalog as published, not as this CLI parsed it

deehwb7b58a8b53aa
diff --git a/cli/src/dsh/catalog.spec.ts b/cli/src/dsh/catalog.spec.ts
index 33b4a015f..7a3f359a2 100644
--- a/cli/src/dsh/catalog.spec.ts
+++ b/cli/src/dsh/catalog.spec.ts
@@ -77,6 +77,61 @@ describe('live Store catalog', () => {
expect(readFileSync(join(root, 'catalog.json'), 'utf8')).toBe(before)
})
+ it('keeps the catalog on disk as published, so a newer CLI reads the fields this one drops', async () => {
+ const published = { ...first, tagline: 'Arcade games', futureField: 'kept for the next CLI' }
+ fetcher.mockResolvedValueOnce(new Response(JSON.stringify(document([published as DshRegistryEntry, viewer])), { headers: { etag: '"v1"' } }))
+ .mockResolvedValueOnce(new Response(null, { status: 304 }))
+ const catalog = make()
+ await catalog.refresh()
+ expect(catalog.current().find(e => e.id === first.id)).not.toHaveProperty('futureField')
+ const file = join(root, 'catalog.json')
+ const onDisk = () => JSON.parse(readFileSync(file, 'utf8'))
+ expect(onDisk()).toMatchObject({ whole: true, etag: '"v1"', catalog: { spec: 1, entries: [published, viewer] } })
+ // Restarted, it trusts that copy: the ETag is sent, a 304 keeps it, and it is still whole on disk.
+ const restarted = make()
+ expect(restarted.current().find(e => e.id === first.id)?.tagline).toBe('Arcade games')
+ now += CATALOG_REFRESH_MS
+ await restarted.refresh()
+ expect(fetcher.mock.calls[1][1]?.headers).toMatchObject({ 'if-none-match': '"v1"' })
+ expect(onDisk()).toMatchObject({ whole: true, checkedAt: now, catalog: { entries: [published, viewer] } })
+ })
+
+ it('a catalog served without an ETag is saved without one and downloaded in full each time', async () => {
+ fetcher.mockImplementation(async () => new Response(JSON.stringify(document())))
+ await make().refresh()
+ const saved = JSON.parse(readFileSync(join(root, 'catalog.json'), 'utf8'))
+ expect(saved).toMatchObject({ whole: true, catalog: document() })
+ expect(saved).not.toHaveProperty('etag')
+ const restarted = make()
+ now += CATALOG_REFRESH_MS
+ await restarted.refresh()
+ expect(fetcher).toHaveBeenCalledTimes(2)
+ expect(fetcher.mock.calls[1][1]?.headers).not.toHaveProperty('if-none-match')
+ })
+
+ it('downloads the whole catalog again, not revalidating, over a cache an older CLI cut down', async () => {
+ // What a CLI before `whole` wrote: its own parse of the catalog, fields it did not know already gone.
+ const file = join(root, 'catalog.json')
+ const cutDown = { url: 'https://catalog.example.test/catalog.json', checkedAt: now, etag: '"v1"', catalog: document() }
+ writeFileSync(file, JSON.stringify(cutDown))
+ const catalog = make()
+ expect(catalog.current().map(e => e.id)).toEqual([first.id, viewer.id])
+ expect(catalog.current()[0].tagline).toBeUndefined()
+
+ // A 304 it did not ask for changes nothing, and the next refresh tries again.
+ fetcher.mockResolvedValueOnce(new Response(null, { status: 304 }))
+ await catalog.refresh()
+ expect(fetcher.mock.calls[0][1]?.headers).not.toHaveProperty('if-none-match')
+ expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual(cutDown)
+
+ now += 30_000
+ fetcher.mockResolvedValueOnce(new Response(JSON.stringify(document([{ ...first, tagline: 'Arcade games' }, viewer])), { headers: { etag: '"v1"' } }))
+ expect((await catalog.refresh()).find(e => e.id === first.id)?.tagline).toBe('Arcade games')
+ expect(fetcher).toHaveBeenCalledTimes(2)
+ expect(fetcher.mock.calls[1][1]?.headers).not.toHaveProperty('if-none-match')
+ expect(JSON.parse(readFileSync(file, 'utf8'))).toMatchObject({ whole: true, etag: '"v1"' })
+ })
+
it('falls back on first offline launch or a corrupt cache', async () => {
writeFileSync(join(root, 'catalog.json'), '{bad')
fetcher.mockRejectedValue(new Error('offline'))
diff --git a/cli/src/dsh/catalog.ts b/cli/src/dsh/catalog.ts
index fc6acb739..b1465682f 100644
--- a/cli/src/dsh/catalog.ts
+++ b/cli/src/dsh/catalog.ts
@@ -39,7 +39,13 @@ export function parseStoreCatalog(value: unknown): StoreCatalog {
return { spec: 1, entries: entries.sort((a, b) => a.id.localeCompare(b.id)) }
}
-interface Cache { url: string; checkedAt: number; etag?: string; catalog: StoreCatalog }
+/**
+ * [document] is the catalog exactly as published, and it is what goes to disk: [catalog] is that
+ * document parsed by THIS CLI, which drops every field it does not know. A cache of the parsed copy
+ * outlived the CLI that wrote it — upgraded, the next one sent the saved ETag, heard 304 and kept a
+ * catalog cut down to the old CLI's fields (no examples, then no taglines) until the Store next changed.
+ */
+interface Cache { url: string; checkedAt: number; etag?: string; catalog: StoreCatalog; document: unknown }
interface CatalogOptions {
url: string
cacheFile: string
@@ -62,12 +68,15 @@ export class LiveStoreCatalog {
this.fetcher = options.fetch ?? fetch
try {
if (statSync(options.cacheFile).size > MAX_BYTES) return
- const saved = JSON.parse(readFileSync(options.cacheFile, 'utf8')) as Cache
- if (saved.url !== options.url || !Number.isFinite(saved.checkedAt)) return
+ const saved = JSON.parse(readFileSync(options.cacheFile, 'utf8')) as Record<string, unknown>
+ if (saved.url !== options.url || typeof saved.checkedAt !== 'number' || !Number.isFinite(saved.checkedAt)) return
const catalog = parseStoreCatalog(saved.catalog)
- this.cache = { url: saved.url, checkedAt: Math.min(saved.checkedAt, this.now()), catalog,
- ...(typeof saved.etag === 'string' && saved.etag.length < 500 ? { etag: saved.etag } : {}) }
- this.nextAttempt = this.cache.checkedAt + CATALOG_REFRESH_MS
+ // A cache from before `whole` holds only what the CLI that wrote it knew. Serve it, but download
+ // the whole document at the first refresh rather than revalidating a copy that is missing fields.
+ const whole = saved.whole === true
+ this.cache = { url: saved.url, checkedAt: Math.min(saved.checkedAt, this.now()), catalog, document: saved.catalog,
+ ...(whole && typeof saved.etag === 'string' && saved.etag.length < 500 ? { etag: saved.etag } : {}) }
+ this.nextAttempt = whole ? this.cache.checkedAt + CATALOG_REFRESH_MS : 0
} catch { /* First launch, unreadable cache, or old format: use the bundled shelf. */ }
}
@@ -87,7 +96,8 @@ export class LiveStoreCatalog {
signal: AbortSignal.timeout(this.options.timeoutMs ?? 4_000),
redirect: 'error',
})
- if (response.status === 304 && this.cache) {
+ // Only an answer to our own If-None-Match: a copy we did not ask about is never "not modified".
+ if (response.status === 304 && this.cache?.etag) {
this.cache = { ...this.cache, checkedAt: this.now() }
} else {
if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`)
@@ -105,16 +115,19 @@ export class LiveStoreCatalog {
chunks.push(value)
}
} finally { await reader.cancel().catch(() => {}) }
- const catalog = parseStoreCatalog(JSON.parse(Buffer.concat(chunks).toString('utf8')))
+ const document: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'))
+ const catalog = parseStoreCatalog(document)
const etag = response.headers.get('etag')
- this.cache = { url: this.options.url, checkedAt: this.now(), catalog, ...(etag && etag.length < 500 ? { etag } : {}) }
+ this.cache = { url: this.options.url, checkedAt: this.now(), catalog, document, ...(etag && etag.length < 500 ? { etag } : {}) }
}
this.nextAttempt = this.now() + CATALOG_REFRESH_MS
// A read-only disk must not discard metadata that was fetched successfully.
try {
mkdirSync(dirname(this.options.cacheFile), { recursive: true, mode: 0o700 })
const temporary = `${this.options.cacheFile}.${randomUUID()}.tmp`
- writeFileSync(temporary, JSON.stringify(this.cache), { mode: 0o600 })
+ const { url, checkedAt, etag, document } = this.cache
+ // `catalog` keeps its name so an older CLI still reads this file (and parses it for itself).
+ writeFileSync(temporary, JSON.stringify({ url, checkedAt, ...(etag ? { etag } : {}), whole: true, catalog: document }), { mode: 0o600 })
renameSync(temporary, this.options.cacheFile)
} catch { /* The in-memory catalog still works. */ }
} catch {