Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Load the Harness Store catalog independently of CLI releases

deed2fbfce94171
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e618b2def..8f9a1ef02 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -26,7 +26,7 @@ cd ../my-first-harness
```
The first command links the shared viewer from this checkout. Store installations normally install
-that dependency automatically; this also lets you try a new viewer before it reaches a release.
+that dependency automatically; this also lets you try a new viewer before it is published.
**Agent + viewer = harness.** This example has three working files:
@@ -118,10 +118,13 @@ You do not need permission to build a harness or share its repository. Choose th
submit a small entry at `store/registry/<owner>/<name>.json`. You keep the code and its maintenance.
- **This repository:** contribute a folder at `store/agents/<name>/` with a `harness.json` and
`store.json`. Built-in package IDs use `autonomous/<name>`; the `author` field credits the actual
- author or upstream project. The CLI build generates the listing from those files.
+ author or upstream project. The catalog publisher generates the listing from those files.
The [publishing examples](store/README.md#publish-a-harness) show the exact metadata for both routes.
-A Store listing reaches users with a CLI release after it is merged.
+A merge to `main` publishes the live catalog automatically. Running clients pick it up within
+minutes; no app or CLI release is needed for a new harness or viewer that uses the existing
+package format. The catalog describes packages; installation still happens when someone chooses
+Get. Installed packages are not silently replaced.
For a first pull request, give reviewers a short path to the same result you saw:
@@ -165,8 +168,8 @@ Security reports go through [SECURITY.md](SECURITY.md).
currently run manually through **Actions → CI → Run workflow**; a PR does not automatically
exercise the app, real engines, or hardware. Report those checks separately.
5. **Merge and release.** PRs are squash-merged. Rebase on the latest `main` when needed to keep
- the diff readable. Merging and publishing are separate: app, CLI, and firmware releases have
- their own schedules.
+ the diff readable. Harness catalog changes publish automatically after merge. App, CLI, and
+ firmware releases have their own schedules.
## Conventions across this repository
diff --git a/cli/scripts/lib/dshRegistry.mjs b/cli/scripts/lib/dshRegistry.mjs
index 33734fe2f..cfe95eb3d 100644
--- a/cli/scripts/lib/dshRegistry.mjs
+++ b/cli/scripts/lib/dshRegistry.mjs
@@ -22,17 +22,19 @@ export function storeEntry(path, manifest, facts) {
return entry
}
-function readStoreDir(storeDir) {
+function readStoreDir(storeDir, strict) {
const out = []
for (const plural of ['agents', 'viewers']) {
let names
try { names = readdirSync(join(storeDir, plural)).sort() } catch { continue }
for (const name of names) {
const dir = join(storeDir, plural, name)
+ if (!statSync(dir).isDirectory() || name.startsWith('.')) continue
let manifest
- try { manifest = JSON.parse(readFileSync(join(dir, 'harness.json'), 'utf8')) } catch { continue }
+ try { manifest = JSON.parse(readFileSync(join(dir, 'harness.json'), 'utf8')) } catch (error) { if (strict) throw error; continue }
+ if (strict && (manifest.spec !== 1 || manifest.id !== `autonomous/${name}` || (manifest.kind ?? 'agent') !== (plural === 'agents' ? 'agent' : 'viewer'))) throw new Error(`Invalid package identity: ${dir}`)
let facts = {}
- try { facts = JSON.parse(readFileSync(join(dir, 'store.json'), 'utf8')) } catch { facts = {} }
+ try { facts = JSON.parse(readFileSync(join(dir, 'store.json'), 'utf8')) } catch (error) { if (strict) throw error; facts = {} }
out.push(storeEntry(`store/${plural}/${name}`, manifest, facts))
}
}
@@ -61,7 +63,7 @@ function readRegistryDir(root) {
}
/** `storeDir` is the repo's `store/` folder, as a path or a file URL. */
-export function readDshRegistry(storeDir) {
+export function readDshRegistry(storeDir, { strict = false } = {}) {
const root = storeDir instanceof URL ? fileURLToPath(storeDir) : storeDir
- return [...readStoreDir(root), ...readRegistryDir(join(root, 'registry'))]
+ return [...readStoreDir(root, strict), ...readRegistryDir(join(root, 'registry'))]
}
diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.ts
index b16ce9a6c..70703e10d 100644
--- a/cli/src/backendSocket.spec.ts
+++ b/cli/src/backendSocket.spec.ts
@@ -10,6 +10,7 @@ import { registry, type RegisteredSession } from './lib/registry.js'
import * as mediaPreview from './lib/mediaPreview.js'
import * as projectFolder from './lib/projectFolder.js'
import * as projectPreview from './lib/projectPreview.js'
+import * as storeCatalog from './dsh/catalog.js'
import { randomUUID } from 'node:crypto'
const wsMock = vi.hoisted(() => {
@@ -680,6 +681,25 @@ describe('BackendSocket outbound queue', () => {
await socket.stop()
})
+ it('serves live catalog entries without making unrelated RPCs wait for catalog I/O', async () => {
+ let finish!: (entries: Awaited<ReturnType<typeof storeCatalog.refreshDshRegistry>>) => void
+ vi.spyOn(storeCatalog, 'refreshDshRegistry').mockImplementation(() => new Promise(resolve => { finish = resolve }))
+ const socket = new BackendSocket('token')
+ socket.runtimeModelsProvider = async () => []
+ const frames: Array<Record<string, unknown>> = []
+ socket.registerLocalClient('local:catalog', { sendFrame: frame => { frames.push(frame); return true }, sendBinary: () => true })
+ socket.handleLocalFrame('local:catalog', { type: 'dsh_list', payload: { requestId: 'catalog' } })
+ socket.handleLocalFrame('local:catalog', { type: 'models_list', payload: { requestId: 'models' } })
+ await vi.waitFor(() => expect(frames.some(frame => frame.type === 'models_list_result')).toBe(true))
+ expect(frames.some(frame => frame.type === 'dsh_list_result')).toBe(false)
+ finish([{ id: 'acme/published-today', name: 'Published today', engine: 'claude', repo: 'https://example.test/project', tier: 2, viewerUse: 'acme/viewer' }])
+ await vi.waitFor(() => expect(frames).toContainEqual({
+ type: 'dsh_list_result', payload: { requestId: 'catalog', dsh: [expect.objectContaining({ id: 'acme/published-today', installed: false, viewerUse: 'acme/viewer' })] },
+ }))
+ await socket.unregisterLocalClient('local:catalog')
+ await socket.stop()
+ })
+
it('does not let a slow engines_probe block agent_create on the same connection', async () => {
const socket = new BackendSocket('token')
const frames: Array<Record<string, unknown>> = []
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index 18e0a2577..558a3cf68 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -38,6 +38,7 @@ import { projectPreview } from './lib/projectPreview.js'
import { agentFrame, type AgentDshContext, type AgentFrame } from './lib/agentFrame.js'
import { installedDsh } from './dsh/installed.js'
import { DSH_ID_RE } from './dsh/manifest.js'
+import { refreshDshRegistry } from './dsh/catalog.js'
import type { DshInstallProgress } from './dsh/install.js'
import { dshInstallReply, dshInstallRequest, dshInstallStatus, dshListRows, dshRemoveId, dshRemoveReply } from './dsh/wire.js'
import { routeVoiceTask } from './lib/voiceRouter.js'
@@ -1469,9 +1470,10 @@ export class BackendSocket {
}
case 'dsh_list': {
- // Which domain-specific harnesses this machine has, plus what the bundled registry offers —
- // answered here, on the machine in question, for the same reason `engines_probe` is.
- reply(type, requestId, { dsh: dshListRows() })
+ // Keep catalog I/O off this connection's ordered RPC queue.
+ void refreshDshRegistry()
+ .then(catalog => reply(type, requestId, { dsh: dshListRows(undefined, catalog) }))
+ .catch(error => reply(type, requestId, { error: 'INTERNAL', detail: error instanceof Error ? error.message : String(error) }))
return
}
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 848d78654..435cd7260 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -73,7 +73,7 @@ import { buildLaunchOverrides, validateLaunchOverrides, type LaunchOverridesDeps
import { buildHarnessSessionLabel } from './lib/harnessSessionLabel.js'
import { installedDsh } from './dsh/installed.js'
import { dshVerdictPath } from './dsh/manifest.js'
-import { registryEntry } from './dsh/registry.js'
+import { catalogEntry, refreshDshRegistry } from './dsh/catalog.js'
import { installDsh, resolveInstallSource, removeDsh } from './dsh/install.js'
import { preTrustClaudeProject, preTrustCodexProject } from './lib/claudeTrust.js'
import { materializeWorkspace } from './dsh/materialize.js'
@@ -1374,7 +1374,7 @@ async function runForeground(session: AuthSession): Promise<void> {
return {
// The current id, so a face drawn by id survives a rename the agent predates.
id: installed?.id ?? s.dsh,
- name: installed?.manifest.name ?? registryEntry(s.dsh)?.name ?? null,
+ name: installed?.manifest.name ?? catalogEntry(s.dsh)?.name ?? null,
viewerUrl: state?.viewerUrl ?? null,
verdict: state?.verdict ?? null,
}
@@ -2110,6 +2110,8 @@ async function runForeground(session: AuthSession): Promise<void> {
backend.dshFrameProvider = dshFrameContext
backend.onDshRemove = (id) => removeDsh(id)
backend.onDshInstall = async ({ id, url, ref }, progress) => {
+ const catalog = new Map((await refreshDshRegistry(!!id && !catalogEntry(id))).map(entry => [entry.id, entry]))
+ if (id && !catalog.has(id)) return { ok: false, error: 'INVALID_DSH', detail: `${id} is not in this machine's Store catalog` }
const resolved = id ? resolveInstallSource(id) : url ? { source: url, ref } : null
if (!resolved) return { ok: false, error: 'INVALID_DSH', detail: `${id ?? url} is not a known harness` }
// NARRATE THE LINES, NOT ONLY THE PHASES. A toolchain setup is minutes of npm and uv output, and
@@ -2128,6 +2130,8 @@ async function runForeground(session: AuthSession): Promise<void> {
}
const result = await installDsh({
source: resolved.source,
+ expectedId: id,
+ registry: dependencyId => catalog.get(dependencyId),
ref: ref ?? resolved.ref,
path: 'path' in resolved ? resolved.path : undefined,
onProgress: (p) => {
diff --git a/cli/src/config/env.ts b/cli/src/config/env.ts
index 18c720003..cc2c8534e 100644
--- a/cli/src/config/env.ts
+++ b/cli/src/config/env.ts
@@ -241,6 +241,12 @@ const envSchema = z.object({
// its registry entries name (`main`). For trying a store change end to end BEFORE it merges: push
// the branch, run the daemon with HARNESS_STORE_REF=<branch>, and Get in the store fetches from it.
HARNESS_STORE_REF: z.string().regex(/^[A-Za-z0-9._\/-]{1,200}$/).optional().catch(undefined),
+ // Optional catalog mirror. Public HTTPS in production; loopback HTTP supports isolated tests.
+ HARNESS_STORE_CATALOG_URL: z.string().url().refine((value) => {
+ const url = new URL(value)
+ return !url.username && !url.password && (url.protocol === 'https:'
+ || (url.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)))
+ }).optional().catch(undefined),
// This computer's stable id, minted once and never regenerated (see computerIdFile above). Pin it
// explicitly on a box with no durable home — a container or CI job that gets a fresh ~/.harness on
// every boot would otherwise look like a NEW computer each time and collect a machine per start.
diff --git a/cli/src/dsh/catalog.spec.ts b/cli/src/dsh/catalog.spec.ts
new file mode 100644
index 000000000..33b4a015f
--- /dev/null
+++ b/cli/src/dsh/catalog.spec.ts
@@ -0,0 +1,167 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { execFileSync } from 'node:child_process'
+import { CATALOG_REFRESH_MS, LiveStoreCatalog, parseStoreCatalog, refreshDshRegistry, resetLiveDshRegistry } from './catalog.js'
+import { env } from '../config/env.js'
+import { dshCommand } from './command.js'
+import { resolveInstallSource } from './install.js'
+import { installedDsh, invalidateInstalledDsh } from './installed.js'
+import type { DshRegistryEntry } from './registry.js'
+
+const first: DshRegistryEntry = { id: 'acme/new-game', name: 'New Game', engine: 'claude', repo: 'https://example.test/game', viewerUse: 'acme/new-viewer' }
+const viewer: DshRegistryEntry = { id: 'acme/new-viewer', kind: 'viewer', name: 'New Viewer', repo: 'https://example.test/viewer' }
+const fallback: DshRegistryEntry[] = [{ id: 'acme/bundled', name: 'Bundled', engine: 'codex', repo: 'https://example.test/bundled' }]
+const document = (entries = [first, viewer]) => ({ spec: 1, entries })
+const response = (entries = [first, viewer]) => new Response(JSON.stringify(document(entries)), { headers: { etag: '"v1"' } })
+
+describe('live Store catalog', () => {
+ let root: string
+ let now: number
+ let fetcher: ReturnType<typeof vi.fn<typeof fetch>>
+ beforeEach(() => { root = mkdtempSync(join(tmpdir(), 'dsh-catalog-')); now = 1000; fetcher = vi.fn<typeof fetch>() })
+ afterEach(() => { rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks() })
+ const make = () => new LiveStoreCatalog({ url: 'https://catalog.example.test/catalog.json', cacheFile: join(root, 'catalog.json'), fallback: () => fallback, fetch: fetcher, now: () => now })
+
+ it('discovers new entries without rebuilding; removals do not reappear from the bundle', async () => {
+ fetcher.mockResolvedValueOnce(response()).mockResolvedValueOnce(response([viewer]))
+ const catalog = make()
+ expect(catalog.current()).toEqual(fallback)
+ expect((await catalog.refresh()).map(e => e.id)).toEqual([first.id, viewer.id])
+ now += CATALOG_REFRESH_MS
+ expect((await catalog.refresh()).map(e => e.id)).toEqual([viewer.id])
+ })
+
+ it('coalesces concurrent readers and uses the ETag after the refresh interval', async () => {
+ fetcher.mockResolvedValueOnce(response()).mockResolvedValueOnce(new Response(null, { status: 304 }))
+ const catalog = make()
+ await Promise.all(Array.from({ length: 25 }, () => catalog.refresh()))
+ await catalog.refresh()
+ expect(fetcher).toHaveBeenCalledTimes(1)
+ now += CATALOG_REFRESH_MS
+ await catalog.refresh()
+ expect(fetcher).toHaveBeenCalledTimes(2)
+ expect(fetcher.mock.calls[1][1]?.headers).toMatchObject({ 'if-none-match': '"v1"' })
+ expect(catalog.current()).toHaveLength(2)
+ expect(JSON.parse(readFileSync(join(root, 'catalog.json'), 'utf8')).checkedAt).toBe(now)
+ })
+
+ it('restores a valid disk cache without a fetch and keeps it during an outage', async () => {
+ fetcher.mockResolvedValueOnce(response())
+ await make().refresh()
+ const restarted = make()
+ expect((await restarted.refresh()).map(e => e.id)).toContain(first.id)
+ expect(fetcher).toHaveBeenCalledTimes(1)
+ now += CATALOG_REFRESH_MS
+ fetcher.mockRejectedValue(new Error('offline'))
+ expect((await restarted.refresh()).map(e => e.id)).toContain(first.id)
+ await restarted.refresh()
+ expect(fetcher).toHaveBeenCalledTimes(2)
+ })
+
+ it.each([
+ ['invalid JSON', () => new Response('{broken')],
+ ['future envelope', () => new Response(JSON.stringify({ spec: 2, entries: [first] }))],
+ ['damaged entry', () => new Response(JSON.stringify(document([{ ...first, repo: '' }])))],
+ ['duplicate identity', () => response([first, first])],
+ ['HTTP failure', () => new Response('', { status: 503 })],
+ ['oversized body', () => new Response('ignored', { headers: { 'content-length': String(17 * 1024 * 1024) } })],
+ ])('keeps the last good catalog after %s', async (_name, bad) => {
+ fetcher.mockResolvedValueOnce(response()).mockResolvedValueOnce(bad())
+ const catalog = make()
+ await catalog.refresh()
+ const before = readFileSync(join(root, 'catalog.json'), 'utf8')
+ now += CATALOG_REFRESH_MS
+ expect((await catalog.refresh()).map(e => e.id)).toContain(first.id)
+ expect(readFileSync(join(root, 'catalog.json'), 'utf8')).toBe(before)
+ })
+
+ it('falls back on first offline launch or a corrupt cache', async () => {
+ writeFileSync(join(root, 'catalog.json'), '{bad')
+ fetcher.mockRejectedValue(new Error('offline'))
+ expect(await make().refresh()).toEqual(fallback)
+ })
+
+ it('never loads a cache belonging to a different source', async () => {
+ fetcher.mockResolvedValueOnce(response())
+ await make().refresh()
+ const file = join(root, 'catalog.json')
+ const saved = JSON.parse(readFileSync(file, 'utf8'))
+ writeFileSync(file, JSON.stringify({ ...saved, url: 'https://different.test/catalog.json' }))
+ expect(make().current()).toEqual(fallback)
+ })
+
+ it('bounds a network stall and returns the bundled catalog', async () => {
+ fetcher.mockImplementation((_url, init) => new Promise((_resolve, reject) => init!.signal!.addEventListener('abort', () => reject(new Error('timeout')))))
+ const catalog = new LiveStoreCatalog({ url: 'https://catalog.test', cacheFile: join(root, 'cache'), fallback: () => fallback, fetch: fetcher, timeoutMs: 15 })
+ expect(await catalog.refresh()).toEqual(fallback)
+ })
+
+ it('accepts future optional fields and skips unsupported engines without losing supported ones', () => {
+ const parsed = parseStoreCatalog(document([{ ...first, futureField: true }, { ...first, id: 'acme/future', engine: 'future-engine' }, viewer] as DshRegistryEntry[]))
+ expect(parsed.entries.map(e => e.id)).toEqual([first.id, viewer.id])
+ })
+
+ it('does not trust a community package claiming to be first-party', () => {
+ expect(parseStoreCatalog(document([{ ...first, verified: true }])).entries[0].verified).toBe(false)
+ expect(() => parseStoreCatalog(document([{ ...first, repo: '/private/project' }]))).toThrow()
+ expect(() => parseStoreCatalog(document([{ ...first, repo: 'https://secret:[email protected]/repo' }]))).toThrow()
+ })
+
+ it('handles thousands of packages in one response', async () => {
+ const entries = Array.from({ length: 5000 }, (_, i) => ({ ...first, id: `acme/game-${i}` }))
+ fetcher.mockResolvedValueOnce(response(entries))
+ expect(await make().refresh()).toHaveLength(5000)
+ expect(fetcher).toHaveBeenCalledTimes(1)
+ })
+})
+
+describe('catalog → CLI list → install → shared viewer', () => {
+ let root: string
+ const saved = { dsh: env.DSH_DIR, url: env.HARNESS_STORE_CATALOG_URL }
+ beforeEach(() => {
+ root = mkdtempSync(join(tmpdir(), 'dsh-live-install-'))
+ env.DSH_DIR = join(root, 'installed'); env.HARNESS_STORE_CATALOG_URL = 'https://catalog.example.test/live.json'
+ invalidateInstalledDsh(); resetLiveDshRegistry()
+ })
+ afterEach(() => {
+ env.DSH_DIR = saved.dsh; env.HARNESS_STORE_CATALOG_URL = saved.url
+ resetLiveDshRegistry(); invalidateInstalledDsh(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); vi.restoreAllMocks()
+ rmSync(root, { recursive: true, force: true })
+ })
+
+ it('installs a newly published harness and viewer from one pinned commit, without a new CLI', async () => {
+ const repo = join(root, 'source'); mkdirSync(repo)
+ const git = (...args: string[]) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', env: { ...process.env, GIT_AUTHOR_NAME: 'test', GIT_AUTHOR_EMAIL: '[email protected]', GIT_COMMITTER_NAME: 'test', GIT_COMMITTER_EMAIL: '[email protected]' } }).trim()
+ for (const [folder, manifest] of [
+ ['game', { spec: 1, id: first.id, name: first.name, engine: 'claude', viewer: { use: viewer.id } }],
+ ['viewer', { spec: 1, id: viewer.id, name: viewer.name, kind: 'viewer', viewer: { command: 'node viewer.mjs', url: 'http://127.0.0.1:${port}/' } }],
+ ] as const) { mkdirSync(join(repo, folder)); writeFileSync(join(repo, folder, 'harness.json'), JSON.stringify(manifest)) }
+ writeFileSync(join(repo, 'viewer', 'viewer.mjs'), '// fixture viewer\n')
+ git('init', '-q', '-b', 'main'); git('add', '.'); git('commit', '-qm', 'published packages')
+ const ref = git('rev-parse', 'HEAD')
+ writeFileSync(join(repo, 'game', 'unpublished.txt'), 'This later commit is not in the catalog')
+ git('add', '.'); git('commit', '-qm', 'unpublished change')
+ // All Git traffic stays in the fixture repository. No global Git configuration is changed.
+ vi.stubEnv('GIT_CONFIG_COUNT', '1')
+ vi.stubEnv('GIT_CONFIG_KEY_0', `url.file://${repo}.insteadOf`)
+ vi.stubEnv('GIT_CONFIG_VALUE_0', 'https://example.test/packages')
+ const entries = [{ ...first, repo: 'https://example.test/packages', path: 'game', ref }, { ...viewer, repo: 'https://example.test/packages', path: 'viewer', ref }]
+ vi.stubGlobal('fetch', vi.fn<typeof fetch>().mockResolvedValue(response(entries)))
+ const output = vi.spyOn(console, 'log').mockImplementation(() => {})
+ expect(await dshCommand('list', [])).toBe(0)
+ expect(output.mock.calls.flat().join('\n')).toContain('acme/new-game')
+ expect(installedDsh(first.id)).toBeUndefined()
+ expect(installedDsh(viewer.id)).toBeUndefined()
+ expect(resolveInstallSource(first.id)).toMatchObject({ source: entries[0].repo, path: 'game', ref })
+ expect(await dshCommand('install', [first.id])).toBe(0)
+ expect(installedDsh(first.id)?.commit).toBe(ref)
+ expect(installedDsh(viewer.id)?.commit).toBe(ref)
+ const installedAt = installedDsh(viewer.id)?.installedAt
+ expect(await dshCommand('install', [first.id])).toBe(0)
+ expect(installedDsh(viewer.id)?.installedAt).toBe(installedAt)
+ expect(fetch).toHaveBeenCalledTimes(1)
+ expect((await refreshDshRegistry()).map(e => e.id)).toContain(first.id)
+ })
+})
diff --git a/cli/src/dsh/catalog.ts b/cli/src/dsh/catalog.ts
new file mode 100644
index 000000000..fc6acb739
--- /dev/null
+++ b/cli/src/dsh/catalog.ts
@@ -0,0 +1,147 @@
+/** Live Store metadata. Fetching a catalog never installs or executes a package. */
+import { createHash, randomUUID } from 'node:crypto'
+import { mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { env } from '../config/env.js'
+import { ENGINES } from '../engines/types.js'
+import { bundledDshRegistry, DshRegistryEntrySchema, HARNESS_MONOREPO, type DshRegistryEntry } from './registry.js'
+
+export const STORE_CATALOG_URL = 'https://raw.githubusercontent.com/autonomous-ai/openharness/store-catalog/catalog.json'
+export const CATALOG_REFRESH_MS = 5 * 60_000
+const RETRY_MS = 30_000
+const MAX_BYTES = 16 * 1024 * 1024
+
+export interface StoreCatalog { spec: 1; entries: DshRegistryEntry[] }
+
+/** Ignore future optional fields/engines, but never cache a damaged or ambiguous catalog. */
+export function parseStoreCatalog(value: unknown): StoreCatalog {
+ if (!value || typeof value !== 'object') throw new Error('Invalid Store catalog')
+ const catalog = value as Record<string, unknown>
+ if (catalog.spec !== 1 || !Array.isArray(catalog.entries) || catalog.entries.length > 25_000) throw new Error('Unsupported Store catalog')
+ const entries: DshRegistryEntry[] = []
+ const seen = new Set<string>()
+ for (const raw of catalog.entries) {
+ if (!raw || typeof raw !== 'object') throw new Error('Invalid catalog entry')
+ const item = raw as Record<string, unknown>
+ if (item.kind !== undefined && item.kind !== 'agent' && item.kind !== 'viewer') continue
+ if (item.kind !== 'viewer' && typeof item.engine === 'string' && !(ENGINES as readonly string[]).includes(item.engine)) continue
+ const parsed = DshRegistryEntrySchema.strip().parse(item)
+ const url = new URL(parsed.repo)
+ if (url.protocol !== 'https:' || url.username || url.password || /[\x00-\x20\x7f]/.test(parsed.repo)) throw new Error('Catalog sources must be public HTTPS repositories')
+ if (seen.has(parsed.id)) throw new Error(`Duplicate catalog id: ${parsed.id}`)
+ seen.add(parsed.id)
+ // A community entry cannot grant itself first-party installation trust.
+ const firstParty = parsed.repo.replace(/\.git$/, '') === HARNESS_MONOREPO
+ && parsed.id.startsWith('autonomous/')
+ && parsed.path === `store/${parsed.kind === 'viewer' ? 'viewers' : 'agents'}/${parsed.id.slice('autonomous/'.length)}`
+ entries.push({ ...parsed, verified: firstParty && parsed.verified === true })
+ }
+ return { spec: 1, entries: entries.sort((a, b) => a.id.localeCompare(b.id)) }
+}
+
+interface Cache { url: string; checkedAt: number; etag?: string; catalog: StoreCatalog }
+interface CatalogOptions {
+ url: string
+ cacheFile: string
+ fallback: () => DshRegistryEntry[]
+ fetch?: typeof fetch
+ now?: () => number
+ timeoutMs?: number
+}
+
+/** One request per refresh interval, shared by concurrent readers and persisted across restarts. */
+export class LiveStoreCatalog {
+ private cache: Cache | null = null
+ private nextAttempt = 0
+ private inFlight: Promise<DshRegistryEntry[]> | null = null
+ private readonly now: () => number
+ private readonly fetcher: typeof fetch
+
+ constructor(private readonly options: CatalogOptions) {
+ this.now = options.now ?? Date.now
+ this.fetcher = options.fetch ?? fetch
+ try {
+ if (statSync(options.cacheFile).size > MAX_BYTES) return
+ const saved = JSON.parse(readFileSync(options.cacheFile, 'utf8')) as Cache
+ if (saved.url !== options.url || !Number.isFinite(saved.checkedAt)) return
+ const catalog = parseStoreCatalog(saved.catalog)
+ this.cache = { url: saved.url, checkedAt: Math.min(saved.checkedAt, this.now()), catalog,
+ ...(typeof saved.etag === 'string' && saved.etag.length < 500 ? { etag: saved.etag } : {}) }
+ this.nextAttempt = this.cache.checkedAt + CATALOG_REFRESH_MS
+ } catch { /* First launch, unreadable cache, or old format: use the bundled shelf. */ }
+ }
+
+ current(): DshRegistryEntry[] { return this.cache?.catalog.entries ?? this.options.fallback() }
+
+ refresh(force = false): Promise<DshRegistryEntry[]> {
+ if (this.inFlight) return this.inFlight
+ if (!force && this.now() < this.nextAttempt) return Promise.resolve(this.current())
+ this.inFlight = this.update().finally(() => { this.inFlight = null })
+ return this.inFlight
+ }
+
+ private async update(): Promise<DshRegistryEntry[]> {
+ try {
+ const response = await this.fetcher(this.options.url, {
+ headers: { accept: 'application/json', ...(this.cache?.etag ? { 'if-none-match': this.cache.etag } : {}) },
+ signal: AbortSignal.timeout(this.options.timeoutMs ?? 4_000),
+ redirect: 'error',
+ })
+ if (response.status === 304 && this.cache) {
+ this.cache = { ...this.cache, checkedAt: this.now() }
+ } else {
+ if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`)
+ if (Number(response.headers.get('content-length')) > MAX_BYTES) throw new Error('Catalog too large')
+ const reader = response.body?.getReader()
+ if (!reader) throw new Error('Empty catalog response')
+ const chunks: Uint8Array[] = []
+ let bytes = 0
+ try {
+ while (true) {
+ const { done, value } = await reader.read()
+ if (done) break
+ bytes += value.byteLength
+ if (bytes > MAX_BYTES) throw new Error('Catalog too large')
+ chunks.push(value)
+ }
+ } finally { await reader.cancel().catch(() => {}) }
+ const catalog = parseStoreCatalog(JSON.parse(Buffer.concat(chunks).toString('utf8')))
+ const etag = response.headers.get('etag')
+ this.cache = { url: this.options.url, checkedAt: this.now(), catalog, ...(etag && etag.length < 500 ? { etag } : {}) }
+ }
+ this.nextAttempt = this.now() + CATALOG_REFRESH_MS
+ // A read-only disk must not discard metadata that was fetched successfully.
+ try {
+ mkdirSync(dirname(this.options.cacheFile), { recursive: true, mode: 0o700 })
+ const temporary = `${this.options.cacheFile}.${randomUUID()}.tmp`
+ writeFileSync(temporary, JSON.stringify(this.cache), { mode: 0o600 })
+ renameSync(temporary, this.options.cacheFile)
+ } catch { /* The in-memory catalog still works. */ }
+ } catch {
+ this.nextAttempt = this.now() + RETRY_MS
+ // Offline, rate-limited, timed out, or invalid: keep the last complete catalog.
+ }
+ return this.current()
+ }
+}
+
+let live: { key: string; catalog: LiveStoreCatalog } | null = null
+function liveCatalog(): LiveStoreCatalog {
+ const url = env.HARNESS_STORE_CATALOG_URL ?? STORE_CATALOG_URL
+ const cacheFile = join(env.DSH_DIR, '.catalog', `${createHash('sha256').update(url).digest('hex').slice(0, 20)}.json`)
+ const key = `${cacheFile}:${url}`
+ if (live?.key !== key) live = { key, catalog: new LiveStoreCatalog({ url, cacheFile, fallback: bundledDshRegistry }) }
+ return live.catalog
+}
+
+function withStoreRef(entries: DshRegistryEntry[]): DshRegistryEntry[] {
+ const ref = env.HARNESS_STORE_REF
+ return ref ? entries.map(entry => entry.path && entry.repo.replace(/\.git$/, '') === HARNESS_MONOREPO ? { ...entry, ref } : entry) : entries
+}
+
+export function currentDshRegistry(): DshRegistryEntry[] { return withStoreRef(liveCatalog().current()) }
+export async function refreshDshRegistry(force = false): Promise<DshRegistryEntry[]> { return withStoreRef(await liveCatalog().refresh(force)) }
+export function catalogEntry(id: string): DshRegistryEntry | undefined { return currentDshRegistry().find(entry => entry.id === id) }
+
+/** Test seam: also resets the source/cache identity when a test changes DSH_DIR. */
+export function resetLiveDshRegistry(): void { live = null }
diff --git a/cli/src/dsh/catalogPublisher.spec.ts b/cli/src/dsh/catalogPublisher.spec.ts
new file mode 100644
index 000000000..a3a738e40
--- /dev/null
+++ b/cli/src/dsh/catalogPublisher.spec.ts
@@ -0,0 +1,89 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { createHash } from 'node:crypto'
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { parseStoreCatalog } from './catalog.js'
+// @ts-expect-error — dependency-free ESM also runs directly in GitHub Actions.
+import { createStoreCatalog, publishStoreCatalog } from '../../../store/tools/catalog.mjs'
+
+const ref = 'a'.repeat(40)
+const roots: string[] = []
+afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) })
+function fixture() {
+ const root = mkdtempSync(join(tmpdir(), 'catalog-publisher-')); roots.push(root)
+ mkdirSync(join(root, 'agents', 'game'), { recursive: true })
+ writeFileSync(join(root, 'agents', 'game', 'harness.json'), JSON.stringify({ spec: 1, id: 'autonomous/game', name: 'Game', engine: 'claude' }))
+ writeFileSync(join(root, 'agents', 'game', 'store.json'), '{}')
+ return root
+}
+
+describe('Store publication', () => {
+ it('generates a runtime-valid catalog from all real packages, pinning built-ins to the source commit', () => {
+ const catalog = createStoreCatalog(fileURLToPath(new URL('../../../store', import.meta.url)), ref)
+ const parsed = parseStoreCatalog(catalog)
+ expect(parsed.entries).toHaveLength(catalog.entries.length)
+ expect(parsed.entries.length).toBeGreaterThan(10)
+ expect(parsed.entries.filter(entry => entry.verified).every(entry => entry.ref === ref)).toBe(true)
+ })
+
+ it('rejects corrupt manifests instead of silently publishing a partial catalog', () => {
+ const root = fixture()
+ writeFileSync(join(root, 'agents', 'game', 'harness.json'), '{bad')
+ expect(() => createStoreCatalog(root, ref)).toThrow()
+ })
+
+ it('rejects a new harness whose shared viewer is not also published', () => {
+ const root = fixture()
+ const manifest = { spec: 1, id: 'autonomous/game', name: 'Game', engine: 'claude', viewer: { use: 'acme/not-listed' } }
+ writeFileSync(join(root, 'agents', 'game', 'harness.json'), JSON.stringify(manifest))
+ expect(() => createStoreCatalog(root, ref)).toThrow(/viewer acme\/not-listed/)
+ })
+
+ it('keeps community ownership and refs while preventing a verified flag from granting trust', () => {
+ const root = fixture(); mkdirSync(join(root, 'registry', 'acme'), { recursive: true })
+ writeFileSync(join(root, 'registry', 'acme', 'community.json'), JSON.stringify({ id: 'acme/community', name: 'Community', engine: 'codex', repo: 'https://github.com/acme/community', ref: 'v1', verified: true }))
+ expect(createStoreCatalog(root, ref).entries.find((entry: { id: string }) => entry.id === 'acme/community')).toMatchObject({ verified: false, ref: 'v1' })
+ })
+
+ it('creates only catalog.json on the catalog branch, without modifying main or release tags', async () => {
+ const requests: Array<{ path: string; method: string; data: Record<string, unknown> | null }> = []
+ const request = vi.fn<typeof fetch>().mockImplementation(async (url, init) => {
+ const path = String(url).replace('https://api.github.com/repos/autonomous-ai/openharness/', '')
+ requests.push({ path, method: init?.method ?? 'GET', data: init?.body ? JSON.parse(String(init.body)) : null })
+ if (path === 'git/ref/heads/store-catalog') return new Response('', { status: 404 })
+ return Response.json({ sha: path === 'git/blobs' ? 'blob' : path === 'git/trees' ? 'tree' : 'commit' })
+ })
+ expect(await publishStoreCatalog(createStoreCatalog(fixture(), ref), { token: 'fixture-token', fetch: request })).toEqual({ changed: true, sha: 'commit' })
+ expect(requests.map(item => item.path)).toEqual(['git/ref/heads/store-catalog', 'git/blobs', 'git/trees', 'git/commits', 'git/refs'])
+ expect(requests.find(item => item.path === 'git/trees')?.data).toEqual({ tree: [{ path: 'catalog.json', mode: '100644', type: 'blob', sha: 'blob' }] })
+ expect(requests.at(-1)?.data).toEqual({ ref: 'refs/heads/store-catalog', sha: 'commit' })
+ })
+
+ it('skips an unchanged publication', async () => {
+ const catalog = createStoreCatalog(fixture(), ref)
+ const body = JSON.stringify(catalog, null, 2) + '\n'
+ const sha = createHash('sha1').update(`blob ${Buffer.byteLength(body)}\0`).update(body).digest('hex')
+ const request = vi.fn<typeof fetch>()
+ .mockResolvedValueOnce(Response.json({ object: { sha: 'existing' } }))
+ .mockResolvedValueOnce(Response.json({ tree: { sha: 'tree' } }))
+ .mockResolvedValueOnce(Response.json({ tree: [{ path: 'catalog.json', sha }] }))
+ expect(await publishStoreCatalog(catalog, { token: 'fixture-token', fetch: request })).toEqual({ changed: false, sha: 'existing' })
+ expect(request.mock.calls.every(([, init]) => init?.method === 'GET')).toBe(true)
+ })
+
+ it('updates by fast-forward and refuses a conflicting publication', async () => {
+ const request = vi.fn<typeof fetch>()
+ .mockResolvedValueOnce(Response.json({ object: { sha: 'existing' } }))
+ .mockResolvedValueOnce(Response.json({ tree: { sha: 'tree' } }))
+ .mockResolvedValueOnce(Response.json({ tree: [] }))
+ .mockResolvedValueOnce(Response.json({ sha: 'blob' }))
+ .mockResolvedValueOnce(Response.json({ sha: 'tree' }))
+ .mockResolvedValueOnce(Response.json({ sha: 'commit' }))
+ .mockResolvedValueOnce(new Response('', { status: 422 }))
+ await expect(publishStoreCatalog(createStoreCatalog(fixture(), ref), { token: 'fixture-token', fetch: request })).rejects.toThrow(/422/)
+ expect(JSON.parse(String(request.mock.calls.at(-1)?.[1]?.body))).toEqual({ sha: 'commit', force: false })
+ expect(request).toHaveBeenCalledTimes(7)
+ })
+})
diff --git a/cli/src/dsh/command.spec.ts b/cli/src/dsh/command.spec.ts
index 53d943260..ddf9bcd05 100644
--- a/cli/src/dsh/command.spec.ts
+++ b/cli/src/dsh/command.spec.ts
@@ -47,6 +47,9 @@ describe('harness dsh', () => {
env.DSH_DIR = join(root, 'installed')
env.HARNESS_STORE_REF = undefined
invalidateInstalledDsh()
+ // These command-contract fixtures exercise the offline bundled fallback.
+ // catalog.spec.ts covers live discovery and installation separately.
+ vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline fixture')))
out = []
err = []
vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { out.push(args.join(' ')) })
diff --git a/cli/src/dsh/command.ts b/cli/src/dsh/command.ts
index a9bf147f5..fae48d6d3 100644
--- a/cli/src/dsh/command.ts
+++ b/cli/src/dsh/command.ts
@@ -3,8 +3,9 @@
* and for the development loop (`--link` a checkout, iterate, no re-clone).
*/
import { listDshState, installedDsh } from './installed.js'
-import { dshTier } from './manifest.js'
-import { bundledDshRegistry, registrySourceUrl } from './registry.js'
+import { DSH_ID_RE, dshTier } from './manifest.js'
+import { registrySourceUrl } from './registry.js'
+import { catalogEntry, currentDshRegistry, refreshDshRegistry } from './catalog.js'
import { installDsh, removeDsh, resolveInstallSource, runDshDoctor } from './install.js'
import { checkDsh, formatCheck } from './check.js'
@@ -33,7 +34,7 @@ export async function dshCommand(verb: string | undefined, rest: readonly string
switch (verb) {
case 'list': {
const { installed, broken } = listDshState()
- const registry = bundledDshRegistry()
+ const registry = await refreshDshRegistry()
const seen = new Set<string>()
const rows: string[] = []
for (const dsh of installed) {
@@ -54,13 +55,17 @@ export async function dshCommand(verb: string | undefined, rest: readonly string
case 'install': {
const target = args[0]
if (!target) { console.error(dshUsage()); return 1 }
+ if (DSH_ID_RE.test(target) && !rest.includes('--link')) await refreshDshRegistry(!catalogEntry(target))
const resolved = resolveInstallSource(target)
if (!resolved) { console.error(`harness dsh install: ${target} is not an id, URL or path`); return 1 }
const link = rest.includes('--link')
const ref = flagValue(rest, '--ref') ?? resolved.ref
const path = flagValue(rest, '--path') ?? resolved.path
+ const catalog = new Map(currentDshRegistry().map(entry => [entry.id, entry]))
const result = await installDsh({
source: resolved.source,
+ expectedId: resolved.id,
+ registry: resolved.id ? id => catalog.get(id) : undefined,
ref: link ? undefined : ref,
path: link ? undefined : path,
link,
diff --git a/cli/src/dsh/install.spec.ts b/cli/src/dsh/install.spec.ts
index 7a07c85d0..8c0fff526 100644
--- a/cli/src/dsh/install.spec.ts
+++ b/cli/src/dsh/install.spec.ts
@@ -79,6 +79,27 @@ describe('installDsh with a used viewer', () => {
expect(asked).toBe(0)
})
+ it('can install a whole repository at the published commit after its branch advances', async () => {
+ const repo = gitRepo(join(root, 'pinned'), { 'harness.json': JSON.stringify({ spec: 1, id: 'acme/pinned', name: 'Published', engine: 'claude' }) })
+ const ref = execFileSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim()
+ writeFileSync(join(repo, 'harness.json'), JSON.stringify({ spec: 1, id: 'acme/pinned', name: 'Unpublished', engine: 'claude' }))
+ execFileSync('git', ['-C', repo, '-c', 'user.name=test', '-c', '[email protected]', 'commit', '-qam', 'later version'])
+ const result = await installDsh({ source: repo, ref, expectedId: 'acme/pinned' })
+ expect(result.ok, JSON.stringify(result)).toBe(true)
+ expect(installedDsh('acme/pinned')?.commit).toBe(ref)
+ expect(installedDsh('acme/pinned')?.manifest.name).toBe('Published')
+ })
+
+ it('rejects a mismatched catalog identity before installing or running setup', async () => {
+ const repo = gitRepo(join(root, 'wrong-id'), {
+ 'harness.json': JSON.stringify({ spec: 1, id: 'acme/surprise', name: 'Wrong', engine: 'claude', toolchain: { setup: './setup.sh' } }),
+ 'setup.sh': '#!/bin/sh\ntouch did-run\n',
+ })
+ expect(await installDsh({ source: repo, expectedId: 'acme/expected' })).toMatchObject({ ok: false, error: 'PACKAGE_ID_MISMATCH' })
+ expect(listInstalledDsh()).toEqual([])
+ expect(readdirSync(env.DSH_DIR).filter(name => name.startsWith('.tmp-'))).toEqual([])
+ })
+
it('a used viewer that is neither installed nor in the registry is said, and the harness still installs', async () => {
const harnessRepo = gitRepo(join(root, 'src', 'thing'), {
'harness.json': JSON.stringify({ spec: 1, id: 'acme/thing', name: 'Thing', engine: 'claude', viewer: { use: 'acme/missing' } }),
diff --git a/cli/src/dsh/install.ts b/cli/src/dsh/install.ts
index 43573908c..f41f811dd 100644
--- a/cli/src/dsh/install.ts
+++ b/cli/src/dsh/install.ts
@@ -18,7 +18,8 @@ import {
type InstalledDsh, type InstalledDshRecord,
} from './installed.js'
import { readDshManifest, viewerUse, type DshManifest } from './manifest.js'
-import { PACKAGE_PATH_RE, registryEntry, type DshRegistryEntry } from './registry.js'
+import { PACKAGE_PATH_RE, type DshRegistryEntry } from './registry.js'
+import { catalogEntry, refreshDshRegistry } from './catalog.js'
import { resolveDshCommand } from './materialize.js'
import { runDshCommand } from './shell.js'
@@ -42,6 +43,8 @@ export interface DshInstallProgress {
export interface DshInstallOptions {
/** A git URL, or a local path (cloned unless `link`). */
source: string
+ /** A catalog install must resolve to the package the user selected. */
+ expectedId?: string
ref?: string
/**
* The folder inside `source` that is the package (`store/agents/typst` of the Harness monorepo).
@@ -70,7 +73,7 @@ export type DshInstallResult =
/** A registry id (`autonomous/typst`) resolves to its repo, ref and folder; anything else is a source. */
export function resolveInstallSource(
idOrSource: string,
- registry: (id: string) => DshRegistryEntry | undefined = registryEntry,
+ registry: (id: string) => DshRegistryEntry | undefined = catalogEntry,
): { source: string; ref?: string; path?: string; id?: string } | null {
const entry = registry(idOrSource)
if (entry) return { source: entry.repo, ref: entry.ref, ...(entry.path ? { path: entry.path } : {}), id: entry.id }
@@ -129,8 +132,7 @@ async function cloneInstall(
if (path === undefined) {
// `--progress` because stderr is not a tty here and git would otherwise stay silent until the end;
// streamed, not collected, so "Receiving objects: 39%" reaches the dialog while it is true.
- const args = ['clone', '--depth', '1', '--progress', ...(ref ? ['--branch', ref] : []), '--', source, tmpDir]
- const clone = await streamGit(args, onLine)
+ const clone = await cloneRepo(source, ref, tmpDir, false, onLine)
if (!clone.ok) return fail('CLONE_FAILED', clone.detail)
commit = await gitHead(tmpDir)
} else {
@@ -140,10 +142,7 @@ async function cloneInstall(
// moved out and the rest of the clone thrown away, so the install is laid out exactly like a
// whole-repo one (the manifest at its root) and nothing else of the monorepo lands on the machine.
const repoDir = join(root, `.tmp-${randomUUID()}`)
- const clone = await streamGit(
- ['clone', '--depth', '1', '--filter=blob:none', '--sparse', '--progress', ...(ref ? ['--branch', ref] : []), '--', source, repoDir],
- onLine,
- )
+ const clone = await cloneRepo(source, ref, repoDir, true, onLine)
if (!clone.ok) return fail('CLONE_FAILED', clone.detail, repoDir)
const sparse = await streamGit(['-C', repoDir, 'sparse-checkout', 'set', '--', path], onLine)
if (!sparse.ok) return fail('CLONE_FAILED', sparse.detail, repoDir)
@@ -163,6 +162,25 @@ async function cloneInstall(
/** How long one git command may take: a clone of a large repository over a slow link, not a hang. */
const GIT_TIMEOUT_MS = 10 * 60_000
+/** Catalogs pin built-ins to the commit they describe; `git clone --branch` cannot take a SHA. */
+async function cloneRepo(source: string, ref: string | undefined, dir: string, sparse: boolean, onLine: DshInstallOptions['onLine']): Promise<{ ok: true } | { ok: false; detail: string }> {
+ if (!ref || !/^[a-f0-9]{40}$/i.test(ref)) {
+ return streamGit(['clone', '--depth', '1', ...(sparse ? ['--filter=blob:none', '--sparse'] : []), '--progress', ...(ref ? ['--branch', ref] : []), '--', source, dir], onLine)
+ }
+ const steps = [
+ ['init', '--quiet', dir],
+ ['-C', dir, 'remote', 'add', 'origin', source],
+ ['-C', dir, 'fetch', '--depth', '1', ...(sparse ? ['--filter=blob:none'] : []), '--progress', 'origin', ref],
+ ...(sparse ? [['-C', dir, 'sparse-checkout', 'init', '--cone']] : []),
+ ['-C', dir, 'checkout', '--detach', 'FETCH_HEAD'],
+ ]
+ for (const args of steps) {
+ const result = await streamGit(args, onLine)
+ if (!result.ok) return result
+ }
+ return { ok: true }
+}
+
/** Run git, handing each stderr line (and each carriage-return progress segment) to `onLine` as it lands. */
function streamGit(args: string[], onLine: ((line: string) => void) | undefined): Promise<{ ok: true } | { ok: false; detail: string }> {
return new Promise((resolve) => {
@@ -239,6 +257,11 @@ export async function runDshDoctor(installed: InstalledDsh, onLine?: (line: stri
export async function installDsh(opts: DshInstallOptions): Promise<DshInstallResult> {
const progress = (p: DshInstallProgress): void => opts.onProgress?.(p)
+ const wrongId = (actual: string): DshInstallResult => {
+ const detail = `Catalog requested ${opts.expectedId}, but the package declares ${actual}`
+ progress({ id: opts.expectedId ?? null, phase: 'failed', detail })
+ return { ok: false, error: 'PACKAGE_ID_MISMATCH', detail }
+ }
let manifest: DshManifest
let dir: string
let commit: string | null = null
@@ -249,6 +272,7 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallRes
const linked = linkInstall(opts.source)
if (!linked.ok) { progress({ id: null, phase: 'failed', detail: linked.detail }); return linked }
manifest = linked.manifest
+ if (opts.expectedId && manifest.id !== opts.expectedId) return wrongId(manifest.id)
realDir = linked.realDir
dir = placeAt(manifest.id, { linkTo: realDir })
commit = await gitHead(realDir)
@@ -256,6 +280,10 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallRes
const cloned = await cloneInstall(opts.source, opts.ref, opts.path, opts.onLine)
if (!cloned.ok) { progress({ id: null, phase: 'failed', detail: cloned.detail }); return cloned }
manifest = cloned.manifest
+ if (opts.expectedId && manifest.id !== opts.expectedId) {
+ rmSync(cloned.tmpDir, { recursive: true, force: true })
+ return wrongId(manifest.id)
+ }
commit = cloned.commit
dir = placeAt(manifest.id, { tmpDir: cloned.tmpDir })
realDir = realpathSync(dir)
@@ -297,7 +325,8 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallRes
// first (`harness dsh install <url>`), and the doctor says so rather than the pane going blank.
const uses = viewerUse(manifest)
if (uses && !installedDsh(uses)) {
- const entry = (opts.registry ?? registryEntry)(uses)
+ if (!opts.registry && !catalogEntry(uses)) await refreshDshRegistry()
+ const entry = (opts.registry ?? catalogEntry)(uses)
if (entry) {
opts.onLine?.(`viewer ${uses} · installing`)
// The viewer's own phases are narrated UNDER THE HARNESS: the dialog watches the id it asked
@@ -305,7 +334,7 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallRes
// install would read as hung for the minutes OpenCascade takes to arrive. The viewer's `done`
// is not the harness's done, so it reports as the harness's setup still going.
const dep = await installDsh({
- source: entry.repo, ref: entry.ref, path: entry.path, setupTimeoutMs: opts.setupTimeoutMs, onLine: opts.onLine,
+ source: entry.repo, expectedId: entry.id, ref: entry.ref, path: entry.path, registry: opts.registry, setupTimeoutMs: opts.setupTimeoutMs, onLine: opts.onLine,
onProgress: (p) => {
if (p.phase === 'failed') return // reported below, once, with the viewer named
const phase: DshInstallPhase = p.phase === 'done' ? 'setup' : p.phase
diff --git a/cli/src/dsh/registry.ts b/cli/src/dsh/registry.ts
index 9217a7bd6..83804d1a6 100644
--- a/cli/src/dsh/registry.ts
+++ b/cli/src/dsh/registry.ts
@@ -1,5 +1,5 @@
/**
- * The bundled registry, baked into the CLI at build time the same way the version is
+ * The offline fallback registry, baked into the CLI at build time the same way the version is
* (`__DSH_REGISTRY__`, an esbuild `define` in both `build.mjs` and `build-bundle.mjs`). Two sources
* at the repo root: every built-in package folder, `store/<agents|viewers>/<name>` — its entry built
* from its `harness.json` and `store.json`, so no fact is written twice — and
@@ -9,7 +9,8 @@
*
* A registry entry is how the desktop can offer "Install Typst" for a package this machine does not
* have yet: it names the repo and the ref to clone, and — for the built-in shelf, which lives in this
- * monorepo under `store/` — the folder inside that repo that IS the package. Nothing here runs code.
+ * monorepo under `store/` — the folder inside that repo that IS the package. `catalog.ts` refreshes
+ * the live shelf; this module supplies its schema and first-offline-launch fallback. Nothing here runs code.
*/
import { readdirSync, readFileSync, statSync } from 'node:fs'
import { join } from 'node:path'
diff --git a/cli/src/dsh/wire.ts b/cli/src/dsh/wire.ts
index 5bd063374..f0cc226cf 100644
--- a/cli/src/dsh/wire.ts
+++ b/cli/src/dsh/wire.ts
@@ -6,7 +6,8 @@
import type { DshInstallProgress } from './install.js'
import { listInstalledDsh, type InstalledDsh } from './installed.js'
import { DSH_ID_RE, dshTier, viewerUse } from './manifest.js'
-import { bundledDshRegistry, registrySourceUrl, type DshRegistryEntry } from './registry.js'
+import { registrySourceUrl, type DshRegistryEntry } from './registry.js'
+import { currentDshRegistry } from './catalog.js'
/**
* `dsh_list`: the harnesses installed on this machine, then what the registry offers that is not.
@@ -15,10 +16,11 @@ import { bundledDshRegistry, registrySourceUrl, type DshRegistryEntry } from './
*/
export function dshListRows(
installed: readonly InstalledDsh[] = listInstalledDsh(),
- registry: readonly DshRegistryEntry[] = bundledDshRegistry(),
+ registry: readonly DshRegistryEntry[] = currentDshRegistry(),
): Record<string, unknown>[] {
+ const byId = new Map(registry.map(entry => [entry.id, entry]))
const facts = (id: string): Record<string, unknown> => {
- const known = registry.find((entry) => entry.id === id)
+ const known = byId.get(id)
return {
verified: known?.verified === true,
// Where a person can read the package: its folder page for a built-in (store/…) one.
diff --git a/cli/vitest.setup.ts b/cli/vitest.setup.ts
index adbaeb556..44739c302 100644
--- a/cli/vitest.setup.ts
+++ b/cli/vitest.setup.ts
@@ -11,3 +11,4 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
process.env.ADAPTER_DATA_DIR = mkdtempSync(join(tmpdir(), 'adapter-test-data-'))
+process.env.DSH_DIR = join(process.env.ADAPTER_DATA_DIR, 'dsh')
diff --git a/desktop/lib/store/store_screen.dart b/desktop/lib/store/store_screen.dart
index e4b34cae4..85804f2b1 100644
--- a/desktop/lib/store/store_screen.dart
+++ b/desktop/lib/store/store_screen.dart
@@ -120,6 +120,7 @@ class _StoreTabState extends State<StoreTab> {
late final _search = TextEditingController(
text: widget.initialHarness == null ? _place.query : '',
);
+ Timer? _catalogRefresh;
void _remember() {
_place
@@ -154,6 +155,17 @@ class _StoreTabState extends State<StoreTab> {
_ask(machine);
}
widget.notifier.addListener(_onAppChanged);
+ // A Store left open picks up new publications; engines do not need reprobes.
+ // The daemon caches catalog HTTP requests for five minutes.
+ _catalogRefresh = Timer.periodic(const Duration(minutes: 1), (_) {
+ for (final machine in widget.notifier.machineStates.values) {
+ if (machine.connectionStatus == ConnectionStatus.connected) {
+ unawaited(
+ widget.notifier.probeDsh(machine.machine.machineId, force: true),
+ );
+ }
+ }
+ });
});
}
@@ -201,6 +213,7 @@ class _StoreTabState extends State<StoreTab> {
@override
void dispose() {
+ _catalogRefresh?.cancel();
widget.notifier.removeListener(_onAppChanged);
_search.dispose();
_store.dispose();
diff --git a/desktop/test/store_screen_test.dart b/desktop/test/store_screen_test.dart
index 4e3e50229..b255517de 100644
--- a/desktop/test/store_screen_test.dart
+++ b/desktop/test/store_screen_test.dart
@@ -280,6 +280,26 @@ void main() {
},
);
+ testWidgets(
+ 'an open Store refreshes its catalog and stops polling when closed',
+ (tester) async {
+ final (notifier, _) = await open(tester);
+ notifier.machineStates['machine-1']!.connectionStatus =
+ ConnectionStatus.connected;
+ notifier.notifyListeners();
+ await tester.pump();
+ final probes = notifier.probes;
+ final engines = notifier.engineProbes;
+ await tester.pump(const Duration(minutes: 1));
+ expect(notifier.probes, probes + 1);
+ expect(notifier.engineProbes, engines);
+ await tester.pumpWidget(const SizedBox());
+ await tester.pump(const Duration(minutes: 2));
+ expect(notifier.probes, probes + 1);
+ expect(tester.takeException(), isNull);
+ },
+ );
+
testWidgets(
'a machine that is offline, unlinked or on an older CLI says so instead of offering Get',
(tester) async {
diff --git a/store/README.md b/store/README.md
index 5e87d17be..1c3170f8f 100644
--- a/store/README.md
+++ b/store/README.md
@@ -127,7 +127,7 @@ Two ways, depending on where the package lives.
**In this repository**, as a built-in: the folder is `store/agents/<name>` (or `store/viewers/<name>`),
its id `autonomous/<name>`, and beside `harness.json` a `store.json` with what the store page shows
-that a manifest does not know. The CLI build lists every such folder; there is no entry to write.
+that a manifest does not know. The catalog publisher lists every such folder; there is no entry to write.
```json
{ "homepage": "https://typst.app", "upstream": "https://github.com/typst/typst", "license": "MIT",
@@ -147,10 +147,40 @@ matching viewer ID) in the registry entry so the Store can show its dependencies
Built-in entries derive this field from the manifest automatically.
A package that is one folder of a bigger repository names it with `"path"`; install then fetches
-that folder alone. Run the conformance check and include a real example in the PR; CI is currently
-triggered manually. Once merged and included in a CLI release, the app offers the tile before the
-package is installed and installs it on Create; `verified: true` marks first-party packages, which
-every built-in is, and everything else shows its git URL on install.
+that folder alone. Run the conformance check and include a real example in the PR; the full test
+suite is currently triggered manually. Once merged, the catalog publisher lists it automatically.
+The app offers the tile before the package is installed. `verified: true` marks built-in packages;
+community packages show their source on install.
+
+### Live catalog
+
+Package changes merged into `main` trigger
+[`Publish Store catalog`](../.github/workflows/publish-store-catalog.yml). It validates the complete
+catalog, checks viewer dependencies, pins built-in packages to the source commit, and writes one
+`catalog.json` to the `store-catalog` branch. It uses GitHub only; it does not build or release the
+app, run package setup scripts, or access cloud infrastructure. Authors do not edit a generated index.
+
+The CLI reads that public JSON over HTTPS, using conditional requests and a five-minute cache.
+Concurrent requests share one fetch. An open Store asks connected machines again every minute;
+reopening it also asks. New harnesses and shared viewers appear without a client restart or another
+app/CLI release. GitHub caching and the refresh interval mean publication is not instantaneous.
+
+The last validated catalog is saved on disk. If GitHub is unavailable, slow, or serves an invalid
+response, clients keep that catalog; a first offline launch uses the bundled catalog. Installed
+harnesses remain listed even if their Store entry is removed. Refreshing metadata does not install,
+update, or execute packages. Get resolves both the harness and its viewer from the live catalog,
+and an already-installed viewer is reused.
+
+Validate publication locally without writing to GitHub:
+
+```bash
+node store/tools/catalog.mjs
+```
+
+`HARNESS_STORE_CATALOG_URL` selects a catalog mirror or a loopback HTTP fixture for development.
+`HARNESS_STORE_REF` still overrides built-in install refs for testing a branch. Existing clients
+need the CLI version containing the live reader once; subsequent package publications do not need
+releases. Changes to the runtime or package protocol can still require a client update.
## Tiers
@@ -186,7 +216,8 @@ The packages Autonomous maintains are folders here, and the rules below hold for
`blender`, `text-to-cad`, `autonomous-circuit`. A wrapper never renames what it wraps.
- **One place per fact.** Name, category, author, description and engine are the manifest's; homepage,
upstream, licence and screenshots are `store.json`'s. The CLI build turns each folder into its
- registry entry (repo this repository, ref `main`, path the folder, tier from what the manifest ships).
+ registry entry (repo this repository, path the folder, tier from what the manifest ships). The
+ live publisher pins its ref to the source commit; the offline build fallback uses `main`.
- **`harness dsh check` passes** on the folder as it is committed.
- **Credit travels with the code.** A `LICENSE` for the wrapper, the upstream's licence beside it when
anything of theirs is in the folder, and a README whose "Credit and stewardship" section says whose
diff --git a/store/tools/catalog.mjs b/store/tools/catalog.mjs
new file mode 100644
index 000000000..b9295ed6c
--- /dev/null
+++ b/store/tools/catalog.mjs
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+// Publish metadata, never run a harness's setup or load its code. No npm dependencies required.
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { readDshRegistry } from '../../cli/scripts/lib/dshRegistry.mjs';
+
+const MONOREPO='https://github.com/autonomous-ai/openharness';
+const ID=/^[a-z0-9][a-z0-9-]{0,63}\/[a-z0-9][a-z0-9-]{0,63}$/;
+const PACKAGE_PATH=/^(?!\/)(?!.*\/$)(?!.*\/\/)(?!(?:.*\/)?\.{1,2}(?:\/|$))[A-Za-z0-9._\-/]+$/;
+const LIMITS={id:160,name:40,description:300,category:24,author:80,repo:2048,ref:200,path:512,homepage:2048,upstream:2048,license:40,viewerUse:160};
+
+export function createStoreCatalog(storeDir, ref) {
+ if (!/^[a-f0-9]{40}$/i.test(ref)) throw new Error('Publish from a complete git commit SHA');
+ const entries=readDshRegistry(storeDir,{strict:true}).sort((a,b)=>a.id.localeCompare(b.id));
+ const byId=new Map();
+ for (const entry of entries) {
+ for(const [key,max]of Object.entries(LIMITS))if(entry[key]!==undefined&&(typeof entry[key]!=='string'||entry[key].length>max||/[\x00-\x1f\x7f]/.test(entry[key])))throw new Error(`${entry.id}: invalid ${key}`);
+ if(!ID.test(entry.id)||!entry.name||!entry.repo||byId.has(entry.id))throw new Error(`Invalid or duplicate catalog identity: ${entry.id}`);
+ if(entry.kind!==undefined&&!['agent','viewer'].includes(entry.kind))throw new Error(`${entry.id}: invalid kind`);
+ if(entry.kind!=='viewer'&&(typeof entry.engine!=='string'||!entry.engine))throw new Error(`${entry.id}: missing engine`);
+ const url=new URL(entry.repo);
+ if(url.protocol!=='https:'||url.username||url.password)throw new Error(`${entry.id}: use a public HTTPS repository`);
+ if(entry.path!==undefined&&!PACKAGE_PATH.test(entry.path))throw new Error(`${entry.id}: invalid package path`);
+ if(entry.viewerUse!==undefined&&!ID.test(entry.viewerUse))throw new Error(`${entry.id}: invalid viewer dependency`);
+ if(entry.tier!==undefined&&![0,1,2].includes(entry.tier))throw new Error(`${entry.id}: invalid tier`);
+ for(const key of ['homepage','upstream'])if(entry[key]!==undefined)new URL(entry[key]);
+ if(entry.screenshots!==undefined){if(!Array.isArray(entry.screenshots)||entry.screenshots.length>8)throw new Error(`${entry.id}: invalid screenshots`);for(const shot of entry.screenshots){if(typeof shot!=='string'||shot.length>2048)throw new Error(`${entry.id}: invalid screenshot`);new URL(shot);}}
+ const builtIn=entry.repo.replace(/\.git$/,'')===MONOREPO&&entry.path===`store/${entry.kind==='viewer'?'viewers':'agents'}/${entry.id.slice('autonomous/'.length)}`&&entry.id.startsWith('autonomous/');
+ if(entry.id.startsWith('autonomous/')&&!builtIn)throw new Error(`${entry.id}: autonomous IDs belong to built-in packages`);
+ if(builtIn)entry.ref=ref;
+ entry.verified=builtIn;
+ byId.set(entry.id,entry);
+ }
+ for(const entry of entries)if(entry.viewerUse&&byId.get(entry.viewerUse)?.kind!=='viewer')throw new Error(`${entry.id}: viewer ${entry.viewerUse} must be listed in the same catalog`);
+ if(!entries.length)throw new Error('Refusing to publish an empty Store');
+ return {spec:1,entries};
+}
+
+/** One immutable catalog commit; optimistic ref updates cannot overwrite another publication. */
+export async function publishStoreCatalog(catalog, {repository='autonomous-ai/openharness',token,fetch:request=fetch}={}) {
+ if(!token)throw new Error('GITHUB_TOKEN is required to publish the catalog');
+ const body=JSON.stringify(catalog,null,2)+'\n';
+ const blobSha=createHash('sha1').update(`blob ${Buffer.byteLength(body)}\0`).update(body).digest('hex');
+ const api=async(path,method='GET',data)=>{
+ const response=await request(`https://api.github.com/repos/${repository}/${path}`,{method,headers:{authorization:`Bearer ${token}`,accept:'application/vnd.github+json','content-type':'application/json','x-github-api-version':'2022-11-28'},...(data?{body:JSON.stringify(data)}:{}),signal:AbortSignal.timeout(30_000)});
+ if(response.status===404&&method==='GET')return null;
+ if(!response.ok)throw new Error(`Catalog publication failed: ${method} ${path} (${response.status})`);
+ return response.json();
+ };
+ const current=await api('git/ref/heads/store-catalog');
+ if(current){const previous=await api(`git/commits/${current.object.sha}`);const tree=await api(`git/trees/${previous.tree.sha}`);if(tree?.tree?.length===1&&tree.tree[0].path==='catalog.json'&&tree.tree[0].sha===blobSha)return {changed:false,sha:current.object.sha};}
+ const blob=await api('git/blobs','POST',{content:body,encoding:'utf-8'});
+ const tree=await api('git/trees','POST',{tree:[{path:'catalog.json',mode:'100644',type:'blob',sha:blob.sha}]});
+ const commit=await api('git/commits','POST',{message:'Publish Harness Store catalog',tree:tree.sha,parents:current?[current.object.sha]:[]});
+ if(current)await api('git/refs/heads/store-catalog','PATCH',{sha:commit.sha,force:false});
+ else await api('git/refs','POST',{ref:'refs/heads/store-catalog',sha:commit.sha});
+ return {changed:true,sha:commit.sha};
+}
+
+if(process.argv[1]&&resolve(process.argv[1])===fileURLToPath(import.meta.url)){
+ const args=process.argv.slice(2),value=flag=>args[args.indexOf(flag)+1];
+ const storeDir=resolve(args.includes('--store')?value('--store'):fileURLToPath(new URL('..',import.meta.url)));
+ const ref=args.includes('--ref')?value('--ref'):execFileSync('git',['-C',storeDir,'rev-parse','HEAD'],{encoding:'utf8'}).trim();
+ const catalog=createStoreCatalog(storeDir,ref);
+ if(args.includes('--output'))writeFileSync(value('--output'),JSON.stringify(catalog,null,2)+'\n');
+ if(args.includes('--publish'))console.log(await publishStoreCatalog(catalog,{token:process.env.GITHUB_TOKEN,repository:process.env.GITHUB_REPOSITORY||'autonomous-ai/openharness'}));
+ else console.log(`Catalog valid: ${catalog.entries.length} packages at ${ref}`);
+}