The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Load the Harness Store catalog independently of CLI releases
deed2fbfce94171
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex e618b2def..8f9a1ef02 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -26,7 +26,7 @@ cd ../my-first-harness```The first command links the shared viewer from this checkout. Store installations normally install-that dependency automatically; this also lets you try a new viewer before it reaches a release.+that dependency automatically; this also lets you try a new viewer before it is published.**Agent + viewer = harness.** This example has three working files:@@ -118,10 +118,13 @@ You do not need permission to build a harness or share its repository. Choose thsubmit a small entry at `store/registry/<owner>/<name>.json`. You keep the code and its maintenance.- **This repository:** contribute a folder at `store/agents/<name>/` with a `harness.json` and`store.json`. Built-in package IDs use `autonomous/<name>`; the `author` field credits the actual- author or upstream project. The CLI build generates the listing from those files.+ author or upstream project. The catalog publisher generates the listing from those files.The [publishing examples](store/README.md#publish-a-harness) show the exact metadata for both routes.-A Store listing reaches users with a CLI release after it is merged.+A merge to `main` publishes the live catalog automatically. Running clients pick it up within+minutes; no app or CLI release is needed for a new harness or viewer that uses the existing+package format. The catalog describes packages; installation still happens when someone chooses+Get. Installed packages are not silently replaced.For a first pull request, give reviewers a short path to the same result you saw:@@ -165,8 +168,8 @@ Security reports go through [SECURITY.md](SECURITY.md).currently run manually through **Actions → CI → Run workflow**; a PR does not automaticallyexercise the app, real engines, or hardware. Report those checks separately.5. **Merge and release.** PRs are squash-merged. Rebase on the latest `main` when needed to keep- the diff readable. Merging and publishing are separate: app, CLI, and firmware releases have- their own schedules.+ the diff readable. Harness catalog changes publish automatically after merge. App, CLI, and+ firmware releases have their own schedules.## Conventions across this repositorydiff --git a/cli/scripts/lib/dshRegistry.mjs b/cli/scripts/lib/dshRegistry.mjsindex 33734fe2f..cfe95eb3d 100644--- a/cli/scripts/lib/dshRegistry.mjs+++ b/cli/scripts/lib/dshRegistry.mjs@@ -22,17 +22,19 @@ export function storeEntry(path, manifest, facts) {return entry}-function readStoreDir(storeDir) {+function readStoreDir(storeDir, strict) {const out = []for (const plural of ['agents', 'viewers']) {let namestry { names = readdirSync(join(storeDir, plural)).sort() } catch { continue }for (const name of names) {const dir = join(storeDir, plural, name)+ if (!statSync(dir).isDirectory() || name.startsWith('.')) continuelet manifest- try { manifest = JSON.parse(readFileSync(join(dir, 'harness.json'), 'utf8')) } catch { continue }+ try { manifest = JSON.parse(readFileSync(join(dir, 'harness.json'), 'utf8')) } catch (error) { if (strict) throw error; continue }+ if (strict && (manifest.spec !== 1 || manifest.id !== `autonomous/${name}` || (manifest.kind ?? 'agent') !== (plural === 'agents' ? 'agent' : 'viewer'))) throw new Error(`Invalid package identity: ${dir}`)let facts = {}- try { facts = JSON.parse(readFileSync(join(dir, 'store.json'), 'utf8')) } catch { facts = {} }+ try { facts = JSON.parse(readFileSync(join(dir, 'store.json'), 'utf8')) } catch (error) { if (strict) throw error; facts = {} }out.push(storeEntry(`store/${plural}/${name}`, manifest, facts))}}@@ -61,7 +63,7 @@ function readRegistryDir(root) {}/** `storeDir` is the repo's `store/` folder, as a path or a file URL. */-export function readDshRegistry(storeDir) {+export function readDshRegistry(storeDir, { strict = false } = {}) {const root = storeDir instanceof URL ? fileURLToPath(storeDir) : storeDir- return [...readStoreDir(root), ...readRegistryDir(join(root, 'registry'))]+ return [...readStoreDir(root, strict), ...readRegistryDir(join(root, 'registry'))]}diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.tsindex b16ce9a6c..70703e10d 100644--- a/cli/src/backendSocket.spec.ts+++ b/cli/src/backendSocket.spec.ts@@ -10,6 +10,7 @@ import { registry, type RegisteredSession } from './lib/registry.js'import * as mediaPreview from './lib/mediaPreview.js'import * as projectFolder from './lib/projectFolder.js'import * as projectPreview from './lib/projectPreview.js'+import * as storeCatalog from './dsh/catalog.js'import { randomUUID } from 'node:crypto'const wsMock = vi.hoisted(() => {@@ -680,6 +681,25 @@ describe('BackendSocket outbound queue', () => {await socket.stop()})+ it('serves live catalog entries without making unrelated RPCs wait for catalog I/O', async () => {+ let finish!: (entries: Awaited<ReturnType<typeof storeCatalog.refreshDshRegistry>>) => void+ vi.spyOn(storeCatalog, 'refreshDshRegistry').mockImplementation(() => new Promise(resolve => { finish = resolve }))+ const socket = new BackendSocket('token')+ socket.runtimeModelsProvider = async () => []+ const frames: Array<Record<string, unknown>> = []+ socket.registerLocalClient('local:catalog', { sendFrame: frame => { frames.push(frame); return true }, sendBinary: () => true })+ socket.handleLocalFrame('local:catalog', { type: 'dsh_list', payload: { requestId: 'catalog' } })+ socket.handleLocalFrame('local:catalog', { type: 'models_list', payload: { requestId: 'models' } })+ await vi.waitFor(() => expect(frames.some(frame => frame.type === 'models_list_result')).toBe(true))+ expect(frames.some(frame => frame.type === 'dsh_list_result')).toBe(false)+ finish([{ id: 'acme/published-today', name: 'Published today', engine: 'claude', repo: 'https://example.test/project', tier: 2, viewerUse: 'acme/viewer' }])+ await vi.waitFor(() => expect(frames).toContainEqual({+ type: 'dsh_list_result', payload: { requestId: 'catalog', dsh: [expect.objectContaining({ id: 'acme/published-today', installed: false, viewerUse: 'acme/viewer' })] },+ }))+ await socket.unregisterLocalClient('local:catalog')+ await socket.stop()+ })+it('does not let a slow engines_probe block agent_create on the same connection', async () => {const socket = new BackendSocket('token')const frames: Array<Record<string, unknown>> = []diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex 18e0a2577..558a3cf68 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -38,6 +38,7 @@ import { projectPreview } from './lib/projectPreview.js'import { agentFrame, type AgentDshContext, type AgentFrame } from './lib/agentFrame.js'import { installedDsh } from './dsh/installed.js'import { DSH_ID_RE } from './dsh/manifest.js'+import { refreshDshRegistry } from './dsh/catalog.js'import type { DshInstallProgress } from './dsh/install.js'import { dshInstallReply, dshInstallRequest, dshInstallStatus, dshListRows, dshRemoveId, dshRemoveReply } from './dsh/wire.js'import { routeVoiceTask } from './lib/voiceRouter.js'@@ -1469,9 +1470,10 @@ export class BackendSocket {}case 'dsh_list': {- // Which domain-specific harnesses this machine has, plus what the bundled registry offers —- // answered here, on the machine in question, for the same reason `engines_probe` is.- reply(type, requestId, { dsh: dshListRows() })+ // Keep catalog I/O off this connection's ordered RPC queue.+ void refreshDshRegistry()+ .then(catalog => reply(type, requestId, { dsh: dshListRows(undefined, catalog) }))+ .catch(error => reply(type, requestId, { error: 'INTERNAL', detail: error instanceof Error ? error.message : String(error) }))return}diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 848d78654..435cd7260 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -73,7 +73,7 @@ import { buildLaunchOverrides, validateLaunchOverrides, type LaunchOverridesDepsimport { buildHarnessSessionLabel } from './lib/harnessSessionLabel.js'import { installedDsh } from './dsh/installed.js'import { dshVerdictPath } from './dsh/manifest.js'-import { registryEntry } from './dsh/registry.js'+import { catalogEntry, refreshDshRegistry } from './dsh/catalog.js'import { installDsh, resolveInstallSource, removeDsh } from './dsh/install.js'import { preTrustClaudeProject, preTrustCodexProject } from './lib/claudeTrust.js'import { materializeWorkspace } from './dsh/materialize.js'@@ -1374,7 +1374,7 @@ async function runForeground(session: AuthSession): Promise<void> {return {// The current id, so a face drawn by id survives a rename the agent predates.id: installed?.id ?? s.dsh,- name: installed?.manifest.name ?? registryEntry(s.dsh)?.name ?? null,+ name: installed?.manifest.name ?? catalogEntry(s.dsh)?.name ?? null,viewerUrl: state?.viewerUrl ?? null,verdict: state?.verdict ?? null,}@@ -2110,6 +2110,8 @@ async function runForeground(session: AuthSession): Promise<void> {backend.dshFrameProvider = dshFrameContextbackend.onDshRemove = (id) => removeDsh(id)backend.onDshInstall = async ({ id, url, ref }, progress) => {+ const catalog = new Map((await refreshDshRegistry(!!id && !catalogEntry(id))).map(entry => [entry.id, entry]))+ if (id && !catalog.has(id)) return { ok: false, error: 'INVALID_DSH', detail: `${id} is not in this machine's Store catalog` }const resolved = id ? resolveInstallSource(id) : url ? { source: url, ref } : nullif (!resolved) return { ok: false, error: 'INVALID_DSH', detail: `${id ?? url} is not a known harness` }// NARRATE THE LINES, NOT ONLY THE PHASES. A toolchain setup is minutes of npm and uv output, and@@ -2128,6 +2130,8 @@ async function runForeground(session: AuthSession): Promise<void> {}const result = await installDsh({source: resolved.source,+ expectedId: id,+ registry: dependencyId => catalog.get(dependencyId),ref: ref ?? resolved.ref,path: 'path' in resolved ? resolved.path : undefined,onProgress: (p) => {diff --git a/cli/src/config/env.ts b/cli/src/config/env.tsindex 18c720003..cc2c8534e 100644--- a/cli/src/config/env.ts+++ b/cli/src/config/env.ts@@ -241,6 +241,12 @@ const envSchema = z.object({// its registry entries name (`main`). For trying a store change end to end BEFORE it merges: push// the branch, run the daemon with HARNESS_STORE_REF=<branch>, and Get in the store fetches from it.HARNESS_STORE_REF: z.string().regex(/^[A-Za-z0-9._\/-]{1,200}$/).optional().catch(undefined),+ // Optional catalog mirror. Public HTTPS in production; loopback HTTP supports isolated tests.+ HARNESS_STORE_CATALOG_URL: z.string().url().refine((value) => {+ const url = new URL(value)+ return !url.username && !url.password && (url.protocol === 'https:'+ || (url.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname)))+ }).optional().catch(undefined),// This computer's stable id, minted once and never regenerated (see computerIdFile above). Pin it// explicitly on a box with no durable home — a container or CI job that gets a fresh ~/.harness on// every boot would otherwise look like a NEW computer each time and collect a machine per start.diff --git a/cli/src/dsh/catalog.spec.ts b/cli/src/dsh/catalog.spec.tsnew file mode 100644index 000000000..33b4a015f--- /dev/null+++ b/cli/src/dsh/catalog.spec.ts@@ -0,0 +1,167 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { join } from 'node:path'+import { tmpdir } from 'node:os'+import { execFileSync } from 'node:child_process'+import { CATALOG_REFRESH_MS, LiveStoreCatalog, parseStoreCatalog, refreshDshRegistry, resetLiveDshRegistry } from './catalog.js'+import { env } from '../config/env.js'+import { dshCommand } from './command.js'+import { resolveInstallSource } from './install.js'+import { installedDsh, invalidateInstalledDsh } from './installed.js'+import type { DshRegistryEntry } from './registry.js'++const first: DshRegistryEntry = { id: 'acme/new-game', name: 'New Game', engine: 'claude', repo: 'https://example.test/game', viewerUse: 'acme/new-viewer' }+const viewer: DshRegistryEntry = { id: 'acme/new-viewer', kind: 'viewer', name: 'New Viewer', repo: 'https://example.test/viewer' }+const fallback: DshRegistryEntry[] = [{ id: 'acme/bundled', name: 'Bundled', engine: 'codex', repo: 'https://example.test/bundled' }]+const document = (entries = [first, viewer]) => ({ spec: 1, entries })+const response = (entries = [first, viewer]) => new Response(JSON.stringify(document(entries)), { headers: { etag: '"v1"' } })++describe('live Store catalog', () => {+ let root: string+ let now: number+ let fetcher: ReturnType<typeof vi.fn<typeof fetch>>+ beforeEach(() => { root = mkdtempSync(join(tmpdir(), 'dsh-catalog-')); now = 1000; fetcher = vi.fn<typeof fetch>() })+ afterEach(() => { rmSync(root, { recursive: true, force: true }); vi.restoreAllMocks() })+ const make = () => new LiveStoreCatalog({ url: 'https://catalog.example.test/catalog.json', cacheFile: join(root, 'catalog.json'), fallback: () => fallback, fetch: fetcher, now: () => now })++ it('discovers new entries without rebuilding; removals do not reappear from the bundle', async () => {+ fetcher.mockResolvedValueOnce(response()).mockResolvedValueOnce(response([viewer]))+ const catalog = make()+ expect(catalog.current()).toEqual(fallback)+ expect((await catalog.refresh()).map(e => e.id)).toEqual([first.id, viewer.id])+ now += CATALOG_REFRESH_MS+ expect((await catalog.refresh()).map(e => e.id)).toEqual([viewer.id])+ })++ it('coalesces concurrent readers and uses the ETag after the refresh interval', async () => {+ fetcher.mockResolvedValueOnce(response()).mockResolvedValueOnce(new Response(null, { status: 304 }))+ const catalog = make()+ await Promise.all(Array.from({ length: 25 }, () => catalog.refresh()))+ await catalog.refresh()+ expect(fetcher).toHaveBeenCalledTimes(1)+ now += CATALOG_REFRESH_MS+ await catalog.refresh()+ expect(fetcher).toHaveBeenCalledTimes(2)+ expect(fetcher.mock.calls[1][1]?.headers).toMatchObject({ 'if-none-match': '"v1"' })+ expect(catalog.current()).toHaveLength(2)+ expect(JSON.parse(readFileSync(join(root, 'catalog.json'), 'utf8')).checkedAt).toBe(now)+ })++ it('restores a valid disk cache without a fetch and keeps it during an outage', async () => {+ fetcher.mockResolvedValueOnce(response())+ await make().refresh()+ const restarted = make()+ expect((await restarted.refresh()).map(e => e.id)).toContain(first.id)+ expect(fetcher).toHaveBeenCalledTimes(1)+ now += CATALOG_REFRESH_MS+ fetcher.mockRejectedValue(new Error('offline'))+ expect((await restarted.refresh()).map(e => e.id)).toContain(first.id)+ await restarted.refresh()+ expect(fetcher).toHaveBeenCalledTimes(2)+ })++ it.each([+ ['invalid JSON', () => new Response('{broken')],+ ['future envelope', () => new Response(JSON.stringify({ spec: 2, entries: [first] }))],+ ['damaged entry', () => new Response(JSON.stringify(document([{ ...first, repo: '' }])))],+ ['duplicate identity', () => response([first, first])],+ ['HTTP failure', () => new Response('', { status: 503 })],+ ['oversized body', () => new Response('ignored', { headers: { 'content-length': String(17 * 1024 * 1024) } })],+ ])('keeps the last good catalog after %s', async (_name, bad) => {+ fetcher.mockResolvedValueOnce(response()).mockResolvedValueOnce(bad())+ const catalog = make()+ await catalog.refresh()+ const before = readFileSync(join(root, 'catalog.json'), 'utf8')+ now += CATALOG_REFRESH_MS+ expect((await catalog.refresh()).map(e => e.id)).toContain(first.id)+ expect(readFileSync(join(root, 'catalog.json'), 'utf8')).toBe(before)+ })++ it('falls back on first offline launch or a corrupt cache', async () => {+ writeFileSync(join(root, 'catalog.json'), '{bad')+ fetcher.mockRejectedValue(new Error('offline'))+ expect(await make().refresh()).toEqual(fallback)+ })++ it('never loads a cache belonging to a different source', async () => {+ fetcher.mockResolvedValueOnce(response())+ await make().refresh()+ const file = join(root, 'catalog.json')+ const saved = JSON.parse(readFileSync(file, 'utf8'))+ writeFileSync(file, JSON.stringify({ ...saved, url: 'https://different.test/catalog.json' }))+ expect(make().current()).toEqual(fallback)+ })++ it('bounds a network stall and returns the bundled catalog', async () => {+ fetcher.mockImplementation((_url, init) => new Promise((_resolve, reject) => init!.signal!.addEventListener('abort', () => reject(new Error('timeout')))))+ const catalog = new LiveStoreCatalog({ url: 'https://catalog.test', cacheFile: join(root, 'cache'), fallback: () => fallback, fetch: fetcher, timeoutMs: 15 })+ expect(await catalog.refresh()).toEqual(fallback)+ })++ it('accepts future optional fields and skips unsupported engines without losing supported ones', () => {+ const parsed = parseStoreCatalog(document([{ ...first, futureField: true }, { ...first, id: 'acme/future', engine: 'future-engine' }, viewer] as DshRegistryEntry[]))+ expect(parsed.entries.map(e => e.id)).toEqual([first.id, viewer.id])+ })++ it('does not trust a community package claiming to be first-party', () => {+ expect(parseStoreCatalog(document([{ ...first, verified: true }])).entries[0].verified).toBe(false)+ expect(() => parseStoreCatalog(document([{ ...first, repo: '/private/project' }]))).toThrow()+ expect(() => parseStoreCatalog(document([{ ...first, repo: 'https://secret:[email protected]/repo' }]))).toThrow()+ })++ it('handles thousands of packages in one response', async () => {+ const entries = Array.from({ length: 5000 }, (_, i) => ({ ...first, id: `acme/game-${i}` }))+ fetcher.mockResolvedValueOnce(response(entries))+ expect(await make().refresh()).toHaveLength(5000)+ expect(fetcher).toHaveBeenCalledTimes(1)+ })+})++describe('catalog → CLI list → install → shared viewer', () => {+ let root: string+ const saved = { dsh: env.DSH_DIR, url: env.HARNESS_STORE_CATALOG_URL }+ beforeEach(() => {+ root = mkdtempSync(join(tmpdir(), 'dsh-live-install-'))+ env.DSH_DIR = join(root, 'installed'); env.HARNESS_STORE_CATALOG_URL = 'https://catalog.example.test/live.json'+ invalidateInstalledDsh(); resetLiveDshRegistry()+ })+ afterEach(() => {+ env.DSH_DIR = saved.dsh; env.HARNESS_STORE_CATALOG_URL = saved.url+ resetLiveDshRegistry(); invalidateInstalledDsh(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); vi.restoreAllMocks()+ rmSync(root, { recursive: true, force: true })+ })++ it('installs a newly published harness and viewer from one pinned commit, without a new CLI', async () => {+ const repo = join(root, 'source'); mkdirSync(repo)+ const git = (...args: string[]) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', env: { ...process.env, GIT_AUTHOR_NAME: 'test', GIT_AUTHOR_EMAIL: '[email protected]', GIT_COMMITTER_NAME: 'test', GIT_COMMITTER_EMAIL: '[email protected]' } }).trim()+ for (const [folder, manifest] of [+ ['game', { spec: 1, id: first.id, name: first.name, engine: 'claude', viewer: { use: viewer.id } }],+ ['viewer', { spec: 1, id: viewer.id, name: viewer.name, kind: 'viewer', viewer: { command: 'node viewer.mjs', url: 'http://127.0.0.1:${port}/' } }],+ ] as const) { mkdirSync(join(repo, folder)); writeFileSync(join(repo, folder, 'harness.json'), JSON.stringify(manifest)) }+ writeFileSync(join(repo, 'viewer', 'viewer.mjs'), '// fixture viewer\n')+ git('init', '-q', '-b', 'main'); git('add', '.'); git('commit', '-qm', 'published packages')+ const ref = git('rev-parse', 'HEAD')+ writeFileSync(join(repo, 'game', 'unpublished.txt'), 'This later commit is not in the catalog')+ git('add', '.'); git('commit', '-qm', 'unpublished change')+ // All Git traffic stays in the fixture repository. No global Git configuration is changed.+ vi.stubEnv('GIT_CONFIG_COUNT', '1')+ vi.stubEnv('GIT_CONFIG_KEY_0', `url.file://${repo}.insteadOf`)+ vi.stubEnv('GIT_CONFIG_VALUE_0', 'https://example.test/packages')+ const entries = [{ ...first, repo: 'https://example.test/packages', path: 'game', ref }, { ...viewer, repo: 'https://example.test/packages', path: 'viewer', ref }]+ vi.stubGlobal('fetch', vi.fn<typeof fetch>().mockResolvedValue(response(entries)))+ const output = vi.spyOn(console, 'log').mockImplementation(() => {})+ expect(await dshCommand('list', [])).toBe(0)+ expect(output.mock.calls.flat().join('\n')).toContain('acme/new-game')+ expect(installedDsh(first.id)).toBeUndefined()+ expect(installedDsh(viewer.id)).toBeUndefined()+ expect(resolveInstallSource(first.id)).toMatchObject({ source: entries[0].repo, path: 'game', ref })+ expect(await dshCommand('install', [first.id])).toBe(0)+ expect(installedDsh(first.id)?.commit).toBe(ref)+ expect(installedDsh(viewer.id)?.commit).toBe(ref)+ const installedAt = installedDsh(viewer.id)?.installedAt+ expect(await dshCommand('install', [first.id])).toBe(0)+ expect(installedDsh(viewer.id)?.installedAt).toBe(installedAt)+ expect(fetch).toHaveBeenCalledTimes(1)+ expect((await refreshDshRegistry()).map(e => e.id)).toContain(first.id)+ })+})diff --git a/cli/src/dsh/catalog.ts b/cli/src/dsh/catalog.tsnew file mode 100644index 000000000..fc6acb739--- /dev/null+++ b/cli/src/dsh/catalog.ts@@ -0,0 +1,147 @@+/** Live Store metadata. Fetching a catalog never installs or executes a package. */+import { createHash, randomUUID } from 'node:crypto'+import { mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'+import { dirname, join } from 'node:path'+import { env } from '../config/env.js'+import { ENGINES } from '../engines/types.js'+import { bundledDshRegistry, DshRegistryEntrySchema, HARNESS_MONOREPO, type DshRegistryEntry } from './registry.js'++export const STORE_CATALOG_URL = 'https://raw.githubusercontent.com/autonomous-ai/openharness/store-catalog/catalog.json'+export const CATALOG_REFRESH_MS = 5 * 60_000+const RETRY_MS = 30_000+const MAX_BYTES = 16 * 1024 * 1024++export interface StoreCatalog { spec: 1; entries: DshRegistryEntry[] }++/** Ignore future optional fields/engines, but never cache a damaged or ambiguous catalog. */+export function parseStoreCatalog(value: unknown): StoreCatalog {+ if (!value || typeof value !== 'object') throw new Error('Invalid Store catalog')+ const catalog = value as Record<string, unknown>+ if (catalog.spec !== 1 || !Array.isArray(catalog.entries) || catalog.entries.length > 25_000) throw new Error('Unsupported Store catalog')+ const entries: DshRegistryEntry[] = []+ const seen = new Set<string>()+ for (const raw of catalog.entries) {+ if (!raw || typeof raw !== 'object') throw new Error('Invalid catalog entry')+ const item = raw as Record<string, unknown>+ if (item.kind !== undefined && item.kind !== 'agent' && item.kind !== 'viewer') continue+ if (item.kind !== 'viewer' && typeof item.engine === 'string' && !(ENGINES as readonly string[]).includes(item.engine)) continue+ const parsed = DshRegistryEntrySchema.strip().parse(item)+ const url = new URL(parsed.repo)+ if (url.protocol !== 'https:' || url.username || url.password || /[\x00-\x20\x7f]/.test(parsed.repo)) throw new Error('Catalog sources must be public HTTPS repositories')+ if (seen.has(parsed.id)) throw new Error(`Duplicate catalog id: ${parsed.id}`)+ seen.add(parsed.id)+ // A community entry cannot grant itself first-party installation trust.+ const firstParty = parsed.repo.replace(/\.git$/, '') === HARNESS_MONOREPO+ && parsed.id.startsWith('autonomous/')+ && parsed.path === `store/${parsed.kind === 'viewer' ? 'viewers' : 'agents'}/${parsed.id.slice('autonomous/'.length)}`+ entries.push({ ...parsed, verified: firstParty && parsed.verified === true })+ }+ return { spec: 1, entries: entries.sort((a, b) => a.id.localeCompare(b.id)) }+}++interface Cache { url: string; checkedAt: number; etag?: string; catalog: StoreCatalog }+interface CatalogOptions {+ url: string+ cacheFile: string+ fallback: () => DshRegistryEntry[]+ fetch?: typeof fetch+ now?: () => number+ timeoutMs?: number+}++/** One request per refresh interval, shared by concurrent readers and persisted across restarts. */+export class LiveStoreCatalog {+ private cache: Cache | null = null+ private nextAttempt = 0+ private inFlight: Promise<DshRegistryEntry[]> | null = null+ private readonly now: () => number+ private readonly fetcher: typeof fetch++ constructor(private readonly options: CatalogOptions) {+ this.now = options.now ?? Date.now+ this.fetcher = options.fetch ?? fetch+ try {+ if (statSync(options.cacheFile).size > MAX_BYTES) return+ const saved = JSON.parse(readFileSync(options.cacheFile, 'utf8')) as Cache+ if (saved.url !== options.url || !Number.isFinite(saved.checkedAt)) return+ const catalog = parseStoreCatalog(saved.catalog)+ this.cache = { url: saved.url, checkedAt: Math.min(saved.checkedAt, this.now()), catalog,+ ...(typeof saved.etag === 'string' && saved.etag.length < 500 ? { etag: saved.etag } : {}) }+ this.nextAttempt = this.cache.checkedAt + CATALOG_REFRESH_MS+ } catch { /* First launch, unreadable cache, or old format: use the bundled shelf. */ }+ }++ current(): DshRegistryEntry[] { return this.cache?.catalog.entries ?? this.options.fallback() }++ refresh(force = false): Promise<DshRegistryEntry[]> {+ if (this.inFlight) return this.inFlight+ if (!force && this.now() < this.nextAttempt) return Promise.resolve(this.current())+ this.inFlight = this.update().finally(() => { this.inFlight = null })+ return this.inFlight+ }++ private async update(): Promise<DshRegistryEntry[]> {+ try {+ const response = await this.fetcher(this.options.url, {+ headers: { accept: 'application/json', ...(this.cache?.etag ? { 'if-none-match': this.cache.etag } : {}) },+ signal: AbortSignal.timeout(this.options.timeoutMs ?? 4_000),+ redirect: 'error',+ })+ if (response.status === 304 && this.cache) {+ this.cache = { ...this.cache, checkedAt: this.now() }+ } else {+ if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`)+ if (Number(response.headers.get('content-length')) > MAX_BYTES) throw new Error('Catalog too large')+ const reader = response.body?.getReader()+ if (!reader) throw new Error('Empty catalog response')+ const chunks: Uint8Array[] = []+ let bytes = 0+ try {+ while (true) {+ const { done, value } = await reader.read()+ if (done) break+ bytes += value.byteLength+ if (bytes > MAX_BYTES) throw new Error('Catalog too large')+ chunks.push(value)+ }+ } finally { await reader.cancel().catch(() => {}) }+ const catalog = parseStoreCatalog(JSON.parse(Buffer.concat(chunks).toString('utf8')))+ const etag = response.headers.get('etag')+ this.cache = { url: this.options.url, checkedAt: this.now(), catalog, ...(etag && etag.length < 500 ? { etag } : {}) }+ }+ this.nextAttempt = this.now() + CATALOG_REFRESH_MS+ // A read-only disk must not discard metadata that was fetched successfully.+ try {+ mkdirSync(dirname(this.options.cacheFile), { recursive: true, mode: 0o700 })+ const temporary = `${this.options.cacheFile}.${randomUUID()}.tmp`+ writeFileSync(temporary, JSON.stringify(this.cache), { mode: 0o600 })+ renameSync(temporary, this.options.cacheFile)+ } catch { /* The in-memory catalog still works. */ }+ } catch {+ this.nextAttempt = this.now() + RETRY_MS+ // Offline, rate-limited, timed out, or invalid: keep the last complete catalog.+ }+ return this.current()+ }+}++let live: { key: string; catalog: LiveStoreCatalog } | null = null+function liveCatalog(): LiveStoreCatalog {+ const url = env.HARNESS_STORE_CATALOG_URL ?? STORE_CATALOG_URL+ const cacheFile = join(env.DSH_DIR, '.catalog', `${createHash('sha256').update(url).digest('hex').slice(0, 20)}.json`)+ const key = `${cacheFile}:${url}`+ if (live?.key !== key) live = { key, catalog: new LiveStoreCatalog({ url, cacheFile, fallback: bundledDshRegistry }) }+ return live.catalog+}++function withStoreRef(entries: DshRegistryEntry[]): DshRegistryEntry[] {+ const ref = env.HARNESS_STORE_REF+ return ref ? entries.map(entry => entry.path && entry.repo.replace(/\.git$/, '') === HARNESS_MONOREPO ? { ...entry, ref } : entry) : entries+}++export function currentDshRegistry(): DshRegistryEntry[] { return withStoreRef(liveCatalog().current()) }+export async function refreshDshRegistry(force = false): Promise<DshRegistryEntry[]> { return withStoreRef(await liveCatalog().refresh(force)) }+export function catalogEntry(id: string): DshRegistryEntry | undefined { return currentDshRegistry().find(entry => entry.id === id) }++/** Test seam: also resets the source/cache identity when a test changes DSH_DIR. */+export function resetLiveDshRegistry(): void { live = null }diff --git a/cli/src/dsh/catalogPublisher.spec.ts b/cli/src/dsh/catalogPublisher.spec.tsnew file mode 100644index 000000000..a3a738e40--- /dev/null+++ b/cli/src/dsh/catalogPublisher.spec.ts@@ -0,0 +1,89 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import { createHash } from 'node:crypto'+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { fileURLToPath } from 'node:url'+import { parseStoreCatalog } from './catalog.js'+// @ts-expect-error — dependency-free ESM also runs directly in GitHub Actions.+import { createStoreCatalog, publishStoreCatalog } from '../../../store/tools/catalog.mjs'++const ref = 'a'.repeat(40)+const roots: string[] = []+afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) })+function fixture() {+ const root = mkdtempSync(join(tmpdir(), 'catalog-publisher-')); roots.push(root)+ mkdirSync(join(root, 'agents', 'game'), { recursive: true })+ writeFileSync(join(root, 'agents', 'game', 'harness.json'), JSON.stringify({ spec: 1, id: 'autonomous/game', name: 'Game', engine: 'claude' }))+ writeFileSync(join(root, 'agents', 'game', 'store.json'), '{}')+ return root+}++describe('Store publication', () => {+ it('generates a runtime-valid catalog from all real packages, pinning built-ins to the source commit', () => {+ const catalog = createStoreCatalog(fileURLToPath(new URL('../../../store', import.meta.url)), ref)+ const parsed = parseStoreCatalog(catalog)+ expect(parsed.entries).toHaveLength(catalog.entries.length)+ expect(parsed.entries.length).toBeGreaterThan(10)+ expect(parsed.entries.filter(entry => entry.verified).every(entry => entry.ref === ref)).toBe(true)+ })++ it('rejects corrupt manifests instead of silently publishing a partial catalog', () => {+ const root = fixture()+ writeFileSync(join(root, 'agents', 'game', 'harness.json'), '{bad')+ expect(() => createStoreCatalog(root, ref)).toThrow()+ })++ it('rejects a new harness whose shared viewer is not also published', () => {+ const root = fixture()+ const manifest = { spec: 1, id: 'autonomous/game', name: 'Game', engine: 'claude', viewer: { use: 'acme/not-listed' } }+ writeFileSync(join(root, 'agents', 'game', 'harness.json'), JSON.stringify(manifest))+ expect(() => createStoreCatalog(root, ref)).toThrow(/viewer acme\/not-listed/)+ })++ it('keeps community ownership and refs while preventing a verified flag from granting trust', () => {+ const root = fixture(); mkdirSync(join(root, 'registry', 'acme'), { recursive: true })+ writeFileSync(join(root, 'registry', 'acme', 'community.json'), JSON.stringify({ id: 'acme/community', name: 'Community', engine: 'codex', repo: 'https://github.com/acme/community', ref: 'v1', verified: true }))+ expect(createStoreCatalog(root, ref).entries.find((entry: { id: string }) => entry.id === 'acme/community')).toMatchObject({ verified: false, ref: 'v1' })+ })++ it('creates only catalog.json on the catalog branch, without modifying main or release tags', async () => {+ const requests: Array<{ path: string; method: string; data: Record<string, unknown> | null }> = []+ const request = vi.fn<typeof fetch>().mockImplementation(async (url, init) => {+ const path = String(url).replace('https://api.github.com/repos/autonomous-ai/openharness/', '')+ requests.push({ path, method: init?.method ?? 'GET', data: init?.body ? JSON.parse(String(init.body)) : null })+ if (path === 'git/ref/heads/store-catalog') return new Response('', { status: 404 })+ return Response.json({ sha: path === 'git/blobs' ? 'blob' : path === 'git/trees' ? 'tree' : 'commit' })+ })+ expect(await publishStoreCatalog(createStoreCatalog(fixture(), ref), { token: 'fixture-token', fetch: request })).toEqual({ changed: true, sha: 'commit' })+ expect(requests.map(item => item.path)).toEqual(['git/ref/heads/store-catalog', 'git/blobs', 'git/trees', 'git/commits', 'git/refs'])+ expect(requests.find(item => item.path === 'git/trees')?.data).toEqual({ tree: [{ path: 'catalog.json', mode: '100644', type: 'blob', sha: 'blob' }] })+ expect(requests.at(-1)?.data).toEqual({ ref: 'refs/heads/store-catalog', sha: 'commit' })+ })++ it('skips an unchanged publication', async () => {+ const catalog = createStoreCatalog(fixture(), ref)+ const body = JSON.stringify(catalog, null, 2) + '\n'+ const sha = createHash('sha1').update(`blob ${Buffer.byteLength(body)}\0`).update(body).digest('hex')+ const request = vi.fn<typeof fetch>()+ .mockResolvedValueOnce(Response.json({ object: { sha: 'existing' } }))+ .mockResolvedValueOnce(Response.json({ tree: { sha: 'tree' } }))+ .mockResolvedValueOnce(Response.json({ tree: [{ path: 'catalog.json', sha }] }))+ expect(await publishStoreCatalog(catalog, { token: 'fixture-token', fetch: request })).toEqual({ changed: false, sha: 'existing' })+ expect(request.mock.calls.every(([, init]) => init?.method === 'GET')).toBe(true)+ })++ it('updates by fast-forward and refuses a conflicting publication', async () => {+ const request = vi.fn<typeof fetch>()+ .mockResolvedValueOnce(Response.json({ object: { sha: 'existing' } }))+ .mockResolvedValueOnce(Response.json({ tree: { sha: 'tree' } }))+ .mockResolvedValueOnce(Response.json({ tree: [] }))+ .mockResolvedValueOnce(Response.json({ sha: 'blob' }))+ .mockResolvedValueOnce(Response.json({ sha: 'tree' }))+ .mockResolvedValueOnce(Response.json({ sha: 'commit' }))+ .mockResolvedValueOnce(new Response('', { status: 422 }))+ await expect(publishStoreCatalog(createStoreCatalog(fixture(), ref), { token: 'fixture-token', fetch: request })).rejects.toThrow(/422/)+ expect(JSON.parse(String(request.mock.calls.at(-1)?.[1]?.body))).toEqual({ sha: 'commit', force: false })+ expect(request).toHaveBeenCalledTimes(7)+ })+})diff --git a/cli/src/dsh/command.spec.ts b/cli/src/dsh/command.spec.tsindex 53d943260..ddf9bcd05 100644--- a/cli/src/dsh/command.spec.ts+++ b/cli/src/dsh/command.spec.ts@@ -47,6 +47,9 @@ describe('harness dsh', () => {env.DSH_DIR = join(root, 'installed')env.HARNESS_STORE_REF = undefinedinvalidateInstalledDsh()+ // These command-contract fixtures exercise the offline bundled fallback.+ // catalog.spec.ts covers live discovery and installation separately.+ vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline fixture')))out = []err = []vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { out.push(args.join(' ')) })diff --git a/cli/src/dsh/command.ts b/cli/src/dsh/command.tsindex a9bf147f5..fae48d6d3 100644--- a/cli/src/dsh/command.ts+++ b/cli/src/dsh/command.ts@@ -3,8 +3,9 @@* and for the development loop (`--link` a checkout, iterate, no re-clone).*/import { listDshState, installedDsh } from './installed.js'-import { dshTier } from './manifest.js'-import { bundledDshRegistry, registrySourceUrl } from './registry.js'+import { DSH_ID_RE, dshTier } from './manifest.js'+import { registrySourceUrl } from './registry.js'+import { catalogEntry, currentDshRegistry, refreshDshRegistry } from './catalog.js'import { installDsh, removeDsh, resolveInstallSource, runDshDoctor } from './install.js'import { checkDsh, formatCheck } from './check.js'@@ -33,7 +34,7 @@ export async function dshCommand(verb: string | undefined, rest: readonly stringswitch (verb) {case 'list': {const { installed, broken } = listDshState()- const registry = bundledDshRegistry()+ const registry = await refreshDshRegistry()const seen = new Set<string>()const rows: string[] = []for (const dsh of installed) {@@ -54,13 +55,17 @@ export async function dshCommand(verb: string | undefined, rest: readonly stringcase 'install': {const target = args[0]if (!target) { console.error(dshUsage()); return 1 }+ if (DSH_ID_RE.test(target) && !rest.includes('--link')) await refreshDshRegistry(!catalogEntry(target))const resolved = resolveInstallSource(target)if (!resolved) { console.error(`harness dsh install: ${target} is not an id, URL or path`); return 1 }const link = rest.includes('--link')const ref = flagValue(rest, '--ref') ?? resolved.refconst path = flagValue(rest, '--path') ?? resolved.path+ const catalog = new Map(currentDshRegistry().map(entry => [entry.id, entry]))const result = await installDsh({source: resolved.source,+ expectedId: resolved.id,+ registry: resolved.id ? id => catalog.get(id) : undefined,ref: link ? undefined : ref,path: link ? undefined : path,link,diff --git a/cli/src/dsh/install.spec.ts b/cli/src/dsh/install.spec.tsindex 7a07c85d0..8c0fff526 100644--- a/cli/src/dsh/install.spec.ts+++ b/cli/src/dsh/install.spec.ts@@ -79,6 +79,27 @@ describe('installDsh with a used viewer', () => {expect(asked).toBe(0)})+ it('can install a whole repository at the published commit after its branch advances', async () => {+ const repo = gitRepo(join(root, 'pinned'), { 'harness.json': JSON.stringify({ spec: 1, id: 'acme/pinned', name: 'Published', engine: 'claude' }) })+ const ref = execFileSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim()+ writeFileSync(join(repo, 'harness.json'), JSON.stringify({ spec: 1, id: 'acme/pinned', name: 'Unpublished', engine: 'claude' }))+ execFileSync('git', ['-C', repo, '-c', 'user.name=test', '-c', '[email protected]', 'commit', '-qam', 'later version'])+ const result = await installDsh({ source: repo, ref, expectedId: 'acme/pinned' })+ expect(result.ok, JSON.stringify(result)).toBe(true)+ expect(installedDsh('acme/pinned')?.commit).toBe(ref)+ expect(installedDsh('acme/pinned')?.manifest.name).toBe('Published')+ })++ it('rejects a mismatched catalog identity before installing or running setup', async () => {+ const repo = gitRepo(join(root, 'wrong-id'), {+ 'harness.json': JSON.stringify({ spec: 1, id: 'acme/surprise', name: 'Wrong', engine: 'claude', toolchain: { setup: './setup.sh' } }),+ 'setup.sh': '#!/bin/sh\ntouch did-run\n',+ })+ expect(await installDsh({ source: repo, expectedId: 'acme/expected' })).toMatchObject({ ok: false, error: 'PACKAGE_ID_MISMATCH' })+ expect(listInstalledDsh()).toEqual([])+ expect(readdirSync(env.DSH_DIR).filter(name => name.startsWith('.tmp-'))).toEqual([])+ })+it('a used viewer that is neither installed nor in the registry is said, and the harness still installs', async () => {const harnessRepo = gitRepo(join(root, 'src', 'thing'), {'harness.json': JSON.stringify({ spec: 1, id: 'acme/thing', name: 'Thing', engine: 'claude', viewer: { use: 'acme/missing' } }),diff --git a/cli/src/dsh/install.ts b/cli/src/dsh/install.tsindex 43573908c..f41f811dd 100644--- a/cli/src/dsh/install.ts+++ b/cli/src/dsh/install.ts@@ -18,7 +18,8 @@ import {type InstalledDsh, type InstalledDshRecord,} from './installed.js'import { readDshManifest, viewerUse, type DshManifest } from './manifest.js'-import { PACKAGE_PATH_RE, registryEntry, type DshRegistryEntry } from './registry.js'+import { PACKAGE_PATH_RE, type DshRegistryEntry } from './registry.js'+import { catalogEntry, refreshDshRegistry } from './catalog.js'import { resolveDshCommand } from './materialize.js'import { runDshCommand } from './shell.js'@@ -42,6 +43,8 @@ export interface DshInstallProgress {export interface DshInstallOptions {/** A git URL, or a local path (cloned unless `link`). */source: string+ /** A catalog install must resolve to the package the user selected. */+ expectedId?: stringref?: string/*** The folder inside `source` that is the package (`store/agents/typst` of the Harness monorepo).@@ -70,7 +73,7 @@ export type DshInstallResult =/** A registry id (`autonomous/typst`) resolves to its repo, ref and folder; anything else is a source. */export function resolveInstallSource(idOrSource: string,- registry: (id: string) => DshRegistryEntry | undefined = registryEntry,+ registry: (id: string) => DshRegistryEntry | undefined = catalogEntry,): { source: string; ref?: string; path?: string; id?: string } | null {const entry = registry(idOrSource)if (entry) return { source: entry.repo, ref: entry.ref, ...(entry.path ? { path: entry.path } : {}), id: entry.id }@@ -129,8 +132,7 @@ async function cloneInstall(if (path === undefined) {// `--progress` because stderr is not a tty here and git would otherwise stay silent until the end;// streamed, not collected, so "Receiving objects: 39%" reaches the dialog while it is true.- const args = ['clone', '--depth', '1', '--progress', ...(ref ? ['--branch', ref] : []), '--', source, tmpDir]- const clone = await streamGit(args, onLine)+ const clone = await cloneRepo(source, ref, tmpDir, false, onLine)if (!clone.ok) return fail('CLONE_FAILED', clone.detail)commit = await gitHead(tmpDir)} else {@@ -140,10 +142,7 @@ async function cloneInstall(// moved out and the rest of the clone thrown away, so the install is laid out exactly like a// whole-repo one (the manifest at its root) and nothing else of the monorepo lands on the machine.const repoDir = join(root, `.tmp-${randomUUID()}`)- const clone = await streamGit(- ['clone', '--depth', '1', '--filter=blob:none', '--sparse', '--progress', ...(ref ? ['--branch', ref] : []), '--', source, repoDir],- onLine,- )+ const clone = await cloneRepo(source, ref, repoDir, true, onLine)if (!clone.ok) return fail('CLONE_FAILED', clone.detail, repoDir)const sparse = await streamGit(['-C', repoDir, 'sparse-checkout', 'set', '--', path], onLine)if (!sparse.ok) return fail('CLONE_FAILED', sparse.detail, repoDir)@@ -163,6 +162,25 @@ async function cloneInstall(/** How long one git command may take: a clone of a large repository over a slow link, not a hang. */const GIT_TIMEOUT_MS = 10 * 60_000+/** Catalogs pin built-ins to the commit they describe; `git clone --branch` cannot take a SHA. */+async function cloneRepo(source: string, ref: string | undefined, dir: string, sparse: boolean, onLine: DshInstallOptions['onLine']): Promise<{ ok: true } | { ok: false; detail: string }> {+ if (!ref || !/^[a-f0-9]{40}$/i.test(ref)) {+ return streamGit(['clone', '--depth', '1', ...(sparse ? ['--filter=blob:none', '--sparse'] : []), '--progress', ...(ref ? ['--branch', ref] : []), '--', source, dir], onLine)+ }+ const steps = [+ ['init', '--quiet', dir],+ ['-C', dir, 'remote', 'add', 'origin', source],+ ['-C', dir, 'fetch', '--depth', '1', ...(sparse ? ['--filter=blob:none'] : []), '--progress', 'origin', ref],+ ...(sparse ? [['-C', dir, 'sparse-checkout', 'init', '--cone']] : []),+ ['-C', dir, 'checkout', '--detach', 'FETCH_HEAD'],+ ]+ for (const args of steps) {+ const result = await streamGit(args, onLine)+ if (!result.ok) return result+ }+ return { ok: true }+}+/** Run git, handing each stderr line (and each carriage-return progress segment) to `onLine` as it lands. */function streamGit(args: string[], onLine: ((line: string) => void) | undefined): Promise<{ ok: true } | { ok: false; detail: string }> {return new Promise((resolve) => {@@ -239,6 +257,11 @@ export async function runDshDoctor(installed: InstalledDsh, onLine?: (line: striexport async function installDsh(opts: DshInstallOptions): Promise<DshInstallResult> {const progress = (p: DshInstallProgress): void => opts.onProgress?.(p)+ const wrongId = (actual: string): DshInstallResult => {+ const detail = `Catalog requested ${opts.expectedId}, but the package declares ${actual}`+ progress({ id: opts.expectedId ?? null, phase: 'failed', detail })+ return { ok: false, error: 'PACKAGE_ID_MISMATCH', detail }+ }let manifest: DshManifestlet dir: stringlet commit: string | null = null@@ -249,6 +272,7 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallResconst linked = linkInstall(opts.source)if (!linked.ok) { progress({ id: null, phase: 'failed', detail: linked.detail }); return linked }manifest = linked.manifest+ if (opts.expectedId && manifest.id !== opts.expectedId) return wrongId(manifest.id)realDir = linked.realDirdir = placeAt(manifest.id, { linkTo: realDir })commit = await gitHead(realDir)@@ -256,6 +280,10 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallResconst cloned = await cloneInstall(opts.source, opts.ref, opts.path, opts.onLine)if (!cloned.ok) { progress({ id: null, phase: 'failed', detail: cloned.detail }); return cloned }manifest = cloned.manifest+ if (opts.expectedId && manifest.id !== opts.expectedId) {+ rmSync(cloned.tmpDir, { recursive: true, force: true })+ return wrongId(manifest.id)+ }commit = cloned.commitdir = placeAt(manifest.id, { tmpDir: cloned.tmpDir })realDir = realpathSync(dir)@@ -297,7 +325,8 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallRes// first (`harness dsh install <url>`), and the doctor says so rather than the pane going blank.const uses = viewerUse(manifest)if (uses && !installedDsh(uses)) {- const entry = (opts.registry ?? registryEntry)(uses)+ if (!opts.registry && !catalogEntry(uses)) await refreshDshRegistry()+ const entry = (opts.registry ?? catalogEntry)(uses)if (entry) {opts.onLine?.(`viewer ${uses} · installing`)// The viewer's own phases are narrated UNDER THE HARNESS: the dialog watches the id it asked@@ -305,7 +334,7 @@ export async function installDsh(opts: DshInstallOptions): Promise<DshInstallRes// install would read as hung for the minutes OpenCascade takes to arrive. The viewer's `done`// is not the harness's done, so it reports as the harness's setup still going.const dep = await installDsh({- source: entry.repo, ref: entry.ref, path: entry.path, setupTimeoutMs: opts.setupTimeoutMs, onLine: opts.onLine,+ source: entry.repo, expectedId: entry.id, ref: entry.ref, path: entry.path, registry: opts.registry, setupTimeoutMs: opts.setupTimeoutMs, onLine: opts.onLine,onProgress: (p) => {if (p.phase === 'failed') return // reported below, once, with the viewer namedconst phase: DshInstallPhase = p.phase === 'done' ? 'setup' : p.phasediff --git a/cli/src/dsh/registry.ts b/cli/src/dsh/registry.tsindex 9217a7bd6..83804d1a6 100644--- a/cli/src/dsh/registry.ts+++ b/cli/src/dsh/registry.ts@@ -1,5 +1,5 @@/**- * The bundled registry, baked into the CLI at build time the same way the version is+ * The offline fallback registry, baked into the CLI at build time the same way the version is* (`__DSH_REGISTRY__`, an esbuild `define` in both `build.mjs` and `build-bundle.mjs`). Two sources* at the repo root: every built-in package folder, `store/<agents|viewers>/<name>` — its entry built* from its `harness.json` and `store.json`, so no fact is written twice — and@@ -9,7 +9,8 @@** A registry entry is how the desktop can offer "Install Typst" for a package this machine does not* have yet: it names the repo and the ref to clone, and — for the built-in shelf, which lives in this- * monorepo under `store/` — the folder inside that repo that IS the package. Nothing here runs code.+ * monorepo under `store/` — the folder inside that repo that IS the package. `catalog.ts` refreshes+ * the live shelf; this module supplies its schema and first-offline-launch fallback. Nothing here runs code.*/import { readdirSync, readFileSync, statSync } from 'node:fs'import { join } from 'node:path'diff --git a/cli/src/dsh/wire.ts b/cli/src/dsh/wire.tsindex 5bd063374..f0cc226cf 100644--- a/cli/src/dsh/wire.ts+++ b/cli/src/dsh/wire.ts@@ -6,7 +6,8 @@import type { DshInstallProgress } from './install.js'import { listInstalledDsh, type InstalledDsh } from './installed.js'import { DSH_ID_RE, dshTier, viewerUse } from './manifest.js'-import { bundledDshRegistry, registrySourceUrl, type DshRegistryEntry } from './registry.js'+import { registrySourceUrl, type DshRegistryEntry } from './registry.js'+import { currentDshRegistry } from './catalog.js'/*** `dsh_list`: the harnesses installed on this machine, then what the registry offers that is not.@@ -15,10 +16,11 @@ import { bundledDshRegistry, registrySourceUrl, type DshRegistryEntry } from './*/export function dshListRows(installed: readonly InstalledDsh[] = listInstalledDsh(),- registry: readonly DshRegistryEntry[] = bundledDshRegistry(),+ registry: readonly DshRegistryEntry[] = currentDshRegistry(),): Record<string, unknown>[] {+ const byId = new Map(registry.map(entry => [entry.id, entry]))const facts = (id: string): Record<string, unknown> => {- const known = registry.find((entry) => entry.id === id)+ const known = byId.get(id)return {verified: known?.verified === true,// Where a person can read the package: its folder page for a built-in (store/…) one.diff --git a/cli/vitest.setup.ts b/cli/vitest.setup.tsindex adbaeb556..44739c302 100644--- a/cli/vitest.setup.ts+++ b/cli/vitest.setup.ts@@ -11,3 +11,4 @@ import { tmpdir } from 'node:os'import { join } from 'node:path'process.env.ADAPTER_DATA_DIR = mkdtempSync(join(tmpdir(), 'adapter-test-data-'))+process.env.DSH_DIR = join(process.env.ADAPTER_DATA_DIR, 'dsh')diff --git a/desktop/lib/store/store_screen.dart b/desktop/lib/store/store_screen.dartindex e4b34cae4..85804f2b1 100644--- a/desktop/lib/store/store_screen.dart+++ b/desktop/lib/store/store_screen.dart@@ -120,6 +120,7 @@ class _StoreTabState extends State<StoreTab> {late final _search = TextEditingController(text: widget.initialHarness == null ? _place.query : '',);+ Timer? _catalogRefresh;void _remember() {_place@@ -154,6 +155,17 @@ class _StoreTabState extends State<StoreTab> {_ask(machine);}widget.notifier.addListener(_onAppChanged);+ // A Store left open picks up new publications; engines do not need reprobes.+ // The daemon caches catalog HTTP requests for five minutes.+ _catalogRefresh = Timer.periodic(const Duration(minutes: 1), (_) {+ for (final machine in widget.notifier.machineStates.values) {+ if (machine.connectionStatus == ConnectionStatus.connected) {+ unawaited(+ widget.notifier.probeDsh(machine.machine.machineId, force: true),+ );+ }+ }+ });});}@@ -201,6 +213,7 @@ class _StoreTabState extends State<StoreTab> {@overridevoid dispose() {+ _catalogRefresh?.cancel();widget.notifier.removeListener(_onAppChanged);_search.dispose();_store.dispose();diff --git a/desktop/test/store_screen_test.dart b/desktop/test/store_screen_test.dartindex 4e3e50229..b255517de 100644--- a/desktop/test/store_screen_test.dart+++ b/desktop/test/store_screen_test.dart@@ -280,6 +280,26 @@ void main() {},);+ testWidgets(+ 'an open Store refreshes its catalog and stops polling when closed',+ (tester) async {+ final (notifier, _) = await open(tester);+ notifier.machineStates['machine-1']!.connectionStatus =+ ConnectionStatus.connected;+ notifier.notifyListeners();+ await tester.pump();+ final probes = notifier.probes;+ final engines = notifier.engineProbes;+ await tester.pump(const Duration(minutes: 1));+ expect(notifier.probes, probes + 1);+ expect(notifier.engineProbes, engines);+ await tester.pumpWidget(const SizedBox());+ await tester.pump(const Duration(minutes: 2));+ expect(notifier.probes, probes + 1);+ expect(tester.takeException(), isNull);+ },+ );+testWidgets('a machine that is offline, unlinked or on an older CLI says so instead of offering Get',(tester) async {diff --git a/store/README.md b/store/README.mdindex 5e87d17be..1c3170f8f 100644--- a/store/README.md+++ b/store/README.md@@ -127,7 +127,7 @@ Two ways, depending on where the package lives.**In this repository**, as a built-in: the folder is `store/agents/<name>` (or `store/viewers/<name>`),its id `autonomous/<name>`, and beside `harness.json` a `store.json` with what the store page shows-that a manifest does not know. The CLI build lists every such folder; there is no entry to write.+that a manifest does not know. The catalog publisher lists every such folder; there is no entry to write.```json{ "homepage": "https://typst.app", "upstream": "https://github.com/typst/typst", "license": "MIT",@@ -147,10 +147,40 @@ matching viewer ID) in the registry entry so the Store can show its dependenciesBuilt-in entries derive this field from the manifest automatically.A package that is one folder of a bigger repository names it with `"path"`; install then fetches-that folder alone. Run the conformance check and include a real example in the PR; CI is currently-triggered manually. Once merged and included in a CLI release, the app offers the tile before the-package is installed and installs it on Create; `verified: true` marks first-party packages, which-every built-in is, and everything else shows its git URL on install.+that folder alone. Run the conformance check and include a real example in the PR; the full test+suite is currently triggered manually. Once merged, the catalog publisher lists it automatically.+The app offers the tile before the package is installed. `verified: true` marks built-in packages;+community packages show their source on install.++### Live catalog++Package changes merged into `main` trigger+[`Publish Store catalog`](../.github/workflows/publish-store-catalog.yml). It validates the complete+catalog, checks viewer dependencies, pins built-in packages to the source commit, and writes one+`catalog.json` to the `store-catalog` branch. It uses GitHub only; it does not build or release the+app, run package setup scripts, or access cloud infrastructure. Authors do not edit a generated index.++The CLI reads that public JSON over HTTPS, using conditional requests and a five-minute cache.+Concurrent requests share one fetch. An open Store asks connected machines again every minute;+reopening it also asks. New harnesses and shared viewers appear without a client restart or another+app/CLI release. GitHub caching and the refresh interval mean publication is not instantaneous.++The last validated catalog is saved on disk. If GitHub is unavailable, slow, or serves an invalid+response, clients keep that catalog; a first offline launch uses the bundled catalog. Installed+harnesses remain listed even if their Store entry is removed. Refreshing metadata does not install,+update, or execute packages. Get resolves both the harness and its viewer from the live catalog,+and an already-installed viewer is reused.++Validate publication locally without writing to GitHub:++```bash+node store/tools/catalog.mjs+```++`HARNESS_STORE_CATALOG_URL` selects a catalog mirror or a loopback HTTP fixture for development.+`HARNESS_STORE_REF` still overrides built-in install refs for testing a branch. Existing clients+need the CLI version containing the live reader once; subsequent package publications do not need+releases. Changes to the runtime or package protocol can still require a client update.## Tiers@@ -186,7 +216,8 @@ The packages Autonomous maintains are folders here, and the rules below hold for`blender`, `text-to-cad`, `autonomous-circuit`. A wrapper never renames what it wraps.- **One place per fact.** Name, category, author, description and engine are the manifest's; homepage,upstream, licence and screenshots are `store.json`'s. The CLI build turns each folder into its- registry entry (repo this repository, ref `main`, path the folder, tier from what the manifest ships).+ registry entry (repo this repository, path the folder, tier from what the manifest ships). The+ live publisher pins its ref to the source commit; the offline build fallback uses `main`.- **`harness dsh check` passes** on the folder as it is committed.- **Credit travels with the code.** A `LICENSE` for the wrapper, the upstream's licence beside it whenanything of theirs is in the folder, and a README whose "Credit and stewardship" section says whosediff --git a/store/tools/catalog.mjs b/store/tools/catalog.mjsnew file mode 100644index 000000000..b9295ed6c--- /dev/null+++ b/store/tools/catalog.mjs@@ -0,0 +1,71 @@+#!/usr/bin/env node+// Publish metadata, never run a harness's setup or load its code. No npm dependencies required.+import { writeFileSync } from 'node:fs';+import { resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import { execFileSync } from 'node:child_process';+import { createHash } from 'node:crypto';+import { readDshRegistry } from '../../cli/scripts/lib/dshRegistry.mjs';++const MONOREPO='https://github.com/autonomous-ai/openharness';+const ID=/^[a-z0-9][a-z0-9-]{0,63}\/[a-z0-9][a-z0-9-]{0,63}$/;+const PACKAGE_PATH=/^(?!\/)(?!.*\/$)(?!.*\/\/)(?!(?:.*\/)?\.{1,2}(?:\/|$))[A-Za-z0-9._\-/]+$/;+const LIMITS={id:160,name:40,description:300,category:24,author:80,repo:2048,ref:200,path:512,homepage:2048,upstream:2048,license:40,viewerUse:160};++export function createStoreCatalog(storeDir, ref) {+ if (!/^[a-f0-9]{40}$/i.test(ref)) throw new Error('Publish from a complete git commit SHA');+ const entries=readDshRegistry(storeDir,{strict:true}).sort((a,b)=>a.id.localeCompare(b.id));+ const byId=new Map();+ for (const entry of entries) {+ for(const [key,max]of Object.entries(LIMITS))if(entry[key]!==undefined&&(typeof entry[key]!=='string'||entry[key].length>max||/[\x00-\x1f\x7f]/.test(entry[key])))throw new Error(`${entry.id}: invalid ${key}`);+ if(!ID.test(entry.id)||!entry.name||!entry.repo||byId.has(entry.id))throw new Error(`Invalid or duplicate catalog identity: ${entry.id}`);+ if(entry.kind!==undefined&&!['agent','viewer'].includes(entry.kind))throw new Error(`${entry.id}: invalid kind`);+ if(entry.kind!=='viewer'&&(typeof entry.engine!=='string'||!entry.engine))throw new Error(`${entry.id}: missing engine`);+ const url=new URL(entry.repo);+ if(url.protocol!=='https:'||url.username||url.password)throw new Error(`${entry.id}: use a public HTTPS repository`);+ if(entry.path!==undefined&&!PACKAGE_PATH.test(entry.path))throw new Error(`${entry.id}: invalid package path`);+ if(entry.viewerUse!==undefined&&!ID.test(entry.viewerUse))throw new Error(`${entry.id}: invalid viewer dependency`);+ if(entry.tier!==undefined&&![0,1,2].includes(entry.tier))throw new Error(`${entry.id}: invalid tier`);+ for(const key of ['homepage','upstream'])if(entry[key]!==undefined)new URL(entry[key]);+ if(entry.screenshots!==undefined){if(!Array.isArray(entry.screenshots)||entry.screenshots.length>8)throw new Error(`${entry.id}: invalid screenshots`);for(const shot of entry.screenshots){if(typeof shot!=='string'||shot.length>2048)throw new Error(`${entry.id}: invalid screenshot`);new URL(shot);}}+ const builtIn=entry.repo.replace(/\.git$/,'')===MONOREPO&&entry.path===`store/${entry.kind==='viewer'?'viewers':'agents'}/${entry.id.slice('autonomous/'.length)}`&&entry.id.startsWith('autonomous/');+ if(entry.id.startsWith('autonomous/')&&!builtIn)throw new Error(`${entry.id}: autonomous IDs belong to built-in packages`);+ if(builtIn)entry.ref=ref;+ entry.verified=builtIn;+ byId.set(entry.id,entry);+ }+ for(const entry of entries)if(entry.viewerUse&&byId.get(entry.viewerUse)?.kind!=='viewer')throw new Error(`${entry.id}: viewer ${entry.viewerUse} must be listed in the same catalog`);+ if(!entries.length)throw new Error('Refusing to publish an empty Store');+ return {spec:1,entries};+}++/** One immutable catalog commit; optimistic ref updates cannot overwrite another publication. */+export async function publishStoreCatalog(catalog, {repository='autonomous-ai/openharness',token,fetch:request=fetch}={}) {+ if(!token)throw new Error('GITHUB_TOKEN is required to publish the catalog');+ const body=JSON.stringify(catalog,null,2)+'\n';+ const blobSha=createHash('sha1').update(`blob ${Buffer.byteLength(body)}\0`).update(body).digest('hex');+ const api=async(path,method='GET',data)=>{+ const response=await request(`https://api.github.com/repos/${repository}/${path}`,{method,headers:{authorization:`Bearer ${token}`,accept:'application/vnd.github+json','content-type':'application/json','x-github-api-version':'2022-11-28'},...(data?{body:JSON.stringify(data)}:{}),signal:AbortSignal.timeout(30_000)});+ if(response.status===404&&method==='GET')return null;+ if(!response.ok)throw new Error(`Catalog publication failed: ${method} ${path} (${response.status})`);+ return response.json();+ };+ const current=await api('git/ref/heads/store-catalog');+ if(current){const previous=await api(`git/commits/${current.object.sha}`);const tree=await api(`git/trees/${previous.tree.sha}`);if(tree?.tree?.length===1&&tree.tree[0].path==='catalog.json'&&tree.tree[0].sha===blobSha)return {changed:false,sha:current.object.sha};}+ const blob=await api('git/blobs','POST',{content:body,encoding:'utf-8'});+ const tree=await api('git/trees','POST',{tree:[{path:'catalog.json',mode:'100644',type:'blob',sha:blob.sha}]});+ const commit=await api('git/commits','POST',{message:'Publish Harness Store catalog',tree:tree.sha,parents:current?[current.object.sha]:[]});+ if(current)await api('git/refs/heads/store-catalog','PATCH',{sha:commit.sha,force:false});+ else await api('git/refs','POST',{ref:'refs/heads/store-catalog',sha:commit.sha});+ return {changed:true,sha:commit.sha};+}++if(process.argv[1]&&resolve(process.argv[1])===fileURLToPath(import.meta.url)){+ const args=process.argv.slice(2),value=flag=>args[args.indexOf(flag)+1];+ const storeDir=resolve(args.includes('--store')?value('--store'):fileURLToPath(new URL('..',import.meta.url)));+ const ref=args.includes('--ref')?value('--ref'):execFileSync('git',['-C',storeDir,'rev-parse','HEAD'],{encoding:'utf8'}).trim();+ const catalog=createStoreCatalog(storeDir,ref);+ if(args.includes('--output'))writeFileSync(value('--output'),JSON.stringify(catalog,null,2)+'\n');+ if(args.includes('--publish'))console.log(await publishStoreCatalog(catalog,{token:process.env.GITHUB_TOKEN,repository:process.env.GITHUB_REPOSITORY||'autonomous-ai/openharness'}));+ else console.log(`Catalog valid: ${catalog.entries.length} packages at ${ref}`);+}