Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Upgrade stranded shipped stage prompts (#12222)
Adam Eivy2763a0dd7cf2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex beeb506751..3681445b93 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -154,6 +154,26 @@ jobs:echo "- Suite selection reasons: \`${SUITE_REASONS}\`"} >> "$GITHUB_STEP_SUMMARY"+ prompt-history:+ name: Shipped prompt history+ runs-on: ubuntu-latest+ timeout-minutes: 10+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ fetch-depth: 0+ persist-credentials: false++ - name: Use Node.js 24.x+ uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0+ with:+ node-version: 24.x++ - name: Check shipped stage-prompt history+ run: node scripts/migrations/check-shipped-prompt-history.js+server:# Sharded on a full plan — see docs/GITHUB_ACTIONS.md "Full-suite sharding".# The matrix comes from the planner; once-only steps pin to shard 1, and the@@ -1050,7 +1070,7 @@ jobs:gate:name: CI Gateif: always()- needs: [impact, server, client, lint, database, browser, windows-server, node-minimum, linux-containment]+ needs: [impact, prompt-history, server, client, lint, database, browser, windows-server, node-minimum, linux-containment]runs-on: ubuntu-latesttimeout-minutes: 10# READ only: the reporter lists this run's jobs to name the one whose STEP@@ -1082,6 +1102,7 @@ jobs:GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}CI_GATE_LABEL: CI GateCI_GATE_RESULT_IMPACT: ${{ needs.impact.result }}+ CI_GATE_RESULT_PROMPT_HISTORY: ${{ needs.prompt-history.result }}CI_GATE_RESULT_SERVER: ${{ needs.server.result }}CI_GATE_RESULT_CLIENT: ${{ needs.client.result }}CI_GATE_RESULT_LINT: ${{ needs.lint.result }}diff --git a/scripts/ci-base-sha.test.js b/scripts/ci-base-sha.test.jsindex a3caf15ea0..dffb5e0560 100644--- a/scripts/ci-base-sha.test.js+++ b/scripts/ci-base-sha.test.js@@ -47,12 +47,12 @@ describe('resolveBaseSha', () => {describe('ci.yml checkout depth', () => {const jobs = workflowJobs(WORKFLOW);- it('never asks for full history', () => {- // fetch-depth: 0 clones every commit in the repo on a job that only ever- // diffs the merge ref against its own first parent.- expect(WORKFLOW).not.toMatch(/fetch-depth:\s*0\b/);- // Negative control: the assertion above can fail.- expect(' with:\n fetch-depth: 0\n').toMatch(/fetch-depth:\s*0\b/);+ it('fetches full history only for the shipped prompt guard', () => {+ const jobs = workflowJobs(WORKFLOW);+ expect(jobs['prompt-history']).toMatch(/fetch-depth:\s*0\b/);+ for (const [id, body] of Object.entries(jobs)) {+ if (id !== 'prompt-history') expect(body, id).not.toMatch(/fetch-depth:\s*0\b/);+ }});it('resolves the diff base in every job that consumes it', () => {diff --git a/scripts/ci-test-plan.js b/scripts/ci-test-plan.jsindex 76d5b280e4..e30c8dcc4b 100644--- a/scripts/ci-test-plan.js+++ b/scripts/ci-test-plan.js@@ -294,6 +294,7 @@ export const ALWAYS_RUN_TESTS = ['scripts/direct-invocation-drift.test.js','scripts/ensure-deps.test.js','scripts/migrations/promptBumpMigrations.guard.test.js',+ 'scripts/migrations/shippedPromptHistory.guard.test.js','scripts/node-version-drift.test.js',// Whole-tree markdown scanner: a docs-only PR can reintroduce the stale band// in prose with no import edge back to this test (#11851).diff --git a/scripts/migrations/385-stranded-stage-prompt-history.js b/scripts/migrations/385-stranded-stage-prompt-history.jsnew file mode 100644index 0000000000..66fb8f366e--- /dev/null+++ b/scripts/migrations/385-stranded-stage-prompt-history.js@@ -0,0 +1,111 @@+import { makePromptReplaceMigration } from './_lib.js';++export const ACCEPTED_OLD_MD5 = {+ 'catalog-ideas-scenes-concepts.md': ['243074821ebc58eec63e26d284101b29'],+ 'fableloom-branch-node.md': [+ 'e5005826c4577623ba362a0292741c7c',+ 'ff5d40b3090c775fc9c1f48c2ea96bbd',+ ],+ 'fableloom-feedback-episode.md': ['9ff21b47c4251585da9500c306d91a28'],+ 'fableloom-feedback-series-plan.md': ['c56a3de3776cf7ef8ba16782f0691144'],+ 'fableloom-review-series-plan.md': ['d7d3b306c82cea4d7f13a78512a46948'],+ 'fableloom-weave-episode.md': ['990cdecf9e46fa049ae4aae27e3b172d'],+ 'manuscript-reformat.md': ['4df6381d5c685c683a518326b7908e18'],+ 'pipeline-comic-script.md': ['9fe2451b573fc60eb6c971e1b8808ef3'],+ 'pipeline-editorial-appearance-continuity.md': [+ '25ccc731f1596a99d21600919896a8e3',+ ],+ 'pipeline-editorial-arc-transitions.md': ['4c297e92df472965be8641f8b9ce9853'],+ 'pipeline-editorial-chekhov.md': ['21246b605f442f8c749fc2835bd293b5'],+ 'pipeline-editorial-climax-agency.md': [+ '55c25664b9904806146f92cc739d7208',+ 'f6611ff22e36d6738b3b4faa9c1310df',+ ],+ 'pipeline-editorial-endings-cliffhanger.md': [+ '3ef5e6fbc160f9f33902ab22d9284281',+ ],+ 'pipeline-editorial-eyeline-match.md': ['5d0b8d0d4605225d208786165e26abb4'],+ 'pipeline-editorial-head-hopping.md': ['460e6449c4f1daec815b9ff0a6573f97'],+ 'pipeline-editorial-object-weight-proportionality.md': [+ '89f6f6c6856ebcf0183b7760d71a6898',+ 'c1b7b58e0255c86b6ba5cece4330ac48',+ 'f4f91a4ea55a9fd89eaebc70074a89ec',+ ],+ 'pipeline-editorial-on-the-nose.md': ['8d9038a2b197728f6edeedb84699f364'],+ 'pipeline-editorial-reaction-proportionality.md': [+ '38ca1b6edaa55c8989e22e7ffdbd76c7',+ '8787e6e40f8bb46067b8303b866c7ff8',+ ],+ 'pipeline-editorial-secondary-arc.md': ['6ad9eb7c4969d8baeb2c8c38d8f6695e'],+ 'pipeline-editorial-timeline-contradiction.md': [+ 'f39272a486d4cc031ebb7b3b8fc01ed3',+ ],+ 'pipeline-editorial-unmodeled-names.md': [+ '2e562a414726b5b412537ad4ecbdbff5',+ '50029186d6009d50c3850f3f4b90a5d6',+ '9b9732f729c96dd7154175b77755fdd2',+ 'd34323ee9bf05fffc3584ad9d91146bd',+ ],+ 'pipeline-editorial-world-cost-free-power.md': [+ '42db5691fa6e4b2db9a146a821ed1fe7',+ ],+ 'pipeline-editorial-world-unforeshadowed-solution.md': [+ '9a23180e49fbac1699c846ced1625575',+ ],+ 'pipeline-idea-expansion.md': ['c5b2e719e232a281b3829a5348cb29c1'],+ 'pipeline-manuscript-fix.md': ['87c6e47030df07b37ace44d2a15372cf'],+ 'pipeline-pov-rewrite.md': ['cd073c8dbca2bf25eea07b6103e27d81'],+ 'pipeline-script-verify.md': ['cec35763c134fcac8bd4e9669ab5505a'],+ 'story-builder-reader-map-refine.md': ['befd154bb319c221f16ca357a23f1aa7'],+ 'story-builder-reader-map.md': ['af375710531a34b8fe265b425be42b3d'],+};++export const NEW_SHIPPED_MD5 = {+ 'catalog-ideas-scenes-concepts.md': 'e6cce39c76689ea9c5233f233cd0f153',+ 'fableloom-branch-node.md': '39a208c8cc593d0531af50760e3cf0da',+ 'fableloom-feedback-episode.md': '1aaa6f17acad6a3215e48dcce14e8670',+ 'fableloom-feedback-series-plan.md': 'ece17ecf3ee0d6018f3ca350794d8d52',+ 'fableloom-review-series-plan.md': 'd82ff6df1d41c3e53fb0cc7ea49f8dc5',+ 'fableloom-weave-episode.md': '94f22c0807924dadf151341c30720c71',+ 'manuscript-reformat.md': '1a02cf3ac59b5c253359357608c1c4ad',+ 'pipeline-comic-script.md': '49af30c05f008b20f6998a0f113f7d87',+ 'pipeline-editorial-appearance-continuity.md':+ '8ef03aa3ed15557febe85d7ee66b6cc6',+ 'pipeline-editorial-arc-transitions.md': '46cb5444f41d05fb7bdbc219d62619b2',+ 'pipeline-editorial-chekhov.md': '1f8a1696b5e4f476051dc5b2e5737db9',+ 'pipeline-editorial-climax-agency.md': '255ad29214f48e6dd1c17adbb5887478',+ 'pipeline-editorial-endings-cliffhanger.md':+ 'deaebfa026d40a6e576479ecc4f1e98b',+ 'pipeline-editorial-eyeline-match.md': '7c66926cbf8f7174ca17043e1f2e2251',+ 'pipeline-editorial-head-hopping.md': '00e596e9dea15b3154a6a9f3509e90a5',+ 'pipeline-editorial-object-weight-proportionality.md':+ '761890e3dcdc2cd828ec2be2a5f9cb82',+ 'pipeline-editorial-on-the-nose.md': 'e5786fb019e5bf19c7aa6ed0c8b35cda',+ 'pipeline-editorial-reaction-proportionality.md':+ 'b15471388a184d6f805e6dfac76fb164',+ 'pipeline-editorial-secondary-arc.md': 'a9bd4dd9a06bc571f0363e0031d9a5d7',+ 'pipeline-editorial-timeline-contradiction.md':+ '2d797a9fa3b55f4c9f85c22cc0baa547',+ 'pipeline-editorial-unmodeled-names.md': '6b35274e08ea9b5727c42f6ef6653cd5',+ 'pipeline-editorial-world-cost-free-power.md':+ '2b761163f63c2e4fbe8dc6d7f6e577f7',+ 'pipeline-editorial-world-unforeshadowed-solution.md':+ '09d5997e8c9ea77a5cf1bf446ef211f1',+ 'pipeline-idea-expansion.md': 'a032e4a724251ed3e3495d33c4dbab8e',+ 'pipeline-manuscript-fix.md': 'e2baaf0f2f53c8aa1e934a428c0ca583',+ 'pipeline-pov-rewrite.md': 'eedcc03aaaf7e22e7726491f08d52974',+ 'pipeline-script-verify.md': '722c62462d05462603cf67ca0ed1dee8',+ 'story-builder-reader-map-refine.md': 'd8f02936a237a9e4e964d7250a118e17',+ 'story-builder-reader-map.md': '21929c9f7a7bd54c17c9dfdf00213987',+};++const { applyMigration, up } = makePromptReplaceMigration({+ accepted: ACCEPTED_OLD_MD5,+ current: NEW_SHIPPED_MD5,+ label: 'Previously stranded shipped stage-prompt history',+ customizedHint: (filename) =>+ ` Merge the current shipped guidance from data.reference/prompts/stages/${filename} into the customized prompt.`,+});++export { applyMigration };+export default { up };diff --git a/scripts/migrations/385-stranded-stage-prompt-history.test.js b/scripts/migrations/385-stranded-stage-prompt-history.test.jsnew file mode 100644index 0000000000..86ca87a1e0--- /dev/null+++ b/scripts/migrations/385-stranded-stage-prompt-history.test.js@@ -0,0 +1,87 @@+import { execFileSync } from 'child_process';+import {+ mkdtempSync,+ mkdirSync,+ readFileSync,+ rmSync,+ writeFileSync,+} from 'fs';+import { tmpdir } from 'os';+import { join } from 'path';+import { describe, expect, it } from 'vitest';+import migration, {+ applyMigration,+ ACCEPTED_OLD_MD5,+ NEW_SHIPPED_MD5,+} from './385-stranded-stage-prompt-history.js';+import { repoRoot, runPromptMigrationTests } from './_testHelpers.js';+import { isShallowRepository } from './shippedPromptHistory.js';++const strandedBodyCommit = '95ecbc86c5';+const promptPath =+ 'data.reference/prompts/stages/fableloom-review-series-plan.md';++describe('migration 385 — previously stranded shipped stage-prompt history', () => {+ runPromptMigrationTests({+ migration,+ applyMigration,+ ACCEPTED_OLD_MD5,+ NEW_SHIPPED_MD5,+ prefix: 'migration-385-stranded-stage-prompt-history-',+ });++ const historyAvailable = !isShallowRepository(repoRoot);+ if (!historyAvailable) {+ console.warn(+ '⏭️ Skipping the original FableLoom prompt-body fixture: checkout has shallow git history',+ );+ }++ it.skipIf(!historyAvailable)(+ 'upgrades the first shipped FableLoom plan-review body',+ async () => {+ const oldBody = execFileSync(+ 'git',+ ['show', `${strandedBodyCommit}:${promptPath}`],+ {+ cwd: repoRoot,+ encoding: 'utf8',+ },+ );+ const rootDir = mkdtempSync(+ join(tmpdir(), 'migration-385-original-body-'),+ );+ try {+ const dataPath = join(+ rootDir,+ 'data',+ 'prompts',+ 'stages',+ 'fableloom-review-series-plan.md',+ );+ const samplePath = join(rootDir, promptPath);+ mkdirSync(join(rootDir, 'data', 'prompts', 'stages'), {+ recursive: true,+ });+ mkdirSync(join(rootDir, 'data.reference', 'prompts', 'stages'), {+ recursive: true,+ });+ writeFileSync(dataPath, oldBody);+ writeFileSync(+ samplePath,+ readFileSync(join(repoRoot, promptPath), 'utf8'),+ );++ await expect(applyMigration({ rootDir })).resolves.toMatchObject({+ updated: 1,+ skipped: 0,+ });+ expect(readFileSync(dataPath, 'utf8')).toBe(+ readFileSync(samplePath, 'utf8'),+ );+ } finally {+ rmSync(rootDir, { recursive: true, force: true });+ }+ },+ );+});diff --git a/scripts/migrations/check-shipped-prompt-history.js b/scripts/migrations/check-shipped-prompt-history.jsnew file mode 100644index 0000000000..9fb3f5d2cf--- /dev/null+++ b/scripts/migrations/check-shipped-prompt-history.js@@ -0,0 +1,32 @@+#!/usr/bin/env node++import { dirname, join } from 'path';+import { fileURLToPath } from 'url';+import { inspectShippedPromptHistory } from './shippedPromptHistory.js';++const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..');+const result = await inspectShippedPromptHistory({+ repoRoot,+ migrationsDir: join(repoRoot, 'scripts', 'migrations'),+ stageDir: join(repoRoot, 'data.reference', 'prompts', 'stages'),+});++if (result.skipped) {+ console.log(+ `⏭️ Skipping shipped stage-prompt history guard: ${result.skipped}`,+ );+ process.exit(0);+}++if (result.stranded.length) {+ for (const { file, hash, commit } of result.stranded) {+ console.error(+ `❌ ${file}: historical hash ${hash} from ${commit.slice(0, 12)} is not accepted by a prompt migration`,+ );+ }+ process.exitCode = 1;+} else {+ console.log(+ '✅ Every historical shipped stage-prompt body is accepted by a prompt migration',+ );+}diff --git a/scripts/migrations/shippedPromptHistory.guard.test.js b/scripts/migrations/shippedPromptHistory.guard.test.jsnew file mode 100644index 0000000000..5be3cf1dba--- /dev/null+++ b/scripts/migrations/shippedPromptHistory.guard.test.js@@ -0,0 +1,53 @@+import { describe, expect, it } from 'vitest';+import { dirname, join } from 'path';+import { fileURLToPath } from 'url';+import {+ findUnacceptedPromptHistory,+ isShallowRepository,+ inspectShippedPromptHistory,+} from './shippedPromptHistory.js';++const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..');+const historyAvailable = !isShallowRepository(repoRoot);++if (!historyAvailable) {+ console.warn(+ '⏭️ Skipping shipped stage-prompt history scan: checkout has shallow git history',+ );+}++describe('shipped stage-prompt history guard', () => {+ it('identifies historical hashes missing from every migration table', () => {+ expect(+ findUnacceptedPromptHistory({+ histories: { 'example.md': [{ hash: 'stranded', commit: 'abc' }] },+ accepted: { 'example.md': ['known-old'] },+ current: { 'example.md': 'current' },+ }),+ ).toEqual([{ file: 'example.md', hash: 'stranded', commit: 'abc' }]);+ });++ it('accepts current and migrated historical hashes', () => {+ expect(+ findUnacceptedPromptHistory({+ histories: {+ 'example.md': [{ hash: 'known-old' }, { hash: 'current' }],+ },+ accepted: { 'example.md': ['known-old'] },+ current: { 'example.md': 'current' },+ }),+ ).toEqual([]);+ });++ it.skipIf(!historyAvailable)(+ 'checks all shipped stage-prompt history on a full-history checkout',+ async () => {+ const result = await inspectShippedPromptHistory({+ repoRoot,+ migrationsDir: join(repoRoot, 'scripts', 'migrations'),+ stageDir: join(repoRoot, 'data.reference', 'prompts', 'stages'),+ });+ expect(result.stranded).toEqual([]);+ },+ );+});diff --git a/scripts/migrations/shippedPromptHistory.js b/scripts/migrations/shippedPromptHistory.jsnew file mode 100644index 0000000000..928360f628--- /dev/null+++ b/scripts/migrations/shippedPromptHistory.js@@ -0,0 +1,144 @@+import { execFileSync } from 'child_process';+import { createHash } from 'crypto';+import { readdir, readFile } from 'fs/promises';+import { join } from 'path';+import { pathToFileURL } from 'url';++const hash = (body) =>+ createHash('md5')+ .update(body.replace(/\r\n/g, '\n').replace(/\r/g, '\n'))+ .digest('hex');++const runGit = (args, cwd, encoding = 'utf8') =>+ execFileSync('git', args, {+ cwd,+ encoding,+ maxBuffer: 64 * 1024 * 1024,+ stdio: ['ignore', 'pipe', 'pipe'],+ });++export const isShallowRepository = (repoRoot) =>+ runGit(['rev-parse', '--is-shallow-repository'], repoRoot).trim() === 'true';++const historicalHashes = (repoRoot, files) => {+ const wanted = new Set(+ files.map((file) => `data.reference/prompts/stages/${file}`),+ );+ const lines = runGit(+ [+ 'log',+ '--format=%H',+ '--name-only',+ '--',+ 'data.reference/prompts/stages',+ ],+ repoRoot,+ ).split('\n');+ const objects = [];+ let commit = null;+ for (const line of lines) {+ if (/^[0-9a-f]{40}$/.test(line)) {+ commit = line;+ } else if (line && commit && wanted.has(line)) {+ objects.push({ commit, path: line });+ }+ }++ if (!objects.length) return {};+ const input = `${objects.map(({ commit: revision, path }) => `${revision}:${path}`).join('\n')}\n`;+ const blobs = execFileSync('git', ['cat-file', '--batch'], {+ cwd: repoRoot,+ input,+ encoding: null,+ maxBuffer: 256 * 1024 * 1024,+ stdio: ['pipe', 'pipe', 'pipe'],+ });++ const histories = Object.fromEntries(files.map((file) => [file, []]));+ let offset = 0;+ for (const object of objects) {+ const newline = blobs.indexOf(10, offset);+ if (newline < 0) throw new Error('Malformed `git cat-file --batch` header');+ const header = blobs.subarray(offset, newline).toString('utf8');+ offset = newline + 1;+ if (header.endsWith(' missing')) continue;+ const size = Number(header.split(' ').at(-1));+ if (!Number.isInteger(size) || size < 0)+ throw new Error(`Malformed git object header: ${header}`);+ const body = blobs.subarray(offset, offset + size).toString('utf8');+ offset += size + 1;+ histories[object.path.split('/').at(-1)].push({+ commit: object.commit,+ hash: hash(body),+ });+ }+ return histories;+};++export const findUnacceptedPromptHistory = ({+ histories,+ accepted,+ current,+}) => {+ const stranded = [];+ for (const [file, versions] of Object.entries(histories)) {+ const covered = new Set([current[file], ...(accepted[file] || [])]);+ for (const version of versions) {+ if (!covered.has(version.hash)) {+ stranded.push({ file, hash: version.hash, commit: version.commit });+ }+ }+ }+ return stranded;+};++const migrationHashes = async (migrationsDir) => {+ const accepted = {};+ const files = (await readdir(migrationsDir))+ .filter((file) => /^\d.*\.js$/.test(file) && !file.endsWith('.test.js'))+ .sort();+ for (const file of files) {+ const sourcePath = join(migrationsDir, file);+ const source = await readFile(sourcePath, 'utf8');+ if (+ !source.includes('ACCEPTED_OLD_MD5') &&+ !source.includes('NEW_SHIPPED_MD5')+ )+ continue;+ const migration = await import(pathToFileURL(sourcePath).href);+ for (const [name, hashes] of Object.entries(+ migration.ACCEPTED_OLD_MD5 || {},+ )) {+ accepted[name] ||= [];+ accepted[name].push(...hashes);+ }+ }+ return { accepted };+};++export const inspectShippedPromptHistory = async ({+ repoRoot,+ migrationsDir,+ stageDir,+}) => {+ if (isShallowRepository(repoRoot))+ return { skipped: 'requires full git history', stranded: [] };++ const files = (await readdir(stageDir))+ .filter((file) => file.endsWith('.md'))+ .sort();+ const histories = historicalHashes(repoRoot, files);+ const { accepted } = await migrationHashes(migrationsDir);+ const current = Object.fromEntries(+ await Promise.all(+ files.map(async (file) => [+ file,+ hash(await readFile(join(stageDir, file), 'utf8')),+ ]),+ ),+ );+ return {+ skipped: null,+ stranded: findUnacceptedPromptHistory({ histories, accepted, current }),+ };+};