Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Upgrade stranded shipped stage prompts (#12222)

Adam Eivy2763a0dd7cf2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index beeb506751..3681445b93 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -154,6 +154,26 @@ jobs:
echo "- Suite selection reasons: \`${SUITE_REASONS}\`"
} >> "$GITHUB_STEP_SUMMARY"
+ prompt-history:
+ name: Shipped prompt history
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Use Node.js 24.x
+ uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+ with:
+ node-version: 24.x
+
+ - name: Check shipped stage-prompt history
+ run: node scripts/migrations/check-shipped-prompt-history.js
+
server:
# Sharded on a full plan — see docs/GITHUB_ACTIONS.md "Full-suite sharding".
# The matrix comes from the planner; once-only steps pin to shard 1, and the
@@ -1050,7 +1070,7 @@ jobs:
gate:
name: CI Gate
if: always()
- needs: [impact, server, client, lint, database, browser, windows-server, node-minimum, linux-containment]
+ needs: [impact, prompt-history, server, client, lint, database, browser, windows-server, node-minimum, linux-containment]
runs-on: ubuntu-latest
timeout-minutes: 10
# READ only: the reporter lists this run's jobs to name the one whose STEP
@@ -1082,6 +1102,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CI_GATE_LABEL: CI Gate
CI_GATE_RESULT_IMPACT: ${{ needs.impact.result }}
+ CI_GATE_RESULT_PROMPT_HISTORY: ${{ needs.prompt-history.result }}
CI_GATE_RESULT_SERVER: ${{ needs.server.result }}
CI_GATE_RESULT_CLIENT: ${{ needs.client.result }}
CI_GATE_RESULT_LINT: ${{ needs.lint.result }}
diff --git a/scripts/ci-base-sha.test.js b/scripts/ci-base-sha.test.js
index a3caf15ea0..dffb5e0560 100644
--- a/scripts/ci-base-sha.test.js
+++ b/scripts/ci-base-sha.test.js
@@ -47,12 +47,12 @@ describe('resolveBaseSha', () => {
describe('ci.yml checkout depth', () => {
const jobs = workflowJobs(WORKFLOW);
- it('never asks for full history', () => {
- // fetch-depth: 0 clones every commit in the repo on a job that only ever
- // diffs the merge ref against its own first parent.
- expect(WORKFLOW).not.toMatch(/fetch-depth:\s*0\b/);
- // Negative control: the assertion above can fail.
- expect(' with:\n fetch-depth: 0\n').toMatch(/fetch-depth:\s*0\b/);
+ it('fetches full history only for the shipped prompt guard', () => {
+ const jobs = workflowJobs(WORKFLOW);
+ expect(jobs['prompt-history']).toMatch(/fetch-depth:\s*0\b/);
+ for (const [id, body] of Object.entries(jobs)) {
+ if (id !== 'prompt-history') expect(body, id).not.toMatch(/fetch-depth:\s*0\b/);
+ }
});
it('resolves the diff base in every job that consumes it', () => {
diff --git a/scripts/ci-test-plan.js b/scripts/ci-test-plan.js
index 76d5b280e4..e30c8dcc4b 100644
--- a/scripts/ci-test-plan.js
+++ b/scripts/ci-test-plan.js
@@ -294,6 +294,7 @@ export const ALWAYS_RUN_TESTS = [
'scripts/direct-invocation-drift.test.js',
'scripts/ensure-deps.test.js',
'scripts/migrations/promptBumpMigrations.guard.test.js',
+ 'scripts/migrations/shippedPromptHistory.guard.test.js',
'scripts/node-version-drift.test.js',
// Whole-tree markdown scanner: a docs-only PR can reintroduce the stale band
// in prose with no import edge back to this test (#11851).
diff --git a/scripts/migrations/385-stranded-stage-prompt-history.js b/scripts/migrations/385-stranded-stage-prompt-history.js
new file mode 100644
index 0000000000..66fb8f366e
--- /dev/null
+++ b/scripts/migrations/385-stranded-stage-prompt-history.js
@@ -0,0 +1,111 @@
+import { makePromptReplaceMigration } from './_lib.js';
+
+export const ACCEPTED_OLD_MD5 = {
+ 'catalog-ideas-scenes-concepts.md': ['243074821ebc58eec63e26d284101b29'],
+ 'fableloom-branch-node.md': [
+ 'e5005826c4577623ba362a0292741c7c',
+ 'ff5d40b3090c775fc9c1f48c2ea96bbd',
+ ],
+ 'fableloom-feedback-episode.md': ['9ff21b47c4251585da9500c306d91a28'],
+ 'fableloom-feedback-series-plan.md': ['c56a3de3776cf7ef8ba16782f0691144'],
+ 'fableloom-review-series-plan.md': ['d7d3b306c82cea4d7f13a78512a46948'],
+ 'fableloom-weave-episode.md': ['990cdecf9e46fa049ae4aae27e3b172d'],
+ 'manuscript-reformat.md': ['4df6381d5c685c683a518326b7908e18'],
+ 'pipeline-comic-script.md': ['9fe2451b573fc60eb6c971e1b8808ef3'],
+ 'pipeline-editorial-appearance-continuity.md': [
+ '25ccc731f1596a99d21600919896a8e3',
+ ],
+ 'pipeline-editorial-arc-transitions.md': ['4c297e92df472965be8641f8b9ce9853'],
+ 'pipeline-editorial-chekhov.md': ['21246b605f442f8c749fc2835bd293b5'],
+ 'pipeline-editorial-climax-agency.md': [
+ '55c25664b9904806146f92cc739d7208',
+ 'f6611ff22e36d6738b3b4faa9c1310df',
+ ],
+ 'pipeline-editorial-endings-cliffhanger.md': [
+ '3ef5e6fbc160f9f33902ab22d9284281',
+ ],
+ 'pipeline-editorial-eyeline-match.md': ['5d0b8d0d4605225d208786165e26abb4'],
+ 'pipeline-editorial-head-hopping.md': ['460e6449c4f1daec815b9ff0a6573f97'],
+ 'pipeline-editorial-object-weight-proportionality.md': [
+ '89f6f6c6856ebcf0183b7760d71a6898',
+ 'c1b7b58e0255c86b6ba5cece4330ac48',
+ 'f4f91a4ea55a9fd89eaebc70074a89ec',
+ ],
+ 'pipeline-editorial-on-the-nose.md': ['8d9038a2b197728f6edeedb84699f364'],
+ 'pipeline-editorial-reaction-proportionality.md': [
+ '38ca1b6edaa55c8989e22e7ffdbd76c7',
+ '8787e6e40f8bb46067b8303b866c7ff8',
+ ],
+ 'pipeline-editorial-secondary-arc.md': ['6ad9eb7c4969d8baeb2c8c38d8f6695e'],
+ 'pipeline-editorial-timeline-contradiction.md': [
+ 'f39272a486d4cc031ebb7b3b8fc01ed3',
+ ],
+ 'pipeline-editorial-unmodeled-names.md': [
+ '2e562a414726b5b412537ad4ecbdbff5',
+ '50029186d6009d50c3850f3f4b90a5d6',
+ '9b9732f729c96dd7154175b77755fdd2',
+ 'd34323ee9bf05fffc3584ad9d91146bd',
+ ],
+ 'pipeline-editorial-world-cost-free-power.md': [
+ '42db5691fa6e4b2db9a146a821ed1fe7',
+ ],
+ 'pipeline-editorial-world-unforeshadowed-solution.md': [
+ '9a23180e49fbac1699c846ced1625575',
+ ],
+ 'pipeline-idea-expansion.md': ['c5b2e719e232a281b3829a5348cb29c1'],
+ 'pipeline-manuscript-fix.md': ['87c6e47030df07b37ace44d2a15372cf'],
+ 'pipeline-pov-rewrite.md': ['cd073c8dbca2bf25eea07b6103e27d81'],
+ 'pipeline-script-verify.md': ['cec35763c134fcac8bd4e9669ab5505a'],
+ 'story-builder-reader-map-refine.md': ['befd154bb319c221f16ca357a23f1aa7'],
+ 'story-builder-reader-map.md': ['af375710531a34b8fe265b425be42b3d'],
+};
+
+export const NEW_SHIPPED_MD5 = {
+ 'catalog-ideas-scenes-concepts.md': 'e6cce39c76689ea9c5233f233cd0f153',
+ 'fableloom-branch-node.md': '39a208c8cc593d0531af50760e3cf0da',
+ 'fableloom-feedback-episode.md': '1aaa6f17acad6a3215e48dcce14e8670',
+ 'fableloom-feedback-series-plan.md': 'ece17ecf3ee0d6018f3ca350794d8d52',
+ 'fableloom-review-series-plan.md': 'd82ff6df1d41c3e53fb0cc7ea49f8dc5',
+ 'fableloom-weave-episode.md': '94f22c0807924dadf151341c30720c71',
+ 'manuscript-reformat.md': '1a02cf3ac59b5c253359357608c1c4ad',
+ 'pipeline-comic-script.md': '49af30c05f008b20f6998a0f113f7d87',
+ 'pipeline-editorial-appearance-continuity.md':
+ '8ef03aa3ed15557febe85d7ee66b6cc6',
+ 'pipeline-editorial-arc-transitions.md': '46cb5444f41d05fb7bdbc219d62619b2',
+ 'pipeline-editorial-chekhov.md': '1f8a1696b5e4f476051dc5b2e5737db9',
+ 'pipeline-editorial-climax-agency.md': '255ad29214f48e6dd1c17adbb5887478',
+ 'pipeline-editorial-endings-cliffhanger.md':
+ 'deaebfa026d40a6e576479ecc4f1e98b',
+ 'pipeline-editorial-eyeline-match.md': '7c66926cbf8f7174ca17043e1f2e2251',
+ 'pipeline-editorial-head-hopping.md': '00e596e9dea15b3154a6a9f3509e90a5',
+ 'pipeline-editorial-object-weight-proportionality.md':
+ '761890e3dcdc2cd828ec2be2a5f9cb82',
+ 'pipeline-editorial-on-the-nose.md': 'e5786fb019e5bf19c7aa6ed0c8b35cda',
+ 'pipeline-editorial-reaction-proportionality.md':
+ 'b15471388a184d6f805e6dfac76fb164',
+ 'pipeline-editorial-secondary-arc.md': 'a9bd4dd9a06bc571f0363e0031d9a5d7',
+ 'pipeline-editorial-timeline-contradiction.md':
+ '2d797a9fa3b55f4c9f85c22cc0baa547',
+ 'pipeline-editorial-unmodeled-names.md': '6b35274e08ea9b5727c42f6ef6653cd5',
+ 'pipeline-editorial-world-cost-free-power.md':
+ '2b761163f63c2e4fbe8dc6d7f6e577f7',
+ 'pipeline-editorial-world-unforeshadowed-solution.md':
+ '09d5997e8c9ea77a5cf1bf446ef211f1',
+ 'pipeline-idea-expansion.md': 'a032e4a724251ed3e3495d33c4dbab8e',
+ 'pipeline-manuscript-fix.md': 'e2baaf0f2f53c8aa1e934a428c0ca583',
+ 'pipeline-pov-rewrite.md': 'eedcc03aaaf7e22e7726491f08d52974',
+ 'pipeline-script-verify.md': '722c62462d05462603cf67ca0ed1dee8',
+ 'story-builder-reader-map-refine.md': 'd8f02936a237a9e4e964d7250a118e17',
+ 'story-builder-reader-map.md': '21929c9f7a7bd54c17c9dfdf00213987',
+};
+
+const { applyMigration, up } = makePromptReplaceMigration({
+ accepted: ACCEPTED_OLD_MD5,
+ current: NEW_SHIPPED_MD5,
+ label: 'Previously stranded shipped stage-prompt history',
+ customizedHint: (filename) =>
+ ` Merge the current shipped guidance from data.reference/prompts/stages/${filename} into the customized prompt.`,
+});
+
+export { applyMigration };
+export default { up };
diff --git a/scripts/migrations/385-stranded-stage-prompt-history.test.js b/scripts/migrations/385-stranded-stage-prompt-history.test.js
new file mode 100644
index 0000000000..86ca87a1e0
--- /dev/null
+++ b/scripts/migrations/385-stranded-stage-prompt-history.test.js
@@ -0,0 +1,87 @@
+import { execFileSync } from 'child_process';
+import {
+ mkdtempSync,
+ mkdirSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { describe, expect, it } from 'vitest';
+import migration, {
+ applyMigration,
+ ACCEPTED_OLD_MD5,
+ NEW_SHIPPED_MD5,
+} from './385-stranded-stage-prompt-history.js';
+import { repoRoot, runPromptMigrationTests } from './_testHelpers.js';
+import { isShallowRepository } from './shippedPromptHistory.js';
+
+const strandedBodyCommit = '95ecbc86c5';
+const promptPath =
+ 'data.reference/prompts/stages/fableloom-review-series-plan.md';
+
+describe('migration 385 — previously stranded shipped stage-prompt history', () => {
+ runPromptMigrationTests({
+ migration,
+ applyMigration,
+ ACCEPTED_OLD_MD5,
+ NEW_SHIPPED_MD5,
+ prefix: 'migration-385-stranded-stage-prompt-history-',
+ });
+
+ const historyAvailable = !isShallowRepository(repoRoot);
+ if (!historyAvailable) {
+ console.warn(
+ '⏭️ Skipping the original FableLoom prompt-body fixture: checkout has shallow git history',
+ );
+ }
+
+ it.skipIf(!historyAvailable)(
+ 'upgrades the first shipped FableLoom plan-review body',
+ async () => {
+ const oldBody = execFileSync(
+ 'git',
+ ['show', `${strandedBodyCommit}:${promptPath}`],
+ {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ },
+ );
+ const rootDir = mkdtempSync(
+ join(tmpdir(), 'migration-385-original-body-'),
+ );
+ try {
+ const dataPath = join(
+ rootDir,
+ 'data',
+ 'prompts',
+ 'stages',
+ 'fableloom-review-series-plan.md',
+ );
+ const samplePath = join(rootDir, promptPath);
+ mkdirSync(join(rootDir, 'data', 'prompts', 'stages'), {
+ recursive: true,
+ });
+ mkdirSync(join(rootDir, 'data.reference', 'prompts', 'stages'), {
+ recursive: true,
+ });
+ writeFileSync(dataPath, oldBody);
+ writeFileSync(
+ samplePath,
+ readFileSync(join(repoRoot, promptPath), 'utf8'),
+ );
+
+ await expect(applyMigration({ rootDir })).resolves.toMatchObject({
+ updated: 1,
+ skipped: 0,
+ });
+ expect(readFileSync(dataPath, 'utf8')).toBe(
+ readFileSync(samplePath, 'utf8'),
+ );
+ } finally {
+ rmSync(rootDir, { recursive: true, force: true });
+ }
+ },
+ );
+});
diff --git a/scripts/migrations/check-shipped-prompt-history.js b/scripts/migrations/check-shipped-prompt-history.js
new file mode 100644
index 0000000000..9fb3f5d2cf
--- /dev/null
+++ b/scripts/migrations/check-shipped-prompt-history.js
@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import { inspectShippedPromptHistory } from './shippedPromptHistory.js';
+
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const result = await inspectShippedPromptHistory({
+ repoRoot,
+ migrationsDir: join(repoRoot, 'scripts', 'migrations'),
+ stageDir: join(repoRoot, 'data.reference', 'prompts', 'stages'),
+});
+
+if (result.skipped) {
+ console.log(
+ `⏭️ Skipping shipped stage-prompt history guard: ${result.skipped}`,
+ );
+ process.exit(0);
+}
+
+if (result.stranded.length) {
+ for (const { file, hash, commit } of result.stranded) {
+ console.error(
+ `❌ ${file}: historical hash ${hash} from ${commit.slice(0, 12)} is not accepted by a prompt migration`,
+ );
+ }
+ process.exitCode = 1;
+} else {
+ console.log(
+ '✅ Every historical shipped stage-prompt body is accepted by a prompt migration',
+ );
+}
diff --git a/scripts/migrations/shippedPromptHistory.guard.test.js b/scripts/migrations/shippedPromptHistory.guard.test.js
new file mode 100644
index 0000000000..5be3cf1dba
--- /dev/null
+++ b/scripts/migrations/shippedPromptHistory.guard.test.js
@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import {
+ findUnacceptedPromptHistory,
+ isShallowRepository,
+ inspectShippedPromptHistory,
+} from './shippedPromptHistory.js';
+
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const historyAvailable = !isShallowRepository(repoRoot);
+
+if (!historyAvailable) {
+ console.warn(
+ '⏭️ Skipping shipped stage-prompt history scan: checkout has shallow git history',
+ );
+}
+
+describe('shipped stage-prompt history guard', () => {
+ it('identifies historical hashes missing from every migration table', () => {
+ expect(
+ findUnacceptedPromptHistory({
+ histories: { 'example.md': [{ hash: 'stranded', commit: 'abc' }] },
+ accepted: { 'example.md': ['known-old'] },
+ current: { 'example.md': 'current' },
+ }),
+ ).toEqual([{ file: 'example.md', hash: 'stranded', commit: 'abc' }]);
+ });
+
+ it('accepts current and migrated historical hashes', () => {
+ expect(
+ findUnacceptedPromptHistory({
+ histories: {
+ 'example.md': [{ hash: 'known-old' }, { hash: 'current' }],
+ },
+ accepted: { 'example.md': ['known-old'] },
+ current: { 'example.md': 'current' },
+ }),
+ ).toEqual([]);
+ });
+
+ it.skipIf(!historyAvailable)(
+ 'checks all shipped stage-prompt history on a full-history checkout',
+ async () => {
+ const result = await inspectShippedPromptHistory({
+ repoRoot,
+ migrationsDir: join(repoRoot, 'scripts', 'migrations'),
+ stageDir: join(repoRoot, 'data.reference', 'prompts', 'stages'),
+ });
+ expect(result.stranded).toEqual([]);
+ },
+ );
+});
diff --git a/scripts/migrations/shippedPromptHistory.js b/scripts/migrations/shippedPromptHistory.js
new file mode 100644
index 0000000000..928360f628
--- /dev/null
+++ b/scripts/migrations/shippedPromptHistory.js
@@ -0,0 +1,144 @@
+import { execFileSync } from 'child_process';
+import { createHash } from 'crypto';
+import { readdir, readFile } from 'fs/promises';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+
+const hash = (body) =>
+ createHash('md5')
+ .update(body.replace(/\r\n/g, '\n').replace(/\r/g, '\n'))
+ .digest('hex');
+
+const runGit = (args, cwd, encoding = 'utf8') =>
+ execFileSync('git', args, {
+ cwd,
+ encoding,
+ maxBuffer: 64 * 1024 * 1024,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+
+export const isShallowRepository = (repoRoot) =>
+ runGit(['rev-parse', '--is-shallow-repository'], repoRoot).trim() === 'true';
+
+const historicalHashes = (repoRoot, files) => {
+ const wanted = new Set(
+ files.map((file) => `data.reference/prompts/stages/${file}`),
+ );
+ const lines = runGit(
+ [
+ 'log',
+ '--format=%H',
+ '--name-only',
+ '--',
+ 'data.reference/prompts/stages',
+ ],
+ repoRoot,
+ ).split('\n');
+ const objects = [];
+ let commit = null;
+ for (const line of lines) {
+ if (/^[0-9a-f]{40}$/.test(line)) {
+ commit = line;
+ } else if (line && commit && wanted.has(line)) {
+ objects.push({ commit, path: line });
+ }
+ }
+
+ if (!objects.length) return {};
+ const input = `${objects.map(({ commit: revision, path }) => `${revision}:${path}`).join('\n')}\n`;
+ const blobs = execFileSync('git', ['cat-file', '--batch'], {
+ cwd: repoRoot,
+ input,
+ encoding: null,
+ maxBuffer: 256 * 1024 * 1024,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+
+ const histories = Object.fromEntries(files.map((file) => [file, []]));
+ let offset = 0;
+ for (const object of objects) {
+ const newline = blobs.indexOf(10, offset);
+ if (newline < 0) throw new Error('Malformed `git cat-file --batch` header');
+ const header = blobs.subarray(offset, newline).toString('utf8');
+ offset = newline + 1;
+ if (header.endsWith(' missing')) continue;
+ const size = Number(header.split(' ').at(-1));
+ if (!Number.isInteger(size) || size < 0)
+ throw new Error(`Malformed git object header: ${header}`);
+ const body = blobs.subarray(offset, offset + size).toString('utf8');
+ offset += size + 1;
+ histories[object.path.split('/').at(-1)].push({
+ commit: object.commit,
+ hash: hash(body),
+ });
+ }
+ return histories;
+};
+
+export const findUnacceptedPromptHistory = ({
+ histories,
+ accepted,
+ current,
+}) => {
+ const stranded = [];
+ for (const [file, versions] of Object.entries(histories)) {
+ const covered = new Set([current[file], ...(accepted[file] || [])]);
+ for (const version of versions) {
+ if (!covered.has(version.hash)) {
+ stranded.push({ file, hash: version.hash, commit: version.commit });
+ }
+ }
+ }
+ return stranded;
+};
+
+const migrationHashes = async (migrationsDir) => {
+ const accepted = {};
+ const files = (await readdir(migrationsDir))
+ .filter((file) => /^\d.*\.js$/.test(file) && !file.endsWith('.test.js'))
+ .sort();
+ for (const file of files) {
+ const sourcePath = join(migrationsDir, file);
+ const source = await readFile(sourcePath, 'utf8');
+ if (
+ !source.includes('ACCEPTED_OLD_MD5') &&
+ !source.includes('NEW_SHIPPED_MD5')
+ )
+ continue;
+ const migration = await import(pathToFileURL(sourcePath).href);
+ for (const [name, hashes] of Object.entries(
+ migration.ACCEPTED_OLD_MD5 || {},
+ )) {
+ accepted[name] ||= [];
+ accepted[name].push(...hashes);
+ }
+ }
+ return { accepted };
+};
+
+export const inspectShippedPromptHistory = async ({
+ repoRoot,
+ migrationsDir,
+ stageDir,
+}) => {
+ if (isShallowRepository(repoRoot))
+ return { skipped: 'requires full git history', stranded: [] };
+
+ const files = (await readdir(stageDir))
+ .filter((file) => file.endsWith('.md'))
+ .sort();
+ const histories = historicalHashes(repoRoot, files);
+ const { accepted } = await migrationHashes(migrationsDir);
+ const current = Object.fromEntries(
+ await Promise.all(
+ files.map(async (file) => [
+ file,
+ hash(await readFile(join(stageDir, file), 'utf8')),
+ ]),
+ ),
+ );
+ return {
+ skipped: null,
+ stranded: findUnacceptedPromptHistory({ histories, accepted, current }),
+ };
+};