Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: admit manuscript import commits one at a time per series (#10763)
Adam Eivye5e2da7de448
diff --git a/server/services/importer.js b/server/services/importer.jsindex 67f6b96ce0..723ff53ff9 100644--- a/server/services/importer.js+++ b/server/services/importer.js@@ -40,6 +40,7 @@ import { IMPORTER_CONTENT_TYPES, IMPORTER_PROSE_EXCERPT_MAX } from '../lib/validexport { IMPORTER_PROSE_EXCERPT_MAX };import { mergeExtractedBible, BIBLE_KIND } from '../lib/storyBible.js';import { isStr } from '../lib/textUtils.js';+import { createKeyCachedQueue } from '../lib/createKeyCachedQueue.js';import {deriveImportId, getImportSession, recordImportProgress, withImportLock, SESSION_STATUS,} from './importerSessions.js';@@ -1117,13 +1118,59 @@ export async function retryIssueSplit({* a second tab, a retry after a reload dropped the client's own markers —* replays the recorded result or resumes at the issues instead of re-sending* canon/arc and duplicating the issue set. Absent, the commit behaves as before.+ *+ * Every commit into a series — any importId, a same-id retry, or an older+ * client sending none — is admitted one at a time per series (#10763). The+ * position checks read the series' issue inventory and allocate from it; two+ * manuscripts committed side by side would otherwise both pass against the+ * same snapshot and land duplicate arcPositions.*/export async function commitImport(args = {}) {const { importId = null } = args;return importId ? withImportLock(importId, () => commitImportOnce(args)) : commitImportOnce(args);}-async function commitImportOnce({+// Per-series commit admission (#10763). Held from the inventory read through+// the last issue write, so the collision checks and the positions the plan+// allocates stay true until the issues are published. Different series run+// concurrently. Never re-entered from inside a commit, and the import lock is+// always taken outside it, so the two cannot deadlock.+const seriesCommitQueue = createKeyCachedQueue();++async function commitImportOnce(args) {+ const { universeId, seriesId, issues, contentType = null, cleanupFormatting = false } = args;+ // Payload-only checks — no reads — so a malformed commit fails before it+ // queues behind another import or spends an LLM call.+ if (!isStr(universeId)) throw codedError('universeId is required', ERR_VALIDATION);+ if (!isStr(seriesId)) throw codedError('seriesId is required', ERR_VALIDATION);+ if (!Array.isArray(issues) || issues.length === 0) {+ throw codedError('At least one issue is required', ERR_VALIDATION);+ }+ let formattedExcerpts = null;+ if (cleanupFormatting) {+ // The optional AI cleanup (issue #1335) is one LLM call per issue — minutes+ // for a long manuscript. Check the commit against current state first so a+ // doomed or already-committed import never burns those calls, then run them+ // OUTSIDE the series admission so one slow import doesn't hold up every+ // other commit into the series. Admission re-checks fresh state and takes+ // only the cleaned excerpts from here — never the positions this precheck+ // saw, which may be stale by then.+ const precheck = await admitAndCommit(args, { precheck: true });+ if (!precheck.replay) {+ const cleaned = await reformatSeededExcerpts(precheck.pendingProposals, { contentType });+ formattedExcerpts = cleaned.map((proposal) => proposal?.proseExcerpt);+ }+ }+ return seriesCommitQueue(seriesId, () => admitAndCommit(args, { formattedExcerpts }));+}++// Validates and plans the commit against fresh state, then writes it. Runs+// inside the series admission, except for the `precheck` pass (a fail-fast+// before formatting, which writes nothing): that returns `{ replay: true }`+// for an import already committed, else `{ pendingProposals }` — the excerpts+// the formatting pass should clean. `formattedExcerpts[i]` (from that pass)+// replaces proposal i's excerpt when it is still to be created.+async function admitAndCommit({universeId,seriesId,importId = null,@@ -1134,31 +1181,20 @@ async function commitImportOnce({// Content type drives which stage each issue's verbatim excerpt seeds (see// the stage-seed block below). Absent → prose-seed (prior behavior).contentType = null,- // Opt-in AI cleanup: run each seeded excerpt through the manuscript-reformat- // pass before seeding so imported prose/scripts arrive already-clean (issue- // #1335). Best-effort + per-issue (one LLM call each), so it defaults off.- cleanupFormatting = false,// Destructive replace: wipes existing issues, overwrites arc + seasons.// Universe canon still merges additively (it's shared across series, so// a per-series destructive replace would be too coarse). Default false.replaceMode = false,-} = {}) {- if (!isStr(universeId)) throw codedError('universeId is required', ERR_VALIDATION);- if (!isStr(seriesId)) throw codedError('seriesId is required', ERR_VALIDATION);- if (!Array.isArray(issues) || issues.length === 0) {- throw codedError('At least one issue is required', ERR_VALIDATION);- }-+} = {}, { precheck = false, formattedExcerpts = null } = {}) {// Read for validation (universeId/seriesId linkage, lock checks, the// arcPosition/season-number gates below) and for the up-front counts used// to size auto-assigned positions and the additive issue-count target.- // NOT the source of truth the universe/series writes below merge against —- // `cleanupFormatting` can spend minutes in per-issue LLM calls after this- // read, so a concurrent edit to universe canon or series seasons could- // land in that window. The actual canon/season merges use the mutator form- // of `updateUniverse`/`updateSeries`, which re-reads the freshest persisted- // record inside the write queue (issue #8453) — this snapshot is stale by- // the time those writes run.+ // The series admission keeps other importer commits out of this series, but+ // NOT edits made outside the importer, and the universe is shared across+ // series — so this snapshot is not the source of truth the universe/series+ // writes below merge against. Those use the mutator form of+ // `updateUniverse`/`updateSeries`, which re-reads the freshest persisted+ // record inside the write queue (issue #8453).const universe = await getUniverse(universeId);const series = await getSeries(seriesId);@@ -1186,6 +1222,8 @@ async function commitImportOnce({// the session; additive mode would duplicate, so it does.const session = importId && !replaceMode ? await resolveImportSession(importId, series.id) : null;if (session?.status === SESSION_STATUS.COMMITTED) {+ // The precheck writes nothing; admission answers the replay.+ if (precheck) return { replay: true };// Answered before the locked-arc gate: nothing new is written, so a lock// added since the commit cannot turn a recorded success into an error.// A receipt lost after the last issue landed (#10762) is completed here.@@ -1398,13 +1436,12 @@ async function commitImportOnce({}}- // Optional AI cleanup of the seeded excerpts (issue #1335). Runs AFTER all- // the cheap fail-fast validation (so a bad payload never burns LLM calls) but- // BEFORE any destructive write (the wipe + universe/series writes below), so- // the heavy per-issue reformat happens while the on-disk state is untouched.+ // Optional AI cleanup of the seeded excerpts (issue #1335) runs between the+ // precheck pass and admission (see commitImportOnce): AFTER all the cheap+ // fail-fast validation above (so a bad payload never burns LLM calls) but+ // BEFORE any destructive write (the wipe + universe/series writes below).// Best-effort — `reformatSeededExcerpts` falls back to the verbatim excerpt- // per issue, so import never breaks. Returns a fresh array; the create loop- // below seeds from it.+ // per issue, so import never breaks.// A resume only reformats the issues it still has to create — the excerpt is// dropped from a surviving one so it costs no LLM call (it is never re-written).const pendingProposals = resumePlan@@ -1412,8 +1449,13 @@ async function commitImportOnce({? { ...proposal, proseExcerpt: undefined }: proposal)): issuesWithPositions;- const issuesToCreate = cleanupFormatting- ? await reformatSeededExcerpts(pendingProposals, { contentType })+ if (precheck) return { pendingProposals };+ // An issue that became pending only since the precheck (a survivor deleted in+ // between) has no cleaned excerpt and keeps its verbatim one.+ const issuesToCreate = formattedExcerpts+ ? pendingProposals.map((proposal, i) => (proposal.proseExcerpt && isStr(formattedExcerpts[i])+ ? { ...proposal, proseExcerpt: formattedExcerpts[i] }+ : proposal)): pendingProposals;// Issues already on the series that this import does not own — the base for// the additive issue-count target below.@@ -1485,13 +1527,13 @@ async function commitImportOnce({// If the user supplied no canon at all (arc-only import), skip the// updateUniverse round-trip entirely.//- // Mutator form (issue #8453): `cleanupFormatting` above can spend minutes- // in per-issue LLM calls between the up-front `getUniverse` read and this- // write. Merging against the stale `universe` snapshot here would silently- // drop any canon edit a concurrent tab/job made to the SAME array during- // that window (mergeExtractedBible rebuilds the whole array from its base- // list). Merging against `latest` — the freshest persisted record, read- // inside updateUniverse's write queue — closes that race.+ // Mutator form (issue #8453): the series admission serializes importer+ // commits into ONE series, but the universe is shared across series and is+ // edited outside the importer too, so the up-front `universe` snapshot can+ // already be stale. Merging against it would silently drop a concurrent+ // canon edit to the SAME array (mergeExtractedBible rebuilds the whole array+ // from its base list). Merging against `latest` — the freshest persisted+ // record, read inside updateUniverse's write queue — closes that race.const updatedUniverse = selectedKindMap.length > 0? await updateUniverse(universe.id, (latest) => Object.fromEntries(selectedKindMap.map(([selectionKey, kind, storageKey]) => [@@ -1546,9 +1588,9 @@ async function commitImportOnce({...biblePatch,});} else if (sanitizedArc || seasons.length > 0 || Object.keys(biblePatch).length > 0) {- // Mutator form (issue #8453): same race as the universe write above —- // `cleanupFormatting` can run for minutes before this write, during which- // a concurrent tab/job may add or edit a season on this series.+ // Mutator form (issue #8453): same race as the universe write above — a+ // tab or job outside the importer may add or edit a season on this series+ // between the up-front read and this write.// `mergeSeasons` rebuilds the whole array from its base list, so merging// against the stale `series` snapshot captured before that window would// silently drop the concurrent edit. Recompute the seasons merge and the@@ -1690,6 +1732,11 @@ async function commitImportOnce({// the actual state: input present, generation not yet performed.stages.idea = { status: 'empty', input: ideaSeed };}+ // The series admission keeps other imports out, but not an ordinary+ // issue create; ensureIssueWithId re-checks the planned position against+ // the live issues inside the series write queue and refuses one taken+ // since the read above (#10763), so the race fails here loudly and rolls+ // back instead of landing a duplicate position.const { created } = await ensureIssueWithId(item.issueId, {seriesId: updatedSeries.id,title: proposal.title,diff --git a/server/services/importer.test.js b/server/services/importer.test.jsindex 66b571639e..77a07c984d 100644--- a/server/services/importer.test.js+++ b/server/services/importer.test.js@@ -1994,6 +1994,167 @@ describe('commitImport import sessions (#9943)', () => {});});+// #10763: commits into one series are admitted one at a time, whatever their+// importId, and the issue publication rechecks positions against fresh state.+// Each interleaving test holds the FIRST commit's first issue write at a+// barrier — the point where the issue's reproduction found both commits had+// already passed the collision check — and lets the second commit run into it.+describe('commitImport series admission (#10763)', () => {+ function holdFirstPublication() {+ let reached;+ let release;+ const reachedP = new Promise((resolve) => { reached = resolve; });+ const releaseP = new Promise((resolve) => { release = resolve; });+ mockEnsureIssue.mockImplementationOnce(async (...args) => {+ reached();+ await releaseP;+ return realIssuesRef.current.ensureIssueWithId(...args);+ });+ return { reached: reachedP, release };+ }+ const importIdFor = (ser, source) => sessionsSvc.deriveImportId({ seriesId: ser.id, source });+ const livePositions = async (ser) => (await issuesSvc.listIssues({ seriesId: ser.id }))+ .map((i) => i.arcPosition).sort((a, b) => a - b);++ it('two manuscripts claiming one explicit position: one commits, the other is refused before any write', async () => {+ const { uni, ser } = await setupForCommit();+ const held = holdFirstPublication();+ const first = importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, importId: importIdFor(ser, 'manuscript A'),+ canonSelections: { characters: [{ name: 'Aria' }] },+ issues: [{ title: 'A1', arcPosition: 1, proseExcerpt: 'a' }],+ });+ await held.reached;+ const second = importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, importId: importIdFor(ser, 'manuscript B'),+ canonSelections: { characters: [{ name: 'Intruder' }] },+ arc: { logline: 'B logline', summary: 'B summary', shape: 'rags-to-riches' },+ issues: [{ title: 'B1', arcPosition: 1, proseExcerpt: 'b' }],+ });+ const secondOutcome = second.then(() => null, (err) => err);+ held.release();++ await expect(first).resolves.toMatchObject({ createdIssueIds: [expect.any(String)] });+ const err = await secondOutcome;+ expect(err?.code).toBe(importerSvc.ERR_VALIDATION);+ expect(err.message).toMatch(/collides with an existing issue/);+ expect((await issuesSvc.listIssues({ seriesId: ser.id })).map((i) => i.title)).toEqual(['A1']);+ // The refused batch wrote nothing: no canon, no arc.+ const universeAfter = await universeSvc.getUniverse(uni.id);+ expect(universeAfter.characters.map((c) => c.name)).toEqual(['Aria']);+ expect((await seriesSvc.getSeries(ser.id)).arc?.shape).not.toBe('rags-to-riches');+ });++ it('concurrent manuscripts with omitted positions get distinct sequential positions', async () => {+ const { uni, ser } = await setupForCommit();+ const held = holdFirstPublication();+ const first = importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, importId: importIdFor(ser, 'manuscript A'),+ issues: [{ title: 'A1', proseExcerpt: 'a' }, { title: 'A2', proseExcerpt: 'a2' }],+ });+ await held.reached;+ const second = importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, importId: importIdFor(ser, 'manuscript B'),+ issues: [{ title: 'B1', proseExcerpt: 'b' }],+ });+ held.release();+ await Promise.all([first, second]);+ expect(await livePositions(ser)).toEqual([1, 2, 3]);+ });++ it('applies the same admission to legacy commits with no importId', async () => {+ const { uni, ser } = await setupForCommit();+ const held = holdFirstPublication();+ const body = (title) => ({ universeId: uni.id, seriesId: ser.id, issues: [{ title, arcPosition: 1, proseExcerpt: 'p' }] });+ const first = importerSvc.commitImport(body('Legacy A'));+ await held.reached;+ const second = importerSvc.commitImport(body('Legacy B')).then(() => null, (err) => err);+ held.release();+ await first;+ expect((await second)?.code).toBe(importerSvc.ERR_VALIDATION);+ expect(await livePositions(ser)).toEqual([1]);+ });++ it('lets a commit into an unrelated series proceed while one series is held', async () => {+ const { uni, ser } = await setupForCommit();+ const other = await seriesSvc.createSeries({ name: 'Other S', universeId: uni.id, ephemeral: true });+ const held = holdFirstPublication();+ const first = importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, issues: [{ title: 'Held', arcPosition: 1, proseExcerpt: 'h' }],+ });+ await held.reached;+ // Resolves while the first series is still held — no shared admission.+ const result = await importerSvc.commitImport({+ universeId: uni.id, seriesId: other.id, issues: [{ title: 'Free', arcPosition: 1, proseExcerpt: 'f' }],+ });+ expect(result.createdIssueIds).toHaveLength(1);+ held.release();+ await first;+ expect(await livePositions(ser)).toEqual([1]);+ });++ it.each([+ ['explicit', { arcPosition: 1 }],+ ['omitted', {}],+ ])('a plain issue create racing the import cannot silently duplicate an %s position', async (_label, position) => {+ const { uni, ser } = await setupForCommit();+ // An ordinary create lands on the series after admission planned the+ // position, before the import publishes it.+ mockEnsureIssue.mockImplementationOnce(async (...args) => {+ await realIssuesRef.current.createIssue({ seriesId: ser.id, title: 'Manual', arcPosition: 1 });+ return realIssuesRef.current.ensureIssueWithId(...args);+ });+ await expect(importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, issues: [{ title: 'Imported', ...position, proseExcerpt: 'i' }],+ })).rejects.toMatchObject({+ code: importerSvc.ERR_PARTIAL_COMMIT_ISSUES,+ message: expect.stringMatching(/already held by another issue/),+ });+ expect((await issuesSvc.listIssues({ seriesId: ser.id })).map((i) => i.title)).toEqual(['Manual']);+ });++ it('releases admission after a failed commit so the next one into the series proceeds', async () => {+ const { uni, ser } = await setupForCommit();+ mockEnsureIssue.mockRejectedValueOnce(new Error('simulated write failure'));+ await expect(importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, issues: [{ title: 'Fails', arcPosition: 1, proseExcerpt: 'f' }],+ })).rejects.toMatchObject({ code: importerSvc.ERR_PARTIAL_COMMIT_ISSUES });+ const result = await importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, issues: [{ title: 'Works', arcPosition: 1, proseExcerpt: 'w' }],+ });+ expect(result.createdIssueIds).toHaveLength(1);+ });++ it('runs cleanupFormatting outside the admission: another commit lands while the reformat is in flight', async () => {+ const { uni, ser } = await setupForCommit();+ let releaseReformat;+ const reformatGate = new Promise((resolve) => { releaseReformat = resolve; });+ let reformatStarted;+ const started = new Promise((resolve) => { reformatStarted = resolve; });+ mockRunStagedLLM.mockImplementation(async () => {+ reformatStarted();+ await reformatGate;+ return { content: 'The vault loomed in the dark.', runId: 'r1' };+ });+ const slow = importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, contentType: 'short-story', cleanupFormatting: true,+ issues: [{ title: 'Slow', proseExcerpt: 'The vault loomed in the\ndark.' }],+ });+ await started;+ // Completes while the slow commit's LLM pass is still pending.+ await importerSvc.commitImport({+ universeId: uni.id, seriesId: ser.id, issues: [{ title: 'Fast', proseExcerpt: 'f' }],+ });+ releaseReformat();+ await slow;+ // The slow commit allocated from the state it found AFTER formatting, and+ // seeded the cleaned excerpt.+ const after = await issuesSvc.listIssues({ seriesId: ser.id });+ expect(Object.fromEntries(after.map((i) => [i.title, i.arcPosition]))).toEqual({ Fast: 1, Slow: 2 });+ expect(after.find((i) => i.title === 'Slow').stages.prose.output).toBe('The vault loomed in the dark.');+ });+});+describe('mergeSeasons (pure helper)', () => {const stubBuildSeason = (input) => ({id: `built-${input.number}`,diff --git a/server/services/pipeline/issueCrud.js b/server/services/pipeline/issueCrud.jsindex 07c4e1f39c..9ae94cadaf 100644--- a/server/services/pipeline/issueCrud.js+++ b/server/services/pipeline/issueCrud.js@@ -18,7 +18,7 @@ import {store, queueSeriesIssuesWrite, readState, readStateForSeries,saveIssueNow, saveIssuesNow, renumberInline, sanitizeIssue,snapshotRunHistory, ISSUE_ID_RE,- ERR_NOT_FOUND, ERR_VALIDATION, ERR_DUPLICATE, ERR_SEASON_LOCKED,+ ERR_NOT_FOUND, ERR_VALIDATION, ERR_DUPLICATE, ERR_SEASON_LOCKED, ERR_ARC_POSITION_CONFLICT,TITLE_MAX, SERIES_ID_MAX, ISSUES_PER_RESPONSE_MAX,} from './issuesShared.js';import { isNonBlankStr, isStr, trimTo } from '../../lib/textUtils.js';@@ -161,6 +161,12 @@ export function createIssue(input = {}, { preloadedSeries = null } = {}) {** A tombstoned id, or one living in another series, is refused (ERR_DUPLICATE)* rather than resurrected — the caller re-plans a fresh id for that slot.+ *+ * A NEW issue's planned `arcPosition` is re-checked against the series' live+ * issues inside the series write queue (#10763): the caller validated it+ * against an earlier read, and any other writer queued in between may have+ * taken it. A taken position is refused (ERR_ARC_POSITION_CONFLICT) rather+ * than landed as a duplicate.*/export function ensureIssueWithId(id, input = {}) {if (!isStr(id) || !ISSUE_ID_RE.test(id)) {@@ -181,6 +187,11 @@ function queueNewIssue(id, input, { preloadedSeries = null, reuseExisting = falsconst existing = state.issues.find((i) => i.id === id);if (existing && !existing.deleted && existing.seriesId === seriesId) return { issue: existing, created: false };if (existing) throw codedError(`Issue id already used: ${id}`, ERR_DUPLICATE);+ const pos = input.arcPosition;+ if (Number.isInteger(pos) && pos >= 1+ && state.issues.some((i) => i.seriesId === seriesId && !i.deleted && i.arcPosition === pos)) {+ throw codedError(`arcPosition ${pos} is already held by another issue in this series`, ERR_ARC_POSITION_CONFLICT);+ }}const next = sanitizeIssue({id,diff --git a/server/services/pipeline/issuesShared.js b/server/services/pipeline/issuesShared.jsindex 4c7d1d7452..7d7b1539c2 100644--- a/server/services/pipeline/issuesShared.js+++ b/server/services/pipeline/issuesShared.js@@ -80,6 +80,7 @@ export const ERR_NOT_FOUND = 'PIPELINE_ISSUE_NOT_FOUND';export const ERR_VALIDATION = 'PIPELINE_ISSUE_VALIDATION';export const ERR_DUPLICATE = 'PIPELINE_ISSUE_DUPLICATE';export const ERR_SEASON_LOCKED = 'PIPELINE_ISSUE_SEASON_LOCKED';+export const ERR_ARC_POSITION_CONFLICT = 'PIPELINE_ISSUE_ARC_POSITION_CONFLICT';export const ERR_STAGE_LOCKED = 'PIPELINE_STAGE_LOCKED';const ISSUE_ID_RE = /^iss-[A-Za-z0-9-]+$/;