Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
reconcile on failure
Julius Marminge9e2324c2e567
diff --git a/.plans/16c-pr89-remediation-checklist.md b/.plans/16c-pr89-remediation-checklist.mdindex 61b8c2c9c..ba7686dfd 100644--- a/.plans/16c-pr89-remediation-checklist.md+++ b/.plans/16c-pr89-remediation-checklist.md@@ -11,7 +11,7 @@ Status values:- `DONE`: Implemented and verified- `CLOSED_INVALID`: Stale/invalid review finding-Counts: active `53` (`valid=34`, `partially-valid=19`), closed-invalid `5`+Counts: active `52` (`valid=33`, `partially-valid=19`), closed-invalid `5`## Active Checklist@@ -163,8 +163,8 @@ Counts: active `53` (`valid=34`, `partially-valid=19`), closed-invalid `5`- Threads: PRRT_kwDORLtfbc5wzp4T- Audit note: Append/project/receipt are non-atomic; retry can duplicate events.-- [ ] `C013` If `projectionPipeline.projectEvent` fails after `eventStore.append` succeeds, the event is persisted but `readModel` isn't updated, causing desync. Consider updating the in-memory `readModel` immediately after append (before the external projection), so local state stays consistent regardless of downstream failures.- - Status: `TODO`+- [x] `C013` If `projectionPipeline.projectEvent` fails after `eventStore.append` succeeds, the event is persisted but `readModel` isn't updated, causing desync. Consider updating the in-memory `readModel` immediately after append (before the external projection), so local state stays consistent regardless of downstream failures.+ - Status: `DONE`- Verdict: `valid`- Severity: `Medium`- Area: `Event ordering and state consistency`diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.tsindex e2c8ab35f..81c46f07d 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts@@ -467,6 +467,107 @@ describe("OrchestrationEngine", () => {await runtime.dispose();});+ it("reconciles in-memory state when append persists but projection fails", async () => {+ type StoredEvent =+ ReturnType<OrchestrationEventStoreShape["append"]> extends Effect.Effect<infer A, any, any>+ ? A+ : never;+ const events: StoredEvent[] = [];+ let nextSequence = 1;++ const nonTransactionalStore: OrchestrationEventStoreShape = {+ append(event) {+ const savedEvent = {+ ...event,+ sequence: nextSequence,+ } as StoredEvent;+ nextSequence += 1;+ events.push(savedEvent);+ return Effect.succeed(savedEvent);+ },+ readFromSequence(sequenceExclusive) {+ return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));+ },+ readAll() {+ return Stream.fromIterable(events);+ },+ };++ let shouldFailProjection = true;+ const flakyProjectionPipeline: OrchestrationProjectionPipelineShape = {+ bootstrap: Effect.void,+ projectEvent: (event) => {+ if (+ shouldFailProjection &&+ event.commandId === CommandId.makeUnsafe("cmd-thread-meta-sync-fail")+ ) {+ shouldFailProjection = false;+ return Effect.fail(+ new PersistenceSqlError({+ operation: "test.projection",+ detail: "projection failed",+ }),+ );+ }+ return Effect.void;+ },+ };++ const runtime = ManagedRuntime.make(+ OrchestrationEngineLive.pipe(+ Layer.provide(Layer.succeed(OrchestrationProjectionPipeline, flakyProjectionPipeline)),+ Layer.provide(Layer.succeed(OrchestrationEventStore, nonTransactionalStore)),+ Layer.provide(OrchestrationCommandReceiptRepositoryLive),+ Layer.provide(SqlitePersistenceMemory),+ ),+ );+ const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService));+ const createdAt = now();++ await runtime.runPromise(+ engine.dispatch({+ type: "project.create",+ commandId: CommandId.makeUnsafe("cmd-project-sync-create"),+ projectId: asProjectId("project-sync"),+ title: "Sync Project",+ workspaceRoot: "/tmp/project-sync",+ defaultModel: "gpt-5-codex",+ createdAt,+ }),+ );+ await runtime.runPromise(+ engine.dispatch({+ type: "thread.create",+ commandId: CommandId.makeUnsafe("cmd-thread-sync-create"),+ threadId: ThreadId.makeUnsafe("thread-sync"),+ projectId: asProjectId("project-sync"),+ title: "sync-before",+ model: "gpt-5-codex",+ branch: null,+ worktreePath: null,+ createdAt,+ }),+ );++ await expect(+ runtime.runPromise(+ engine.dispatch({+ type: "thread.meta.update",+ commandId: CommandId.makeUnsafe("cmd-thread-meta-sync-fail"),+ threadId: ThreadId.makeUnsafe("thread-sync"),+ title: "sync-after-failed-projection",+ }),+ ),+ ).rejects.toThrow("projection failed");++ const readModelAfterFailure = await runtime.runPromise(engine.getReadModel());+ const updatedThread = readModelAfterFailure.threads.find((thread) => thread.id === "thread-sync");+ expect(readModelAfterFailure.snapshotSequence).toBe(3);+ expect(updatedThread?.title).toBe("sync-after-failed-projection");++ await runtime.dispose();+ });+it("fails command dispatch when command invariants are violated", async () => {const system = await createOrchestrationSystem();const { engine } = system;diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.tsindex ebe554770..ce48f1e1b 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts@@ -67,8 +67,28 @@ const makeOrchestrationEngine = Effect.gen(function* () {const commandQueue = yield* Queue.unbounded<CommandEnvelope>();const eventPubSub = yield* PubSub.unbounded<OrchestrationEvent>();- const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> =>- Effect.gen(function* () {+ const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {+ const dispatchStartSequence = readModel.snapshotSequence;+ const reconcileReadModelAfterDispatchFailure = Effect.gen(function* () {+ const persistedEvents = yield* Stream.runCollect(+ eventStore.readFromSequence(dispatchStartSequence),+ ).pipe(Effect.map((chunk): OrchestrationEvent[] => Array.from(chunk)));+ if (persistedEvents.length === 0) {+ return;+ }++ let nextReadModel = readModel;+ for (const persistedEvent of persistedEvents) {+ nextReadModel = yield* projectEvent(nextReadModel, persistedEvent);+ }+ readModel = nextReadModel;++ for (const persistedEvent of persistedEvents) {+ yield* PubSub.publish(eventPubSub, persistedEvent);+ }+ });++ return Effect.gen(function* () {const existingReceipt = yield* commandReceiptRepository.getByCommandId({commandId: envelope.command.commandId,});@@ -102,8 +122,8 @@ const makeOrchestrationEngine = Effect.gen(function* () {for (const nextEvent of eventBases) {const savedEvent = yield* eventStore.append(nextEvent);- yield* projectionPipeline.projectEvent(savedEvent);nextReadModel = yield* projectEvent(nextReadModel, savedEvent);+ yield* projectionPipeline.projectEvent(savedEvent);committedEvents.push(savedEvent);}@@ -148,6 +168,19 @@ const makeOrchestrationEngine = Effect.gen(function* () {}).pipe(Effect.catch((error) =>Effect.gen(function* () {+ yield* reconcileReadModelAfterDispatchFailure.pipe(+ Effect.catch(() =>+ Effect.logWarning(+ "failed to reconcile orchestration read model after dispatch failure",+ ).pipe(+ Effect.annotateLogs({+ commandId: envelope.command.commandId,+ snapshotSequence: readModel.snapshotSequence,+ }),+ ),+ ),+ );+if (error instanceof OrchestrationCommandInvariantError) {const aggregateRef = commandToAggregateRef(envelope.command);yield* commandReceiptRepository@@ -166,6 +199,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {}),),);+ };yield* projectionPipeline.bootstrap;