TypeScript
test(server): strengthen v1 cutover proof with event-stream waits and content snapshots (#11869)
Alex Southwell221d8b8812d1
diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.tsindex 83c34c7edab..fe8e999eec0 100644--- a/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts+++ b/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts@@ -1,4 +1,5 @@import { assert, describe, it } from "@effect/vitest";+import * as NodeCrypto from "@effect/platform-node/NodeCrypto";import * as NodeServices from "@effect/platform-node/NodeServices";import {CommandId,@@ -14,8 +15,10 @@ import {type OrchestrationV2ThreadProjection,} from "@t3tools/contracts";import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";+import * as Crypto from "effect/Crypto";import * as DateTime from "effect/DateTime";import * as Effect from "effect/Effect";+import * as Hex from "effect/encoding/Hex";import * as FileSystem from "effect/FileSystem";import * as Layer from "effect/Layer";import * as Logger from "effect/Logger";@@ -400,6 +403,37 @@ const seedV1Database = (fixturePath: string, workspace: string) =>});yield* sql`PRAGMA wal_checkpoint(TRUNCATE);`;++ // Deterministic pre-migration snapshot of the legacy recovery tables;+ // post-boot assertions compare preserved column contents, not counts.+ // Migrations may append columns, so the comparison is restricted to the+ // column set recorded here.+ const tableColumns = Effect.fn("seedV1Database.tableColumns")(function* (table: string) {+ const info = yield* sql<{+ readonly name: string;+ }>`SELECT name FROM pragma_table_info(${table})`;+ return info.map((row) => row.name);+ });+ return {+ projects: {+ columns: yield* tableColumns("projection_projects"),+ rows: yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_projects ORDER BY project_id+ `,+ },+ threads: {+ columns: yield* tableColumns("projection_threads"),+ rows: yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_threads ORDER BY thread_id+ `,+ },+ messages: {+ columns: yield* tableColumns("projection_thread_messages"),+ rows: yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_thread_messages ORDER BY message_id+ `,+ },+ };}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: fixturePath }))),);@@ -551,32 +585,62 @@ const makeCodexAdapter = (capturedTurns: Ref.Ref<ReadonlyArray<CapturedTurn>>) =}),}) satisfies ProviderAdapter.ProviderAdapterV2["Service"];-const waitForIdle = Effect.fn("LegacyV1Cutover.waitForIdle")(function* (threadId: ThreadId) {- const orchestrator = yield* Orchestrator.OrchestratorV2;- const eventSink = yield* EventSink.EventSinkV2;- const isIdle = (projection: OrchestrationV2ThreadProjection) =>- projection.runs.every(- (run) => !["queued", "starting", "running", "waiting"].includes(run.status),+// Await the dispatched run's terminal event on the persisted domain-event stream+// instead of polling the projection, then drain remaining outbox effects.+const dispatchMessageAwaitingRun = Effect.fn("LegacyV1Cutover.dispatchMessageAwaitingRun")(+ function* (input: {+ readonly threadId: ThreadId;+ readonly messageId: string;+ readonly commandId: string;+ readonly text: string;+ }) {+ const orchestrator = yield* Orchestrator.OrchestratorV2;+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;+ const eventSink = yield* EventSink.EventSinkV2;+ const messageId = MessageId.make(input.messageId);+ // Capture the cursor before dispatch so even a late subscriber replays+ // the terminal event instead of starting its live tail after completion.+ const afterSequence = yield* eventSink.latestSequence();+ const terminal = eventSink.stream({ afterSequence }).pipe(+ Stream.map((stored) => stored.event),+ Stream.filter((event) => event.type === "run.updated"),+ Stream.filter(+ (event) =>+ ["completed", "failed", "interrupted", "cancelled", "rolled_back"].includes(+ event.payload.status,+ ) &&+ event.payload.threadId === input.threadId &&+ event.payload.userMessageId === messageId,+ ),+ Stream.take(1),+ Stream.runForEach((event) =>+ Effect.sync(() =>+ assert.equal(event.payload.status, "completed", `Run for ${messageId} did not complete`),+ ),+ ),);- const afterSequence = yield* eventSink.latestSequence();- const current = yield* orchestrator.getThreadProjection(threadId);- if (isIdle(current)) {- return current;- }- // Projections commit with their events, so each later event is the only time to re-read.- const [idle] = yield* eventSink.stream({ afterSequence }).pipe(- Stream.mapEffect(() => orchestrator.getThreadProjection(threadId)),- Stream.filter(isIdle),- Stream.take(1),- Stream.runCollect,- );- if (idle === undefined) {- return yield* Effect.die(- new Error("Cutover test event stream ended before the thread went idle"),+ yield* orchestrator.dispatch({+ type: "message.dispatch",+ createdBy: "user",+ creationSource: "web",+ commandId: CommandId.make(input.commandId),+ threadId: input.threadId,+ messageId,+ text: input.text,+ attachments: [],+ dispatchMode: { type: "start_immediately" },+ });+ yield* worker.drain();+ yield* terminal.pipe(+ Effect.timeoutOrElse({+ duration: "30 seconds",+ orElse: () => Effect.die(`No terminal run.updated for ${messageId}`),+ }),);- }- return idle;-});+ yield* worker.drain();+ return yield* orchestrator.getThreadProjection(input.threadId);+ },+);const layerBoot = (input: {readonly name: string;@@ -639,6 +703,26 @@ const makeCapturingLogger = (logs: CapturedLog[]) =>});});+const sha256File = Effect.fn("LegacyV1Cutover.sha256File")(function* (filePath: string) {+ const fs = yield* FileSystem.FileSystem;+ const crypto = yield* Crypto.Crypto;+ return Hex.encode(+ yield* crypto.digest("SHA-256", yield* fs.readFile(filePath)).pipe(Effect.orDie),+ );+}, Effect.provide(NodeCrypto.layer));++interface LegacyTableSnapshot {+ readonly columns: ReadonlyArray<string>;+ readonly rows: ReadonlyArray<Record<string, unknown>>;+}++// Migrations legitimately append columns; preservation means the columns the+// v1 schema already had keep their exact values.+const restrictToSnapshotColumns = (+ rows: ReadonlyArray<Record<string, unknown>>,+ snapshot: LegacyTableSnapshot,+) => rows.map((row) => Object.fromEntries(snapshot.columns.map((column) => [column, row[column]])));+const messageOrdinals = (projection: OrchestrationV2ThreadProjection) =>projection.turnItems.filter(@@ -663,7 +747,8 @@ describe("orchestration v2 legacy v1 cutover", () => {const copyPath = path.join(stateDir, "userdata", "state.sqlite");yield* fs.makeDirectory(path.join(stateDir, "userdata"), { recursive: true });- yield* seedV1Database(fixturePath, workspace);+ const v1Snapshot = yield* seedV1Database(fixturePath, workspace);+ const fixtureHashBefore = yield* sha256File(fixturePath);yield* fs.copyFile(fixturePath, copyPath);const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);@@ -676,7 +761,6 @@ describe("orchestration v2 legacy v1 cutover", () => {const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter;const maintenance = yield* ProjectionReplay.ProjectionReplay;const projections = yield* ProjectionStore.ProjectionStoreV2;- const orchestrator = yield* Orchestrator.OrchestratorV2;assert.equal(yield* importer.pendingThreadCount, ALL_THREADS.length);const shellImport = yield* importer.reconcileShells;@@ -808,6 +892,22 @@ describe("orchestration v2 legacy v1 cutover", () => {`;assert.isNull(snoozedImportRow[0]?.transcript_imported_at);+ // A failed hydration leaves the legacy rows and pending marker+ // available for retry in a fresh runtime after restart.+ yield* sql`+ CREATE TRIGGER fail_snoozed_hydration+ BEFORE INSERT ON orchestration_events+ WHEN NEW.event_id = 'migration:v1:message:message:cutover:snoozed:2'+ BEGIN+ SELECT RAISE(ABORT, 'injected cutover hydration failure');+ END+ `;+ const hydrationError = yield* importer+ .ensureTranscript(ThreadId.make(SNOOZED_THREAD))+ .pipe(Effect.flip);+ assert.equal(hydrationError._tag, "LegacyV1ThreadImportError");+ assert.equal(yield* importer.pendingThreadCount, 1);+const interrupted = yield* projections.getThreadProjection(ThreadId.make(INTERRUPTED_THREAD),);@@ -816,18 +916,12 @@ describe("orchestration v2 legacy v1 cutover", () => {["message:cutover:interrupted:2", 2, "interrupted"],]);- yield* orchestrator.dispatch({- type: "message.dispatch",- createdBy: "user",- creationSource: "web",- commandId: CommandId.make("command:cutover:continue"),+ const continued = yield* dispatchMessageAwaitingRun({threadId: longThreadId,- messageId: MessageId.make("message:cutover:long:continuation"),+ messageId: "message:cutover:long:continuation",+ commandId: "command:cutover:continue",text: CONTINUATION_PROMPT,- attachments: [],- dispatchMode: { type: "start_immediately" },});- const continued = yield* waitForIdle(longThreadId);const runs = continued.runs.filter((run) => run.ordinal >= 1);assert.equal(runs.at(-1)?.status, "completed");@@ -844,18 +938,12 @@ describe("orchestration v2 legacy v1 cutover", () => {assert.include(turns[0]!.text, "Context handoff (manual_context):");assert.include(turns[0]!.text, `User message:\n${CONTINUATION_PROMPT}`);- yield* orchestrator.dispatch({- type: "message.dispatch",- createdBy: "user",- creationSource: "web",- commandId: CommandId.make("command:cutover:continue:again"),+ const continuedAgain = yield* dispatchMessageAwaitingRun({threadId: longThreadId,- messageId: MessageId.make("message:cutover:long:continuation:2"),+ messageId: "message:cutover:long:continuation:2",+ commandId: "command:cutover:continue:again",text: "One more.",- attachments: [],- dispatchMode: { type: "start_immediately" },});- const continuedAgain = yield* waitForIdle(longThreadId);assert.equal(continuedAgain.contextHandoffs.length, 1);assert.equal((yield* Ref.get(capturedTurns)).length, 2);@@ -876,11 +964,14 @@ describe("orchestration v2 legacy v1 cutover", () => {FROM orchestration_v2_legacy_importsORDER BY thread_id`;- const legacyMessageCount = yield* sql<{ readonly count: number }>`- SELECT COUNT(*) AS count FROM projection_thread_messages+ const legacyProjects = yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_projects ORDER BY project_id+ `;+ const legacyThreads = yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_threads ORDER BY thread_id`;- const legacyThreadCount = yield* sql<{ readonly count: number }>`- SELECT COUNT(*) AS count FROM projection_threads+ const legacyMessages = yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_thread_messages ORDER BY message_id`;const recordedMigration41 = yield* sql<{ readonly name: string }>`SELECT name FROM effect_sql_migrations WHERE migration_id = 41@@ -891,8 +982,9 @@ describe("orchestration v2 legacy v1 cutover", () => {return {migrationEventCount: migrationEventCount[0]?.count ?? 0,importRows,- legacyMessageCount: legacyMessageCount[0]?.count ?? 0,- legacyThreadCount: legacyThreadCount[0]?.count ?? 0,+ legacyProjects,+ legacyThreads,+ legacyMessages,longProjection: continuedAgain,migration41Name: recordedMigration41[0]?.name ?? null,authSessionColumnNames: authSessionColumns.map((column) => column.name),@@ -924,6 +1016,21 @@ describe("orchestration v2 legacy v1 cutover", () => {assert.notInclude(firstBoot.authSessionColumnNames, "client_surface");assert.notInclude(firstBoot.authSessionColumnNames, "client_app_version");+ // Full row contents of the legacy recovery tables survive migration+ // and continuation unchanged, not just their counts.+ assert.deepStrictEqual(+ restrictToSnapshotColumns(firstBoot.legacyProjects, v1Snapshot.projects),+ v1Snapshot.projects.rows,+ );+ assert.deepStrictEqual(+ restrictToSnapshotColumns(firstBoot.legacyThreads, v1Snapshot.threads),+ v1Snapshot.threads.rows,+ );+ assert.deepStrictEqual(+ restrictToSnapshotColumns(firstBoot.legacyMessages, v1Snapshot.messages),+ v1Snapshot.messages.rows,+ );+assert.equal(firstBoot.importRows.length, ALL_THREADS.length);const unhydratedRows = firstBoot.importRows.filter((row) => row.transcript_imported_at === null,@@ -945,6 +1052,7 @@ describe("orchestration v2 legacy v1 cutover", () => {importedThreadCount: 0,importedMessageCount: 0,});+ yield* sql`DROP TRIGGER fail_snoozed_hydration`;assert.deepStrictEqual(yield* importer.ensureTranscript(ThreadId.make(SNOOZED_THREAD)),{ importedThreadCount: 1, importedMessageCount: 1 },@@ -983,14 +1091,33 @@ describe("orchestration v2 legacy v1 cutover", () => {firstBoot.longProjection.runs.map((run) => run.status),);- const legacyMessageCount = yield* sql<{ readonly count: number }>`- SELECT COUNT(*) AS count FROM projection_thread_messages- `;- const legacyThreadCount = yield* sql<{ readonly count: number }>`- SELECT COUNT(*) AS count FROM projection_threads- `;- assert.equal(legacyMessageCount[0]?.count, firstBoot.legacyMessageCount);- assert.equal(legacyThreadCount[0]?.count, firstBoot.legacyThreadCount);+ assert.deepStrictEqual(+ restrictToSnapshotColumns(+ yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_projects ORDER BY project_id+ `,+ v1Snapshot.projects,+ ),+ v1Snapshot.projects.rows,+ );+ assert.deepStrictEqual(+ restrictToSnapshotColumns(+ yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_threads ORDER BY thread_id+ `,+ v1Snapshot.threads,+ ),+ v1Snapshot.threads.rows,+ );+ assert.deepStrictEqual(+ restrictToSnapshotColumns(+ yield* sql<Record<string, unknown>>`+ SELECT * FROM projection_thread_messages ORDER BY message_id+ `,+ v1Snapshot.messages,+ ),+ v1Snapshot.messages.rows,+ );const quickCheck = yield* sql<{ readonly quick_check: string }>`PRAGMA quick_check`;@@ -1017,6 +1144,10 @@ describe("orchestration v2 legacy v1 cutover", () => {assert.isTrue(boot2Logs.some((log) => String(log.message).includes("migration history diverges")),);++ // The seeded source file is the untouched recovery copy: migration+ // only ever ran against the file copied out of it.+ assert.equal(yield* sha256File(fixturePath), fixtureHashBefore);}).pipe(Effect.provide(NodeServices.layer)),),);