Skip to content
TypeScript

test(server): strengthen v1 cutover proof with event-stream waits and content snapshots (#11869)

Alex Southwell221d8b8812d1
diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts
index 83c34c7edab..fe8e999eec0 100644
--- a/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts
+++ b/apps/server/src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts
@@ -1,4 +1,5 @@
import { assert, describe, it } from "@effect/vitest";
+import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import {
CommandId,
@@ -14,8 +15,10 @@ import {
type OrchestrationV2ThreadProjection,
} from "@t3tools/contracts";
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
+import * as Crypto from "effect/Crypto";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
+import * as Hex from "effect/encoding/Hex";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Logger from "effect/Logger";
@@ -400,6 +403,37 @@ const seedV1Database = (fixturePath: string, workspace: string) =>
});
yield* sql`PRAGMA wal_checkpoint(TRUNCATE);`;
+
+ // Deterministic pre-migration snapshot of the legacy recovery tables;
+ // post-boot assertions compare preserved column contents, not counts.
+ // Migrations may append columns, so the comparison is restricted to the
+ // column set recorded here.
+ const tableColumns = Effect.fn("seedV1Database.tableColumns")(function* (table: string) {
+ const info = yield* sql<{
+ readonly name: string;
+ }>`SELECT name FROM pragma_table_info(${table})`;
+ return info.map((row) => row.name);
+ });
+ return {
+ projects: {
+ columns: yield* tableColumns("projection_projects"),
+ rows: yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_projects ORDER BY project_id
+ `,
+ },
+ threads: {
+ columns: yield* tableColumns("projection_threads"),
+ rows: yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_threads ORDER BY thread_id
+ `,
+ },
+ messages: {
+ columns: yield* tableColumns("projection_thread_messages"),
+ rows: yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_thread_messages ORDER BY message_id
+ `,
+ },
+ };
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: fixturePath }))),
);
@@ -551,32 +585,62 @@ const makeCodexAdapter = (capturedTurns: Ref.Ref<ReadonlyArray<CapturedTurn>>) =
}),
}) satisfies ProviderAdapter.ProviderAdapterV2["Service"];
-const waitForIdle = Effect.fn("LegacyV1Cutover.waitForIdle")(function* (threadId: ThreadId) {
- const orchestrator = yield* Orchestrator.OrchestratorV2;
- const eventSink = yield* EventSink.EventSinkV2;
- const isIdle = (projection: OrchestrationV2ThreadProjection) =>
- projection.runs.every(
- (run) => !["queued", "starting", "running", "waiting"].includes(run.status),
+// Await the dispatched run's terminal event on the persisted domain-event stream
+// instead of polling the projection, then drain remaining outbox effects.
+const dispatchMessageAwaitingRun = Effect.fn("LegacyV1Cutover.dispatchMessageAwaitingRun")(
+ function* (input: {
+ readonly threadId: ThreadId;
+ readonly messageId: string;
+ readonly commandId: string;
+ readonly text: string;
+ }) {
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
+ const eventSink = yield* EventSink.EventSinkV2;
+ const messageId = MessageId.make(input.messageId);
+ // Capture the cursor before dispatch so even a late subscriber replays
+ // the terminal event instead of starting its live tail after completion.
+ const afterSequence = yield* eventSink.latestSequence();
+ const terminal = eventSink.stream({ afterSequence }).pipe(
+ Stream.map((stored) => stored.event),
+ Stream.filter((event) => event.type === "run.updated"),
+ Stream.filter(
+ (event) =>
+ ["completed", "failed", "interrupted", "cancelled", "rolled_back"].includes(
+ event.payload.status,
+ ) &&
+ event.payload.threadId === input.threadId &&
+ event.payload.userMessageId === messageId,
+ ),
+ Stream.take(1),
+ Stream.runForEach((event) =>
+ Effect.sync(() =>
+ assert.equal(event.payload.status, "completed", `Run for ${messageId} did not complete`),
+ ),
+ ),
);
- const afterSequence = yield* eventSink.latestSequence();
- const current = yield* orchestrator.getThreadProjection(threadId);
- if (isIdle(current)) {
- return current;
- }
- // Projections commit with their events, so each later event is the only time to re-read.
- const [idle] = yield* eventSink.stream({ afterSequence }).pipe(
- Stream.mapEffect(() => orchestrator.getThreadProjection(threadId)),
- Stream.filter(isIdle),
- Stream.take(1),
- Stream.runCollect,
- );
- if (idle === undefined) {
- return yield* Effect.die(
- new Error("Cutover test event stream ended before the thread went idle"),
+ yield* orchestrator.dispatch({
+ type: "message.dispatch",
+ createdBy: "user",
+ creationSource: "web",
+ commandId: CommandId.make(input.commandId),
+ threadId: input.threadId,
+ messageId,
+ text: input.text,
+ attachments: [],
+ dispatchMode: { type: "start_immediately" },
+ });
+ yield* worker.drain();
+ yield* terminal.pipe(
+ Effect.timeoutOrElse({
+ duration: "30 seconds",
+ orElse: () => Effect.die(`No terminal run.updated for ${messageId}`),
+ }),
);
- }
- return idle;
-});
+ yield* worker.drain();
+ return yield* orchestrator.getThreadProjection(input.threadId);
+ },
+);
const layerBoot = (input: {
readonly name: string;
@@ -639,6 +703,26 @@ const makeCapturingLogger = (logs: CapturedLog[]) =>
});
});
+const sha256File = Effect.fn("LegacyV1Cutover.sha256File")(function* (filePath: string) {
+ const fs = yield* FileSystem.FileSystem;
+ const crypto = yield* Crypto.Crypto;
+ return Hex.encode(
+ yield* crypto.digest("SHA-256", yield* fs.readFile(filePath)).pipe(Effect.orDie),
+ );
+}, Effect.provide(NodeCrypto.layer));
+
+interface LegacyTableSnapshot {
+ readonly columns: ReadonlyArray<string>;
+ readonly rows: ReadonlyArray<Record<string, unknown>>;
+}
+
+// Migrations legitimately append columns; preservation means the columns the
+// v1 schema already had keep their exact values.
+const restrictToSnapshotColumns = (
+ rows: ReadonlyArray<Record<string, unknown>>,
+ snapshot: LegacyTableSnapshot,
+) => rows.map((row) => Object.fromEntries(snapshot.columns.map((column) => [column, row[column]])));
+
const messageOrdinals = (projection: OrchestrationV2ThreadProjection) =>
projection.turnItems
.filter(
@@ -663,7 +747,8 @@ describe("orchestration v2 legacy v1 cutover", () => {
const copyPath = path.join(stateDir, "userdata", "state.sqlite");
yield* fs.makeDirectory(path.join(stateDir, "userdata"), { recursive: true });
- yield* seedV1Database(fixturePath, workspace);
+ const v1Snapshot = yield* seedV1Database(fixturePath, workspace);
+ const fixtureHashBefore = yield* sha256File(fixturePath);
yield* fs.copyFile(fixturePath, copyPath);
const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
@@ -676,7 +761,6 @@ describe("orchestration v2 legacy v1 cutover", () => {
const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter;
const maintenance = yield* ProjectionReplay.ProjectionReplay;
const projections = yield* ProjectionStore.ProjectionStoreV2;
- const orchestrator = yield* Orchestrator.OrchestratorV2;
assert.equal(yield* importer.pendingThreadCount, ALL_THREADS.length);
const shellImport = yield* importer.reconcileShells;
@@ -808,6 +892,22 @@ describe("orchestration v2 legacy v1 cutover", () => {
`;
assert.isNull(snoozedImportRow[0]?.transcript_imported_at);
+ // A failed hydration leaves the legacy rows and pending marker
+ // available for retry in a fresh runtime after restart.
+ yield* sql`
+ CREATE TRIGGER fail_snoozed_hydration
+ BEFORE INSERT ON orchestration_events
+ WHEN NEW.event_id = 'migration:v1:message:message:cutover:snoozed:2'
+ BEGIN
+ SELECT RAISE(ABORT, 'injected cutover hydration failure');
+ END
+ `;
+ const hydrationError = yield* importer
+ .ensureTranscript(ThreadId.make(SNOOZED_THREAD))
+ .pipe(Effect.flip);
+ assert.equal(hydrationError._tag, "LegacyV1ThreadImportError");
+ assert.equal(yield* importer.pendingThreadCount, 1);
+
const interrupted = yield* projections.getThreadProjection(
ThreadId.make(INTERRUPTED_THREAD),
);
@@ -816,18 +916,12 @@ describe("orchestration v2 legacy v1 cutover", () => {
["message:cutover:interrupted:2", 2, "interrupted"],
]);
- yield* orchestrator.dispatch({
- type: "message.dispatch",
- createdBy: "user",
- creationSource: "web",
- commandId: CommandId.make("command:cutover:continue"),
+ const continued = yield* dispatchMessageAwaitingRun({
threadId: longThreadId,
- messageId: MessageId.make("message:cutover:long:continuation"),
+ messageId: "message:cutover:long:continuation",
+ commandId: "command:cutover:continue",
text: CONTINUATION_PROMPT,
- attachments: [],
- dispatchMode: { type: "start_immediately" },
});
- const continued = yield* waitForIdle(longThreadId);
const runs = continued.runs.filter((run) => run.ordinal >= 1);
assert.equal(runs.at(-1)?.status, "completed");
@@ -844,18 +938,12 @@ describe("orchestration v2 legacy v1 cutover", () => {
assert.include(turns[0]!.text, "Context handoff (manual_context):");
assert.include(turns[0]!.text, `User message:\n${CONTINUATION_PROMPT}`);
- yield* orchestrator.dispatch({
- type: "message.dispatch",
- createdBy: "user",
- creationSource: "web",
- commandId: CommandId.make("command:cutover:continue:again"),
+ const continuedAgain = yield* dispatchMessageAwaitingRun({
threadId: longThreadId,
- messageId: MessageId.make("message:cutover:long:continuation:2"),
+ messageId: "message:cutover:long:continuation:2",
+ commandId: "command:cutover:continue:again",
text: "One more.",
- attachments: [],
- dispatchMode: { type: "start_immediately" },
});
- const continuedAgain = yield* waitForIdle(longThreadId);
assert.equal(continuedAgain.contextHandoffs.length, 1);
assert.equal((yield* Ref.get(capturedTurns)).length, 2);
@@ -876,11 +964,14 @@ describe("orchestration v2 legacy v1 cutover", () => {
FROM orchestration_v2_legacy_imports
ORDER BY thread_id
`;
- const legacyMessageCount = yield* sql<{ readonly count: number }>`
- SELECT COUNT(*) AS count FROM projection_thread_messages
+ const legacyProjects = yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_projects ORDER BY project_id
+ `;
+ const legacyThreads = yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_threads ORDER BY thread_id
`;
- const legacyThreadCount = yield* sql<{ readonly count: number }>`
- SELECT COUNT(*) AS count FROM projection_threads
+ const legacyMessages = yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_thread_messages ORDER BY message_id
`;
const recordedMigration41 = yield* sql<{ readonly name: string }>`
SELECT name FROM effect_sql_migrations WHERE migration_id = 41
@@ -891,8 +982,9 @@ describe("orchestration v2 legacy v1 cutover", () => {
return {
migrationEventCount: migrationEventCount[0]?.count ?? 0,
importRows,
- legacyMessageCount: legacyMessageCount[0]?.count ?? 0,
- legacyThreadCount: legacyThreadCount[0]?.count ?? 0,
+ legacyProjects,
+ legacyThreads,
+ legacyMessages,
longProjection: continuedAgain,
migration41Name: recordedMigration41[0]?.name ?? null,
authSessionColumnNames: authSessionColumns.map((column) => column.name),
@@ -924,6 +1016,21 @@ describe("orchestration v2 legacy v1 cutover", () => {
assert.notInclude(firstBoot.authSessionColumnNames, "client_surface");
assert.notInclude(firstBoot.authSessionColumnNames, "client_app_version");
+ // Full row contents of the legacy recovery tables survive migration
+ // and continuation unchanged, not just their counts.
+ assert.deepStrictEqual(
+ restrictToSnapshotColumns(firstBoot.legacyProjects, v1Snapshot.projects),
+ v1Snapshot.projects.rows,
+ );
+ assert.deepStrictEqual(
+ restrictToSnapshotColumns(firstBoot.legacyThreads, v1Snapshot.threads),
+ v1Snapshot.threads.rows,
+ );
+ assert.deepStrictEqual(
+ restrictToSnapshotColumns(firstBoot.legacyMessages, v1Snapshot.messages),
+ v1Snapshot.messages.rows,
+ );
+
assert.equal(firstBoot.importRows.length, ALL_THREADS.length);
const unhydratedRows = firstBoot.importRows.filter(
(row) => row.transcript_imported_at === null,
@@ -945,6 +1052,7 @@ describe("orchestration v2 legacy v1 cutover", () => {
importedThreadCount: 0,
importedMessageCount: 0,
});
+ yield* sql`DROP TRIGGER fail_snoozed_hydration`;
assert.deepStrictEqual(
yield* importer.ensureTranscript(ThreadId.make(SNOOZED_THREAD)),
{ importedThreadCount: 1, importedMessageCount: 1 },
@@ -983,14 +1091,33 @@ describe("orchestration v2 legacy v1 cutover", () => {
firstBoot.longProjection.runs.map((run) => run.status),
);
- const legacyMessageCount = yield* sql<{ readonly count: number }>`
- SELECT COUNT(*) AS count FROM projection_thread_messages
- `;
- const legacyThreadCount = yield* sql<{ readonly count: number }>`
- SELECT COUNT(*) AS count FROM projection_threads
- `;
- assert.equal(legacyMessageCount[0]?.count, firstBoot.legacyMessageCount);
- assert.equal(legacyThreadCount[0]?.count, firstBoot.legacyThreadCount);
+ assert.deepStrictEqual(
+ restrictToSnapshotColumns(
+ yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_projects ORDER BY project_id
+ `,
+ v1Snapshot.projects,
+ ),
+ v1Snapshot.projects.rows,
+ );
+ assert.deepStrictEqual(
+ restrictToSnapshotColumns(
+ yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_threads ORDER BY thread_id
+ `,
+ v1Snapshot.threads,
+ ),
+ v1Snapshot.threads.rows,
+ );
+ assert.deepStrictEqual(
+ restrictToSnapshotColumns(
+ yield* sql<Record<string, unknown>>`
+ SELECT * FROM projection_thread_messages ORDER BY message_id
+ `,
+ v1Snapshot.messages,
+ ),
+ v1Snapshot.messages.rows,
+ );
const quickCheck = yield* sql<{ readonly quick_check: string }>`
PRAGMA quick_check
`;
@@ -1017,6 +1144,10 @@ describe("orchestration v2 legacy v1 cutover", () => {
assert.isTrue(
boot2Logs.some((log) => String(log.message).includes("migration history diverges")),
);
+
+ // The seeded source file is the untouched recovery copy: migration
+ // only ever ran against the file copied out of it.
+ assert.equal(yield* sha256File(fixturePath), fixtureHashBefore);
}).pipe(Effect.provide(NodeServices.layer)),
),
);