Skip to content

Open-source SEO, GEO, and marketing skills for AI agents.

fix: pre-landing review fixes

Yuting Zhong659d717a445d
diff --git a/notfair/src/components/chat/transcript-model.test.ts b/notfair/src/components/chat/transcript-model.test.ts
index 7dece81b..ee5f6e0f 100644
--- a/notfair/src/components/chat/transcript-model.test.ts
+++ b/notfair/src/components/chat/transcript-model.test.ts
@@ -38,6 +38,17 @@ const result = (id: string, tcid: string, ok = true): TranscriptEvent => ({
summary: null,
ok,
});
+const lifecycle = (
+ id: string,
+ phase: string,
+ ok?: boolean,
+): TranscriptEvent => ({
+ kind: "lifecycle",
+ id,
+ ts: t,
+ phase,
+ ok,
+});
describe("collapseEvents", () => {
it("pairs calls with results and groups contiguous tools", () => {
@@ -101,6 +112,78 @@ describe("collapseEvents", () => {
>;
expect(group.tools[0]).toMatchObject({ toolCallId: "t9", done: true, ok: false });
});
+
+ it("closes an unmatched tool call when the turn reaches a terminal boundary", () => {
+ const items = collapseEvents([
+ call("c1", "t1"),
+ text("a1", "finished"),
+ lifecycle("done-1", "done"),
+ ]);
+ const group = items[0] as Extract<
+ ReturnType<typeof collapseEvents>[number],
+ { kind: "tool_group" }
+ >;
+ expect(group.tools[0]).toMatchObject({ toolCallId: "t1", done: true });
+ });
+
+ it("marks an unmatched tool failed when the terminal boundary is an error", () => {
+ const items = collapseEvents([
+ call("c1", "t1"),
+ text("err-1", "⚠ harness crashed"),
+ lifecycle("done-1", "done", false),
+ ]);
+ const group = items[0] as Extract<
+ ReturnType<typeof collapseEvents>[number],
+ { kind: "tool_group" }
+ >;
+ expect(group.tools[0]).toMatchObject({
+ toolCallId: "t1",
+ done: true,
+ ok: false,
+ });
+ });
+
+ it("closes abandoned tools before a later turn reuses their id", () => {
+ const items = collapseEvents([
+ call("c1", "item_1"),
+ user("u2", "try again"),
+ lifecycle("start-2", "start"),
+ call("c2", "item_1"),
+ ]);
+ const groups = items.filter((item) => item.kind === "tool_group");
+ expect(groups).toHaveLength(2);
+ expect(groups[0]!.tools[0]).toMatchObject({
+ toolCallId: "item_1",
+ done: true,
+ ok: false,
+ });
+ expect(groups[1]!.tools[0]).toMatchObject({
+ toolCallId: "item_1",
+ done: false,
+ });
+ });
+
+ it("closes an unmatched tool when a fresh lifecycle starts without a user row", () => {
+ const items = collapseEvents([
+ call("c1", "item_1"),
+ lifecycle("start-2", "start"),
+ call("c2", "item_1"),
+ ]);
+ const group = items[0] as Extract<
+ ReturnType<typeof collapseEvents>[number],
+ { kind: "tool_group" }
+ >;
+ expect(group.tools).toHaveLength(2);
+ expect(group.tools[0]).toMatchObject({
+ toolCallId: "item_1",
+ done: true,
+ ok: false,
+ });
+ expect(group.tools[1]).toMatchObject({
+ toolCallId: "item_1",
+ done: false,
+ });
+ });
});
describe("upsertToolEntry", () => {
diff --git a/notfair/src/components/chat/transcript-model.ts b/notfair/src/components/chat/transcript-model.ts
index a783bc18..bdba06e8 100644
--- a/notfair/src/components/chat/transcript-model.ts
+++ b/notfair/src/components/chat/transcript-model.ts
@@ -61,6 +61,15 @@ export function collapseEvents(events: TranscriptEvent[]): RenderedItem[] {
| { tag: "msg"; item: RenderedItem };
const steps: Step[] = [];
const callIndex = new Map<string, number>();
+ const closeOpenTools = (ok: boolean) => {
+ for (const idx of callIndex.values()) {
+ const step = steps[idx];
+ if (step?.tag === "tool" && !step.entry.done) {
+ step.entry = { ...step.entry, done: true, ok };
+ }
+ }
+ callIndex.clear();
+ };
for (const e of events) {
if (e.kind === "tool_call") {
callIndex.set(e.tool_call_id, steps.length);
@@ -87,6 +96,7 @@ export function collapseEvents(events: TranscriptEvent[]): RenderedItem[] {
ok: e.ok,
done: true,
};
+ callIndex.delete(e.tool_call_id);
continue;
}
steps.push({
@@ -103,6 +113,11 @@ export function collapseEvents(events: TranscriptEvent[]): RenderedItem[] {
continue;
}
if (e.kind === "user_message") {
+ // A new user turn is a hard boundary. If the previous harness turn
+ // vanished before emitting a tool result, keep that historical row
+ // static and failed instead of animating it forever or pairing a
+ // later turn's reused tool id with it.
+ closeOpenTools(false);
steps.push({
tag: "msg",
item: { kind: "user_message", key: e.id, body: e.body, system: e.system },
@@ -116,6 +131,17 @@ export function collapseEvents(events: TranscriptEvent[]): RenderedItem[] {
});
continue;
}
+ if (e.kind === "lifecycle") {
+ if (e.phase === "done") {
+ // A final/error event is surfaced as lifecycle "done" by
+ // transcript-tail. The turn is terminal even if a harness omitted
+ // one tool-result event, so no tool from it may remain "live".
+ closeOpenTools(e.ok !== false);
+ } else if (e.phase === "start") {
+ closeOpenTools(false);
+ }
+ continue;
+ }
if (e.kind === "unknown") {
steps.push({
tag: "msg",
diff --git a/notfair/src/server/db/db.test.ts b/notfair/src/server/db/db.test.ts
index 1806a49d..2d80283a 100644
--- a/notfair/src/server/db/db.test.ts
+++ b/notfair/src/server/db/db.test.ts
@@ -31,6 +31,10 @@ describe("getDb", () => {
expect(names).toContain("projects");
expect(names).toContain("sessions");
expect(names).toContain("transcript_events");
+ const tickColumns = db.pragma("table_info(goal_ticks)") as Array<{
+ name: string;
+ }>;
+ expect(tickColumns.map((column) => column.name)).toContain("owner_pid");
});
it("configures WAL journaling and foreign keys", () => {
diff --git a/notfair/src/server/db/db.ts b/notfair/src/server/db/db.ts
index b99c8e55..9646e3a5 100644
--- a/notfair/src/server/db/db.ts
+++ b/notfair/src/server/db/db.ts
@@ -22,6 +22,7 @@ export function getDb(): Database.Database {
db.pragma("busy_timeout = 5000");
db.exec(SCHEMA);
+ ensureGoalTickOwnerColumn(db);
cached = db;
return db;
@@ -30,3 +31,20 @@ export function getDb(): Database.Database {
export function getDbPath(): string {
return DB_PATH;
}
+
+function ensureGoalTickOwnerColumn(db: Database.Database): void {
+ const hasOwnerPid = () =>
+ (
+ db.pragma("table_info(goal_ticks)") as Array<{ name: string }>
+ ).some((column) => column.name === "owner_pid");
+ if (hasOwnerPid()) return;
+
+ try {
+ db.exec("ALTER TABLE goal_ticks ADD COLUMN owner_pid INTEGER");
+ } catch (error) {
+ // Two local processes can open the same data directory concurrently.
+ // Treat a raced migration as success only when the other process
+ // actually installed the column.
+ if (!hasOwnerPid()) throw error;
+ }
+}
diff --git a/notfair/src/server/db/goals.ts b/notfair/src/server/db/goals.ts
index a65af111..c6b623e7 100644
--- a/notfair/src/server/db/goals.ts
+++ b/notfair/src/server/db/goals.ts
@@ -121,6 +121,7 @@ export type GoalTick = {
goal_id: string;
tick_number: number;
trigger_kind: GoalTickTrigger;
+ owner_pid: number | null;
session_id: string | null;
metric_value: number | null;
metric_error: string | null;
@@ -831,9 +832,9 @@ export function createGoalTick(input: {
const ts = now();
db.prepare(
`INSERT INTO goal_ticks
- (id, goal_id, tick_number, trigger_kind, status, started_at)
- VALUES (?, ?, ?, ?, 'running', ?)`,
- ).run(id, input.goal_id, input.tick_number, input.trigger_kind, ts);
+ (id, goal_id, tick_number, trigger_kind, owner_pid, status, started_at)
+ VALUES (?, ?, ?, ?, ?, 'running', ?)`,
+ ).run(id, input.goal_id, input.tick_number, input.trigger_kind, process.pid, ts);
return getGoalTick(id)!;
}
@@ -869,6 +870,22 @@ export function finishGoalTick(
.run(status, summary ?? null, now(), id);
}
+/** Terminal transition used by crash recovery; never overwrite live progress. */
+export function finishRunningGoalTick(
+ id: string,
+ status: Extract<GoalTickStatus, "done" | "failed">,
+ summary?: string | null,
+): boolean {
+ const result = getDb()
+ .prepare(
+ `UPDATE goal_ticks
+ SET status = ?, summary = ?, finished_at = ?
+ WHERE id = ? AND status = 'running'`,
+ )
+ .run(status, summary ?? null, now(), id);
+ return result.changes === 1;
+}
+
export function listGoalTicks(
goal_id: string,
limit = 30,
diff --git a/notfair/src/server/db/schema.ts b/notfair/src/server/db/schema.ts
index 8cc0e582..f5ae6acb 100644
--- a/notfair/src/server/db/schema.ts
+++ b/notfair/src/server/db/schema.ts
@@ -299,6 +299,7 @@ CREATE TABLE IF NOT EXISTS goal_ticks (
tick_number INTEGER NOT NULL,
trigger_kind TEXT NOT NULL DEFAULT 'heartbeat'
CHECK (trigger_kind IN ('heartbeat','manual','approval','intake')),
+ owner_pid INTEGER,
session_id TEXT,
metric_value REAL,
metric_error TEXT,
diff --git a/notfair/src/server/goals/checks.test.ts b/notfair/src/server/goals/checks.test.ts
index 8f9ae91d..638b097b 100644
--- a/notfair/src/server/goals/checks.test.ts
+++ b/notfair/src/server/goals/checks.test.ts
@@ -80,6 +80,14 @@ function seedTickToolCalls(tick_number: number, toolNames: string[]): void {
}
describe("listCheckRows paging", () => {
+ it("records the process that owns a running tick", () => {
+ seedTicks(1);
+ const row = getDb()
+ .prepare("SELECT owner_pid FROM goal_ticks WHERE goal_id = ?")
+ .get(goalId) as { owner_pid: number };
+ expect(row.owner_pid).toBe(process.pid);
+ });
+
it("returns the newest page with hasMore when older checks exist", () => {
seedTicks(CHECKS_PAGE_SIZE + 2);
const { rows, hasMore } = listCheckRows(goalId);
diff --git a/notfair/src/server/goals/recovery.test.ts b/notfair/src/server/goals/recovery.test.ts
new file mode 100644
index 00000000..0ec29a27
--- /dev/null
+++ b/notfair/src/server/goals/recovery.test.ts
@@ -0,0 +1,260 @@
+import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.hoisted(() => {
+ const { mkdtempSync } = require("node:fs") as typeof import("node:fs");
+ const { tmpdir } = require("node:os") as typeof import("node:os");
+ const { join } = require("node:path") as typeof import("node:path");
+ process.env.NOTFAIR_DATA_DIR = mkdtempSync(join(tmpdir(), "notfair-recovery-"));
+});
+
+import { getDb } from "@/server/db/db";
+import { getGoalTick } from "@/server/db/goals";
+import {
+ appendTranscriptEvent,
+ getOrCreateSession,
+ listTranscriptEvents,
+} from "@/server/sessions";
+import {
+ INTERRUPTED_TICK_SUMMARY,
+ recoverInterruptedGoalTicks,
+} from "./recovery";
+
+const NOW = "2026-07-22T20:00:00.000Z";
+
+function insertRunningTick(input: {
+ id: string;
+ tickNumber: number;
+ sessionId?: string;
+ ownerPid?: number;
+}): void {
+ getDb()
+ .prepare(
+ `INSERT INTO goal_ticks
+ (id, goal_id, tick_number, trigger_kind, owner_pid, session_id, status, started_at)
+ VALUES (?, 'goal-1', ?, 'heartbeat', ?, ?, 'running', ?)`,
+ )
+ .run(
+ input.id,
+ input.tickNumber,
+ input.ownerPid ?? null,
+ input.sessionId ?? null,
+ NOW,
+ );
+}
+
+beforeAll(() => {
+ const db = getDb();
+ db.prepare(
+ "INSERT INTO projects (id, slug, display_name, created_at, harness_adapter) VALUES ('p1', 'proj', 'Proj', ?, 'codex-local')",
+ ).run(NOW);
+ db.prepare(
+ `INSERT INTO goals
+ (id, project_slug, agent_id, statement, status, cadence_cron, created_at, updated_at)
+ VALUES ('goal-1', 'proj', 'agent-1', 'Keep errors low', 'active', '0 * * * *', ?, ?)`,
+ ).run(NOW, NOW);
+});
+
+beforeEach(() => {
+ const db = getDb();
+ db.prepare("DELETE FROM goal_ticks").run();
+ db.prepare("DELETE FROM transcript_events").run();
+ db.prepare("DELETE FROM sessions").run();
+});
+
+describe("recoverInterruptedGoalTicks", () => {
+ it("fails an orphaned running tick and closes its partial transcript", () => {
+ const session = getOrCreateSession({
+ project_slug: "proj",
+ agent_id: "agent-1",
+ label: "tick-1",
+ harness_adapter: "codex-local",
+ });
+ appendTranscriptEvent(session.id, "user", { text: "check" });
+ appendTranscriptEvent(session.id, "lifecycle", {
+ kind: "lifecycle",
+ phase: "start",
+ });
+ insertRunningTick({
+ id: "tick-1",
+ tickNumber: 1,
+ sessionId: session.id,
+ ownerPid: 4040,
+ });
+
+ expect(recoverInterruptedGoalTicks(() => false)).toEqual({
+ recovered: 1,
+ completed: 0,
+ failed: 1,
+ active: 0,
+ });
+
+ expect(getGoalTick("tick-1")).toMatchObject({
+ status: "failed",
+ summary: INTERRUPTED_TICK_SUMMARY,
+ });
+ const events = listTranscriptEvents(session.id);
+ expect(events.at(-1)).toMatchObject({ kind: "error" });
+ expect(JSON.parse(events.at(-1)!.payload_json)).toMatchObject({
+ message: INTERRUPTED_TICK_SUMMARY,
+ });
+
+ expect(recoverInterruptedGoalTicks(() => false)).toEqual({
+ recovered: 0,
+ completed: 0,
+ failed: 0,
+ active: 0,
+ });
+ expect(listTranscriptEvents(session.id)).toHaveLength(3);
+ });
+
+ it("recovers a final event as done instead of replaying the tick", () => {
+ const session = getOrCreateSession({
+ project_slug: "proj",
+ agent_id: "agent-1",
+ label: "tick-2",
+ harness_adapter: "codex-local",
+ });
+ appendTranscriptEvent(session.id, "final", {
+ kind: "final",
+ text: "The check completed before shutdown.",
+ });
+ insertRunningTick({ id: "tick-2", tickNumber: 2, sessionId: session.id });
+
+ expect(recoverInterruptedGoalTicks()).toEqual({
+ recovered: 1,
+ completed: 1,
+ failed: 0,
+ active: 0,
+ });
+ expect(getGoalTick("tick-2")).toMatchObject({
+ status: "done",
+ summary: "The check completed before shutdown.",
+ });
+ expect(listTranscriptEvents(session.id)).toHaveLength(1);
+ });
+
+ it("preserves a terminal error as the failed tick summary without duplicating it", () => {
+ const session = getOrCreateSession({
+ project_slug: "proj",
+ agent_id: "agent-1",
+ label: "tick-3",
+ harness_adapter: "codex-local",
+ });
+ appendTranscriptEvent(session.id, "error", {
+ kind: "error",
+ message: "The harness stopped unexpectedly.",
+ transient: false,
+ });
+ insertRunningTick({ id: "tick-3", tickNumber: 3, sessionId: session.id });
+
+ expect(recoverInterruptedGoalTicks()).toEqual({
+ recovered: 1,
+ completed: 0,
+ failed: 1,
+ active: 0,
+ });
+ expect(getGoalTick("tick-3")).toMatchObject({
+ status: "failed",
+ summary: "The harness stopped unexpectedly.",
+ });
+ expect(listTranscriptEvents(session.id)).toHaveLength(1);
+ });
+
+ it("recovers a stored lifecycle completion as done", () => {
+ const session = getOrCreateSession({
+ project_slug: "proj",
+ agent_id: "agent-1",
+ label: "tick-4",
+ harness_adapter: "codex-local",
+ });
+ appendTranscriptEvent(session.id, "lifecycle", {
+ kind: "lifecycle",
+ phase: "done",
+ });
+ insertRunningTick({ id: "tick-4", tickNumber: 4, sessionId: session.id });
+
+ expect(recoverInterruptedGoalTicks()).toEqual({
+ recovered: 1,
+ completed: 1,
+ failed: 0,
+ active: 0,
+ });
+ expect(getGoalTick("tick-4")).toMatchObject({
+ status: "done",
+ summary: "Check completed before NotFair restarted.",
+ });
+ expect(listTranscriptEvents(session.id)).toHaveLength(1);
+ });
+
+ it("fails a running row without a session", () => {
+ insertRunningTick({ id: "tick-5", tickNumber: 5, ownerPid: 5050 });
+
+ expect(recoverInterruptedGoalTicks(() => false)).toEqual({
+ recovered: 1,
+ completed: 0,
+ failed: 1,
+ active: 0,
+ });
+ expect(getGoalTick("tick-5")).toMatchObject({
+ status: "failed",
+ summary: INTERRUPTED_TICK_SUMMARY,
+ });
+ });
+
+ it("leaves a tick owned by another live process running", () => {
+ insertRunningTick({ id: "tick-6", tickNumber: 6, ownerPid: 4242 });
+
+ expect(recoverInterruptedGoalTicks((pid) => pid === 4242)).toEqual({
+ recovered: 0,
+ completed: 0,
+ failed: 0,
+ active: 1,
+ });
+ expect(getGoalTick("tick-6")).toMatchObject({
+ status: "running",
+ owner_pid: 4242,
+ });
+ });
+
+ it("leaves a legacy ownerless partial tick untouched", () => {
+ insertRunningTick({ id: "tick-7", tickNumber: 7 });
+
+ expect(recoverInterruptedGoalTicks()).toEqual({
+ recovered: 0,
+ completed: 0,
+ failed: 0,
+ active: 1,
+ });
+ expect(getGoalTick("tick-7")).toMatchObject({
+ status: "running",
+ owner_pid: null,
+ });
+ });
+
+ it("does not treat a transient error as proof a legacy turn ended", () => {
+ const session = getOrCreateSession({
+ project_slug: "proj",
+ agent_id: "agent-1",
+ label: "tick-8",
+ harness_adapter: "codex-local",
+ });
+ appendTranscriptEvent(session.id, "error", {
+ kind: "error",
+ message: "Reconnecting...",
+ transient: true,
+ });
+ insertRunningTick({
+ id: "tick-8",
+ tickNumber: 8,
+ sessionId: session.id,
+ });
+
+ expect(recoverInterruptedGoalTicks()).toEqual({
+ recovered: 0,
+ completed: 0,
+ failed: 0,
+ active: 1,
+ });
+ expect(getGoalTick("tick-8")?.status).toBe("running");
+ });
+});
diff --git a/notfair/src/server/goals/recovery.ts b/notfair/src/server/goals/recovery.ts
new file mode 100644
index 00000000..6013f85e
--- /dev/null
+++ b/notfair/src/server/goals/recovery.ts
@@ -0,0 +1,156 @@
+import { getDb } from "@/server/db/db";
+import { finishRunningGoalTick, type GoalTick } from "@/server/db/goals";
+import { appendTranscriptEvent } from "@/server/sessions";
+
+export const INTERRUPTED_TICK_SUMMARY =
+ "Check interrupted because NotFair stopped or restarted before it finished.";
+
+const RECOVERED_SUMMARY_MAX = 1_000;
+
+type TerminalEvent = {
+ kind: "final" | "error" | "lifecycle";
+ payload_json: string;
+};
+
+export type TickRecoveryResult = {
+ recovered: number;
+ completed: number;
+ failed: number;
+ active: number;
+};
+
+function isProcessAlive(pid: number): boolean {
+ if (pid === process.pid) return true;
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code === "EPERM";
+ }
+}
+
+function payloadText(
+ event: TerminalEvent,
+ key: "text" | "message",
+): string | null {
+ try {
+ const payload = JSON.parse(event.payload_json) as Record<string, unknown>;
+ const value = payload[key];
+ return typeof value === "string" && value.trim()
+ ? value.trim().slice(0, RECOVERED_SUMMARY_MAX)
+ : null;
+ } catch {
+ return null;
+ }
+}
+
+function terminalEvent(sessionId: string): TerminalEvent | null {
+ const row = getDb()
+ .prepare(
+ `SELECT kind, payload_json
+ FROM transcript_events
+ WHERE session_id = ?
+ AND (
+ kind IN ('final', 'error')
+ OR (
+ kind = 'lifecycle'
+ AND json_extract(payload_json, '$.phase') = 'done'
+ )
+ )
+ AND (
+ kind <> 'error'
+ OR COALESCE(json_extract(payload_json, '$.transient'), 0) = 0
+ )
+ ORDER BY seq DESC
+ LIMIT 1`,
+ )
+ .get(sessionId) as TerminalEvent | undefined;
+ return row ?? null;
+}
+
+function sessionExists(sessionId: string): boolean {
+ return Boolean(
+ getDb().prepare("SELECT 1 FROM sessions WHERE id = ?").get(sessionId),
+ );
+}
+
+/**
+ * Reconcile check rows left "running" by the previous server process.
+ *
+ * A local harness subprocess belongs to the process that spawned it. At
+ * boot there cannot be a live in-memory owner for a persisted running row,
+ * and replaying it could duplicate external mutations. We therefore:
+ *
+ * - preserve a terminal final as a completed check;
+ * - preserve a terminal error as a failed check;
+ * - close every other partial transcript with an interruption error and
+ * mark its check failed, allowing the normal next heartbeat to continue.
+ *
+ * Called exactly once by the scheduler before its first sweep.
+ */
+export function recoverInterruptedGoalTicks(
+ processAlive: (pid: number) => boolean = isProcessAlive,
+): TickRecoveryResult {
+ const rows = getDb()
+ .prepare("SELECT * FROM goal_ticks WHERE status = 'running' ORDER BY started_at ASC")
+ .all() as GoalTick[];
+ const result: TickRecoveryResult = {
+ recovered: 0,
+ completed: 0,
+ failed: 0,
+ active: 0,
+ };
+
+ for (const tick of rows) {
+ const terminal = tick.session_id ? terminalEvent(tick.session_id) : null;
+ // A second server can share the data directory (for example, an
+ // orphaned process whose server.json entry was replaced). Never
+ // recover work while its owning process is still alive. Legacy rows
+ // without an owner are only safe to reconcile when their transcript
+ // already proves the turn ended.
+ if (tick.owner_pid !== null && processAlive(tick.owner_pid)) {
+ result.active += 1;
+ continue;
+ }
+ if (tick.owner_pid === null && !terminal) {
+ result.active += 1;
+ continue;
+ }
+
+ if (terminal?.kind === "final" || terminal?.kind === "lifecycle") {
+ const summary =
+ terminal.kind === "final"
+ ? payloadText(terminal, "text")
+ : null;
+ if (!finishRunningGoalTick(
+ tick.id,
+ "done",
+ summary ?? tick.summary ?? "Check completed before NotFair restarted.",
+ )) {
+ continue;
+ }
+ result.recovered += 1;
+ result.completed += 1;
+ continue;
+ }
+
+ const summary =
+ terminal?.kind === "error"
+ ? payloadText(terminal, "message") ?? INTERRUPTED_TICK_SUMMARY
+ : INTERRUPTED_TICK_SUMMARY;
+ if (!finishRunningGoalTick(tick.id, "failed", summary)) {
+ continue;
+ }
+ if (!terminal && tick.session_id && sessionExists(tick.session_id)) {
+ appendTranscriptEvent(tick.session_id, "error", {
+ kind: "error",
+ message: INTERRUPTED_TICK_SUMMARY,
+ transient: true,
+ });
+ }
+ result.recovered += 1;
+ result.failed += 1;
+ }
+
+ return result;
+}
diff --git a/notfair/src/server/scheduler/tick.test.ts b/notfair/src/server/scheduler/tick.test.ts
index b5e88884..9afe5619 100644
--- a/notfair/src/server/scheduler/tick.test.ts
+++ b/notfair/src/server/scheduler/tick.test.ts
@@ -1,17 +1,31 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
-const mocks = vi.hoisted(() => ({ runTicks: vi.fn(), syncPrs: vi.fn() }));
+const mocks = vi.hoisted(() => ({
+ recoverTicks: vi.fn(),
+ runTicks: vi.fn(),
+ syncPrs: vi.fn(),
+}));
vi.mock("@/server/goals/tick", () => ({ runDueGoalTicks: mocks.runTicks }));
vi.mock("@/server/goals/pr-sync", () => ({ syncDueGoalPrs: mocks.syncPrs }));
+vi.mock("@/server/goals/recovery", () => ({
+ recoverInterruptedGoalTicks: mocks.recoverTicks,
+}));
import { ensureSchedulerRunning, stopScheduler } from "./tick";
beforeEach(() => {
vi.useFakeTimers();
vi.clearAllMocks();
+ mocks.recoverTicks.mockReturnValue({
+ recovered: 0,
+ completed: 0,
+ failed: 0,
+ active: 0,
+ });
mocks.runTicks.mockResolvedValue(undefined);
mocks.syncPrs.mockResolvedValue(undefined);
vi.spyOn(console, "error").mockImplementation(() => {});
+ vi.spyOn(console, "warn").mockImplementation(() => {});
stopScheduler();
});
@@ -24,6 +38,7 @@ afterEach(() => {
it("runs immediately, repeats every 30s, and starts only once", async () => {
ensureSchedulerRunning();
ensureSchedulerRunning();
+ expect(mocks.recoverTicks).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(0);
expect(mocks.runTicks).toHaveBeenCalledTimes(1);
expect(mocks.syncPrs).toHaveBeenCalledTimes(1);
@@ -34,6 +49,32 @@ it("runs immediately, repeats every 30s, and starts only once", async () => {
expect(mocks.runTicks).toHaveBeenCalledTimes(2);
});
+it("logs recovery failures but still starts the scheduler", async () => {
+ mocks.recoverTicks.mockImplementationOnce(() => {
+ throw new Error("recovery failed");
+ });
+ ensureSchedulerRunning();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(console.error).toHaveBeenCalledWith(
+ "[scheduler] interrupted tick recovery failed:",
+ expect.any(Error),
+ );
+ expect(mocks.runTicks).toHaveBeenCalledTimes(1);
+});
+
+it("reports interrupted ticks recovered during startup", () => {
+ mocks.recoverTicks.mockReturnValueOnce({
+ recovered: 3,
+ completed: 1,
+ failed: 2,
+ active: 0,
+ });
+ ensureSchedulerRunning();
+ expect(console.warn).toHaveBeenCalledWith(
+ "[scheduler] recovered 3 interrupted goal tick(s): 1 completed, 2 failed",
+ );
+});
+
it("logs independent tick and PR sweep failures", async () => {
mocks.runTicks.mockRejectedValue(new Error("tick failed"));
mocks.syncPrs.mockRejectedValue(new Error("pr failed"));
diff --git a/notfair/src/server/scheduler/tick.ts b/notfair/src/server/scheduler/tick.ts
index 05e527d3..48ae52bd 100644
--- a/notfair/src/server/scheduler/tick.ts
+++ b/notfair/src/server/scheduler/tick.ts
@@ -1,5 +1,6 @@
import { runDueGoalTicks } from "@/server/goals/tick";
import { syncDueGoalPrs } from "@/server/goals/pr-sync";
+import { recoverInterruptedGoalTicks } from "@/server/goals/recovery";
/**
* The heartbeat loop. Started once on boot (src/instrumentation.ts); a
@@ -28,6 +29,22 @@ function sweep(): void {
export function ensureSchedulerRunning(): void {
if (started) return;
started = true;
+ try {
+ const recovered = recoverInterruptedGoalTicks();
+ if (recovered.recovered > 0) {
+ console.warn(
+ `[scheduler] recovered ${recovered.recovered} interrupted goal tick(s): ` +
+ `${recovered.completed} completed, ${recovered.failed} failed` +
+ (recovered.active > 0
+ ? `; left ${recovered.active} owned by a live process`
+ : ""),
+ );
+ }
+ } catch (err) {
+ // Recovery must never prevent future heartbeats from starting. Any
+ // untouched rows remain "running" and will be retried at next boot.
+ console.error("[scheduler] interrupted tick recovery failed:", err);
+ }
timer = setInterval(sweep, TICK_INTERVAL_MS);
// First sweep on the next event-loop turn so callers return immediately.
setImmediate(sweep);
diff --git a/notfair/src/server/sessions/transcript-tail.test.ts b/notfair/src/server/sessions/transcript-tail.test.ts
index 560fa785..b96142c3 100644
--- a/notfair/src/server/sessions/transcript-tail.test.ts
+++ b/notfair/src/server/sessions/transcript-tail.test.ts
@@ -57,9 +57,19 @@ it("normalizes every transcript event kind and advances the cursor", () => {
expect.objectContaining({ kind: "tool_call", id: "tc2", name: "tool", label: null }),
expect.objectContaining({ kind: "tool_result", id: "tr", ok: true, summary: null }),
expect.objectContaining({ kind: "lifecycle", id: "life", phase: "unknown" }),
- expect.objectContaining({ kind: "lifecycle", id: "fin", phase: "done" }),
+ expect.objectContaining({
+ kind: "lifecycle",
+ id: "fin",
+ phase: "done",
+ ok: true,
+ }),
expect.objectContaining({ kind: "assistant_text", id: "err", body: "⚠ boom" }),
- expect.objectContaining({ kind: "lifecycle", id: "err:done", phase: "done" }),
+ expect.objectContaining({
+ kind: "lifecycle",
+ id: "err:done",
+ phase: "done",
+ ok: false,
+ }),
expect.objectContaining({ kind: "unknown", id: "x", raw_type: "other" }),
]));
expect(result.events).toHaveLength(13);
diff --git a/notfair/src/server/sessions/transcript-tail.ts b/notfair/src/server/sessions/transcript-tail.ts
index 9b9e1879..6b37def0 100644
--- a/notfair/src/server/sessions/transcript-tail.ts
+++ b/notfair/src/server/sessions/transcript-tail.ts
@@ -26,7 +26,7 @@ export type TranscriptEvent =
summary: string | null;
ok: boolean;
}
- | { kind: "lifecycle"; id: string; ts: number; phase: string }
+ | { kind: "lifecycle"; id: string; ts: number; phase: string; ok?: boolean }
| { kind: "unknown"; id: string; ts: number; raw_type: string };
/**
@@ -118,11 +118,12 @@ export function readTranscriptTail(
}
} else if (row.kind === "lifecycle") {
const phase = typeof payload.phase === "string" ? payload.phase : "unknown";
- events.push({ kind: "lifecycle", id, ts, phase });
+ const ok = typeof payload.ok === "boolean" ? payload.ok : undefined;
+ events.push({ kind: "lifecycle", id, ts, phase, ok });
} else if (row.kind === "final") {
// Already covered by the accumulated delta stream; surface as a no-op
// lifecycle so clients can show "done" if they want.
- events.push({ kind: "lifecycle", id, ts, phase: "done" });
+ events.push({ kind: "lifecycle", id, ts, phase: "done", ok: true });
} else if (row.kind === "error") {
const text = typeof payload.message === "string" ? payload.message : "";
events.push({ kind: "assistant_text", id, ts, body: `⚠ ${text}` });
@@ -130,7 +131,13 @@ export function readTranscriptTail(
// its final, so no lifecycle "done" will ever come. Emit a synthetic
// boundary so the client's open-turn detection closes immediately
// instead of showing "still working" until the staleness cutoff.
- events.push({ kind: "lifecycle", id: `${id}:done`, ts, phase: "done" });
+ events.push({
+ kind: "lifecycle",
+ id: `${id}:done`,
+ ts,
+ phase: "done",
+ ok: false,
+ });
} else {
events.push({ kind: "unknown", id, ts, raw_type: row.kind });
}