Comp AI CRM is an open source, CRM designed for AI agents. Agentic-first CRM.
TypeScript
feat: enrichment queue widget (CMP-92) (#159)
grim5e11482bee23
diff --git a/.gitignore b/.gitignoreindex 06b9c681..85385299 100644--- a/.gitignore+++ b/.gitignore@@ -46,3 +46,7 @@ yarn-error.log*# eve build output.eve.output+/dogfood-output++# Agent scratch files+.scratch/diff --git a/apps/agent/agent/channels/crm.ts b/apps/agent/agent/channels/crm.tsindex fabb45c4..aff3af45 100644--- a/apps/agent/agent/channels/crm.ts+++ b/apps/agent/agent/channels/crm.ts@@ -31,6 +31,7 @@ import { settle } from "../lib/enrichment";import { finishRun, runResultOf } from "../lib/run-runtime";import { attribute } from "../lib/session-purpose";import { createSlackChannel } from "../lib/slack-membership";+import { reconcileStaleTasks } from "../lib/stale-tasks";import { completeTask, taskSubject } from "../lib/tasks";const TASK_MARKER = "task:";@@ -84,6 +85,21 @@ export function taskFromToken(token: string | undefined): string | null {return id.length > 0 ? id : null;}+export async function closeTask(+ token: string | undefined,+ outcome: string,+ status: EnrichmentStatus = EnrichmentStatus.COMPLETE,+): Promise<boolean> {+ const taskId = taskFromToken(token);+ if (!taskId) return false;++ const subject =+ (await completeTask(taskId, outcome)) ?? (await taskSubject(taskId));+ if (subject) await settle(subject, status);++ return true;+}+export default defineChannel({routes: [GET("/internal/crm/dispatch-health", async (request) => {@@ -122,6 +138,7 @@ export default defineChannel({waitUntil((async () => {+ await reconcileStaleTasks();await drainAll((task) =>send(brief(task), {auth: taskAuth(task),@@ -248,12 +265,7 @@ export default defineChannel({},async "session.waiting"(_data, channel) {- const taskId = taskFromToken(channel.continuationToken);- if (taskId) {- const subject = await completeTask(taskId, "ran");- if (subject) await settle(subject, EnrichmentStatus.COMPLETE);- return;- }+ if (await closeTask(channel.continuationToken, "ran")) return;const conversationId = builderIdFromToken(channel.continuationToken);if (!conversationId) return;@@ -295,6 +307,8 @@ export default defineChannel({},async "session.completed"(_data, channel) {+ if (await closeTask(channel.continuationToken, "ran")) return;+const conversationId = builderIdFromToken(channel.continuationToken);if (conversationId) {const { db } = await import("@crm/db");@@ -330,6 +344,16 @@ export default defineChannel({},async "turn.cancelled"(_data, channel) {+ if (+ await closeTask(+ channel.continuationToken,+ "stopped",+ EnrichmentStatus.SKIPPED,+ )+ ) {+ return;+ }+const conversationId = builderIdFromToken(channel.continuationToken);if (conversationId) {const { db } = await import("@crm/db");diff --git a/apps/agent/agent/lib/dispatch-config.ts b/apps/agent/agent/lib/dispatch-config.tsindex 916fe465..4382b4a8 100644--- a/apps/agent/agent/lib/dispatch-config.ts+++ b/apps/agent/agent/lib/dispatch-config.ts@@ -32,6 +32,11 @@ export const DISPATCH = {leaseMs: 10 * MINUTE_MS,},+ reconcile: {+ scan: 200,+ retire: 100,+ },+sweep: {timeoutMs: 4 * MINUTE_MS,staleQueueMs: 5 * MINUTE_MS,diff --git a/apps/agent/agent/lib/dispatch.ts b/apps/agent/agent/lib/dispatch.tsindex eaf70d22..029dbd15 100644--- a/apps/agent/agent/lib/dispatch.ts+++ b/apps/agent/agent/lib/dispatch.ts@@ -9,14 +9,13 @@ import { collapsing, runLimited } from "./pool";import { runPortrait } from "./portrait";import { runSlackChannelJoin } from "./slack-join-task";import { runSlackPeopleMatch } from "./slack-people";+import { staleTaskSweep } from "./stale-tasks";import {claimDue,completeTask,DIRECT_KINDS,type LeasedTask,noteSession,- retireExhausted,- type TaskSubject,} from "./tasks";export const VISIBLE_BATCH = DISPATCH.visible.batch;@@ -26,24 +25,6 @@ export const VISIBLE_LEASE_MS = DISPATCH.visible.leaseMs;export const RESEARCH_BATCH = DISPATCH.research.batch;export const RESEARCH_LEASE_MS = DISPATCH.research.leaseMs;-export async function retireAbandoned(): Promise<void> {- let abandoned: TaskSubject[] = [];-- try {- abandoned = await retireExhausted();- } catch {- return;- }-- for (const task of abandoned) {- await settle(- task,- EnrichmentStatus.FAILED,- "Research was attempted several times and never completed.",- ).catch(() => {});- }-}-export async function runVisibleLane(signal?: AbortSignal): Promise<number> {let handled = 0;@@ -322,6 +303,7 @@ export function dispatchHealth() {pendingStarts,pendingItems,unlinkedSessions,+ staleTasks: staleTaskSweep(),lastError: lastSweepError,};}@@ -343,7 +325,6 @@ export const drainAll = collapsing(const signal = controller.signal;const sweep = (async () => {- await retireAbandoned();await Promise.all([runVisibleLane(signal),runResearchLane(start, signal),diff --git a/apps/agent/agent/lib/stale-tasks.ts b/apps/agent/agent/lib/stale-tasks.tsnew file mode 100644index 00000000..1d2ade28--- /dev/null+++ b/apps/agent/agent/lib/stale-tasks.ts@@ -0,0 +1,207 @@+import { db, EnrichmentStatus, type Prisma } from "@crm/db";+import { isEnrichmentKind, MAX_ATTEMPTS } from "@crm/db/agent-tasks";+import { DISPATCH } from "./dispatch-config";+import { settle } from "./enrichment";+import { retireExhausted, type TaskSubject } from "./tasks";++const SCAN = DISPATCH.reconcile.scan;++const LANDED_OUTCOME =+ "The record was already up to date when this was checked again.";++const RETIRED_ERROR =+ "Research was attempted several times and never completed.";++export type StaleTaskSweep = {+ scanned: number;+ closed: number;+ retired: number;+ released: number;+ waiting: number;+ unscanned: number;+ error: string | null;+};++type OpenTask = {+ id: string;+ kind: string;+ contactId: string | null;+ companyId: string | null;+ attempts: number;+ leasedUntil: Date | null;+ startedAt: Date | null;+};++let lastSweep: StaleTaskSweep | null = null;++export function staleTaskSweep(): StaleTaskSweep | null {+ return lastSweep;+}++export async function retireAbandoned(): Promise<TaskSubject[]> {+ let abandoned: TaskSubject[] = [];++ try {+ abandoned = await retireExhausted();+ } catch {+ return [];+ }++ for (const task of abandoned) {+ await settle(task, EnrichmentStatus.FAILED, RETIRED_ERROR).catch(() => {});+ }++ return abandoned;+}++export async function reconcileStaleTasks(): Promise<StaleTaskSweep> {+ const sweep: StaleTaskSweep = {+ scanned: 0,+ closed: 0,+ retired: 0,+ released: 0,+ waiting: 0,+ unscanned: 0,+ error: null,+ };++ try {+ await runSweep(sweep);+ } catch (cause) {+ sweep.error = cause instanceof Error ? cause.message : String(cause);+ console.error(`[agent] Stale task reconciliation failed: ${sweep.error}`);+ }++ lastSweep = sweep;+ return sweep;+}++async function runSweep(sweep: StaleTaskSweep): Promise<void> {+ const now = new Date();++ const where: Prisma.AgentTaskWhereInput = {+ finishedAt: null,+ dueAt: { lte: now },+ OR: [{ leasedUntil: null }, { leasedUntil: { lt: now } }],+ };++ const [open, tasks] = await Promise.all([+ db.agentTask.count({ where }),+ db.agentTask.findMany({+ where,+ orderBy: [{ dueAt: "asc" }],+ take: SCAN,+ select: {+ id: true,+ kind: true,+ contactId: true,+ companyId: true,+ attempts: true,+ leasedUntil: true,+ startedAt: true,+ },+ }),+ ]);++ sweep.scanned = tasks.length;+ sweep.unscanned = Math.max(0, open - tasks.length);++ const completed = await completedSubjects(tasks);++ const landed: string[] = [];+ const dead: string[] = [];++ for (const task of tasks) {+ if (finishedElsewhere(task, completed)) {+ landed.push(task.id);+ continue;+ }++ if (task.attempts >= MAX_ATTEMPTS) continue;++ if (task.leasedUntil !== null) dead.push(task.id);+ else sweep.waiting += 1;+ }++ if (landed.length > 0) {+ const { count } = await db.agentTask.updateMany({+ where: {+ id: { in: landed },+ finishedAt: null,+ OR: [{ leasedUntil: null }, { leasedUntil: { lt: now } }],+ },+ data: { finishedAt: now, outcome: LANDED_OUTCOME },+ });++ sweep.closed = count;+ }++ sweep.retired = (await retireAbandoned()).length;++ if (dead.length > 0) {+ const { count } = await db.agentTask.updateMany({+ where: { id: { in: dead }, finishedAt: null, leasedUntil: { lt: now } },+ data: { leasedUntil: null },+ });++ sweep.released = count;+ }+}++async function completedSubjects(+ tasks: readonly OpenTask[],+): Promise<Map<string, Date>> {+ const contactIds = unique(tasks.map((task) => task.contactId));+ const companyIds = unique(tasks.map((task) => task.companyId));++ const [contacts, companies] = await Promise.all([+ contactIds.length === 0+ ? []+ : db.contact.findMany({+ where: {+ id: { in: contactIds },+ enrichmentStatus: EnrichmentStatus.COMPLETE,+ enrichedAt: { not: null },+ },+ select: { id: true, enrichedAt: true },+ }),+ companyIds.length === 0+ ? []+ : db.company.findMany({+ where: {+ id: { in: companyIds },+ enrichmentStatus: EnrichmentStatus.COMPLETE,+ enrichedAt: { not: null },+ },+ select: { id: true, enrichedAt: true },+ }),+ ]);++ const completed = new Map<string, Date>();++ for (const row of [...contacts, ...companies]) {+ if (row.enrichedAt) completed.set(row.id, row.enrichedAt);+ }++ return completed;+}++function finishedElsewhere(+ task: OpenTask,+ completed: Map<string, Date>,+): boolean {+ if (!isEnrichmentKind(task.kind)) return false;+ if (task.attempts === 0 || task.startedAt === null) return false;++ const subjectId = task.contactId ?? task.companyId;+ if (!subjectId) return false;++ const enrichedAt = completed.get(subjectId);+ if (!enrichedAt) return false;++ return enrichedAt.getTime() >= task.startedAt.getTime();+}++function unique(ids: readonly (string | null)[]): string[] {+ return [...new Set(ids.filter((id): id is string => id !== null))];+}diff --git a/apps/agent/agent/lib/tasks.ts b/apps/agent/agent/lib/tasks.tsindex a29637f9..9d8a912c 100644--- a/apps/agent/agent/lib/tasks.ts+++ b/apps/agent/agent/lib/tasks.ts@@ -70,16 +70,25 @@ export async function claimDue();}-export async function retireExhausted(): Promise<TaskSubject[]> {+export async function retireExhausted(+ limit: number = DISPATCH.reconcile.retire,+): Promise<TaskSubject[]> {const now = new Date();return db.$queryRaw<TaskSubject[]>`UPDATE "agentTask" AS tSET "finishedAt" = ${now},"outcome" = ${RETIRED_OUTCOME}- WHERE t."finishedAt" IS NULL- AND t."attempts" >= ${MAX_ATTEMPTS}- AND (t."leasedUntil" IS NULL OR t."leasedUntil" < ${now})+ WHERE t.id IN (+ SELECT c.id+ FROM "agentTask" AS c+ WHERE c."finishedAt" IS NULL+ AND c."attempts" >= ${MAX_ATTEMPTS}+ AND (c."leasedUntil" IS NULL OR c."leasedUntil" < ${now})+ ORDER BY c."dueAt" ASC+ LIMIT ${limit}+ FOR UPDATE SKIP LOCKED+ )RETURNING t.id, t."contactId", t."companyId", t."dealId", t.kind;`;}diff --git a/apps/agent/agent/schedules/dispatch.ts b/apps/agent/agent/schedules/dispatch.tsindex c3b650af..1aa080fe 100644--- a/apps/agent/agent/schedules/dispatch.ts+++ b/apps/agent/agent/schedules/dispatch.ts@@ -7,6 +7,7 @@ import {queueDueAgentRuns,} from "../lib/custom-agent-dispatch";import { brief, drainAll, taskAuth } from "../lib/dispatch";+import { reconcileStaleTasks } from "../lib/stale-tasks";export default defineSchedule({cron: "* * * * *",@@ -16,6 +17,7 @@ export default defineSchedule({sweepBlankFacts(),(async () => {+ await reconcileStaleTasks();await drainAll((task) =>receive(crm, {message: brief(task),diff --git a/apps/agent/test/close-task.integration.spec.ts b/apps/agent/test/close-task.integration.spec.tsnew file mode 100644index 00000000..a92746df--- /dev/null+++ b/apps/agent/test/close-task.integration.spec.ts@@ -0,0 +1,82 @@+import { afterEach, beforeEach, describe, expect, it } from "bun:test";+import { db, EnrichmentStatus } from "@crm/db";+import { closeTask, taskToken } from "../agent/channels/crm";++const kind = "test-close-task";+const email = `close-task-${crypto.randomUUID()}@example.test`;++async function clear() {+ await db.agentTask.deleteMany({ where: { kind } });+ await db.contact.deleteMany({ where: { email } });+}++beforeEach(clear);+afterEach(clear);++async function running() {+ const contact = await db.contact.create({+ data: {+ firstName: "Close",+ email,+ enrichmentStatus: EnrichmentStatus.RUNNING,+ },+ select: { id: true },+ });++ const task = await db.agentTask.create({+ data: {+ kind,+ reason: "test",+ dueAt: new Date(Date.now() - 1000),+ budget: 4,+ contactId: contact.id,+ startedAt: new Date(),+ attempts: 1,+ },+ select: { id: true },+ });++ return { contactId: contact.id, taskId: task.id };+}++describe("closeTask", () => {+ it("closes the row and marks the record complete", async () => {+ const { contactId, taskId } = await running();++ expect(await closeTask(taskToken(taskId), "ran")).toBe(true);++ const task = await db.agentTask.findUnique({ where: { id: taskId } });+ expect(task?.finishedAt).not.toBeNull();+ expect(task?.outcome).toBe("ran");++ const contact = await db.contact.findUnique({ where: { id: contactId } });+ expect(contact?.enrichmentStatus).toBe(EnrichmentStatus.COMPLETE);+ expect(contact?.enrichedAt).not.toBeNull();+ });++ it("marks the record skipped when the turn is stopped", async () => {+ const { contactId, taskId } = await running();++ expect(+ await closeTask(taskToken(taskId), "stopped", EnrichmentStatus.SKIPPED),+ ).toBe(true);++ const contact = await db.contact.findUnique({ where: { id: contactId } });+ expect(contact?.enrichmentStatus).toBe(EnrichmentStatus.SKIPPED);+ });++ it("claims a task token it has already closed", async () => {+ const { taskId } = await running();++ await closeTask(taskToken(taskId), "ran");+ expect(await closeTask(taskToken(taskId), "ran again")).toBe(true);++ const task = await db.agentTask.findUnique({ where: { id: taskId } });+ expect(task?.outcome).toBe("ran");+ });++ it("leaves a token that names no task alone", async () => {+ expect(await closeTask("crm:adhoc:1234", "ran")).toBe(false);+ expect(await closeTask(undefined, "ran")).toBe(false);+ });+});diff --git a/apps/agent/test/e2e/retry.e2e.ts b/apps/agent/test/e2e/retry.e2e.tsindex a0bbbb7e..652a6d71 100644--- a/apps/agent/test/e2e/retry.e2e.ts+++ b/apps/agent/test/e2e/retry.e2e.ts@@ -1,6 +1,6 @@import { db } from "@crm/db";import { MAX_ATTEMPTS, RETIRED_OUTCOME } from "@crm/db/agent-tasks";-import { retireAbandoned } from "../../agent/lib/dispatch";+import { retireAbandoned } from "../../agent/lib/stale-tasks";import { claimDue } from "../../agent/lib/tasks";import { reasonOf } from "./e2e-agents";import { E2E } from "./e2e-config";diff --git a/apps/agent/test/stale-tasks.integration.spec.ts b/apps/agent/test/stale-tasks.integration.spec.tsnew file mode 100644index 00000000..4bf530ae--- /dev/null+++ b/apps/agent/test/stale-tasks.integration.spec.ts@@ -0,0 +1,336 @@+import { afterEach, beforeEach, describe, expect, it } from "bun:test";+import { db, EnrichmentStatus } from "@crm/db";+import { MAX_ATTEMPTS, RETIRED_OUTCOME } from "@crm/db/agent-tasks";+import { reconcileStaleTasks } from "../agent/lib/stale-tasks";++const kind = "recheck";++const REASON = "stale-spec";++const MINUTE_MS = 60_000;++async function clear() {+ await db.agentTask.deleteMany({ where: { reason: REASON } });+ await db.contact.deleteMany({ where: { email: { startsWith: "stale-" } } });+ await db.company.deleteMany({ where: { name: { startsWith: "Stale Co " } } });+}++beforeEach(clear);+afterEach(clear);++async function someone(status: EnrichmentStatus, enrichedAt?: Date) {+ return db.contact.create({+ data: {+ firstName: "Stale",+ email: `stale-${crypto.randomUUID()}@example.test`,+ enrichmentStatus: status,+ enrichedAt: enrichedAt ?? null,+ },+ select: { id: true },+ });+}++async function anAccount(status: EnrichmentStatus, enrichedAt?: Date) {+ return db.company.create({+ data: {+ name: `Stale Co ${crypto.randomUUID()}`,+ enrichmentStatus: status,+ enrichedAt: enrichedAt ?? null,+ },+ select: { id: true },+ });+}++async function queue(overrides: {+ kind?: string;+ contactId?: string;+ companyId?: string;+ attempts?: number;+ dueAt?: Date;+ startedAt?: Date | null;+ leasedUntil?: Date | null;+}) {+ return db.agentTask.create({+ data: {+ kind: overrides.kind ?? kind,+ reason: REASON,+ dueAt: overrides.dueAt ?? new Date(Date.now() - MINUTE_MS),+ priority: 0,+ budget: 4,+ contactId: overrides.contactId ?? null,+ companyId: overrides.companyId ?? null,+ attempts: overrides.attempts ?? 0,+ startedAt: overrides.startedAt ?? null,+ leasedUntil: overrides.leasedUntil ?? null,+ },+ select: { id: true },+ });+}++async function row(id: string) {+ return db.agentTask.findUnique({+ where: { id },+ select: { finishedAt: true, outcome: true, leasedUntil: true },+ });+}++describe("a task whose lease ran out", () => {+ it("gives up once the attempts are gone, and says so on the record", async () => {+ const contact = await someone(EnrichmentStatus.RUNNING);+ const task = await queue({+ contactId: contact.id,+ attempts: MAX_ATTEMPTS,+ startedAt: new Date(Date.now() - 30 * MINUTE_MS),+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ await reconcileStaleTasks();++ const closed = await row(task.id);+ expect(closed?.finishedAt).not.toBeNull();+ expect(closed?.outcome).toBe(RETIRED_OUTCOME);++ const settled = await db.contact.findUnique({+ where: { id: contact.id },+ select: { enrichmentStatus: true },+ });+ expect(settled?.enrichmentStatus).toBe(EnrichmentStatus.FAILED);+ });++ it("waits again when it still has attempts left", async () => {+ const contact = await someone(EnrichmentStatus.RUNNING);+ const task = await queue({+ contactId: contact.id,+ attempts: 1,+ startedAt: new Date(Date.now() - 30 * MINUTE_MS),+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ const sweep = await reconcileStaleTasks();+ expect(sweep.released).toBeGreaterThanOrEqual(1);++ const waiting = await row(task.id);+ expect(waiting?.finishedAt).toBeNull();+ expect(waiting?.leasedUntil).toBeNull();+ });+});++describe("a task that is still being worked on", () => {+ it("leaves a live lease alone", async () => {+ const contact = await someone(EnrichmentStatus.RUNNING);+ const task = await queue({+ contactId: contact.id,+ attempts: MAX_ATTEMPTS,+ startedAt: new Date(Date.now() - MINUTE_MS),+ leasedUntil: new Date(Date.now() + 10 * MINUTE_MS),+ });++ await reconcileStaleTasks();++ const held = await row(task.id);+ expect(held?.finishedAt).toBeNull();+ expect(held?.leasedUntil).not.toBeNull();+ });++ it("leaves a live lease alone even once the record reads complete", async () => {+ const account = await anAccount(EnrichmentStatus.COMPLETE, new Date());+ const task = await queue({+ companyId: account.id,+ attempts: 1,+ startedAt: new Date(Date.now() - MINUTE_MS),+ leasedUntil: new Date(Date.now() + 10 * MINUTE_MS),+ });++ await reconcileStaleTasks();+ expect((await row(task.id))?.finishedAt).toBeNull();+ });+});++describe("a task whose record is already done", () => {+ it("closes the row the work never closed", async () => {+ const startedAt = new Date(Date.now() - 30 * MINUTE_MS);+ const account = await anAccount(+ EnrichmentStatus.COMPLETE,+ new Date(Date.now() - 29 * MINUTE_MS),+ );+ const task = await queue({+ companyId: account.id,+ attempts: 1,+ startedAt,+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ const sweep = await reconcileStaleTasks();+ expect(sweep.closed).toBeGreaterThanOrEqual(1);++ const closed = await row(task.id);+ expect(closed?.finishedAt).not.toBeNull();+ expect(closed?.outcome).toContain("already up to date");+ });++ it("keeps event work that names a record the agent just enriched", async () => {+ const account = await anAccount(+ EnrichmentStatus.COMPLETE,+ new Date(Date.now() - 29 * MINUTE_MS),+ );+ const task = await queue({+ kind: "agent-event",+ companyId: account.id,+ attempts: 1,+ startedAt: new Date(Date.now() - 30 * MINUTE_MS),+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ await reconcileStaleTasks();++ expect((await row(task.id))?.finishedAt).toBeNull();+ });++ it("never closes a row another sweep is holding", async () => {+ const account = await anAccount(+ EnrichmentStatus.COMPLETE,+ new Date(Date.now() - 29 * MINUTE_MS),+ );+ const task = await queue({+ companyId: account.id,+ attempts: 1,+ startedAt: new Date(Date.now() - 30 * MINUTE_MS),+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ const findMany = db.agentTask.findMany;+ db.agentTask.findMany = async (...args) => {+ const rows = await findMany.apply(db.agentTask, args);+ await db.agentTask.update({+ where: { id: task.id },+ data: { leasedUntil: new Date(Date.now() + 10 * MINUTE_MS) },+ });++ return rows;+ };++ try {+ await reconcileStaleTasks();+ } finally {+ db.agentTask.findMany = findMany;+ }++ expect((await row(task.id))?.finishedAt).toBeNull();+ });++ it("keeps a record that finished before this work started", async () => {+ const account = await anAccount(+ EnrichmentStatus.COMPLETE,+ new Date(Date.now() - 60 * MINUTE_MS),+ );+ const task = await queue({+ companyId: account.id,+ attempts: 1,+ startedAt: new Date(Date.now() - 30 * MINUTE_MS),+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ await reconcileStaleTasks();+ expect((await row(task.id))?.finishedAt).toBeNull();+ });++ it("keeps work that is booked for a later day", async () => {+ const contact = await someone(EnrichmentStatus.COMPLETE, new Date());+ const task = await queue({+ contactId: contact.id,+ attempts: 0,+ dueAt: new Date(Date.now() + 90 * 24 * 60 * MINUTE_MS),+ });++ await reconcileStaleTasks();+ expect((await row(task.id))?.finishedAt).toBeNull();+ });++ it("keeps work nothing has picked up yet", async () => {+ const contact = await someone(EnrichmentStatus.COMPLETE, new Date());+ const task = await queue({ contactId: contact.id, attempts: 0 });++ await reconcileStaleTasks();+ expect((await row(task.id))?.finishedAt).toBeNull();+ });+});++describe("running it twice", () => {+ it("changes nothing the second time", async () => {+ const startedAt = new Date(Date.now() - 30 * MINUTE_MS);+ const account = await anAccount(+ EnrichmentStatus.COMPLETE,+ new Date(Date.now() - 29 * MINUTE_MS),+ );++ await queue({+ companyId: account.id,+ attempts: 1,+ startedAt,+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });+ await queue({+ contactId: (await someone(EnrichmentStatus.RUNNING)).id,+ attempts: 1,+ startedAt,+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });+ await queue({+ contactId: (await someone(EnrichmentStatus.RUNNING)).id,+ attempts: MAX_ATTEMPTS,+ startedAt,+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ const first = await reconcileStaleTasks();+ expect(first.closed).toBeGreaterThanOrEqual(1);+ expect(first.released).toBeGreaterThanOrEqual(1);+ expect(first.retired).toBeGreaterThanOrEqual(1);++ const second = await reconcileStaleTasks();+ expect(second.closed).toBe(0);+ expect(second.released).toBe(0);+ expect(second.retired).toBe(0);+ });++ it("reports a database failure instead of throwing", async () => {+ const findMany = db.agentTask.findMany;+ db.agentTask.findMany = () => {+ throw new Error("the database is unreachable");+ };++ try {+ const sweep = await reconcileStaleTasks();+ expect(sweep.error).toBe("the database is unreachable");+ expect(sweep.scanned).toBe(0);+ } finally {+ db.agentTask.findMany = findMany;+ }++ expect((await reconcileStaleTasks()).error).toBeNull();+ });++ it("keeps the counters a failed sweep already reached", async () => {+ await queue({+ contactId: (await someone(EnrichmentStatus.RUNNING)).id,+ attempts: 1,+ startedAt: new Date(Date.now() - 30 * MINUTE_MS),+ leasedUntil: new Date(Date.now() - MINUTE_MS),+ });++ const updateMany = db.agentTask.updateMany;+ db.agentTask.updateMany = () => {+ throw new Error("the write failed");+ };++ let sweep: Awaited<ReturnType<typeof reconcileStaleTasks>>;+ try {+ sweep = await reconcileStaleTasks();+ } finally {+ db.agentTask.updateMany = updateMany;+ }++ expect(sweep.error).toBe("the write failed");+ expect(sweep.scanned).toBeGreaterThanOrEqual(1);+ });+});diff --git a/apps/agent/test/tasks.integration.spec.ts b/apps/agent/test/tasks.integration.spec.tsindex 431c93b6..3100f0c1 100644--- a/apps/agent/test/tasks.integration.spec.ts+++ b/apps/agent/test/tasks.integration.spec.ts@@ -179,6 +179,25 @@ describe("retireExhausted", () => {expect(await retireExhausted()).toHaveLength(0);});++ it("retires no more rows than the limit allows", async () => {+ const mine: string[] = [];+ for (let row = 0; row < 3; row++) mine.push((await queue()).id);++ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {+ const claimed = await claimDue(10, RESEARCH);+ for (const task of claimed) await expire(task.id);+ }++ for (let pass = 0; pass < 3; pass++) {+ expect((await retireExhausted(2)).length).toBeLessThanOrEqual(2);+ }++ const open = await db.agentTask.count({+ where: { id: { in: mine }, finishedAt: null },+ });+ expect(open).toBe(0);+ });});describe("completeTask", () => {diff --git a/apps/api/src/agent/agent-queue.service.ts b/apps/api/src/agent/agent-queue.service.tsindex e1bfa2c8..f408f160 100644--- a/apps/api/src/agent/agent-queue.service.ts+++ b/apps/api/src/agent/agent-queue.service.ts@@ -7,45 +7,49 @@ export class AgentQueueService {constructor(@InjectDatabase() private readonly db: Db) {}async queuedCompanies(ids: readonly string[]): Promise<Set<string>> {- return this.queued("companyId", ids);+ if (ids.length === 0) return new Set();++ const rows = await this.db.agentTask.findMany({+ where: this.due({ companyId: { in: [...ids] } }),+ select: { companyId: true },+ distinct: ["companyId"],+ });++ return this.identifiers(rows.map((row) => row.companyId));}async queuedContacts(ids: readonly string[]): Promise<Set<string>> {- return this.queued("contactId", ids);+ if (ids.length === 0) return new Set();++ const rows = await this.db.agentTask.findMany({+ where: this.due({ contactId: { in: [...ids] } }),+ select: { contactId: true },+ distinct: ["contactId"],+ });++ return this.identifiers(rows.map((row) => row.contactId));}async isQueued(subject: {companyId?: string;contactId?: string;}): Promise<boolean> {- const where: Prisma.AgentTaskWhereInput = { finishedAt: null };- if (subject.companyId) where.companyId = subject.companyId;- if (subject.contactId) where.contactId = subject.contactId;-const row = await this.db.agentTask.findFirst({- where,+ where: this.due({+ companyId: subject.companyId,+ contactId: subject.contactId,+ }),select: { id: true },});return row !== null;}- private async queued(- column: "companyId" | "contactId",- ids: readonly string[],- ): Promise<Set<string>> {- if (ids.length === 0) return new Set();-- const rows = await this.db.agentTask.findMany({- where: { finishedAt: null, [column]: { in: [...ids] } },- select: { [column]: true },- distinct: [column],- });+ private due(subject: Prisma.AgentTaskWhereInput): Prisma.AgentTaskWhereInput {+ return { ...subject, finishedAt: null, dueAt: { lte: new Date() } };+ }- return new Set(- rows- .map((row) => (row as Record<string, string | null>)[column])- .filter((id): id is string => id !== null),- );+ private identifiers(values: (string | null)[]): Set<string> {+ return new Set(values.filter((value): value is string => value !== null));}}diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.tsindex fb4178fe..74acf8de 100644--- a/apps/api/src/app.module.ts+++ b/apps/api/src/app.module.ts@@ -16,6 +16,7 @@ import { CurrencyModule } from "./currency/currency.module";import { DashboardModule } from "./dashboard/dashboard.module";import { DatabaseModule } from "./database/database.module";import { DealsModule } from "./deals/deals.module";+import { EnrichmentModule } from "./enrichment/enrichment.module";import { FieldsModule } from "./fields/fields.module";import { GoogleModule } from "./google/google.module";import { HealthModule } from "./health/health.module";@@ -58,6 +59,7 @@ import { WorkspaceModule } from "./workspace/workspace.module";FieldsModule,ActivitiesModule,AgentModule,+ EnrichmentModule,DashboardModule,SearchModule,MailboxModule,diff --git a/apps/api/src/enrichment/enrichment-copy.ts b/apps/api/src/enrichment/enrichment-copy.tsnew file mode 100644index 00000000..af1f1540--- /dev/null+++ b/apps/api/src/enrichment/enrichment-copy.ts@@ -0,0 +1,72 @@+import { MAX_ATTEMPTS, type TaskKind } from "@crm/db/agent-tasks";++export type EnrichmentQueueState = "running" | "queued" | "failed";++const STEPS = {+ brand: "Fetching the logo",+ portrait: "Finding their photo",+ "meeting-prep": "Getting ready for your meeting",+ identify: "Reading their profile",+ profile: "Reading their profile",+ recheck: "Checking for anything new",+ "company-profile": "Reading the company website",+ "workspace-profile": "Reading your own website",+ "field-backfill": "Filling in the blank details",+ "slack-people-match": "Matching people in Slack",+ "slack-channel-join": "Joining a Slack channel",+ "agent-event": "Reacting to a change",+} satisfies Record<TaskKind, string>;++const STEP_BY_KIND = new Map<string, string>(Object.entries(STEPS));++const UNKNOWN_STEP = "Looking them up";+const WAITING = "Waiting";+const GAVE_UP = "Could not look this up";++const SECOND_MS = 1_000;+const DAY_MS = 24 * 60 * 60 * SECOND_MS;+const DAYS_IN_WEEK = 7;+const DAYS_IN_MONTH = 30;+const WEEKS_FROM = 14;+const MONTHS_FROM = 60;++const TODAY = "Later today";+const TOMORROW = "Tomorrow";++export function enrichmentStep(kind: string): string {+ return STEP_BY_KIND.get(kind) ?? UNKNOWN_STEP;+}++export function enrichmentQueueState(+ attempts: number,+ leasedUntil: Date | null,+ now: Date,+): EnrichmentQueueState {+ if (leasedUntil !== null && leasedUntil.getTime() > now.getTime()) {+ return "running";+ }++ return attempts >= MAX_ATTEMPTS ? "failed" : "queued";+}++export function enrichmentQueueLine(+ state: EnrichmentQueueState,+ kind: string,+): string {+ if (state === "queued") return WAITING;+ if (state === "failed") return GAVE_UP;++ return enrichmentStep(kind);+}++export function enrichmentDueLabel(dueAt: Date, now: Date): string {+ const ahead = dueAt.getTime() - now.getTime();+ if (ahead < DAY_MS) return TODAY;++ const days = Math.floor(ahead / DAY_MS);+ if (days === 1) return TOMORROW;+ if (days < WEEKS_FROM) return `In ${days} days`;+ if (days < MONTHS_FROM) return `In ${Math.round(days / DAYS_IN_WEEK)} weeks`;++ return `In ${Math.round(days / DAYS_IN_MONTH)} months`;+}diff --git a/apps/api/src/enrichment/enrichment.module.ts b/apps/api/src/enrichment/enrichment.module.tsnew file mode 100644index 00000000..95208eac--- /dev/null+++ b/apps/api/src/enrichment/enrichment.module.ts@@ -0,0 +1,10 @@+import { Module } from "@nestjs/common";+import { TrpcModule } from "../trpc/trpc.module";+import { EnrichmentRouter } from "./enrichment.router";+import { EnrichmentService } from "./enrichment.service";++@Module({+ imports: [TrpcModule],+ providers: [EnrichmentService, EnrichmentRouter],+})+export class EnrichmentModule {}diff --git a/apps/api/src/enrichment/enrichment.router.ts b/apps/api/src/enrichment/enrichment.router.tsnew file mode 100644index 00000000..5276bef8--- /dev/null+++ b/apps/api/src/enrichment/enrichment.router.ts@@ -0,0 +1,17 @@+import { Inject } from "@nestjs/common";+import { Query, Router, UseMiddlewares } from "nestjs-trpc";+import { AuthMiddleware } from "../trpc/middlewares/auth.middleware";+import { EnrichmentService } from "./enrichment.service";++@Router({ alias: "enrichment" })+@UseMiddlewares(AuthMiddleware)+export class EnrichmentRouter {+ constructor(+ @Inject(EnrichmentService) private readonly enrichment: EnrichmentService,+ ) {}++ @Query()+ async queue() {+ return this.enrichment.queue();+ }+}diff --git a/apps/api/src/enrichment/enrichment.service.ts b/apps/api/src/enrichment/enrichment.service.tsnew file mode 100644index 00000000..ba51ae8b--- /dev/null+++ b/apps/api/src/enrichment/enrichment.service.ts@@ -0,0 +1,220 @@+import type { Db, Prisma } from "@crm/db";+import { Injectable } from "@nestjs/common";+import { InjectDatabase } from "../database/database.constants";+import {+ type EnrichmentQueueState,+ enrichmentDueLabel,+ enrichmentQueueLine,+ enrichmentQueueState,+} from "./enrichment-copy";++const QUEUE_ROWS = 20;++const SCHEDULED_ROWS = 20;++export type EnrichmentQueueSubject =+ | {+ kind: "contact";+ id: string;+ name: string;+ email: string | null;+ imageUrl: string | null;+ }+ | {+ kind: "company";+ id: string;+ name: string;+ logoUrl: string | null;+ logoDarkUrl: string | null;+ logoTone: string | null;+ };++export type EnrichmentQueueRow = {+ id: string;+ state: EnrichmentQueueState;+ line: string;+ startedAt: string | null;+ subject: EnrichmentQueueSubject;+};++export type EnrichmentScheduledRow = {+ id: string;+ due: string;+ subject: EnrichmentQueueSubject;+};++export type EnrichmentQueue = {+ rows: EnrichmentQueueRow[];+ total: number;+ scheduled: EnrichmentScheduledRow[];+ scheduledTotal: number;+};++const TASK_SELECT = {+ id: true,+ contactId: true,+ companyId: true,+ kind: true,+ attempts: true,+ leasedUntil: true,+ startedAt: true,+ dueAt: true,+} as const;++type QueuedTask = {+ id: string;+ contactId: string | null;+ companyId: string | null;+ kind: string;+ attempts: number;+ leasedUntil: Date | null;+ startedAt: Date | null;+ dueAt: Date;+};++@Injectable()+export class EnrichmentService {+ constructor(@InjectDatabase() private readonly db: Db) {}++ async queue(): Promise<EnrichmentQueue> {+ const now = new Date();++ const named: Prisma.AgentTaskWhereInput = {+ finishedAt: null,+ OR: [{ contactId: { not: null } }, { companyId: { not: null } }],+ };++ const dueWhere: Prisma.AgentTaskWhereInput = {+ ...named,+ dueAt: { lte: now },+ };++ const scheduledWhere: Prisma.AgentTaskWhereInput = {+ ...named,+ dueAt: { gt: now },+ };++ const [tasks, total, booked, scheduledTotal] = await Promise.all([+ this.db.agentTask.findMany({+ where: dueWhere,+ orderBy: [{ priority: "desc" }, { dueAt: "asc" }],+ take: QUEUE_ROWS,+ select: TASK_SELECT,+ }),+ this.db.agentTask.count({ where: dueWhere }),+ this.db.agentTask.findMany({+ where: scheduledWhere,+ orderBy: [{ dueAt: "asc" }],+ take: SCHEDULED_ROWS,+ select: TASK_SELECT,+ }),+ this.db.agentTask.count({ where: scheduledWhere }),+ ]);++ const everyTask: QueuedTask[] = [...tasks, ...booked];+ const contactIds = unique(everyTask.map((task) => task.contactId));+ const companyIds = unique(everyTask.map((task) => task.companyId));++ const [contacts, companies] = await Promise.all([+ contactIds.length === 0+ ? []+ : this.db.contact.findMany({+ where: { id: { in: contactIds } },+ select: {+ id: true,+ firstName: true,+ lastName: true,+ email: true,+ imageUrl: true,+ },+ }),+ companyIds.length === 0+ ? []+ : this.db.company.findMany({+ where: { id: { in: companyIds } },+ select: {+ id: true,+ name: true,+ iconUrl: true,+ logoUrl: true,+ iconDarkUrl: true,+ logoDarkUrl: true,+ iconTone: true,+ },+ }),+ ]);++ const contactById = new Map(+ contacts.map((contact) => [+ contact.id,+ {+ kind: "contact" as const,+ id: contact.id,+ name:+ [contact.firstName, contact.lastName].filter(Boolean).join(" ") ||+ contact.email ||+ "Unnamed contact",+ email: contact.email,+ imageUrl: contact.imageUrl,+ },+ ]),+ );++ const companyById = new Map(+ companies.map((company) => [+ company.id,+ {+ kind: "company" as const,+ id: company.id,+ name: company.name,+ logoUrl: company.iconUrl ?? company.logoUrl,+ logoDarkUrl: company.iconDarkUrl ?? company.logoDarkUrl,+ logoTone: company.iconTone,+ },+ ]),+ );++ const subjectOf = (task: QueuedTask): EnrichmentQueueSubject | undefined =>+ task.contactId+ ? contactById.get(task.contactId)+ : task.companyId+ ? companyById.get(task.companyId)+ : undefined;++ const rows: EnrichmentQueueRow[] = [];++ for (const task of tasks) {+ const subject = subjectOf(task);+ if (!subject) continue;++ const state = enrichmentQueueState(task.attempts, task.leasedUntil, now);++ rows.push({+ id: task.id,+ state,+ line: enrichmentQueueLine(state, task.kind),+ startedAt: task.startedAt?.toISOString() ?? null,+ subject,+ });+ }++ const scheduled: EnrichmentScheduledRow[] = [];++ for (const task of booked) {+ const subject = subjectOf(task);+ if (!subject) continue;++ scheduled.push({+ id: task.id,+ due: enrichmentDueLabel(task.dueAt, now),+ subject,+ });+ }++ return { rows, total, scheduled, scheduledTotal };+ }+}++function unique(ids: readonly (string | null)[]): string[] {+ return [...new Set(ids.filter((id): id is string => id !== null))];+}diff --git a/apps/api/src/generated/server.ts b/apps/api/src/generated/server.tsindex b42b0326..84a6eed8 100644--- a/apps/api/src/generated/server.ts+++ b/apps/api/src/generated/server.ts@@ -37,6 +37,7 @@ import type { ConversationsRouter } from "../conversations/conversations.router"import type { CurrencyRouter } from "../currency/currency.router";import type { DashboardRouter } from "../dashboard/dashboard.router";import type { DealsRouter } from "../deals/deals.router";+import type { EnrichmentRouter } from "../enrichment/enrichment.router";import type { FieldsRouter } from "../fields/fields.router";import type { GoogleRouter } from "../google/google.router";import type { MicrosoftRouter } from "../microsoft/microsoft.router";@@ -301,6 +302,10 @@ const appRouter = t.router({.input(dealBulkInput).mutation(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited<ReturnType<DealsRouter["bulkDelete"]>>)}),+ enrichment: t.router({+ queue: publicProcedure+ .query(async () => "PLACEHOLDER_DO_NOT_REMOVE" as unknown as Awaited<ReturnType<EnrichmentRouter["queue"]>>)+ }),fields: t.router({list: publicProcedure.input(fieldListInput)diff --git a/apps/api/test/agent-queue.spec.ts b/apps/api/test/agent-queue.spec.tsnew file mode 100644index 00000000..3a658e23--- /dev/null+++ b/apps/api/test/agent-queue.spec.ts@@ -0,0 +1,76 @@+import { afterAll, beforeAll, describe, expect, it } from "bun:test";+import { db } from "@crm/db";+import { AgentQueueService } from "../src/agent/agent-queue.service";++const suffix = process.env.TEST_RUN_ID ?? "agent-queue-spec";+const email = `badge-${suffix}@example.test`;+const name = `Badge Co ${suffix}`;+const kind = "test-queue-badge";++const queue = new AgentQueueService(db);++const DAY_MS = 86_400_000;++let contactId: string;+let companyId: string;++async function clean() {+ await db.agentTask.deleteMany({ where: { kind } });+ await db.contact.deleteMany({ where: { email } });+ await db.company.deleteMany({ where: { name } });+}++beforeAll(async () => {+ await clean();++ const contact = await db.contact.create({+ data: { firstName: "Badge", lastName: "Later", email },+ select: { id: true },+ });+ const company = await db.company.create({+ data: { name },+ select: { id: true },+ });++ contactId = contact.id;+ companyId = company.id;++ await db.agentTask.create({+ data: {+ kind,+ reason: "recheck in three months",+ dueAt: new Date(Date.now() + 90 * DAY_MS),+ budget: 4,+ contactId,+ },+ });++ await db.agentTask.create({+ data: {+ kind,+ reason: "due now",+ dueAt: new Date(Date.now() - 60_000),+ budget: 4,+ companyId,+ },+ });+});++afterAll(clean);++describe("the queued badge", () => {+ it("ignores a recheck booked for a later date", async () => {+ expect(await queue.queuedContacts([contactId])).not.toContain(contactId);+ expect(await queue.isQueued({ contactId })).toBe(false);+ });++ it("marks work that is due now", async () => {+ expect(await queue.queuedCompanies([companyId])).toContain(companyId);+ expect(await queue.isQueued({ companyId })).toBe(true);+ });++ it("asks for nothing when given no records", async () => {+ expect((await queue.queuedContacts([])).size).toBe(0);+ expect((await queue.queuedCompanies([])).size).toBe(0);+ });+});diff --git a/apps/api/test/enrichment-copy.spec.ts b/apps/api/test/enrichment-copy.spec.tsnew file mode 100644index 00000000..f019bec2--- /dev/null+++ b/apps/api/test/enrichment-copy.spec.ts@@ -0,0 +1,109 @@+import { describe, expect, it } from "bun:test";+import { MAX_ATTEMPTS, TASK_KINDS } from "@crm/db/agent-tasks";+import {+ enrichmentDueLabel,+ enrichmentQueueLine,+ enrichmentQueueState,+ enrichmentStep,+} from "../src/enrichment/enrichment-copy";++const NOW = new Date("2026-08-17T10:00:00.000Z");+const LATER = new Date("2026-08-17T10:05:00.000Z");+const EARLIER = new Date("2026-08-17T09:55:00.000Z");++describe("what a rep reads while the agent works", () => {+ it("gives every task kind a sentence", () => {+ for (const kind of TASK_KINDS) {+ expect(enrichmentStep(kind).length).toBeGreaterThan(0);+ }+ });++ it("says what the agent is doing, not what the code calls it", () => {+ expect(enrichmentStep("brand")).toBe("Fetching the logo");+ expect(enrichmentStep("identify")).toBe("Reading their profile");+ expect(enrichmentStep("portrait")).toBe("Finding their photo");+ expect(enrichmentStep("company-profile")).toBe(+ "Reading the company website",+ );+ });++ it("uses no internal vocabulary", () => {+ const banned = ["tick", "sweep", "enrol", "task", "lane", "dispatch"];++ for (const kind of TASK_KINDS) {+ const sentence = enrichmentStep(kind).toLowerCase();+ for (const word of banned) expect(sentence).not.toContain(word);+ }+ });++ it("falls back to a sentence for a kind it does not know", () => {+ expect(enrichmentStep("something-new")).toBe("Looking them up");+ });+});++describe("which state a row is in", () => {+ it("is running while the lease holds", () => {+ expect(enrichmentQueueState(1, LATER, NOW)).toBe("running");+ });++ it("is waiting before anything claims it", () => {+ expect(enrichmentQueueState(0, null, NOW)).toBe("queued");+ });++ it("is waiting again once a lease expires", () => {+ expect(enrichmentQueueState(1, EARLIER, NOW)).toBe("queued");+ });++ it("has failed once the attempts run out", () => {+ expect(enrichmentQueueState(MAX_ATTEMPTS, null, NOW)).toBe("failed");+ expect(enrichmentQueueState(MAX_ATTEMPTS, EARLIER, NOW)).toBe("failed");+ });++ it("still reads as running on the last attempt while the lease holds", () => {+ expect(enrichmentQueueState(MAX_ATTEMPTS, LATER, NOW)).toBe("running");+ });+});++describe("when work is booked for later", () => {+ const ahead = (days: number, hours = 0) =>+ new Date(NOW.getTime() + days * 86_400_000 + hours * 3_600_000);++ it("says the day in words, never a date", () => {+ expect(enrichmentDueLabel(ahead(0, 6), NOW)).toBe("Later today");+ expect(enrichmentDueLabel(ahead(1, 1), NOW)).toBe("Tomorrow");+ expect(enrichmentDueLabel(ahead(3), NOW)).toBe("In 3 days");+ });++ it("counts weeks from a fortnight, and months from two", () => {+ expect(enrichmentDueLabel(ahead(14), NOW)).toBe("In 2 weeks");+ expect(enrichmentDueLabel(ahead(30), NOW)).toBe("In 4 weeks");+ expect(enrichmentDueLabel(ahead(60), NOW)).toBe("In 2 months");+ expect(enrichmentDueLabel(ahead(90), NOW)).toBe("In 3 months");+ });++ it("uses no internal vocabulary", () => {+ const banned = ["tick", "sweep", "task", "lease", "dispatch", "queue"];++ for (const days of [0, 1, 3, 14, 30, 90, 365]) {+ const sentence = enrichmentDueLabel(ahead(days), NOW).toLowerCase();+ for (const word of banned) expect(sentence).not.toContain(word);+ }+ });+});++describe("the second line of a row", () => {+ it("names the step while it runs", () => {+ expect(enrichmentQueueLine("running", "brand")).toBe("Fetching the logo");+ });++ it("says Waiting for a queued row, whatever the kind", () => {+ expect(enrichmentQueueLine("queued", "brand")).toBe("Waiting");+ expect(enrichmentQueueLine("queued", "identify")).toBe("Waiting");+ });++ it("says why a failed row stopped", () => {+ expect(enrichmentQueueLine("failed", "brand")).toBe(+ "Could not look this up",+ );+ });+});diff --git a/apps/api/test/enrichment-queue.spec.ts b/apps/api/test/enrichment-queue.spec.tsnew file mode 100644index 00000000..6e619157--- /dev/null+++ b/apps/api/test/enrichment-queue.spec.ts@@ -0,0 +1,99 @@+import { afterAll, beforeAll, describe, expect, it } from "bun:test";+import { db } from "@crm/db";+import { EnrichmentService } from "../src/enrichment/enrichment.service";++const suffix = process.env.TEST_RUN_ID ?? "enrichment-queue-spec";+const email = `queue-${suffix}@example.test`;+const name = `Queue Co ${suffix}`;+const kind = "test-queue-split";++const enrichment = new EnrichmentService(db);++const DAY_MS = 86_400_000;++let contactId: string;+let companyId: string;+let dueId: string;+let scheduledId: string;++async function clean() {+ await db.agentTask.deleteMany({ where: { kind } });+ await db.contact.deleteMany({ where: { email } });+ await db.company.deleteMany({ where: { name } });+}++beforeAll(async () => {+ await clean();++ const contact = await db.contact.create({+ data: { firstName: "Queue", lastName: "Split", email },+ select: { id: true },+ });+ const company = await db.company.create({+ data: { name },+ select: { id: true },+ });++ contactId = contact.id;+ companyId = company.id;++ const due = await db.agentTask.create({+ data: {+ kind,+ reason: "due now",+ dueAt: new Date(Date.now() - 60_000),+ budget: 4,+ companyId,+ },+ select: { id: true },+ });+ const scheduled = await db.agentTask.create({+ data: {+ kind,+ reason: "booked for later",+ dueAt: new Date(Date.now() + 90 * DAY_MS),+ budget: 4,+ contactId,+ },+ select: { id: true },+ });++ dueId = due.id;+ scheduledId = scheduled.id;+});++afterAll(clean);++describe("what the enrichment widget reads", () => {+ it("lists work that is due now", async () => {+ const queue = await enrichment.queue();++ expect(queue.rows.map((row) => row.id)).toContain(dueId);+ expect(queue.total).toBeGreaterThanOrEqual(1);+ });++ it("keeps work booked for a later day out of the count", async () => {+ const queue = await enrichment.queue();++ expect(queue.rows.map((row) => row.id)).not.toContain(scheduledId);+ expect(queue.scheduled.map((row) => row.id)).toContain(scheduledId);+ expect(queue.scheduledTotal).toBeGreaterThanOrEqual(1);+ });++ it("says when a booked record is looked at again", async () => {+ const queue = await enrichment.queue();+ const booked = queue.scheduled.find((row) => row.id === scheduledId);++ expect(booked?.due).toBe("In 3 months");+ expect(booked?.subject.id).toBe(contactId);+ expect(booked?.subject.name).toBe("Queue Split");+ });++ it("names the company a due row is about", async () => {+ const queue = await enrichment.queue();+ const row = queue.rows.find((entry) => entry.id === dueId);++ expect(row?.subject.id).toBe(companyId);+ expect(row?.line).toBe("Waiting");+ });+});diff --git a/apps/app/components/app-header.tsx b/apps/app/components/app-header.tsxindex b2762247..807d15be 100644--- a/apps/app/components/app-header.tsx+++ b/apps/app/components/app-header.tsx@@ -22,6 +22,7 @@ import { useQuery } from "@tanstack/react-query";import Link from "next/link";import { useTheme } from "next-themes";import { toast } from "sonner";+import { EnrichmentQueue } from "@/components/enrichment-queue";import { useMobileNav } from "@/components/mobile-nav";import { signOutAndRedirect } from "@/lib/sign-out";import { useTRPC } from "@/lib/trpc/client";@@ -61,6 +62,7 @@ export function AppHeader({ user }: { user: User }) {</div><div className="ml-auto flex shrink-0 items-center gap-1.5">+ <EnrichmentQueue /><UserMenuuser={user}onSignOut={() => {diff --git a/apps/app/components/enrichment-queue.tsx b/apps/app/components/enrichment-queue.tsxnew file mode 100644index 00000000..a09f0962--- /dev/null+++ b/apps/app/components/enrichment-queue.tsx@@ -0,0 +1,297 @@+"use client";++import ChevronRight from "@carbon/icons-react/es/ChevronRight";+import { Button } from "@crm/ui/components/button";+import {+ Collapsible,+ CollapsibleContent,+ CollapsibleTrigger,+} from "@crm/ui/components/collapsible";+import {+ EntityLogo,+ type EntityLogoTone,+} from "@crm/ui/components/entity-logo";+import { PersonAvatar } from "@crm/ui/components/person-avatar";+import {+ Popover,+ PopoverContent,+ PopoverTrigger,+} from "@crm/ui/components/popover";+import {+ IndicatorDot,+ type StatusTone,+} from "@crm/ui/components/status-indicator";+import { cn } from "@crm/ui/lib/utils";+import { useQuery } from "@tanstack/react-query";+import { useState } from "react";+import { useOpenRecord } from "@/components/crm/record-sheet/record-stack";+import { elapsedLabel } from "@/lib/enrichment-queue";+import {+ ENRICHMENT_IDLE_POLL_MS,+ ENRICHMENT_POLL_MS,+} from "@/lib/enrichment-status";+import { useTRPC } from "@/lib/trpc/client";+import type { RouterOutputs } from "@/lib/trpc/types";+import { useSeconds } from "@/lib/use-seconds";++type QueueRow = RouterOutputs["enrichment"]["queue"]["rows"][number];+type ScheduledRow = RouterOutputs["enrichment"]["queue"]["scheduled"][number];+type QueueSubject = QueueRow["subject"];++const VISIBLE_ROWS = 5;++const STATE_TONE = {+ running: "primary",+ queued: "neutral",+ failed: "error",+} satisfies Record<QueueRow["state"], StatusTone>;++export function EnrichmentQueue() {+ const trpc = useTRPC();+ const openRecord = useOpenRecord();+ const [open, setOpen] = useState(false);+ const [expanded, setExpanded] = useState(false);++ const queue = useQuery({+ ...trpc.enrichment.queue.queryOptions(),+ refetchInterval: (query) =>+ (query.state.data?.total ?? 0) > 0+ ? ENRICHMENT_POLL_MS+ : ENRICHMENT_IDLE_POLL_MS,+ });++ const rows = queue.data?.rows ?? [];+ const total = queue.data?.total ?? 0;+ const scheduled = queue.data?.scheduled ?? [];+ const scheduledTotal = queue.data?.scheduledTotal ?? 0;+ const visible = expanded ? rows : rows.slice(0, VISIBLE_ROWS);+ const hidden = rows.length - visible.length;+ const more = total - visible.length;++ const openSubject = (subject: QueueSubject) => {+ setOpen(false);+ openRecord({ kind: subject.kind, id: subject.id });+ };++ return (+ <Popover+ open={open}+ onOpenChange={(next) => {+ setOpen(next);+ if (!next) setExpanded(false);+ }}+ >+ <PopoverTrigger asChild>+ <Button variant="outline" size="sm">+ <IndicatorDot+ tone={total > 0 ? "primary" : "neutral"}+ aria-hidden="true"+ />+ {total > 0 ? `Enriching ${total}` : "Enriching"}+ </Button>+ </PopoverTrigger>++ <PopoverContent align="end" size="panel">+ <div className="flex items-center justify-between border-b px-4 py-3">+ <span className="font-semibold text-popover-foreground text-sm">+ Enriching now+ </span>+ <span className="font-mono text-muted-foreground text-xs">+ {total}+ </span>+ </div>++ {visible.length === 0 ? (+ <div className="flex flex-col gap-1 px-6 py-8 text-center">+ <span className="font-medium text-popover-foreground text-sm">+ Nothing right now+ </span>+ <span className="text-muted-foreground text-xs">+ New companies and contacts show up here while your agents look+ them up.+ </span>+ </div>+ ) : (+ <QueueRows+ rows={visible}+ scroll={expanded}+ onOpen={(row) => openSubject(row.subject)}+ />+ )}++ {more > 0 ? (+ <div className="flex items-center justify-between px-4 py-3 text-xs">+ <span className="text-muted-foreground">{more} more queued</span>+ {hidden > 0 ? (+ <button+ type="button"+ className="cursor-pointer text-primary"+ onClick={() => setExpanded(true)}+ >+ Show all+ </button>+ ) : null}+ </div>+ ) : null}++ {scheduledTotal > 0 ? (+ <ScheduledSection+ rows={scheduled}+ total={scheduledTotal}+ onOpen={(row) => openSubject(row.subject)}+ />+ ) : null}+ </PopoverContent>+ </Popover>+ );+}++function QueueRows({+ rows,+ scroll,+ onOpen,+}: {+ rows: QueueRow[];+ scroll: boolean;+ onOpen: (row: QueueRow) => void;+}) {+ const now = useSeconds();++ return (+ <div className={cn(scroll && "max-h-80 overflow-y-auto")}>+ {rows.map((row) => (+ <QueueRowButton+ key={row.id}+ row={row}+ now={now}+ onOpen={() => onOpen(row)}+ />+ ))}+ </div>+ );+}++function QueueRowButton({+ row,+ now,+ onOpen,+}: {+ row: QueueRow;+ now: number;+ onOpen: () => void;+}) {+ return (+ <button+ type="button"+ onClick={onOpen}+ className="group flex w-full cursor-pointer items-center gap-3 border-b px-4 py-3 text-left last:border-b-0 hover:bg-muted"+ >+ <SubjectFace subject={row.subject} />++ <span className="flex min-w-0 flex-1 flex-col gap-px">+ <span className="truncate font-medium text-popover-foreground text-sm">+ {row.subject.name}+ </span>+ <span+ className={cn(+ "truncate text-xs",+ row.state === "failed"+ ? "text-destructive"+ : "text-muted-foreground",+ )}+ >+ {row.line}+ </span>+ </span>++ <span className="flex w-13 shrink-0 items-center justify-end gap-2">+ <span className="font-mono text-muted-foreground text-xs group-hover:hidden">+ {elapsedLabel(row.state, row.startedAt, now)}+ </span>+ <ChevronRight+ className="hidden size-3.5 text-muted-foreground group-hover:block"+ aria-hidden="true"+ />+ <IndicatorDot tone={STATE_TONE[row.state]} aria-hidden="true" />+ </span>+ </button>+ );+}++function SubjectFace({ subject }: { subject: QueueSubject }) {+ if (subject.kind === "contact") {+ return (+ <PersonAvatar+ size="sm"+ src={subject.imageUrl}+ name={subject.name}+ email={subject.email}+ />+ );+ }++ return (+ <EntityLogo+ src={subject.logoUrl}+ darkSrc={subject.logoDarkUrl}+ tone={entityTone(subject.logoTone)}+ name={subject.name}+ />+ );+}++function ScheduledSection({+ rows,+ total,+ onOpen,+}: {+ rows: ScheduledRow[];+ total: number;+ onOpen: (row: ScheduledRow) => void;+}) {+ const [shown, setShown] = useState(false);++ return (+ <Collapsible className="border-t" open={shown} onOpenChange={setShown}>+ <CollapsibleTrigger className="flex w-full cursor-pointer items-center justify-between px-4 py-3 text-xs hover:bg-muted">+ <span className="text-muted-foreground">+ {total === 1+ ? "1 record is booked for a later look"+ : `${total} records are booked for a later look`}+ </span>+ <span className="text-primary">{shown ? "Hide" : "Show"}</span>+ </CollapsibleTrigger>++ <CollapsibleContent className="max-h-60 overflow-y-auto border-t">+ {rows.map((row) => (+ <button+ key={row.id}+ type="button"+ onClick={() => onOpen(row)}+ className="flex w-full cursor-pointer items-center gap-3 border-b px-4 py-2.5 text-left last:border-b-0 hover:bg-muted"+ >+ <SubjectFace subject={row.subject} />+ <span className="min-w-0 flex-1 truncate text-muted-foreground text-sm">+ {row.subject.name}+ </span>+ <span className="shrink-0 text-muted-foreground text-xs">+ {row.due}+ </span>+ </button>+ ))}++ {total > rows.length ? (+ <div className="px-4 py-3 text-muted-foreground text-xs">+ {total - rows.length} more are booked but not listed here+ </div>+ ) : null}+ </CollapsibleContent>+ </Collapsible>+ );+}++function entityTone(tone: string | null): EntityLogoTone | null {+ if (tone === "dark" || tone === "light" || tone === "opaque") return tone;++ return null;+}diff --git a/apps/app/lib/enrichment-queue.ts b/apps/app/lib/enrichment-queue.tsnew file mode 100644index 00000000..ebdf1619--- /dev/null+++ b/apps/app/lib/enrichment-queue.ts@@ -0,0 +1,32 @@+const SECOND_MS = 1_000;+const MINUTE_S = 60;+const HOUR_S = 60 * MINUTE_S;++export const NO_ELAPSED = "—";++export function formatElapsed(seconds: number): string {+ const whole = Math.max(0, Math.floor(seconds));+ const minutes = Math.floor((whole % HOUR_S) / MINUTE_S);+ const rest = whole % MINUTE_S;++ if (whole < HOUR_S) return `${minutes}:${pad(rest)}`;++ return `${Math.floor(whole / HOUR_S)}:${pad(minutes)}:${pad(rest)}`;+}++export function elapsedLabel(+ state: "running" | "queued" | "failed",+ startedAt: string | null,+ now: number,+): string {+ if (state === "queued" || startedAt === null) return NO_ELAPSED;++ const started = Date.parse(startedAt);+ if (Number.isNaN(started)) return NO_ELAPSED;++ return formatElapsed((now - started) / SECOND_MS);+}++function pad(value: number): string {+ return value.toString().padStart(2, "0");+}diff --git a/apps/app/lib/enrichment-status.ts b/apps/app/lib/enrichment-status.tsindex 7024158d..4cdc3dda 100644--- a/apps/app/lib/enrichment-status.ts+++ b/apps/app/lib/enrichment-status.ts@@ -18,6 +18,8 @@ const QUEUED = { label: "Queued", tone: "neutral" as StatusTone, busy: false };export const ENRICHMENT_POLL_MS = 3_000;+export const ENRICHMENT_IDLE_POLL_MS = 30_000;+export const ENRICHMENT_FACET_OPTIONS = (Object.keys(PRESENTATION) as EnrichmentStatus[]).map((value) => ({ value, label: PRESENTATION[value].label }));diff --git a/apps/app/lib/use-seconds.ts b/apps/app/lib/use-seconds.tsnew file mode 100644index 00000000..9739e814--- /dev/null+++ b/apps/app/lib/use-seconds.ts@@ -0,0 +1,45 @@+"use client";++import { useSyncExternalStore } from "react";++const TICK_MS = 1_000;++const listeners = new Set<() => void>();++let timer: ReturnType<typeof setInterval> | null = null;+let now = Date.now();++export function useSeconds(): number {+ return useSyncExternalStore(subscribe, clientSnapshot, serverSnapshot);+}++function subscribe(listener: () => void) {+ listeners.add(listener);++ if (timer === null) {+ now = Date.now();+ timer = setInterval(tick, TICK_MS);+ }++ return () => {+ listeners.delete(listener);++ if (listeners.size === 0 && timer !== null) {+ clearInterval(timer);+ timer = null;+ }+ };+}++function tick() {+ now = Date.now();+ for (const listener of listeners) listener();+}++function clientSnapshot() {+ return now;+}++function serverSnapshot() {+ return 0;+}diff --git a/apps/app/test/enrichment-queue.spec.ts b/apps/app/test/enrichment-queue.spec.tsnew file mode 100644index 00000000..70ca7f4b--- /dev/null+++ b/apps/app/test/enrichment-queue.spec.ts@@ -0,0 +1,59 @@+import { describe, expect, it } from "bun:test";+import {+ elapsedLabel,+ formatElapsed,+ NO_ELAPSED,+} from "../lib/enrichment-queue";++describe("how long a lookup has been going", () => {+ it("pads the seconds under a minute", () => {+ expect(formatElapsed(0)).toBe("0:00");+ expect(formatElapsed(9)).toBe("0:09");+ expect(formatElapsed(12)).toBe("0:12");+ });++ it("counts minutes up to an hour", () => {+ expect(formatElapsed(60)).toBe("1:00");+ expect(formatElapsed(605)).toBe("10:05");+ expect(formatElapsed(3599)).toBe("59:59");+ });++ it("adds an hours field past an hour", () => {+ expect(formatElapsed(3600)).toBe("1:00:00");+ expect(formatElapsed(3723)).toBe("1:02:03");+ expect(formatElapsed(86_400)).toBe("24:00:00");+ });++ it("drops part seconds instead of rounding up", () => {+ expect(formatElapsed(12.9)).toBe("0:12");+ });++ it("reads a clock skew as zero, never as a negative time", () => {+ expect(formatElapsed(-5)).toBe("0:00");+ });+});++describe("what the meta slot shows", () => {+ const started = "2026-08-17T10:00:00.000Z";+ const now = Date.parse("2026-08-17T10:00:12.000Z");++ it("counts up while the agent works", () => {+ expect(elapsedLabel("running", started, now)).toBe("0:12");+ });++ it("shows a dash for a row that is still waiting", () => {+ expect(elapsedLabel("queued", started, now)).toBe(NO_ELAPSED);+ });++ it("keeps the time on a row that gave up", () => {+ expect(elapsedLabel("failed", started, now)).toBe("0:12");+ });++ it("shows a dash when nothing has started", () => {+ expect(elapsedLabel("running", null, now)).toBe(NO_ELAPSED);+ });++ it("shows a dash rather than NaN for an unreadable date", () => {+ expect(elapsedLabel("running", "not a date", now)).toBe(NO_ELAPSED);+ });+});diff --git a/docs/agent.md b/docs/agent.mdindex 89eb5377..3dd8e33c 100644--- a/docs/agent.md+++ b/docs/agent.md@@ -122,6 +122,32 @@ to ration and nobody to scope it to.chain is not guaranteed to run at all. A cron in the agent is the only trigger herethat is a fact rather than a hope.+### Stale rows are closed on the dispatch tick++`reconcileStaleTasks` (`lib/stale-tasks.ts`) runs before `drainAll`, every minute. Like+the blank-field pass it is a database pass with no session, no model and no credits. It+closes rows that are provably done or provably dead, so the queue a rep reads is the+work that is actually happening.++- **It only looks at work that is due.** A `recheck` booked ninety days out is not+ stale, it is scheduled, and closing it would delete the next look at that contact.+- **A live lease is never touched.** That is work in flight, whatever the record says.+- **A record that completed after the task started closes the task** with a plain+ outcome. That is the row the settle path failed to close: research landed, facts were+ written, and the task sat open holding the contact on *Researching*.+- **A dead lease with attempts left goes back to waiting**, and the lease is cleared so+ the row reads as waiting rather than as running. Exhausted rows are retired by+ `retireExhausted`, which is the one rule for giving up.+- **Idempotent** — a second pass in the same minute closes nothing, because the rows it+ closed have a `finishedAt` and the rows it released have no lease.+- **Scans 200 rows a pass** and reports `unscanned` rather than a clean sweep it did not+ make. The counts are in `dispatchHealth().staleTasks`, so a growing number of stale+ rows is visible at `GET /internal/crm/dispatch-health`.++**It is a safety net, not the fix.** The settle path still fails to close some rows;+this pass keeps that from reaching a rep, and the count it reports is how you see the+underlying bug getting worse.+### BackfillsSign-in sweep covers records never looked up (10 credits/company);diff --git a/packages/db/src/agent-tasks.ts b/packages/db/src/agent-tasks.tsindex 33f60eb1..4f65eec6 100644--- a/packages/db/src/agent-tasks.ts+++ b/packages/db/src/agent-tasks.ts@@ -29,6 +29,22 @@ export function isDirectKind(kind: string): kind is DirectKind {return (DIRECT_KINDS as readonly string[]).includes(kind);}+export const ENRICHMENT_KINDS = [+ "brand",+ "portrait",+ "identify",+ "profile",+ "recheck",+ "company-profile",+ "workspace-profile",+] as const;++export type EnrichmentKind = (typeof ENRICHMENT_KINDS)[number];++export function isEnrichmentKind(kind: string): kind is EnrichmentKind {+ return (ENRICHMENT_KINDS as readonly string[]).includes(kind);+}+export const MAX_ATTEMPTS = 3;export const RETIRED_OUTCOME = `Gave up after ${MAX_ATTEMPTS} attempts: the session never reported back.`;diff --git a/packages/ui/src/components/popover.tsx b/packages/ui/src/components/popover.tsxindex 6420634e..b43c12ef 100644--- a/packages/ui/src/components/popover.tsx+++ b/packages/ui/src/components/popover.tsx@@ -25,6 +25,7 @@ const popoverContentVariants = cva(size: {default: "w-72 gap-2.5 p-2.5",fit: "w-auto overflow-hidden p-0",+ panel: "w-90 overflow-hidden p-0",},},defaultVariants: { size: "default" },