Skip to content

armgit / crm

Public Imported from GitHub
0

Comp AI CRM is an open source, CRM designed for AI agents. Agentic-first CRM.

TypeScript

feat(agent): apply sourced facts to empty fields automatically (#112)

github-actions[bot]0342c8ee6256
diff --git a/apps/agent/agent/channels/crm.ts b/apps/agent/agent/channels/crm.ts
index 43faef02..665438d8 100644
--- a/apps/agent/agent/channels/crm.ts
+++ b/apps/agent/agent/channels/crm.ts
@@ -1,6 +1,7 @@
import { timingSafeEqual } from "node:crypto";
import { EnrichmentStatus } from "@crm/db";
import { defineChannel, POST } from "eve/channels";
+import { sweepBlankFacts } from "../lib/blank-facts";
import { verifyKey } from "../lib/context-dev";
import {
builderIdFromToken,
@@ -87,6 +88,22 @@ export default defineChannel({
},
),
+ POST("/internal/crm/apply-blank-facts", async (request) => {
+ if (!authorised(request)) {
+ return new Response("Unauthorized", { status: 401 });
+ }
+
+ const sweep = await sweepBlankFacts();
+
+ return Response.json({
+ scanned: sweep.scanned,
+ filled: sweep.filled,
+ settled: sweep.settled,
+ waiting: sweep.waiting,
+ unscanned: sweep.unscanned,
+ });
+ }),
+
POST("/internal/crm/verify-key", async (request) => {
if (!authorised(request)) {
return new Response("Unauthorized", { status: 401 });
diff --git a/apps/agent/agent/lib/blank-facts.ts b/apps/agent/agent/lib/blank-facts.ts
new file mode 100644
index 00000000..c3e3d8a4
--- /dev/null
+++ b/apps/agent/agent/lib/blank-facts.ts
@@ -0,0 +1,237 @@
+import { db, FactStatus } from "@crm/db";
+import {
+ type FactField,
+ type FactSubject,
+ factColumn,
+ fillsBlank,
+} from "./facts";
+import { splitName } from "./names";
+
+const SCAN = 2000;
+
+const MAX_FILLS = 500;
+
+const CONTACT_SELECT = {
+ id: true,
+ email: true,
+ firstName: true,
+ lastName: true,
+ title: true,
+ linkedinUrl: true,
+ twitterUrl: true,
+ githubUrl: true,
+} as const;
+
+export type BlankFactFill = {
+ contactId: string;
+ contact: string;
+ field: FactField;
+ value: string;
+ score: number;
+ dropped: number;
+};
+
+export type BlankFactSweep = {
+ scanned: number;
+ filled: number;
+ settled: number;
+ waiting: number;
+ unscanned: number;
+ fills: BlankFactFill[];
+};
+
+export async function sweepBlankFacts(
+ options: { dry?: boolean } = {},
+): Promise<BlankFactSweep> {
+ const [pending, proposals] = await Promise.all([
+ db.contactFact.count({ where: { status: FactStatus.PROPOSED } }),
+ db.contactFact.findMany({
+ where: { status: FactStatus.PROPOSED },
+ select: {
+ id: true,
+ contactId: true,
+ field: true,
+ value: true,
+ score: true,
+ contact: { select: CONTACT_SELECT },
+ },
+ orderBy: [{ score: "desc" }, { observedAt: "desc" }],
+ take: SCAN,
+ }),
+ ]);
+
+ const sweep: BlankFactSweep = {
+ scanned: proposals.length,
+ filled: 0,
+ settled: 0,
+ waiting: 0,
+ unscanned: Math.max(0, pending - proposals.length),
+ fills: [],
+ };
+
+ if (proposals.length === 0) return sweep;
+
+ const applied = await appliedValues([
+ ...new Set(proposals.map((row) => row.contactId)),
+ ]);
+
+ for (const group of groupByField(proposals)) {
+ const [best] = group;
+ const field = best.field as FactField;
+ const contact = best.contact as FactSubject;
+ const column = factColumn(field);
+ const current = applied.get(key(best.contactId, field));
+
+ if (!fillsBlank({ field, contact, hasAgentFact: current !== undefined })) {
+ const value =
+ current ?? (column ? (contact[column] as string | null) : null);
+ const stale = redundant(group, value);
+
+ sweep.waiting += group.length - stale.length;
+ if (stale.length === 0) continue;
+
+ if (!options.dry) {
+ await db.contactFact.updateMany({
+ where: { id: { in: stale.map((row) => row.id) } },
+ data: { status: FactStatus.SUPERSEDED, supersededAt: new Date() },
+ });
+ }
+
+ sweep.settled += stale.length;
+ continue;
+ }
+
+ if (sweep.filled >= MAX_FILLS) {
+ sweep.waiting += group.length;
+ continue;
+ }
+
+ if (!options.dry) {
+ await fill(best.id, best.contactId, field, best.value, column);
+ }
+
+ sweep.filled += 1;
+ sweep.settled += group.length - 1;
+ sweep.fills.push({
+ contactId: best.contactId,
+ contact: [contact.firstName, contact.lastName].filter(Boolean).join(" "),
+ field,
+ value: best.value,
+ score: best.score,
+ dropped: group.length - 1,
+ });
+ }
+
+ return sweep;
+}
+
+type Proposal = {
+ id: string;
+ contactId: string;
+ field: string;
+ value: string;
+ score: number;
+ contact: {
+ id: string;
+ email: string | null;
+ firstName: string;
+ lastName: string | null;
+ title: string | null;
+ linkedinUrl: string | null;
+ twitterUrl: string | null;
+ githubUrl: string | null;
+ };
+};
+
+async function appliedValues(
+ contactIds: string[],
+): Promise<Map<string, string>> {
+ const values = new Map<string, string>();
+
+ for (let start = 0; start < contactIds.length; start += 1000) {
+ const rows = await db.contactFact.findMany({
+ where: {
+ status: FactStatus.APPLIED,
+ contactId: { in: contactIds.slice(start, start + 1000) },
+ },
+ select: { contactId: true, field: true, value: true },
+ });
+
+ for (const row of rows)
+ values.set(key(row.contactId, row.field), row.value);
+ }
+
+ return values;
+}
+
+function groupByField(proposals: Proposal[]): [Proposal, ...Proposal[]][] {
+ const groups = new Map<string, [Proposal, ...Proposal[]]>();
+
+ for (const row of proposals) {
+ const id = key(row.contactId, row.field);
+ const group = groups.get(id);
+ if (group) group.push(row);
+ else groups.set(id, [row]);
+ }
+
+ return [...groups.values()];
+}
+
+function redundant(group: Proposal[], value: string | null): Proposal[] {
+ const kept = new Set<string>();
+
+ return group.filter((row) => {
+ const seen = row.value.trim().toLowerCase();
+ if (value && seen === value.trim().toLowerCase()) return true;
+ if (kept.has(seen)) return true;
+
+ kept.add(seen);
+ return false;
+ });
+}
+
+async function fill(
+ factId: string,
+ contactId: string,
+ field: FactField,
+ value: string,
+ column: string | null,
+): Promise<void> {
+ await db.$transaction(async (tx) => {
+ await tx.contactFact.updateMany({
+ where: {
+ contactId,
+ field,
+ id: { not: factId },
+ status: { in: [FactStatus.APPLIED, FactStatus.PROPOSED] },
+ },
+ data: { status: FactStatus.SUPERSEDED, supersededAt: new Date() },
+ });
+
+ await tx.contactFact.update({
+ where: { id: factId },
+ data: { status: FactStatus.APPLIED },
+ });
+
+ if (column) {
+ await tx.contact.update({
+ where: { id: contactId },
+ data: { [column]: value },
+ });
+ }
+
+ if (field === "name") {
+ const split = splitName(value);
+ if (split) {
+ await tx.contact.update({
+ where: { id: contactId },
+ data: { firstName: split.firstName, lastName: split.lastName },
+ });
+ }
+ }
+ });
+}
+
+function key(contactId: string, field: string): string {
+ return `${contactId}:${field}`;
+}
diff --git a/apps/agent/agent/lib/facts.ts b/apps/agent/agent/lib/facts.ts
index 3b8a866c..83bda180 100644
--- a/apps/agent/agent/lib/facts.ts
+++ b/apps/agent/agent/lib/facts.ts
@@ -20,6 +20,27 @@ export type FactField = keyof typeof FIELDS;
export const FACT_FIELDS = Object.keys(FIELDS) as FactField[];
+export type FactSubject = {
+ email: string | null;
+ firstName: string;
+ lastName: string | null;
+} & Record<string, unknown>;
+
+export function factColumn(field: FactField): string | null {
+ return FIELDS[field].column;
+}
+
+export function fillsBlank(input: {
+ field: FactField;
+ contact: FactSubject;
+ hasAgentFact: boolean;
+}): boolean {
+ const column = FIELDS[input.field].column;
+ if (humanOwns({ ...input, column })) return false;
+
+ return isEmpty({ ...input, column });
+}
+
export type RecordFactInput = {
contactId: string;
field: FactField;
@@ -122,25 +143,47 @@ export async function recordFact(
}
const column = FIELDS[field].column;
+ const hasAgentFact = Boolean(currentApplied);
+
+ if (humanOwns({ field, column, contact, hasAgentFact })) {
+ return {
+ ...base,
+ stored: false,
+ applied: false,
+ reason: `A person already filled in ${field}. That outranks anything found on the web.`,
+ };
+ }
+
+ const applies =
+ scored.band === FactBand.VERIFIED ||
+ fillsBlank({ field, contact, hasAgentFact });
if (
- humanOwns({ field, column, contact, hasAgentFact: Boolean(currentApplied) })
+ !applies &&
+ existing.some(
+ (fact) =>
+ fact.status === FactStatus.PROPOSED && sameValue(fact.value, trimmed),
+ )
) {
return {
...base,
stored: false,
applied: false,
- reason: `A person already filled in ${field}. That outranks anything found on the web.`,
+ reason:
+ "This exact value is already in front of a rep, waiting on them. Offering it twice only makes them read it twice.",
};
}
- const applies = scored.band === FactBand.VERIFIED;
const sessionId = currentFocus().sessionId;
await db.$transaction(async (tx) => {
- if (applies && currentApplied) {
- await tx.contactFact.update({
- where: { id: currentApplied.id },
+ if (applies) {
+ await tx.contactFact.updateMany({
+ where: {
+ contactId,
+ field,
+ status: { in: [FactStatus.APPLIED, FactStatus.PROPOSED] },
+ },
data: { status: FactStatus.SUPERSEDED, supersededAt: new Date() },
});
}
@@ -186,7 +229,7 @@ export async function recordFact(
applied: applies,
reason: applies
? undefined
- : "Kept as a proposal for a rep to accept or dismiss. This is a normal outcome, not a failure — do not try to raise the score.",
+ : "The record already carries a value here, and only VERIFIED evidence may replace one, so this is kept as a proposal for a rep to accept or dismiss. This is a normal outcome, not a failure — do not try to raise the score.",
};
}
@@ -284,6 +327,24 @@ function humanOwns({
return Boolean(contact[column]);
}
+function isEmpty({
+ field,
+ column,
+ contact,
+ hasAgentFact,
+}: {
+ field: FactField;
+ column: string | null;
+ contact: Record<string, unknown>;
+ hasAgentFact: boolean;
+}): boolean {
+ if (hasAgentFact) return false;
+ if (field === "name") return true;
+ if (!column) return true;
+
+ return !contact[column];
+}
+
function sameValue(a: string, b: string): boolean {
return a.trim().toLowerCase() === b.trim().toLowerCase();
}
diff --git a/apps/agent/package.json b/apps/agent/package.json
index b65d0454..d7c6fa60 100644
--- a/apps/agent/package.json
+++ b/apps/agent/package.json
@@ -5,6 +5,7 @@
"type": "module",
"license": "MIT",
"scripts": {
+ "backfill:facts": "bun scripts/backfill-empty-facts.ts",
"backfill:images": "bun scripts/backfill-brand-images.ts",
"dev": "eve dev",
"dev:headless": "eve dev --no-ui",
diff --git a/apps/agent/scripts/backfill-empty-facts.ts b/apps/agent/scripts/backfill-empty-facts.ts
new file mode 100644
index 00000000..87bfa07a
--- /dev/null
+++ b/apps/agent/scripts/backfill-empty-facts.ts
@@ -0,0 +1,33 @@
+import { db } from "@crm/db";
+import { sweepBlankFacts } from "../agent/lib/blank-facts";
+
+const dry = process.argv.includes("--dry");
+
+const sweep = await sweepBlankFacts({ dry });
+
+for (const fill of sweep.fills) {
+ const dropped =
+ fill.dropped > 0
+ ? ` — dropped ${fill.dropped} weaker suggestion${fill.dropped === 1 ? "" : "s"}`
+ : "";
+
+ console.log(
+ `${fill.contact} · ${fill.field}: "${fill.value}" (${fill.score.toFixed(2)})${dropped}`,
+ );
+}
+
+console.log(
+ `\n${dry ? "Would fill" : "Filled"} ${count(sweep.filled, "empty field")} from ${count(sweep.scanned, "waiting suggestion")}. ${count(sweep.settled, "stale suggestion")} cleared, ${sweep.waiting} still need a rep.`,
+);
+
+if (sweep.unscanned > 0) {
+ console.log(
+ `${sweep.unscanned} suggestions were not looked at this pass. Run it again.`,
+ );
+}
+
+await db.$disconnect();
+
+function count(n: number, noun: string): string {
+ return `${n} ${noun}${n === 1 ? "" : "s"}`;
+}
diff --git a/apps/agent/test/blank-facts.integration.spec.ts b/apps/agent/test/blank-facts.integration.spec.ts
new file mode 100644
index 00000000..97a371d5
--- /dev/null
+++ b/apps/agent/test/blank-facts.integration.spec.ts
@@ -0,0 +1,150 @@
+import { afterAll, beforeEach, describe, expect, it } from "bun:test";
+import { db, FactBand, FactStatus } from "@crm/db";
+import { sweepBlankFacts } from "../agent/lib/blank-facts";
+
+const suffix = process.env.TEST_RUN_ID ?? "blank-facts-spec";
+const email = `blank.subject.${suffix}@example.test`;
+
+let contactId: string;
+
+async function propose(input: {
+ field: string;
+ value: string;
+ score: number;
+}): Promise<string> {
+ const fact = await db.contactFact.create({
+ data: {
+ contactId,
+ field: input.field,
+ value: input.value,
+ score: input.score,
+ band: input.score >= 0.55 ? FactBand.PROBABLE : FactBand.POSSIBLE,
+ evidence: [{ kind: "web.cited-claim", detail: "a page said so" }],
+ method: "web",
+ status: FactStatus.PROPOSED,
+ },
+ select: { id: true },
+ });
+
+ return fact.id;
+}
+
+function statusOf(id: string) {
+ return db.contactFact
+ .findUnique({ where: { id }, select: { status: true } })
+ .then((fact) => fact?.status);
+}
+
+beforeEach(async () => {
+ await db.contact.deleteMany({ where: { email } });
+ const contact = await db.contact.create({
+ data: { firstName: "Blank", lastName: "Subject", email },
+ select: { id: true },
+ });
+ contactId = contact.id;
+});
+
+afterAll(async () => {
+ await db.contact.deleteMany({ where: { email } });
+});
+
+describe("sweepBlankFacts", () => {
+ it("fills an empty field from the best-evidenced suggestion", async () => {
+ const weak = await propose({
+ field: "linkedinUrl",
+ value: "https://www.linkedin.com/in/maybe",
+ score: 0.35,
+ });
+ const strong = await propose({
+ field: "linkedinUrl",
+ value: "https://www.linkedin.com/in/subject",
+ score: 0.61,
+ });
+
+ const sweep = await sweepBlankFacts();
+
+ expect(sweep.filled).toBe(1);
+ expect(await statusOf(strong)).toBe("APPLIED");
+ expect(await statusOf(weak)).toBe("SUPERSEDED");
+
+ const contact = await db.contact.findUnique({
+ where: { id: contactId },
+ select: { linkedinUrl: true },
+ });
+ expect(contact?.linkedinUrl).toBe("https://www.linkedin.com/in/subject");
+ });
+
+ it("leaves a suggestion that disagrees with what is already there", async () => {
+ await db.contact.update({
+ where: { id: contactId },
+ data: { title: "Head of Security" },
+ });
+
+ const offer = await propose({
+ field: "title",
+ value: "VP Security",
+ score: 0.61,
+ });
+
+ const sweep = await sweepBlankFacts();
+
+ expect(sweep.filled).toBe(0);
+ expect(sweep.waiting).toBe(1);
+ expect(await statusOf(offer)).toBe("PROPOSED");
+ });
+
+ it("clears a suggestion that only repeats the record", async () => {
+ await db.contact.update({
+ where: { id: contactId },
+ data: { title: "Head of Security" },
+ });
+
+ const echo = await propose({
+ field: "title",
+ value: "head of security",
+ score: 0.61,
+ });
+
+ const sweep = await sweepBlankFacts();
+
+ expect(sweep.settled).toBe(1);
+ expect(sweep.waiting).toBe(0);
+ expect(await statusOf(echo)).toBe("SUPERSEDED");
+ });
+
+ it("keeps one of two suggestions offering the same value", async () => {
+ await db.contact.update({
+ where: { id: contactId },
+ data: { title: "Head of Security" },
+ });
+
+ await propose({ field: "title", value: "VP Security", score: 0.61 });
+ await propose({ field: "title", value: "VP Security", score: 0.61 });
+
+ await sweepBlankFacts();
+
+ const left = await db.contactFact.count({
+ where: { contactId, field: "title", status: FactStatus.PROPOSED },
+ });
+ expect(left).toBe(1);
+ });
+
+ it("counts without writing when it is only reading", async () => {
+ const offer = await propose({
+ field: "githubUrl",
+ value: "https://github.com/subject",
+ score: 0.4,
+ });
+
+ const sweep = await sweepBlankFacts({ dry: true });
+
+ expect(sweep.filled).toBe(1);
+ expect(await statusOf(offer)).toBe("PROPOSED");
+
+ const contact = await db.contact.findUnique({
+ where: { id: contactId },
+ select: { githubUrl: true },
+ });
+ expect(contact?.githubUrl).toBeNull();
+ });
+});
diff --git a/apps/agent/test/facts.integration.spec.ts b/apps/agent/test/facts.integration.spec.ts
index c84fab2b..4c9fc42e 100644
--- a/apps/agent/test/facts.integration.spec.ts
+++ b/apps/agent/test/facts.integration.spec.ts
@@ -46,7 +46,7 @@ describe("recordFact", () => {
expect(contact?.title).toBe("Head of Security");
});
- it("keeps a probable fact off the record and offers it instead", async () => {
+ it("fills an empty field without asking anybody", async () => {
const result = await recordFact({
contactId,
field: "twitterUrl",
@@ -55,15 +55,85 @@ describe("recordFact", () => {
method: "x.handle+citation",
});
+ expect(result.applied).toBe(true);
+ expect(result.band).toBe("PROBABLE");
+
+ const contact = await db.contact.findUnique({
+ where: { id: contactId },
+ select: { twitterUrl: true },
+ });
+ expect(contact?.twitterUrl).toBe("https://x.com/subject");
+ });
+
+ it("fills a field the record keeps no column for", async () => {
+ const result = await recordFact({
+ contactId,
+ field: "location",
+ value: "Brooklyn, NY",
+ evidence: [seen("web.cited-claim")],
+ method: "web",
+ });
+
+ expect(result.applied).toBe(true);
+ expect(result.band).toBe("POSSIBLE");
+ });
+
+ it("offers rather than replaces a value it already found", async () => {
+ const result = await recordFact({
+ contactId,
+ field: "twitterUrl",
+ value: "https://x.com/someone-else",
+ evidence: [seen("handle.name-form"), seen("search.cites-profile")],
+ method: "x.handle+citation",
+ });
+
expect(result.stored).toBe(true);
expect(result.applied).toBe(false);
- expect(result.band).toBe("PROBABLE");
+ expect(result.reason).toContain("already carries a value");
const contact = await db.contact.findUnique({
where: { id: contactId },
select: { twitterUrl: true },
});
- expect(contact?.twitterUrl).toBeNull();
+ expect(contact?.twitterUrl).toBe("https://x.com/subject");
+ });
+
+ it("puts the same suggestion in front of a rep only once", async () => {
+ const result = await recordFact({
+ contactId,
+ field: "twitterUrl",
+ value: "https://x.com/someone-else",
+ evidence: [seen("web.cited-claim"), seen("search.cites-profile")],
+ method: "web",
+ });
+
+ expect(result.stored).toBe(false);
+ expect(result.reason).toContain("already in front of a rep");
+
+ const offers = await db.contactFact.count({
+ where: { contactId, field: "twitterUrl", status: "PROPOSED" },
+ });
+ expect(offers).toBe(1);
+ });
+
+ it("settles the suggestions a stronger fact has answered", async () => {
+ await recordFact({
+ contactId,
+ field: "twitterUrl",
+ value: "https://x.com/the-real-one",
+ evidence: [seen("profile.email-match")],
+ method: "x.profile",
+ });
+
+ const facts = await db.contactFact.findMany({
+ where: { contactId, field: "twitterUrl" },
+ select: { value: true, status: true },
+ });
+
+ expect(facts.filter((fact) => fact.status === "PROPOSED")).toHaveLength(0);
+ expect(
+ facts.find((f) => f.value === "https://x.com/the-real-one")?.status,
+ ).toBe("APPLIED");
});
it("never overwrites what a person typed", async () => {
@@ -91,8 +161,17 @@ describe("recordFact", () => {
});
it("never re-offers a value a person dismissed", async () => {
+ const offer = await recordFact({
+ contactId,
+ field: "title",
+ value: "Head of Trust",
+ evidence: [seen("web.cited-claim"), seen("search.cites-profile")],
+ method: "web",
+ });
+ expect(offer.applied).toBe(false);
+
const proposal = await db.contactFact.findFirst({
- where: { contactId, field: "twitterUrl" },
+ where: { contactId, field: "title", status: "PROPOSED" },
select: { id: true },
});
await db.contactFact.update({
@@ -102,10 +181,10 @@ describe("recordFact", () => {
const result = await recordFact({
contactId,
- field: "twitterUrl",
- value: "https://x.com/subject",
- evidence: [seen("handle.name-form"), seen("search.cites-profile")],
- method: "x.handle+citation",
+ field: "title",
+ value: "Head of Trust",
+ evidence: [seen("web.cited-claim"), seen("search.cites-profile")],
+ method: "web",
});
expect(result.stored).toBe(false);
@@ -139,6 +218,38 @@ describe("recordFact", () => {
expect(facts.find((f) => f.value === "Comp AI")?.status).toBe("APPLIED");
});
+ it("lets verified evidence settle the suggestion it matches", async () => {
+ const offer = await recordFact({
+ contactId,
+ field: "employer",
+ value: "Fleetio Inc",
+ evidence: [seen("web.cited-claim"), seen("search.cites-profile")],
+ method: "web",
+ });
+ expect(offer.applied).toBe(false);
+
+ const result = await recordFact({
+ contactId,
+ field: "employer",
+ value: "Fleetio Inc",
+ evidence: [seen("linkedin.employer-and-name")],
+ method: "linkedin.profile",
+ });
+
+ expect(result.stored).toBe(true);
+ expect(result.applied).toBe(true);
+
+ const facts = await db.contactFact.findMany({
+ where: { contactId, field: "employer" },
+ select: { value: true, status: true, score: true },
+ });
+
+ const live = facts.filter((fact) => fact.status === "APPLIED");
+ expect(live).toHaveLength(1);
+ expect(live[0]?.value).toBe("Fleetio Inc");
+ expect(facts.filter((fact) => fact.status === "PROPOSED")).toHaveLength(0);
+ });
+
it("stores the evidence, so the score can be explained later", async () => {
const fact = await db.contactFact.findFirst({
where: { contactId, field: "title" },
diff --git a/apps/api/.scratch/shape.ts b/apps/api/.scratch/shape.ts
index b3f2ff37..dae49c71 100644
--- a/apps/api/.scratch/shape.ts
+++ b/apps/api/.scratch/shape.ts
@@ -1,5 +1,4 @@
import { initTRPC } from "@trpc/server";
-import { z } from "zod";
import { setResearchKeyInput } from "../src/settings/settings.contracts";
import { formatTrpcError } from "../src/trpc/error-formatter";
diff --git a/apps/api/src/agent/agent.module.ts b/apps/api/src/agent/agent.module.ts
index 2cdae2c2..d0c489ae 100644
--- a/apps/api/src/agent/agent.module.ts
+++ b/apps/api/src/agent/agent.module.ts
@@ -6,6 +6,7 @@ import { AgentQueueService } from "./agent-queue.service";
import { AgentRunsService } from "./agent-runs.service";
import { AgentTriggerService } from "./agent-trigger.service";
import { AgentsRouter } from "./agents.router";
+import { BlankFactsService } from "./blank-facts.service";
import { ResearchKeyService } from "./research-key.service";
@Module({
@@ -17,8 +18,14 @@ import { ResearchKeyService } from "./research-key.service";
AgentRunsService,
AgentTriggerService,
AgentsRouter,
+ BlankFactsService,
+ ResearchKeyService,
+ ],
+ exports: [
+ AgentTriggerService,
+ AgentQueueService,
+ BlankFactsService,
ResearchKeyService,
],
- exports: [AgentTriggerService, AgentQueueService, ResearchKeyService],
})
export class AgentModule {}
diff --git a/apps/api/src/agent/blank-facts.service.ts b/apps/api/src/agent/blank-facts.service.ts
new file mode 100644
index 00000000..bcd7786e
--- /dev/null
+++ b/apps/api/src/agent/blank-facts.service.ts
@@ -0,0 +1,64 @@
+import { Injectable, Logger } from "@nestjs/common";
+import { bridge } from "./bridge";
+
+const SWEEP_TIMEOUT_MS = 60_000;
+
+export type BlankFactSweep = {
+ scanned: number;
+ filled: number;
+ settled: number;
+ waiting: number;
+ unscanned: number;
+};
+
+const NOTHING: BlankFactSweep = {
+ scanned: 0,
+ filled: 0,
+ settled: 0,
+ waiting: 0,
+ unscanned: 0,
+};
+
+/**
+ * Applying a suggestion to a field nobody has filled is the agent's rule, in
+ * the agent's code — `fillsBlank` knows what a human owns and what a derived
+ * name is, and a second copy of that here is the thing `docs/api.md` forbids.
+ * So this asks, and reports what came back.
+ */
+@Injectable()
+export class BlankFactsService {
+ private readonly logger = new Logger(BlankFactsService.name);
+
+ async sweep(): Promise<BlankFactSweep> {
+ const agent = bridge();
+ if (!agent) return NOTHING;
+
+ try {
+ const response = await fetch(
+ agent.url("/internal/crm/apply-blank-facts"),
+ {
+ method: "POST",
+ headers: { authorization: `Bearer ${agent.secret}` },
+ signal: AbortSignal.timeout(SWEEP_TIMEOUT_MS),
+ },
+ );
+
+ if (!response.ok) {
+ this.unreachable(`The agent answered ${response.status}.`);
+ return NOTHING;
+ }
+
+ return { ...NOTHING, ...((await response.json()) as BlankFactSweep) };
+ } catch (error) {
+ this.unreachable(error instanceof Error ? error.message : String(error));
+ return NOTHING;
+ }
+ }
+
+ private unreachable(reason: string): void {
+ this.logger.warn({
+ message: "Could not apply suggestions to empty fields",
+ reason,
+ });
+ }
+}
diff --git a/apps/api/src/backfill/backfill.service.ts b/apps/api/src/backfill/backfill.service.ts
index 8f4b9607..33fcc07d 100644
--- a/apps/api/src/backfill/backfill.service.ts
+++ b/apps/api/src/backfill/backfill.service.ts
@@ -6,6 +6,7 @@ import { CACHE_MANAGER } from "@nestjs/cache-manager";
import { Inject, Injectable, Logger, type OnModuleInit } from "@nestjs/common";
import type { Cache } from "cache-manager";
import { AgentTriggerService } from "../agent/agent-trigger.service";
+import { BlankFactsService } from "../agent/blank-facts.service";
import { FaviconService } from "../companies/favicon.service";
import { InjectDatabase } from "../database/database.constants";
import { ImageMirrorService } from "./image-mirror.service";
@@ -53,6 +54,7 @@ export class BackfillService implements OnModuleInit {
private readonly agent: AgentTriggerService,
private readonly favicon: FaviconService,
private readonly images: ImageMirrorService,
+ private readonly facts: BlankFactsService,
@Inject(CACHE_MANAGER) private readonly cache: Cache,
) {}
@@ -74,6 +76,7 @@ export class BackfillService implements OnModuleInit {
const contacts = await this.runContacts();
const mirrored = await this.images.sweep();
+ const facts = await this.facts.sweep();
this.logger.log({
message: "Automatic backfill swept",
@@ -81,6 +84,9 @@ export class BackfillService implements OnModuleInit {
remaining: companies.remaining + contacts.remaining,
iconsResolving: companies.iconsResolving,
imagesMirrored: mirrored.copied,
+ blankFieldsFilled: facts.filled,
+ suggestionsSettled: facts.settled,
+ suggestionsWaiting: facts.waiting + facts.unscanned,
});
} catch (error) {
this.logger.error(
diff --git a/apps/api/src/contacts/contacts.service.ts b/apps/api/src/contacts/contacts.service.ts
index 9f7df4dd..9060cc3b 100644
--- a/apps/api/src/contacts/contacts.service.ts
+++ b/apps/api/src/contacts/contacts.service.ts
@@ -623,7 +623,8 @@ export class ContactsService {
where: {
contactId: fact.contactId,
field: fact.field,
- status: FactStatus.APPLIED,
+ id: { not: fact.id },
+ status: { in: [FactStatus.APPLIED, FactStatus.PROPOSED] },
},
data: { status: FactStatus.SUPERSEDED, supersededAt: new Date() },
});
diff --git a/apps/app/app/(app)/[slug]/settings/currencies/currency-settings.tsx b/apps/app/app/(app)/[slug]/settings/currencies/currency-settings.tsx
index c47ec66a..5d29edaa 100644
--- a/apps/app/app/(app)/[slug]/settings/currencies/currency-settings.tsx
+++ b/apps/app/app/(app)/[slug]/settings/currencies/currency-settings.tsx
@@ -11,7 +11,6 @@ import {
CardTitle,
} from "@crm/ui/components/card";
import { CardTableEmpty } from "@crm/ui/components/card-table";
-import { EmptyCellValue } from "@crm/ui/components/empty-cell";
import { Field, FieldDescription, FieldLabel } from "@crm/ui/components/field";
import { Input } from "@crm/ui/components/input";
import {
diff --git a/docs/agent.md b/docs/agent.md
index 6003b621..639422d5 100644
--- a/docs/agent.md
+++ b/docs/agent.md
@@ -89,14 +89,36 @@ task row; exists because the API may not call Context.
- **`classifyKey` rejects on `401` and nothing else.**
- **The candidate key, never the stored one.**
+### `POST /internal/crm/apply-blank-facts`
+
+Applies every pending suggestion whose field is still empty, and clears the ones that
+have stopped saying anything. No session, no model, no task row, no credits —
+`sweepBlankFacts` (`lib/blank-facts.ts`) is a database pass. It exists as a route for
+the same reason `verify-key` does: the rule is `fillsBlank`, the rule lives here, and a
+second copy of it in Nest is what `api.md` forbids. `BlankFactsService` carries the
+question and reports the counts.
+
+- **Scans 2000 suggestions and fills at most 500 a pass**, and says what it did not
+ reach (`unscanned`) rather than reporting a clean sweep it did not make.
+- **Idempotent** — a second pass over the same rows fills nothing, because they are no
+ longer blank.
+- **The suggestions left are conflicts**, every one of them against a value already on
+ the record. That number should be small and should be read as work for a rep.
+
### Backfills
Sign-in sweep covers records never looked up (10 credits/company);
`ImageMirrorService` in the same sweep re-hosts off-site pictures (free);
-`backfill:images` fixes enriched records missing only pictures (free).
+the same sweep asks `apply-blank-facts` above (free);
+`backfill:images` fixes enriched records missing only pictures (free);
+`backfill:facts` is the same fact sweep run by hand, with `--dry` to read it first.
- **The image sweep keeps "every picture is ours" true**, not true-since-Tuesday.
25 rows/table/sweep.
+- **Nobody clicks a suggestion into a blank field.** The rule is enforced at the write
+ path, so no queue forms; the sweep is for rows written before it existed and for the
+ field a rep clears while a suggestion is pending. **An unreachable agent leaves the
+ suggestions where they are** — the API cannot apply one itself.
- **A finished `portrait` task stands that contact down for thirty days** — that third
source costs credits and usually finds nothing.
- **No button, deliberately** — a rep cannot know which records predate a resolver.
@@ -117,7 +139,23 @@ wrong in the direction that looks useful.
- **`lib/facts.ts` is the only write path to a contact's fields.** Applies at
`VERIFIED`, proposes below it, and enforces three things a prompt cannot: never
overwrite a human, never re-offer a dismissal, never write without a primary source.
-- **Bands are behaviour.** `PROBABLE` means *a rep decides* — a correct outcome.
+- **A band decides only when there is something to lose.** An empty field is filled by
+ whatever cleared the floor for keeping, whatever the band — approving a sourced guess
+ into a blank is a click that can only say yes, and a rep with four hundred contacts
+ reads none of them. `fillsBlank` is the whole rule: no human value in the way and
+ nothing already found. **`PROBABLE` still means *a rep decides* when the field is
+ already filled**, which is the case where a wrong answer costs something. The sign-in
+ sweep applies the same rule to rows that predate it —
+ `POST /internal/crm/apply-blank-facts`, above.
+- **Applying settles the field's other suggestions.** They were all offers to fill the
+ same blank, and the sheet shows one at a time — so left alone, accepting one reveals
+ the next, forever. The same rule holds when a rep accepts one (`decideFact`).
+- **The same value is never offered twice.** A second `PROPOSED` row with a value
+ already waiting is refused at the write path, not deduplicated on read. **The refusal
+ is for offers only** — the check runs after `applies` is decided, so evidence that has
+ reached `VERIFIED` since the offer was made still lands, settles the suggestion it
+ matches, and replaces the older value. Refusing it there left a weaker value on the
+ record with the answer sitting unread beneath it.
- **A new fact field goes in `FIELDS` (`lib/facts.ts`) *and* `FACT_COLUMNS`**
(`apps/api/src/contacts/contacts.service.ts`).
diff --git a/turbo.json b/turbo.json
index ed196c2e..c7cdc3ca 100644
--- a/turbo.json
+++ b/turbo.json
@@ -23,10 +23,20 @@
* the separately-declared public copy — so they must not become part of the
* cache key. A secret in the cache key means a cache miss every rotation and
* a secret in the cache metadata.
+ *
+ * `TEST_DATABASE_URL` is here for the same reason and learned the same way.
+ * `NODE_ENV` is in `globalEnv`, so `bun test` sets it to `test` in every
+ * package, and `@crm/db` then demands the test database and refuses to fall
+ * back. Undeclared, it was stripped from every test task — so the suite went
+ * red on a machine that had the variable set, complaining it was not set.
+ * Locally it passed throughout, because `@crm/env/load` reads the root `.env`
+ * from disk and CI has no such file.
*/
"globalPassThroughEnv": [
"DATABASE_URL",
"DATABASE_URL_UNPOOLED",
+ "TEST_DATABASE_URL",
+ "TEST_RUN_ID",
"BETTER_AUTH_SECRET",
"BETTER_AUTH_URL",
"ALLOWED_SIGN_IN",