Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

In-app notifications for monitor changes and dashboard crawls (#4928)

Nicolas66ebd15b5da3
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 0b584ce6f..4de3f5591 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -362,6 +362,7 @@ export async function batchScrapeController(
zeroDataRetention,
v1: true,
webhook: req.body.webhook,
+ origin: req.body.origin,
};
if (req.body.appendToId && (!sc || sc.team_id !== req.auth.team_id)) {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 74fe3eea4..7d2deb81b 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -291,6 +291,7 @@ export async function crawlController(
zeroDataRetention,
v1: true,
webhook: req.body.webhook,
+ origin: req.body.origin,
};
const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 3ab5ba6c2..6e9cd196c 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -417,6 +417,7 @@ export async function batchScrapeController(
v1: true,
webhook: req.body.webhook,
requestId: req.body.__agentInterop?.requestId ?? undefined,
+ origin: req.body.origin,
};
if (req.body.appendToId) {
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 4e8b408b0..34dbd6f68 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -357,6 +357,7 @@ export async function crawlController(
zeroDataRetention,
v1: true,
webhook: req.body.webhook,
+ origin: req.body.origin,
};
const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 1132a920c..46b29ea74 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -37,6 +37,9 @@ export type StoredCrawl = {
v1?: boolean;
webhook?: WebhookConfig;
requestId?: string;
+ // The request's `origin`; dashboard-started jobs get an in-app notification
+ // when they finish.
+ origin?: string;
};
export async function saveCrawl(id: string, crawl: StoredCrawl) {
diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.ts
index 0fd2c8b43..6dc43a185 100644
--- a/apps/api/src/services/monitoring/runner.finalization.test.ts
+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts
@@ -21,8 +21,13 @@ vi.mock("../queue-jobs", () => ({}));
vi.mock("../../controllers/v2/types", () => ({}));
vi.mock("../webhook", () => ({}));
vi.mock("./results", () => ({}));
-vi.mock("../notification/monitoring_email", () => ({}));
+vi.mock("../notification/monitoring_email", () => ({
+ sendMonitoringEmailSummary: vi.fn(),
+}));
vi.mock("../notification/monitoring_slack", () => ({}));
+vi.mock("../notification/monitoring_in_app", () => ({
+ recordMonitorInAppNotification: vi.fn(),
+}));
vi.mock("./types", () => ({}));
vi.mock("./interest", () => ({
trackMonitorCheckStartedInterest: async () => {},
@@ -42,6 +47,7 @@ vi.mock("./store", () => ({
updateMonitorCheckIfStatus: vi.fn(),
markMonitorRunning: vi.fn(),
countMonitorCheckPages: vi.fn(),
+ listMonitorCheckPages: vi.fn(),
calculateMonitorCheckActualCredits: vi.fn(),
updateMonitorScheduleAfterRun: vi.fn(),
}));
@@ -63,6 +69,8 @@ import { getACUCTeam } from "../../controllers/auth";
import { autumnService } from "../autumn/autumn.service";
import { getBillingQueue } from "../queue-service";
import { redisEvictConnection } from "../redis";
+import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
+import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";
function deferred() {
let resolve!: () => void;
@@ -220,6 +228,37 @@ describe("monitor check finalization ownership", () => {
expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
});
+ it("records the in-app notification even when the email channel throws", async () => {
+ const lockSet = vi
+ .mocked(redisEvictConnection.set)
+ .getMockImplementation()!;
+ vi.mocked(redisEvictConnection.set).mockImplementation(((
+ key: string,
+ ...rest: unknown[]
+ ) =>
+ key.startsWith("monitor-check-notify:")
+ ? Promise.resolve("OK")
+ : (lockSet as any)(key, ...rest)) as any);
+ vi.mocked(store.listMonitorCheckPages).mockResolvedValue([] as any);
+ vi.mocked(recordMonitorInAppNotification).mockResolvedValue({
+ attempted: true,
+ success: true,
+ });
+ vi.mocked(sendMonitoringEmailSummary).mockRejectedValue(
+ new Error("recipient lookup failed"),
+ );
+
+ await reconcileRunningMonitorChecks();
+
+ expect(sendMonitoringEmailSummary).toHaveBeenCalledTimes(1);
+ expect(recordMonitorInAppNotification).toHaveBeenCalledWith(
+ expect.objectContaining({
+ check: expect.objectContaining({ id: "check-1" }),
+ }),
+ );
+ expect(current.status).toBe("completed");
+ });
+
it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {
current.target_results = [
{ type: "search", targetId: "target-1", searchCompleted: false },
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 6b08c777e..239334436 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -33,6 +33,7 @@ import { createWebhookSender, WebhookEvent } from "../webhook";
import { sendMonitorPageWebhook } from "./results";
import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
import { sendMonitoringSlackSummary } from "../notification/monitoring_slack";
+import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";
import {
bulkUpsertMonitorPages,
calculateMonitorCheckActualCredits,
@@ -439,7 +440,12 @@ async function sendNotifications(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
pages: PageResult[];
-}): Promise<{ webhook?: unknown; email?: unknown; slack?: unknown }> {
+}): Promise<{
+ webhook?: unknown;
+ email?: unknown;
+ slack?: unknown;
+ inApp?: unknown;
+}> {
const payload = {
monitorId: params.monitor.id,
checkId: params.check.id,
@@ -447,6 +453,19 @@ async function sendNotifications(params: {
summary: toSummaryObject(params.check),
};
+ const nonSamePages = params.pages.filter(page => page.status !== "same");
+
+ // Never throws, and goes first so a failing channel below cannot skip it.
+ const inAppStatus = await recordMonitorInAppNotification({
+ monitor: params.monitor,
+ check: params.check,
+ pages: nonSamePages.map(page => ({
+ url: page.url,
+ status: page.status,
+ judgment: page.judgment ?? null,
+ })),
+ });
+
let webhookStatus: unknown = { attempted: false };
if (params.monitor.webhook) {
const sender = await createWebhookSender({
@@ -479,7 +498,6 @@ async function sendNotifications(params: {
}
}
- const nonSamePages = params.pages.filter(page => page.status !== "same");
// Pull diff text for up to 5 meaningful changed pages so the email leads with
// the diff. Errors swallowed per-page so one GCS hiccup doesn't drop the alert.
const diffEligible = nonSamePages
@@ -547,6 +565,7 @@ async function sendNotifications(params: {
webhook: webhookStatus,
email: emailStatus,
slack: slackStatus,
+ inApp: inAppStatus,
};
}
diff --git a/apps/api/src/services/notification/in_app.test.ts b/apps/api/src/services/notification/in_app.test.ts
new file mode 100644
index 000000000..3a8219693
--- /dev/null
+++ b/apps/api/src/services/notification/in_app.test.ts
@@ -0,0 +1,81 @@
+import { createInAppNotification, isDashboardOrigin } from "./in_app";
+import { db } from "../../db/connection";
+import { redisEvictConnection } from "../redis";
+
+const values = vi.fn();
+
+vi.mock("../../config", () => ({ config: { USE_DB_AUTHENTICATION: true } }));
+
+vi.mock("../../db/connection", () => ({
+ db: { insert: vi.fn(() => ({ values })) },
+}));
+
+vi.mock("../redis", () => ({
+ redisEvictConnection: { set: vi.fn(), del: vi.fn() },
+}));
+
+const claims = new Set<string>();
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ claims.clear();
+ values.mockResolvedValue(undefined);
+ vi.mocked(redisEvictConnection.set).mockImplementation((async (
+ key: string,
+ ) => {
+ if (claims.has(key)) return null;
+ claims.add(key);
+ return "OK";
+ }) as any);
+ vi.mocked(redisEvictConnection.del).mockImplementation((async (key: string) =>
+ Number(claims.delete(key))) as any);
+});
+
+const notify = () =>
+ createInAppNotification(
+ "team-1",
+ "crawlCompleted",
+ { jobId: "crawl-1" },
+ { dedupeKey: "crawl-1" },
+ );
+
+describe("deduplicated in-app notifications", () => {
+ test("writes one row per key when the finishing job is redelivered", async () => {
+ expect(await notify()).toBe(true);
+ expect(await notify()).toBe(false);
+
+ expect(db.insert).toHaveBeenCalledTimes(1);
+ expect(values).toHaveBeenCalledWith(
+ expect.objectContaining({
+ team_id: "team-1",
+ notification_type: "crawlCompleted",
+ metadata: { jobId: "crawl-1" },
+ }),
+ );
+ });
+
+ test("releases the claim when the write fails so a retry can notify", async () => {
+ values.mockRejectedValueOnce(new Error("db down"));
+
+ expect(await notify()).toBe(false);
+ expect(await notify()).toBe(true);
+
+ expect(values).toHaveBeenCalledTimes(2);
+ });
+
+ test("does not throw when the claim cannot be taken", async () => {
+ vi.mocked(redisEvictConnection.set).mockRejectedValueOnce(
+ new Error("redis down"),
+ );
+
+ expect(await notify()).toBe(false);
+ expect(db.insert).not.toHaveBeenCalled();
+ });
+});
+
+test("only the exact dashboard origin counts", () => {
+ expect(isDashboardOrigin("website")).toBe(true);
+ expect(isDashboardOrigin("my-website-bot")).toBe(false);
+ expect(isDashboardOrigin("api")).toBe(false);
+ expect(isDashboardOrigin(undefined)).toBe(false);
+});
diff --git a/apps/api/src/services/notification/in_app.ts b/apps/api/src/services/notification/in_app.ts
new file mode 100644
index 000000000..2ae836ba8
--- /dev/null
+++ b/apps/api/src/services/notification/in_app.ts
@@ -0,0 +1,73 @@
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+import { redisEvictConnection } from "../redis";
+
+const logger = _logger.child({ module: "in-app-notification" });
+
+// Outlives queue redelivery of the job that produced the notification.
+const DEDUPE_TTL_SECONDS = 24 * 60 * 60;
+
+// Must match a type firecrawl-web's notification registry knows how to render.
+type InAppNotificationType =
+ | "monitorChangeDetected"
+ | "crawlCompleted"
+ | "batchScrapeCompleted";
+
+/**
+ * Add a row to the team's dashboard notification center. Never throws: a
+ * missing notification must not fail the job that produced it. With a
+ * `dedupeKey`, only the first call per key and type writes a row.
+ */
+export async function createInAppNotification(
+ teamId: string,
+ type: InAppNotificationType,
+ metadata: Record<string, string | number | null>,
+ options: { dedupeKey?: string } = {},
+): Promise<boolean> {
+ // Self-hosted deployments have no dashboard to show these in.
+ if (!config.USE_DB_AUTHENTICATION) return false;
+ const claimKey = options.dedupeKey
+ ? `in-app-notification:${type}:${options.dedupeKey}`
+ : null;
+ let claimed = false;
+ try {
+ if (claimKey) {
+ const result = await redisEvictConnection.set(
+ claimKey,
+ "1",
+ "EX",
+ DEDUPE_TTL_SECONDS,
+ "NX",
+ );
+ if (result !== "OK") return false;
+ claimed = true;
+ }
+ const now = new Date().toISOString();
+ await db.insert(schema.user_notifications).values({
+ team_id: teamId,
+ notification_type: type,
+ sent_date: now,
+ timestamp: now,
+ metadata,
+ });
+ return true;
+ } catch (error) {
+ logger.warn("Failed to create in-app notification", {
+ error,
+ teamId,
+ type,
+ });
+ if (claimed && claimKey) {
+ // Let a redelivered job retry the write.
+ await redisEvictConnection.del(claimKey).catch(() => {});
+ }
+ return false;
+ }
+}
+
+/** Jobs started from the dashboard playground send origin "website". */
+export function isDashboardOrigin(origin: unknown): boolean {
+ return origin === "website";
+}
diff --git a/apps/api/src/services/notification/monitoring_in_app.test.ts b/apps/api/src/services/notification/monitoring_in_app.test.ts
new file mode 100644
index 000000000..941bcfae0
--- /dev/null
+++ b/apps/api/src/services/notification/monitoring_in_app.test.ts
@@ -0,0 +1,110 @@
+import type { Mock } from "vitest";
+import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";
+import { createInAppNotification } from "./in_app";
+import {
+ describeMonitorActivity,
+ recordMonitorInAppNotification,
+} from "./monitoring_in_app";
+
+vi.mock("./in_app", () => ({
+ createInAppNotification: vi.fn(async () => true),
+}));
+
+vi.mock("./monitoring_slack", () => ({
+ shouldSuppressForNoise: vi.fn(
+ (monitor: MonitorRow, _check: MonitorCheckRow, pages: any[]) =>
+ monitor.judge_enabled &&
+ pages.every(page => page.judgment?.meaningful === false),
+ ),
+}));
+
+const counts = (changed: number, added: number, removed: number) => ({
+ changed_count: changed,
+ new_count: added,
+ removed_count: removed,
+});
+
+const monitor = {
+ id: "mon-1",
+ team_id: "team-1",
+ name: "Competitor pricing",
+ judge_enabled: false,
+} as MonitorRow;
+
+const check = { id: "check-1", ...counts(3, 1, 0) } as MonitorCheckRow;
+
+beforeEach(() => vi.clearAllMocks());
+
+describe("describing a monitor check", () => {
+ test("lists only the kinds of activity that happened", () => {
+ expect(describeMonitorActivity(counts(3, 1, 0))).toBe(
+ "3 pages changed, 1 new page.",
+ );
+ expect(describeMonitorActivity(counts(1, 0, 2))).toBe(
+ "1 page changed, 2 pages removed.",
+ );
+ expect(describeMonitorActivity(counts(0, 1200, 0))).toBe(
+ "1,200 new pages.",
+ );
+ });
+
+ test("returns nothing when no page changed", () => {
+ expect(describeMonitorActivity(counts(0, 0, 0))).toBeNull();
+ });
+});
+
+describe("recording a monitor notification", () => {
+ test("writes the monitor's name, link target and summary", async () => {
+ const result = await recordMonitorInAppNotification({
+ monitor,
+ check,
+ pages: [{ url: "https://a.com", status: "changed" }],
+ });
+
+ expect(result).toEqual({ attempted: true, success: true });
+ expect(createInAppNotification).toHaveBeenCalledWith(
+ "team-1",
+ "monitorChangeDetected",
+ {
+ monitorId: "mon-1",
+ monitorName: "Competitor pricing",
+ checkId: "check-1",
+ summary: "3 pages changed, 1 new page.",
+ },
+ );
+ });
+
+ test("reports a failed write without throwing", async () => {
+ (createInAppNotification as Mock).mockResolvedValueOnce(false);
+
+ const result = await recordMonitorInAppNotification({
+ monitor,
+ check,
+ pages: [{ url: "https://a.com", status: "changed" }],
+ });
+
+ expect(result).toEqual({ attempted: true, success: false });
+ });
+
+ test("skips checks with no changes and changes the judge marked as noise", async () => {
+ await recordMonitorInAppNotification({
+ monitor,
+ check: { ...check, ...counts(0, 0, 0) },
+ pages: [],
+ });
+ const suppressed = await recordMonitorInAppNotification({
+ monitor: { ...monitor, judge_enabled: true },
+ check,
+ pages: [
+ {
+ url: "https://a.com",
+ status: "changed",
+ judgment: { meaningful: false, confidence: "high", reason: "ad" },
+ },
+ ],
+ });
+
+ expect(suppressed).toMatchObject({ attempted: false, suppressed: true });
+ expect(createInAppNotification as Mock).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/notification/monitoring_in_app.ts b/apps/api/src/services/notification/monitoring_in_app.ts
new file mode 100644
index 000000000..ad2033834
--- /dev/null
+++ b/apps/api/src/services/notification/monitoring_in_app.ts
@@ -0,0 +1,58 @@
+import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";
+import { createInAppNotification } from "./in_app";
+import { shouldSuppressForNoise } from "./monitoring_slack";
+
+function countPhrase(count: number, singular: string, plural: string): string {
+ return `${count.toLocaleString("en-US")} ${count === 1 ? singular : plural}`;
+}
+
+/** "3 pages changed, 1 new page." or null when nothing changed. */
+export function describeMonitorActivity(
+ check: Pick<MonitorCheckRow, "changed_count" | "new_count" | "removed_count">,
+): string | null {
+ const parts: string[] = [];
+ if (check.changed_count > 0) {
+ parts.push(`${countPhrase(check.changed_count, "page", "pages")} changed`);
+ }
+ if (check.new_count > 0) {
+ parts.push(countPhrase(check.new_count, "new page", "new pages"));
+ }
+ if (check.removed_count > 0) {
+ parts.push(`${countPhrase(check.removed_count, "page", "pages")} removed`);
+ }
+ if (parts.length === 0) return null;
+ const sentence = parts.join(", ");
+ return `${sentence.charAt(0).toUpperCase()}${sentence.slice(1)}.`;
+}
+
+export async function recordMonitorInAppNotification(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ pages: {
+ url: string;
+ status: string;
+ judgment?: {
+ meaningful: boolean;
+ confidence: "high" | "medium" | "low";
+ reason: string;
+ } | null;
+ }[];
+}): Promise<{ attempted: boolean; success: boolean; suppressed?: boolean }> {
+ const summary = describeMonitorActivity(params.check);
+ if (!summary) return { attempted: false, success: true };
+ // Same judge gate as email and Slack, so the bell never fires on noise alone.
+ if (shouldSuppressForNoise(params.monitor, params.check, params.pages)) {
+ return { attempted: false, success: true, suppressed: true };
+ }
+ const success = await createInAppNotification(
+ params.monitor.team_id,
+ "monitorChangeDetected",
+ {
+ monitorId: params.monitor.id,
+ monitorName: params.monitor.name,
+ checkId: params.check.id,
+ summary,
+ },
+ );
+ return { attempted: true, success };
+}
diff --git a/apps/api/src/services/worker/crawl-logic.test.ts b/apps/api/src/services/worker/crawl-logic.test.ts
index 510ec6413..f8074d8eb 100644
--- a/apps/api/src/services/worker/crawl-logic.test.ts
+++ b/apps/api/src/services/worker/crawl-logic.test.ts
@@ -1,8 +1,9 @@
import type { Mock } from "vitest";
import { finishCrawlSuper } from "./crawl-logic";
-import { getCrawl } from "../../lib/crawl-redis";
+import { getCrawl, getDoneJobsOrderedLength } from "../../lib/crawl-redis";
import { logCrawl } from "../logging/log_job";
import { createWebhookSender } from "../webhook/index";
+import { createInAppNotification } from "../notification/in_app";
import type { NuQJob } from "./nuq";
vi.mock("../../lib/crawl-redis", () => ({
@@ -12,6 +13,14 @@ vi.mock("../../lib/crawl-redis", () => ({
getDoneJobsOrderedLength: vi.fn(async () => 2),
}));
+vi.mock("../../lib/request-credits-store", () => ({
+ readRequestCredits: vi.fn(async () => 7),
+}));
+
+vi.mock("../../lib/request-credits-analytics", () => ({
+ readRequestCreditsFromAnalytics: vi.fn(async () => 7),
+}));
+
vi.mock("../../controllers/v1/crawl-status", () => ({
getJobs: vi.fn(async () => []),
}));
@@ -29,6 +38,13 @@ vi.mock("../webhook/index", () => ({
},
}));
+vi.mock("../redis", () => ({ redisEvictConnection: {} }));
+
+vi.mock("../notification/in_app", async importOriginal => ({
+ ...(await importOriginal<typeof import("../notification/in_app")>()),
+ createInAppNotification: vi.fn(async () => true),
+}));
+
const baseSc = {
originUrl: "https://example.com",
crawlerOptions: {},
@@ -92,3 +108,107 @@ test("does not crash or fire a webhook when none is configured", async () => {
await expect(finishCrawlSuper(job)).resolves.not.toThrow();
expect(createWebhookSender).not.toHaveBeenCalled();
});
+
+describe("dashboard notification on finish", () => {
+ const job = (id: string) =>
+ ({
+ id: `job-${id}`,
+ groupId: `crawl-${id}`,
+ ownerId: "team-from-sc",
+ data: null,
+ }) as unknown as NuQJob<any>;
+
+ test("notifies the team when a crawl started from the dashboard finishes", async () => {
+ (getCrawl as Mock).mockResolvedValue({
+ ...baseSc,
+ zeroDataRetention: false,
+ internalOptions: {},
+ v1: true,
+ origin: "website",
+ });
+
+ await finishCrawlSuper(job("dash"));
+
+ expect(createInAppNotification).toHaveBeenCalledWith(
+ "team-from-sc",
+ "crawlCompleted",
+ expect.objectContaining({
+ jobId: "crawl-dash",
+ url: "https://example.com",
+ completed: 2,
+ creditsUsed: 7,
+ link: "/app/logs?q=crawl-dash",
+ }),
+ { dedupeKey: "crawl-dash" },
+ );
+ });
+
+ test("leaves the URL out for zero data retention crawls", async () => {
+ (getCrawl as Mock).mockResolvedValue({
+ ...baseSc,
+ v1: true,
+ origin: "website",
+ });
+
+ await finishCrawlSuper(job("zdr"));
+
+ expect(createInAppNotification).toHaveBeenCalledWith(
+ "team-from-sc",
+ "crawlCompleted",
+ expect.objectContaining({ url: null }),
+ expect.anything(),
+ );
+ });
+
+ test("names a batch scrape as such", async () => {
+ (getCrawl as Mock).mockResolvedValue({
+ ...baseSc,
+ crawlerOptions: null,
+ v1: true,
+ origin: "website",
+ });
+
+ await finishCrawlSuper(job("batch"));
+
+ expect(createInAppNotification).toHaveBeenCalledWith(
+ "team-from-sc",
+ "batchScrapeCompleted",
+ expect.objectContaining({ url: null }),
+ expect.anything(),
+ );
+ });
+
+ test("stays quiet for API-started, cancelled, and fully failed jobs", async () => {
+ (getCrawl as Mock).mockResolvedValueOnce({
+ ...baseSc,
+ v1: true,
+ origin: "api",
+ });
+ await finishCrawlSuper(job("api"));
+
+ (getCrawl as Mock).mockResolvedValueOnce({
+ ...baseSc,
+ v1: true,
+ origin: "not-the-website",
+ });
+ await finishCrawlSuper(job("lookalike"));
+
+ (getCrawl as Mock).mockResolvedValueOnce({
+ ...baseSc,
+ v1: true,
+ origin: "website",
+ });
+ (getDoneJobsOrderedLength as Mock).mockResolvedValueOnce(0);
+ await finishCrawlSuper(job("all-failed"));
+
+ (getCrawl as Mock).mockResolvedValueOnce({
+ ...baseSc,
+ v1: true,
+ origin: "website",
+ cancelled: true,
+ });
+ await finishCrawlSuper(job("cancelled"));
+
+ expect(createInAppNotification).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 132b72a5a..86c5ab3b7 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -11,6 +11,10 @@ import { logCrawl, logBatchScrape } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook/index";
import type { NuQJob } from "./nuq";
import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
+import {
+ createInAppNotification,
+ isDashboardOrigin,
+} from "../notification/in_app";
/**
* How often, and how long apart, finalization re-reads the ClickHouse credits
@@ -223,6 +227,29 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
);
}
+ // The origin is client-supplied, so this only scopes which of the caller's
+ // own jobs reach their team's feed. Runs with no successful page are not
+ // reported as completed.
+ if (
+ !sc.cancelled &&
+ num_docs > 0 &&
+ isDashboardOrigin(sc.origin ?? data?.origin)
+ ) {
+ const isCrawl = sc.crawlerOptions !== null;
+ await createInAppNotification(
+ teamId,
+ isCrawl ? "crawlCompleted" : "batchScrapeCompleted",
+ {
+ jobId: crawlId,
+ url: isCrawl && !zeroDataRetention ? (sc.originUrl ?? null) : null,
+ completed: num_docs,
+ creditsUsed: credits_billed,
+ link: `/app/logs?q=${encodeURIComponent(crawlId)}`,
+ },
+ { dedupeKey: crawlId },
+ );
+ }
+
// v1 web hooks, call when done with no data, but with event completed
if (isV1 && webhook) {
const sender = await createWebhookSender({