Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
In-app notifications for monitor changes and dashboard crawls (#4928)
Nicolas66ebd15b5da3
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 0b584ce6f..4de3f5591 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -362,6 +362,7 @@ export async function batchScrapeController(zeroDataRetention,v1: true,webhook: req.body.webhook,+ origin: req.body.origin,};if (req.body.appendToId && (!sc || sc.team_id !== req.auth.team_id)) {diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 74fe3eea4..7d2deb81b 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -291,6 +291,7 @@ export async function crawlController(zeroDataRetention,v1: true,webhook: req.body.webhook,+ origin: req.body.origin,};const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 3ab5ba6c2..6e9cd196c 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -417,6 +417,7 @@ export async function batchScrapeController(v1: true,webhook: req.body.webhook,requestId: req.body.__agentInterop?.requestId ?? undefined,+ origin: req.body.origin,};if (req.body.appendToId) {diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 4e8b408b0..34dbd6f68 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -357,6 +357,7 @@ export async function crawlController(zeroDataRetention,v1: true,webhook: req.body.webhook,+ origin: req.body.origin,};const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 1132a920c..46b29ea74 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -37,6 +37,9 @@ export type StoredCrawl = {v1?: boolean;webhook?: WebhookConfig;requestId?: string;+ // The request's `origin`; dashboard-started jobs get an in-app notification+ // when they finish.+ origin?: string;};export async function saveCrawl(id: string, crawl: StoredCrawl) {diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.tsindex 0fd2c8b43..6dc43a185 100644--- a/apps/api/src/services/monitoring/runner.finalization.test.ts+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts@@ -21,8 +21,13 @@ vi.mock("../queue-jobs", () => ({}));vi.mock("../../controllers/v2/types", () => ({}));vi.mock("../webhook", () => ({}));vi.mock("./results", () => ({}));-vi.mock("../notification/monitoring_email", () => ({}));+vi.mock("../notification/monitoring_email", () => ({+ sendMonitoringEmailSummary: vi.fn(),+}));vi.mock("../notification/monitoring_slack", () => ({}));+vi.mock("../notification/monitoring_in_app", () => ({+ recordMonitorInAppNotification: vi.fn(),+}));vi.mock("./types", () => ({}));vi.mock("./interest", () => ({trackMonitorCheckStartedInterest: async () => {},@@ -42,6 +47,7 @@ vi.mock("./store", () => ({updateMonitorCheckIfStatus: vi.fn(),markMonitorRunning: vi.fn(),countMonitorCheckPages: vi.fn(),+ listMonitorCheckPages: vi.fn(),calculateMonitorCheckActualCredits: vi.fn(),updateMonitorScheduleAfterRun: vi.fn(),}));@@ -63,6 +69,8 @@ import { getACUCTeam } from "../../controllers/auth";import { autumnService } from "../autumn/autumn.service";import { getBillingQueue } from "../queue-service";import { redisEvictConnection } from "../redis";+import { sendMonitoringEmailSummary } from "../notification/monitoring_email";+import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";function deferred() {let resolve!: () => void;@@ -220,6 +228,37 @@ describe("monitor check finalization ownership", () => {expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);});+ it("records the in-app notification even when the email channel throws", async () => {+ const lockSet = vi+ .mocked(redisEvictConnection.set)+ .getMockImplementation()!;+ vi.mocked(redisEvictConnection.set).mockImplementation(((+ key: string,+ ...rest: unknown[]+ ) =>+ key.startsWith("monitor-check-notify:")+ ? Promise.resolve("OK")+ : (lockSet as any)(key, ...rest)) as any);+ vi.mocked(store.listMonitorCheckPages).mockResolvedValue([] as any);+ vi.mocked(recordMonitorInAppNotification).mockResolvedValue({+ attempted: true,+ success: true,+ });+ vi.mocked(sendMonitoringEmailSummary).mockRejectedValue(+ new Error("recipient lookup failed"),+ );++ await reconcileRunningMonitorChecks();++ expect(sendMonitoringEmailSummary).toHaveBeenCalledTimes(1);+ expect(recordMonitorInAppNotification).toHaveBeenCalledWith(+ expect.objectContaining({+ check: expect.objectContaining({ id: "check-1" }),+ }),+ );+ expect(current.status).toBe("completed");+ });+it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {current.target_results = [{ type: "search", targetId: "target-1", searchCompleted: false },diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 6b08c777e..239334436 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -33,6 +33,7 @@ import { createWebhookSender, WebhookEvent } from "../webhook";import { sendMonitorPageWebhook } from "./results";import { sendMonitoringEmailSummary } from "../notification/monitoring_email";import { sendMonitoringSlackSummary } from "../notification/monitoring_slack";+import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";import {bulkUpsertMonitorPages,calculateMonitorCheckActualCredits,@@ -439,7 +440,12 @@ async function sendNotifications(params: {monitor: MonitorRow;check: MonitorCheckRow;pages: PageResult[];-}): Promise<{ webhook?: unknown; email?: unknown; slack?: unknown }> {+}): Promise<{+ webhook?: unknown;+ email?: unknown;+ slack?: unknown;+ inApp?: unknown;+}> {const payload = {monitorId: params.monitor.id,checkId: params.check.id,@@ -447,6 +453,19 @@ async function sendNotifications(params: {summary: toSummaryObject(params.check),};+ const nonSamePages = params.pages.filter(page => page.status !== "same");++ // Never throws, and goes first so a failing channel below cannot skip it.+ const inAppStatus = await recordMonitorInAppNotification({+ monitor: params.monitor,+ check: params.check,+ pages: nonSamePages.map(page => ({+ url: page.url,+ status: page.status,+ judgment: page.judgment ?? null,+ })),+ });+let webhookStatus: unknown = { attempted: false };if (params.monitor.webhook) {const sender = await createWebhookSender({@@ -479,7 +498,6 @@ async function sendNotifications(params: {}}- const nonSamePages = params.pages.filter(page => page.status !== "same");// Pull diff text for up to 5 meaningful changed pages so the email leads with// the diff. Errors swallowed per-page so one GCS hiccup doesn't drop the alert.const diffEligible = nonSamePages@@ -547,6 +565,7 @@ async function sendNotifications(params: {webhook: webhookStatus,email: emailStatus,slack: slackStatus,+ inApp: inAppStatus,};}diff --git a/apps/api/src/services/notification/in_app.test.ts b/apps/api/src/services/notification/in_app.test.tsnew file mode 100644index 000000000..3a8219693--- /dev/null+++ b/apps/api/src/services/notification/in_app.test.ts@@ -0,0 +1,81 @@+import { createInAppNotification, isDashboardOrigin } from "./in_app";+import { db } from "../../db/connection";+import { redisEvictConnection } from "../redis";++const values = vi.fn();++vi.mock("../../config", () => ({ config: { USE_DB_AUTHENTICATION: true } }));++vi.mock("../../db/connection", () => ({+ db: { insert: vi.fn(() => ({ values })) },+}));++vi.mock("../redis", () => ({+ redisEvictConnection: { set: vi.fn(), del: vi.fn() },+}));++const claims = new Set<string>();++beforeEach(() => {+ vi.clearAllMocks();+ claims.clear();+ values.mockResolvedValue(undefined);+ vi.mocked(redisEvictConnection.set).mockImplementation((async (+ key: string,+ ) => {+ if (claims.has(key)) return null;+ claims.add(key);+ return "OK";+ }) as any);+ vi.mocked(redisEvictConnection.del).mockImplementation((async (key: string) =>+ Number(claims.delete(key))) as any);+});++const notify = () =>+ createInAppNotification(+ "team-1",+ "crawlCompleted",+ { jobId: "crawl-1" },+ { dedupeKey: "crawl-1" },+ );++describe("deduplicated in-app notifications", () => {+ test("writes one row per key when the finishing job is redelivered", async () => {+ expect(await notify()).toBe(true);+ expect(await notify()).toBe(false);++ expect(db.insert).toHaveBeenCalledTimes(1);+ expect(values).toHaveBeenCalledWith(+ expect.objectContaining({+ team_id: "team-1",+ notification_type: "crawlCompleted",+ metadata: { jobId: "crawl-1" },+ }),+ );+ });++ test("releases the claim when the write fails so a retry can notify", async () => {+ values.mockRejectedValueOnce(new Error("db down"));++ expect(await notify()).toBe(false);+ expect(await notify()).toBe(true);++ expect(values).toHaveBeenCalledTimes(2);+ });++ test("does not throw when the claim cannot be taken", async () => {+ vi.mocked(redisEvictConnection.set).mockRejectedValueOnce(+ new Error("redis down"),+ );++ expect(await notify()).toBe(false);+ expect(db.insert).not.toHaveBeenCalled();+ });+});++test("only the exact dashboard origin counts", () => {+ expect(isDashboardOrigin("website")).toBe(true);+ expect(isDashboardOrigin("my-website-bot")).toBe(false);+ expect(isDashboardOrigin("api")).toBe(false);+ expect(isDashboardOrigin(undefined)).toBe(false);+});diff --git a/apps/api/src/services/notification/in_app.ts b/apps/api/src/services/notification/in_app.tsnew file mode 100644index 000000000..2ae836ba8--- /dev/null+++ b/apps/api/src/services/notification/in_app.ts@@ -0,0 +1,73 @@+import { db } from "../../db/connection";+import * as schema from "../../db/schema";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";+import { redisEvictConnection } from "../redis";++const logger = _logger.child({ module: "in-app-notification" });++// Outlives queue redelivery of the job that produced the notification.+const DEDUPE_TTL_SECONDS = 24 * 60 * 60;++// Must match a type firecrawl-web's notification registry knows how to render.+type InAppNotificationType =+ | "monitorChangeDetected"+ | "crawlCompleted"+ | "batchScrapeCompleted";++/**+ * Add a row to the team's dashboard notification center. Never throws: a+ * missing notification must not fail the job that produced it. With a+ * `dedupeKey`, only the first call per key and type writes a row.+ */+export async function createInAppNotification(+ teamId: string,+ type: InAppNotificationType,+ metadata: Record<string, string | number | null>,+ options: { dedupeKey?: string } = {},+): Promise<boolean> {+ // Self-hosted deployments have no dashboard to show these in.+ if (!config.USE_DB_AUTHENTICATION) return false;+ const claimKey = options.dedupeKey+ ? `in-app-notification:${type}:${options.dedupeKey}`+ : null;+ let claimed = false;+ try {+ if (claimKey) {+ const result = await redisEvictConnection.set(+ claimKey,+ "1",+ "EX",+ DEDUPE_TTL_SECONDS,+ "NX",+ );+ if (result !== "OK") return false;+ claimed = true;+ }+ const now = new Date().toISOString();+ await db.insert(schema.user_notifications).values({+ team_id: teamId,+ notification_type: type,+ sent_date: now,+ timestamp: now,+ metadata,+ });+ return true;+ } catch (error) {+ logger.warn("Failed to create in-app notification", {+ error,+ teamId,+ type,+ });+ if (claimed && claimKey) {+ // Let a redelivered job retry the write.+ await redisEvictConnection.del(claimKey).catch(() => {});+ }+ return false;+ }+}++/** Jobs started from the dashboard playground send origin "website". */+export function isDashboardOrigin(origin: unknown): boolean {+ return origin === "website";+}diff --git a/apps/api/src/services/notification/monitoring_in_app.test.ts b/apps/api/src/services/notification/monitoring_in_app.test.tsnew file mode 100644index 000000000..941bcfae0--- /dev/null+++ b/apps/api/src/services/notification/monitoring_in_app.test.ts@@ -0,0 +1,110 @@+import type { Mock } from "vitest";+import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";+import { createInAppNotification } from "./in_app";+import {+ describeMonitorActivity,+ recordMonitorInAppNotification,+} from "./monitoring_in_app";++vi.mock("./in_app", () => ({+ createInAppNotification: vi.fn(async () => true),+}));++vi.mock("./monitoring_slack", () => ({+ shouldSuppressForNoise: vi.fn(+ (monitor: MonitorRow, _check: MonitorCheckRow, pages: any[]) =>+ monitor.judge_enabled &&+ pages.every(page => page.judgment?.meaningful === false),+ ),+}));++const counts = (changed: number, added: number, removed: number) => ({+ changed_count: changed,+ new_count: added,+ removed_count: removed,+});++const monitor = {+ id: "mon-1",+ team_id: "team-1",+ name: "Competitor pricing",+ judge_enabled: false,+} as MonitorRow;++const check = { id: "check-1", ...counts(3, 1, 0) } as MonitorCheckRow;++beforeEach(() => vi.clearAllMocks());++describe("describing a monitor check", () => {+ test("lists only the kinds of activity that happened", () => {+ expect(describeMonitorActivity(counts(3, 1, 0))).toBe(+ "3 pages changed, 1 new page.",+ );+ expect(describeMonitorActivity(counts(1, 0, 2))).toBe(+ "1 page changed, 2 pages removed.",+ );+ expect(describeMonitorActivity(counts(0, 1200, 0))).toBe(+ "1,200 new pages.",+ );+ });++ test("returns nothing when no page changed", () => {+ expect(describeMonitorActivity(counts(0, 0, 0))).toBeNull();+ });+});++describe("recording a monitor notification", () => {+ test("writes the monitor's name, link target and summary", async () => {+ const result = await recordMonitorInAppNotification({+ monitor,+ check,+ pages: [{ url: "https://a.com", status: "changed" }],+ });++ expect(result).toEqual({ attempted: true, success: true });+ expect(createInAppNotification).toHaveBeenCalledWith(+ "team-1",+ "monitorChangeDetected",+ {+ monitorId: "mon-1",+ monitorName: "Competitor pricing",+ checkId: "check-1",+ summary: "3 pages changed, 1 new page.",+ },+ );+ });++ test("reports a failed write without throwing", async () => {+ (createInAppNotification as Mock).mockResolvedValueOnce(false);++ const result = await recordMonitorInAppNotification({+ monitor,+ check,+ pages: [{ url: "https://a.com", status: "changed" }],+ });++ expect(result).toEqual({ attempted: true, success: false });+ });++ test("skips checks with no changes and changes the judge marked as noise", async () => {+ await recordMonitorInAppNotification({+ monitor,+ check: { ...check, ...counts(0, 0, 0) },+ pages: [],+ });+ const suppressed = await recordMonitorInAppNotification({+ monitor: { ...monitor, judge_enabled: true },+ check,+ pages: [+ {+ url: "https://a.com",+ status: "changed",+ judgment: { meaningful: false, confidence: "high", reason: "ad" },+ },+ ],+ });++ expect(suppressed).toMatchObject({ attempted: false, suppressed: true });+ expect(createInAppNotification as Mock).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/notification/monitoring_in_app.ts b/apps/api/src/services/notification/monitoring_in_app.tsnew file mode 100644index 000000000..ad2033834--- /dev/null+++ b/apps/api/src/services/notification/monitoring_in_app.ts@@ -0,0 +1,58 @@+import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";+import { createInAppNotification } from "./in_app";+import { shouldSuppressForNoise } from "./monitoring_slack";++function countPhrase(count: number, singular: string, plural: string): string {+ return `${count.toLocaleString("en-US")} ${count === 1 ? singular : plural}`;+}++/** "3 pages changed, 1 new page." or null when nothing changed. */+export function describeMonitorActivity(+ check: Pick<MonitorCheckRow, "changed_count" | "new_count" | "removed_count">,+): string | null {+ const parts: string[] = [];+ if (check.changed_count > 0) {+ parts.push(`${countPhrase(check.changed_count, "page", "pages")} changed`);+ }+ if (check.new_count > 0) {+ parts.push(countPhrase(check.new_count, "new page", "new pages"));+ }+ if (check.removed_count > 0) {+ parts.push(`${countPhrase(check.removed_count, "page", "pages")} removed`);+ }+ if (parts.length === 0) return null;+ const sentence = parts.join(", ");+ return `${sentence.charAt(0).toUpperCase()}${sentence.slice(1)}.`;+}++export async function recordMonitorInAppNotification(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ pages: {+ url: string;+ status: string;+ judgment?: {+ meaningful: boolean;+ confidence: "high" | "medium" | "low";+ reason: string;+ } | null;+ }[];+}): Promise<{ attempted: boolean; success: boolean; suppressed?: boolean }> {+ const summary = describeMonitorActivity(params.check);+ if (!summary) return { attempted: false, success: true };+ // Same judge gate as email and Slack, so the bell never fires on noise alone.+ if (shouldSuppressForNoise(params.monitor, params.check, params.pages)) {+ return { attempted: false, success: true, suppressed: true };+ }+ const success = await createInAppNotification(+ params.monitor.team_id,+ "monitorChangeDetected",+ {+ monitorId: params.monitor.id,+ monitorName: params.monitor.name,+ checkId: params.check.id,+ summary,+ },+ );+ return { attempted: true, success };+}diff --git a/apps/api/src/services/worker/crawl-logic.test.ts b/apps/api/src/services/worker/crawl-logic.test.tsindex 510ec6413..f8074d8eb 100644--- a/apps/api/src/services/worker/crawl-logic.test.ts+++ b/apps/api/src/services/worker/crawl-logic.test.ts@@ -1,8 +1,9 @@import type { Mock } from "vitest";import { finishCrawlSuper } from "./crawl-logic";-import { getCrawl } from "../../lib/crawl-redis";+import { getCrawl, getDoneJobsOrderedLength } from "../../lib/crawl-redis";import { logCrawl } from "../logging/log_job";import { createWebhookSender } from "../webhook/index";+import { createInAppNotification } from "../notification/in_app";import type { NuQJob } from "./nuq";vi.mock("../../lib/crawl-redis", () => ({@@ -12,6 +13,14 @@ vi.mock("../../lib/crawl-redis", () => ({getDoneJobsOrderedLength: vi.fn(async () => 2),}));+vi.mock("../../lib/request-credits-store", () => ({+ readRequestCredits: vi.fn(async () => 7),+}));++vi.mock("../../lib/request-credits-analytics", () => ({+ readRequestCreditsFromAnalytics: vi.fn(async () => 7),+}));+vi.mock("../../controllers/v1/crawl-status", () => ({getJobs: vi.fn(async () => []),}));@@ -29,6 +38,13 @@ vi.mock("../webhook/index", () => ({},}));+vi.mock("../redis", () => ({ redisEvictConnection: {} }));++vi.mock("../notification/in_app", async importOriginal => ({+ ...(await importOriginal<typeof import("../notification/in_app")>()),+ createInAppNotification: vi.fn(async () => true),+}));+const baseSc = {originUrl: "https://example.com",crawlerOptions: {},@@ -92,3 +108,107 @@ test("does not crash or fire a webhook when none is configured", async () => {await expect(finishCrawlSuper(job)).resolves.not.toThrow();expect(createWebhookSender).not.toHaveBeenCalled();});++describe("dashboard notification on finish", () => {+ const job = (id: string) =>+ ({+ id: `job-${id}`,+ groupId: `crawl-${id}`,+ ownerId: "team-from-sc",+ data: null,+ }) as unknown as NuQJob<any>;++ test("notifies the team when a crawl started from the dashboard finishes", async () => {+ (getCrawl as Mock).mockResolvedValue({+ ...baseSc,+ zeroDataRetention: false,+ internalOptions: {},+ v1: true,+ origin: "website",+ });++ await finishCrawlSuper(job("dash"));++ expect(createInAppNotification).toHaveBeenCalledWith(+ "team-from-sc",+ "crawlCompleted",+ expect.objectContaining({+ jobId: "crawl-dash",+ url: "https://example.com",+ completed: 2,+ creditsUsed: 7,+ link: "/app/logs?q=crawl-dash",+ }),+ { dedupeKey: "crawl-dash" },+ );+ });++ test("leaves the URL out for zero data retention crawls", async () => {+ (getCrawl as Mock).mockResolvedValue({+ ...baseSc,+ v1: true,+ origin: "website",+ });++ await finishCrawlSuper(job("zdr"));++ expect(createInAppNotification).toHaveBeenCalledWith(+ "team-from-sc",+ "crawlCompleted",+ expect.objectContaining({ url: null }),+ expect.anything(),+ );+ });++ test("names a batch scrape as such", async () => {+ (getCrawl as Mock).mockResolvedValue({+ ...baseSc,+ crawlerOptions: null,+ v1: true,+ origin: "website",+ });++ await finishCrawlSuper(job("batch"));++ expect(createInAppNotification).toHaveBeenCalledWith(+ "team-from-sc",+ "batchScrapeCompleted",+ expect.objectContaining({ url: null }),+ expect.anything(),+ );+ });++ test("stays quiet for API-started, cancelled, and fully failed jobs", async () => {+ (getCrawl as Mock).mockResolvedValueOnce({+ ...baseSc,+ v1: true,+ origin: "api",+ });+ await finishCrawlSuper(job("api"));++ (getCrawl as Mock).mockResolvedValueOnce({+ ...baseSc,+ v1: true,+ origin: "not-the-website",+ });+ await finishCrawlSuper(job("lookalike"));++ (getCrawl as Mock).mockResolvedValueOnce({+ ...baseSc,+ v1: true,+ origin: "website",+ });+ (getDoneJobsOrderedLength as Mock).mockResolvedValueOnce(0);+ await finishCrawlSuper(job("all-failed"));++ (getCrawl as Mock).mockResolvedValueOnce({+ ...baseSc,+ v1: true,+ origin: "website",+ cancelled: true,+ });+ await finishCrawlSuper(job("cancelled"));++ expect(createInAppNotification).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 132b72a5a..86c5ab3b7 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -11,6 +11,10 @@ import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";import type { NuQJob } from "./nuq";import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";+import {+ createInAppNotification,+ isDashboardOrigin,+} from "../notification/in_app";/*** How often, and how long apart, finalization re-reads the ClickHouse credits@@ -223,6 +227,29 @@ export async function finishCrawlSuper(job: NuQJob<any>) {);}+ // The origin is client-supplied, so this only scopes which of the caller's+ // own jobs reach their team's feed. Runs with no successful page are not+ // reported as completed.+ if (+ !sc.cancelled &&+ num_docs > 0 &&+ isDashboardOrigin(sc.origin ?? data?.origin)+ ) {+ const isCrawl = sc.crawlerOptions !== null;+ await createInAppNotification(+ teamId,+ isCrawl ? "crawlCompleted" : "batchScrapeCompleted",+ {+ jobId: crawlId,+ url: isCrawl && !zeroDataRetention ? (sc.originUrl ?? null) : null,+ completed: num_docs,+ creditsUsed: credits_billed,+ link: `/app/logs?q=${encodeURIComponent(crawlId)}`,+ },+ { dedupeKey: crawlId },+ );+ }+// v1 web hooks, call when done with no data, but with event completedif (isV1 && webhook) {const sender = await createWebhookSender({