Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Use onConflictDoUpdate for bulk monitor-page upsert
Developers Digeste45a47de1a6a
diff --git a/apps/api/src/services/monitoring/store-bulk-upsert.test.ts b/apps/api/src/services/monitoring/store-bulk-upsert.test.tsnew file mode 100644index 000000000..ad8b0eae8--- /dev/null+++ b/apps/api/src/services/monitoring/store-bulk-upsert.test.ts@@ -0,0 +1,62 @@+// The db client is mocked so the abort/empty guards can assert no query is issued.+// bulkUpsertMonitorPages' actual SQL behavior (the ON CONFLICT field rules) is+// covered by the Docker integration test against a real Postgres β a mock can't+// exercise the enum / CASE semantics, which is exactly where the prior bugs hid.+const { dbInsert } = vi.hoisted(() => ({ dbInsert: vi.fn() }));++vi.mock("../../db/connection", () => ({+ db: { insert: dbInsert },+ dbRr: { select: vi.fn() },+ dbIndex: { select: vi.fn() },+}));++import { bulkUpsertMonitorPages } from "./store";++const hash = (hex: string) => Buffer.from(hex, "hex");++beforeEach(() => {+ dbInsert.mockReset();+});++describe("bulkUpsertMonitorPages guards", () => {+ it("writes nothing when the abort signal is already aborted", async () => {+ const controller = new AbortController();+ controller.abort();++ await expect(+ bulkUpsertMonitorPages({+ monitorId: "m1",+ teamId: "t1",+ targetId: "tg1",+ checkId: "c1",+ rows: [+ {+ url: "https://example.com/a",+ urlHash: hash("aa"),+ status: "new",+ metadata: {},+ source: "discovered",+ scrapeId: null,+ },+ ],+ abortSignal: controller.signal,+ }),+ ).resolves.toBeUndefined();++ expect(dbInsert).not.toHaveBeenCalled();+ });++ it("writes nothing when there are no rows", async () => {+ await expect(+ bulkUpsertMonitorPages({+ monitorId: "m1",+ teamId: "t1",+ targetId: "tg1",+ checkId: "c1",+ rows: [],+ }),+ ).resolves.toBeUndefined();++ expect(dbInsert).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex d13e9b0f3..3939b8cb7 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -1,6 +1,6 @@import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";-import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";+import { and, asc, count, desc, eq, isNull, ne, sql } from "drizzle-orm";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { monitoringClaimDueMonitors } from "../../db/rpc";@@ -1100,6 +1100,108 @@ export async function upsertMonitorPage(params: {);}+type BulkUpsertMonitorPageRow = {+ url: string;+ urlHash?: Buffer;+ status: "same" | "new" | "changed" | "removed" | "error";+ metadata?: unknown;+ source: "explicit" | "discovered";+ scrapeId: string | null;+};++// Bulk equivalent of upsertMonitorPage: collapses an N-page upsert from ~2N+// sequential round-trips (replica read + primary write per page) into ONE atomic+// INSERT ... ON CONFLICT DO UPDATE keyed by the (monitor_id, target_id, url_hash)+// unique index. Per-row field rules mirror upsertMonitorPage exactly, expressed in+// the conflict set via `excluded` + CASE so no read is needed and Drizzle handles+// the enum/jsonb column types (no hand-written casts that can drift from the schema).+export async function bulkUpsertMonitorPages(params: {+ monitorId: string;+ teamId: string;+ targetId: string;+ checkId: string;+ rows: BulkUpsertMonitorPageRow[];+ // When finalize times out the caller aborts this signal; we then skip the whole+ // write so an aborted finalize leaves monitor_pages untouched (no partial baseline).+ abortSignal?: AbortSignal;+}): Promise<void> {+ if (params.abortSignal?.aborted) return;++ // Dedup by url_hash (last wins) so a repeated URL can't double-insert, and sort+ // by url_hash for a deterministic row-lock order.+ const byHash = new Map<+ string,+ BulkUpsertMonitorPageRow & { urlHash: Buffer }+ >();+ for (const row of params.rows) {+ const urlHash = row.urlHash ?? hashMonitorUrl(row.url);+ byHash.set(urlHash.toString("hex"), { ...row, urlHash });+ }+ if (byHash.size === 0) return;+ const rows = [...byHash.values()].sort((a, b) =>+ a.urlHash.toString("hex") < b.urlHash.toString("hex") ? -1 : 1,+ );++ const now = new Date().toISOString();++ // Build every row as if newly inserted; ON CONFLICT applies the existing-row+ // rules via `excluded` + CASE so the whole upsert is ONE atomic statement β no+ // separate read, no separate update β and Drizzle maps the enum/jsonb types from+ // the schema, so there are no hand-written casts that can drift from the columns.+ const values = rows.map(row => {+ const isRemoved = row.status === "removed";+ const isChangedOrNew = row.status === "changed" || row.status === "new";+ return {+ monitor_id: params.monitorId,+ team_id: params.teamId,+ target_id: params.targetId,+ url: row.url,+ url_hash: row.urlHash,+ source: row.source,+ first_seen_check_id: params.checkId,+ last_seen_check_id: isRemoved ? null : params.checkId,+ last_changed_check_id: isChangedOrNew ? params.checkId : null,+ last_scrape_id: row.scrapeId,+ last_status: row.status,+ is_removed: isRemoved,+ removed_at: isRemoved ? now : null,+ metadata: row.metadata ?? null,+ created_at: now,+ updated_at: now,+ };+ });++ if (params.abortSignal?.aborted) return;++ await run(+ () =>+ db+ .insert(schema.monitor_pages)+ .values(values)+ .onConflictDoUpdate({+ target: [+ schema.monitor_pages.monitor_id,+ schema.monitor_pages.target_id,+ schema.monitor_pages.url_hash,+ ],+ set: {+ last_status: sql`excluded.last_status`,+ is_removed: sql`excluded.is_removed`,+ removed_at: sql`excluded.removed_at`,+ // Preserve prior metadata when the new row carries none.+ metadata: sql`coalesce(excluded.metadata, ${schema.monitor_pages.metadata})`,+ // last_seen / last_scrape advance only when not removed; else preserved.+ last_seen_check_id: sql`case when excluded.is_removed then ${schema.monitor_pages.last_seen_check_id} else excluded.last_seen_check_id end`,+ last_scrape_id: sql`case when excluded.is_removed then ${schema.monitor_pages.last_scrape_id} else excluded.last_scrape_id end`,+ // last_changed advances only on new/changed; first_seen is never touched.+ last_changed_check_id: sql`case when excluded.last_status in ('new','changed') then excluded.last_changed_check_id else ${schema.monitor_pages.last_changed_check_id} end`,+ updated_at: sql`excluded.updated_at`,+ },+ }),+ "Failed to bulk upsert monitor pages",+ );+}+export async function listActiveMonitorPages(params: {monitorId: string;targetId: string;