Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Use onConflictDoUpdate for bulk monitor-page upsert

Developers Digeste45a47de1a6a
diff --git a/apps/api/src/services/monitoring/store-bulk-upsert.test.ts b/apps/api/src/services/monitoring/store-bulk-upsert.test.ts
new file mode 100644
index 000000000..ad8b0eae8
--- /dev/null
+++ b/apps/api/src/services/monitoring/store-bulk-upsert.test.ts
@@ -0,0 +1,62 @@
+// The db client is mocked so the abort/empty guards can assert no query is issued.
+// bulkUpsertMonitorPages' actual SQL behavior (the ON CONFLICT field rules) is
+// covered by the Docker integration test against a real Postgres β€” a mock can't
+// exercise the enum / CASE semantics, which is exactly where the prior bugs hid.
+const { dbInsert } = vi.hoisted(() => ({ dbInsert: vi.fn() }));
+
+vi.mock("../../db/connection", () => ({
+ db: { insert: dbInsert },
+ dbRr: { select: vi.fn() },
+ dbIndex: { select: vi.fn() },
+}));
+
+import { bulkUpsertMonitorPages } from "./store";
+
+const hash = (hex: string) => Buffer.from(hex, "hex");
+
+beforeEach(() => {
+ dbInsert.mockReset();
+});
+
+describe("bulkUpsertMonitorPages guards", () => {
+ it("writes nothing when the abort signal is already aborted", async () => {
+ const controller = new AbortController();
+ controller.abort();
+
+ await expect(
+ bulkUpsertMonitorPages({
+ monitorId: "m1",
+ teamId: "t1",
+ targetId: "tg1",
+ checkId: "c1",
+ rows: [
+ {
+ url: "https://example.com/a",
+ urlHash: hash("aa"),
+ status: "new",
+ metadata: {},
+ source: "discovered",
+ scrapeId: null,
+ },
+ ],
+ abortSignal: controller.signal,
+ }),
+ ).resolves.toBeUndefined();
+
+ expect(dbInsert).not.toHaveBeenCalled();
+ });
+
+ it("writes nothing when there are no rows", async () => {
+ await expect(
+ bulkUpsertMonitorPages({
+ monitorId: "m1",
+ teamId: "t1",
+ targetId: "tg1",
+ checkId: "c1",
+ rows: [],
+ }),
+ ).resolves.toBeUndefined();
+
+ expect(dbInsert).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index d13e9b0f3..3939b8cb7 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -1,6 +1,6 @@
import { createHash } from "crypto";
import { v7 as uuidv7 } from "uuid";
-import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";
+import { and, asc, count, desc, eq, isNull, ne, sql } from "drizzle-orm";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { monitoringClaimDueMonitors } from "../../db/rpc";
@@ -1100,6 +1100,108 @@ export async function upsertMonitorPage(params: {
);
}
+type BulkUpsertMonitorPageRow = {
+ url: string;
+ urlHash?: Buffer;
+ status: "same" | "new" | "changed" | "removed" | "error";
+ metadata?: unknown;
+ source: "explicit" | "discovered";
+ scrapeId: string | null;
+};
+
+// Bulk equivalent of upsertMonitorPage: collapses an N-page upsert from ~2N
+// sequential round-trips (replica read + primary write per page) into ONE atomic
+// INSERT ... ON CONFLICT DO UPDATE keyed by the (monitor_id, target_id, url_hash)
+// unique index. Per-row field rules mirror upsertMonitorPage exactly, expressed in
+// the conflict set via `excluded` + CASE so no read is needed and Drizzle handles
+// the enum/jsonb column types (no hand-written casts that can drift from the schema).
+export async function bulkUpsertMonitorPages(params: {
+ monitorId: string;
+ teamId: string;
+ targetId: string;
+ checkId: string;
+ rows: BulkUpsertMonitorPageRow[];
+ // When finalize times out the caller aborts this signal; we then skip the whole
+ // write so an aborted finalize leaves monitor_pages untouched (no partial baseline).
+ abortSignal?: AbortSignal;
+}): Promise<void> {
+ if (params.abortSignal?.aborted) return;
+
+ // Dedup by url_hash (last wins) so a repeated URL can't double-insert, and sort
+ // by url_hash for a deterministic row-lock order.
+ const byHash = new Map<
+ string,
+ BulkUpsertMonitorPageRow & { urlHash: Buffer }
+ >();
+ for (const row of params.rows) {
+ const urlHash = row.urlHash ?? hashMonitorUrl(row.url);
+ byHash.set(urlHash.toString("hex"), { ...row, urlHash });
+ }
+ if (byHash.size === 0) return;
+ const rows = [...byHash.values()].sort((a, b) =>
+ a.urlHash.toString("hex") < b.urlHash.toString("hex") ? -1 : 1,
+ );
+
+ const now = new Date().toISOString();
+
+ // Build every row as if newly inserted; ON CONFLICT applies the existing-row
+ // rules via `excluded` + CASE so the whole upsert is ONE atomic statement β€” no
+ // separate read, no separate update β€” and Drizzle maps the enum/jsonb types from
+ // the schema, so there are no hand-written casts that can drift from the columns.
+ const values = rows.map(row => {
+ const isRemoved = row.status === "removed";
+ const isChangedOrNew = row.status === "changed" || row.status === "new";
+ return {
+ monitor_id: params.monitorId,
+ team_id: params.teamId,
+ target_id: params.targetId,
+ url: row.url,
+ url_hash: row.urlHash,
+ source: row.source,
+ first_seen_check_id: params.checkId,
+ last_seen_check_id: isRemoved ? null : params.checkId,
+ last_changed_check_id: isChangedOrNew ? params.checkId : null,
+ last_scrape_id: row.scrapeId,
+ last_status: row.status,
+ is_removed: isRemoved,
+ removed_at: isRemoved ? now : null,
+ metadata: row.metadata ?? null,
+ created_at: now,
+ updated_at: now,
+ };
+ });
+
+ if (params.abortSignal?.aborted) return;
+
+ await run(
+ () =>
+ db
+ .insert(schema.monitor_pages)
+ .values(values)
+ .onConflictDoUpdate({
+ target: [
+ schema.monitor_pages.monitor_id,
+ schema.monitor_pages.target_id,
+ schema.monitor_pages.url_hash,
+ ],
+ set: {
+ last_status: sql`excluded.last_status`,
+ is_removed: sql`excluded.is_removed`,
+ removed_at: sql`excluded.removed_at`,
+ // Preserve prior metadata when the new row carries none.
+ metadata: sql`coalesce(excluded.metadata, ${schema.monitor_pages.metadata})`,
+ // last_seen / last_scrape advance only when not removed; else preserved.
+ last_seen_check_id: sql`case when excluded.is_removed then ${schema.monitor_pages.last_seen_check_id} else excluded.last_seen_check_id end`,
+ last_scrape_id: sql`case when excluded.is_removed then ${schema.monitor_pages.last_scrape_id} else excluded.last_scrape_id end`,
+ // last_changed advances only on new/changed; first_seen is never touched.
+ last_changed_check_id: sql`case when excluded.last_status in ('new','changed') then excluded.last_changed_check_id else ${schema.monitor_pages.last_changed_check_id} end`,
+ updated_at: sql`excluded.updated_at`,
+ },
+ }),
+ "Failed to bulk upsert monitor pages",
+ );
+}
+
export async function listActiveMonitorPages(params: {
monitorId: string;
targetId: string;