Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import {
createTestIdUrl,
describeIf,
ALLOW_TEST_SUITE_WEBSITE,
TEST_SELF_HOST,
TEST_SUITE_WEBSITE,
} from "../lib";
import {
idmux,
Identity,
monitorCheckRaw,
monitorCreateRaw,
monitorDeleteRaw,
monitorEmailConfirmRaw,
monitorEmailConfirmRawViaQuery,
monitorEmailUnsubscribeRaw,
monitorGetRaw,
monitorListRaw,
monitorPatchRaw,
monitorRunRaw,
scrapeTimeout,
} from "./lib";
describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
let identity: Identity;
beforeAll(async () => {
identity = await idmux({
name: "monitor",
concurrency: 20,
credits: 1000000,
});
}, 10000);
it(
"reserves the known PDF page cost before a repeat monitor run",
async () => {
const create = await monitorCreateRaw(
{
name: "PDF reservation monitor",
schedule: { cron: "0 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [
`${TEST_SUITE_WEBSITE}/example-long.pdf?testId=${crypto.randomUUID()}`,
],
scrapeOptions: { formats: ["markdown"] },
},
],
notification: { email: { enabled: false } },
},
identity,
);
expect(create.statusCode).toBe(200);
const monitorId = create.body.data.id;
const runToCompletion = async () => {
const run = await monitorRunRaw(monitorId, identity);
expect(run.statusCode).toBe(200);
let check: any;
for (let i = 0; i < 90; i++) {
const raw = await monitorCheckRaw(monitorId, run.body.id, identity);
expect(raw.statusCode).toBe(200);
check = raw.body.data;
if (
["completed", "partial", "failed", "skipped_no_credits"].includes(
check.status,
)
)
break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
expect(check.status).toBe("completed");
return check;
};
try {
const first = await runToCompletion();
expect(first.estimatedCredits).toBe(1);
expect(first.actualCredits).toBeGreaterThan(1);
const second = await runToCompletion();
expect(second.estimatedCredits).toBe(first.actualCredits);
expect(second.reservedCredits).toBe(first.actualCredits);
expect(second.actualCredits).toBe(first.actualCredits);
} finally {
await monitorDeleteRaw(monitorId, identity);
}
},
3 * scrapeTimeout,
);
it("creates, lists, gets, pauses, and deletes a monitor", async () => {
const create = await monitorCreateRaw(
{
name: "snips monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl(), createTestIdUrl()],
scrapeOptions: { formats: ["markdown"] },
},
],
notification: { email: { enabled: false } },
},
identity,
);
expect(create.statusCode).toBe(200);
expect(create.body.success).toBe(true);
expect(create.body.data.id).toEqual(expect.any(String));
expect(create.body.data.targets[0].id).toEqual(expect.any(String));
const id = create.body.data.id;
const list = await monitorListRaw(identity);
expect(list.statusCode).toBe(200);
expect(list.body.data.some((x: any) => x.id === id)).toBe(true);
const get = await monitorGetRaw(id, identity);
expect(get.statusCode).toBe(200);
expect(get.body.data.id).toBe(id);
const patch = await monitorPatchRaw(id, { status: "paused" }, identity);
expect(patch.statusCode).toBe(200);
expect(patch.body.data.status).toBe("paused");
const del = await monitorDeleteRaw(id, identity);
expect(del.statusCode).toBe(200);
expect(del.body.success).toBe(true);
});
describe("when the team requires zero data retention for search", () => {
let searchZdr: Identity;
beforeAll(async () => {
searchZdr = await idmux({
name: "monitor/search-zdr",
credits: 10000,
flags: { searchZDR: "forced-zdr" },
});
}, 10000);
it("refuses a search monitor", async () => {
const create = await monitorCreateRaw(
{
name: "search zdr monitor",
schedule: { cron: "0 * * * *", timezone: "UTC" },
goal: "New releases of the firecrawl SDK",
targets: [{ type: "search", queries: ["firecrawl sdk release"] }],
notification: { email: { enabled: false } },
},
searchZdr,
);
expect(create.statusCode).toBe(400);
expect(create.body.success).toBe(false);
expect(create.body.error).toContain("zero data retention");
});
it("keeps a scrape monitor and refuses adding a search target to it", async () => {
const create = await monitorCreateRaw(
{
name: "scrape monitor on a search zdr team",
schedule: { cron: "0 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: { formats: ["markdown"] },
},
],
notification: { email: { enabled: false } },
},
searchZdr,
);
expect(create.statusCode).toBe(200);
const id = create.body.data.id;
try {
const addSearch = await monitorPatchRaw(
id,
{
goal: "New releases of the firecrawl SDK",
targets: [{ type: "search", queries: ["firecrawl sdk release"] }],
},
searchZdr,
);
expect(addSearch.statusCode).toBe(400);
expect(addSearch.body.error).toContain("zero data retention");
const pause = await monitorPatchRaw(
id,
{ status: "paused" },
searchZdr,
);
expect(pause.statusCode).toBe(200);
expect(pause.body.data.status).toBe("paused");
} finally {
await monitorDeleteRaw(id, searchZdr);
}
});
});
it("accepts an origin field in the create body", async () => {
const create = await monitorCreateRaw(
{
name: "origin monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
},
],
origin: "[email protected]",
},
identity,
);
expect(create.statusCode).toBe(200);
expect(create.body.success).toBe(true);
const id = create.body.data.id;
await monitorDeleteRaw(id, identity);
});
it("still rejects unknown keys in the create body", async () => {
const create = await monitorCreateRaw(
{
name: "unknown key monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
},
],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
nonsenseField: "value",
} as any,
identity,
);
expect(create.statusCode).toBe(400);
expect(create.body.success).toBe(false);
});
it("rejects cron schedules under 15 minutes", async () => {
const response = await monitorCreateRaw(
{
name: "too frequent",
schedule: { cron: "*/5 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
},
],
},
identity,
);
expect(response.statusCode).toBe(400);
expect(response.body.success).toBe(false);
expect(response.body.error).toContain("15 minutes");
});
it(
"runs a manual scrape monitor check",
async () => {
const create = await monitorCreateRaw(
{
name: "manual monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl(), createTestIdUrl()],
scrapeOptions: { formats: ["markdown"] },
},
],
},
identity,
);
expect(create.statusCode).toBe(200);
const monitorId = create.body.data.id;
const run = await monitorRunRaw(monitorId, identity);
expect(run.statusCode).toBe(200);
const checkId = run.body.id;
let check: any;
for (let i = 0; i < 90; i++) {
const raw = await monitorCheckRaw(monitorId, checkId, identity);
expect(raw.statusCode).toBe(200);
check = raw.body.data;
if (["completed", "partial", "failed"].includes(check.status)) break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
expect(["completed", "partial"]).toContain(check.status);
expect(check.summary.totalPages).toBeGreaterThanOrEqual(2);
expect(check.pages.length).toBeGreaterThanOrEqual(1);
expect(check.next).toBeUndefined();
const firstPage = await monitorCheckRaw(monitorId, checkId, identity, {
limit: 1,
});
expect(firstPage.statusCode).toBe(200);
expect(firstPage.body.next).toContain("skip=1");
expect(firstPage.body.next).toContain("limit=1");
expect(firstPage.body.data.next).toBe(firstPage.body.next);
expect(firstPage.body.data.pages).toHaveLength(1);
await monitorDeleteRaw(monitorId, identity);
},
2 * scrapeTimeout,
);
it(
"runs a JSON-mode monitor and surfaces a snapshot",
async () => {
const jsonSchema = {
type: "object",
additionalProperties: false,
required: ["title"],
properties: {
title: {
type: "string",
description: "The page title, verbatim.",
},
},
};
const create = await monitorCreateRaw(
{
name: "json-mode monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: {
formats: [
"markdown",
{
type: "json",
prompt: "Extract the page title verbatim.",
schema: jsonSchema,
},
],
},
},
],
},
identity,
);
expect(create.statusCode).toBe(200);
const monitorId = create.body.data.id;
const run = await monitorRunRaw(monitorId, identity);
expect(run.statusCode).toBe(200);
const checkId = run.body.id;
let check: any;
for (let i = 0; i < 90; i++) {
const raw = await monitorCheckRaw(monitorId, checkId, identity);
expect(raw.statusCode).toBe(200);
check = raw.body.data;
if (["completed", "partial", "failed"].includes(check.status)) break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
expect(["completed", "partial"]).toContain(check.status);
// First run: status is "new" (no previous scrape to diff against), no
// snapshot persisted to GCS. We can't assert snapshot.json here
// without a mutating fixture; a second run with a changed page would
// be needed. The contract assertion is: when JSON mode is requested,
// the monitor doesn't fall through the markdown path and crash.
expect(check.pages[0].status).toBe("new");
await monitorDeleteRaw(monitorId, identity);
},
2 * scrapeTimeout,
);
it(
"runs a deterministicJson-mode monitor target",
async () => {
const jsonSchema = {
type: "object",
additionalProperties: false,
required: ["title"],
properties: {
title: {
type: "string",
description: "The page title, verbatim.",
},
},
};
const create = await monitorCreateRaw(
{
name: "deterministic-json monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: {
formats: [
"markdown",
{
type: "deterministicJson",
prompt: "Extract the page title verbatim.",
schema: jsonSchema,
},
],
},
},
],
},
identity,
);
expect(create.statusCode).toBe(200);
const monitorId = create.body.data.id;
const run = await monitorRunRaw(monitorId, identity);
expect(run.statusCode).toBe(200);
const checkId = run.body.id;
let check: any;
for (let i = 0; i < 90; i++) {
const raw = await monitorCheckRaw(monitorId, checkId, identity);
expect(raw.statusCode).toBe(200);
check = raw.body.data;
if (["completed", "partial", "failed"].includes(check.status)) break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Contract: deterministicJson is accepted, the reusable-json-mode extractor
// populates document.json via the code-sandbox, and the check flows through
// the JSON diff path (status "new" on first run) rather than crashing.
expect(["completed", "partial"]).toContain(check.status);
expect(check.pages[0].status).toBe("new");
await monitorDeleteRaw(monitorId, identity);
},
2 * scrapeTimeout,
);
it(
"accepts changeTracking-json format on a monitor target",
async () => {
const jsonSchema = {
type: "object",
additionalProperties: false,
required: ["title"],
properties: {
title: { type: "string", description: "The page title." },
},
};
const create = await monitorCreateRaw(
{
name: "ct-json monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: {
formats: [
"markdown",
{
type: "changeTracking",
modes: ["json"],
prompt: "Extract the title.",
schema: jsonSchema,
},
],
},
},
],
},
identity,
);
expect(create.statusCode).toBe(200);
const monitorId = create.body.data.id;
const run = await monitorRunRaw(monitorId, identity);
expect(run.statusCode).toBe(200);
const checkId = run.body.id;
let check: any;
for (let i = 0; i < 90; i++) {
const raw = await monitorCheckRaw(monitorId, checkId, identity);
expect(raw.statusCode).toBe(200);
check = raw.body.data;
if (["completed", "partial", "failed"].includes(check.status)) break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
expect(["completed", "partial"]).toContain(check.status);
expect(check.pages[0].status).toBe("new");
await monitorDeleteRaw(monitorId, identity);
},
2 * scrapeTimeout,
);
describe("email recipient opt-in", () => {
function externalRecipient(): string {
const id = Math.random().toString(36).slice(2, 10);
return `optin-${id}@external-test.example`;
}
it("starts external recipients as pending without sending notifications", async () => {
const recipient = externalRecipient();
const create = await monitorCreateRaw(
{
name: "opt-in monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: { formats: ["markdown"] },
},
],
notification: {
email: { enabled: true, recipients: [recipient] },
},
},
identity,
);
expect(create.statusCode).toBe(200);
expect(create.body.data.emailRecipientSubscriptions).toEqual(
expect.arrayContaining([
expect.objectContaining({
email: recipient.toLowerCase(),
status: "pending",
source: "opt_in",
}),
]),
);
const get = await monitorGetRaw(create.body.data.id, identity);
expect(get.body.data.emailRecipientSubscriptions).toEqual(
expect.arrayContaining([
expect.objectContaining({
email: recipient.toLowerCase(),
status: "pending",
}),
]),
);
await monitorDeleteRaw(create.body.data.id, identity);
});
it("rejects malformed confirm/unsubscribe tokens with a 400 JSON error", async () => {
const badConfirm = await monitorEmailConfirmRaw("x");
expect(badConfirm.statusCode).toBe(400);
expect(badConfirm.body).toEqual({
success: false,
error: "invalid_token",
});
const badUnsub = await monitorEmailUnsubscribeRaw("x");
expect(badUnsub.statusCode).toBe(400);
expect(badUnsub.body).toEqual({
success: false,
error: "invalid_token",
});
});
it("rejects tokens sent in the query string (body-only)", async () => {
// Tokens in URLs leak into access/proxy logs and APM URL tags, so
// the controller deliberately ignores query params.
const unknownToken = "a".repeat(43);
const response = await monitorEmailConfirmRawViaQuery(unknownToken);
expect(response.statusCode).toBe(400);
expect(response.body).toEqual({
success: false,
error: "invalid_token",
});
});
it("returns 404 JSON for unknown but well-formed tokens", async () => {
const unknownToken = "a".repeat(43);
const confirm = await monitorEmailConfirmRaw(unknownToken);
expect(confirm.statusCode).toBe(404);
expect(confirm.body).toEqual({
success: false,
error: "not_found",
});
const unsub = await monitorEmailUnsubscribeRaw(unknownToken);
expect(unsub.statusCode).toBe(404);
expect(unsub.body).toEqual({
success: false,
error: "not_found",
});
});
it("does not require opt-in when recipients are unset (team-default path)", async () => {
const create = await monitorCreateRaw(
{
name: "team default monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: { formats: ["markdown"] },
},
],
notification: { email: { enabled: true } },
},
identity,
);
expect(create.statusCode).toBe(200);
expect(create.body.data.emailRecipientSubscriptions).toEqual([]);
await monitorDeleteRaw(create.body.data.id, identity);
});
});
it(
"accepts mixed json + git-diff changeTracking modes",
async () => {
const jsonSchema = {
type: "object",
additionalProperties: false,
required: ["title"],
properties: {
title: { type: "string", description: "The page title." },
},
};
const create = await monitorCreateRaw(
{
name: "ct-mixed monitor",
schedule: { cron: "*/30 * * * *", timezone: "UTC" },
targets: [
{
type: "scrape",
urls: [createTestIdUrl()],
scrapeOptions: {
formats: [
"markdown",
{
type: "changeTracking",
modes: ["json", "git-diff"],
prompt: "Extract the title.",
schema: jsonSchema,
},
],
},
},
],
},
identity,
);
expect(create.statusCode).toBe(200);
const monitorId = create.body.data.id;
const run = await monitorRunRaw(monitorId, identity);
expect(run.statusCode).toBe(200);
const checkId = run.body.id;
let check: any;
for (let i = 0; i < 90; i++) {
const raw = await monitorCheckRaw(monitorId, checkId, identity);
expect(raw.statusCode).toBe(200);
check = raw.body.data;
if (["completed", "partial", "failed"].includes(check.status)) break;
await new Promise(resolve => setTimeout(resolve, 1000));
}
expect(["completed", "partial"]).toContain(check.status);
// First run is always "new"; the assertion here is that mixed-mode
// configuration is accepted end-to-end without erroring.
expect(check.pages[0].status).toBe("new");
await monitorDeleteRaw(monitorId, identity);
},
2 * scrapeTimeout,
);
});